1
0
mirror of https://github.com/aquatix/digimarks.git synced 2025-12-06 22:05:09 +01:00

Refactoring to FastAPI and SQLAlchemy

This commit is contained in:
2023-07-30 21:19:51 +02:00
parent 96e7ef16d4
commit 7e397f9d2b
6 changed files with 407 additions and 157 deletions

View File

@@ -46,6 +46,18 @@ url's when wanted.
Url's are of the form https://marks.example.com/<userkey>/<action>
digimarks can also be run from the command line: ``uvicorn digimarks:app --reload``
Be sure to export/set the ``SECRETKEY`` environment variable before running, it's needed for some management URI's.
Run ``gunicorn -k uvicorn.workers.UvicornWorker`` for production. For an example of how to set up a server `see this article <https://www.slingacademy.com/article/deploying-fastapi-on-ubuntu-with-nginx-and-lets-encrypt/>`_ with configuration for nginx, uvicorn, systemd, security and such.
The RQ background worker can be run from the command line: ``rq worker --with-scheduler``
Url's are of the form https://hook.example.com/app/<appkey>/<triggerkey>
API documentation is auto-generated, and can be browsed at https://hook.example.com/docs
Bookmarklet
~~~~~~~~~~~
@@ -74,8 +86,9 @@ If you for whatever reason would lose this user key, just either look on the con
Server configuration
~~~~~~~~~~~~~~~~~~~~
* `vhost for Apache2.4`_
* `uwsgi.ini`_
* `systemd for digimarks API`_ which uses the `gunicorn config`_
* `nginx for digimarks API`_
* `more config`_
What's new?
@@ -91,7 +104,6 @@ Attributions
.. _digimarks: https://github.com/aquatix/digimarks
.. _webhook: https://en.wikipedia.org/wiki/Webhook
.. |PyPI version| image:: https://img.shields.io/pypi/v/digimarks.svg
:target: https://pypi.python.org/pypi/digimarks/
.. |PyPI license| image:: https://img.shields.io/github/license/aquatix/digimarks.svg
@@ -107,3 +119,6 @@ Attributions
.. _uwsgi.ini: https://github.com/aquatix/digimarks/blob/master/example_config/uwsgi.ini
.. _Changelog: https://github.com/aquatix/digimarks/blob/master/CHANGELOG.md
.. _Freepik: http://www.flaticon.com/free-icon/letter-m_2041
.. _systemd for digimarks API: https://github.com/aquatix/digimarks/blob/master/example_config/systemd/digimarks.service
.. _gunicorn config: https://github.com/aquatix/digimarks/blob/master/example_config/gunicorn_digimarks_conf.py
.. _more config: https://github.com/aquatix/digimarks/tree/master/example_config

View File

@@ -1,3 +1,8 @@
-r requirements.in
black
pylint
ruff
build
twine

View File

@@ -1,71 +1,223 @@
#
# This file is autogenerated by pip-compile with python 3.10
# To update, run:
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile requirements-dev.in
#
anyio==3.6.1
# via starlette
astroid==2.11.7
annotated-types==0.5.0
# via pydantic
anyio==3.7.1
# via
# httpcore
# starlette
# watchfiles
astroid==2.15.6
# via pylint
beautifulsoup4==4.11.1
beautifulsoup4==4.12.2
# via bs4
black==23.7.0
# via -r requirements-dev.in
bleach==6.0.0
# via readme-renderer
bs4==0.0.1
# via -r requirements.in
certifi==2022.6.15
build==0.10.0
# via -r requirements-dev.in
certifi==2023.7.22
# via
# httpcore
# httpx
# requests
cffi==1.15.1
# via cryptography
charset-normalizer==3.2.0
# via requests
charset-normalizer==2.1.0
# via requests
dill==0.3.5.1
click==8.1.6
# via
# black
# uvicorn
cryptography==41.0.2
# via secretstorage
dill==0.3.7
# via pylint
fastapi==0.79.0
dnspython==2.4.1
# via email-validator
docutils==0.20.1
# via readme-renderer
email-validator==2.0.0.post2
# via fastapi
exceptiongroup==1.1.2
# via anyio
fastapi[all]==0.100.1
# via -r requirements.in
feedgen==0.9.0
# via -r requirements.in
greenlet==1.1.2
greenlet==2.0.2
# via sqlalchemy
idna==3.3
h11==0.14.0
# via
# httpcore
# uvicorn
httpcore==0.17.3
# via httpx
httptools==0.6.0
# via uvicorn
httpx==0.24.1
# via fastapi
idna==3.4
# via
# anyio
# email-validator
# httpx
# requests
isort==5.10.1
importlib-metadata==6.8.0
# via
# keyring
# twine
isort==5.12.0
# via pylint
lazy-object-proxy==1.7.1
itsdangerous==2.1.2
# via fastapi
jaraco-classes==3.3.0
# via keyring
jeepney==0.8.0
# via
# keyring
# secretstorage
jinja2==3.1.2
# via fastapi
keyring==24.2.0
# via twine
lazy-object-proxy==1.9.0
# via astroid
lxml==4.9.1
lxml==4.9.3
# via feedgen
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.3
# via jinja2
mccabe==0.7.0
# via pylint
platformdirs==2.5.2
# via pylint
pydantic==1.9.1
mdurl==0.1.2
# via markdown-it-py
more-itertools==10.0.0
# via jaraco-classes
mypy-extensions==1.0.0
# via black
orjson==3.9.2
# via fastapi
pylint==2.14.5
packaging==23.1
# via
# black
# build
pathspec==0.11.2
# via black
pkginfo==1.9.6
# via twine
platformdirs==3.10.0
# via
# black
# pylint
pycparser==2.21
# via cffi
pydantic==2.1.1
# via
# fastapi
# pydantic-extra-types
# pydantic-settings
pydantic-core==2.4.0
# via pydantic
pydantic-extra-types==2.0.0
# via fastapi
pydantic-settings==2.0.2
# via fastapi
pygments==2.15.1
# via
# readme-renderer
# rich
pylint==2.17.5
# via -r requirements-dev.in
pyproject-hooks==1.0.0
# via build
python-dateutil==2.8.2
# via feedgen
requests==2.28.1
# via -r requirements.in
python-dotenv==1.0.0
# via
# pydantic-settings
# uvicorn
python-multipart==0.0.6
# via fastapi
pyyaml==6.0.1
# via
# fastapi
# uvicorn
readme-renderer==40.0
# via twine
requests==2.31.0
# via
# -r requirements.in
# requests-toolbelt
# twine
requests-toolbelt==1.0.0
# via twine
rfc3986==2.0.0
# via twine
rich==13.5.0
# via twine
ruff==0.0.280
# via -r requirements-dev.in
secretstorage==3.3.3
# via keyring
six==1.16.0
# via python-dateutil
sniffio==1.2.0
# via anyio
soupsieve==2.3.2.post1
# via
# bleach
# python-dateutil
sniffio==1.3.0
# via
# anyio
# httpcore
# httpx
soupsieve==2.4.1
# via beautifulsoup4
sqlalchemy==1.4.39
sqlalchemy==2.0.19
# via -r requirements.in
starlette==0.19.1
starlette==0.27.0
# via fastapi
tomli==2.0.1
# via
# black
# build
# pylint
# pyproject-hooks
tomlkit==0.12.1
# via pylint
tomlkit==0.11.1
# via pylint
typing-extensions==4.3.0
# via pydantic
urllib3==1.26.11
# via requests
wrapt==1.14.1
twine==4.0.2
# via -r requirements-dev.in
typing-extensions==4.7.1
# via
# astroid
# fastapi
# pydantic
# pydantic-core
# sqlalchemy
# uvicorn
ujson==5.8.0
# via fastapi
urllib3==2.0.4
# via
# requests
# twine
uvicorn[standard]==0.23.1
# via fastapi
uvloop==0.17.0
# via uvicorn
watchfiles==0.19.0
# via uvicorn
webencodings==0.5.1
# via bleach
websockets==11.0.3
# via uvicorn
wrapt==1.15.0
# via astroid
# The following packages are considered to be unsafe in a requirements file:
# setuptools
zipp==3.16.2
# via importlib-metadata

View File

@@ -1,5 +1,5 @@
# Core application
fastapi
fastapi[all]
sqlalchemy
# Fetch title etc from links

View File

@@ -1,44 +1,121 @@
#
# This file is autogenerated by pip-compile with python 3.10
# To update, run:
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile requirements.in
#
beautifulsoup4==4.11.1
annotated-types==0.5.0
# via pydantic
anyio==3.7.1
# via
# httpcore
# starlette
# watchfiles
beautifulsoup4==4.12.2
# via bs4
bs4==0.0.1
# via -r requirements.in
certifi==2022.6.15
certifi==2023.7.22
# via
# httpcore
# httpx
# requests
charset-normalizer==3.2.0
# via requests
charset-normalizer==2.1.0
# via requests
click==8.1.3
# via flask
click==8.1.6
# via uvicorn
dnspython==2.4.1
# via email-validator
email-validator==2.0.0.post2
# via fastapi
exceptiongroup==1.1.2
# via anyio
fastapi[all]==0.100.1
# via -r requirements.in
feedgen==0.9.0
# via -r requirements.in
flask==2.1.3
# via -r requirements.in
idna==3.3
# via requests
greenlet==2.0.2
# via sqlalchemy
h11==0.14.0
# via
# httpcore
# uvicorn
httpcore==0.17.3
# via httpx
httptools==0.6.0
# via uvicorn
httpx==0.24.1
# via fastapi
idna==3.4
# via
# anyio
# email-validator
# httpx
# requests
itsdangerous==2.1.2
# via flask
# via fastapi
jinja2==3.1.2
# via flask
lxml==4.9.1
# via fastapi
lxml==4.9.3
# via feedgen
markupsafe==2.1.1
markupsafe==2.1.3
# via jinja2
peewee==3.15.1
# via -r requirements.in
orjson==3.9.2
# via fastapi
pydantic==2.1.1
# via
# fastapi
# pydantic-extra-types
# pydantic-settings
pydantic-core==2.4.0
# via pydantic
pydantic-extra-types==2.0.0
# via fastapi
pydantic-settings==2.0.2
# via fastapi
python-dateutil==2.8.2
# via feedgen
requests==2.28.1
python-dotenv==1.0.0
# via
# pydantic-settings
# uvicorn
python-multipart==0.0.6
# via fastapi
pyyaml==6.0.1
# via
# fastapi
# uvicorn
requests==2.31.0
# via -r requirements.in
six==1.16.0
# via python-dateutil
soupsieve==2.3.2.post1
sniffio==1.3.0
# via
# anyio
# httpcore
# httpx
soupsieve==2.4.1
# via beautifulsoup4
urllib3==1.26.10
sqlalchemy==2.0.19
# via -r requirements.in
starlette==0.27.0
# via fastapi
typing-extensions==4.7.1
# via
# fastapi
# pydantic
# pydantic-core
# sqlalchemy
# uvicorn
ujson==5.8.0
# via fastapi
urllib3==2.0.4
# via requests
werkzeug==2.1.2
# via flask
uvicorn[standard]==0.23.1
# via fastapi
uvloop==0.17.0
# via uvicorn
watchfiles==0.19.0
# via uvicorn
websockets==11.0.3
# via uvicorn

View File

@@ -1,28 +1,25 @@
from __future__ import print_function
import binascii
import datetime
import gzip
import hashlib
import logging
import os
import shutil
import sys
from urllib.parse import urljoin, urlparse, urlunparse
import bs4
import requests
from dateutil import tz
from feedgen.feed import FeedGenerator
#from flask import (Flask, abort, jsonify, make_response, redirect,
# render_template, request, url_for)
from typing import List
import databases
import sqlalchemy
from fastapi import FastAPI
from pydantic import BaseModel
from urlparse import urljoin, urlparse, urlunparse
from fastapi import Depends, FastAPI, HTTPException, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from feedgen.feed import FeedGenerator
from pydantic import BaseModel, DirectoryPath, FilePath, validator
from pydantic_settings import BaseSettings
from sqlalchemy import VARCHAR, Boolean, Column, DateTime, ForeignKey, Integer, String, Text, create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
DIGIMARKS_USER_AGENT = 'digimarks/2.0.0-dev'
@@ -150,44 +147,44 @@ themes = {
}
}
try:
import settings
except ImportError:
print('Copy settings_example.py to settings.py and set the configuration to your own preferences')
sys.exit(1)
# app configuration
APP_ROOT = os.path.dirname(os.path.realpath(__file__))
MEDIA_ROOT = os.path.join(APP_ROOT, 'static')
MEDIA_URL = '/static/'
DATABASE = {
'name': os.path.join(APP_ROOT, 'bookmarks.db'),
'engine': 'peewee.SqliteDatabase',
}
DATABASE_URL = os.path.join(APP_ROOT, 'bookmarks.db')
#PHANTOM = '/usr/local/bin/phantomjs'
#SCRIPT = os.path.join(APP_ROOT, 'screenshot.js')
class Settings(BaseSettings):
"""Configuration needed for digimarks to find its database, favicons, API integrations"""
# create our flask app and a database wrapper
#app = Flask(__name__)
#app.config.from_object(__name__)
#database = SqliteDatabase(os.path.join(APP_ROOT, 'bookmarks.db'))
database_file: FilePath = './bookmarks.db'
media_dir: DirectoryPath
media_url: str = '/static/'
database = databases.Database(DATABASE_URL)
mashape_api_key: str
metadata = sqlalchemy.MetaData()
debug: bool = False
# Strip unnecessary whitespace due to jinja2 codeblocks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# set custom url for the app, for example '/bookmarks'
try:
app.config['APPLICATION_ROOT'] = settings.APPLICATION_ROOT
except AttributeError:
pass
settings = Settings()
engine = create_engine(
f'sqlite:///{settings.database_file}', connect_args={'check_same_thread': False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
app = FastAPI()
logger = logging.getLogger('digimarks')
if settings.debug:
logger.setLevel(logging.DEBUG)
# CORS configuration
app.add_middleware(
CORSMiddleware,
allow_origins=["*"], # Allow requests from everywhere
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Cache the tags
all_tags = {}
usersettings = {}
@@ -202,9 +199,11 @@ def ifilterfalse(predicate, iterable):
def unique_everseen(iterable, key=None):
"List unique elements, preserving order. Remember all elements ever seen."
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
# unique_everseen('ABBCcAD', str.lower) --> A B C D
"""List unique elements, preserving order. Remember all elements ever seen.
unique_everseen('AAAABBBCCDAABBB') --> A B C D
unique_everseen('ABBCcAD', str.lower) --> A B C D
"""
seen = set()
seen_add = seen.add
if key is None:
@@ -218,6 +217,7 @@ def unique_everseen(iterable, key=None):
seen_add(k)
yield element
def clean_tags(tags_list):
tags_res = [x.strip() for x in tags_list]
tags_res = list(unique_everseen(tags_res))
@@ -244,17 +244,13 @@ def file_type(filename):
return "no match"
class BaseModel(Model):
class Meta:
database = database
class User(BaseModel):
class User(Base):
""" User account """
username = CharField()
key = CharField()
theme = CharField(default=DEFAULT_THEME)
created_date = DateTimeField(default=datetime.datetime.now)
username = Column(VARCHAR(255))
key = Column(VARCHAR(255))
# theme = CharField(default=DEFAULT_THEME)
theme = Column(VARCHAR(20), default=DEFAULT_THEME)
created_date = Column(DateTime, default=datetime.datetime.now)
def generate_key(self):
""" Generate userkey """
@@ -262,21 +258,21 @@ class User(BaseModel):
return self.key
class Bookmark(BaseModel):
class Bookmark(Base):
""" Bookmark instance, connected to User """
# Foreign key to User
userkey = CharField()
userkey = Column(VARCHAR(255))
title = CharField(default='')
url = CharField()
note = TextField(default='')
title = Column(VARCHAR(255), default='')
url = Column(VARCHAR(255))
note = Column(Text, default='')
#image = CharField(default='')
url_hash = CharField(default='')
tags = CharField(default='')
starred = BooleanField(default=False)
url_hash = Column(VARCHAR(255) , default='')
tags = Column(VARCHAR(255), default='')
starred = Column(Boolean, default=False)
# Website (domain) favicon
favicon = CharField(null=True)
favicon = Column(VARCHAR(255), null=True)
# Status code: 200 is OK, 404 is not found, for example (showing an error)
HTTP_CONNECTIONERROR = 0
@@ -285,17 +281,17 @@ class Bookmark(BaseModel):
HTTP_MOVEDTEMPORARILY = 304
HTTP_NOTFOUND = 404
http_status = IntegerField(default=200)
http_status = Column(Integer, default=200)
redirect_uri = None
created_date = DateTimeField(default=datetime.datetime.now)
modified_date = DateTimeField(null=True)
deleted_date = DateTimeField(null=True)
created_date = Column(DateTime, default=datetime.datetime.now)
modified_date = Column(DateTime, null=True)
deleted_date = Column(DateTime, null=True)
# Bookmark status; deleting doesn't remove from DB
VISIBLE = 0
DELETED = 1
status = IntegerField(default=VISIBLE)
status = Column(Integer, default=VISIBLE)
class Meta:
@@ -345,7 +341,7 @@ class Bookmark(BaseModel):
stream=True,
headers={'User-Agent': DIGIMARKS_USER_AGENT}
)
filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
filename = os.path.join(settings.media_dir, 'favicons/', domain + fileextension)
with open(filename, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
del response
@@ -387,7 +383,7 @@ class Bookmark(BaseModel):
fileextension = '.jpg'
if response.headers['content-type'] == 'image/x-icon':
fileextension = '.ico'
filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
filename = os.path.join(settings.media_dir, 'favicons/', domain + fileextension)
with open(filename, 'wb') as out_file:
shutil.copyfileobj(response.raw, out_file)
del response
@@ -406,11 +402,11 @@ class Bookmark(BaseModel):
""" Fetch favicon for the domain """
u = urlparse(self.url)
domain = u.netloc
if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.png')):
if os.path.isfile(os.path.join(settings.media_dir, 'favicons/', domain + '.png')):
# If file exists, don't re-download it
self.favicon = domain + '.png'
return
if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.ico')):
if os.path.isfile(os.path.join(settings.media_dir, 'favicons/', domain + '.ico')):
# If file exists, don't re-download it
self.favicon = domain + '.ico'
return
@@ -466,10 +462,10 @@ class Bookmark(BaseModel):
class PublicTag(BaseModel):
""" Publicly shared tag """
tagkey = CharField()
userkey = CharField()
tag = CharField()
created_date = DateTimeField(default=datetime.datetime.now)
tagkey = Column(VARCHAR(255))
userkey = Column(VARCHAR(255))
tag = Column(VARCHAR(255))
created_date = Column(DateTime, default=datetime.datetime.now)
def generate_key(self):
""" Generate hash-based key for publicly shared tag """
@@ -958,8 +954,8 @@ def publictag_json(tagkey):
abort(404)
@app.route('/pub/<tagkey>/feed')
def publictag_feed(tagkey):
@app.get('/pub/<tagkey>/feed')
async def publictag_feed(request: Request, tagkey: str):
""" rss/atom representation of the Read-only overview of the bookmarks in the userkey/tag of this PublicTag """
try:
this_tag = PublicTag.get(PublicTag.tagkey == tagkey)
@@ -973,7 +969,7 @@ def publictag_feed(tagkey):
feed.title(this_tag.tag)
feed.id(request.url)
feed.link(href=request.url, rel='self')
feed.link(href=make_external(url_for('publictag_page', tagkey=tagkey)))
feed.link(href=make_external(app.url_path_for('publictag_page', tagkey=tagkey)))
for bookmark in bookmarks:
entry = feed.add_entry()
@@ -993,20 +989,21 @@ def publictag_feed(tagkey):
entry.published(bookmark.created_date.replace(tzinfo=tz.tzlocal()))
entry.updated(updated_date.replace(tzinfo=tz.tzlocal()))
response = make_response(feed.atom_str(pretty=True))
response = Response(data=feed.atom_str(pretty=True), media_type='application/xml')
response.headers.set('Content-Type', 'application/atom+xml')
return response
except PublicTag.DoesNotExist:
abort(404)
raise HTTPException(status_code=404, detail='Tag not found')
@app.route('/<userkey>/<tag>/makepublic', methods=['GET', 'POST'])
def addpublictag(userkey, tag):
#user = get_object_or_404(User.get(User.key == userkey))
@app.get('/<userkey>/<tag>/makepublic')
@app.post('/<userkey>/<tag>/makepublic')
async def addpublictag(userkey: str, tag: str):
try:
User.get(User.key == userkey)
except User.DoesNotExist:
abort(404)
raise HTTPException(status_code=404, detail='User not found')
try:
publictag = PublicTag.get(PublicTag.userkey == userkey, PublicTag.tag == tag)
except PublicTag.DoesNotExist:
@@ -1019,10 +1016,14 @@ def addpublictag(userkey, tag):
newpublictag.save()
message = 'Public link to this tag created'
return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message))
message = 'Public link already existed'
return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message))
success = True
# return RedirectResponse(url=url_path_for('tag_page', userkey=userkey, tag=tag, message=message))
else:
message = 'Public link already existed'
success = False
# return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message))
url = app.url_path_for('tag_page', userkey=userkey, tag=tag, message=message)
return {'success': success, 'message': message, 'url': url}
@app.route('/<userkey>/<tag>/removepublic/<tagkey>', methods=['GET', 'POST'])