From 7e397f9d2b63647b428d2b5f09206dbbaaac7933 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Sun, 30 Jul 2023 21:19:51 +0200 Subject: [PATCH] Refactoring to FastAPI and SQLAlchemy --- README.rst | 21 +++- requirements-dev.in | 5 + requirements-dev.txt | 228 +++++++++++++++++++++++++++++++++++------- requirements.in | 2 +- requirements.txt | 123 ++++++++++++++++++----- src/digimarks/main.py | 185 +++++++++++++++++----------------- 6 files changed, 407 insertions(+), 157 deletions(-) diff --git a/README.rst b/README.rst index 06cf965..e079140 100644 --- a/README.rst +++ b/README.rst @@ -46,6 +46,18 @@ url's when wanted. Url's are of the form https://marks.example.com// +digimarks can also be run from the command line: ``uvicorn digimarks:app --reload`` + +Be sure to export/set the ``SECRETKEY`` environment variable before running, it's needed for some management URI's. + +Run ``gunicorn -k uvicorn.workers.UvicornWorker`` for production. For an example of how to set up a server `see this article `_ with configuration for nginx, uvicorn, systemd, security and such. + +The RQ background worker can be run from the command line: ``rq worker --with-scheduler`` + +Url's are of the form https://hook.example.com/app// + +API documentation is auto-generated, and can be browsed at https://hook.example.com/docs + Bookmarklet ~~~~~~~~~~~ @@ -74,8 +86,9 @@ If you for whatever reason would lose this user key, just either look on the con Server configuration ~~~~~~~~~~~~~~~~~~~~ -* `vhost for Apache2.4`_ -* `uwsgi.ini`_ +* `systemd for digimarks API`_ which uses the `gunicorn config`_ +* `nginx for digimarks API`_ +* `more config`_ What's new? @@ -91,7 +104,6 @@ Attributions .. _digimarks: https://github.com/aquatix/digimarks -.. _webhook: https://en.wikipedia.org/wiki/Webhook .. |PyPI version| image:: https://img.shields.io/pypi/v/digimarks.svg :target: https://pypi.python.org/pypi/digimarks/ .. |PyPI license| image:: https://img.shields.io/github/license/aquatix/digimarks.svg @@ -107,3 +119,6 @@ Attributions .. _uwsgi.ini: https://github.com/aquatix/digimarks/blob/master/example_config/uwsgi.ini .. _Changelog: https://github.com/aquatix/digimarks/blob/master/CHANGELOG.md .. _Freepik: http://www.flaticon.com/free-icon/letter-m_2041 +.. _systemd for digimarks API: https://github.com/aquatix/digimarks/blob/master/example_config/systemd/digimarks.service +.. _gunicorn config: https://github.com/aquatix/digimarks/blob/master/example_config/gunicorn_digimarks_conf.py +.. _more config: https://github.com/aquatix/digimarks/tree/master/example_config diff --git a/requirements-dev.in b/requirements-dev.in index 7ddd3dc..2536ded 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,3 +1,8 @@ -r requirements.in +black pylint +ruff + +build +twine diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b45dfc..4f469b3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,71 +1,223 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements-dev.in # -anyio==3.6.1 - # via starlette -astroid==2.11.7 +annotated-types==0.5.0 + # via pydantic +anyio==3.7.1 + # via + # httpcore + # starlette + # watchfiles +astroid==2.15.6 # via pylint -beautifulsoup4==4.11.1 +beautifulsoup4==4.12.2 # via bs4 +black==23.7.0 + # via -r requirements-dev.in +bleach==6.0.0 + # via readme-renderer bs4==0.0.1 # via -r requirements.in -certifi==2022.6.15 +build==0.10.0 + # via -r requirements-dev.in +certifi==2023.7.22 + # via + # httpcore + # httpx + # requests +cffi==1.15.1 + # via cryptography +charset-normalizer==3.2.0 # via requests -charset-normalizer==2.1.0 - # via requests -dill==0.3.5.1 +click==8.1.6 + # via + # black + # uvicorn +cryptography==41.0.2 + # via secretstorage +dill==0.3.7 # via pylint -fastapi==0.79.0 +dnspython==2.4.1 + # via email-validator +docutils==0.20.1 + # via readme-renderer +email-validator==2.0.0.post2 + # via fastapi +exceptiongroup==1.1.2 + # via anyio +fastapi[all]==0.100.1 # via -r requirements.in feedgen==0.9.0 # via -r requirements.in -greenlet==1.1.2 +greenlet==2.0.2 # via sqlalchemy -idna==3.3 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==0.17.3 + # via httpx +httptools==0.6.0 + # via uvicorn +httpx==0.24.1 + # via fastapi +idna==3.4 # via # anyio + # email-validator + # httpx # requests -isort==5.10.1 +importlib-metadata==6.8.0 + # via + # keyring + # twine +isort==5.12.0 # via pylint -lazy-object-proxy==1.7.1 +itsdangerous==2.1.2 + # via fastapi +jaraco-classes==3.3.0 + # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.2 + # via fastapi +keyring==24.2.0 + # via twine +lazy-object-proxy==1.9.0 # via astroid -lxml==4.9.1 +lxml==4.9.3 # via feedgen +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via jinja2 mccabe==0.7.0 # via pylint -platformdirs==2.5.2 - # via pylint -pydantic==1.9.1 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.0.0 + # via jaraco-classes +mypy-extensions==1.0.0 + # via black +orjson==3.9.2 # via fastapi -pylint==2.14.5 +packaging==23.1 + # via + # black + # build +pathspec==0.11.2 + # via black +pkginfo==1.9.6 + # via twine +platformdirs==3.10.0 + # via + # black + # pylint +pycparser==2.21 + # via cffi +pydantic==2.1.1 + # via + # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.4.0 + # via pydantic +pydantic-extra-types==2.0.0 + # via fastapi +pydantic-settings==2.0.2 + # via fastapi +pygments==2.15.1 + # via + # readme-renderer + # rich +pylint==2.17.5 # via -r requirements-dev.in +pyproject-hooks==1.0.0 + # via build python-dateutil==2.8.2 # via feedgen -requests==2.28.1 - # via -r requirements.in +python-dotenv==1.0.0 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.6 + # via fastapi +pyyaml==6.0.1 + # via + # fastapi + # uvicorn +readme-renderer==40.0 + # via twine +requests==2.31.0 + # via + # -r requirements.in + # requests-toolbelt + # twine +requests-toolbelt==1.0.0 + # via twine +rfc3986==2.0.0 + # via twine +rich==13.5.0 + # via twine +ruff==0.0.280 + # via -r requirements-dev.in +secretstorage==3.3.3 + # via keyring six==1.16.0 - # via python-dateutil -sniffio==1.2.0 - # via anyio -soupsieve==2.3.2.post1 + # via + # bleach + # python-dateutil +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +soupsieve==2.4.1 # via beautifulsoup4 -sqlalchemy==1.4.39 +sqlalchemy==2.0.19 # via -r requirements.in -starlette==0.19.1 +starlette==0.27.0 # via fastapi tomli==2.0.1 + # via + # black + # build + # pylint + # pyproject-hooks +tomlkit==0.12.1 # via pylint -tomlkit==0.11.1 - # via pylint -typing-extensions==4.3.0 - # via pydantic -urllib3==1.26.11 - # via requests -wrapt==1.14.1 +twine==4.0.2 + # via -r requirements-dev.in +typing-extensions==4.7.1 + # via + # astroid + # fastapi + # pydantic + # pydantic-core + # sqlalchemy + # uvicorn +ujson==5.8.0 + # via fastapi +urllib3==2.0.4 + # via + # requests + # twine +uvicorn[standard]==0.23.1 + # via fastapi +uvloop==0.17.0 + # via uvicorn +watchfiles==0.19.0 + # via uvicorn +webencodings==0.5.1 + # via bleach +websockets==11.0.3 + # via uvicorn +wrapt==1.15.0 # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +zipp==3.16.2 + # via importlib-metadata diff --git a/requirements.in b/requirements.in index 7eac957..8182352 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ # Core application -fastapi +fastapi[all] sqlalchemy # Fetch title etc from links diff --git a/requirements.txt b/requirements.txt index 6944184..b1c9617 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,44 +1,121 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements.in # -beautifulsoup4==4.11.1 +annotated-types==0.5.0 + # via pydantic +anyio==3.7.1 + # via + # httpcore + # starlette + # watchfiles +beautifulsoup4==4.12.2 # via bs4 bs4==0.0.1 # via -r requirements.in -certifi==2022.6.15 +certifi==2023.7.22 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.2.0 # via requests -charset-normalizer==2.1.0 - # via requests -click==8.1.3 - # via flask +click==8.1.6 + # via uvicorn +dnspython==2.4.1 + # via email-validator +email-validator==2.0.0.post2 + # via fastapi +exceptiongroup==1.1.2 + # via anyio +fastapi[all]==0.100.1 + # via -r requirements.in feedgen==0.9.0 # via -r requirements.in -flask==2.1.3 - # via -r requirements.in -idna==3.3 - # via requests +greenlet==2.0.2 + # via sqlalchemy +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==0.17.3 + # via httpx +httptools==0.6.0 + # via uvicorn +httpx==0.24.1 + # via fastapi +idna==3.4 + # via + # anyio + # email-validator + # httpx + # requests itsdangerous==2.1.2 - # via flask + # via fastapi jinja2==3.1.2 - # via flask -lxml==4.9.1 + # via fastapi +lxml==4.9.3 # via feedgen -markupsafe==2.1.1 +markupsafe==2.1.3 # via jinja2 -peewee==3.15.1 - # via -r requirements.in +orjson==3.9.2 + # via fastapi +pydantic==2.1.1 + # via + # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.4.0 + # via pydantic +pydantic-extra-types==2.0.0 + # via fastapi +pydantic-settings==2.0.2 + # via fastapi python-dateutil==2.8.2 # via feedgen -requests==2.28.1 +python-dotenv==1.0.0 + # via + # pydantic-settings + # uvicorn +python-multipart==0.0.6 + # via fastapi +pyyaml==6.0.1 + # via + # fastapi + # uvicorn +requests==2.31.0 # via -r requirements.in six==1.16.0 # via python-dateutil -soupsieve==2.3.2.post1 +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +soupsieve==2.4.1 # via beautifulsoup4 -urllib3==1.26.10 +sqlalchemy==2.0.19 + # via -r requirements.in +starlette==0.27.0 + # via fastapi +typing-extensions==4.7.1 + # via + # fastapi + # pydantic + # pydantic-core + # sqlalchemy + # uvicorn +ujson==5.8.0 + # via fastapi +urllib3==2.0.4 # via requests -werkzeug==2.1.2 - # via flask +uvicorn[standard]==0.23.1 + # via fastapi +uvloop==0.17.0 + # via uvicorn +watchfiles==0.19.0 + # via uvicorn +websockets==11.0.3 + # via uvicorn diff --git a/src/digimarks/main.py b/src/digimarks/main.py index af2e9d9..f95acc8 100644 --- a/src/digimarks/main.py +++ b/src/digimarks/main.py @@ -1,28 +1,25 @@ -from __future__ import print_function - import binascii import datetime import gzip import hashlib +import logging import os import shutil -import sys +from urllib.parse import urljoin, urlparse, urlunparse import bs4 import requests from dateutil import tz -from feedgen.feed import FeedGenerator #from flask import (Flask, abort, jsonify, make_response, redirect, # render_template, request, url_for) -from typing import List - -import databases -import sqlalchemy -from fastapi import FastAPI -from pydantic import BaseModel - -from urlparse import urljoin, urlparse, urlunparse - +from fastapi import Depends, FastAPI, HTTPException, Request, Response +from fastapi.middleware.cors import CORSMiddleware +from feedgen.feed import FeedGenerator +from pydantic import BaseModel, DirectoryPath, FilePath, validator +from pydantic_settings import BaseSettings +from sqlalchemy import VARCHAR, Boolean, Column, DateTime, ForeignKey, Integer, String, Text, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker DIGIMARKS_USER_AGENT = 'digimarks/2.0.0-dev' @@ -150,44 +147,44 @@ themes = { } } -try: - import settings -except ImportError: - print('Copy settings_example.py to settings.py and set the configuration to your own preferences') - sys.exit(1) -# app configuration -APP_ROOT = os.path.dirname(os.path.realpath(__file__)) -MEDIA_ROOT = os.path.join(APP_ROOT, 'static') -MEDIA_URL = '/static/' -DATABASE = { - 'name': os.path.join(APP_ROOT, 'bookmarks.db'), - 'engine': 'peewee.SqliteDatabase', -} -DATABASE_URL = os.path.join(APP_ROOT, 'bookmarks.db') -#PHANTOM = '/usr/local/bin/phantomjs' -#SCRIPT = os.path.join(APP_ROOT, 'screenshot.js') +class Settings(BaseSettings): + """Configuration needed for digimarks to find its database, favicons, API integrations""" -# create our flask app and a database wrapper -#app = Flask(__name__) -#app.config.from_object(__name__) -#database = SqliteDatabase(os.path.join(APP_ROOT, 'bookmarks.db')) + database_file: FilePath = './bookmarks.db' + media_dir: DirectoryPath + media_url: str = '/static/' -database = databases.Database(DATABASE_URL) + mashape_api_key: str -metadata = sqlalchemy.MetaData() + debug: bool = False -# Strip unnecessary whitespace due to jinja2 codeblocks -app.jinja_env.trim_blocks = True -app.jinja_env.lstrip_blocks = True -# set custom url for the app, for example '/bookmarks' -try: - app.config['APPLICATION_ROOT'] = settings.APPLICATION_ROOT -except AttributeError: - pass +settings = Settings() + +engine = create_engine( + f'sqlite:///{settings.database_file}', connect_args={'check_same_thread': False} +) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() + +app = FastAPI() + +logger = logging.getLogger('digimarks') +if settings.debug: + logger.setLevel(logging.DEBUG) + +# CORS configuration +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # Allow requests from everywhere + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + -# Cache the tags all_tags = {} usersettings = {} @@ -202,9 +199,11 @@ def ifilterfalse(predicate, iterable): def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D + """List unique elements, preserving order. Remember all elements ever seen. + + unique_everseen('AAAABBBCCDAABBB') --> A B C D + unique_everseen('ABBCcAD', str.lower) --> A B C D + """ seen = set() seen_add = seen.add if key is None: @@ -218,6 +217,7 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element + def clean_tags(tags_list): tags_res = [x.strip() for x in tags_list] tags_res = list(unique_everseen(tags_res)) @@ -244,17 +244,13 @@ def file_type(filename): return "no match" -class BaseModel(Model): - class Meta: - database = database - - -class User(BaseModel): +class User(Base): """ User account """ - username = CharField() - key = CharField() - theme = CharField(default=DEFAULT_THEME) - created_date = DateTimeField(default=datetime.datetime.now) + username = Column(VARCHAR(255)) + key = Column(VARCHAR(255)) + # theme = CharField(default=DEFAULT_THEME) + theme = Column(VARCHAR(20), default=DEFAULT_THEME) + created_date = Column(DateTime, default=datetime.datetime.now) def generate_key(self): """ Generate userkey """ @@ -262,21 +258,21 @@ class User(BaseModel): return self.key -class Bookmark(BaseModel): +class Bookmark(Base): """ Bookmark instance, connected to User """ # Foreign key to User - userkey = CharField() + userkey = Column(VARCHAR(255)) - title = CharField(default='') - url = CharField() - note = TextField(default='') + title = Column(VARCHAR(255), default='') + url = Column(VARCHAR(255)) + note = Column(Text, default='') #image = CharField(default='') - url_hash = CharField(default='') - tags = CharField(default='') - starred = BooleanField(default=False) + url_hash = Column(VARCHAR(255) , default='') + tags = Column(VARCHAR(255), default='') + starred = Column(Boolean, default=False) # Website (domain) favicon - favicon = CharField(null=True) + favicon = Column(VARCHAR(255), null=True) # Status code: 200 is OK, 404 is not found, for example (showing an error) HTTP_CONNECTIONERROR = 0 @@ -285,17 +281,17 @@ class Bookmark(BaseModel): HTTP_MOVEDTEMPORARILY = 304 HTTP_NOTFOUND = 404 - http_status = IntegerField(default=200) + http_status = Column(Integer, default=200) redirect_uri = None - created_date = DateTimeField(default=datetime.datetime.now) - modified_date = DateTimeField(null=True) - deleted_date = DateTimeField(null=True) + created_date = Column(DateTime, default=datetime.datetime.now) + modified_date = Column(DateTime, null=True) + deleted_date = Column(DateTime, null=True) # Bookmark status; deleting doesn't remove from DB VISIBLE = 0 DELETED = 1 - status = IntegerField(default=VISIBLE) + status = Column(Integer, default=VISIBLE) class Meta: @@ -345,7 +341,7 @@ class Bookmark(BaseModel): stream=True, headers={'User-Agent': DIGIMARKS_USER_AGENT} ) - filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension) + filename = os.path.join(settings.media_dir, 'favicons/', domain + fileextension) with open(filename, 'wb') as out_file: shutil.copyfileobj(response.raw, out_file) del response @@ -387,7 +383,7 @@ class Bookmark(BaseModel): fileextension = '.jpg' if response.headers['content-type'] == 'image/x-icon': fileextension = '.ico' - filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension) + filename = os.path.join(settings.media_dir, 'favicons/', domain + fileextension) with open(filename, 'wb') as out_file: shutil.copyfileobj(response.raw, out_file) del response @@ -406,11 +402,11 @@ class Bookmark(BaseModel): """ Fetch favicon for the domain """ u = urlparse(self.url) domain = u.netloc - if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.png')): + if os.path.isfile(os.path.join(settings.media_dir, 'favicons/', domain + '.png')): # If file exists, don't re-download it self.favicon = domain + '.png' return - if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.ico')): + if os.path.isfile(os.path.join(settings.media_dir, 'favicons/', domain + '.ico')): # If file exists, don't re-download it self.favicon = domain + '.ico' return @@ -466,10 +462,10 @@ class Bookmark(BaseModel): class PublicTag(BaseModel): """ Publicly shared tag """ - tagkey = CharField() - userkey = CharField() - tag = CharField() - created_date = DateTimeField(default=datetime.datetime.now) + tagkey = Column(VARCHAR(255)) + userkey = Column(VARCHAR(255)) + tag = Column(VARCHAR(255)) + created_date = Column(DateTime, default=datetime.datetime.now) def generate_key(self): """ Generate hash-based key for publicly shared tag """ @@ -958,8 +954,8 @@ def publictag_json(tagkey): abort(404) -@app.route('/pub//feed') -def publictag_feed(tagkey): +@app.get('/pub//feed') +async def publictag_feed(request: Request, tagkey: str): """ rss/atom representation of the Read-only overview of the bookmarks in the userkey/tag of this PublicTag """ try: this_tag = PublicTag.get(PublicTag.tagkey == tagkey) @@ -973,7 +969,7 @@ def publictag_feed(tagkey): feed.title(this_tag.tag) feed.id(request.url) feed.link(href=request.url, rel='self') - feed.link(href=make_external(url_for('publictag_page', tagkey=tagkey))) + feed.link(href=make_external(app.url_path_for('publictag_page', tagkey=tagkey))) for bookmark in bookmarks: entry = feed.add_entry() @@ -993,20 +989,21 @@ def publictag_feed(tagkey): entry.published(bookmark.created_date.replace(tzinfo=tz.tzlocal())) entry.updated(updated_date.replace(tzinfo=tz.tzlocal())) - response = make_response(feed.atom_str(pretty=True)) + response = Response(data=feed.atom_str(pretty=True), media_type='application/xml') + response.headers.set('Content-Type', 'application/atom+xml') return response except PublicTag.DoesNotExist: - abort(404) + raise HTTPException(status_code=404, detail='Tag not found') -@app.route('///makepublic', methods=['GET', 'POST']) -def addpublictag(userkey, tag): - #user = get_object_or_404(User.get(User.key == userkey)) +@app.get('///makepublic') +@app.post('///makepublic') +async def addpublictag(userkey: str, tag: str): try: User.get(User.key == userkey) except User.DoesNotExist: - abort(404) + raise HTTPException(status_code=404, detail='User not found') try: publictag = PublicTag.get(PublicTag.userkey == userkey, PublicTag.tag == tag) except PublicTag.DoesNotExist: @@ -1019,10 +1016,14 @@ def addpublictag(userkey, tag): newpublictag.save() message = 'Public link to this tag created' - return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message)) - - message = 'Public link already existed' - return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message)) + success = True + # return RedirectResponse(url=url_path_for('tag_page', userkey=userkey, tag=tag, message=message)) + else: + message = 'Public link already existed' + success = False + # return redirect(url_for('tag_page', userkey=userkey, tag=tag, message=message)) + url = app.url_path_for('tag_page', userkey=userkey, tag=tag, message=message) + return {'success': success, 'message': message, 'url': url} @app.route('///removepublic/', methods=['GET', 'POST'])