From 80f585487a388066e51309a5dc51d576395ac433 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Thu, 11 Sep 2025 16:50:26 +0200 Subject: [PATCH 1/8] Introducing alembic migrations --- alembic.ini | 147 ++++++++++++++++++++++++++++++++++++++ migrations/README | 1 + migrations/env.py | 89 +++++++++++++++++++++++ migrations/script.py.mako | 28 ++++++++ requirements.in | 1 + 5 files changed, 266 insertions(+) create mode 100644 alembic.ini create mode 100644 migrations/README create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..563c6d9 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,147 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..e0d0858 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..9f2d519 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,89 @@ +import asyncio +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/requirements.in b/requirements.in index 5e8c8f3..2e21251 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,7 @@ # Core application fastapi[all] sqlmodel +alembic # Fetch title etc from links beautifulsoup4 From 121937118594be4b1c3c7561fa395b40972d7718 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 12:03:07 +0200 Subject: [PATCH 2/8] async sqlite migrations config --- alembic.ini | 2 +- migrations/env.py | 14 +++++++------- requirements.in | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/alembic.ini b/alembic.ini index 563c6d9..0bc9b86 100644 --- a/alembic.ini +++ b/alembic.ini @@ -84,7 +84,7 @@ path_separator = os # database URL. This is consumed by the user-maintained env.py script only. # other means of configuring database URLs may be customized within the env.py # file. -sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = sqlite+aiosqlite:///bookmarks.db [post_write_hooks] diff --git a/migrations/env.py b/migrations/env.py index 9f2d519..b9ac505 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,11 +1,13 @@ import asyncio from logging.config import fileConfig +from alembic import context from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config +from sqlmodel import SQLModel -from alembic import context +# from app.models import Bookmark # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -20,7 +22,7 @@ if config.config_file_name is not None: # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -target_metadata = None +target_metadata = SQLModel.metadata # other values from the config, defined by the needs of env.py, # can be acquired: @@ -40,12 +42,12 @@ def run_migrations_offline() -> None: script output. """ - url = config.get_main_option("sqlalchemy.url") + url = config.get_main_option('sqlalchemy.url') context.configure( url=url, target_metadata=target_metadata, literal_binds=True, - dialect_opts={"paramstyle": "named"}, + dialect_opts={'paramstyle': 'named'}, ) with context.begin_transaction(): @@ -64,10 +66,9 @@ async def run_async_migrations() -> None: and associate a connection with the context. """ - connectable = async_engine_from_config( config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", + prefix='sqlalchemy.', poolclass=pool.NullPool, ) @@ -79,7 +80,6 @@ async def run_async_migrations() -> None: def run_migrations_online() -> None: """Run migrations in 'online' mode.""" - asyncio.run(run_async_migrations()) diff --git a/requirements.in b/requirements.in index 2e21251..4e915a1 100644 --- a/requirements.in +++ b/requirements.in @@ -2,6 +2,7 @@ fastapi[all] sqlmodel alembic +aiosqlite # Fetch title etc from links beautifulsoup4 From 3a87485b9a392998890aa2a905a8326948e5e813 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 12:03:12 +0200 Subject: [PATCH 3/8] Application now uses async DB session everywhere --- src/digimarks/main.py | 86 +++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/src/digimarks/main.py b/src/digimarks/main.py index 414156d..7c1cee3 100644 --- a/src/digimarks/main.py +++ b/src/digimarks/main.py @@ -20,7 +20,10 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from pydantic import AnyUrl, DirectoryPath, FilePath, computed_field from pydantic_settings import BaseSettings -from sqlmodel import AutoString, Field, Session, SQLModel, create_engine, desc, select +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.orm import sessionmaker +from sqlmodel import AutoString, Field, SQLModel, desc, select +from sqlmodel.ext.asyncio.session import AsyncSession DIGIMARKS_USER_AGENT = 'digimarks/2.0.0-dev' DIGIMARKS_VERSION = '2.0.0a1' @@ -49,16 +52,17 @@ class Settings(BaseSettings): settings = Settings() print(settings.model_dump()) -engine = create_engine(f'sqlite:///{settings.database_file}', connect_args={'check_same_thread': False}) +engine = create_async_engine(f'sqlite+aiosqlite:///{settings.database_file}', connect_args={'check_same_thread': False}) -def get_session(): +async def get_session() -> AsyncSession: """SQLAlchemy session factory.""" - with Session(engine) as session: + async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with async_session() as session: yield session -SessionDep = Annotated[Session, Depends(get_session)] +SessionDep = Annotated[AsyncSession, Depends(get_session)] @asynccontextmanager @@ -358,7 +362,7 @@ def index(request: Request): @app.get('/api/v1/admin/{system_key}/users/{user_id}', response_model=User) -def get_user(session: SessionDep, system_key: str, user_id: int) -> Type[User]: +async def get_user(session: SessionDep, system_key: str, user_id: int) -> Type[User]: """Show user information.""" if system_key != settings.system_key: raise HTTPException(status_code=404) @@ -371,7 +375,7 @@ def get_user(session: SessionDep, system_key: str, user_id: int) -> Type[User]: # @app.get('/admin/{system_key}/users/', response_model=list[User]) @app.get('/api/v1/admin/{system_key}/users/') -def list_users( +async def list_users( session: SessionDep, system_key: str, offset: int = 0, @@ -394,40 +398,42 @@ def list_users( @app.get('/api/v1/{user_key}/bookmarks/') -def list_bookmarks( +async def list_bookmarks( session: SessionDep, user_key: str, offset: int = 0, limit: Annotated[int, Query(le=10000)] = 100, ) -> list[Bookmark]: """List all bookmarks in the database. By default 100 items are returned.""" - bookmarks = session.exec( + result = await session.exec( select(Bookmark) .where(Bookmark.userkey == user_key, Bookmark.status != Visibility.DELETED) .offset(offset) .limit(limit) - ).all() + ) + bookmarks = result.all() return bookmarks @app.get('/api/v1/{user_key}/bookmarks/{url_hash}') -def get_bookmark( +async def get_bookmark( session: SessionDep, user_key: str, url_hash: str, ) -> Bookmark: """Show bookmark details.""" - bookmark = session.exec( + result = await session.exec( select(Bookmark).where( Bookmark.userkey == user_key, Bookmark.url_hash == url_hash, Bookmark.status != Visibility.DELETED ) - ).first() + ) + bookmark = result.first() # bookmark = session.get(Bookmark, {'url_hash': url_hash, 'userkey': user_key}) return bookmark @app.post('/api/v1/{user_key}/autocomplete_bookmark/', response_model=Bookmark) -def autocomplete_bookmark( +async def autocomplete_bookmark( session: SessionDep, request: Request, user_key: str, @@ -441,11 +447,12 @@ def autocomplete_bookmark( update_bookmark_with_info(bookmark, request, strip_params) url_hash = generate_hash(str(bookmark.url)) - bookmark_db = session.exec( + result = await session.exec( select(Bookmark).where( Bookmark.userkey == user_key, Bookmark.url_hash == url_hash, Bookmark.status != Visibility.DELETED ) - ).first() + ) + bookmark_db = result.first() if bookmark_db: # Bookmark with this URL already exists, provide the hash so the frontend can look it up and the user can # merge them if so wanted @@ -455,7 +462,7 @@ def autocomplete_bookmark( @app.post('/api/v1/{user_key}/bookmarks/', response_model=Bookmark) -def add_bookmark( +async def add_bookmark( session: SessionDep, request: Request, user_key: str, @@ -470,13 +477,13 @@ def add_bookmark( bookmark.url_hash = generate_hash(str(bookmark.url)) session.add(bookmark) - session.commit() - session.refresh(bookmark) + await session.commit() + await session.refresh(bookmark) return bookmark @app.patch('/api/v1/{user_key}/bookmarks/{url_hash}', response_model=Bookmark) -def update_bookmark( +async def update_bookmark( session: SessionDep, request: Request, user_key: str, @@ -485,11 +492,12 @@ def update_bookmark( strip_params: bool = False, ): """Update existing bookmark `bookmark_key` for user `user_key`.""" - bookmark_db = session.exec( + result = await session.exec( select(Bookmark).where( Bookmark.userkey == user_key, Bookmark.url_hash == url_hash, Bookmark.status != Visibility.DELETED ) - ).first() + ) + bookmark_db = result.first() if not bookmark_db: raise HTTPException(status_code=404, detail='Bookmark not found') @@ -510,13 +518,14 @@ def update_bookmark( @app.delete('/api/v1/{user_key}/bookmarks/{url_hash}', response_model=Bookmark) -def delete_bookmark( +async def delete_bookmark( session: SessionDep, user_key: str, url_hash: str, ): """(Soft)Delete bookmark `bookmark_key` for user `user_key`.""" - bookmark = session.get(Bookmark, {'url_hash': url_hash, 'userkey': user_key}) + result = await session.get(Bookmark, {'url_hash': url_hash, 'userkey': user_key}) + bookmark = result if not bookmark: raise HTTPException(status_code=404, detail='Bookmark not found') bookmark.deleted_date = datetime.now(UTC) @@ -527,21 +536,23 @@ def delete_bookmark( @app.get('/api/v1/{user_key}/latest_changes/') -def bookmarks_changed_since( +async def bookmarks_changed_since( session: SessionDep, user_key: str, ): """Last update on server, so the (browser) client knows whether to fetch an update.""" - latest_modified_bookmark = session.exec( + result = await session.exec( select(Bookmark) .where(Bookmark.userkey == user_key, Bookmark.status != Visibility.DELETED) .order_by(desc(Bookmark.modified_date)) - ).first() - latest_created_bookmark = session.exec( + ) + latest_modified_bookmark = result.first() + result = await session.exec( select(Bookmark) .where(Bookmark.userkey == user_key, Bookmark.status != Visibility.DELETED) .order_by(desc(Bookmark.created_date)) - ).first() + ) + latest_created_bookmark = result.first() latest_modification = max(latest_modified_bookmark.modified_date, latest_created_bookmark.created_date) @@ -554,14 +565,15 @@ def bookmarks_changed_since( @app.get('/api/v1/{user_key}/tags/') -def list_tags_for_user( +async def list_tags_for_user( session: SessionDep, user_key: str, ) -> list[str]: """List all tags in use by the user.""" - bookmarks = session.exec( + result = await session.exec( select(Bookmark).where(Bookmark.userkey == user_key, Bookmark.status != Visibility.DELETED) - ).all() + ) + bookmarks = result.all() tags = [] for bookmark in bookmarks: tags += bookmark.tag_list @@ -569,23 +581,25 @@ def list_tags_for_user( @app.get('/api/v1/{user_key}/tags/{tag_key}') -def list_tags_for_user( +async def list_tags_for_user( session: SessionDep, user_key: str, ) -> list[str]: """List all tags in use by the user.""" - bookmarks = session.exec(select(Bookmark).where(Bookmark.userkey == user_key)).all() + result = await session.exec(select(Bookmark).where(Bookmark.userkey == user_key)) + bookmarks = result.all() return list_tags_for_bookmarks(bookmarks) @app.get('/{user_key}', response_class=HTMLResponse) -def page_user_landing( +async def page_user_landing( session: SessionDep, request: Request, user_key: str, ): """HTML page with the main view for the user.""" - user = session.exec(select(User).where(User.key == user_key)).first() + result = await session.exec(select(User).where(User.key == user_key)) + user = result.first() if not user: raise HTTPException(status_code=404, detail='User not found') language = 'en' From b6a81fded46370988659130760983d0a05fb4ad2 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 12:15:25 +0200 Subject: [PATCH 4/8] Make alembic comprehend/use sqlmodel --- migrations/env.py | 7 ++----- migrations/script.py.mako | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/migrations/env.py b/migrations/env.py index b9ac505..db84064 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -7,7 +7,7 @@ from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config from sqlmodel import SQLModel -# from app.models import Bookmark +from src.digimarks.main import Bookmark, PublicTag, User # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -62,10 +62,7 @@ def do_run_migrations(connection: Connection) -> None: async def run_async_migrations() -> None: - """In this scenario we need to create an Engine - and associate a connection with the context. - - """ + """In this scenario we need to create an Engine and associate a connection with the context.""" connectable = async_engine_from_config( config.get_section(config.config_ini_section, {}), prefix='sqlalchemy.', diff --git a/migrations/script.py.mako b/migrations/script.py.mako index 1101630..697cf67 100644 --- a/migrations/script.py.mako +++ b/migrations/script.py.mako @@ -9,6 +9,7 @@ from typing import Sequence, Union from alembic import op import sqlalchemy as sa +import sqlmodel ${imports if imports else ""} # revision identifiers, used by Alembic. From 59205166cb3d3bdd0fb0326549188b2e53b8b164 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 12:26:59 +0200 Subject: [PATCH 5/8] Moved DB models to their own module --- src/digimarks/main.py | 110 ++-------------------------------------- src/digimarks/models.py | 107 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 105 deletions(-) create mode 100644 src/digimarks/models.py diff --git a/src/digimarks/main.py b/src/digimarks/main.py index 7c1cee3..69b426e 100644 --- a/src/digimarks/main.py +++ b/src/digimarks/main.py @@ -6,8 +6,7 @@ import logging import os from contextlib import asynccontextmanager from datetime import UTC, datetime -from http import HTTPStatus -from typing import Annotated, Optional, Sequence, Type, TypeVar +from typing import Annotated, Sequence, Type from urllib.parse import urlparse, urlunparse import bs4 @@ -18,18 +17,18 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from pydantic import AnyUrl, DirectoryPath, FilePath, computed_field +from pydantic import AnyUrl, DirectoryPath, FilePath from pydantic_settings import BaseSettings from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.orm import sessionmaker -from sqlmodel import AutoString, Field, SQLModel, desc, select +from sqlmodel import desc, select from sqlmodel.ext.asyncio.session import AsyncSession +from src.digimarks.models import DEFAULT_THEME, Bookmark, User, Visibility + DIGIMARKS_USER_AGENT = 'digimarks/2.0.0-dev' DIGIMARKS_VERSION = '2.0.0a1' -DEFAULT_THEME = 'freshgreen' - class Settings(BaseSettings): """Configuration needed for digimarks to find its database, favicons, API integrations.""" @@ -176,39 +175,6 @@ def generate_key() -> str: return str(binascii.hexlify(os.urandom(24))) -# Type var used for building custom types for the DB -T = TypeVar('T') - - -def build_custom_type(internal_type: Type[T]) -> Type[AutoString]: - """Create a type that is compatible with the database. - - Based on https://github.com/fastapi/sqlmodel/discussions/847 - """ - - class CustomType(AutoString): - def process_bind_param(self, value, dialect) -> Optional[str]: - if value is None: - return None - - if isinstance(value, str): - # Test if value is valid to avoid `process_result_value` failing - try: - internal_type(value) # type: ignore[call-arg] - except ValueError as e: - raise ValueError(f'Invalid value for {internal_type.__name__}: {e}') from e - - return str(value) - - def process_result_value(self, value, dialect) -> Optional[T]: - if value is None: - return None - - return internal_type(value) # type: ignore[call-arg] - - return CustomType - - def get_favicon(html_content: str, root_url: str) -> str: """Fetch the favicon from `html_content` using `root_url`.""" favicons = from_html(html_content, root_url=root_url, include_fallbacks=True) @@ -217,60 +183,6 @@ def get_favicon(html_content: str, root_url: str) -> str: # TODO: save the preferred image to file and return -class User(SQLModel, table=True): - """User account.""" - - __tablename__ = 'user' - - id: int = Field(primary_key=True) - username: str - key: str - theme: str = Field(default=DEFAULT_THEME) - created_date: datetime - - -class Visibility: - """Options for visibility of an object.""" - - VISIBLE = 0 - DELETED = 1 - - -class Bookmark(SQLModel, table=True): - """Bookmark object.""" - - __tablename__ = 'bookmark' - - id: int = Field(primary_key=True) - userkey: str = Field(foreign_key='user.key') - title: str = Field(default='') - url: AnyUrl = Field(default='', sa_type=build_custom_type(AnyUrl)) - note: str = Field(default='') - # image: str = Field(default='') - url_hash: str = Field(default='') - tags: str = Field(default='') - starred: bool = Field(default=False) - - favicon: str | None = Field(default=None) - - http_status: int = Field(default=HTTPStatus.OK) - - created_date: datetime = Field(default=datetime.now(UTC)) - modified_date: datetime = Field(default=None) - deleted_date: datetime = Field(default=None) - - status: int = Field(default=Visibility.VISIBLE) - - @computed_field - @property - def tag_list(self) -> list: - """The tags but as a proper list.""" - if self.tags: - return self.tags.split(',') - # Not tags, return empty list instead of [''] that split returns in that case - return [] - - async def set_information_from_source(bookmark: Bookmark, request: Request) -> Bookmark: """Request the title by requesting the source url.""" logger.info('Extracting information from url %s', bookmark.url) @@ -338,18 +250,6 @@ def update_bookmark_with_info(bookmark: Bookmark, request: Request, strip_params set_tags(bookmark, bookmark.tags) -class PublicTag(SQLModel, table=True): - """Public tag object.""" - - __tablename__ = 'public_tag' - - id: int = Field(primary_key=True) - tagkey: str - userkey: str = Field(foreign_key='user.key') - tag: str - created_date: datetime = Field(default=datetime.now(UTC)) - - @app.get('/', response_class=HTMLResponse) @app.head('/', response_class=HTMLResponse) def index(request: Request): diff --git a/src/digimarks/models.py b/src/digimarks/models.py new file mode 100644 index 0000000..6e2c47d --- /dev/null +++ b/src/digimarks/models.py @@ -0,0 +1,107 @@ +from datetime import UTC, datetime +from http import HTTPStatus +from typing import Optional, Type, TypeVar + +from pydantic import AnyUrl, computed_field +from sqlmodel import AutoString, Field, SQLModel + +DEFAULT_THEME = 'freshgreen' + + +class User(SQLModel, table=True): + """User account.""" + + __tablename__ = 'user' + + id: int = Field(primary_key=True) + username: str + key: str + theme: str = Field(default=DEFAULT_THEME) + created_date: datetime + + +class Visibility: + """Options for visibility of an object.""" + + VISIBLE = 0 + DELETED = 1 + + +# Type var used for building custom types for the DB +T = TypeVar('T') + + +def build_custom_type(internal_type: Type[T]) -> Type[AutoString]: + """Create a type that is compatible with the database. + + Based on https://github.com/fastapi/sqlmodel/discussions/847 + """ + + class CustomType(AutoString): + def process_bind_param(self, value, dialect) -> Optional[str]: + if value is None: + return None + + if isinstance(value, str): + # Test if value is valid to avoid `process_result_value` failing + try: + internal_type(value) # type: ignore[call-arg] + except ValueError as e: + raise ValueError(f'Invalid value for {internal_type.__name__}: {e}') from e + + return str(value) + + def process_result_value(self, value, dialect) -> Optional[T]: + if value is None: + return None + + return internal_type(value) # type: ignore[call-arg] + + return CustomType + + +class Bookmark(SQLModel, table=True): + """Bookmark object.""" + + __tablename__ = 'bookmark' + + id: int = Field(primary_key=True) + userkey: str = Field(foreign_key='user.key') + title: str = Field(default='') + url: AnyUrl = Field(default='', sa_type=build_custom_type(AnyUrl)) + note: str = Field(default='') + # image: str = Field(default='') + url_hash: str = Field(default='') + tags: str = Field(default='') + starred: bool = Field(default=False) + + favicon: str | None = Field(default=None) + + http_status: int = Field(default=HTTPStatus.OK) + + created_date: datetime = Field(default=datetime.now(UTC)) + modified_date: datetime = Field(default=None) + deleted_date: datetime = Field(default=None) + + status: int = Field(default=Visibility.VISIBLE) + + @computed_field + @property + def tag_list(self) -> list: + """The tags but as a proper list.""" + if self.tags: + return self.tags.split(',') + # Not tags, return empty list instead of [''] that split returns in that case + return [] + + +class PublicTag(SQLModel, table=True): + """Public tag object.""" + + __tablename__ = 'public_tag' + + id: int = Field(primary_key=True) + tagkey: str + userkey: str = Field(foreign_key='user.key') + tag: str + created_date: datetime = Field(default=datetime.now(UTC)) From f4afa34f69304e0145f5eaa6cee76684be106240 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 12:30:14 +0200 Subject: [PATCH 6/8] Point alembic config to the new models module --- migrations/env.py | 2 +- src/digimarks/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/env.py b/migrations/env.py index db84064..caa32bf 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -7,7 +7,7 @@ from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config from sqlmodel import SQLModel -from src.digimarks.main import Bookmark, PublicTag, User +from src.digimarks.models import Bookmark, PublicTag, User # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/src/digimarks/models.py b/src/digimarks/models.py index 6e2c47d..bf1c731 100644 --- a/src/digimarks/models.py +++ b/src/digimarks/models.py @@ -98,7 +98,7 @@ class Bookmark(SQLModel, table=True): class PublicTag(SQLModel, table=True): """Public tag object.""" - __tablename__ = 'public_tag' + __tablename__ = 'publictag' id: int = Field(primary_key=True) tagkey: str From ad92e23804527a4e9772f44ed0b929fa690fba94 Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 16:29:17 +0200 Subject: [PATCH 7/8] Initial migration with original state from peewee ORM --- .../115bcd2e1a38_initial_migration.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 migrations/versions/115bcd2e1a38_initial_migration.py diff --git a/migrations/versions/115bcd2e1a38_initial_migration.py b/migrations/versions/115bcd2e1a38_initial_migration.py new file mode 100644 index 0000000..61f43b8 --- /dev/null +++ b/migrations/versions/115bcd2e1a38_initial_migration.py @@ -0,0 +1,66 @@ +"""Initial migration + +Revision ID: 115bcd2e1a38 +Revises: +Create Date: 2025-09-12 16:06:16.479075 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '115bcd2e1a38' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('bookmark', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('userkey', sa.String(length=255), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('url', sa.String(length=255), nullable=False), + sa.Column('created_date', sa.DateTime(), nullable=False), + sa.Column('url_hash', sa.String(length=255), nullable=False), + sa.Column('tags', sa.String(length=255), nullable=False), + sa.Column('http_status', sa.Integer(), nullable=False), + sa.Column('modified_date', sa.DateTime(), nullable=True), + sa.Column('favicon', sa.String(length=255), nullable=True), + sa.Column('starred', sa.Boolean(), server_default=sa.text('0'), nullable=True), + sa.Column('deleted_date', sa.DateTime(), server_default=sa.text('(null)'), nullable=True), + sa.Column('status', sa.Integer(), server_default=sa.text('0'), nullable=True), + sa.Column('note', sa.Text(), server_default=sa.text('(null)'), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('publictag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tagkey', sa.String(length=255), nullable=False), + sa.Column('userkey', sa.String(length=255), nullable=False), + sa.Column('tag', sa.String(length=255), nullable=False), + sa.Column('created_date', sa.DateTime(), server_default=sa.text('(null)'), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('username', sa.String(length=255), nullable=False), + sa.Column('key', sa.String(length=255), nullable=False), + sa.Column('created_date', sa.DateTime(), nullable=False), + sa.Column('theme', sa.String(length=20), server_default=sa.text("'green'"), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('user') + op.drop_table('publictag') + op.drop_table('bookmark') + # ### end Alembic commands ### From ad7f7df21cfe5b7b8a26827afe766267f06e3d5f Mon Sep 17 00:00:00 2001 From: Michiel Scholten Date: Fri, 12 Sep 2025 17:19:11 +0200 Subject: [PATCH 8/8] Migration to sqlmodel way of doing things --- .../a8d8e45f60a1_migrate_to_sqlmodel.py | 95 +++++++++++++++++++ src/digimarks/models.py | 6 +- 2 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 migrations/versions/a8d8e45f60a1_migrate_to_sqlmodel.py diff --git a/migrations/versions/a8d8e45f60a1_migrate_to_sqlmodel.py b/migrations/versions/a8d8e45f60a1_migrate_to_sqlmodel.py new file mode 100644 index 0000000..02b8712 --- /dev/null +++ b/migrations/versions/a8d8e45f60a1_migrate_to_sqlmodel.py @@ -0,0 +1,95 @@ +"""Migrate to sqlmodel. + +Revision ID: a8d8e45f60a1 +Revises: 115bcd2e1a38 +Create Date: 2025-09-12 16:10:41.378716 + +""" +from typing import Sequence, Union + +from alembic import op +from datetime import UTC, datetime +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = 'a8d8e45f60a1' +down_revision: Union[str, Sequence[str], None] = '115bcd2e1a38' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('bookmark', schema=None) as batch_op: + batch_op.alter_column('note', + existing_type=sa.TEXT(), + type_=sqlmodel.sql.sqltypes.AutoString(), + nullable=True, + existing_server_default=sa.text('(null)')) + batch_op.alter_column('starred', + existing_type=sa.BOOLEAN(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.alter_column('modified_date', + existing_type=sa.DATETIME(), + nullable=True) + batch_op.alter_column('deleted_date', + existing_type=sa.DATETIME(), + nullable=True, + existing_server_default=sa.text('(null)')) + batch_op.alter_column('status', + existing_type=sa.INTEGER(), + nullable=False, + existing_server_default=sa.text('0')) + batch_op.create_foreign_key('bookmark_user', 'user', ['userkey'], ['key']) + with op.batch_alter_table('publictag', schema=None) as batch_op: + batch_op.alter_column('created_date', + existing_type=sa.DATETIME(), + nullable=True, + existing_server_default=sa.text(str(datetime.now(UTC)))) + batch_op.create_foreign_key('publictag_user', 'user', ['userkey'], ['key']) + with op.batch_alter_table('user', schema=None) as batch_op: + batch_op.alter_column('theme', + existing_type=sa.VARCHAR(length=20), + nullable=False, + existing_server_default=sa.text("'green'")) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('user', 'theme', + existing_type=sa.VARCHAR(length=20), + nullable=True, + existing_server_default=sa.text("'green'")) + op.drop_constraint(None, 'publictag', type_='foreignkey') + op.alter_column('publictag', 'created_date', + existing_type=sa.DATETIME(), + nullable=True, + existing_server_default=sa.text('(null)')) + op.drop_constraint(None, 'bookmark', type_='foreignkey') + op.alter_column('bookmark', 'status', + existing_type=sa.INTEGER(), + nullable=True, + existing_server_default=sa.text('0')) + op.alter_column('bookmark', 'deleted_date', + existing_type=sa.DATETIME(), + nullable=True, + existing_server_default=sa.text('(null)')) + op.alter_column('bookmark', 'modified_date', + existing_type=sa.DATETIME(), + nullable=True) + op.alter_column('bookmark', 'starred', + existing_type=sa.BOOLEAN(), + nullable=True, + existing_server_default=sa.text('0')) + op.alter_column('bookmark', 'note', + existing_type=sqlmodel.sql.sqltypes.AutoString(), + type_=sa.TEXT(), + nullable=True, + existing_server_default=sa.text('(null)')) + # ### end Alembic commands ### diff --git a/src/digimarks/models.py b/src/digimarks/models.py index bf1c731..bada175 100644 --- a/src/digimarks/models.py +++ b/src/digimarks/models.py @@ -69,7 +69,7 @@ class Bookmark(SQLModel, table=True): userkey: str = Field(foreign_key='user.key') title: str = Field(default='') url: AnyUrl = Field(default='', sa_type=build_custom_type(AnyUrl)) - note: str = Field(default='') + note: str = Field(default='', nullable=True) # image: str = Field(default='') url_hash: str = Field(default='') tags: str = Field(default='') @@ -80,8 +80,8 @@ class Bookmark(SQLModel, table=True): http_status: int = Field(default=HTTPStatus.OK) created_date: datetime = Field(default=datetime.now(UTC)) - modified_date: datetime = Field(default=None) - deleted_date: datetime = Field(default=None) + modified_date: datetime = Field(default=None, nullable=True) + deleted_date: datetime = Field(default=None, nullable=True) status: int = Field(default=Visibility.VISIBLE)