diginaut/digimarks
1
0
Fork 0
mirror of https://github.com/aquatix/digimarks.git synced 2025-12-06 23:05:10 +01:00
Code Issues Releases Wiki Activity

Use async httpx instead of requests to do external calls

Browse Source
This commit is contained in:
Michiel Scholten
2024-03-31 21:30:28 +02:00
parent 29c3ccca59
commit 7cbde4cc37
1 changed files with 31 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/digimarks/main.py
View File

@@ -9,7 +9,8 @@ from typing import Optional
from urllib.parse import urljoin, urlparse, urlunparse
import bs4
import requests
import httpx
from contextlib import asynccontextmanager
from dateutil import tz
# from flask import (Flask, abort, jsonify, make_response, redirect,
@@ -53,7 +54,14 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
app = FastAPI()
@asynccontextmanager
async def lifespan(the_app: FastAPI):
"""Upon start, initialise an AsyncClient and assign it to an attribute named requests_client on the app object"""
the_app.requests_client = httpx.AsyncClient()
yield
await the_app.requests_client.aclose()
app = FastAPI(lifespan=lifespan)
templates = Jinja2Templates(directory='templates')
@@ -206,10 +214,10 @@ class Bookmark(Base):
"""Generate hash."""
self.url_hash = hashlib.md5(self.url.encode('utf-8')).hexdigest()
def set_title_from_source(self) -> str:
def set_title_from_source(self, request: Request) -> str:
"""Request the title by requesting the source url."""
try:
result = requests.get(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
result = request.app.requests_client.get(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
self.http_status = result.status_code
except:
# For example 'MissingSchema: Invalid URL 'abc': No schema supplied. Perhaps you meant http://abc?'
@@ -222,19 +230,20 @@ class Bookmark(Base):
self.title = ''
return self.title
def set_status_code(self) -> int:
def set_status_code(self, request: Request) -> int:
"""Check the HTTP status of the url, as it might not exist for example."""
try:
result = requests.head(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT}, timeout=30)
result = request.app.requests_client.head(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT}, timeout=30)
self.http_status = result.status_code
except requests.ConnectionError:
except httpx.HTTPError as e:
logger.error(f'Failed to do head info fetching for {self.url}: {e}')
self.http_status = self.HTTP_CONNECTIONERROR
return self.http_status
def _set_favicon_with_iconsbetterideaorg(self, domain):
def _set_favicon_with_iconsbetterideaorg(self, request: Request, domain):
"""Fetch favicon for the domain."""
fileextension = '.png'
meta = requests.head(
meta = request.app.requests_client.head(
'http://icons.better-idea.org/icon?size=60&url=' + domain,
allow_redirects=True,
headers={'User-Agent': DIGIMARKS_USER_AGENT},
@@ -242,7 +251,7 @@ class Bookmark(Base):
)
if meta.url[-3:].lower() == 'ico':
fileextension = '.ico'
response = requests.get(
response = request.app.requests_client.get(
'http://icons.better-idea.org/icon?size=60&url=' + domain,
stream=True,
headers={'User-Agent': DIGIMARKS_USER_AGENT},
@@ -263,16 +272,16 @@ class Bookmark(Base):
new.write(origcontent)
self.favicon = domain + fileextension
def _set_favicon_with_realfavicongenerator(self, domain):
def _set_favicon_with_realfavicongenerator(self, request: Request, domain: str):
"""Fetch favicon for the domain."""
response = requests.get(
response = request.app.requests_client.get(
'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=android_chrome&site=' + domain,
stream=True,
headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': settings.MASHAPE_API_KEY},
)
if response.status_code == 404:
# Fall back to desktop favicon
response = requests.get(
response = request.app.requests_client.get(
'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=desktop&site=' + domain,
stream=True,
headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': settings.MASHAPE_API_KEY},
@@ -305,7 +314,7 @@ class Bookmark(Base):
new.write(origcontent)
self.favicon = domain + fileextension
def set_favicon(self):
def set_favicon(self, request: Request):
"""Fetch favicon for the domain."""
u = urlparse(self.url)
domain = u.netloc
@@ -318,7 +327,7 @@ class Bookmark(Base):
self.favicon = f'{domain}.ico'
return
# self._set_favicon_with_iconsbetterideaorg(domain)
self._set_favicon_with_realfavicongenerator(domain)
self._set_favicon_with_realfavicongenerator(request, domain)
def set_tags(self, new_tags):
"""Set tags from `tags`, strip and sort them."""
@@ -326,12 +335,12 @@ class Bookmark(Base):
tags_clean = clean_tags(tags_split)
self.tags = ','.join(tags_clean)
def get_redirect_uri(self):
def get_redirect_uri(self, request: Request):
"""Derive where to redirect to."""
if self.redirect_uri:
return self.redirect_uri
if self.http_status in (301, 302):
result = requests.head(
result = request.app.requests_client.head(
self.url, allow_redirects=True, headers={'User-Agent': DIGIMARKS_USER_AGENT}, timeout=30
)
self.http_status = result.status_code
@@ -689,11 +698,11 @@ def update_bookmark(request: Request, userkey, urlhash=None):
# bookmark.fetch_image()
if not title:
# Title was empty, automatically fetch it from the url, will also update the status code
bookmark.set_title_from_source()
bookmark.set_title_from_source(request)
else:
bookmark.set_status_code()
bookmark.set_status_code(request)
if bookmark.http_status == 200 or bookmark.http_status == 202:
if bookmark.http_status in (200, 202):
try:
bookmark.set_favicon()
except IOError:
@@ -998,7 +1007,7 @@ def refresh_favicons(systemkey):
@app.route('/<systemkey>/findmissingfavicons')
def find_missing_favicons(systemkey):
def find_missing_favicons(request: Request, systemkey: str):
"""Add user endpoint, convenience."""
if systemkey == settings.SYSTEMKEY:
bookmarks = Bookmark.select()
@@ -1012,7 +1021,7 @@ def find_missing_favicons(systemkey):
bookmark.favicon = None
bookmark.save()
# Try to fetch and save new favicon
bookmark.set_favicon()
bookmark.set_favicon(request)
bookmark.save()
except OSError as e:
print(e)