Use environment vars; fix modularising a bit

README.rst

@@ -39,8 +39,6 @@ Usage / example configuration
 Copy ``settings.py`` from example_config to the parent directory and
 configure to your needs (*at the least* change the value of `SYSTEMKEY`).
 
-Do not forget to fill in the `MASHAPE_API_KEY` value, which you [can request on the RapidAPI website](https://rapidapi.com/realfavicongenerator/api/realfavicongenerator).
-
 Run digimarks as a service under nginx or apache and call the appropriate
 url's when wanted.
 

digimarks/__init__.py

@@ -0,0 +1,29 @@
+import os
+
+from flask import Flask
+
+
+def create_app(test_config=None):
+    # create our flask app and a database wrapper
+    app = Flask(__name__)
+    app.config.from_object(__name__)
+
+    if test_config is None:
+        # load the instance config, if it exists, when not testing
+        app.config.from_pyfile('settings.py', silent=True)
+    else:
+        # load the test config if passed in
+        app.config.from_mapping(test_config)
+
+    # Strip unnecessary whitespace due to jinja2 codeblocks
+    app.jinja_env.trim_blocks = True
+    app.jinja_env.lstrip_blocks = True
+
+    # set custom url for the app, for example '/bookmarks'
+    try:
+        # TODO: get settings from ENV vars
+        app.config['APPLICATION_ROOT'] = os.environ['APPLICATION_ROOT']
+    except AttributeError:
+        pass
+
+    return app

digimarks/marks.py

@@ -1,154 +1,17 @@
 from __future__ import print_function
 
-import binascii
 import datetime
-import gzip
-import hashlib
 import os
-import shutil
 import sys
+from urllib.parse import urljoin
 
-import bs4
-import requests
-from dateutil import tz
-from feedgen.feed import FeedGenerator
-from flask import (Flask, abort, jsonify, make_response, redirect,
-                   render_template, request, url_for)
-from peewee import *  # noqa
+from flask import abort, jsonify, redirect, render_template, request, url_for
+from werkzeug.contrib.atom import AtomFeed
 
-try:
-    # Python 3
-    from urllib.parse import urljoin, urlparse, urlunparse
-except ImportError:
-    # Python 2
-    from urlparse import urljoin, urlparse, urlunparse
+from . import themes
+from .models import Bookmark, PublicTag, User, get_tags_for_user
 
-
-DIGIMARKS_USER_AGENT = 'digimarks/1.2.0-dev'
-
-DEFAULT_THEME = 'freshgreen'
-themes = {
-    'green': {
-        'BROWSERCHROME': '#2e7d32',  # green darken-2
-        'BODY': 'grey lighten-4',
-        'TEXT': 'black-text',
-        'TEXTHEX': '#000',
-        'NAV': 'green darken-3',
-        'PAGEHEADER': 'grey-text lighten-5',
-        'MESSAGE_BACKGROUND': 'orange lighten-2',
-        'MESSAGE_TEXT': 'white-text',
-        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
-        'ERRORMESSAGE_TEXT': 'white-text',
-        'BUTTON': '#1b5e20', # green darken-4
-        'BUTTON_ACTIVE': '#43a047',  # green darken-1
-        'LINK_TEXT': '#1b5e20',  # green darken-4
-        'CARD_BACKGROUND': 'green darken-3',
-        'CARD_TEXT': 'white-text',
-        'CARD_LINK': '#FFF',  # white-text
-        'CHIP_TEXT': '#1b5e20',  # green darken-4
-        'FAB': 'red',
-
-        'STAR': 'yellow-text',
-        'PROBLEM': 'red-text',
-        'COMMENT': '',
-    },
-    'freshgreen': {
-        'BROWSERCHROME': '#43a047',  # green darken-1
-        'BODY': 'grey lighten-5',
-        'TEXT': 'black-text',
-        'TEXTHEX': '#000',
-        'NAV': 'green darken-1',
-        'PAGEHEADER': 'grey-text lighten-5',
-        'MESSAGE_BACKGROUND': 'orange lighten-2',
-        'MESSAGE_TEXT': 'white-text',
-        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
-        'ERRORMESSAGE_TEXT': 'white-text',
-        'BUTTON': '#1b5e20', # green darken-4
-        'BUTTON_ACTIVE': '#43a047',  # green darken-1
-        'LINK_TEXT': '#1b5e20',  # green darken-4
-        'CARD_BACKGROUND': 'green darken-1',
-        'CARD_TEXT': 'white-text',
-        'CARD_LINK': '#FFF',  # white-text
-        'CHIP_TEXT': '#1b5e20',  # green darken-4
-        'FAB': 'red',
-
-        'STAR': 'yellow-text',
-        'PROBLEM': 'red-text',
-        'COMMENT': '',
-    },
-    'lightblue': {
-        'BROWSERCHROME': '#0288d1',  # light-blue darken-2
-        'BODY': 'white',
-        'TEXT': 'black-text',
-        'TEXTHEX': '#000',
-        'NAV': 'light-blue darken-2',
-        'PAGEHEADER': 'grey-text lighten-5',
-        'MESSAGE_BACKGROUND': 'orange lighten-2',
-        'MESSAGE_TEXT': 'white-text',
-        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
-        'ERRORMESSAGE_TEXT': 'white-text',
-        'BUTTON': '#fb8c00', # orange darken-1
-        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
-        'LINK_TEXT': '#FFF',  # white
-        'CARD_BACKGROUND': 'light-blue lighten-2',
-        'CARD_TEXT': 'black-text',
-        'CARD_LINK': '#263238',  # blue-grey-text darken-4
-        'CHIP_TEXT': '#FFF',  # white
-        'FAB': 'light-blue darken-4',
-
-        'STAR': 'yellow-text',
-        'PROBLEM': 'red-text',
-        'COMMENT': '',
-    },
-    'dark': {
-        'BROWSERCHROME': '#212121',  # grey darken-4
-        'BODY': 'grey darken-4',
-        'TEXT': 'grey-text lighten-1',
-        'TEXTHEX': '#bdbdbd',
-        'NAV': 'grey darken-3',
-        'PAGEHEADER': 'grey-text lighten-1',
-        'MESSAGE_BACKGROUND': 'orange lighten-2',
-        'MESSAGE_TEXT': 'white-text',
-        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
-        'ERRORMESSAGE_TEXT': 'white-text',
-        'BUTTON': '#fb8c00', # orange darken-1
-        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
-        'LINK_TEXT': '#fb8c00',  # orange-text darken-1
-        'CARD_BACKGROUND': 'grey darken-3',
-        'CARD_TEXT': 'grey-text lighten-1',
-        'CARD_LINK': '#fb8c00',  # orange-text darken-1
-        'CHIP_TEXT': '#fb8c00',  # orange-text darken-1
-        'FAB': 'red',
-
-        'STAR': 'yellow-text',
-        'PROBLEM': 'red-text',
-        'COMMENT': '',
-    },
-    'amoled': {
-        'BROWSERCHROME': '#000',  # grey darken-4
-        'BODY': 'black',
-        'TEXT': 'grey-text lighten-1',
-        'TEXTHEX': '#bdbdbd',
-        'NAV': 'grey darken-3',
-        'PAGEHEADER': 'grey-text lighten-1',
-        'MESSAGE_BACKGROUND': 'orange lighten-2',
-        'MESSAGE_TEXT': 'white-text',
-        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
-        'ERRORMESSAGE_TEXT': 'white-text',
-        'BUTTON': '#fb8c00', # orange darken-1
-        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
-        'LINK_TEXT': '#fb8c00',  # orange-text darken-1
-        'CARD_BACKGROUND': 'grey darken-3',
-        'CARD_TEXT': 'grey-text lighten-1',
-        'CARD_LINK': '#fb8c00',  # orange-text darken-1
-        'CHIP_TEXT': '#fb8c00',  # orange-text darken-1
-        'FAB': 'red',
-
-        'STAR': 'yellow-text',
-        'PROBLEM': 'red-text',
-        'COMMENT': '',
-    }
-}
+DIGIMARKS_USER_AGENT = 'digimarks/2.0.0-dev'
 
 try:
     import settings
@@ -166,320 +29,23 @@ DATABASE = {
 }
 #PHANTOM = '/usr/local/bin/phantomjs'
 #SCRIPT = os.path.join(APP_ROOT, 'screenshot.js')
-
-# create our flask app and a database wrapper
-app = Flask(__name__)
-app.config.from_object(__name__)
-database = SqliteDatabase(os.path.join(APP_ROOT, 'bookmarks.db'))
-
-# Strip unnecessary whitespace due to jinja2 codeblocks
-app.jinja_env.trim_blocks = True
-app.jinja_env.lstrip_blocks = True
-
-# set custom url for the app, for example '/bookmarks'
+SYSTEMKEY = None
 try:
-    app.config['APPLICATION_ROOT'] = settings.APPLICATION_ROOT
-except AttributeError:
-    pass
+    SYSTEMKEY = os.environ['SYSTEMKEY']
+except KeyError:
+    print('No ENV var found for SYSTEMKEY')
+
+MASHAPE_API_KEY = None
+try:
+    MASHAPE_API_KEY = os.environ['MASHAPE_API_KEY']
+except KeyError:
+    print('No ENV var found for MASHAPE_API_KEY')
 
 # Cache the tags
 all_tags = {}
 usersettings = {}
 
 
-def ifilterfalse(predicate, iterable):
-    # ifilterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8
-    if predicate is None:
-        predicate = bool
-    for x in iterable:
-        if not predicate(x):
-            yield x
-
-
-def unique_everseen(iterable, key=None):
-    "List unique elements, preserving order. Remember all elements ever seen."
-    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
-    # unique_everseen('ABBCcAD', str.lower) --> A B C D
-    seen = set()
-    seen_add = seen.add
-    if key is None:
-        for element in ifilterfalse(seen.__contains__, iterable):
-            seen_add(element)
-            yield element
-    else:
-        for element in iterable:
-            k = key(element)
-            if k not in seen:
-                seen_add(k)
-                yield element
-
-def clean_tags(tags_list):
-    tags_res = [x.strip() for x in tags_list]
-    tags_res = list(unique_everseen(tags_res))
-    tags_res.sort()
-    if tags_res and tags_res[0] == '':
-        del tags_res[0]
-    return tags_res
-
-
-magic_dict = {
-    b"\x1f\x8b\x08": "gz",
-    b"\x42\x5a\x68": "bz2",
-    b"\x50\x4b\x03\x04": "zip"
-    }
-
-max_len = max(len(x) for x in magic_dict)
-
-def file_type(filename):
-    with open(filename, "rb") as f:
-        file_start = f.read(max_len)
-    for magic, filetype in magic_dict.items():
-        if file_start.startswith(magic):
-            return filetype
-    return "no match"
-
-
-class BaseModel(Model):
-    class Meta:
-        database = database
-
-
-class User(BaseModel):
-    """ User account """
-    username = CharField()
-    key = CharField()
-    theme = CharField(default=DEFAULT_THEME)
-    created_date = DateTimeField(default=datetime.datetime.now)
-
-    def generate_key(self):
-        """ Generate userkey """
-        self.key = binascii.hexlify(os.urandom(24))
-        return self.key
-
-
-class Bookmark(BaseModel):
-    """ Bookmark instance, connected to User """
-    # Foreign key to User
-    userkey = CharField()
-
-    title = CharField(default='')
-    url = CharField()
-    note = TextField(default='')
-    #image = CharField(default='')
-    url_hash = CharField(default='')
-    tags = CharField(default='')
-    starred = BooleanField(default=False)
-
-    # Website (domain) favicon
-    favicon = CharField(null=True)
-
-    # Status code: 200 is OK, 404 is not found, for example (showing an error)
-    HTTP_CONNECTIONERROR = 0
-    HTTP_OK = 200
-    HTTP_ACCEPTED = 202
-    HTTP_MOVEDTEMPORARILY = 304
-    HTTP_NOTFOUND = 404
-
-    http_status = IntegerField(default=200)
-    redirect_uri = None
-
-    created_date = DateTimeField(default=datetime.datetime.now)
-    modified_date = DateTimeField(null=True)
-    deleted_date = DateTimeField(null=True)
-
-    # Bookmark status; deleting doesn't remove from DB
-    VISIBLE = 0
-    DELETED = 1
-    status = IntegerField(default=VISIBLE)
-
-
-    class Meta:
-        ordering = (('created_date', 'desc'),)
-
-    def set_hash(self):
-        """ Generate hash """
-        self.url_hash = hashlib.md5(self.url.encode('utf-8')).hexdigest()
-
-    def set_title_from_source(self):
-        """ Request the title by requesting the source url """
-        try:
-            result = requests.get(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
-            self.http_status = result.status_code
-        except:
-            # For example 'MissingSchema: Invalid URL 'abc': No schema supplied. Perhaps you meant http://abc?'
-            self.http_status = 404
-        if self.http_status == 200 or self.http_status == 202:
-            html = bs4.BeautifulSoup(result.text, 'html.parser')
-            try:
-                self.title = html.title.text.strip()
-            except AttributeError:
-                self.title = ''
-        return self.title
-
-    def set_status_code(self):
-        """ Check the HTTP status of the url, as it might not exist for example """
-        try:
-            result = requests.head(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
-            self.http_status = result.status_code
-        except requests.ConnectionError:
-            self.http_status = self.HTTP_CONNECTIONERROR
-        return self.http_status
-
-    def _set_favicon_with_iconsbetterideaorg(self, domain):
-        """ Fetch favicon for the domain """
-        fileextension = '.png'
-        meta = requests.head(
-            'http://icons.better-idea.org/icon?size=60&url=' + domain,
-            allow_redirects=True,
-            headers={'User-Agent': DIGIMARKS_USER_AGENT}
-        )
-        if meta.url[-3:].lower() == 'ico':
-            fileextension = '.ico'
-        response = requests.get(
-            'http://icons.better-idea.org/icon?size=60&url=' + domain,
-            stream=True,
-            headers={'User-Agent': DIGIMARKS_USER_AGENT}
-        )
-        filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
-        with open(filename, 'wb') as out_file:
-            shutil.copyfileobj(response.raw, out_file)
-        del response
-        filetype = file_type(filename)
-        if filetype == 'gz':
-            # decompress
-            orig = gzip.GzipFile(filename, 'rb')
-            origcontent = orig.read()
-            orig.close()
-            os.remove(filename)
-            with open(filename, 'wb') as new:
-                new.write(origcontent)
-        self.favicon = domain + fileextension
-
-    def _set_favicon_with_realfavicongenerator(self, domain):
-        """ Fetch favicon for the domain """
-        response = requests.get(
-            'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=android_chrome&site=' + domain,
-            stream=True,
-            headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': settings.MASHAPE_API_KEY}
-        )
-        if response.status_code == 404:
-            # Fall back to desktop favicon
-            response = requests.get(
-                'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=desktop&site=' + domain,
-                stream=True,
-                headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': settings.MASHAPE_API_KEY}
-            )
-        # Debug for the moment
-        print(domain)
-        print(response.headers)
-        if 'Content-Length' in response.headers and response.headers['Content-Length'] == '0':
-            # No favicon found, likely
-            print('Skipping this favicon, needs fallback')
-            return
-        # Default to 'image/png'
-        fileextension = '.png'
-        if response.headers['content-type'] == 'image/jpeg':
-            fileextension = '.jpg'
-        if response.headers['content-type'] == 'image/x-icon':
-            fileextension = '.ico'
-        filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
-        with open(filename, 'wb') as out_file:
-            shutil.copyfileobj(response.raw, out_file)
-        del response
-        filetype = file_type(filename)
-        if filetype == 'gz':
-            # decompress
-            orig = gzip.GzipFile(filename, 'rb')
-            origcontent = orig.read()
-            orig.close()
-            os.remove(filename)
-            with open(filename, 'wb') as new:
-                new.write(origcontent)
-        self.favicon = domain + fileextension
-
-    def set_favicon(self):
-        """ Fetch favicon for the domain """
-        u = urlparse(self.url)
-        domain = u.netloc
-        if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.png')):
-            # If file exists, don't re-download it
-            self.favicon = domain + '.png'
-            return
-        if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.ico')):
-            # If file exists, don't re-download it
-            self.favicon = domain + '.ico'
-            return
-        #self._set_favicon_with_iconsbetterideaorg(domain)
-        self._set_favicon_with_realfavicongenerator(domain)
-
-    def set_tags(self, newtags):
-        """ Set tags from `tags`, strip and sort them """
-        tags_split = newtags.split(',')
-        tags_clean = clean_tags(tags_split)
-        self.tags = ','.join(tags_clean)
-
-    def get_redirect_uri(self):
-        if self.redirect_uri:
-            return self.redirect_uri
-        if self.http_status == 301 or self.http_status == 302:
-            result = requests.head(self.url, allow_redirects=True, headers={'User-Agent': DIGIMARKS_USER_AGENT})
-            self.http_status = result.status_code
-            self.redirect_uri = result.url
-            return result.url
-        return None
-
-    def get_uri_domain(self):
-        parsed = urlparse(self.url)
-        return parsed.hostname
-
-    @classmethod
-    def strip_url_params(cls, url):
-        parsed = urlparse(url)
-        return urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, '', parsed.fragment))
-
-    @property
-    def tags_list(self):
-        """ Get the tags as a list, iterable in template """
-        if self.tags:
-            return self.tags.split(',')
-        return []
-
-    def to_dict(self):
-        result = {
-            'title': self.title,
-            'url': self.url,
-            'created':  self.created_date.strftime('%Y-%m-%d %H:%M:%S'),
-            'url_hash': self.url_hash,
-            'tags': self.tags,
-        }
-        return result
-
-    @property
-    def serialize(self):
-        return self.to_dict()
-
-
-class PublicTag(BaseModel):
-    """ Publicly shared tag """
-    tagkey = CharField()
-    userkey = CharField()
-    tag = CharField()
-    created_date = DateTimeField(default=datetime.datetime.now)
-
-    def generate_key(self):
-        """ Generate hash-based key for publicly shared tag """
-        self.tagkey = binascii.hexlify(os.urandom(16))
-
-
-def get_tags_for_user(userkey):
-    """ Extract all tags from the bookmarks """
-    bookmarks = Bookmark.select().filter(Bookmark.userkey == userkey, Bookmark.status == Bookmark.VISIBLE)
-    tags = []
-    for bookmark in bookmarks:
-        tags += bookmark.tags_list
-    return clean_tags(tags)
-
-
 def get_cached_tags(userkey):
     """ Fail-safe way to get the cached tags for `userkey` """
     try:
@@ -491,9 +57,9 @@ def get_cached_tags(userkey):
 def get_theme(userkey):
     try:
         usertheme = usersettings[userkey]['theme']
-        return themes[usertheme]
+        return themes.themes[usertheme]
     except KeyError:
-        return themes[DEFAULT_THEME]  # default
+        return themes.themes[themes.DEFAULT_THEME]  # default
 
 
 def make_external(url):
@@ -514,14 +80,14 @@ def _find_bookmarks(userkey, filter_text):
 
 @app.errorhandler(404)
 def page_not_found(e):
-    theme = themes[DEFAULT_THEME]
+    theme = themes.themes[themes.DEFAULT_THEME]
     return render_template('404.html', error=e, theme=theme), 404
 
 
 @app.route('/')
 def index():
     """ Homepage, point visitors to project page """
-    theme = themes[DEFAULT_THEME]
+    theme = themes.themes[themes.DEFAULT_THEME]
     return render_template('index.html', theme=theme)
 
 
@@ -605,12 +171,10 @@ def bookmarks_js(userkey):
         Bookmark.userkey == userkey,
         Bookmark.status == Bookmark.VISIBLE
     ).order_by(Bookmark.created_date.desc())
-    resp = make_response(render_template(
+    return render_template(
         'bookmarks.js',
         bookmarks=bookmarks
-    ))
-    resp.headers['Content-type'] = 'text/javascript; charset=utf-8'
-    return resp
+    )
 
 
 @app.route('/r/<userkey>/<urlhash>')
@@ -922,7 +486,7 @@ def publictag_page(tagkey):
     #this_tag = get_object_or_404(PublicTag.select().where(PublicTag.tagkey == tagkey))
     try:
         this_tag, bookmarks = get_publictag(tagkey)
-        theme = themes[DEFAULT_THEME]
+        theme = themes.themes[themes.DEFAULT_THEME]
         return render_template(
             'publicbookmarks.html',
             bookmarks=bookmarks,
@@ -963,34 +527,23 @@ def publictag_feed(tagkey):
             Bookmark.tags.contains(this_tag.tag),
             Bookmark.status == Bookmark.VISIBLE
         )
-
-        feed = FeedGenerator()
-        feed.title(this_tag.tag)
-        feed.id(request.url)
-        feed.link(href=request.url, rel='self')
-        feed.link(href=make_external(url_for('publictag_page', tagkey=tagkey)))
-
+        feed = AtomFeed(this_tag.tag, feed_url=request.url, url=make_external(url_for('publictag_page', tagkey=tagkey)))
         for bookmark in bookmarks:
-            entry = feed.add_entry()
-
             updated_date = bookmark.modified_date
             if not bookmark.modified_date:
                 updated_date = bookmark.created_date
             bookmarktitle = '{} (no title)'.format(bookmark.url)
             if bookmark.title:
                 bookmarktitle = bookmark.title
-
-            entry.id(bookmark.url)
-            entry.title(bookmarktitle)
-            entry.link(href=bookmark.url)
-            entry.author(name='digimarks')
-            entry.pubdate(bookmark.created_date.replace(tzinfo=tz.tzlocal()))
-            entry.published(bookmark.created_date.replace(tzinfo=tz.tzlocal()))
-            entry.updated(updated_date.replace(tzinfo=tz.tzlocal()))
-
-        response = make_response(feed.atom_str(pretty=True))
-        response.headers.set('Content-Type', 'application/atom+xml')
-        return response
+            feed.add(
+                bookmarktitle,
+                content_type='html',
+                author='digimarks',
+                url=bookmark.url,
+                updated=updated_date,
+                published=bookmark.created_date
+            )
+        return feed.get_response()
     except PublicTag.DoesNotExist:
         abort(404)
 
@@ -1083,6 +636,7 @@ def findmissingfavicons(systemkey):
 
 
 # Initialisation == create the bookmark, user and public tag tables if they do not exist
+# TODO: move to __init__.py
 Bookmark.create_table(True)
 User.create_table(True)
 PublicTag.create_table(True)
@@ -1093,8 +647,3 @@ for user in users:
     all_tags[user.key] = get_tags_for_user(user.key)
     usersettings[user.key] = {'theme': user.theme}
     print(user.key)
-
-# Run when called standalone
-if __name__ == '__main__':
-    # run the application
-    app.run(host='0.0.0.0', port=9999, debug=True)

digimarks/models.py

@@ -0,0 +1,312 @@
+"""digimarks data models and accompanying convenience functions"""
+import binascii
+import datetime
+import gzip
+import hashlib
+import os
+import shutil
+from urllib.parse import urlparse, urlunparse
+
+import bs4
+import requests
+from peewee import *  # noqa
+
+from . import themes
+
+DATABASE_PATH = os.path.dirname(os.path.realpath(__file__))
+if 'DIGIMARKS_DB_PATH' in os.environ:
+    DATABASE_PATH = os.environ['DIGIMARKS_DB_PATH']
+database = SqliteDatabase(os.path.join(DATABASE_PATH, 'bookmarks.db'))
+
+
+def ifilterfalse(predicate, iterable):
+    # ifilterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8
+    if predicate is None:
+        predicate = bool
+    for x in iterable:
+        if not predicate(x):
+            yield x
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in ifilterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
+
+def clean_tags(tags_list):
+    tags_res = [x.strip() for x in tags_list]
+    tags_res = list(unique_everseen(tags_res))
+    tags_res.sort()
+    if tags_res and tags_res[0] == '':
+        del tags_res[0]
+    return tags_res
+
+
+magic_dict = {
+    b"\x1f\x8b\x08": "gz",
+    b"\x42\x5a\x68": "bz2",
+    b"\x50\x4b\x03\x04": "zip"
+    }
+
+max_len = max(len(x) for x in magic_dict)
+
+def file_type(filename):
+    with open(filename, "rb") as f:
+        file_start = f.read(max_len)
+    for magic, filetype in magic_dict.items():
+        if file_start.startswith(magic):
+            return filetype
+    return "no match"
+
+
+class BaseModel(Model):
+    class Meta:
+        database = database
+
+
+class User(BaseModel):
+    """ User account """
+    username = CharField()
+    key = CharField()
+    theme = CharField(default=themes.DEFAULT_THEME)
+    created_date = DateTimeField(default=datetime.datetime.now)
+
+    def generate_key(self):
+        """ Generate userkey """
+        self.key = binascii.hexlify(os.urandom(24))
+        return self.key
+
+
+class Bookmark(BaseModel):
+    """ Bookmark instance, connected to User """
+    # Foreign key to User
+    userkey = CharField()
+
+    title = CharField(default='')
+    url = CharField()
+    note = TextField(default='')
+    #image = CharField(default='')
+    url_hash = CharField(default='')
+    tags = CharField(default='')
+    starred = BooleanField(default=False)
+
+    # Website (domain) favicon
+    favicon = CharField(null=True)
+
+    # Status code: 200 is OK, 404 is not found, for example (showing an error)
+    HTTP_CONNECTIONERROR = 0
+    HTTP_OK = 200
+    HTTP_ACCEPTED = 202
+    HTTP_MOVEDTEMPORARILY = 304
+    HTTP_NOTFOUND = 404
+
+    http_status = IntegerField(default=200)
+    redirect_uri = None
+
+    created_date = DateTimeField(default=datetime.datetime.now)
+    modified_date = DateTimeField(null=True)
+    deleted_date = DateTimeField(null=True)
+
+    # Bookmark status; deleting doesn't remove from DB
+    VISIBLE = 0
+    DELETED = 1
+    status = IntegerField(default=VISIBLE)
+
+
+    class Meta:
+        ordering = (('created_date', 'desc'),)
+
+    def set_hash(self):
+        """ Generate hash """
+        self.url_hash = hashlib.md5(self.url.encode('utf-8')).hexdigest()
+
+    def set_title_from_source(self):
+        """ Request the title by requesting the source url """
+        try:
+            result = requests.get(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
+            self.http_status = result.status_code
+        except:
+            # For example 'MissingSchema: Invalid URL 'abc': No schema supplied. Perhaps you meant http://abc?'
+            self.http_status = 404
+        if self.http_status == 200 or self.http_status == 202:
+            html = bs4.BeautifulSoup(result.text, 'html.parser')
+            try:
+                self.title = html.title.text.strip()
+            except AttributeError:
+                self.title = ''
+        return self.title
+
+    def set_status_code(self):
+        """ Check the HTTP status of the url, as it might not exist for example """
+        try:
+            result = requests.head(self.url, headers={'User-Agent': DIGIMARKS_USER_AGENT})
+            self.http_status = result.status_code
+        except requests.ConnectionError:
+            self.http_status = self.HTTP_CONNECTIONERROR
+        return self.http_status
+
+    def _set_favicon_with_iconsbetterideaorg(self, domain):
+        """ Fetch favicon for the domain """
+        fileextension = '.png'
+        meta = requests.head(
+            'http://icons.better-idea.org/icon?size=60&url=' + domain,
+            allow_redirects=True,
+            headers={'User-Agent': DIGIMARKS_USER_AGENT}
+        )
+        if meta.url[-3:].lower() == 'ico':
+            fileextension = '.ico'
+        response = requests.get(
+            'http://icons.better-idea.org/icon?size=60&url=' + domain,
+            stream=True,
+            headers={'User-Agent': DIGIMARKS_USER_AGENT}
+        )
+        filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
+        with open(filename, 'wb') as out_file:
+            shutil.copyfileobj(response.raw, out_file)
+        del response
+        filetype = file_type(filename)
+        if filetype == 'gz':
+            # decompress
+            orig = gzip.GzipFile(filename, 'rb')
+            origcontent = orig.read()
+            orig.close()
+            os.remove(filename)
+            with open(filename, 'wb') as new:
+                new.write(origcontent)
+        self.favicon = domain + fileextension
+
+    def _set_favicon_with_realfavicongenerator(self, domain):
+        """ Fetch favicon for the domain """
+        response = requests.get(
+            'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=android_chrome&site=' + domain,
+            stream=True,
+            headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': MASHAPE_API_KEY}
+        )
+        if response.status_code == 404:
+            # Fall back to desktop favicon
+            response = requests.get(
+                'https://realfavicongenerator.p.rapidapi.com/favicon/icon?platform=desktop&site=' + domain,
+                stream=True,
+                headers={'User-Agent': DIGIMARKS_USER_AGENT, 'X-Mashape-Key': settings.MASHAPE_API_KEY}
+            )
+        # Debug for the moment
+        print(domain)
+        print(response.headers)
+        if 'Content-Length' in response.headers and response.headers['Content-Length'] == '0':
+            # No favicon found, likely
+            print('Skipping this favicon, needs fallback')
+            return
+        # Default to 'image/png'
+        fileextension = '.png'
+        if response.headers['content-type'] == 'image/jpeg':
+            fileextension = '.jpg'
+        if response.headers['content-type'] == 'image/x-icon':
+            fileextension = '.ico'
+        filename = os.path.join(MEDIA_ROOT, 'favicons/' + domain + fileextension)
+        with open(filename, 'wb') as out_file:
+            shutil.copyfileobj(response.raw, out_file)
+        del response
+        filetype = file_type(filename)
+        if filetype == 'gz':
+            # decompress
+            orig = gzip.GzipFile(filename, 'rb')
+            origcontent = orig.read()
+            orig.close()
+            os.remove(filename)
+            with open(filename, 'wb') as new:
+                new.write(origcontent)
+        self.favicon = domain + fileextension
+
+    def set_favicon(self):
+        """ Fetch favicon for the domain """
+        u = urlparse(self.url)
+        domain = u.netloc
+        if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.png')):
+            # If file exists, don't re-download it
+            self.favicon = domain + '.png'
+            return
+        if os.path.isfile(os.path.join(MEDIA_ROOT, 'favicons/' + domain + '.ico')):
+            # If file exists, don't re-download it
+            self.favicon = domain + '.ico'
+            return
+        #self._set_favicon_with_iconsbetterideaorg(domain)
+        self._set_favicon_with_realfavicongenerator(domain)
+
+    def set_tags(self, newtags):
+        """ Set tags from `tags`, strip and sort them """
+        tags_split = newtags.split(',')
+        tags_clean = clean_tags(tags_split)
+        self.tags = ','.join(tags_clean)
+
+    def get_redirect_uri(self):
+        if self.redirect_uri:
+            return self.redirect_uri
+        if self.http_status == 301 or self.http_status == 302:
+            result = requests.head(self.url, allow_redirects=True, headers={'User-Agent': DIGIMARKS_USER_AGENT})
+            self.http_status = result.status_code
+            self.redirect_uri = result.url
+            return result.url
+        return None
+
+    def get_uri_domain(self):
+        parsed = urlparse(self.url)
+        return parsed.hostname
+
+    @classmethod
+    def strip_url_params(cls, url):
+        parsed = urlparse(url)
+        return urlunparse((parsed.scheme, parsed.netloc, parsed.path, parsed.params, '', parsed.fragment))
+
+    @property
+    def tags_list(self):
+        """ Get the tags as a list, iterable in template """
+        if self.tags:
+            return self.tags.split(',')
+        return []
+
+    def to_dict(self):
+        result = {
+            'title': self.title,
+            'url': self.url,
+            'created':  self.created_date.strftime('%Y-%m-%d %H:%M:%S'),
+            'url_hash': self.url_hash,
+            'tags': self.tags,
+        }
+        return result
+
+    @property
+    def serialize(self):
+        return self.to_dict()
+
+
+class PublicTag(BaseModel):
+    """ Publicly shared tag """
+    tagkey = CharField()
+    userkey = CharField()
+    tag = CharField()
+    created_date = DateTimeField(default=datetime.datetime.now)
+
+    def generate_key(self):
+        """ Generate hash-based key for publicly shared tag """
+        self.tagkey = binascii.hexlify(os.urandom(16))
+
+
+def get_tags_for_user(userkey):
+    """ Extract all tags from the bookmarks """
+    bookmarks = Bookmark.select().filter(Bookmark.userkey == userkey, Bookmark.status == Bookmark.VISIBLE)
+    tags = []
+    for bookmark in bookmarks:
+        tags += bookmark.tags_list
+    return clean_tags(tags)

digimarks/themes.py

@@ -0,0 +1,125 @@
+"""digimarks theme definitions"""
+
+DEFAULT_THEME = 'freshgreen'
+themes = {
+    'green': {
+        'BROWSERCHROME': '#2e7d32',  # green darken-2
+        'BODY': 'grey lighten-4',
+        'TEXT': 'black-text',
+        'TEXTHEX': '#000',
+        'NAV': 'green darken-3',
+        'PAGEHEADER': 'grey-text lighten-5',
+        'MESSAGE_BACKGROUND': 'orange lighten-2',
+        'MESSAGE_TEXT': 'white-text',
+        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
+        'ERRORMESSAGE_TEXT': 'white-text',
+        'BUTTON': '#1b5e20', # green darken-4
+        'BUTTON_ACTIVE': '#43a047',  # green darken-1
+        'LINK_TEXT': '#1b5e20',  # green darken-4
+        'CARD_BACKGROUND': 'green darken-3',
+        'CARD_TEXT': 'white-text',
+        'CARD_LINK': '#FFF',  # white-text
+        'CHIP_TEXT': '#1b5e20',  # green darken-4
+        'FAB': 'red',
+
+        'STAR': 'yellow-text',
+        'PROBLEM': 'red-text',
+        'COMMENT': '',
+    },
+    'freshgreen': {
+        'BROWSERCHROME': '#43a047',  # green darken-1
+        'BODY': 'grey lighten-5',
+        'TEXT': 'black-text',
+        'TEXTHEX': '#000',
+        'NAV': 'green darken-1',
+        'PAGEHEADER': 'grey-text lighten-5',
+        'MESSAGE_BACKGROUND': 'orange lighten-2',
+        'MESSAGE_TEXT': 'white-text',
+        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
+        'ERRORMESSAGE_TEXT': 'white-text',
+        'BUTTON': '#1b5e20', # green darken-4
+        'BUTTON_ACTIVE': '#43a047',  # green darken-1
+        'LINK_TEXT': '#1b5e20',  # green darken-4
+        'CARD_BACKGROUND': 'green darken-1',
+        'CARD_TEXT': 'white-text',
+        'CARD_LINK': '#FFF',  # white-text
+        'CHIP_TEXT': '#1b5e20',  # green darken-4
+        'FAB': 'red',
+
+        'STAR': 'yellow-text',
+        'PROBLEM': 'red-text',
+        'COMMENT': '',
+    },
+    'lightblue': {
+        'BROWSERCHROME': '#0288d1',  # light-blue darken-2
+        'BODY': 'white',
+        'TEXT': 'black-text',
+        'TEXTHEX': '#000',
+        'NAV': 'light-blue darken-2',
+        'PAGEHEADER': 'grey-text lighten-5',
+        'MESSAGE_BACKGROUND': 'orange lighten-2',
+        'MESSAGE_TEXT': 'white-text',
+        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
+        'ERRORMESSAGE_TEXT': 'white-text',
+        'BUTTON': '#fb8c00', # orange darken-1
+        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
+        'LINK_TEXT': '#FFF',  # white
+        'CARD_BACKGROUND': 'light-blue lighten-2',
+        'CARD_TEXT': 'black-text',
+        'CARD_LINK': '#263238',  # blue-grey-text darken-4
+        'CHIP_TEXT': '#FFF',  # white
+        'FAB': 'light-blue darken-4',
+
+        'STAR': 'yellow-text',
+        'PROBLEM': 'red-text',
+        'COMMENT': '',
+    },
+    'dark': {
+        'BROWSERCHROME': '#212121',  # grey darken-4
+        'BODY': 'grey darken-4',
+        'TEXT': 'grey-text lighten-1',
+        'TEXTHEX': '#bdbdbd',
+        'NAV': 'grey darken-3',
+        'PAGEHEADER': 'grey-text lighten-1',
+        'MESSAGE_BACKGROUND': 'orange lighten-2',
+        'MESSAGE_TEXT': 'white-text',
+        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
+        'ERRORMESSAGE_TEXT': 'white-text',
+        'BUTTON': '#fb8c00', # orange darken-1
+        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
+        'LINK_TEXT': '#fb8c00',  # orange-text darken-1
+        'CARD_BACKGROUND': 'grey darken-3',
+        'CARD_TEXT': 'grey-text lighten-1',
+        'CARD_LINK': '#fb8c00',  # orange-text darken-1
+        'CHIP_TEXT': '#fb8c00',  # orange-text darken-1
+        'FAB': 'red',
+
+        'STAR': 'yellow-text',
+        'PROBLEM': 'red-text',
+        'COMMENT': '',
+    },
+    'amoled': {
+        'BROWSERCHROME': '#000',  # grey darken-4
+        'BODY': 'black',
+        'TEXT': 'grey-text lighten-1',
+        'TEXTHEX': '#bdbdbd',
+        'NAV': 'grey darken-3',
+        'PAGEHEADER': 'grey-text lighten-1',
+        'MESSAGE_BACKGROUND': 'orange lighten-2',
+        'MESSAGE_TEXT': 'white-text',
+        'ERRORMESSAGE_BACKGROUND': 'red darken-1',
+        'ERRORMESSAGE_TEXT': 'white-text',
+        'BUTTON': '#fb8c00', # orange darken-1
+        'BUTTON_ACTIVE': '#ffa726',  # orange lighten-1
+        'LINK_TEXT': '#fb8c00',  # orange-text darken-1
+        'CARD_BACKGROUND': 'grey darken-3',
+        'CARD_TEXT': 'grey-text lighten-1',
+        'CARD_LINK': '#fb8c00',  # orange-text darken-1
+        'CHIP_TEXT': '#fb8c00',  # orange-text darken-1
+        'FAB': 'red',
+
+        'STAR': 'yellow-text',
+        'PROBLEM': 'red-text',
+        'COMMENT': '',
+    }
+}

example_config/settings.py

@@ -10,10 +10,6 @@ DEBUG = False
 # echo -n "yourstring" | sha1sum
 SYSTEMKEY = 'S3kr1t'
 
-# RapidAPI key for favicons
-# https://rapidapi.com/realfavicongenerator/api/realfavicongenerator
-MASHAPE_API_KEY = 'your_MASHAPE_key'
-
 LOG_LOCATION = 'digimarks.log'
 #LOG_LOCATION = '/var/log/digimarks/digimarks.log'
 # How many logs to keep in log rotation:

requirements-dev.txt

@@ -1,61 +1,29 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile requirements-dev.in
-astroid==3.3.10
-    # via pylint
-beautifulsoup4==4.13.4
-    # via bs4
-blinker==1.9.0
-    # via flask
-bs4==0.0.2
-    # via -r requirements.in
-certifi==2025.6.15
-    # via requests
-charset-normalizer==3.4.2
-    # via requests
-click==8.2.1
-    # via flask
-dill==0.4.0
-    # via pylint
-feedgen==1.0.0
-    # via -r requirements.in
-flask==3.1.1
-    # via -r requirements.in
-idna==3.10
-    # via requests
-isort==6.0.1
-    # via pylint
-itsdangerous==2.2.0
-    # via flask
-jinja2==3.1.6
-    # via flask
-lxml==6.0.0
-    # via feedgen
-markupsafe==3.0.2
-    # via
-    #   flask
-    #   jinja2
-    #   werkzeug
-mccabe==0.7.0
-    # via pylint
-peewee==3.18.2
-    # via -r requirements.in
-platformdirs==4.3.8
-    # via pylint
-pylint==3.3.7
-    # via -r requirements-dev.in
-python-dateutil==2.9.0.post0
-    # via feedgen
-requests==2.32.4
-    # via -r requirements.in
-six==1.17.0
-    # via python-dateutil
-soupsieve==2.7
-    # via beautifulsoup4
-tomlkit==0.13.3
-    # via pylint
-typing-extensions==4.14.1
-    # via beautifulsoup4
-urllib3==2.5.0
-    # via requests
-werkzeug==3.1.3
-    # via flask
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements-dev.in
+#
+astroid==2.3.3            # via pylint
+beautifulsoup4==4.8.2     # via bs4
+bs4==0.0.1
+certifi==2019.11.28       # via requests
+chardet==3.0.4            # via requests
+click==7.0                # via flask
+flask==1.1.1
+idna==2.8                 # via requests
+isort==4.3.21             # via pylint
+itsdangerous==1.1.0       # via flask
+jinja2==2.10.3            # via flask
+lazy-object-proxy==1.4.3  # via astroid
+markupsafe==1.1.1         # via jinja2
+mccabe==0.6.1             # via pylint
+peewee==3.13.1
+pylint==2.4.4
+requests==2.22.0
+six==1.13.0               # via astroid
+soupsieve==1.9.5          # via beautifulsoup4
+typed-ast==1.4.0          # via astroid
+urllib3==1.25.7           # via requests
+werkzeug==0.16.0          # via flask
+wrapt==1.11.2             # via astroid

requirements.in

@@ -1,10 +1,4 @@
-# Core application
 flask
 peewee
-
-# Fetch title etc from links
 bs4
 requests
-
-# Generate (atom) feeds for tags and such
-feedgen

requirements.txt

@@ -1,47 +1,21 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile requirements.in
-beautifulsoup4==4.13.4
-    # via bs4
-blinker==1.9.0
-    # via flask
-bs4==0.0.2
-    # via -r requirements.in
-certifi==2025.6.15
-    # via requests
-charset-normalizer==3.4.2
-    # via requests
-click==8.2.1
-    # via flask
-feedgen==1.0.0
-    # via -r requirements.in
-flask==3.1.1
-    # via -r requirements.in
-idna==3.10
-    # via requests
-itsdangerous==2.2.0
-    # via flask
-jinja2==3.1.6
-    # via flask
-lxml==6.0.0
-    # via feedgen
-markupsafe==3.0.2
-    # via
-    #   flask
-    #   jinja2
-    #   werkzeug
-peewee==3.18.2
-    # via -r requirements.in
-python-dateutil==2.9.0.post0
-    # via feedgen
-requests==2.32.4
-    # via -r requirements.in
-six==1.17.0
-    # via python-dateutil
-soupsieve==2.7
-    # via beautifulsoup4
-typing-extensions==4.14.1
-    # via beautifulsoup4
-urllib3==2.5.0
-    # via requests
-werkzeug==3.1.3
-    # via flask
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements.in --upgrade
+#
+beautifulsoup4==4.7.1     # via bs4
+bs4==0.0.1
+certifi==2019.3.9         # via requests
+chardet==3.0.4            # via requests
+click==7.0                # via flask
+flask==1.0.2
+idna==2.8                 # via requests
+itsdangerous==1.1.0       # via flask
+jinja2==2.10.1            # via flask
+markupsafe==1.1.1         # via jinja2
+peewee==3.9.3
+requests==2.21.0
+soupsieve==1.9            # via beautifulsoup4
+urllib3==1.24.1           # via requests
+werkzeug==0.15.2          # via flask

setup.py

@@ -5,11 +5,12 @@ https://packaging.python.org/en/latest/distributing.html
 https://github.com/pypa/sampleproject
 """
 
-from setuptools import setup
 # To use a consistent encoding
 from codecs import open as codecopen
 from os import path
 
+from setuptools import setup
+
 here = path.abspath(path.dirname(__file__))
 
 # Get the long description from the relevant file
@@ -18,7 +19,8 @@ with codecopen(path.join(here, 'README.rst'), encoding='utf-8') as f:
 
 setup(
     name='digimarks', # pip install digimarks
-    description='Simple bookmarking service, using a SQLite database to store bookmarks, supporting tags, automatic title fetching and REST API calls.',
+    description='Simple bookmarking service, using a SQLite database to store bookmarks, supporting tags, automatic '
+                'title fetching and REST API calls.',
     #long_description=open('README.md', 'rt').read(),
     long_description=long_description,
 
@@ -35,7 +37,7 @@ setup(
 
     # as a practice no need to hard code version unless you know program wont
     # work unless the specific versions are used
-    install_requires=['Flask', 'Peewee', 'Flask-Peewee', 'requests', 'bs4'],
+    install_requires=['Flask', 'Peewee', 'requests', 'bs4'],
 
     py_modules=['digimarks'],