diff --git a/digimarks.py b/digimarks.py index 8f33310..b59be7c 100644 --- a/digimarks.py +++ b/digimarks.py @@ -7,7 +7,6 @@ import sys import requests import shutil import bs4 -from more_itertools import unique_everseen from urlparse import urlparse, urlunparse, urljoin from utilkit import datetimeutil @@ -49,6 +48,22 @@ except AttributeError: # Cache the tags all_tags = {} +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in ifilterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element def clean_tags(tags_list): tags_res = [x.strip() for x in tags_list] diff --git a/requirements.txt b/requirements.txt index 920b612..362bc8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,5 @@ flask peewee flask-peewee bs4 -more_itertools requests utilkit