Merge pull request #1 from jelmer/unique-everseen

Avoid dependency on more_itertools.

digimarks.py

@@ -7,7 +7,6 @@ import sys
 import requests
 import shutil
 import bs4
-from more_itertools import unique_everseen
 from urlparse import urlparse, urlunparse, urljoin
 
 from utilkit import datetimeutil
@@ -49,6 +48,22 @@ except AttributeError:
 # Cache the tags
 all_tags = {}
 
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in ifilterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
 
 def clean_tags(tags_list):
     tags_res = [x.strip() for x in tags_list]

requirements.txt

@@ -2,6 +2,5 @@ flask
 peewee
 flask-peewee
 bs4
-more_itertools
 requests
 utilkit