1
0
mirror of https://github.com/aquatix/digimarks.git synced 2025-12-06 20:55:10 +01:00

Avoid dependency on more_itertools.

Instead, use version of unique_everseen as seen in itertools
documentation.
This commit is contained in:
Jelmer Vernooij
2016-11-28 23:25:09 +00:00
parent 698a417bac
commit 9e6d67a0ff
2 changed files with 16 additions and 2 deletions

View File

@@ -7,7 +7,6 @@ import sys
import requests
import shutil
import bs4
from more_itertools import unique_everseen
from urlparse import urlparse, urlunparse, urljoin
from utilkit import datetimeutil
@@ -49,6 +48,22 @@ except AttributeError:
# Cache the tags
all_tags = {}
def unique_everseen(iterable, key=None):
"List unique elements, preserving order. Remember all elements ever seen."
# unique_everseen('AAAABBBCCDAABBB') --> A B C D
# unique_everseen('ABBCcAD', str.lower) --> A B C D
seen = set()
seen_add = seen.add
if key is None:
for element in ifilterfalse(seen.__contains__, iterable):
seen_add(element)
yield element
else:
for element in iterable:
k = key(element)
if k not in seen:
seen_add(k)
yield element
def clean_tags(tags_list):
tags_res = [x.strip() for x in tags_list]

View File

@@ -2,6 +2,5 @@ flask
peewee
flask-peewee
bs4
more_itertools
requests
utilkit