extend tests

1 files changed, 43 insertions, 0 deletions

diff --git a/fuzzycat/utils.py b/fuzzycat/utils.py
index f269b11..5ded48c 100644
--- a/fuzzycat/utils.py
+++ b/fuzzycat/utils.py
@@ -1,3 +1,4 @@
+import itertools
 import io
 import string
 
@@ -24,3 +25,45 @@ def cut(f: int = 0, sep: str = '\t', ignore_missing_column: bool = True):
         return parts[f]
 
     return func
+
+def author_similarity_score(u, v):
+    """
+    Given two author strings, return a similarity score between 0 and 1.
+    """
+    return jaccard(set(token_n_grams(u)), set(token_n_grams(v)))
+
+
+def jaccard(a, b):
+    """
+    Jaccard of sets a and b.
+    """
+    if len(a | b) == 0:
+        return 0
+    return len(a & b) / len(a | b)
+
+
+def token_n_grams(s):
+    """
+    Return n-grams, calculated per token.
+    """
+    return ["".join(v) for v in itertools.chain(*[nwise(v, n=2) for v in tokenize_string(s)])]
+
+
+def tokenize_string(s):
+    """
+    Normalize and tokenize, should be broken up.
+    """
+    return [token for token in s.lower().split()]
+
+
+def nwise(iterable, n=2):
+    """
+    Generalized: func: `pairwise`. Split an iterable after every
+    `n` items.
+    """
+    i = iter(iterable)
+    piece = tuple(itertools.islice(i, n))
+    while piece:
+        yield piece
+        piece = tuple(itertools.islice(i, n))
+