add matching submodule

2 files changed, 149 insertions, 0 deletions

diff --git a/fuzzycat/__init__.py b/fuzzycat/__init__.py
index 6f0eb54..39cdbc6 100644
--- a/fuzzycat/__init__.py
+++ b/fuzzycat/__init__.py
@@ -1,3 +1,5 @@
 __version__ = "0.1.1"
 
 from fuzzycat.status import MatchStatus
+from fuzzycat.utils import *
+from fuzzycat.matching import compare_container_name
diff --git a/fuzzycat/matching.py b/fuzzycat/matching.py
new file mode 100644
index 0000000..0c482e0
--- /dev/null
+++ b/fuzzycat/matching.py
@@ -0,0 +1,147 @@
+import string
+import re
+from unidecode import unidecode
+from ftfy import fix_text
+
+from fuzzycat import MatchStatus, StringPipeline, StringAnnotator
+from fuzzycat.utils import *
+
+
+def compare_container_name(a: str, b: str) -> MatchStatus:
+    """
+    Given two strings representing container names, return a match status. This
+    would be a subproblem of verify_container_match in cases where only a
+    string is given or the entity has only a name. Factored out for ease of
+    testing. TODO(martin): incorporate abbreviations mapping, other synonyms.
+
+    Some name stats over 146302 real names from fatcat.
+
+        In [11]: len(df)
+        Out[11]: 146302
+
+        In [12]: df.head()
+        Out[12]:
+                                                        name  nlen
+        0                       Sartre Studies International    28
+        1                                Revolutionary world    19
+        2  Monograph Series on Nonlinear Science and Comp...    52
+        3                                  Hepatitis Monthly    17
+        4                                             TRACEY     6
+
+        In [13]: df.describe()
+        Out[13]:
+                        nlen
+        count  146302.000000
+        mean       33.891861
+        std        18.955551
+        min         2.000000
+        25%        20.000000
+        50%        31.000000
+        75%        44.000000
+        max       286.000000
+
+    Aroung 4000 names which are not [a-zA-z ], e.g.:
+
+        In [23]: df[df.is_alpha_only == False].sample(n=5)
+        Out[23]:
+                                                             name  nlen  is_alpha_only
+        118497                     Журнал Фронтирных Исследований    30          False
+        124885  Õpetatud Eesti Seltsi Aastaraamat/Yearbook of ...    74          False
+        142217             Études économiques de l'OCDE : Norvège    38          False
+        34681             حولیة کلیة أصول الدین والدعوة بالمنوفیة    39          False
+        132251  Известия Российской академии наук Теория и сис...    61          False
+
+
+    """
+    if a is None or b is None:
+        raise ValueError("strings required, got: a = {}, b = {}".format(a, b))
+
+    # Whitespace cleanup.Try to remove superfluous whitespace, which should
+    # never matter, "HNO    Praxis"
+    string_cleanups = StringPipeline([
+        str.lower,
+        str.strip,
+        fix_text,
+        lambda s: re.sub(r"\s{2,}", " ", s),
+        lambda s: s.replace("&", "and"),
+    ])
+    a = string_cleanups.run(a)
+    b = string_cleanups.run(b)
+
+    # Derive some characteristics of the string. The keys are free form which
+    # may or may not be a problem. TODO(martin): maybe subclass str and just
+    # add additional methods?
+    sa = StringAnnotator([
+        lambda s: {
+            "is_short_string": len(s) < 15
+        },
+        lambda s: {
+            "is_printable_only": all(c in string.printable for c in s)
+        },
+        lambda s: {
+            "is_single_token": len(s.split()) < 2
+        },
+        lambda s: {
+            "letter_to_non_letter_ratio": letter_to_non_letter_ratio(s)
+        },
+        lambda s: {
+            "alphanumeric_ratio": alphanumeric_ratio(s)
+        },
+        lambda s: {
+            "has_diacritics": s != unidecode(s)
+        },
+        lambda s: {
+            "startswith_the": s.startswith("the ")
+        },
+        lambda s: {
+            "parenthesized_year": parenthesized_year(s)
+        },
+        lambda s: {
+            "alphanumeric_only": alphanumeric_only(s)
+        },
+    ])
+    asa = sa.run(a)
+    bsa = sa.run(b)
+
+    if asa["is_short_string"] and asa["letter_to_non_letter_ratio"] > 0.4:
+        if a == b:
+            return MatchStatus.EXACT
+
+    if not asa["is_short_string"] and not asa["is_single_token"]:
+        if a == b:
+            return MatchStatus.EXACT
+
+    # Short, single (ascii) word titles, like "Language" and the like. Single
+    # token "臨床皮膚科" needs to pass.
+    if asa["is_printable_only"] and asa["is_single_token"]:
+        return MatchStatus.AMBIGIOUS
+
+    if a == b:
+        return MatchStatus.EXACT
+
+    # Mostly ASCII, but with some possible artifacts.
+    if (asa["alphanumeric_ratio"] > 0.9 and asa["alphanumeric_only"] == bsa["alphanumeric_only"]):
+        return MatchStatus.STRONG
+
+    # Year in parentheses case, e.g. "Conf X (2018)" and "Conf X (2019)" should
+    # be different; about 3% of names contain a '(', 1% some possible date.
+    if (asa["parenthesized_year"] and asa["parenthesized_year"] == bsa["parenthesized_year"]):
+        return MatchStatus.DIFFERENT
+
+    # Common prefixes (maybe curate these manually):
+    common_prefixes = ("precarpathian bulletin of the shevchenko scientific society", )
+    for prefix in common_prefixes:
+        if a.startswith(prefix) and a != b:
+            return MatchStatus.DIFFERENT
+
+    if (not asa["is_short"] and not bsa["is_short"] and common_prefix_length_ratio(a, b) > 0.9):
+        return MatchStatus.STRONG
+
+    if (not asa["is_short"] and not bsa["is_short"] and common_prefix_length_ratio(a, b) > 0.7):
+        return MatchStatus.WEAK
+
+    # Address e.g. a char flip, but only, if we do not have diacritics.
+    if (not asa["is_short_string"] and not asa["is_single_token"] and not asa["has_diacritics"] and hamming_distance(a, b) < 2):
+        return MatchStatus.STRONG
+
+    return MatchStatus.AMBIGIOUS