From 08a9242e2ed19aaec14d92fe174bee21bb4232eb Mon Sep 17 00:00:00 2001 From: Martin Czygan Date: Tue, 21 Sep 2021 15:54:46 +0200 Subject: style: apply formatting --- fuzzycat/config.py | 1 - fuzzycat/matching.py | 3 ++- fuzzycat/simple.py | 2 +- fuzzycat/utils.py | 2 ++ fuzzycat/verify.py | 9 ++++----- tests/test_matching.py | 15 ++++++++++++--- tests/test_utils.py | 1 + 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/fuzzycat/config.py b/fuzzycat/config.py index 61050d1..d6989cf 100644 --- a/fuzzycat/config.py +++ b/fuzzycat/config.py @@ -1,4 +1,3 @@ - from dynaconf import Dynaconf settings = Dynaconf(envvar_prefix="FUZZYCAT") diff --git a/fuzzycat/matching.py b/fuzzycat/matching.py index c94a308..33e130e 100644 --- a/fuzzycat/matching.py +++ b/fuzzycat/matching.py @@ -10,9 +10,9 @@ import requests from fatcat_openapi_client import ContainerEntity, DefaultApi, ReleaseEntity from fatcat_openapi_client.rest import ApiException +from fuzzycat.config import settings from fuzzycat.entities import entity_from_dict, entity_from_json from fuzzycat.utils import es_compat_hits_total -from fuzzycat.config import settings FATCAT_API_URL = settings.get("FATCAT_API_URL", "https://api.fatcat.wiki/v0") @@ -106,6 +106,7 @@ def match_release_fuzzy( if es_compat_hits_total(resp) > 0: return response_to_entity_list(resp, entity_type=ReleaseEntity, size=size, api=api) + # TODO: perform more queries on other fields. return [] diff --git a/fuzzycat/simple.py b/fuzzycat/simple.py index 8b206b1..ff59ba2 100644 --- a/fuzzycat/simple.py +++ b/fuzzycat/simple.py @@ -25,8 +25,8 @@ from fuzzycat.common import Reason, Status from fuzzycat.entities import entity_to_dict from fuzzycat.grobid_unstructured import grobid_parse_unstructured from fuzzycat.matching import match_release_fuzzy -from fuzzycat.verify import verify from fuzzycat.utils import clean_doi +from fuzzycat.verify import verify @dataclass diff --git a/fuzzycat/utils.py b/fuzzycat/utils.py index a1c5124..303daf6 100644 --- a/fuzzycat/utils.py +++ b/fuzzycat/utils.py @@ -81,6 +81,7 @@ def dict_key_exists(doc, path): else: return True + def clean_doi(raw: Optional[str]) -> Optional[str]: if not raw: return None @@ -95,6 +96,7 @@ def clean_doi(raw: Optional[str]) -> Optional[str]: raw = raw[:8] + raw[9:] return raw + def doi_prefix(v): """ Return the prefix of a DOI. diff --git a/fuzzycat/verify.py b/fuzzycat/verify.py index 1eeea40..5b90c47 100644 --- a/fuzzycat/verify.py +++ b/fuzzycat/verify.py @@ -90,9 +90,9 @@ from fuzzycat.common import Reason, Status from fuzzycat.data import (CONTAINER_NAME_BLACKLIST, PUBLISHER_BLACKLIST, TITLE_BLACKLIST, TITLE_FRAGMENT_BLACKLIST) from fuzzycat.entities import entity_to_dict -from fuzzycat.utils import (author_similarity_score, contains_chemical_formula, dict_key_exists, - doi_prefix, has_doi_prefix, jaccard, num_project, parse_page_string, - slugify_string, clean_doi) +from fuzzycat.utils import (author_similarity_score, clean_doi, contains_chemical_formula, + dict_key_exists, doi_prefix, has_doi_prefix, jaccard, num_project, + parse_page_string, slugify_string) Verify = collections.namedtuple("Verify", "status reason") @@ -597,8 +597,7 @@ def verify(a: Dict, b: Dict, min_title_length=5) -> Tuple[str, str]: try: a_parsed_pages = parse_page_string(glom(a, "pages")) b_parsed_pages = parse_page_string(glom(b, "pages")) - if (a_parsed_pages.count != None - and b_parsed_pages.count != None + if (a_parsed_pages.count != None and b_parsed_pages.count != None and abs(a_parsed_pages.count - b_parsed_pages.count) > 5): return Verify(Status.DIFFERENT, Reason.PAGE_COUNT) except (ValueError, PathAccessError): diff --git a/tests/test_matching.py b/tests/test_matching.py index 7ab7b11..2122144 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -9,7 +9,8 @@ from fatcat_openapi_client import ReleaseEntity from fuzzycat.entities import entity_from_dict from fuzzycat.matching import anything_to_entity, match_release_fuzzy -warnings.filterwarnings("ignore") # InsecureRequestWarning: Unverified HTTPS request is being made to host ... +warnings.filterwarnings( + "ignore") # InsecureRequestWarning: Unverified HTTPS request is being made to host ... from fuzzycat.matching import anything_to_entity, match_release_fuzzy from fuzzycat.config import settings @@ -28,6 +29,7 @@ FATCAT_SEARCH_URL = settings.get("FATCAT_SEARCH_URL", "https://search.fatcat.wik def is_not_reachable(url, timeout=3): return not is_reachable(url) + def is_reachable(url, timeout=3): """ Return true, if URL is reachable and returns HTTP 200. @@ -37,14 +39,21 @@ def is_reachable(url, timeout=3): except Exception: return False + @pytest.fixture def es_client(): return elasticsearch.Elasticsearch([FATCAT_SEARCH_URL]) -@pytest.mark.skipif(is_not_reachable(FATCAT_SEARCH_URL), - reason="{} not reachable, use e.g. FUZZYCAT_FATCAT_SEARCH_URL=localhost:9200 to override".format(FATCAT_SEARCH_URL)) +@pytest.mark.skipif( + is_not_reachable(FATCAT_SEARCH_URL), + reason="{} not reachable, use e.g. FUZZYCAT_FATCAT_SEARCH_URL=localhost:9200 to override". + format(FATCAT_SEARCH_URL)) def test_match_release_fuzzy(es_client, caplog): + """ + This test is tied to the current index contents, so if that changes, this + test may fail as well. + """ cases = ( ("wtv64ahbdzgwnan7rllwr3nurm", 1), ("eqcgtpav3na5jh56o5vjsvb4ei", 1), diff --git a/tests/test_utils.py b/tests/test_utils.py index 21b85a4..957203f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -124,6 +124,7 @@ def test_es_compat_hits_total(): for r, expected in cases: assert es_compat_hits_total(r) == expected + def test_clean_doi(): assert clean_doi(None) == None assert clean_doi("blah") == None -- cgit v1.2.3