aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fatcat_scholar/query_citation.py175
1 files changed, 175 insertions, 0 deletions
diff --git a/fatcat_scholar/query_citation.py b/fatcat_scholar/query_citation.py
new file mode 100644
index 0000000..970c235
--- /dev/null
+++ b/fatcat_scholar/query_citation.py
@@ -0,0 +1,175 @@
+"""
+This file contains helpers to fuzzy match a raw citation string:
+
+- try to parse it with GROBID into structured form
+- transform the GROBID XML response to a simple dict/struct
+- run fuzzycat lookup
+
+Note that this chain hits several external services, and should be wrapped in a
+timeout and try/except! In the future, perhaps should be async so it can run in
+parallel with "regular" query?
+"""
+
+import io
+import sys
+from typing import Optional, Any, Tuple
+import xml.etree.ElementTree as ET
+
+import requests
+from fatcat_openapi_client import ReleaseEntity, ReleaseExtIds, ReleaseContrib
+from fatcat_scholar.api_entities import entity_to_dict
+from fuzzycat.matching import match_release_fuzzy
+import fuzzycat.common
+import fuzzycat.verify
+
+from fatcat_scholar.grobid2json import biblio_info
+
+
+def grobid_process_citation(raw: str) -> Optional[str]:
+ GROBID_URL = "https://grobid.qa.fatcat.wiki"
+ try:
+ grobid_response = requests.post(
+ GROBID_URL + "/api/processCitation",
+ data={"citations": raw, "consolidateCitations": 0,},
+ timeout=180.0,
+ )
+ except requests.Timeout:
+ print("GROBID request (HTTP POST) timeout", file=sys.stderr)
+ return None
+ if grobid_response.status_code != 200:
+ print(f"GROBID request (HTTP POST) failed: {grobid_response}", file=sys.stderr)
+ return None
+ return grobid_response.text
+
+
+def transform_grobid(raw_xml: str) -> Optional[dict]:
+ # first, remove any xmlns stuff
+ raw_xml = raw_xml.replace('xmlns="http://www.tei-c.org/ns/1.0"', "")
+ tree = ET.parse(io.StringIO(raw_xml))
+ root = tree.getroot()
+ ref = biblio_info(root, ns="")
+ return ref
+
+
+def ref_to_release(ref: dict) -> ReleaseEntity:
+ contribs = []
+ for author in ref.get("authors") or []:
+ contribs.append(
+ ReleaseContrib(
+ raw_name=author.get("name"),
+ given_name=author.get("given_name"),
+ surname=author.get("surname"),
+ )
+ )
+ release = ReleaseEntity(
+ title=ref.get("title"),
+ contribs=contribs,
+ volume=ref.get("volume"),
+ issue=ref.get("issue"),
+ pages=ref.get("pages"),
+ ext_ids=ReleaseExtIds(
+ doi=ref.get("doi"),
+ pmid=ref.get("pmid"),
+ pmcid=ref.get("pmcid"),
+ arxiv=ref.get("arxiv_id"),
+ ),
+ )
+ if ref.get("journal"):
+ release.extra = {"container_name": ref.get("journal")}
+ if ref.get("date"):
+ if len(ref["date"]) == 4 and ref["date"].isdigit():
+ release.release_year = int(ref["date"])
+ return release
+
+
+def fuzzy_match(
+ release: ReleaseEntity, es_client: Any, api_client: Any
+) -> Optional[Tuple[str, str, ReleaseEntity]]:
+ """
+ This helper function uses fuzzycat (and elasticsearch) to look for
+ existing release entities with similar metadata.
+
+ Returns None if there was no match of any kind, or a single tuple
+ (status: str, reason: str, existing: ReleaseEntity) if there was a match.
+
+ Status string is one of the fuzzycat.common.Status, with "strongest
+ match" in this sorted order:
+
+ - EXACT
+ - STRONG
+ - WEAK
+ - AMBIGUOUS
+
+ Eg, if there is any EXACT match that is always returned; an AMBIGIOUS
+ result is only returned if all the candidate matches were ambiguous.
+ """
+
+ # this map used to establish priority order of verified matches
+ STATUS_SORT = {
+ fuzzycat.common.Status.TODO: 0,
+ fuzzycat.common.Status.EXACT: 10,
+ fuzzycat.common.Status.STRONG: 20,
+ fuzzycat.common.Status.WEAK: 30,
+ fuzzycat.common.Status.AMBIGUOUS: 40,
+ fuzzycat.common.Status.DIFFERENT: 60,
+ }
+
+ # TODO: the size here is a first guess; what should it really be?
+ candidates = match_release_fuzzy(release, size=10, es=es_client)
+ if not candidates:
+ return None
+
+ release_dict = entity_to_dict(release, api_client=api_client.api_client)
+ verified = [
+ (
+ fuzzycat.verify.verify(
+ release_dict, entity_to_dict(c, api_client=api_client.api_client)
+ ),
+ c,
+ )
+ for c in candidates
+ ]
+
+ # chose the "closest" match
+ closest = sorted(verified, key=lambda v: STATUS_SORT[v[0].status])[0]
+ if closest[0].status == fuzzycat.common.Status.DIFFERENT:
+ return None
+ elif closest[0].status == fuzzycat.common.Status.TODO:
+ raise NotImplementedError("fuzzycat verify hit a Status.TODO")
+ else:
+ return (closest[0].status.name, closest[0].reason.value, closest[1])
+
+
+if __name__ == "__main__":
+ """
+ Demo showing how to integrate the above functions together.
+ """
+ import os
+ import elasticsearch
+ import fatcat_openapi_client
+
+ citation = sys.argv[1]
+ print("Sending to GROBID...")
+ resp = grobid_process_citation(citation)
+ print(resp)
+ if not resp:
+ sys.exit(0)
+ ref = transform_grobid(resp)
+ print(ref)
+ if not ref:
+ sys.exit(0)
+ release = ref_to_release(ref)
+ print(release)
+
+ es_backend = os.environ.get("ELASTICSEARCH_BACKEND", "https://search.fatcat.wiki")
+ es_client = elasticsearch.Elasticsearch(es_backend)
+ api_conf = fatcat_openapi_client.Configuration()
+ api_conf.host = os.environ.get("FATCAT_API_HOST", "https://api.fatcat.wiki/v0")
+ api_client = fatcat_openapi_client.DefaultApi(
+ fatcat_openapi_client.ApiClient(api_conf)
+ )
+ matches = fuzzy_match(release, es_client=es_client, api_client=api_client)
+ print(matches)
+ if not matches or matches[0] not in ("EXACT", "STRONG", "WEAK"):
+ sys.exit(0)
+ print(matches[2].work_id)