aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/grobid.py36
1 files changed, 34 insertions, 2 deletions
diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py
index cdd2093..c57ea7c 100644
--- a/python/sandcrawler/grobid.py
+++ b/python/sandcrawler/grobid.py
@@ -1,5 +1,6 @@
-from typing import Any, Dict, List, Optional
import sys
+import unicodedata
+from typing import Any, Dict, List, Optional
import requests
from grobid_tei_xml import GrobidBiblio, parse_citation_list_xml, parse_document_xml
@@ -9,6 +10,30 @@ from .misc import gen_file_metadata
from .workers import SandcrawlerFetchWorker, SandcrawlerWorker
+def clean_ref_str(raw: str) -> str:
+ """
+ When comparing raw unstructured strings (from upstream sources) to
+ GROBID-returned citations, we sometimes want to do exact comparisons to
+ match up records (eg, from crossref). GROBID does some (totally reasonable)
+ arbitrary normalizations of strings, like simplifying whitespace.
+
+ This routine is to make comparisons against when GROBID returned and
+ original strings easier.
+ """
+ # TODO: dead test code
+ # raw = unicodedata.normalize('NFKC', raw)
+ # raw = raw.replace('\u00a0', ' ').replace('\u2013', '-').replace('\u2014', '-').strip()
+ # raw = ' '.join(raw.split())
+ raw = raw.replace(" ", " ")
+ return raw
+
+
+def test_clean_ref_str() -> None:
+ raw_with_nbsp = """Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W. Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. (2018) 385–394. Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W. Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. (2018) 385–394."""
+ raw_without_nbsp = """Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W. Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. (2018) 385-394. Qingyao Ai Keping Bi Cheng Luo Jiafeng Guo and W. Bruce Croft. 2018. Unbiased Learning to Rank with Unbiased Propensity Estimation. (2018) 385-394."""
+ assert clean_ref_str(raw_with_nbsp) == raw_without_nbsp
+
+
class GrobidClient(object):
def __init__(self, host_url: str = "https://grobid.qa.fatcat.wiki", **kwargs):
self.host_url = host_url
@@ -190,7 +215,14 @@ class GrobidClient(object):
refs_json = []
for i in range(len(refs)):
refs[i].id = unstructured_refs[i].get("key")
- assert refs[i].unstructured == unstructured_refs[i]["unstructured"]
+ original = unstructured_refs[i]["unstructured"]
+ original_clean = clean_ref_str(unstructured_refs[i]["unstructured"])
+ assert (
+ refs[i].unstructured == original or refs[i].unstructured == original_clean
+ ), f'raw citation mismatch (GROBID then original cleaned): \n{refs[i].unstructured.encode("utf-8")}\n{original_clean.encode("utf-8")}'
+ # intentionally put "unclean" original string in, to allow later
+ # exact byte-accurate comparisons
+ refs[i].unstructured = original
refs_json.append(refs[i].to_dict())
ret["refs_json"] = refs_json
return ret