aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2021-11-03 18:41:30 -0700
committerBryan Newbold <bnewbold@archive.org>2021-11-04 17:19:52 -0700
commit1f078fe94a5cf5322527b97dcdf0cb054e0c7540 (patch)
tree7bc5419530c16f9fc64f112113942398ec62eb9a /python
parent8577d50b644dd45bce5275675eed4d43bb816b67 (diff)
downloadsandcrawler-1f078fe94a5cf5322527b97dcdf0cb054e0c7540.tar.gz
sandcrawler-1f078fe94a5cf5322527b97dcdf0cb054e0c7540.zip
grobid: handle weird whitespace unstructured from crossref
See also: https://github.com/kermitt2/grobid/issues/849
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/grobid.py11
1 files changed, 10 insertions, 1 deletions
diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py
index e28c488..7d7f6b5 100644
--- a/python/sandcrawler/grobid.py
+++ b/python/sandcrawler/grobid.py
@@ -27,6 +27,7 @@ def clean_crossref_unstructured(raw: str) -> str:
raw = html.unescape(raw)
raw.replace(" ", " ")
+ raw = raw.strip()
return raw
@@ -56,6 +57,14 @@ def test_clean_ref_str() -> None:
== """Ronald L. Rivest and Butler W. Lampson. 1996. SDSI: A Simple Distributed Security Infrastructure. In Advances in Cryptology — CRYPTO ’96. Springer Berlin Heidelberg."""
)
+ # all non-breaking whitespace
+ assert (
+ clean_crossref_unstructured(
+ "\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"
+ )
+ == ""
+ )
+
class GrobidClient(object):
def __init__(self, host_url: str = "https://grobid.qa.fatcat.wiki", **kwargs):
@@ -173,7 +182,7 @@ class GrobidClient(object):
"""
if ref.get("DOI"):
return False
- if len(ref.get("unstructured", "")) <= 6:
+ if len(ref.get("unstructured", "").strip()) <= 6:
return False
# TODO: what other combinations are enough to skip parsing?