From 64d8a0e64b0b4f5d1c5927c7a45317f5bc65a421 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 26 Dec 2019 19:18:39 -0800 Subject: be more parsimonious with GROBID metadata Because these are getting persisted in database (as well as kafka), don't write out empty keys. --- python/sandcrawler/grobid.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'python/sandcrawler') diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py index 70f7b16..9fd5ad4 100644 --- a/python/sandcrawler/grobid.py +++ b/python/sandcrawler/grobid.py @@ -62,10 +62,12 @@ class GrobidClient(object): meta = dict() biblio = dict() for k in ('title', 'authors', 'journal', 'date', 'doi', ): - biblio[k] = tei_json.get(k) + if tei_json.get(k): + biblio[k] = tei_json[k] meta['biblio'] = biblio for k in ('grobid_version', 'grobid_timestamp', 'fatcat_release', 'language_code'): - meta[k] = tei_json.get(k) + if tei_json.get(k): + meta[k] = tei_json[k] return meta class GrobidWorker(SandcrawlerWorker): -- cgit v1.2.3