diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-12-26 19:18:39 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-01-02 18:12:58 -0800 |
| commit | 64d8a0e64b0b4f5d1c5927c7a45317f5bc65a421 (patch) | |
| tree | 4239cd45fcc9490f0a2baf4261b04b2a622e930d /python/sandcrawler | |
| parent | 03b04aabc9d9b63ff54a80f52590b619aee06159 (diff) | |
| download | sandcrawler-64d8a0e64b0b4f5d1c5927c7a45317f5bc65a421.tar.gz sandcrawler-64d8a0e64b0b4f5d1c5927c7a45317f5bc65a421.zip | |
be more parsimonious with GROBID metadata
Because these are getting persisted in database (as well as kafka),
don't write out empty keys.
Diffstat (limited to 'python/sandcrawler')
| -rw-r--r-- | python/sandcrawler/grobid.py | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py index 70f7b16..9fd5ad4 100644 --- a/python/sandcrawler/grobid.py +++ b/python/sandcrawler/grobid.py @@ -62,10 +62,12 @@ class GrobidClient(object): meta = dict() biblio = dict() for k in ('title', 'authors', 'journal', 'date', 'doi', ): - biblio[k] = tei_json.get(k) + if tei_json.get(k): + biblio[k] = tei_json[k] meta['biblio'] = biblio for k in ('grobid_version', 'grobid_timestamp', 'fatcat_release', 'language_code'): - meta[k] = tei_json.get(k) + if tei_json.get(k): + meta[k] = tei_json[k] return meta class GrobidWorker(SandcrawlerWorker): |