From f60d37fb2f9079e6707f9a253983b6ea07964e18 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 6 Dec 2021 16:25:21 -0800 Subject: fetch GROBID-parsed refs along with crossref metadata --- fatcat_scholar/sandcrawler.py | 6 ++++-- fatcat_scholar/work_pipeline.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'fatcat_scholar') diff --git a/fatcat_scholar/sandcrawler.py b/fatcat_scholar/sandcrawler.py index 087cdc6..5580841 100644 --- a/fatcat_scholar/sandcrawler.py +++ b/fatcat_scholar/sandcrawler.py @@ -39,8 +39,10 @@ class SandcrawlerPostgrestClient: else: return None - def get_crossref(self, doi: str) -> Optional[Dict[str, Any]]: - resp = requests.get(self.api_url + "/crossref", params=dict(doi="eq." + doi)) + def get_crossref_with_refs(self, doi: str) -> Optional[Dict[str, Any]]: + resp = requests.get( + self.api_url + "/crossref_with_refs", params=dict(doi="eq." + doi) + ) resp.raise_for_status() resp_json = resp.json() if resp_json: diff --git a/fatcat_scholar/work_pipeline.py b/fatcat_scholar/work_pipeline.py index 9dce006..fa1a7bc 100644 --- a/fatcat_scholar/work_pipeline.py +++ b/fatcat_scholar/work_pipeline.py @@ -276,13 +276,14 @@ class WorkPipeline: # if this is definitely a Datacite or JALC DOI, can skip the Crossref cache lookup return None doi = re.ext_ids.doi.lower() - crossref_meta = self.sandcrawler_db_client.get_crossref(doi) + crossref_meta = self.sandcrawler_db_client.get_crossref_with_refs(doi) if not crossref_meta or not crossref_meta.get("record"): return None return dict( release_ident=re.ident, doi=doi, record=crossref_meta["record"], + grobid_refs=crossref_meta["refs_json"] or [], ) def lookup_sim(self, release: ReleaseEntity) -> Optional[SimIssueRow]: -- cgit v1.2.3