diff options
| -rwxr-xr-x | python/fatcat_ingest.py | 4 | ||||
| -rw-r--r-- | python/fatcat_tools/transforms/ingest.py | 12 |
2 files changed, 10 insertions, 6 deletions
diff --git a/python/fatcat_ingest.py b/python/fatcat_ingest.py index 68676ad2..b9d71a7c 100755 --- a/python/fatcat_ingest.py +++ b/python/fatcat_ingest.py @@ -87,6 +87,7 @@ def _run_search_dump(args, search): ingest_request = release_ingest_request( release, ingest_request_source="fatcat-ingest", + ingest_type=args.ingest_type, ) if not ingest_request: continue @@ -214,6 +215,9 @@ def main(): parser.add_argument('--force-recrawl', action='store_true', help="Tell ingest worker to skip GWB history lookup and do SPNv2 crawl") + parser.add_argument('--ingest-type', + default="pdf", + help="What medium to ingest (pdf, xml, html)") subparsers = parser.add_subparsers() sub_container = subparsers.add_parser('container', diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py index 2f4e2271..59831017 100644 --- a/python/fatcat_tools/transforms/ingest.py +++ b/python/fatcat_tools/transforms/ingest.py @@ -15,15 +15,19 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type= if release.state != 'active': return None + # TODO: infer ingest type based on release_type or container metadata? + if not ingest_type: + ingest_type = 'pdf' + # generate a URL where we expect to find fulltext url = None link_source = None link_source_id = None - if release.ext_ids.arxiv: + if release.ext_ids.arxiv and ingest_type == "pdf": url = "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv) link_source = "arxiv" link_source_id = release.ext_ids.arxiv - elif release.ext_ids.pmcid: + elif release.ext_ids.pmcid and ingest_type == "pdf": # TODO: how to tell if an author manuscript in PMC vs. published? #url = "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/".format(release.ext_ids.pmcid) url = "http://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.ext_ids.pmcid) @@ -40,10 +44,6 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type= ext_ids = release.ext_ids.to_dict() ext_ids = dict([(k, v) for (k, v) in ext_ids.items() if v]) - # TODO: infer ingest type based on release_type or container metadata? - if not ingest_type: - ingest_type = 'pdf' - ingest_request = { 'ingest_type': ingest_type, 'ingest_request_source': ingest_request_source, |