ingest tool: support for setting ingest type

2 files changed, 10 insertions, 6 deletions

diff --git a/python/fatcat_ingest.py b/python/fatcat_ingest.py
index 68676ad2..b9d71a7c 100755
--- a/python/fatcat_ingest.py
+++ b/python/fatcat_ingest.py
@@ -87,6 +87,7 @@ def _run_search_dump(args, search):
         ingest_request = release_ingest_request(
             release,
             ingest_request_source="fatcat-ingest",
+            ingest_type=args.ingest_type,
         )
         if not ingest_request:
             continue
@@ -214,6 +215,9 @@ def main():
     parser.add_argument('--force-recrawl',
         action='store_true',
         help="Tell ingest worker to skip GWB history lookup and do SPNv2 crawl")
+    parser.add_argument('--ingest-type',
+        default="pdf",
+        help="What medium to ingest (pdf, xml, html)")
     subparsers = parser.add_subparsers()
 
     sub_container = subparsers.add_parser('container',
diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py
index 2f4e2271..59831017 100644
--- a/python/fatcat_tools/transforms/ingest.py
+++ b/python/fatcat_tools/transforms/ingest.py
@@ -15,15 +15,19 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type=
     if release.state != 'active':
         return None
 
+    # TODO: infer ingest type based on release_type or container metadata?
+    if not ingest_type:
+        ingest_type = 'pdf'
+
     # generate a URL where we expect to find fulltext
     url = None
     link_source = None
     link_source_id = None
-    if release.ext_ids.arxiv:
+    if release.ext_ids.arxiv and ingest_type == "pdf":
         url = "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv)
         link_source = "arxiv"
         link_source_id = release.ext_ids.arxiv
-    elif release.ext_ids.pmcid:
+    elif release.ext_ids.pmcid and ingest_type == "pdf":
         # TODO: how to tell if an author manuscript in PMC vs. published?
         #url = "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/".format(release.ext_ids.pmcid)
         url = "http://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.ext_ids.pmcid)
@@ -40,10 +44,6 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type=
     ext_ids = release.ext_ids.to_dict()
     ext_ids = dict([(k, v) for (k, v) in ext_ids.items() if v])
 
-    # TODO: infer ingest type based on release_type or container metadata?
-    if not ingest_type:
-        ingest_type = 'pdf'
-
     ingest_request = {
         'ingest_type': ingest_type,
         'ingest_request_source': ingest_request_source,