summaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools/transforms/ingest.py
diff options
context:
space:
mode:
authorMartin Czygan <martin@archive.org>2020-11-19 22:36:55 +0000
committerMartin Czygan <martin@archive.org>2020-11-19 22:36:55 +0000
commit03eadfc7e2bee4213345f6464378e87b8f741d20 (patch)
tree3e5b13af8ba46b240f9ae53d5f522fb7ee02c219 /python/fatcat_tools/transforms/ingest.py
parent5afde4690a4653db53fe4962af5da3eb9188d9a2 (diff)
parenta73b73c2944b3df2a62886c4e6b69c93f5e74222 (diff)
downloadfatcat-03eadfc7e2bee4213345f6464378e87b8f741d20.tar.gz
fatcat-03eadfc7e2bee4213345f6464378e87b8f741d20.zip
Merge branch 'bnewbold-xml-html-ingest' into 'master'
HTML webcapture ingest (and XML file ingest) See merge request webgroup/fatcat!88
Diffstat (limited to 'python/fatcat_tools/transforms/ingest.py')
-rw-r--r--python/fatcat_tools/transforms/ingest.py12
1 files changed, 6 insertions, 6 deletions
diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py
index 2f4e2271..59831017 100644
--- a/python/fatcat_tools/transforms/ingest.py
+++ b/python/fatcat_tools/transforms/ingest.py
@@ -15,15 +15,19 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type=
if release.state != 'active':
return None
+ # TODO: infer ingest type based on release_type or container metadata?
+ if not ingest_type:
+ ingest_type = 'pdf'
+
# generate a URL where we expect to find fulltext
url = None
link_source = None
link_source_id = None
- if release.ext_ids.arxiv:
+ if release.ext_ids.arxiv and ingest_type == "pdf":
url = "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv)
link_source = "arxiv"
link_source_id = release.ext_ids.arxiv
- elif release.ext_ids.pmcid:
+ elif release.ext_ids.pmcid and ingest_type == "pdf":
# TODO: how to tell if an author manuscript in PMC vs. published?
#url = "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/".format(release.ext_ids.pmcid)
url = "http://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.ext_ids.pmcid)
@@ -40,10 +44,6 @@ def release_ingest_request(release, ingest_request_source='fatcat', ingest_type=
ext_ids = release.ext_ids.to_dict()
ext_ids = dict([(k, v) for (k, v) in ext_ids.items() if v])
- # TODO: infer ingest type based on release_type or container metadata?
- if not ingest_type:
- ingest_type = 'pdf'
-
ingest_request = {
'ingest_type': ingest_type,
'ingest_request_source': ingest_request_source,