summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-11-12 21:13:16 -0800
committerBryan Newbold <bnewbold@robocracy.org>2019-11-15 16:46:26 -0800
commit06cf64414d4c7fa497c9ddb83b7c066b3779c4d2 (patch)
tree202d60c467b6404202c76c6a03c763ecf5c8186c
parent5038de89f5539cfed150b302d672d5c04f380a65 (diff)
downloadfatcat-06cf64414d4c7fa497c9ddb83b7c066b3779c4d2.tar.gz
fatcat-06cf64414d4c7fa497c9ddb83b7c066b3779c4d2.zip
add ingest request transform (and test)
-rw-r--r--python/fatcat_tools/transforms/__init__.py1
-rw-r--r--python/fatcat_tools/transforms/ingest.py66
-rw-r--r--python/tests/transform_tests.py2
3 files changed, 68 insertions, 1 deletions
diff --git a/python/fatcat_tools/transforms/__init__.py b/python/fatcat_tools/transforms/__init__.py
index 735d1b29..6a4b1bba 100644
--- a/python/fatcat_tools/transforms/__init__.py
+++ b/python/fatcat_tools/transforms/__init__.py
@@ -2,3 +2,4 @@
from .entities import entity_to_dict, entity_from_json, entity_from_dict
from .elasticsearch import release_to_elasticsearch, container_to_elasticsearch, changelog_to_elasticsearch
from .csl import release_to_csl, citeproc_csl
+from .ingest import release_ingest_request
diff --git a/python/fatcat_tools/transforms/ingest.py b/python/fatcat_tools/transforms/ingest.py
new file mode 100644
index 00000000..eee60630
--- /dev/null
+++ b/python/fatcat_tools/transforms/ingest.py
@@ -0,0 +1,66 @@
+
+from .elasticsearch import release_to_elasticsearch
+
+def release_ingest_request(release, oa_only=False, project='fatcat'):
+ """
+ Takes a full release entity object and returns an ingest request (as dict),
+ or None if it seems like this release shouldn't be ingested.
+
+ The release entity should have the container, file, fileset, and webcapture
+ fields set.
+
+ The 'oa_only' boolean flag indicates that we should only return an ingest
+ request if we have reason to believe this is an OA release (or, eg, in
+ arxiv or pubmed central). Respecting this flag means we are likely to miss
+ a lot of "hybrid" and "bronze" content, but could reduce load
+ significantly.
+
+ The type of the ingest request may depend on release type and container
+ metadata (eg, as to whether we expect a PDF, datasets, web page), so
+ calling code should check the returned type field.
+ """
+
+ if release.state != 'active':
+ return None
+
+ # generate a URL where we expect to find fulltext
+ url = None
+ expect_mimetypes = []
+ if release.ext_ids.arxiv:
+ url = "https://arxiv.org/pdf/{}.pdf".format(release.ext_ids.arxiv)
+ expect_mimetypes = ['application/pdf']
+ elif release.ext_ids.pmcid:
+ #url = "https://www.ncbi.nlm.nih.gov/pmc/articles/{}/pdf/".format(release.pmcid)
+ url = "http://europepmc.org/backend/ptpmcrender.fcgi?accid={}&blobtype=pdf".format(release.pmcid)
+ expect_mimetypes = ['application/pdf']
+ elif release.ext_ids.doi:
+ url = "https://doi.org/{}".format(release.ext_ids.doi)
+
+ if not url:
+ return None
+
+ ext_ids = dict()
+ for k in ('doi', 'pmid', 'pmcid', 'arxiv'):
+ v = getattr(release.ext_ids, k)
+ if v:
+ ext_ids[k] = v
+
+ if oa_only and not ext_ids['arxiv'] and not ext_ids['pmcid']:
+ es = release_to_elasticsearch(release)
+ if not es['is_oa']:
+ return None
+
+ ingest_request = {
+ 'ingest_type': 'file',
+ 'project': project,
+ 'base_url': url,
+ 'fatcat': {
+ 'release_stage': release.release_stage,
+ 'release_ident': release.ident,
+ 'work_ident': release.work_id,
+ },
+ 'ext_ids': ext_ids,
+ 'expect_mimetypes': expect_mimetypes or None,
+ }
+ return ingest_request
+
diff --git a/python/tests/transform_tests.py b/python/tests/transform_tests.py
index 5e657190..f254e117 100644
--- a/python/tests/transform_tests.py
+++ b/python/tests/transform_tests.py
@@ -16,7 +16,7 @@ def test_basic_elasticsearch_convert(crossref_importer):
r.state = 'active'
release_to_elasticsearch(r)
-def test_rich_elasticsearch_convert(crossref_importer):
+def test_rich_elasticsearch_convert():
r = ReleaseEntity(
title="something",
release_year=1234,