From e9dd3c73f036d3fba2680eeaff8e62ecf2dbf9a1 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 10 Aug 2020 15:07:19 -0700 Subject: update crawl blocklist for SPNv2 requests which mostly fail --- python/fatcat_tools/workers/changelog.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'python/fatcat_tools/workers') diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index d5891ad1..1ac7a865 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -101,12 +101,20 @@ class EntityUpdatesWorker(FatcatWorker): "10.3932/", # ccdc.cam.ac.uk: crystal structures "10.5517/", + # researchgate: mostly blocks our crawler + "10.13140/", + # springerlink: mostly blocks crawler + "10.1007/", + # nature group: mostly blocks crawler + "10.1038/", + # SAGE: mostly blocks crawler + "10.1177/", + # IOP: mostly blocks crawler + "10.1088/", ] self.live_pdf_ingest_doi_prefix_acceptlist = [ # biorxiv and medrxiv "10.1101/", - # researchgate - "10.13140/", # the lancet (often hybrid OA) "10.1016/s0140-6736", "10.1016/s2213-2600", -- cgit v1.2.3 From 2a492914082444690f853a55ab1394fc0cf50108 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 11 Aug 2020 14:52:47 -0700 Subject: entity update: skip ingest of figshare+zenodo 'group' DOIs --- python/fatcat_tools/workers/changelog.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'python/fatcat_tools/workers') diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index 1ac7a865..dc5ef299 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -175,6 +175,7 @@ class EntityUpdatesWorker(FatcatWorker): 'paper-conference', 'patent', 'peer_review', + 'post', 'report', 'retraction', 'review', @@ -217,6 +218,20 @@ class EntityUpdatesWorker(FatcatWorker): if doi.startswith(prefix): return False + # figshare + if doi and doi.startswith('10.6084/') or doi.startswith('10.25384/'): + # don't crawl "most recent version" (aka "group") DOIs + if not release.version: + return False + + # zenodo + if doi and doi.startswith('10.5281/'): + # if this is a "grouping" DOI of multiple "version" DOIs, do not crawl (will crawl the versioned DOIs) + if release.extra and release.extra.get('relations'): + for rel in release.extra['relations']: + if (rel.get('relationType') == 'HasVersion' and rel.get('relatedIdentifier', '').startswith('10.5281/')): + return False + return True def run(self): -- cgit v1.2.3 From 5eddc9b9aefbd7ae197d441b8a7af1fded940e2d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 11 Aug 2020 15:23:25 -0700 Subject: entity update: default to ingest non-OA works --- python/fatcat_tools/workers/changelog.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'python/fatcat_tools/workers') diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index dc5ef299..f7df6748 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -89,7 +89,7 @@ class EntityUpdatesWorker(FatcatWorker): self.ingest_file_request_topic = ingest_file_request_topic self.poll_interval = poll_interval self.consumer_group = "entity-updates" - self.ingest_oa_only = True + self.ingest_oa_only = False self.ingest_pdf_doi_prefix_blocklist = [ # gbif.org: many DOIs, not PDF fulltext "10.15468/", @@ -191,15 +191,16 @@ class EntityUpdatesWorker(FatcatWorker): 'stub', ) - # accept list sets a default "crawl it" despite OA metadata for - # known-OA DOI prefixes - in_acceptlist = False - if doi: - for prefix in self.live_pdf_ingest_doi_prefix_acceptlist: - if doi.startswith(prefix): - in_acceptlist = True - if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'): + + # accept list sets a default "crawl it" despite OA metadata for + # known-OA DOI prefixes + in_acceptlist = False + if doi: + for prefix in self.live_pdf_ingest_doi_prefix_acceptlist: + if doi.startswith(prefix): + in_acceptlist = True + es = release_to_elasticsearch(release) # most datacite documents are in IRs and should be crawled is_datacite_doc = False -- cgit v1.2.3 From 03d2004717d36962aef1bd373d59ce799d7db9ab Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 11 Aug 2020 15:45:36 -0700 Subject: entity update: change big5 ingest behavior In addition to changing the OA default, this was the main intended behavior change in this group of commits: want to ingest fewer attempts that we *expect* to fail, but default to ingest/crawl attempt if we are uncertain. This is because there is a long tail of journals that register DOIs and are defacto OA (fulltext is available), but we don't have metadata indicating them as such. --- python/fatcat_tools/workers/changelog.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'python/fatcat_tools/workers') diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index f7df6748..65a8fcd8 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -159,6 +159,7 @@ class EntityUpdatesWorker(FatcatWorker): link_source = ingest_request.get('ingest_request') ingest_type = ingest_request.get('ingest_type') doi = ingest_request.get('ext_ids', {}).get('doi') + es = release_to_elasticsearch(release) is_document = release.release_type in ( 'article', @@ -191,17 +192,16 @@ class EntityUpdatesWorker(FatcatWorker): 'stub', ) - if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'): + # accept list sets a default "crawl it" despite OA metadata for + # known-OA DOI prefixes + in_acceptlist = False + if doi: + for prefix in self.live_pdf_ingest_doi_prefix_acceptlist: + if doi.startswith(prefix): + in_acceptlist = True - # accept list sets a default "crawl it" despite OA metadata for - # known-OA DOI prefixes - in_acceptlist = False - if doi: - for prefix in self.live_pdf_ingest_doi_prefix_acceptlist: - if doi.startswith(prefix): - in_acceptlist = True + if self.ingest_oa_only and link_source not in ('arxiv', 'pmc'): - es = release_to_elasticsearch(release) # most datacite documents are in IRs and should be crawled is_datacite_doc = False if release.extra and ('datacite' in release.extra) and is_document: @@ -209,6 +209,12 @@ class EntityUpdatesWorker(FatcatWorker): if not (es['is_oa'] or in_acceptlist or is_datacite_doc): return False + # big publishers *generally* have accurate OA metadata, use + # preservation networks, and block our crawlers. So unless OA, or + # explicitly on accept list, or not preserved, skip crawling + if es['publisher_type'] == 'big5' and es['is_preserved'] and not (es['is_oa'] or in_acceptlist): + return False + # if ingest_type is pdf but release_type is almost certainly not a PDF, # skip it. This is mostly a datacite thing. if ingest_type == "pdf" and is_not_pdf: -- cgit v1.2.3