aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xpython/fatcat_import.py2
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py19
-rw-r--r--python/fatcat_tools/importers/common.py15
-rw-r--r--python/fatcat_tools/importers/crossref.py4
-rw-r--r--python/fatcat_tools/importers/grobid_metadata.py13
-rw-r--r--python/fatcat_tools/importers/issn.py9
-rw-r--r--python/fatcat_tools/workers/changelog.py2
-rw-r--r--python/fatcat_web/routes.py10
-rwxr-xr-xpython/fatcat_worker.py2
9 files changed, 47 insertions, 29 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index a5527b8c..cdf04db1 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -3,7 +3,7 @@
import sys
import argparse
from fatcat_tools.importers import CrossrefImporter, OrcidImporter, \
- IssnImporter, MatchedImporter, GrobidMetadataImporter
+ IssnImporter, MatchedImporter, GrobidMetadataImporter
def run_crossref(args):
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index 1a6807d2..ed80cfc9 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -3,6 +3,7 @@ import re
import sys
import csv
import json
+import time
import requests
import itertools
import datetime
@@ -10,6 +11,11 @@ from pykafka import KafkaClient
from fatcat_tools.workers.worker_common import most_recent_message
+# Skip pylint due to:
+# AttributeError: 'NoneType' object has no attribute 'scope'
+# in 'astroid/node_classes.py'
+# pylint: skip-file
+
DATE_FMT = "%Y-%m-%d"
@@ -79,7 +85,7 @@ class HarvestCrossrefWorker:
date_str, date_str)
if self.is_update_filter is not None:
filter_param += ',is_update:{}'.format(bool(self.is_update_filter))
- params = {
+ return {
'filter': filter_param,
'rows': self.api_batch_size,
'cursor': '*',
@@ -93,7 +99,7 @@ class HarvestCrossrefWorker:
state_topic = self.kafka.topics[self.state_topic]
produce_topic = self.kafka.topics[self.produce_topic]
-
+
date_str = date.strftime(DATE_FMT)
params = self.params(date_str)
headers = {
@@ -103,12 +109,12 @@ class HarvestCrossrefWorker:
with produce_topic.get_producer() as producer:
while True:
http_resp = requests.get(self.api_host_url, params, headers=headers)
- if http_resp.status_code is 503:
+ if http_resp.status_code == 503:
# crud backoff
print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code))
time.sleep(30.0)
continue
- assert http_resp.status_code is 200
+ assert http_resp.status_code == 200
resp = http_resp.json()
items = self.extract_items(resp)
count += len(items)
@@ -135,7 +141,7 @@ class HarvestCrossrefWorker:
today_utc = datetime.datetime.utcnow().date()
if self.start_date is None:
self.start_date = self.get_latest_date()
- if self.start_date:
+ if self.start_date:
# if we are continuing, start day after last success
self.start_date = self.start_date + datetime.timedelta(days=1)
if self.start_date is None:
@@ -167,7 +173,7 @@ class HarvestDataciteWorker(HarvestCrossrefWorker):
"""
datacite has a REST API as well as OAI-PMH endpoint.
- have about 8 million
+ have about 8 million
bulk export notes: https://github.com/datacite/datacite/issues/188
@@ -206,4 +212,3 @@ class HarvestDataciteWorker(HarvestCrossrefWorker):
def update_params(self, params, resp):
params['page[number]'] = resp['meta']['page'] + 1
return params
-
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 9cf92b41..e1efde80 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -37,12 +37,21 @@ class FatcatImporter:
print("Processed {} lines, inserted {}, updated {}.".format(
self.counts['processed_lines'], self.counts['insert'], self.counts['update']))
+ def create_row(self, row, editgroup_id=None):
+ # sub-classes expected to implement this
+ raise NotImplementedError
+
+ def create_batch(self, rows, editgroup_id=None):
+ # sub-classes expected to implement this
+ raise NotImplementedError
+
def process_source(self, source, group_size=100):
"""Creates and auto-accepts editgroup every group_size rows"""
eg = self.api.create_editgroup(
fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae'))
+ i = 0
for i, row in enumerate(source):
- self.create_row(row, editgroup=eg.id)
+ self.create_row(row, editgroup_id=eg.id)
if i > 0 and (i % group_size) == 0:
self.api.accept_editgroup(eg.id)
eg = self.api.create_editgroup(
@@ -57,7 +66,7 @@ class FatcatImporter:
self.counts['processed_lines'] += len(rows)
eg = self.api.create_editgroup(
fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae'))
- self.create_batch(rows, editgroup=eg.id)
+ self.create_batch(rows, editgroup_id=eg.id)
def process_csv_source(self, source, group_size=100, delimiter=','):
reader = csv.DictReader(source, delimiter=delimiter)
@@ -85,7 +94,7 @@ class FatcatImporter:
return container_id
def is_orcid(self, orcid):
- return self._orcid_regex.match(orcid) != None
+ return self._orcid_regex.match(orcid) is not None
def lookup_orcid(self, orcid):
"""Caches calls to the Orcid lookup API endpoint in a local dict"""
diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py
index fac8f32b..d0a69cd6 100644
--- a/python/fatcat_tools/importers/crossref.py
+++ b/python/fatcat_tools/importers/crossref.py
@@ -112,7 +112,7 @@ class CrossrefImporter(FatcatImporter):
extra['sequence'] = am.get('sequence')
if not extra:
extra = None
- assert(ctype in ("author", "editor", "translator"))
+ assert ctype in ("author", "editor", "translator")
contribs.append(fatcat_client.ReleaseContrib(
creator_id=creator_id,
index=index,
@@ -133,7 +133,7 @@ class CrossrefImporter(FatcatImporter):
publisher = obj.get('publisher')
ce = None
- if (container_id is None and self.create_containers and issnl != None
+ if (container_id is None and self.create_containers and (issnl is not None)
and obj.get('container-title') and len(obj['container-title']) > 0):
ce = fatcat_client.ContainerEntity(
issnl=issnl,
diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py
index ba8a4e6f..d525d4f7 100644
--- a/python/fatcat_tools/importers/grobid_metadata.py
+++ b/python/fatcat_tools/importers/grobid_metadata.py
@@ -21,7 +21,6 @@ class GrobidMetadataImporter(FatcatImporter):
if not obj.get('title'):
return None
- release = dict()
extra = dict()
if obj.get('abstract') and len(obj.get('abstract')) < MAX_ABSTRACT_BYTES:
@@ -35,7 +34,6 @@ class GrobidMetadataImporter(FatcatImporter):
contribs = []
for i, a in enumerate(obj.get('authors', [])):
- c = dict(raw_name=a['name'], role="author")
contribs.append(fatcat_client.ReleaseContrib(
index=i,
raw_name=a['name'],
@@ -67,7 +65,6 @@ class GrobidMetadataImporter(FatcatImporter):
ref['extra'] = cite_extra
refs.append(ref)
- release_type = "article-journal"
release_date = None
if obj.get('date'):
# TODO: only returns year, ever? how to handle?
@@ -77,7 +74,7 @@ class GrobidMetadataImporter(FatcatImporter):
extra['doi'] = obj['doi']
if obj['journal'] and obj['journal'].get('name'):
extra['container_name'] = obj['journal']['name']
-
+
extra['is_longtail_oa'] = True
# TODO: ISSN/eISSN handling? or just journal name lookup?
@@ -89,6 +86,8 @@ class GrobidMetadataImporter(FatcatImporter):
re = fatcat_client.ReleaseEntity(
title=obj['title'].strip(),
+ release_type="article-journal",
+ release_date=release_date,
contribs=contribs,
refs=refs,
publisher=obj['journal'].get('publisher'),
@@ -97,7 +96,7 @@ class GrobidMetadataImporter(FatcatImporter):
abstracts=abstracts,
extra=extra)
return re
-
+
# TODO: make this a common function somewhere
def make_url(self, raw):
rel = self.default_link_rel
@@ -111,7 +110,7 @@ class GrobidMetadataImporter(FatcatImporter):
return fatcat_client.FileEntityUrls(url=raw, rel=rel)
def parse_file_metadata(self, sha1_key, cdx, mimetype, file_size):
-
+
sha1 = base64.b16encode(base64.b32decode(sha1_key.replace('sha1:', ''))).decode('ascii').lower()
# lookup existing SHA1, or create new entity
@@ -141,7 +140,7 @@ class GrobidMetadataImporter(FatcatImporter):
fe.urls.append(
fatcat_client.FileEntityUrls(url=wayback, rel="webarchive"))
original_url = self.make_url(original)
- if original_url != None:
+ if original_url is not None:
fe.urls.append(original_url)
return fe
diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py
index 0b0efccb..f702dc60 100644
--- a/python/fatcat_tools/importers/issn.py
+++ b/python/fatcat_tools/importers/issn.py
@@ -17,6 +17,7 @@ def truthy(s):
if s is None:
return None
s = s.lower()
+
if s in ('true', 't', 'yes', 'y', '1'):
return True
elif s in ('false', 'f', 'no', 'n', '0'):
@@ -37,12 +38,12 @@ class IssnImporter(FatcatImporter):
def parse_issn_row(self, row):
"""
row is a python dict (parsed from CSV).
- returns a ContainerEntity
+ returns a ContainerEntity (or None if invalid or couldn't parse)
"""
title = or_none(row['title'])
issnl = or_none(row['ISSN-L'])
if title is None or issnl is None:
- return
+ return None
extra = dict(
in_doaj=truthy(row['in_doaj']),
in_road=truthy(row['in_road']),
@@ -72,7 +73,7 @@ class IssnImporter(FatcatImporter):
def create_batch(self, batch, editgroup=None):
"""Reads and processes in batches (not API-call-per-line)"""
objects = [self.parse_issn_row(l)
- for l in batch if l != None]
- objects = [o for o in objects if o != None]
+ for l in batch if (l is not None)]
+ objects = [o for o in objects if (o is not None)]
self.api.create_container_batch(objects, autoaccept="true", editgroup=editgroup)
self.counts['insert'] += len(objects)
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py
index e64c043b..e803e2d0 100644
--- a/python/fatcat_tools/workers/changelog.py
+++ b/python/fatcat_tools/workers/changelog.py
@@ -27,7 +27,7 @@ class ChangelogWorker(FatcatWorker):
# topic
if self.offset is None:
print("Checking for most recent changelog offset...")
- msg = self.most_recent_message(topic)
+ msg = most_recent_message(topic)
if msg:
self.offset = json.loads(msg.decode('utf-8'))['index']
else:
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index 62096a93..64d5931e 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -50,11 +50,13 @@ def container_create_view():
@app.route('/container/create', methods=['POST'])
def container_create():
+ raise NotImplementedError
params = dict()
for k in request.form:
if k.startswith('container_'):
params[k[10:]] = request.form[k]
- edit = api.create_container(params=params)
+ container = None
+ edit = api.create_container(container, params=params)
return redirect("/container/{}".format(edit.ident))
@app.route('/container/lookup', methods=['GET'])
@@ -174,11 +176,13 @@ def release_create_view():
@app.route('/release/create', methods=['POST'])
def release_create():
+ raise NotImplementedError
params = dict()
for k in request.form:
if k.startswith('release_'):
params[k[10:]] = request.form[k]
- edit = api.create_release(params=params)
+ release = None
+ edit = api.create_release(release, params=params)
return redirect("/release/{}".format(edit.ident))
@app.route('/release/<ident>/history', methods=['GET'])
@@ -260,7 +264,7 @@ def work_view(ident):
@app.route('/editgroup/current', methods=['GET'])
def editgroup_current():
- raise NotImplemented()
+ raise NotImplementedError
#eg = api.get_or_create_editgroup()
#return redirect('/editgroup/{}'.format(eg.id))
diff --git a/python/fatcat_worker.py b/python/fatcat_worker.py
index 2f883fe0..e0ac48d8 100755
--- a/python/fatcat_worker.py
+++ b/python/fatcat_worker.py
@@ -21,7 +21,7 @@ def run_entity_updates(args):
def run_elasticsearch_release(args):
consume_topic = "fatcat-{}.release-updates".format(args.env)
- worker = ReleaseWorker(args.kafka_hosts,
+ worker = ElasticsearchReleaseWorker(args.kafka_hosts,
consume_topic, elasticsearch_backend=args.elasticsearch_backend,
elasticsearch_index=args.elasticsearch_index)
worker.run()