bunch of pylint cleanup

9 files changed, 47 insertions, 29 deletions

diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index a5527b8c..cdf04db1 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -3,7 +3,7 @@
 import sys
 import argparse
 from fatcat_tools.importers import CrossrefImporter, OrcidImporter, \
-    IssnImporter, MatchedImporter, GrobidMetadataImporter 
+    IssnImporter, MatchedImporter, GrobidMetadataImporter
 
 
 def run_crossref(args):
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index 1a6807d2..ed80cfc9 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -3,6 +3,7 @@ import re
 import sys
 import csv
 import json
+import time
 import requests
 import itertools
 import datetime
@@ -10,6 +11,11 @@ from pykafka import KafkaClient
 
 from fatcat_tools.workers.worker_common import most_recent_message
 
+# Skip pylint due to:
+#   AttributeError: 'NoneType' object has no attribute 'scope'
+# in 'astroid/node_classes.py'
+# pylint: skip-file
+
 DATE_FMT = "%Y-%m-%d"
 
 
@@ -79,7 +85,7 @@ class HarvestCrossrefWorker:
             date_str, date_str)
         if self.is_update_filter is not None:
             filter_param += ',is_update:{}'.format(bool(self.is_update_filter))
-        params = {
+        return {
             'filter': filter_param,
             'rows': self.api_batch_size,
             'cursor': '*',
@@ -93,7 +99,7 @@ class HarvestCrossrefWorker:
 
         state_topic = self.kafka.topics[self.state_topic]
         produce_topic = self.kafka.topics[self.produce_topic]
- 
+
         date_str = date.strftime(DATE_FMT)
         params = self.params(date_str)
         headers = {
@@ -103,12 +109,12 @@ class HarvestCrossrefWorker:
         with produce_topic.get_producer() as producer:
             while True:
                 http_resp = requests.get(self.api_host_url, params, headers=headers)
-                if http_resp.status_code is 503:
+                if http_resp.status_code == 503:
                     # crud backoff
                     print("got HTTP {}, pausing for 30 seconds".format(http_resp.status_code))
                     time.sleep(30.0)
                     continue
-                assert http_resp.status_code is 200
+                assert http_resp.status_code == 200
                 resp = http_resp.json()
                 items = self.extract_items(resp)
                 count += len(items)
@@ -135,7 +141,7 @@ class HarvestCrossrefWorker:
         today_utc = datetime.datetime.utcnow().date()
         if self.start_date is None:
             self.start_date = self.get_latest_date()
-            if self.start_date: 
+            if self.start_date:
                 # if we are continuing, start day after last success
                 self.start_date = self.start_date + datetime.timedelta(days=1)
         if self.start_date is None:
@@ -167,7 +173,7 @@ class HarvestDataciteWorker(HarvestCrossrefWorker):
     """
     datacite has a REST API as well as OAI-PMH endpoint.
 
-    have about 8 million 
+    have about 8 million
 
     bulk export notes: https://github.com/datacite/datacite/issues/188
 
@@ -206,4 +212,3 @@ class HarvestDataciteWorker(HarvestCrossrefWorker):
     def update_params(self, params, resp):
         params['page[number]'] = resp['meta']['page'] + 1
         return params
-
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 9cf92b41..e1efde80 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -37,12 +37,21 @@ class FatcatImporter:
         print("Processed {} lines, inserted {}, updated {}.".format(
             self.counts['processed_lines'], self.counts['insert'], self.counts['update']))
 
+    def create_row(self, row, editgroup_id=None):
+        # sub-classes expected to implement this
+        raise NotImplementedError
+
+    def create_batch(self, rows, editgroup_id=None):
+        # sub-classes expected to implement this
+        raise NotImplementedError
+
     def process_source(self, source, group_size=100):
         """Creates and auto-accepts editgroup every group_size rows"""
         eg = self.api.create_editgroup(
             fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae'))
+        i = 0
         for i, row in enumerate(source):
-            self.create_row(row, editgroup=eg.id)
+            self.create_row(row, editgroup_id=eg.id)
             if i > 0 and (i % group_size) == 0:
                 self.api.accept_editgroup(eg.id)
                 eg = self.api.create_editgroup(
@@ -57,7 +66,7 @@ class FatcatImporter:
             self.counts['processed_lines'] += len(rows)
             eg = self.api.create_editgroup(
                 fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae'))
-            self.create_batch(rows, editgroup=eg.id)
+            self.create_batch(rows, editgroup_id=eg.id)
 
     def process_csv_source(self, source, group_size=100, delimiter=','):
         reader = csv.DictReader(source, delimiter=delimiter)
@@ -85,7 +94,7 @@ class FatcatImporter:
         return container_id
 
     def is_orcid(self, orcid):
-        return self._orcid_regex.match(orcid) != None
+        return self._orcid_regex.match(orcid) is not None
 
     def lookup_orcid(self, orcid):
         """Caches calls to the Orcid lookup API endpoint in a local dict"""
diff --git a/python/fatcat_tools/importers/crossref.py b/python/fatcat_tools/importers/crossref.py
index fac8f32b..d0a69cd6 100644
--- a/python/fatcat_tools/importers/crossref.py
+++ b/python/fatcat_tools/importers/crossref.py
@@ -112,7 +112,7 @@ class CrossrefImporter(FatcatImporter):
                     extra['sequence'] = am.get('sequence')
                 if not extra:
                     extra = None
-                assert(ctype in ("author", "editor", "translator"))
+                assert ctype in ("author", "editor", "translator")
                 contribs.append(fatcat_client.ReleaseContrib(
                     creator_id=creator_id,
                     index=index,
@@ -133,7 +133,7 @@ class CrossrefImporter(FatcatImporter):
         publisher = obj.get('publisher')
 
         ce = None
-        if (container_id is None and self.create_containers and issnl != None 
+        if (container_id is None and self.create_containers and (issnl is not None)
             and obj.get('container-title') and len(obj['container-title']) > 0):
             ce = fatcat_client.ContainerEntity(
                 issnl=issnl,
diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py
index ba8a4e6f..d525d4f7 100644
--- a/python/fatcat_tools/importers/grobid_metadata.py
+++ b/python/fatcat_tools/importers/grobid_metadata.py
@@ -21,7 +21,6 @@ class GrobidMetadataImporter(FatcatImporter):
         if not obj.get('title'):
             return None
 
-        release = dict()
         extra = dict()
 
         if obj.get('abstract') and len(obj.get('abstract')) < MAX_ABSTRACT_BYTES:
@@ -35,7 +34,6 @@ class GrobidMetadataImporter(FatcatImporter):
 
         contribs = []
         for i, a in enumerate(obj.get('authors', [])):
-            c = dict(raw_name=a['name'], role="author")
             contribs.append(fatcat_client.ReleaseContrib(
                 index=i,
                 raw_name=a['name'],
@@ -67,7 +65,6 @@ class GrobidMetadataImporter(FatcatImporter):
             ref['extra'] = cite_extra
             refs.append(ref)
 
-        release_type = "article-journal"
         release_date = None
         if obj.get('date'):
             # TODO: only returns year, ever? how to handle?
@@ -77,7 +74,7 @@ class GrobidMetadataImporter(FatcatImporter):
             extra['doi'] = obj['doi']
         if obj['journal'] and obj['journal'].get('name'):
             extra['container_name'] = obj['journal']['name']
-        
+
         extra['is_longtail_oa'] = True
 
         # TODO: ISSN/eISSN handling? or just journal name lookup?
@@ -89,6 +86,8 @@ class GrobidMetadataImporter(FatcatImporter):
 
         re = fatcat_client.ReleaseEntity(
             title=obj['title'].strip(),
+            release_type="article-journal",
+            release_date=release_date,
             contribs=contribs,
             refs=refs,
             publisher=obj['journal'].get('publisher'),
@@ -97,7 +96,7 @@ class GrobidMetadataImporter(FatcatImporter):
             abstracts=abstracts,
             extra=extra)
         return re
-    
+
     # TODO: make this a common function somewhere
     def make_url(self, raw):
         rel = self.default_link_rel
@@ -111,7 +110,7 @@ class GrobidMetadataImporter(FatcatImporter):
         return fatcat_client.FileEntityUrls(url=raw, rel=rel)
 
     def parse_file_metadata(self, sha1_key, cdx, mimetype, file_size):
-        
+
         sha1 = base64.b16encode(base64.b32decode(sha1_key.replace('sha1:', ''))).decode('ascii').lower()
 
         # lookup existing SHA1, or create new entity
@@ -141,7 +140,7 @@ class GrobidMetadataImporter(FatcatImporter):
         fe.urls.append(
             fatcat_client.FileEntityUrls(url=wayback, rel="webarchive"))
         original_url = self.make_url(original)
-        if original_url != None:
+        if original_url is not None:
             fe.urls.append(original_url)
 
         return fe
diff --git a/python/fatcat_tools/importers/issn.py b/python/fatcat_tools/importers/issn.py
index 0b0efccb..f702dc60 100644
--- a/python/fatcat_tools/importers/issn.py
+++ b/python/fatcat_tools/importers/issn.py
@@ -17,6 +17,7 @@ def truthy(s):
     if s is None:
         return None
     s = s.lower()
+
     if s in ('true', 't', 'yes', 'y', '1'):
         return True
     elif s in ('false', 'f', 'no', 'n', '0'):
@@ -37,12 +38,12 @@ class IssnImporter(FatcatImporter):
     def parse_issn_row(self, row):
         """
         row is a python dict (parsed from CSV).
-        returns a ContainerEntity
+        returns a ContainerEntity (or None if invalid or couldn't parse)
         """
         title = or_none(row['title'])
         issnl = or_none(row['ISSN-L'])
         if title is None or issnl is None:
-            return
+            return None
         extra = dict(
             in_doaj=truthy(row['in_doaj']),
             in_road=truthy(row['in_road']),
@@ -72,7 +73,7 @@ class IssnImporter(FatcatImporter):
     def create_batch(self, batch, editgroup=None):
         """Reads and processes in batches (not API-call-per-line)"""
         objects = [self.parse_issn_row(l)
-                   for l in batch if l != None]
-        objects = [o for o in objects if o != None]
+                   for l in batch if (l is not None)]
+        objects = [o for o in objects if (o is not None)]
         self.api.create_container_batch(objects, autoaccept="true", editgroup=editgroup)
         self.counts['insert'] += len(objects)
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py
index e64c043b..e803e2d0 100644
--- a/python/fatcat_tools/workers/changelog.py
+++ b/python/fatcat_tools/workers/changelog.py
@@ -27,7 +27,7 @@ class ChangelogWorker(FatcatWorker):
         # topic
         if self.offset is None:
             print("Checking for most recent changelog offset...")
-            msg = self.most_recent_message(topic)
+            msg = most_recent_message(topic)
             if msg:
                 self.offset = json.loads(msg.decode('utf-8'))['index']
             else:
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index 62096a93..64d5931e 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -50,11 +50,13 @@ def container_create_view():
 
 @app.route('/container/create', methods=['POST'])
 def container_create():
+    raise NotImplementedError
     params = dict()
     for k in request.form:
         if k.startswith('container_'):
             params[k[10:]] = request.form[k]
-    edit = api.create_container(params=params)
+    container = None
+    edit = api.create_container(container, params=params)
     return redirect("/container/{}".format(edit.ident))
 
 @app.route('/container/lookup', methods=['GET'])
@@ -174,11 +176,13 @@ def release_create_view():
 
 @app.route('/release/create', methods=['POST'])
 def release_create():
+    raise NotImplementedError
     params = dict()
     for k in request.form:
         if k.startswith('release_'):
             params[k[10:]] = request.form[k]
-    edit = api.create_release(params=params)
+    release = None
+    edit = api.create_release(release, params=params)
     return redirect("/release/{}".format(edit.ident))
 
 @app.route('/release/<ident>/history', methods=['GET'])
@@ -260,7 +264,7 @@ def work_view(ident):
 
 @app.route('/editgroup/current', methods=['GET'])
 def editgroup_current():
-    raise NotImplemented()
+    raise NotImplementedError
     #eg = api.get_or_create_editgroup()
     #return redirect('/editgroup/{}'.format(eg.id))
 
diff --git a/python/fatcat_worker.py b/python/fatcat_worker.py
index 2f883fe0..e0ac48d8 100755
--- a/python/fatcat_worker.py
+++ b/python/fatcat_worker.py
@@ -21,7 +21,7 @@ def run_entity_updates(args):
 
 def run_elasticsearch_release(args):
     consume_topic = "fatcat-{}.release-updates".format(args.env)
-    worker = ReleaseWorker(args.kafka_hosts,
+    worker = ElasticsearchReleaseWorker(args.kafka_hosts,
         consume_topic, elasticsearch_backend=args.elasticsearch_backend,
         elasticsearch_index=args.elasticsearch_index)
     worker.run()