Merge branch 'bnewbold-py37-cleanups' into 'master'

py37 cleanups

See merge request webgroup/fatcat!44

7 files changed, 39 insertions, 64 deletions

diff --git a/python/Pipfile b/python/Pipfile
index 1bcb55cd..eb706815 100644
--- a/python/Pipfile
+++ b/python/Pipfile
@@ -11,7 +11,7 @@ name = "pypi"
 pytest = ">=5,<6.0.0"
 pytest-pythonpath = "*"
 pytest-pylint = "*"
-ipython = "<7.0.0"
+ipython = "*"
 responses = ">=0.10"
 pytest-cov = "*"
 pylint = "*"
@@ -55,12 +55,8 @@ pathlib2 = "*"
 pycountry = "*"
 tldextract = "*"
 
-# this is only to lock to a python3.5-compatible version. needed by an
-# importlib-metadata, under pytest
-zipp = "<2.0.0"
-
 [requires]
-# Python 3.5 is the bundled (system) version of python for Ubuntu 16.04
-# (Xenial), currently the default on Internet Archive production VMs. But we
-# install Python 3.7 using an external package repository.
+# We install Python 3.7 using a PPA (deadsnakes) on Internet Archive cluster
+# machines, which as of Spring 2020 are still running Ubuntu Xenial (16.04),
+# which has a system default of Python 3.5.
 python_version = "3.7"
diff --git a/python/Pipfile.lock b/python/Pipfile.lock
index ba9b8972..4efd165d 100644
--- a/python/Pipfile.lock
+++ b/python/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "dc6b4bafa0214cc4ef81cc0946767ae8d71c76da670e404083015bf0b3996e42"
+            "sha256": "d93f0b9522e1aa9a60030febce10f8941c6eafc638aeb1663e915721e94aff79"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -128,6 +128,7 @@
                 "sha256:4ec0fc40eea26d21586e09eea9046efc3f24df78911b14decf36f613568bfa7e",
                 "sha256:54b101bfe4b073712cd2d4523ecda56138eb3737f4825dcf3575bb9183334774",
                 "sha256:57f7a2570587fef85ef0a97c11f05ef2e9e9929fd752351974b75e2c8c65e585",
+                "sha256:598ebb288d708d62d087b1736b51e98a6f5fa799db8da9f826197ee0744dfc70",
                 "sha256:5bc72cafd9a857618b3ef1f851bc3c9e023182cd74ec3a711d0f335f31ae1803",
                 "sha256:6e99142b37ea58274f42ddd40ecea46520d14cd40689e03ed2dfedb2863c127e",
                 "sha256:74c68a1c6c63eaf4056e55b5162e9f56f26fbd872571660ab629195c63b92198",
@@ -654,14 +655,6 @@
             ],
             "index": "pypi",
             "version": "==2.2.1"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:c70410551488251b0fee67b460fb9a536af8d6f9f008ad10ac51f615b6a521b1",
-                "sha256:e0d9e63797e483a30d27e09fffd308c59a700d365ec34e93cc100844168bf921"
-            ],
-            "index": "pypi",
-            "version": "==1.2.0"
         }
     },
     "develop": {
@@ -760,11 +753,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:007dcd929c14631f83daff35df0147ea51d1af420da303fd078343878bd5fb62",
-                "sha256:b0f2ef9eada4a68ef63ee10b6dde4f35c840035c50fd24265f8052c98947d5a4"
+                "sha256:ca478e52ae1f88da0102360e57e528b92f3ae4316aabac80a2cd7f7ab2efb48a",
+                "sha256:eb8d075de37f678424527b5ef6ea23f7b80240ca031c2dd6de5879d687a65333"
             ],
             "index": "pypi",
-            "version": "==6.5.0"
+            "version": "==7.13.0"
         },
         "ipython-genutils": {
             "hashes": [
@@ -782,10 +775,10 @@
         },
         "jedi": {
             "hashes": [
-                "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2",
-                "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5"
+                "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798",
+                "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030"
             ],
-            "version": "==0.16.0"
+            "version": "==0.17.0"
         },
         "lazy-object-proxy": {
             "hashes": [
@@ -841,14 +834,6 @@
             ],
             "version": "==0.7.0"
         },
-        "pathlib2": {
-            "hashes": [
-                "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db",
-                "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868"
-            ],
-            "index": "pypi",
-            "version": "==2.3.5"
-        },
         "pexpect": {
             "hashes": [
                 "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
@@ -880,11 +865,10 @@
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:37925b37a4af1f6448c76b7606e0285f79f434ad246dda007a27411cca730c6d",
-                "sha256:dd4fca02c8069497ad931a2d09914c6b0d1b50151ce876bc15bde4c747090126",
-                "sha256:f7eec66105baf40eda9ab026cd8b2e251337eea8d111196695d82e0c5f0af852"
+                "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8",
+                "sha256:df7e9e63aea609b1da3a65641ceaf5bc7d05e0a04de5bd45d05dbeffbabf9e04"
             ],
-            "version": "==1.0.18"
+            "version": "==3.0.5"
         },
         "psycopg2": {
             "hashes": [
@@ -995,12 +979,6 @@
             "index": "pypi",
             "version": "==0.10.12"
         },
-        "simplegeneric": {
-            "hashes": [
-                "sha256:dc972e06094b9af5b855b3df4a646395e43d1c9d0d39ed345b7393560d0b9173"
-            ],
-            "version": "==0.8.1"
-        },
         "six": {
             "hashes": [
                 "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a",
@@ -1064,11 +1042,10 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:c70410551488251b0fee67b460fb9a536af8d6f9f008ad10ac51f615b6a521b1",
-                "sha256:e0d9e63797e483a30d27e09fffd308c59a700d365ec34e93cc100844168bf921"
+                "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b",
+                "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96"
             ],
-            "index": "pypi",
-            "version": "==1.2.0"
+            "version": "==3.1.0"
         }
     }
 }
diff --git a/python/fatcat_harvest.py b/python/fatcat_harvest.py
index a45b44f8..5f67c679 100755
--- a/python/fatcat_harvest.py
+++ b/python/fatcat_harvest.py
@@ -15,8 +15,8 @@ sentry_client = raven.Client()
 def run_crossref(args):
     worker = HarvestCrossrefWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.api-crossref".format(args.env),
-        state_topic="fatcat-{}.api-crossref-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.api-crossref",
+        state_topic=f"fatcat-{args.env}.api-crossref-state",
         contact_email=args.contact_email,
         start_date=args.start_date,
         end_date=args.end_date)
@@ -25,8 +25,8 @@ def run_crossref(args):
 def run_datacite(args):
     worker = HarvestDataciteWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.api-datacite".format(args.env),
-        state_topic="fatcat-{}.api-datacite-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.api-datacite",
+        state_topic=f"fatcat-{args.env}.api-datacite-state",
         contact_email=args.contact_email,
         start_date=args.start_date,
         end_date=args.end_date)
@@ -35,8 +35,8 @@ def run_datacite(args):
 def run_arxiv(args):
     worker = HarvestArxivWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.oaipmh-arxiv".format(args.env),
-        state_topic="fatcat-{}.oaipmh-arxiv-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.oaipmh-arxiv",
+        state_topic=f"fatcat-{args.env}.oaipmh-arxiv-state",
         start_date=args.start_date,
         end_date=args.end_date)
     worker.run(continuous=args.continuous)
@@ -44,8 +44,8 @@ def run_arxiv(args):
 def run_pubmed(args):
     worker = PubmedFTPWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.ftp-pubmed".format(args.env),
-        state_topic="fatcat-{}.ftp-pubmed-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.ftp-pubmed",
+        state_topic=f"fatcat-{args.env}.ftp-pubmed-state",
         start_date=args.start_date,
         end_date=args.end_date)
     worker.run(continuous=args.continuous)
@@ -53,8 +53,8 @@ def run_pubmed(args):
 def run_doaj_article(args):
     worker = HarvestDoajArticleWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.oaipmh-doaj-article".format(args.env),
-        state_topic="fatcat-{}.oaipmh-doaj-article-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.oaipmh-doaj-article",
+        state_topic="fatcat-{args.env}.oaipmh-doaj-article-state",
         start_date=args.start_date,
         end_date=args.end_date)
     worker.run(continuous=args.continuous)
@@ -62,8 +62,8 @@ def run_doaj_article(args):
 def run_doaj_journal(args):
     worker = HarvestDoajJournalWorker(
         kafka_hosts=args.kafka_hosts,
-        produce_topic="fatcat-{}.oaipmh-doaj-journal".format(args.env),
-        state_topic="fatcat-{}.oaipmh-doaj-journal-state".format(args.env),
+        produce_topic=f"fatcat-{args.env}.oaipmh-doaj-journal",
+        state_topic=f"fatcat-{args.env}.oaipmh-doaj-journal-state",
         start_date=args.start_date,
         end_date=args.end_date)
     worker.run(continuous=args.continuous)
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index da611ecb..99c330a6 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -306,7 +306,7 @@ class EntityImporter:
 
         self._issnl_id_map = dict()
         self._orcid_id_map = dict()
-        self._orcid_regex = re.compile("^\\d{4}-\\d{4}-\\d{4}-\\d{3}[\\dX]$")
+        self._orcid_regex = re.compile(r"^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$")
         self._doi_id_map = dict()
         self._pmid_id_map = dict()
 
diff --git a/python/fatcat_tools/normal.py b/python/fatcat_tools/normal.py
index 7a2b5fd9..528a822e 100644
--- a/python/fatcat_tools/normal.py
+++ b/python/fatcat_tools/normal.py
@@ -6,7 +6,7 @@ free-form input, titles, etc.
 
 import re
 
-DOI_REGEX = re.compile("^10.\d{3,6}/\S+$")
+DOI_REGEX = re.compile(r"^10.\d{3,6}/\S+$")
 
 def clean_doi(raw):
     """
@@ -66,7 +66,7 @@ def test_clean_doi():
     assert clean_doi("doi:10.1234/ asdf ") == None
     assert clean_doi("10.4149/gpb¬_2017042") == None  # "logical negation" character
 
-ARXIV_ID_REGEX = re.compile("^(\d{4}.\d{4,5}|[a-z\-]+(\.[A-Z]{2})?/\d{7})(v\d+)?$")
+ARXIV_ID_REGEX = re.compile(r"^(\d{4}.\d{4,5}|[a-z\-]+(\.[A-Z]{2})?/\d{7})(v\d+)?$")
 
 def clean_arxiv_id(raw):
     """
@@ -175,7 +175,7 @@ def test_clean_sha256():
     assert clean_sha256("6cc853f2ae75696b2e45f476c76b946b0fc2df7c52bb38287cb074aceb77bc7f") == "6cc853f2ae75696b2e45f476c76b946b0fc2df7c52bb38287cb074aceb77bc7f"
     assert clean_sha256("0fba3fba0e1937aa0297de3836b768b5dfb23d7b") == None
 
-ISSN_REGEX = re.compile("^\d{4}-\d{3}[0-9X]$")
+ISSN_REGEX = re.compile(r"^\d{4}-\d{3}[0-9X]$")
 
 def clean_issn(raw):
     if not raw:
@@ -193,7 +193,7 @@ def test_clean_issn():
     assert clean_issn("134-4567") == None
     assert clean_issn("123X-4567") == None
 
-ISBN13_REGEX = re.compile("^97(?:8|9)-\d{1,5}-\d{1,7}-\d{1,6}-\d$")
+ISBN13_REGEX = re.compile(r"^97(?:8|9)-\d{1,5}-\d{1,7}-\d{1,6}-\d$")
 
 def clean_isbn13(raw):
     if not raw:
@@ -209,7 +209,7 @@ def test_clean_isbn13():
     assert clean_isbn13("978-1-56619-909-4 ") == "978-1-56619-909-4"
     assert clean_isbn13("9781566199094") == None
 
-ORCID_REGEX = re.compile("^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$")
+ORCID_REGEX = re.compile(r"^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$")
 
 def clean_orcid(raw):
     if not raw:
diff --git a/python/fatcat_web/forms.py b/python/fatcat_web/forms.py
index dd322cff..377e35aa 100644
--- a/python/fatcat_web/forms.py
+++ b/python/fatcat_web/forms.py
@@ -91,7 +91,7 @@ class ReleaseEntityForm(EntityEditForm):
         [validators.Optional(True)])
     #release_year
     doi = StringField('DOI',
-        [validators.Regexp('^10\..*\/.*', message="DOI must be valid"),
+        [validators.Regexp(r'^10\..*\/.*', message="DOI must be valid"),
          validators.Optional(True)])
     wikidata_qid = StringField('Wikidata QID')
     isbn13 = StringField('ISBN-13')
diff --git a/python/pytest.ini b/python/pytest.ini
index 069102b1..ba1ac946 100644
--- a/python/pytest.ini
+++ b/python/pytest.ini
@@ -12,5 +12,7 @@ filterwarnings =
     ignore:.*common_exception_handling.*StopIteration:PendingDeprecationWarning
     ignore:passing extensions and flags as constants is deprecated:DeprecationWarning
     ignore:.*authlib.specs.*:authlib.deprecate.AuthlibDeprecationWarning
+    ignore:Using or importing the ABCs.*:DeprecationWarning
+    ignore:direct construction of PyLintItem.*:_pytest.warning_types.PytestDeprecationWarning
 
 log_level = INFO