aboutsummaryrefslogtreecommitdiffstats
path: root/python/fatcat_tools
diff options
context:
space:
mode:
Diffstat (limited to 'python/fatcat_tools')
-rw-r--r--python/fatcat_tools/api_auth.py1
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py3
-rw-r--r--python/fatcat_tools/harvest/harvest_common.py1
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py11
-rw-r--r--python/fatcat_tools/importers/common.py4
-rw-r--r--python/fatcat_tools/importers/grobid_metadata.py1
-rw-r--r--python/fatcat_tools/importers/journal_metadata.py3
-rw-r--r--python/fatcat_tools/workers/changelog.py3
-rw-r--r--python/fatcat_tools/workers/worker_common.py4
9 files changed, 12 insertions, 19 deletions
diff --git a/python/fatcat_tools/api_auth.py b/python/fatcat_tools/api_auth.py
index c49051f6..3c643cea 100644
--- a/python/fatcat_tools/api_auth.py
+++ b/python/fatcat_tools/api_auth.py
@@ -37,4 +37,3 @@ def authenticated_api(host_uri, token=None):
api.auth_check()
return api
-
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index 4a0cb8db..55d85ef9 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -4,9 +4,9 @@ import sys
import csv
import json
import time
-import requests
import itertools
import datetime
+import requests
from pykafka import KafkaClient
from fatcat_tools.workers import most_recent_message
@@ -182,4 +182,3 @@ class HarvestDataciteWorker(HarvestCrossrefWorker):
def update_params(self, params, resp):
params['page[number]'] = resp['meta']['page'] + 1
return params
-
diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py
index 6041a36f..f4d74be2 100644
--- a/python/fatcat_tools/harvest/harvest_common.py
+++ b/python/fatcat_tools/harvest/harvest_common.py
@@ -121,4 +121,3 @@ class HarvestState:
self.update(msg.value.decode('utf-8'))
c += 1
print("... got {} state update messages, done".format(c))
-
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index abd917e0..923500fc 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -4,9 +4,9 @@ import sys
import csv
import json
import time
-import requests
import itertools
import datetime
+import requests
from pykafka import KafkaClient
import sickle
@@ -102,7 +102,7 @@ class HarvestArxivWorker(HarvestOaiPmhWorker):
"""
def __init__(self, **kwargs):
- super().__init__(**kwargs)
+ super().__init__(**kwargs)
self.endpoint_url = "https://export.arxiv.org/oai2"
self.metadata_prefix = "arXiv"
self.name = "arxiv"
@@ -117,7 +117,7 @@ class HarvestPubmedWorker(HarvestOaiPmhWorker):
"""
def __init__(self, **kwargs):
- super().__init__(**kwargs)
+ super().__init__(**kwargs)
self.endpoint_url = "https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi"
self.metadata_prefix = "pmc_fm"
self.name = "pubmed"
@@ -132,7 +132,7 @@ class HarvestDoajJournalWorker(HarvestOaiPmhWorker):
"""
def __init__(self, **kwargs):
- super().__init__(**kwargs)
+ super().__init__(**kwargs)
self.endpoint_url = "https://www.doaj.org/oai"
self.metadata_prefix = "oai_dc"
self.name = "doaj-journal"
@@ -144,8 +144,7 @@ class HarvestDoajArticleWorker(HarvestOaiPmhWorker):
"""
def __init__(self, **kwargs):
- super().__init__(**kwargs)
+ super().__init__(**kwargs)
self.endpoint_url = "https://www.doaj.org/oai.article"
self.metadata_prefix = "oai_doaj"
self.name = "doaj-article"
-
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 32f7b4d5..b89c3828 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -111,7 +111,7 @@ def test_make_rel_url():
class EntityImporter:
"""
Base class for fatcat entity importers.
-
+
The API exposed to record iterator is:
push_record(raw_record)
@@ -143,7 +143,7 @@ class EntityImporter:
eg_extra['git_rev'] = eg_extra.get('git_rev',
subprocess.check_output(["git", "describe", "--always"]).strip()).decode('utf-8')
eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.EntityImporter')
-
+
self.api = api
self.bezerk_mode = kwargs.get('bezerk_mode', False)
self.edit_batch_size = kwargs.get('edit_batch_size', 100)
diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py
index d693bc0a..7281a7a1 100644
--- a/python/fatcat_tools/importers/grobid_metadata.py
+++ b/python/fatcat_tools/importers/grobid_metadata.py
@@ -194,4 +194,3 @@ class GrobidMetadataImporter(EntityImporter):
autoaccept=True,
description=self.editgroup_description,
extra=json.dumps(self.editgroup_extra))
-
diff --git a/python/fatcat_tools/importers/journal_metadata.py b/python/fatcat_tools/importers/journal_metadata.py
index 75ee34d8..9bba2b12 100644
--- a/python/fatcat_tools/importers/journal_metadata.py
+++ b/python/fatcat_tools/importers/journal_metadata.py
@@ -118,7 +118,7 @@ class JournalMetadataImporter(EntityImporter):
if existing:
self.counts['exists'] += 1
return False
-
+
return True
def insert_batch(self, batch):
@@ -126,4 +126,3 @@ class JournalMetadataImporter(EntityImporter):
autoaccept=True,
description=self.editgroup_description,
extra=json.dumps(self.editgroup_extra))
-
diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py
index 636ed304..39a84f18 100644
--- a/python/fatcat_tools/workers/changelog.py
+++ b/python/fatcat_tools/workers/changelog.py
@@ -34,7 +34,7 @@ class ChangelogWorker(FatcatWorker):
self.offset = 1
with topic.get_producer() as producer:
- while True:
+ while True:
latest = int(self.api.get_changelog(limit=1)[0].index)
if latest > self.offset:
print("Fetching changelogs from {} through {}".format(
@@ -101,4 +101,3 @@ class EntityUpdatesWorker(FatcatWorker):
timestamp=None,
)
#consumer.commit_offsets()
-
diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py
index b84341c7..57fb710c 100644
--- a/python/fatcat_tools/workers/worker_common.py
+++ b/python/fatcat_tools/workers/worker_common.py
@@ -4,10 +4,11 @@ import sys
import csv
import json
import itertools
-import fatcat_client
from itertools import islice
from pykafka import KafkaClient
from pykafka.common import OffsetType
+
+import fatcat_client
from fatcat_client.rest import ApiException
@@ -51,4 +52,3 @@ class FatcatWorker:
self.kafka = KafkaClient(hosts=kafka_hosts, broker_version="1.0.0")
self.produce_topic = produce_topic
self.consume_topic = consume_topic
-