From fbbd7f2dc2d4a82d3da9327a7297f47103196544 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 22 Feb 2019 11:48:41 -0800 Subject: bunch of lint/whitespace cleanups --- python/fatcat_tools/api_auth.py | 1 - python/fatcat_tools/harvest/doi_registrars.py | 3 +-- python/fatcat_tools/harvest/harvest_common.py | 1 - python/fatcat_tools/harvest/oaipmh.py | 11 +++++------ python/fatcat_tools/importers/common.py | 4 ++-- python/fatcat_tools/importers/grobid_metadata.py | 1 - python/fatcat_tools/importers/journal_metadata.py | 3 +-- python/fatcat_tools/workers/changelog.py | 3 +-- python/fatcat_tools/workers/worker_common.py | 4 ++-- 9 files changed, 12 insertions(+), 19 deletions(-) (limited to 'python/fatcat_tools') diff --git a/python/fatcat_tools/api_auth.py b/python/fatcat_tools/api_auth.py index c49051f6..3c643cea 100644 --- a/python/fatcat_tools/api_auth.py +++ b/python/fatcat_tools/api_auth.py @@ -37,4 +37,3 @@ def authenticated_api(host_uri, token=None): api.auth_check() return api - diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py index 4a0cb8db..55d85ef9 100644 --- a/python/fatcat_tools/harvest/doi_registrars.py +++ b/python/fatcat_tools/harvest/doi_registrars.py @@ -4,9 +4,9 @@ import sys import csv import json import time -import requests import itertools import datetime +import requests from pykafka import KafkaClient from fatcat_tools.workers import most_recent_message @@ -182,4 +182,3 @@ class HarvestDataciteWorker(HarvestCrossrefWorker): def update_params(self, params, resp): params['page[number]'] = resp['meta']['page'] + 1 return params - diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py index 6041a36f..f4d74be2 100644 --- a/python/fatcat_tools/harvest/harvest_common.py +++ b/python/fatcat_tools/harvest/harvest_common.py @@ -121,4 +121,3 @@ class HarvestState: self.update(msg.value.decode('utf-8')) c += 1 print("... got {} state update messages, done".format(c)) - diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py index abd917e0..923500fc 100644 --- a/python/fatcat_tools/harvest/oaipmh.py +++ b/python/fatcat_tools/harvest/oaipmh.py @@ -4,9 +4,9 @@ import sys import csv import json import time -import requests import itertools import datetime +import requests from pykafka import KafkaClient import sickle @@ -102,7 +102,7 @@ class HarvestArxivWorker(HarvestOaiPmhWorker): """ def __init__(self, **kwargs): - super().__init__(**kwargs) + super().__init__(**kwargs) self.endpoint_url = "https://export.arxiv.org/oai2" self.metadata_prefix = "arXiv" self.name = "arxiv" @@ -117,7 +117,7 @@ class HarvestPubmedWorker(HarvestOaiPmhWorker): """ def __init__(self, **kwargs): - super().__init__(**kwargs) + super().__init__(**kwargs) self.endpoint_url = "https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi" self.metadata_prefix = "pmc_fm" self.name = "pubmed" @@ -132,7 +132,7 @@ class HarvestDoajJournalWorker(HarvestOaiPmhWorker): """ def __init__(self, **kwargs): - super().__init__(**kwargs) + super().__init__(**kwargs) self.endpoint_url = "https://www.doaj.org/oai" self.metadata_prefix = "oai_dc" self.name = "doaj-journal" @@ -144,8 +144,7 @@ class HarvestDoajArticleWorker(HarvestOaiPmhWorker): """ def __init__(self, **kwargs): - super().__init__(**kwargs) + super().__init__(**kwargs) self.endpoint_url = "https://www.doaj.org/oai.article" self.metadata_prefix = "oai_doaj" self.name = "doaj-article" - diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py index 32f7b4d5..b89c3828 100644 --- a/python/fatcat_tools/importers/common.py +++ b/python/fatcat_tools/importers/common.py @@ -111,7 +111,7 @@ def test_make_rel_url(): class EntityImporter: """ Base class for fatcat entity importers. - + The API exposed to record iterator is: push_record(raw_record) @@ -143,7 +143,7 @@ class EntityImporter: eg_extra['git_rev'] = eg_extra.get('git_rev', subprocess.check_output(["git", "describe", "--always"]).strip()).decode('utf-8') eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.EntityImporter') - + self.api = api self.bezerk_mode = kwargs.get('bezerk_mode', False) self.edit_batch_size = kwargs.get('edit_batch_size', 100) diff --git a/python/fatcat_tools/importers/grobid_metadata.py b/python/fatcat_tools/importers/grobid_metadata.py index d693bc0a..7281a7a1 100644 --- a/python/fatcat_tools/importers/grobid_metadata.py +++ b/python/fatcat_tools/importers/grobid_metadata.py @@ -194,4 +194,3 @@ class GrobidMetadataImporter(EntityImporter): autoaccept=True, description=self.editgroup_description, extra=json.dumps(self.editgroup_extra)) - diff --git a/python/fatcat_tools/importers/journal_metadata.py b/python/fatcat_tools/importers/journal_metadata.py index 75ee34d8..9bba2b12 100644 --- a/python/fatcat_tools/importers/journal_metadata.py +++ b/python/fatcat_tools/importers/journal_metadata.py @@ -118,7 +118,7 @@ class JournalMetadataImporter(EntityImporter): if existing: self.counts['exists'] += 1 return False - + return True def insert_batch(self, batch): @@ -126,4 +126,3 @@ class JournalMetadataImporter(EntityImporter): autoaccept=True, description=self.editgroup_description, extra=json.dumps(self.editgroup_extra)) - diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py index 636ed304..39a84f18 100644 --- a/python/fatcat_tools/workers/changelog.py +++ b/python/fatcat_tools/workers/changelog.py @@ -34,7 +34,7 @@ class ChangelogWorker(FatcatWorker): self.offset = 1 with topic.get_producer() as producer: - while True: + while True: latest = int(self.api.get_changelog(limit=1)[0].index) if latest > self.offset: print("Fetching changelogs from {} through {}".format( @@ -101,4 +101,3 @@ class EntityUpdatesWorker(FatcatWorker): timestamp=None, ) #consumer.commit_offsets() - diff --git a/python/fatcat_tools/workers/worker_common.py b/python/fatcat_tools/workers/worker_common.py index b84341c7..57fb710c 100644 --- a/python/fatcat_tools/workers/worker_common.py +++ b/python/fatcat_tools/workers/worker_common.py @@ -4,10 +4,11 @@ import sys import csv import json import itertools -import fatcat_client from itertools import islice from pykafka import KafkaClient from pykafka.common import OffsetType + +import fatcat_client from fatcat_client.rest import ApiException @@ -51,4 +52,3 @@ class FatcatWorker: self.kafka = KafkaClient(hosts=kafka_hosts, broker_version="1.0.0") self.produce_topic = produce_topic self.consume_topic = consume_topic - -- cgit v1.2.3