aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Czygan <martin@archive.org>2020-05-27 23:25:00 +0000
committerMartin Czygan <martin@archive.org>2020-05-27 23:25:00 +0000
commit174cf39a02b85c69ada9bea609be2fc06c172e68 (patch)
tree46fdb09a5cfe629b2581eaf195fd93ec87b399a3
parentd7ebedc840d8fe27fe0b952986ec9d9161964123 (diff)
parent670aed3800873869550b477846f48cb2b4193005 (diff)
downloadfatcat-174cf39a02b85c69ada9bea609be2fc06c172e68.tar.gz
fatcat-174cf39a02b85c69ada9bea609be2fc06c172e68.zip
Merge branch 'bnewbold-harvest-state-next-span' into 'master'
rename HarvestState.next() to HarvestState.next_span() See merge request webgroup/fatcat!53
-rw-r--r--python/fatcat_tools/harvest/doi_registrars.py2
-rw-r--r--python/fatcat_tools/harvest/harvest_common.py4
-rw-r--r--python/fatcat_tools/harvest/oaipmh.py2
-rw-r--r--python/fatcat_tools/harvest/pubmed.py2
-rw-r--r--python/tests/harvest_state.py4
5 files changed, 7 insertions, 7 deletions
diff --git a/python/fatcat_tools/harvest/doi_registrars.py b/python/fatcat_tools/harvest/doi_registrars.py
index 4e027738..f84acb24 100644
--- a/python/fatcat_tools/harvest/doi_registrars.py
+++ b/python/fatcat_tools/harvest/doi_registrars.py
@@ -174,7 +174,7 @@ class HarvestCrossrefWorker:
def run(self, continuous=False):
while True:
- current = self.state.next(continuous) # pylint: disable=not-callable
+ current = self.state.next_span(continuous)
if current:
print("Fetching DOIs updated on {} (UTC)".format(current), file=sys.stderr)
self.fetch_date(current)
diff --git a/python/fatcat_tools/harvest/harvest_common.py b/python/fatcat_tools/harvest/harvest_common.py
index 5f7aa084..27ab8b4a 100644
--- a/python/fatcat_tools/harvest/harvest_common.py
+++ b/python/fatcat_tools/harvest/harvest_common.py
@@ -83,12 +83,12 @@ class HarvestState:
self.to_process.add(current)
current += datetime.timedelta(days=1)
- def next(self, continuous=False):
+ def next_span(self, continuous=False):
"""
Gets next timespan (date) to be processed, or returns None if completed.
If 'continuous' arg is True, will try to enqueue recent possibly valid
- timespans; the idea is to call next() repeatedly, and it will return a
+ timespans; the idea is to call next_span() repeatedly, and it will return a
new timespan when it becomes "available".
"""
if continuous:
diff --git a/python/fatcat_tools/harvest/oaipmh.py b/python/fatcat_tools/harvest/oaipmh.py
index af1ca0d5..d30f9507 100644
--- a/python/fatcat_tools/harvest/oaipmh.py
+++ b/python/fatcat_tools/harvest/oaipmh.py
@@ -98,7 +98,7 @@ class HarvestOaiPmhWorker:
def run(self, continuous=False):
while True:
- current = self.state.next(continuous) # pylint: disable=not-callable
+ current = self.state.next_span(continuous)
if current:
print("Fetching DOIs updated on {} (UTC)".format(current), file=sys.stderr)
self.fetch_date(current)
diff --git a/python/fatcat_tools/harvest/pubmed.py b/python/fatcat_tools/harvest/pubmed.py
index d78045c6..f6301b8d 100644
--- a/python/fatcat_tools/harvest/pubmed.py
+++ b/python/fatcat_tools/harvest/pubmed.py
@@ -144,7 +144,7 @@ class PubmedFTPWorker:
if len(self.date_file_map) == 0:
raise ValueError("map from dates to files should not be empty, maybe the HTML changed?")
- current = self.state.next(continuous) # pylint: disable=not-callable
+ current = self.state.next_span(continuous)
if current:
print("Fetching citations updated on {} (UTC)".format(current), file=sys.stderr)
self.fetch_date(current)
diff --git a/python/tests/harvest_state.py b/python/tests/harvest_state.py
index 4273830f..8b7deba6 100644
--- a/python/tests/harvest_state.py
+++ b/python/tests/harvest_state.py
@@ -16,7 +16,7 @@ def test_harvest_state():
for d in list(hs.to_process):
hs.complete(d)
- assert hs.next() is None # pylint: disable=not-callable
+ assert hs.next_span() is None
hs = HarvestState(
start_date=datetime.date(2000,1,1),
@@ -30,7 +30,7 @@ def test_harvest_state():
assert len(hs.to_process) == 5
hs = HarvestState(catchup_days=0)
- assert hs.next() is None # pylint: disable=not-callable
+ assert hs.next_span() is None
hs.enqueue_period(
start_date=datetime.date(2000,1,1),
end_date=datetime.date(2000,1,3),