aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-01-10 17:45:32 -0800
committerBryan Newbold <bnewbold@archive.org>2020-01-10 17:45:32 -0800
commit578c2bd737856c8cddb4589bb3cb87f1bc9d7b3b (patch)
tree7c08420b60c9747dba11f2206d40bd5a4ebfa3b4
parentf725845758c7dfdf88de1037bb58256905db5302 (diff)
downloadsandcrawler-578c2bd737856c8cddb4589bb3cb87f1bc9d7b3b.tar.gz
sandcrawler-578c2bd737856c8cddb4589bb3cb87f1bc9d7b3b.zip
better print() output
-rw-r--r--python/sandcrawler/ia.py6
-rw-r--r--python/sandcrawler/ingest.py2
2 files changed, 4 insertions, 4 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index d91844b..4b4875d 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -151,14 +151,14 @@ class CdxApiClient:
resp = self._query_api(params)
if not resp:
if retry_sleep:
- print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep))
+ print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep), file=sys.stderr)
time.sleep(retry_sleep)
return self.fetch(url, datetime, filter_status_code=filter_status_code, retry_sleep=None)
raise KeyError("CDX url/datetime not found: {} {}".format(url, datetime))
row = resp[0]
if not (row.url == url and row.datetime == datetime):
if retry_sleep:
- print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep))
+ print("CDX fetch failed; will sleep {}sec and try again".format(retry_sleep), file=sys.stderr)
time.sleep(retry_sleep)
return self.fetch(url, datetime, filter_status_code=filter_status_code, retry_sleep=None)
raise KeyError("Didn't get exact CDX url/datetime match. url:{} dt:{} got:{}".format(url, datetime, row))
@@ -731,7 +731,7 @@ class SavePageNowClient:
retry_sleep=10.0,
)
except KeyError as ke:
- print(str(ke), file=sys.stderr)
+ print("CDX KeyError: {}".format(ke), file=sys.stderr)
return ResourceResult(
start_url=start_url,
hit=False,
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py
index 331df11..8c77d65 100644
--- a/python/sandcrawler/ingest.py
+++ b/python/sandcrawler/ingest.py
@@ -214,7 +214,7 @@ class IngestFileWorker(SandcrawlerWorker):
return result
next_url = fulltext_url.get('pdf_url') or fulltext_url.get('next_url')
assert next_url
- print("[EXTRACT\t] {}\t{}".format(
+ print("[PARSE\t] {}\t{}".format(
fulltext_url.get('technique'),
next_url,
),