diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-03-23 10:32:47 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-03-23 10:32:50 -0700 |
| commit | 84eeefbd3c55ea31bcf552f9c129c0e1576717ae (patch) | |
| tree | a66a381fc9535ff3ed6b3760c573b4476e3ab043 /python/sandcrawler/ingest.py | |
| parent | e5ad7bddbcb55471b96ce30397ed85fe98e3b098 (diff) | |
| download | sandcrawler-84eeefbd3c55ea31bcf552f9c129c0e1576717ae.tar.gz sandcrawler-84eeefbd3c55ea31bcf552f9c129c0e1576717ae.zip | |
ingest: clean_url() in more places
Some 'cdx-error' results were due to URLs with ':' after the hostname or
trailing newline ("\n") characters in the URL. This attempts to work
around this categroy of error.
Diffstat (limited to 'python/sandcrawler/ingest.py')
| -rw-r--r-- | python/sandcrawler/ingest.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index c9a697c..4159e26 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -354,6 +354,7 @@ class IngestFileWorker(SandcrawlerWorker): return result next_url = fulltext_url.get('pdf_url') or fulltext_url.get('next_url') assert next_url + next_url = clean_url(next_url) print("[PARSE\t] {}\t{}".format( fulltext_url.get('technique'), next_url, |