diff options
30 files changed, 86 insertions, 149 deletions
diff --git a/python/grobid_tool.py b/python/grobid_tool.py index 4ba9540..c36fe0a 100755 --- a/python/grobid_tool.py +++ b/python/grobid_tool.py @@ -9,7 +9,6 @@ Example of large parallel run, locally: """ import argparse -import datetime import json import sys diff --git a/python/ingest_tool.py b/python/ingest_tool.py index 305c3a8..eb1047d 100755 --- a/python/ingest_tool.py +++ b/python/ingest_tool.py @@ -42,8 +42,8 @@ def run_requests(args): html_quick_mode=args.html_quick_mode, ) fileset_worker = IngestFilesetWorker(try_spn2=not args.no_spn2, ) - for l in args.json_file: - request = json.loads(l.strip()) + for line in args.json_file: + request = json.loads(line.strip()) if request['ingest_type'] in [ 'dataset', ]: diff --git a/python/pdfextract_tool.py b/python/pdfextract_tool.py index 717b743..f3df6e3 100755 --- a/python/pdfextract_tool.py +++ b/python/pdfextract_tool.py @@ -4,11 +4,8 @@ KNOWN ISSUE: thumbnails are not published to kafka in multi-processing mode """ import argparse -import datetime -import json import sys -from grobid2json import teixml2json from sandcrawler import * diff --git a/python/pdftrio_tool.py b/python/pdftrio_tool.py index 9316313..dbe5b10 100755 --- a/python/pdftrio_tool.py +++ b/python/pdftrio_tool.py @@ -9,8 +9,6 @@ cat /srv/sandcrawler/tasks/something.cdx | pv -l | parallel -j30 --pipe ./pdftri """ import argparse -import datetime -import json import sys from sandcrawler import * diff --git a/python/sandcrawler/fileset_platforms.py b/python/sandcrawler/fileset_platforms.py index f3441c9..2811100 100644 --- a/python/sandcrawler/fileset_platforms.py +++ b/python/sandcrawler/fileset_platforms.py @@ -1,10 +1,5 @@ -import gzip -import json -import sys -import time import urllib.parse -from collections import namedtuple -from typing import Any, Dict, List, Optional, Tuple +from typing import Optional, Tuple import internetarchive import requests @@ -175,12 +170,12 @@ class DataverseHelper(FilesetPlatformHelper): try: parsed_id = self.parse_dataverse_persistentid(platform_id) except ValueError: - raise PlatformScopeError(f"not actually in scope") + raise PlatformScopeError("not actually in scope") if parsed_id['file_id']: # XXX: maybe we could support this? raise PlatformScopeError( - f"only entire dataverse datasets can be archived with this tool") + "only entire dataverse datasets can be archived with this tool") # 1b. if we didn't get a version number from URL, fetch it from API if not dataset_version: @@ -277,13 +272,6 @@ def test_parse_dataverse_persistentid(): "dataset_id": "LL6WXZ", "file_id": None, }, - "doi:10.25625/LL6WXZ": { - "type": "doi", - "authority": "10.25625", - "shoulder": None, - "dataset_id": "LL6WXZ", - "file_id": None, - }, "doi:10.5072/FK2/J8SJZB": { "type": "doi", "authority": "10.5072", @@ -423,7 +411,7 @@ class FigshareHelper(FilesetPlatformHelper): resp.raise_for_status() obj = resp.json() - figshare_type = obj['defined_type_name'] + _figshare_type = obj['defined_type_name'] if not obj['is_public']: raise PlatformRestrictedError(f'record not public: {platform_id} {dataset_version}') diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index 6c25276..4e44d97 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -1,19 +1,13 @@ -import gzip -import json import os import shutil import sys -import time -from collections import namedtuple -from typing import Any, Dict, List, Optional, Tuple +from typing import Optional import internetarchive -from sandcrawler.fileset_types import (ArchiveStrategyResult, FilesetManifestFile, - FilesetPlatformItem, IngestStrategy, PlatformScopeError) -from sandcrawler.html_metadata import BiblioMetadata -from sandcrawler.ia import (ResourceResult, SavePageNowClient, WaybackClient, - fix_transfer_encoding) +from sandcrawler.fileset_types import (ArchiveStrategyResult, FilesetPlatformItem, + IngestStrategy, PlatformScopeError) +from sandcrawler.ia import SavePageNowClient, WaybackClient, fix_transfer_encoding from sandcrawler.misc import gen_file_metadata, gen_file_metadata_path, sanitize_fs_path @@ -233,7 +227,7 @@ class WebFilesetStrategy(FilesetIngestStrategy): via = "wayback" resource = self.wayback_client.lookup_resource(fetch_url, m.mimetype) - if self.try_spn2 and (resource == None or + if self.try_spn2 and (resource is None or (resource and resource.status == 'no-capture')): if len(item.manifest) > self.max_spn_manifest: m.status = 'too-much-spn' diff --git a/python/sandcrawler/fileset_types.py b/python/sandcrawler/fileset_types.py index 606af07..f543ede 100644 --- a/python/sandcrawler/fileset_types.py +++ b/python/sandcrawler/fileset_types.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from pydantic import BaseModel diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py index 16bbb01..d0b7f7e 100644 --- a/python/sandcrawler/grobid.py +++ b/python/sandcrawler/grobid.py @@ -23,7 +23,7 @@ class GrobidClient(object): """ assert blob - if consolidate_mode == None: + if consolidate_mode is None: consolidate_mode = self.consolidate_mode try: @@ -100,8 +100,6 @@ class GrobidWorker(SandcrawlerFetchWorker): ) def process(self, record, key=None): - default_key = record['sha1hex'] - fetch_result = self.fetch_blob(record) if fetch_result['status'] != 'success': return fetch_result diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py index a44fc67..5b9742a 100644 --- a/python/sandcrawler/html.py +++ b/python/sandcrawler/html.py @@ -53,12 +53,12 @@ def extract_fulltext_url(html_url, html_body): print(f"\tdoi.org in citation_pdf_url (loop?): {url}", file=sys.stderr) elif url.startswith('/'): if host_prefix + url == html_url: - print(f"\tavoiding citation_pdf_url link-loop", file=sys.stderr) + print("\tavoiding citation_pdf_url link-loop", file=sys.stderr) else: return dict(pdf_url=host_prefix + url, technique='citation_pdf_url') elif url.startswith('http'): if url == html_url: - print(f"\tavoiding citation_pdf_url link-loop", file=sys.stderr) + print("\tavoiding citation_pdf_url link-loop", file=sys.stderr) else: return dict(pdf_url=url, technique='citation_pdf_url') else: diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py index 6d27a3a..15a9f2b 100644 --- a/python/sandcrawler/html_metadata.py +++ b/python/sandcrawler/html_metadata.py @@ -1,7 +1,7 @@ import datetime import sys import urllib.parse -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, List, Optional, Tuple import braveblock import dateparser @@ -687,7 +687,7 @@ def html_extract_fulltext_url(doc_url: str, doc: HTMLParser, continue return (val, pattern.get('technique', 'unknown')) if self_doc_url: - print(f" WARN: returning fulltext URL pointing to self", file=sys.stderr) + print(" WARN: returning fulltext URL pointing to self", file=sys.stderr) return self_doc_url return None @@ -864,7 +864,7 @@ def html_extract_resources(doc_url: str, doc: HTMLParser, # filter using adblocker resources = [ r for r in resources if adblock.check_network_urls( - r['url'], source_url=doc_url, request_type=r['type']) == False + r['url'], source_url=doc_url, request_type=r['type']) is False ] # remove duplicates diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index a8ce193..fe739bb 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -11,15 +11,14 @@ import sys import time import urllib.parse from collections import namedtuple +from http.client import IncompleteRead from typing import Tuple import requests import urllib3.exceptions # not sure this will really work. Should go before wayback imports. -http.client._MAXHEADERS = 1000 # type: ignore - -from http.client import IncompleteRead +http.client._MAXHEADERS = 1000 # noqa import wayback.exception from gwb.loader import CDXLoaderFactory3 @@ -128,18 +127,18 @@ def fuzzy_match_url(left, right): def test_fuzzy_match_url(): - assert fuzzy_match_url("http://thing.com", "http://thing.com") == True - assert fuzzy_match_url("http://thing.com", "https://thing.com") == True - assert fuzzy_match_url("http://thing.com", "ftp://thing.com") == True - assert fuzzy_match_url("http://thing.com", "http://thing.com/") == True - assert fuzzy_match_url("https://thing.com", "http://thing.com/") == True - assert fuzzy_match_url("https://thing.com/", "http://thing.com") == True - assert fuzzy_match_url("http://thing.com", "http://thing.com/blue") == False + assert fuzzy_match_url("http://thing.com", "http://thing.com") is True + assert fuzzy_match_url("http://thing.com", "https://thing.com") is True + assert fuzzy_match_url("http://thing.com", "ftp://thing.com") is True + assert fuzzy_match_url("http://thing.com", "http://thing.com/") is True + assert fuzzy_match_url("https://thing.com", "http://thing.com/") is True + assert fuzzy_match_url("https://thing.com/", "http://thing.com") is True + assert fuzzy_match_url("http://thing.com", "http://thing.com/blue") is False # should probably handle these? - assert fuzzy_match_url("http://thing.com", "http://www.thing.com") == False - assert fuzzy_match_url("http://www.thing.com", "http://www2.thing.com") == False - assert fuzzy_match_url("http://www.thing.com", "https://www2.thing.com") == False + assert fuzzy_match_url("http://thing.com", "http://www.thing.com") is False + assert fuzzy_match_url("http://www.thing.com", "http://www2.thing.com") is False + assert fuzzy_match_url("http://www.thing.com", "https://www2.thing.com") is False class CdxApiError(Exception): @@ -951,7 +950,7 @@ class SavePageNowClient: resp = self.v2_session.get("{}/status/{}".format(self.v2endpoint, job_id)) try: resp.raise_for_status() - except: + except Exception: raise SavePageNowError(resp.content) status = resp.json()['status'] if status == 'pending': @@ -975,7 +974,7 @@ class SavePageNowClient: final_json['original_job_id'])) try: resp.raise_for_status() - except: + except Exception: raise SavePageNowError(resp.content) final_json = resp.json() diff --git a/python/sandcrawler/ingest_file.py b/python/sandcrawler/ingest_file.py index b480cc2..556e573 100644 --- a/python/sandcrawler/ingest_file.py +++ b/python/sandcrawler/ingest_file.py @@ -1,23 +1,19 @@ -import base64 -import gzip import json import sys import time import xml.etree.ElementTree -from collections import namedtuple -from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import Any, Dict, List, Optional, Tuple +from http.server import BaseHTTPRequestHandler +from typing import Any, Dict, List, Optional -import requests from selectolax.parser import HTMLParser from sandcrawler.db import SandcrawlerPostgrestClient from sandcrawler.grobid import GrobidClient from sandcrawler.html import extract_fulltext_url -from sandcrawler.html_metadata import (BiblioMetadata, html_extract_biblio, - html_extract_resources, load_adblock_rules) -from sandcrawler.ia import (CdxApiClient, CdxApiError, NoCaptureError, PetaboxError, - ResourceResult, SavePageNowClient, SavePageNowError, WaybackClient, +from sandcrawler.html_metadata import (html_extract_biblio, html_extract_resources, + load_adblock_rules) +from sandcrawler.ia import (CdxApiError, NoCaptureError, PetaboxError, ResourceResult, + SavePageNowClient, SavePageNowError, WaybackClient, WaybackContentError, WaybackError, cdx_to_dict, fix_transfer_encoding) from sandcrawler.ingest_html import (WebResource, fetch_html_resources, @@ -211,7 +207,7 @@ class IngestFileWorker(SandcrawlerWorker): return None existing = self.pgrest_client.get_ingest_file_result(ingest_type, base_url) # TODO: filter on more flags? - if existing and existing['hit'] == True: + if existing and existing['hit'] is True: return existing else: return None @@ -249,7 +245,7 @@ class IngestFileWorker(SandcrawlerWorker): if resource and not resource.hit and resource.terminal_dt and resource.terminal_dt < '20190000000000': old_failure = True - if self.try_spn2 and (resource == None or (resource and resource.status == 'no-capture') + if self.try_spn2 and (resource is None or (resource and resource.status == 'no-capture') or soft404 or old_failure): via = "spn2" resource = self.spn_client.crawl_resource(url, self.wayback_client) @@ -751,7 +747,7 @@ class IngestFileWorker(SandcrawlerWorker): # fetch must be a hit if we got this far (though not necessarily an ingest hit!) assert resource - assert resource.hit == True + assert resource.hit is True assert resource.terminal_status_code in (200, 226) if resource.terminal_url: diff --git a/python/sandcrawler/ingest_fileset.py b/python/sandcrawler/ingest_fileset.py index 5cbb908..4376c89 100644 --- a/python/sandcrawler/ingest_fileset.py +++ b/python/sandcrawler/ingest_fileset.py @@ -1,30 +1,19 @@ -import gzip import json import sys import time -from collections import namedtuple -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional import requests from selectolax.parser import HTMLParser -from sandcrawler.db import SandcrawlerPostgrestClient -from sandcrawler.fileset_platforms import DATASET_PLATFORM_HELPER_TABLE, FilesetPlatformHelper -from sandcrawler.fileset_strategies import FILESET_STRATEGY_HELPER_TABLE, FilesetIngestStrategy +from sandcrawler.fileset_platforms import DATASET_PLATFORM_HELPER_TABLE +from sandcrawler.fileset_strategies import FILESET_STRATEGY_HELPER_TABLE from sandcrawler.fileset_types import PlatformRestrictedError, PlatformScopeError -from sandcrawler.html import extract_fulltext_url -from sandcrawler.html_metadata import (BiblioMetadata, html_extract_biblio, - html_extract_resources, load_adblock_rules) -from sandcrawler.ia import (CdxApiClient, CdxApiError, NoCaptureError, PetaboxError, - ResourceResult, SavePageNowClient, SavePageNowError, WaybackClient, - WaybackContentError, WaybackError, cdx_to_dict, - fix_transfer_encoding) +from sandcrawler.html_metadata import html_extract_biblio +from sandcrawler.ia import (CdxApiError, PetaboxError, SavePageNowError, WaybackContentError, + WaybackError, cdx_to_dict, fix_transfer_encoding) from sandcrawler.ingest_file import IngestFileWorker -from sandcrawler.ingest_html import (WebResource, fetch_html_resources, - html_extract_body_teixml, html_guess_platform, - html_guess_scope, quick_fetch_html_resources) -from sandcrawler.misc import clean_url, gen_file_metadata, parse_cdx_datetime -from sandcrawler.workers import SandcrawlerWorker +from sandcrawler.misc import clean_url, gen_file_metadata MAX_BODY_SIZE_BYTES = 128 * 1024 * 1024 @@ -61,7 +50,7 @@ class IngestFilesetWorker(IngestFileWorker): return None existing = self.pgrest_client.get_ingest_fileset_result(ingest_type, base_url) # TODO: filter on more flags? - if existing and existing['hit'] == True: + if existing and existing['hit'] is True: return existing else: return None @@ -196,7 +185,7 @@ class IngestFilesetWorker(IngestFileWorker): # fetch must be a hit if we got this far (though not necessarily an ingest hit!) assert resource - assert resource.hit == True + assert resource.hit is True assert resource.terminal_status_code in (200, 226) if resource.terminal_url: diff --git a/python/sandcrawler/ingest_html.py b/python/sandcrawler/ingest_html.py index bf25d5d..91e5c6e 100644 --- a/python/sandcrawler/ingest_html.py +++ b/python/sandcrawler/ingest_html.py @@ -1,6 +1,5 @@ import argparse import datetime -import io import json import sys import xml.etree.ElementTree as ET @@ -12,9 +11,9 @@ from selectolax.parser import HTMLParser from sandcrawler.html_metadata import (BiblioMetadata, html_extract_biblio, html_extract_resources, load_adblock_rules) -from sandcrawler.ia import (CdxApiClient, NoCaptureError, ResourceResult, WaybackClient, - WaybackContentError, cdx_to_dict, fix_transfer_encoding) -from sandcrawler.misc import (clean_url, datetime_to_cdx, gen_file_metadata, parse_cdx_datetime, +from sandcrawler.ia import (CdxApiClient, NoCaptureError, WaybackClient, WaybackContentError, + cdx_to_dict, fix_transfer_encoding) +from sandcrawler.misc import (datetime_to_cdx, gen_file_metadata, parse_cdx_datetime, url_fuzzy_equal) TRAFILATURA_AGENT = f"trafilatura/{trafilatura.__version__}" @@ -147,7 +146,7 @@ def quick_fetch_html_resources(resources: List[dict], cdx_client: CdxApiClient, file=sys.stderr) if not cdx_row.status_code: # TODO: fall back to a full fetch? - print(f" WARN: skipping revisit record", file=sys.stderr) + print(" WARN: skipping revisit record", file=sys.stderr) continue full.append( WebResource( diff --git a/python/sandcrawler/minio.py b/python/sandcrawler/minio.py index 188621f..046db9e 100644 --- a/python/sandcrawler/minio.py +++ b/python/sandcrawler/minio.py @@ -1,6 +1,5 @@ import hashlib import io -import os import minio diff --git a/python/sandcrawler/misc.py b/python/sandcrawler/misc.py index ddbd95a..5ca7a4b 100644 --- a/python/sandcrawler/misc.py +++ b/python/sandcrawler/misc.py @@ -35,10 +35,10 @@ def url_fuzzy_equal(left: str, right: str) -> bool: def test_url_fuzzy_equal() -> None: - assert True == url_fuzzy_equal( + assert url_fuzzy_equal( "http://www.annalsofian.org/article.asp?issn=0972-2327;year=2014;volume=17;issue=4;spage=463;epage=465;aulast=Nithyashree", "http://annalsofian.org/article.asp?issn=0972-2327;year=2014;volume=17;issue=4;spage=463;epage=465;aulast=Nithyashree" - ) + ) is True def gen_file_metadata(blob: bytes, allow_empty: bool = False) -> dict: @@ -239,8 +239,8 @@ def parse_cdx_datetime(dt_str: str) -> Optional[datetime.datetime]: def test_parse_cdx_datetime() -> None: - assert parse_cdx_datetime("") == None - assert parse_cdx_datetime("asdf") == None + assert parse_cdx_datetime("") is None + assert parse_cdx_datetime("asdf") is None assert parse_cdx_datetime("19930203123045") != None assert parse_cdx_datetime("20201028235103") == datetime.datetime(year=2020, month=10, diff --git a/python/sandcrawler/pdfextract.py b/python/sandcrawler/pdfextract.py index 190672d..9392136 100644 --- a/python/sandcrawler/pdfextract.py +++ b/python/sandcrawler/pdfextract.py @@ -293,7 +293,7 @@ def process_pdf(blob: bytes, thumb_size=(180, 300), thumb_type="JPEG") -> PdfExt return PdfExtractResult( sha1hex=sha1hex, status='bad-pdf', - error_msg=f"PDF known to cause processing issues", + error_msg="PDF known to cause processing issues", file_meta=file_meta, ) diff --git a/python/sandcrawler/pdftrio.py b/python/sandcrawler/pdftrio.py index e3d4a54..ba875cd 100644 --- a/python/sandcrawler/pdftrio.py +++ b/python/sandcrawler/pdftrio.py @@ -75,7 +75,6 @@ class PdfTrioWorker(SandcrawlerFetchWorker): def process(self, record, key=None): start_process = time.time() - default_key = record['sha1hex'] fetch_sec = None start = time.time() diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index 7135f4c..8c604fb 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -116,7 +116,6 @@ class SandcrawlerFetchWorker(SandcrawlerWorker): self.wayback_client = wayback_client def fetch_blob(self, record): - start_process = time.time() default_key = record['sha1hex'] wayback_sec = None petabox_sec = None diff --git a/python/sandcrawler_worker.py b/python/sandcrawler_worker.py index 3c76c17..3e35807 100755 --- a/python/sandcrawler_worker.py +++ b/python/sandcrawler_worker.py @@ -6,7 +6,6 @@ or S3 (SeaweedFS). """ import argparse -import datetime import os import sys @@ -18,7 +17,7 @@ from sandcrawler.persist import PersistHtmlTeiXmlWorker, PersistXmlDocWorker # Yep, a global. Gets DSN from `SENTRY_DSN` environment variable try: git_sha = raven.fetch_git_sha('..') -except Exception as e: +except Exception: git_sha = None sentry_client = raven.Client(release=git_sha) diff --git a/python/tests/test_grobid.py b/python/tests/test_grobid.py index 55636dc..15d43fb 100644 --- a/python/tests/test_grobid.py +++ b/python/tests/test_grobid.py @@ -2,9 +2,9 @@ import struct import pytest import responses -from test_wayback import cdx_client, wayback_client +from test_wayback import cdx_client, wayback_client # noqa:F401 -from sandcrawler import BlackholeSink, CdxLinePusher, GrobidClient, GrobidWorker, WaybackClient +from sandcrawler import BlackholeSink, CdxLinePusher, GrobidClient, GrobidWorker FAKE_PDF_BYTES = b"%PDF SOME JUNK" + struct.pack("!q", 112853843) @@ -58,7 +58,7 @@ def test_grobid_success(grobid_client): @responses.activate -def test_grobid_worker_cdx(grobid_client, wayback_client): +def test_grobid_worker_cdx(grobid_client, wayback_client): # noqa: F811 sink = BlackholeSink() worker = GrobidWorker(grobid_client, wayback_client, sink=sink) diff --git a/python/tests/test_html.py b/python/tests/test_html.py index c5f422e..1caca15 100644 --- a/python/tests/test_html.py +++ b/python/tests/test_html.py @@ -1,8 +1,3 @@ -import json - -import pytest -import responses - from sandcrawler.html import extract_fulltext_url diff --git a/python/tests/test_html_ingest.py b/python/tests/test_html_ingest.py index 3bf94e2..727fef9 100644 --- a/python/tests/test_html_ingest.py +++ b/python/tests/test_html_ingest.py @@ -1,7 +1,3 @@ -import datetime - -import pytest - from sandcrawler.ingest_html import * diff --git a/python/tests/test_ingest.py b/python/tests/test_ingest.py index 79f50f4..f2318c2 100644 --- a/python/tests/test_ingest.py +++ b/python/tests/test_ingest.py @@ -87,7 +87,7 @@ def test_ingest_success(ingest_worker_pdf): resp = ingest_worker_pdf.process(request) print(resp) - assert resp['hit'] == True + assert resp['hit'] is True assert resp['status'] == "success" assert resp['request'] == request assert resp['terminal']['terminal_sha1hex'] == resp['file_meta']['sha1hex'] @@ -156,7 +156,7 @@ def test_ingest_landing(ingest_worker): resp = ingest_worker.process(request) print(resp) - assert resp['hit'] == False + assert resp['hit'] is False assert resp['status'] == "no-pdf-link" assert resp['request'] == request assert 'terminal' in resp @@ -179,7 +179,7 @@ def test_ingest_blocklist(ingest_worker): resp = ingest_worker.process(request) - assert resp['hit'] == False + assert resp['hit'] is False assert resp['status'] == "skip-url-blocklist" assert resp['request'] == request @@ -197,7 +197,7 @@ def test_ingest_wall_blocklist(ingest_worker): resp = ingest_worker.process(request) - assert resp['hit'] == False + assert resp['hit'] is False assert resp['status'] == "skip-wall" assert resp['request'] == request @@ -212,6 +212,6 @@ def test_ingest_cookie_blocklist(ingest_worker): resp = ingest_worker.process(request) - assert resp['hit'] == False + assert resp['hit'] is False assert resp['status'] == "blocked-cookie" assert resp['request'] == request diff --git a/python/tests/test_live_wayback.py b/python/tests/test_live_wayback.py index 0ff4902..bc74916 100644 --- a/python/tests/test_live_wayback.py +++ b/python/tests/test_live_wayback.py @@ -6,12 +6,9 @@ automatically in CI. Simply uncomment lines to run. """ -import json - import pytest -from sandcrawler import (CdxApiClient, CdxApiError, CdxPartial, PetaboxError, SavePageNowClient, - SavePageNowError, WaybackClient, WaybackError, gen_file_metadata) +from sandcrawler import CdxApiClient, SavePageNowClient, WaybackClient, gen_file_metadata @pytest.fixture @@ -89,7 +86,7 @@ def test_lookup_resource_success(wayback_client): url = "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0093949&type=printable" resp = wayback_client.lookup_resource(url) - assert resp.hit == True + assert resp.hit is True assert resp.status == "success" assert resp.terminal_url in (url, url.replace("https://", "http://")) assert resp.cdx.url in (url, url.replace("https://", "http://")) @@ -139,7 +136,7 @@ def test_lookup_ftp(wayback_client): url = "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_pdf/ad/ab/mmr-17-05-6969.PMC5928650.pdf" resp = wayback_client.lookup_resource(url) - assert resp.hit == True + assert resp.hit is True assert resp.status == "success" assert resp.terminal_url == url assert resp.terminal_status_code == 226 @@ -154,7 +151,7 @@ def test_lookup_ftp(wayback_client): url = "ftp://ftp.cs.utexas.edu/pub/qsim/papers/Xu-crv-08.pdf" resp = wayback_client.lookup_resource(url) - assert resp.hit == True + assert resp.hit is True assert resp.status == "success" assert resp.terminal_url == url assert resp.terminal_status_code == 226 @@ -171,10 +168,10 @@ def test_crawl_ftp(spn_client, wayback_client): resp = spn_client.crawl_resource(url, wayback_client) # FTP isn't supported yet! - #assert resp.hit == True + #assert resp.hit is True #assert resp.status == "success" #assert resp.terminal_url == url #assert resp.cdx.url == url - assert resp.hit == False + assert resp.hit is False assert resp.status == "spn2-no-ftp" diff --git a/python/tests/test_misc.py b/python/tests/test_misc.py index dcc1202..7d3e755 100644 --- a/python/tests/test_misc.py +++ b/python/tests/test_misc.py @@ -83,7 +83,7 @@ def test_invalid_cdx(): print("missing warc") raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 20170828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233 -" - assert parse_cdx_line(raw) == None + assert parse_cdx_line(raw) is None print("bad datetime") raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 2070828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233i SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz" diff --git a/python/tests/test_pdfextract.py b/python/tests/test_pdfextract.py index 146b138..086243a 100644 --- a/python/tests/test_pdfextract.py +++ b/python/tests/test_pdfextract.py @@ -2,11 +2,9 @@ import struct import poppler import pytest -import responses -from test_wayback import cdx_client, wayback_client +from test_wayback import cdx_client, wayback_client # noqa:F401 -from sandcrawler import (BlackholeSink, CdxLinePusher, PdfExtractBlobWorker, PdfExtractWorker, - WaybackClient) +from sandcrawler import BlackholeSink, CdxLinePusher, PdfExtractBlobWorker, PdfExtractWorker from sandcrawler.pdfextract import process_pdf FAKE_PDF_BYTES = b"%PDF SOME JUNK" + struct.pack("!q", 112853843) @@ -43,7 +41,7 @@ def test_process_dummy_pdf(): assert resp.pdf_extra['page_count'] == 1 -def test_pdfextract_worker_cdx(wayback_client): +def test_pdfextract_worker_cdx(wayback_client): # noqa: F811 sink = BlackholeSink() worker = PdfExtractWorker(wayback_client, sink=sink, thumbnail_sink=sink) diff --git a/python/tests/test_pushers.py b/python/tests/test_pushers.py index 63f90d3..353a560 100644 --- a/python/tests/test_pushers.py +++ b/python/tests/test_pushers.py @@ -1,5 +1,3 @@ -import pytest - from sandcrawler.workers import BlackholeSink, CdxLinePusher diff --git a/python/tests/test_savepagenow.py b/python/tests/test_savepagenow.py index 80334d9..37f0bc9 100644 --- a/python/tests/test_savepagenow.py +++ b/python/tests/test_savepagenow.py @@ -120,7 +120,7 @@ def test_savepagenow_success(spn_client): assert len(responses.calls) == 4 - assert resp.success == True + assert resp.success is True assert resp.status == "success" assert resp.request_url == TARGET assert resp.terminal_url == TARGET + "/redirect" @@ -151,12 +151,12 @@ def test_savepagenow_remote_error(spn_client): assert len(responses.calls) == 3 - assert resp.success == False + assert resp.success is False assert resp.status == ERROR_BODY['status_ext'] assert resp.request_url == TARGET - assert resp.terminal_url == None - assert resp.terminal_dt == None - assert resp.resources == None + assert resp.terminal_url is None + assert resp.terminal_dt is None + assert resp.resources is None @responses.activate @@ -214,7 +214,7 @@ def test_crawl_resource(spn_client, wayback_client): assert len(responses.calls) == 5 - assert resp.hit == True + assert resp.hit is True assert resp.status == "success" assert resp.body == WARC_BODY assert resp.cdx.sha1b32 == CDX_BEST_SHA1B32 diff --git a/python/tests/test_wayback.py b/python/tests/test_wayback.py index 6ccf775..9861db2 100644 --- a/python/tests/test_wayback.py +++ b/python/tests/test_wayback.py @@ -3,7 +3,7 @@ import json import pytest import responses -from sandcrawler import CdxApiClient, CdxApiError, PetaboxError, WaybackClient, WaybackError +from sandcrawler import CdxApiClient, WaybackClient CDX_TARGET = "http://fatcat.wiki/" CDX_DT = "20180812220054" @@ -215,4 +215,4 @@ def test_lookup_resource_success(wayback_client): resp = wayback_client.lookup_resource(CDX_TARGET) - assert resp.hit == True + assert resp.hit is True |