From 689f495ef5c2317a3db356bca758d36073da4622 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 27 Oct 2020 15:52:04 -0700 Subject: pylint: fix error on selectolax import --- python/.pylintrc | 1 + 1 file changed, 1 insertion(+) (limited to 'python/.pylintrc') diff --git a/python/.pylintrc b/python/.pylintrc index 80e203d..91fea7c 100644 --- a/python/.pylintrc +++ b/python/.pylintrc @@ -12,3 +12,4 @@ notes=FIXME,XXX,DELETEME [TYPECHECK] ignored-modules=responses +extension-pkg-whitelist=selectolax -- cgit v1.2.3 From ae37c8e32f1289816b69cd5a502a6bc5fd862414 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 29 Oct 2020 15:21:57 -0700 Subject: improve test running and config --- python/.pylintrc | 3 +-- python/Makefile | 5 ++++- python/tests/test_pdfextract.py | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'python/.pylintrc') diff --git a/python/.pylintrc b/python/.pylintrc index 91fea7c..387bca1 100644 --- a/python/.pylintrc +++ b/python/.pylintrc @@ -11,5 +11,4 @@ include-ids=yes notes=FIXME,XXX,DELETEME [TYPECHECK] -ignored-modules=responses -extension-pkg-whitelist=selectolax +extension-pkg-whitelist=selectolax,pydantic,responses diff --git a/python/Makefile b/python/Makefile index f783d0e..0a97437 100644 --- a/python/Makefile +++ b/python/Makefile @@ -17,6 +17,7 @@ lint: ## Run lints (eg, flake8, mypy) #pipenv run flake8 . --exit-zero pipenv run flake8 . --select=E9,F63,F7,F82 --exit-zero pipenv run mypy *.py sandcrawler/ tests/ --ignore-missing-imports + pipenv run pylint --rcfile=.pylintrc -E --jobs=4 sandcrawler tests *.py #pipenv run pytype sandcrawler/ .PHONY: fmt @@ -24,7 +25,9 @@ fmt: ## Run code formating on all source code pipenv run black *.py sandcrawler/ tests/ .PHONY: test -test: lint ## Run all tests and lints +test: ## Run all tests and lints + pipenv run flake8 . --select=E9,F63,F7,F82 --exit-zero + pipenv run mypy *.py sandcrawler/ tests/ --ignore-missing-imports pipenv run pytest .PHONY: coverage diff --git a/python/tests/test_pdfextract.py b/python/tests/test_pdfextract.py index ed93341..64e3137 100644 --- a/python/tests/test_pdfextract.py +++ b/python/tests/test_pdfextract.py @@ -2,6 +2,7 @@ import pytest import struct import responses +import poppler from sandcrawler import PdfExtractWorker, PdfExtractBlobWorker, CdxLinePusher, BlackholeSink, WaybackClient from sandcrawler.pdfextract import process_pdf @@ -20,6 +21,7 @@ def test_process_fake_pdf(): resp = process_pdf(pdf_bytes) assert resp.status == 'not-pdf' +@pytest.mark.skipif(poppler.__version__ == '0.2.1', reason="unsupported version of poppler") def test_process_dummy_pdf(): with open('tests/files/dummy.pdf', 'rb') as f: pdf_bytes = f.read() -- cgit v1.2.3