aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-10-29 15:21:57 -0700
committerBryan Newbold <bnewbold@archive.org>2020-10-29 15:21:57 -0700
commitae37c8e32f1289816b69cd5a502a6bc5fd862414 (patch)
tree8daad96a990c5a625e80587d9a2d6e0fdcc65e1b
parente70f5bcaafe370da12cbfea4133b835f40614b44 (diff)
downloadsandcrawler-ae37c8e32f1289816b69cd5a502a6bc5fd862414.tar.gz
sandcrawler-ae37c8e32f1289816b69cd5a502a6bc5fd862414.zip
improve test running and config
-rw-r--r--python/.pylintrc3
-rw-r--r--python/Makefile5
-rw-r--r--python/tests/test_pdfextract.py2
3 files changed, 7 insertions, 3 deletions
diff --git a/python/.pylintrc b/python/.pylintrc
index 91fea7c..387bca1 100644
--- a/python/.pylintrc
+++ b/python/.pylintrc
@@ -11,5 +11,4 @@ include-ids=yes
notes=FIXME,XXX,DELETEME
[TYPECHECK]
-ignored-modules=responses
-extension-pkg-whitelist=selectolax
+extension-pkg-whitelist=selectolax,pydantic,responses
diff --git a/python/Makefile b/python/Makefile
index f783d0e..0a97437 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -17,6 +17,7 @@ lint: ## Run lints (eg, flake8, mypy)
#pipenv run flake8 . --exit-zero
pipenv run flake8 . --select=E9,F63,F7,F82 --exit-zero
pipenv run mypy *.py sandcrawler/ tests/ --ignore-missing-imports
+ pipenv run pylint --rcfile=.pylintrc -E --jobs=4 sandcrawler tests *.py
#pipenv run pytype sandcrawler/
.PHONY: fmt
@@ -24,7 +25,9 @@ fmt: ## Run code formating on all source code
pipenv run black *.py sandcrawler/ tests/
.PHONY: test
-test: lint ## Run all tests and lints
+test: ## Run all tests and lints
+ pipenv run flake8 . --select=E9,F63,F7,F82 --exit-zero
+ pipenv run mypy *.py sandcrawler/ tests/ --ignore-missing-imports
pipenv run pytest
.PHONY: coverage
diff --git a/python/tests/test_pdfextract.py b/python/tests/test_pdfextract.py
index ed93341..64e3137 100644
--- a/python/tests/test_pdfextract.py
+++ b/python/tests/test_pdfextract.py
@@ -2,6 +2,7 @@
import pytest
import struct
import responses
+import poppler
from sandcrawler import PdfExtractWorker, PdfExtractBlobWorker, CdxLinePusher, BlackholeSink, WaybackClient
from sandcrawler.pdfextract import process_pdf
@@ -20,6 +21,7 @@ def test_process_fake_pdf():
resp = process_pdf(pdf_bytes)
assert resp.status == 'not-pdf'
+@pytest.mark.skipif(poppler.__version__ == '0.2.1', reason="unsupported version of poppler")
def test_process_dummy_pdf():
with open('tests/files/dummy.pdf', 'rb') as f:
pdf_bytes = f.read()