diff options
author | Bryan Newbold <bnewbold@robocracy.org> | 2019-11-13 00:27:48 -0800 |
---|---|---|
committer | Bryan Newbold <bnewbold@robocracy.org> | 2019-11-15 16:46:26 -0800 |
commit | a4db9ee2e18a18b23eb7ece484f95914421f877d (patch) | |
tree | 6d4856c8fb7854a75dbd43c983179d4492769039 /python/tests | |
parent | 169477c39dc772c0eb1d45f8097215e73f0f6044 (diff) | |
download | fatcat-a4db9ee2e18a18b23eb7ece484f95914421f877d.tar.gz fatcat-a4db9ee2e18a18b23eb7ece484f95914421f877d.zip |
ingest file result importer
Diffstat (limited to 'python/tests')
-rw-r--r-- | python/tests/files/example_ingest.json | 1 | ||||
-rw-r--r-- | python/tests/import_ingest.py | 58 |
2 files changed, 59 insertions, 0 deletions
diff --git a/python/tests/files/example_ingest.json b/python/tests/files/example_ingest.json new file mode 100644 index 00000000..005d8742 --- /dev/null +++ b/python/tests/files/example_ingest.json @@ -0,0 +1 @@ +{"file_meta": {"sha1hex": "00242a192acc258bdfdb151943419437f440c313", "md5hex": "f4de91152c7ab9fdc2a128f962faebff", "sha256hex": "ffc1005680cb620eec4c913437dfabbf311b535cfe16cbaeb2faec1f92afc362", "size_bytes": 255629, "mimetype": "application/pdf"}, "request": {"project": "unit-tests", "ext_ids": {"doi": "10.123/abc"}}, "cdx": { "datetime": "20170227164644", "url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable" }, "grobid": {"status_code": 200 } } diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py new file mode 100644 index 00000000..7c0a85cd --- /dev/null +++ b/python/tests/import_ingest.py @@ -0,0 +1,58 @@ + +import json +import pytest +from fatcat_tools.importers import IngestFileResultImporter, JsonLinePusher +from fixtures import api + + +@pytest.fixture(scope="function") +def ingest_importer(api): + yield IngestFileResultImporter(api) + +# TODO: use API to check that entities actually created... +def test_ingest_importer_basic(ingest_importer): + with open('tests/files/example_ingest.json', 'r') as f: + JsonLinePusher(ingest_importer, f).run() + +@pytest.mark.skip("tests not flushed out yet") +def test_ingest_importer(ingest_importer): + last_index = ingest_importer.api.get_changelog(limit=1)[0].index + with open('tests/files/example_ingest.json', 'r') as f: + ingest_importer.bezerk_mode = True + counts = JsonLinePusher(ingest_importer, f).run() + assert counts['insert'] == 2 + assert counts['exists'] == 0 + assert counts['skip'] == 11 + + # fetch most recent editgroup + change = ingest_importer.api.get_changelog_entry(index=last_index+1) + eg = change.editgroup + assert eg.description + assert "crawled from web" in eg.description.lower() + assert eg.extra['git_rev'] + assert "fatcat_tools.IngestFileResultImporter" in eg.extra['agent'] + + # re-insert; should skip + with open('tests/files/example_ingest.json', 'r') as f: + ingest_importer.reset() + ingest_importer.bezerk_mode = False + counts = JsonLinePusher(ingest_importer, f).run() + assert counts['insert'] == 0 + assert counts['exists'] == 2 + assert counts['skip'] == 11 + +def test_ingest_dict_parse(ingest_importer): + with open('tests/files/example_ingest.json', 'r') as f: + raw = json.loads(f.readline()) + f = ingest_importer.parse_record(raw) + assert f.sha1 == "00242a192acc258bdfdb151943419437f440c313" + assert f.md5 == "f4de91152c7ab9fdc2a128f962faebff" + assert f.mimetype == "application/pdf" + assert f.size == 255629 + assert len(f.urls) == 2 + for u in f.urls: + if u.rel == "web": + assert u.url.startswith("http://journals.plos.org") + if u.rel == "webarchive": + assert u.url.startswith("https://web.archive.org/") + assert len(f.release_ids) == 1 |