ingest file result importer

2 files changed, 59 insertions, 0 deletions

diff --git a/python/tests/files/example_ingest.json b/python/tests/files/example_ingest.json
new file mode 100644
index 00000000..005d8742
--- /dev/null
+++ b/python/tests/files/example_ingest.json
@@ -0,0 +1 @@
+{"file_meta": {"sha1hex": "00242a192acc258bdfdb151943419437f440c313", "md5hex": "f4de91152c7ab9fdc2a128f962faebff", "sha256hex": "ffc1005680cb620eec4c913437dfabbf311b535cfe16cbaeb2faec1f92afc362", "size_bytes": 255629, "mimetype": "application/pdf"}, "request": {"project": "unit-tests", "ext_ids": {"doi": "10.123/abc"}}, "cdx": { "datetime": "20170227164644", "url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable" }, "grobid": {"status_code": 200 } }
diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py
new file mode 100644
index 00000000..7c0a85cd
--- /dev/null
+++ b/python/tests/import_ingest.py
@@ -0,0 +1,58 @@
+
+import json
+import pytest
+from fatcat_tools.importers import IngestFileResultImporter, JsonLinePusher
+from fixtures import api
+
+
+@pytest.fixture(scope="function")
+def ingest_importer(api):
+    yield IngestFileResultImporter(api)
+
+# TODO: use API to check that entities actually created...
+def test_ingest_importer_basic(ingest_importer):
+    with open('tests/files/example_ingest.json', 'r') as f:
+        JsonLinePusher(ingest_importer, f).run()
+
+@pytest.mark.skip("tests not flushed out yet")
+def test_ingest_importer(ingest_importer):
+    last_index = ingest_importer.api.get_changelog(limit=1)[0].index
+    with open('tests/files/example_ingest.json', 'r') as f:
+        ingest_importer.bezerk_mode = True
+        counts = JsonLinePusher(ingest_importer, f).run()
+    assert counts['insert'] == 2
+    assert counts['exists'] == 0
+    assert counts['skip'] == 11
+
+    # fetch most recent editgroup
+    change = ingest_importer.api.get_changelog_entry(index=last_index+1)
+    eg = change.editgroup
+    assert eg.description
+    assert "crawled from web" in eg.description.lower()
+    assert eg.extra['git_rev']
+    assert "fatcat_tools.IngestFileResultImporter" in eg.extra['agent']
+
+    # re-insert; should skip
+    with open('tests/files/example_ingest.json', 'r') as f:
+        ingest_importer.reset()
+        ingest_importer.bezerk_mode = False
+        counts = JsonLinePusher(ingest_importer, f).run()
+    assert counts['insert'] == 0
+    assert counts['exists'] == 2
+    assert counts['skip'] == 11
+
+def test_ingest_dict_parse(ingest_importer):
+    with open('tests/files/example_ingest.json', 'r') as f:
+        raw = json.loads(f.readline())
+        f = ingest_importer.parse_record(raw)
+        assert f.sha1 == "00242a192acc258bdfdb151943419437f440c313"
+        assert f.md5 == "f4de91152c7ab9fdc2a128f962faebff"
+        assert f.mimetype == "application/pdf"
+        assert f.size == 255629
+        assert len(f.urls) == 2
+        for u in f.urls:
+            if u.rel == "web":
+                assert u.url.startswith("http://journals.plos.org")
+            if u.rel == "webarchive":
+                assert u.url.startswith("https://web.archive.org/")
+        assert len(f.release_ids) == 1