fixes and test coverage for file_meta importer

2 files changed, 68 insertions, 0 deletions

diff --git a/python/tests/files/example_file_meta.json b/python/tests/files/example_file_meta.json
new file mode 100644
index 00000000..7cc42213
--- /dev/null
+++ b/python/tests/files/example_file_meta.json
@@ -0,0 +1,7 @@
+{"sha1hex":"00000088bbc15a03ab89d8da6c356bf25aea9519","sha256hex":"593f5a260129bb89ed316c9ddcb7b2f9c2e3da8adf87d29f212b423de32a2c59","md5hex":"f4308d58dc8806232c30edc56a896412","size_bytes":354118,"mimetype":"application/pdf"}
+{"sha1hex":"0000020e28273e4ed93fb8cf7556dbff6aca94d1","sha256hex":"23722f8899c8d246d31c42b920c35c94ad2a4f3c55ca8444dd41649d9df21374","md5hex":"bb57fac6eaa09ffa93ebb16f01a33fd4","size_bytes":945918,"mimetype":"application/pdf"}
+{"sha1hex":"000002d15f55047daafae1abf7e81c76b64e8062","sha256hex":"1d3f5440160359679f66c957061f51cf5c0a2f38c1fef34502aa1e020499aea8","md5hex":"4424c346ef22168ee63b76e9808d7788","size_bytes":130761,"mimetype":"application/pdf"}
+{"sha1hex":"00000376ad49f56145721503f1eb5e6e49e779fd","sha256hex":"87b56ac0438bc8a6d6911f2ecb821395ef39c09899e51285dfb4cafd5a670f1c","md5hex":"c9f9056276fa33d08e309854700129db","size_bytes":462901,"mimetype":"application/pdf"}
+{"sha1hex":"0000045687dad717ed6512e395b04ec9c00995b7","sha256hex":"51bdc9e40cc175089fcb60b0b188e6cbcdcddb1ff8acbe6b039b8f8fff0afff0","md5hex":"e1fd97475c8aa102568f5d70a1bd0c07","size_bytes":372121,"mimetype":"application/pdf"}
+{"sha1hex":"000005fcdf1fbdb06d978bb0e2b60d2b3fdfeade","sha256hex":"6bef5dc02fdf158da42540fe12021f7964587c6d3c34669874758a6a43a2b5e0","md5hex":"bcff3dfbb33c21b10dc2c4c46c1c6d03","size_bytes":112587,"mimetype":"application/pdf"}
+{"sha1hex":"000005fcdf1fbdb06d978bb0e2b60d2b3fdfeade","sha256hex":"","md5hex":"bcff3dfbb33c21b10dc2c4c46c1c6d03","size_bytes":112587,"mimetype":"application/pdf"}
diff --git a/python/tests/import_file_meta.py b/python/tests/import_file_meta.py
new file mode 100644
index 00000000..b59356b6
--- /dev/null
+++ b/python/tests/import_file_meta.py
@@ -0,0 +1,61 @@
+
+import json
+import pytest
+
+from fatcat_tools.importers import FileMetaImporter, JsonLinePusher
+from fatcat_openapi_client import FileEntity
+from fixtures import *
+
+
+@pytest.fixture(scope="function")
+def file_meta_importer(api):
+    yield FileMetaImporter(api)
+
+def test_file_meta_importer_basic(file_meta_importer):
+
+    # insert two file entities
+    api = file_meta_importer.api
+    eg = quick_eg(file_meta_importer.api)
+    # with full metadata
+    f1edit = api.create_file(eg.editgroup_id, FileEntity(
+        size=372121,
+        md5="e1fd97475c8aa102568f5d70a1bd0c07",
+        sha1="0000045687dad717ed6512e395b04ec9c00995b7",
+        sha256="51bdc9e40cc175089fcb60b0b188e6cbcdcddb1ff8acbe6b039b8f8fff0afff0",
+        mimetype="application/pdf",
+    ))
+    # partial/stub metadata
+    f2edit = api.create_file(eg.editgroup_id, FileEntity(
+        sha1="00000376ad49f56145721503f1eb5e6e49e779fd",
+        mimetype="application/pdf",
+    ))
+    api.accept_editgroup(eg.editgroup_id)
+
+    last_index = file_meta_importer.api.get_changelog(limit=1)[0].index
+
+    with open('tests/files/example_file_meta.json', 'r') as f:
+        counts = JsonLinePusher(file_meta_importer, f).run()
+
+    assert counts['insert'] == 0
+    assert counts['exists'] == 0
+    assert counts['update'] == 1
+    assert counts['skip-no-match'] == 4
+    assert counts['skip-missing-field'] == 1
+    assert counts['skip-existing-complete'] == 1
+
+    # cleanup file entities
+    eg = quick_eg(file_meta_importer.api)
+    api.delete_file(eg.editgroup_id, f1edit.ident)
+    api.delete_file(eg.editgroup_id, f2edit.ident)
+    api.accept_editgroup(eg.editgroup_id)
+
+def test_file_meta_dict_parse(file_meta_importer):
+    with open('tests/files/example_file_meta.json', 'r') as f:
+        raw = json.loads(f.readline())
+        f = file_meta_importer.parse_record(raw)
+
+        assert f.sha1 == "00000088bbc15a03ab89d8da6c356bf25aea9519"
+        assert f.md5 == "f4308d58dc8806232c30edc56a896412"
+        assert f.sha256 == "593f5a260129bb89ed316c9ddcb7b2f9c2e3da8adf87d29f212b423de32a2c59"
+        assert f.mimetype == "application/pdf"
+        assert f.size == 354118