2 files changed, 71 insertions, 0 deletions
diff --git a/python/tests/files/example_shadow.json b/python/tests/files/example_shadow.json
new file mode 100644
index 00000000..f84a61a5
--- /dev/null
+++ b/python/tests/files/example_shadow.json
@@ -0,0 +1,12 @@
+{"shadow":{"shadow_corpus":"scimag","shadow_id":"8149931","sha1hex":"000008bc38cb80636b647b38653fc1574936c03e","doi":"10.1371/journal.pmed.0020124","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"000008bc38cb80636b647b38653fc1574936c03e","sha256hex":"18b341119bbbf297a7dfa21aca86211da446617600baa153df70b4209c2c6e84","md5hex":"629e84885be85bc8d88345b98cffa0b0","size_bytes":39955,"mimetype":null},"cdx":{"url":"https://link.springer.com/content/pdf/10.1007%2Fs11626-008-9119-8.pdf","datetime":"20180729135948","sha1hex":"000008bc38cb80636b647b38653fc1574936c03e","cdx_sha1hex":null,"mimetype":"application/pdf","warc_path":"UNPAYWALL-PDF-CRAWL-2018-07-20180729132538992-15980-16048-wbgrp-svc281/UNPAYWALL-PDF-CRAWL-2018-07-20180729135708800-16009-11693~wbgrp-svc281.us.archive.org~8443.warc.gz","warc_csize":32497,"warc_offset":105265425,"row_created":"2019-08-09T23:25:44.571943+00:00"}}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"33139096","sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","doi":"10.0000/cyberleninka.ru/article/n/analiz-primeneniya-fazochastotnyh-algoritmov-proslezhivaniya-signalov-dlya-izmereniya-urovnya-zhidkosti-v-neftedobyvayuschih","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","sha256hex":"99f15c58c2343f46c8cae75ff01c11b1b9e3c6d2f57189ec78df94e234b2c633","md5hex":"488681b249f6e9292bcde1fab1422550","size_bytes":182449,"mimetype":null},"cdx":{"url":"http://www.lib.tpu.ru/fulltext/v/Bulletin_TPU/2011/v319/i5/12.pdf","datetime":"20180412144307","sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","cdx_sha1hex":null,"mimetype":"application/pdf","warc_path":"OA-JOURNAL-TESTCRAWL-TWO-2018-20180412133030095-00799-00808-wbgrp-svc284/OA-JOURNAL-TESTCRAWL-TWO-2018-20180412142334247-00807-23249~wbgrp-svc284.us.archive.org~8443.warc.gz","warc_csize":126165,"warc_offset":924893749,"row_created":"2019-08-09T05:16:39.785581+00:00"}}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"33139096","sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","doi":null,"pmid":"54321","isbn13":null},"file_meta":{"sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","sha256hex":"99f15c58c2343f46c8cae75ff01c11b1b9e3c6d2f57189ec78df94e234b2c633","md5hex":"488681b249f6e9292bcde1fab1422550","size_bytes":182449,"mimetype":null},"cdx":{"url":"https://cyberleninka.ru/article/n/analiz-primeneniya-fazochastotnyh-algoritmov-proslezhivaniya-signalov-dlya-izmereniya-urovnya-zhidkosti-v-neftedobyvayuschih.pdf","datetime":"20180506175847","sha1hex":"00000c4296e2c5f8f70ab265c683235fbf5e354b","cdx_sha1hex":null,"mimetype":"application/pdf","warc_path":"OA-JOURNAL-TESTCRAWL-TWO-2018-20180506171133875-05766-05775-wbgrp-svc284/OA-JOURNAL-TESTCRAWL-TWO-2018-20180506174415763-05771-23249~wbgrp-svc284.us.archive.org~8443.warc.gz","warc_csize":126144,"warc_offset":532659301,"row_created":"2019-08-09T05:16:39.785581+00:00"}}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"21389391","sha1hex":"00000d5508d7d7106560ade65c33c628c54d7c75","doi":"10.1038/nn.3419","pmid":"23727820","isbn13":null},"file_meta":{"sha1hex":"00000d5508d7d7106560ade65c33c628c54d7c75","sha256hex":"8c48dd68b974ed117f839dc88db44884e7e1df9ddef30f26c541437d7f390d96","md5hex":"c2a43160b62ef0f13256c789270ec2a9","size_bytes":1375452,"mimetype":null},"cdx":{"url":"https://www.janelia.org/sites/default/files/Library/nn.3419.pdf","datetime":"20170829032635","sha1hex":"00000d5508d7d7106560ade65c33c628c54d7c75","cdx_sha1hex":null,"mimetype":"application/pdf","warc_path":"SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170829031124939-00100-00109-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170829032404137-00107-3480~wbgrp-svc284.us.archive.org~8443.warc.gz","warc_csize":973733,"warc_offset":262621802,"row_created":"2019-08-09T05:21:39.486744+00:00"}}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"16462885","sha1hex":"0000102db78329a149d3b6319f6ccf8cc90483e2","doi":"10.1016/j.cell.2007.04.022","pmid":"17482536","isbn13":null},"file_meta":{"sha1hex":"0000102db78329a149d3b6319f6ccf8cc90483e2","sha256hex":null,"md5hex":"995e7145d09d50eadccf322780e474d3","size_bytes":206812,"mimetype":"application/pdf"},"cdx":{"url":"http://publisher-connector.core.ac.uk/resourcesync/data/elsevier/pdf/212/aHR0cDovL2FwaS5lbHNldmllci5jb20vY29udGVudC9hcnRpY2xlL3BpaS9zMDA5Mjg2NzQwNzAwNTI4NA%3D%3D.pdf","datetime":"20170925031039","sha1hex":"0000102db78329a149d3b6319f6ccf8cc90483e2","cdx_sha1hex":null,"mimetype":"application/pdf","warc_path":"TARGETED-PDF-CRAWL-2017-08-04-20170925022437721-01046-01055-wbgrp-svc284/TARGETED-PDF-CRAWL-2017-08-04-20170925024811552-01048-15075~wbgrp-svc284.us.archive.org~8443.warc.gz","warc_csize":188232,"warc_offset":677858259,"row_created":"2019-08-10T03:02:21.656961+00:00"}}
+{"shadow":{"shadow_corpus":"scimag","shadow_id":"12703034","sha1hex":"0000002922264275f11cca7b1c3fb662070d0dd7","doi":"10.1007/s11061-011-9281-1","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"0000002922264275f11cca7b1c3fb662070d0dd7","sha256hex":"b4728210cc0f70d8a8f8c39bd97fcbbab3eaca4309ac4bdfbce5df3b66c82f79","md5hex":"debd8db178fa08a7a0aaec6e42832a8e","size_bytes":206121,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"51052483","sha1hex":"00000119fa780ce368ebd96563afdb3eebb90ad3","doi":"10.1191/0266355403gh289oa","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"00000119fa780ce368ebd96563afdb3eebb90ad3","sha256hex":"57ce460db4410b9bfaf500ed652fd29e64d46b40c17e28f1156ba03736edf91b","md5hex":"96133eec3a6c533993213e7bdf446251","size_bytes":164344,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"2476283","sha1hex":"0000017a31547caf347fab66282a40831b9ceb08","doi":"10.1016/0042-207x(62)90512-2","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"0000017a31547caf347fab66282a40831b9ceb08","sha256hex":"e8d0c607b024ff6ffd58a35f76c454844b70ad19fe3f78a573af1ae53f53ad9d","md5hex":"b53318522b9f35a42b7e53f150fe70b2","size_bytes":116735,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"8760871","sha1hex":"000001abf3dbf936d5053d14f41699722531b8c6","doi":"10.1016/s0042-207x(79)80945-8","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"000001abf3dbf936d5053d14f41699722531b8c6","sha256hex":"8a69b4a6dff98682ad43e7d4139221c1557c1bd202b615490af8a2c7dcbb71d2","md5hex":"29e1cfac8ecfbc8be57a1ec8b465c4be","size_bytes":138218,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"11473618","sha1hex":"0000022e387be46ef797f6686d36c9899cbd6856","doi":"10.1038/ng.2339","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"0000022e387be46ef797f6686d36c9899cbd6856","sha256hex":"a72517e8e72d78bc07a6ef7ff3a6d1d3e04325df986cb8f1bbb4e809f7a9dbdd","md5hex":"9cb8a6e056c9cc740d3bed0c50cd53dc","size_bytes":80992,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"47301218","sha1hex":"0000029209536bda5f22e5110e573c5bd8ceb43a","doi":"10.2307/23406551","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"0000029209536bda5f22e5110e573c5bd8ceb43a","sha256hex":"315f1d39a00ccf256fa15d92a14869dbda48d31500989aaacb11368f906a5827","md5hex":"8141b42ec3bb41fa87099633a1b61d93","size_bytes":305236,"mimetype":null},"cdx":null}
+ {"shadow":{"shadow_corpus":"scimag","shadow_id":"30603850","sha1hex":"000002c1abd521f18aa23d9e8f464e697e218ab1","doi":"10.1109/spire.1998.712983","pmid":null,"isbn13":null},"file_meta":{"sha1hex":"000002c1abd521f18aa23d9e8f464e697e218ab1","sha256hex":"777e2c472e9d2fec3bbd26bad788562cf1e08e5850315c25cfb6e46d38e7e4af","md5hex":"3a3c92fabaf6cf437bb596d9e9255ff6","size_bytes":113768,"mimetype":null},"cdx":null}
diff --git a/python/tests/import_shadow.py b/python/tests/import_shadow.py
new file mode 100644
index 00000000..30e1724f
--- /dev/null
+++ b/python/tests/import_shadow.py
@@ -0,0 +1,59 @@
+
+import json
+import pytest
+from fatcat_tools.importers import ShadowLibraryImporter, JsonLinePusher
+from fixtures import api
+
+
+@pytest.fixture(scope="function")
+def shadow_importer(api):
+    yield ShadowLibraryImporter(api)
+
+# TODO: use API to check that entities actually created...
+def test_shadow_importer_basic(shadow_importer):
+    with open('tests/files/example_shadow.json', 'r') as f:
+        JsonLinePusher(shadow_importer, f).run()
+
+def test_shadow_importer(shadow_importer):
+    last_index = shadow_importer.api.get_changelog(limit=1)[0].index
+    with open('tests/files/example_shadow.json', 'r') as f:
+        shadow_importer.bezerk_mode = True
+        counts = JsonLinePusher(shadow_importer, f).run()
+    assert counts['insert'] == 2
+    assert counts['exists'] == 0
+    assert counts['skip'] == 10
+
+    # fetch most recent editgroup
+    change = shadow_importer.api.get_changelog_entry(index=last_index+1)
+    eg = change.editgroup
+    assert eg.description
+    assert "shadow library" in eg.description.lower()
+    assert eg.extra['git_rev']
+    assert "fatcat_tools.ShadowLibraryImporter" in eg.extra['agent']
+
+    # re-insert; should skip
+    with open('tests/files/example_shadow.json', 'r') as f:
+        shadow_importer.reset()
+        shadow_importer.bezerk_mode = False
+        counts = JsonLinePusher(shadow_importer, f).run()
+    assert counts['insert'] == 0
+    assert counts['exists'] == 2
+    assert counts['skip'] == 10
+
+def test_shadow_dict_parse(shadow_importer):
+    with open('tests/files/example_shadow.json', 'r') as f:
+        raw = json.loads(f.readline())
+        f = shadow_importer.parse_record(raw)
+        assert f.sha1 == "000008bc38cb80636b647b38653fc1574936c03e"
+        assert f.md5 == "629e84885be85bc8d88345b98cffa0b0"
+        assert f.mimetype == None # "application/pdf"
+        assert f.size == 39955
+        assert len(f.urls) == 2
+        for u in f.urls:
+            if u.rel == "publisher":
+                assert u.url.startswith("https://link.springer.com/content/pdf/10.1007%2Fs11626-008-9119-8.pdf")
+            if u.rel == "webarchive":
+                assert u.url.startswith("https://web.archive.org/")
+                assert "20180729135948" in u.url
+        assert len(f.release_ids) == 1
+