ingest: progress on HTML ingest

2 files changed, 44 insertions, 2 deletions

diff --git a/python/tests/files/example_ingest_html.json b/python/tests/files/example_ingest_html.json
new file mode 100644
index 00000000..6c646814
--- /dev/null
+++ b/python/tests/files/example_ingest_html.json
@@ -0,0 +1 @@
+{"cdx": {"datetime": "20200708025309", "mimetype": "text/html", "sha1b32": "THJFFZJR2VYN2FAR7X7LHFGRU2X5IC2U", "sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "status_code": 200, "surt": "py,una,iics,scielo)/scielo.php?lng=en&nrm=iso&pid=s1683-98032015000200002&script=sci_arttext&tlng=es", "url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es", "warc_csize": 13123, "warc_offset": 77579308, "warc_path": "SCIELO-CRAWL-2020-07-20200707211940442-00279-00347-wbgrp-svc206/SCIELO-CRAWL-2020-07-20200708024511243-00332-13069~wbgrp-svc206.us.archive.org~8443.warc.gz"}, "file_meta": {"md5hex": "515a61845a2f898438e3986e4506da8f", "mimetype": "text/html", "sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "sha256hex": "c4559d548476a325891461b71c796beee717e820d6a00cb8411176ce83a0f23f", "size_bytes": 47442}, "hit": true, "hops": ["http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es"], "html_biblio": {"container_issn": "1683-9803", "container_name": "Pediatr\u00eda (Asunci\u00f3n)", "contrib_names": ["Ruiz Valiente, Syntia Carolina", "Ruiz Ca\u00f1ete, Manuel", "Cohene Velazquez, Bartola"], "doi": "10.18004/ped.2015.agosto.102-107", "first_page": "102", "html_fulltext_url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es", "issue": "2", "last_page": "107", "pdf_fulltext_url": "http://scielo.iics.una.py/pdf/ped/v42n2/v42n2a02.pdf", "publisher": "Sociedad Paraguaya de Pediatr\u00eda", "release_date": "2015-08-06", "title": "Prevalence of malnutrition and eating habits in children under 5 years of age in indigenous communities in Azote'y and Yby Yau, 2011", "volume": "42", "xml_fulltext_url": "http://scielo.iics.una.py/scieloOrg/php/articleXML.php?pid=S1683-98032015000200002&lang=en"}, "html_body": {"agent": "trafilatura/0.5.1", "status": "success", "word_count": 3500}, "html_resources": [{"mimetype": "image/gif", "resource_type": "image", "sha1hex": "4991aa771874daf8cba79be38d18d534f946b5d6", "sha256hex": "5e76fad755b873a439dd5e775684696c547008d45cc901606132e9a1ed970757", "size": 220, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/alpha.gif", "timestamp": "2020-10-31T14:07:30", "url": "http://scielo.iics.una.py/img/en/alpha.gif"}, {"mimetype": "text/plain", "resource_type": "script", "sha1hex": "fd28e342fa1b40b84cc17dc66d22df3bf260170b", "sha256hex": "9cf2e81dd65d5a64200970bbd1cd9497b46b2af232e2fbfb79fef95b070f23d1", "size": 3653, "status_code": 200, "surt": "py,una,iics,scielo)/applications/scielo-org/js/toolbox.js", "timestamp": "2020-10-31T20:14:35", "url": "http://scielo.iics.una.py/applications/scielo-org/js/toolbox.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "c5ea6229ce6a97f2dc2b2e2c8ffac26400dfcd58", "sha256hex": "7fb3d59ea14ab060c2b6cbdd5e63d57e158d6cc9e613ceb05ab1e6ec60d64995", "size": 382, "status_code": 200, "surt": "py,una,iics,scielo)/img/common/iconpermalink.gif", "timestamp": "2020-10-31T20:14:52", "url": "http://scielo.iics.una.py/img/common/iconPermalink.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "fbd3488e6b8cd241605fa2db14ba15e0f037d3a7", "sha256hex": "5492829967d521386bec4323f0d7ef951e9a0b16caa1bcd8e75576dc41bd3b55", "size": 26759, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f1.jpg", "timestamp": "2020-07-08T02:53:11", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f1.jpg"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "9f1833948223109dfaca2c37fbdbacb81002a346", "sha256hex": "f5b08a2022fce73ae04c3b9fe368645a084132a942bb29950bba705ed89e6d91", "size": 35440, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02t1.jpg", "timestamp": "2020-07-08T02:53:18", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02t1.jpg"}, {"mimetype": "image/png", "resource_type": "image", "sha1hex": "0d2d329000cba763e5eec45bd8ee2743393ebd62", "sha256hex": "d964eed5974264b8f107a905b74796cb3d5e60f78da1c500bb547a419538915e", "size": 3091, "status_code": 200, "surt": "py,una,iics,scielo)/img/common/icon-close.png", "timestamp": "2020-10-24T10:17:13", "url": "http://scielo.iics.una.py/img/common/icon-close.png"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "5812dc64389992d7d59d10e57449407778bbd0c0", "sha256hex": "605ce931ded871d924f31765c6bbf778eb8b5194b3396f49638a88331f53dc21", "size": 652, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconxmldocument.gif", "timestamp": "2020-10-24T14:58:51", "url": "http://scielo.iics.una.py/img/en/iconXMLDocument.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "f20584095b9c7d06250140bf7f51f7bd91e2ba08", "sha256hex": "f8292c0c25d5eec546fe16e8a53101b4933adb2e75e58d7335158dc94b2bae91", "size": 239, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/artsrc.gif", "timestamp": "2020-10-29T17:34:52", "url": "http://scielo.iics.una.py/img/en/artsrc.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "09fe461f38958a267695edf5675f668323f754ec", "sha256hex": "d0792cfc52df6414126a541e8cd32ba151d75f87225c63d38a9ddad389b913b3", "size": 229, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/subject.gif", "timestamp": "2020-10-28T12:59:36", "url": "http://scielo.iics.una.py/img/en/subject.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "7b2f78593847928d8f0f8a2068b0cb366501c3e5", "sha256hex": "97dfc989c7af7a0139950696e533fe71c373539091200edba96f151efb045f8d", "size": 181, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/grp1c.gif", "timestamp": "2020-10-24T10:17:25", "url": "http://scielo.iics.una.py/img/en/grp1c.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "a1e6d8818d56678a52a18859b0cf919b8663a5aa", "sha256hex": "2d34923f1bb8e417a4c244ba5be13b7fe52e0dc6dba9dbcdf512a9fb3cb84d91", "size": 27383, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f2.jpg", "timestamp": "2020-07-08T02:53:13", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f2.jpg"}, {"mimetype": "text/plain", "resource_type": "stylesheet", "sha1hex": "3754bfd4a8608ec125c79ccc7b62ead02c323bbc", "sha256hex": "4dc9b9edd3fc1e58d7a1c39c64551ac07530bedf0721323fc2c820a90a7b4a64", "size": 87, "status_code": 200, "surt": "py,una,iics,scielo)/css/screen.css", "timestamp": "2020-10-24T11:51:22", "url": "http://scielo.iics.una.py/css/screen.css"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "818ff217eae41fe796f21e9b56336011d8806de0", "sha256hex": "a3853400c16b0628dd226487d1ad7710f44a2e6ea8de85f2b2a6a34b7334d5b6", "size": 210, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/search.gif", "timestamp": "2020-10-28T12:59:22", "url": "http://scielo.iics.una.py/img/en/search.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "a6300c0530bdc13b1bf75a7f380cef6c1be48cc7", "sha256hex": "aa7fa5a5bedea888ddbb89f20838207eb303323c98c414452f632f96acaccbfe", "size": 660, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconemail.gif", "timestamp": "2020-11-01T12:16:11", "url": "http://scielo.iics.una.py/img/en/iconEmail.gif"}, {"mimetype": "image/png", "resource_type": "image", "sha1hex": "100a6b57582fbf383f96c289c92fbbc9aaa63f06", "sha256hex": "f43d4d35e7ac1e815dc0c8897806e30d928ee62e1aa6ac20f49c649f8b694004", "size": 430, "status_code": 200, "surt": "net,licensebuttons)/l/by/4.0/80x15.png", "timestamp": "2020-07-08T21:51:45", "url": "https://licensebuttons.net/l/by/4.0/80x15.png"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "dbfce64d671bbb03591a297983c81ede279b051d", "sha256hex": "2ef85ef9dd7926099287dd33ab43fc6819b393446e85ddb754897ad457c56282", "size": 244, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/prev.gif", "timestamp": "2020-10-29T01:22:04", "url": "http://scielo.iics.una.py/img/en/prev.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "bf5bed62fc6cc82a8a7e862fceac9ce8ffb12cb8", "sha256hex": "1a90de599f61e3191fec24d504798c372b72ccbc511c3c44d48070a0dddefe25", "size": 262, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconrelatedoff.gif", "timestamp": "2020-10-28T12:59:51", "url": "http://scielo.iics.una.py/img/en/iconRelatedOff.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "4a59b2b1d57a210252311d563eea138afcc7a886", "sha256hex": "f96585d38fb34040d9bd81e83538a7beade916bc2d4456e75d5911181281cb6f", "size": 586, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/e-mailt.gif", "timestamp": "2020-10-31T20:53:28", "url": "http://scielo.iics.una.py/img/en/e-mailt.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "d465c5da9dea2a6e10b5d340c5af6af6cc10f3ec", "sha256hex": "ffc1411a8185c8df5ca9c0725fbfab41380706d213ca42c8552f32323b67901d", "size": 33992, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f3.jpg", "timestamp": "2020-07-08T02:53:19", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f3.jpg"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "86dbd3881975bc15fef15536e5cbd54bad53271c", "sha256hex": "75cbc76c44915b46c6c44fdeeeabd1bfab774ecf692d37ed8d1b4674f5ee583d", "size": 628, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconpdfdocument.gif", "timestamp": "2020-10-24T11:51:12", "url": "http://scielo.iics.una.py/img/en/iconPDFDocument.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "080333f92aa899e53bcd30a4e734d9b36d7ac7a4", "sha256hex": "e6834ac24d48ec9d75b178de59964eb9fb66e9cff05b439ad247f5af5d5fc1ff", "size": 374, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconreferences.gif", "timestamp": "2020-10-29T05:40:05", "url": "http://scielo.iics.una.py/img/en/iconReferences.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "fe879762fd80c756df0af9c81e3424d651fa1b6a", "sha256hex": "4333f6c0ccd89f3240b6c8bb9b2c109792da6d0513e618c35033e2474981b55d", "size": 578, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/icontranslation.gif", "timestamp": "2020-10-28T22:59:05", "url": "http://scielo.iics.una.py/img/en/iconTranslation.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "8846563b9722db2f3b832c03ad2ee9b6318c1d0e", "sha256hex": "6843f628c71f39631ec5d501f6b62506ae9f8454c0a3cd957f4dc67985c371bb", "size": 219, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/author.gif", "timestamp": "2020-10-24T10:16:50", "url": "http://scielo.iics.una.py/img/en/author.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "db35a7e4171a130d25632d6e1ba9c3806eec1e87", "sha256hex": "bd6496501a92a6ed3c5e8c16ce0af4ac9b4cece3562934010d0878f6ea06ead0", "size": 288, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconcitedoff.gif", "timestamp": "2020-10-29T17:34:57", "url": "http://scielo.iics.una.py/img/en/iconCitedOff.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "0d844cf48e3ed3849d7e2deed30fb2e7318107b0", "sha256hex": "f257802855722fed0b2b6936a9aede3a4869fc347e91860a26cc81c1ba9df3a3", "size": 164, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/toc.gif", "timestamp": "2020-10-28T22:59:18", "url": "http://scielo.iics.una.py/img/en/toc.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "33e2d3699619eb6dac7c91c207c748599def84f0", "sha256hex": "d8af84c5c4c10e724a081409b0f0e50eb08b9c2cd3d3e0ee0b33cc9eaa20086c", "size": 193, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/next.gif", "timestamp": "2020-11-01T12:16:02", "url": "http://scielo.iics.una.py/img/en/next.gif"}, {"mimetype": "text/html", "resource_type": "script", "sha1hex": "731cf720a546953efe311566a2d874fae715bfc6", "sha256hex": "0d3602fb417d811e15e1a7bd6725384e5bda874dab0eb7be7ee59cd26d64dbd1", "size": 8231, "status_code": 200, "surt": "py,una,iics,scielo)/article.js", "timestamp": "2020-10-29T05:40:31", "url": "http://scielo.iics.una.py/article.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "756779e5ff89d107b2eb4843cc47dd4b63efc829", "sha256hex": "b52a6dc8cbbf4212790cf57af7489b9dc21c040ae7372c07bb6aa18473098759", "size": 190, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/home.gif", "timestamp": "2020-07-06T21:30:58", "url": "http://scielo.iics.una.py/img/en/home.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "e541555caec87c313c9c1859c4b5913b31f955c5", "sha256hex": "64eeb2c0e97f96d9144aa83d027fb5b9d57d96c74681611c39af99d49b148c6e", "size": 643, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/fulltxt.gif", "timestamp": "2020-10-29T22:47:53", "url": "http://scielo.iics.una.py/img/en/fulltxt.gif"}, {"mimetype": "text/plain", "resource_type": "script", "sha1hex": "65cbff4e9d95d47a6f31d96ab4ea361c1f538a7b", "sha256hex": "e23a2a4e2d7c2b41ebcdd8ffc0679df7140eb7f52e1eebabf827a88182643c59", "size": 72174, "status_code": 200, "surt": "py,una,iics,scielo)/applications/scielo-org/js/jquery-1.4.2.min.js", "timestamp": "2020-07-06T21:17:01", "url": "http://scielo.iics.una.py/applications/scielo-org/js/jquery-1.4.2.min.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "20368dd206a00eaf8bb117f98291a30eb0cc8e73", "sha256hex": "534434f1716e29928e0376d0e5dc113808c96d9cedab8675adff7dbf22cb9fd1", "size": 1353, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/fbpelogp.gif", "timestamp": "2020-07-06T21:16:38", "url": "http://scielo.iics.una.py/img/en/fbpelogp.gif"}], "request": {"link_source": "doi", "ingest_request_source": "fatcat-changelog", "base_url": "http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "ext_ids": {"doi": "10.123/abc"}, "fatcat": {"release_ident": null}, "ingest_type": "html"}, "scope": "article-fulltext", "status": "success", "terminal": {"terminal_dt": "20200708025309", "terminal_sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "terminal_status_code": 200, "terminal_url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es"}}
diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py
index 21552fb9..92539f1a 100644
--- a/python/tests/import_ingest.py
+++ b/python/tests/import_ingest.py
@@ -2,7 +2,7 @@
 import json
 import pytest
 
-from fatcat_tools.importers import IngestFileResultImporter, JsonLinePusher
+from fatcat_tools.importers import IngestFileResultImporter, IngestWebResultImporter, JsonLinePusher
 from fixtures import *
 
 
@@ -10,6 +10,10 @@ from fixtures import *
 def ingest_importer(api):
     yield IngestFileResultImporter(api)
 
+@pytest.fixture(scope="function")
+def ingest_web_importer(api):
+    yield IngestWebResultImporter(api)
+
 # TODO: use API to check that entities actually created...
 def test_ingest_importer_basic(ingest_importer):
     with open('tests/files/example_ingest.json', 'r') as f:
@@ -46,6 +50,7 @@ def test_ingest_importer_xml(ingest_importer):
     with open('tests/files/example_ingest_xml.json', 'r') as f:
         ingest_importer.bezerk_mode = True
         counts = JsonLinePusher(ingest_importer, f).run()
+    print(counts)
     assert counts['insert'] == 1
     assert counts['exists'] == 0
     assert counts['skip'] == 0
@@ -58,6 +63,42 @@ def test_ingest_importer_xml(ingest_importer):
     assert eg.extra['git_rev']
     assert "fatcat_tools.IngestFileResultImporter" in eg.extra['agent']
 
+    # re-import should skip
+    with open('tests/files/example_ingest_xml.json', 'r') as f:
+        ingest_importer.reset()
+        ingest_importer.bezerk_mode = False
+        counts = JsonLinePusher(ingest_importer, f).run()
+    assert counts['insert'] == 0
+    assert counts['exists'] == 1
+    assert counts['skip'] == 0
+
+def test_ingest_importer_web(ingest_web_importer):
+    last_index = ingest_web_importer.api.get_changelog(limit=1)[0].index
+    with open('tests/files/example_ingest_html.json', 'r') as f:
+        ingest_web_importer.bezerk_mode = True
+        counts = JsonLinePusher(ingest_web_importer, f).run()
+    print(counts)
+    assert counts['insert'] == 1
+    assert counts['exists'] == 0
+    assert counts['skip'] == 0
+
+    # fetch most recent editgroup
+    change = ingest_web_importer.api.get_changelog_entry(index=last_index+1)
+    eg = change.editgroup
+    assert eg.description
+    assert "crawled from web" in eg.description.lower()
+    assert eg.extra['git_rev']
+    assert "fatcat_tools.IngestWebResultImporter" in eg.extra['agent']
+
+    # re-import should skip
+    with open('tests/files/example_ingest_html.json', 'r') as f:
+        ingest_web_importer.reset()
+        ingest_web_importer.bezerk_mode = False
+        counts = JsonLinePusher(ingest_web_importer, f).run()
+    assert counts['insert'] == 0
+    assert counts['exists'] == 1
+    assert counts['skip'] == 0
+
 def test_ingest_importer_stage(ingest_importer, api):
     """
     Tests that ingest importer correctly handles release stage matching
@@ -74,7 +115,7 @@ def test_ingest_importer_stage(ingest_importer, api):
     with open('tests/files/example_ingest.json', 'r') as f:
         raw = json.loads(f.readline())
     for row in test_table:
-        print(row)
+        #print(row)
 
         # set dummy record stage
         eg = quick_eg(api)