refactored crossref importer to new style

2 files changed, 32 insertions, 29 deletions

diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py
index 89ce9fc9..5e0a150f 100644
--- a/python/tests/import_crossref.py
+++ b/python/tests/import_crossref.py
@@ -1,27 +1,29 @@
 
 import json
 import pytest
-from fatcat_tools.importers import CrossrefImporter
+from fatcat_tools.importers import CrossrefImporter, JsonLinePusher
 from fixtures import api
 
 
 @pytest.fixture(scope="function")
 def crossref_importer(api):
     with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
-        yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', check_existing=False)
+        yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=True)
 
 @pytest.fixture(scope="function")
 def crossref_importer_existing(api):
     with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
-        yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', check_existing=True)
+        yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=False)
 
 def test_crossref_importer_batch(crossref_importer):
     with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
-        crossref_importer.process_batch(f)
+        pusher = JsonLinePusher(crossref_importer, f)
+        pusher.run()
 
 def test_crossref_importer(crossref_importer):
     with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
-        crossref_importer.process_source(f)
+        pusher = JsonLinePusher(crossref_importer, f)
+        pusher.run()
     # fetch most recent editgroup
     changes = crossref_importer.api.get_changelog(limit=1)
     eg = changes[0].editgroup
@@ -39,13 +41,14 @@ def test_crossref_mappings(crossref_importer):
 def test_crossref_importer_create(crossref_importer):
     crossref_importer.create_containers = True
     with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
-        crossref_importer.process_source(f)
+        pusher = JsonLinePusher(crossref_importer, f)
+        pusher.run()
 
 def test_crossref_dict_parse(crossref_importer):
     with open('tests/files/crossref-works.single.json', 'r') as f:
         # not a single line
         raw = json.loads(f.read())
-        (r, c) = crossref_importer.parse_crossref_dict(raw)
+        r = crossref_importer.parse_record(raw)
         extra = r.extra['crossref']
         assert r.title == "Renormalized perturbation theory by the moment method for degenerate states: Anharmonic oscillators"
         assert r.doi == "10.1002/(sici)1097-461x(1998)66:4<261::aid-qua1>3.0.co;2-t"
@@ -79,8 +82,10 @@ def test_crossref_dict_parse(crossref_importer):
 def test_stateful_checking(crossref_importer_existing):
     with open('tests/files/crossref-works.single.json', 'r') as f:
         # not a single line, a whole document
-        raw = json.loads(f.read())
+        raw = f.read()
         # might not exist yet...
-        crossref_importer_existing.process_source([json.dumps(raw)])
-        # ok, make sure we get 'None' back
-        assert crossref_importer_existing.parse_crossref_dict(raw) is None
+        crossref_importer_existing.push_record(json.loads(raw))
+        crossref_importer_existing.finish()
+        # make sure we wouldn't insert again
+        entity = crossref_importer_existing.parse_record(json.loads(raw))
+        assert crossref_importer_existing.try_update(entity) is False
diff --git a/python/tests/importer.py b/python/tests/importer.py
index 34efa5d8..9308ba84 100644
--- a/python/tests/importer.py
+++ b/python/tests/importer.py
@@ -1,13 +1,13 @@
 
 
 import pytest
-from fatcat_tools.importers import FatcatImporter
+from fatcat_tools.importers import CrossrefImporter, OrcidImporter
 from fixtures import api
 
 
 def test_issnl_mapping_lookup(api):
     with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
-        fi = FatcatImporter(api, issn_map_file=issn_file)
+        fi = CrossrefImporter(api, issn_map_file=issn_file)
 
     assert fi.issn2issnl('0000-0027') == '0002-0027'
     assert fi.issn2issnl('0002-0027') == '0002-0027'
@@ -18,20 +18,18 @@ def test_issnl_mapping_lookup(api):
 def test_identifiers(api):
 
     with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
-        fi = FatcatImporter(api, issn_map_file=issn_file)
-
-    assert fi.is_issnl("1234-5678") == True
-    assert fi.is_issnl("1234-5678.") == False
-    assert fi.is_issnl("12345678") == False
-    assert fi.is_issnl("1-2345678") == False
-
-    assert fi.is_doi("10.1234/56789") == True
-    assert fi.is_doi("101234/56789") == False
-    assert fi.is_doi("10.1234_56789") == False
-
-    assert fi.is_orcid("0000-0003-3118-6591") == True
-    assert fi.is_orcid("0000-0003-3953-765X") == True
-    assert fi.is_orcid("0000-00x3-3118-659") == False
-    assert fi.is_orcid("0000-00033118-659") == False
-    assert fi.is_orcid("0000-0003-3118-659.") == False
+        ci = CrossrefImporter(api, issn_map_file=issn_file)
+
+    assert ci.is_issnl("1234-5678") == True
+    assert ci.is_issnl("1234-5678.") == False
+    assert ci.is_issnl("12345678") == False
+    assert ci.is_issnl("1-2345678") == False
+
+    oi = OrcidImporter(api)
+
+    assert oi.is_orcid("0000-0003-3118-6591") == True
+    assert oi.is_orcid("0000-0003-3953-765X") == True
+    assert oi.is_orcid("0000-00x3-3118-659") == False
+    assert oi.is_orcid("0000-00033118-659") == False
+    assert oi.is_orcid("0000-0003-3118-659.") == False