summaryrefslogtreecommitdiffstats
path: root/python/tests/import_crossref.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2021-11-02 18:13:14 -0700
committerBryan Newbold <bnewbold@robocracy.org>2021-11-02 18:13:14 -0700
commitcdfd6b85b386b7bbf9d5a5179ef26970b6e5a4e7 (patch)
tree5e4034027b51f3ee4d2a488bb2cbb7a75c3bd0d8 /python/tests/import_crossref.py
parent78f08280edea4ff65ca613ad30005c45cc48dea6 (diff)
downloadfatcat-cdfd6b85b386b7bbf9d5a5179ef26970b6e5a4e7.tar.gz
fatcat-cdfd6b85b386b7bbf9d5a5179ef26970b6e5a4e7.zip
fmt (black): tests/
Diffstat (limited to 'python/tests/import_crossref.py')
-rw-r--r--python/tests/import_crossref.py123
1 files changed, 76 insertions, 47 deletions
diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py
index a8a1ec44..eb931eb1 100644
--- a/python/tests/import_crossref.py
+++ b/python/tests/import_crossref.py
@@ -1,4 +1,3 @@
-
import gzip
import json
@@ -10,67 +9,79 @@ from fatcat_tools.importers import CrossrefImporter, JsonLinePusher
@pytest.fixture(scope="function")
def crossref_importer(api):
- with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
- yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=True)
+ with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file:
+ yield CrossrefImporter(
+ api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True
+ )
+
@pytest.fixture(scope="function")
def crossref_importer_existing(api):
- with open('tests/files/ISSN-to-ISSN-L.snip.txt', 'r') as issn_file:
- yield CrossrefImporter(api, issn_file, extid_map_file='tests/files/example_map.sqlite3', bezerk_mode=False)
+ with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file:
+ yield CrossrefImporter(
+ api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False
+ )
+
-@pytest.mark.skip(reason="slow/huge crossref import is a corner-case and slows tests significantly")
+@pytest.mark.skip(
+ reason="slow/huge crossref import is a corner-case and slows tests significantly"
+)
def test_crossref_importer_huge(crossref_importer):
last_index = crossref_importer.api.get_changelog(limit=1)[0].index
- with gzip.open('tests/files/huge_crossref_doi.json.gz', 'rt') as f:
+ with gzip.open("tests/files/huge_crossref_doi.json.gz", "rt") as f:
crossref_importer.bezerk_mode = True
line = f.readline()
mega_blob = [line for i in range(95)]
counts = JsonLinePusher(crossref_importer, mega_blob).run()
- assert counts['insert'] == 95
- change = crossref_importer.api.get_changelog_entry(index=last_index+1)
+ assert counts["insert"] == 95
+ change = crossref_importer.api.get_changelog_entry(index=last_index + 1)
release = crossref_importer.api.get_release(change.editgroup.edits.releases[0].ident)
assert len(release.contribs) == 1014
+
def test_crossref_importer(crossref_importer):
last_index = crossref_importer.api.get_changelog(limit=1)[0].index
- with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
+ with open("tests/files/crossref-works.2018-01-21.badsample.json", "r") as f:
crossref_importer.bezerk_mode = True
counts = JsonLinePusher(crossref_importer, f).run()
- assert counts['insert'] == 14
- assert counts['exists'] == 0
- assert counts['skip'] == 0
+ assert counts["insert"] == 14
+ assert counts["exists"] == 0
+ assert counts["skip"] == 0
# fetch most recent editgroup
- change = crossref_importer.api.get_changelog_entry(index=last_index+1)
+ change = crossref_importer.api.get_changelog_entry(index=last_index + 1)
eg = change.editgroup
assert eg.description
assert "crossref" in eg.description.lower()
- assert eg.extra['git_rev']
- assert "fatcat_tools.CrossrefImporter" in eg.extra['agent']
+ assert eg.extra["git_rev"]
+ assert "fatcat_tools.CrossrefImporter" in eg.extra["agent"]
last_index = crossref_importer.api.get_changelog(limit=1)[0].index
- with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
+ with open("tests/files/crossref-works.2018-01-21.badsample.json", "r") as f:
crossref_importer.bezerk_mode = False
crossref_importer.reset()
counts = JsonLinePusher(crossref_importer, f).run()
- assert counts['insert'] == 0
- assert counts['exists'] == 14
- assert counts['skip'] == 0
+ assert counts["insert"] == 0
+ assert counts["exists"] == 14
+ assert counts["skip"] == 0
assert last_index == crossref_importer.api.get_changelog(limit=1)[0].index
+
def test_crossref_mappings(crossref_importer):
- assert crossref_importer.map_release_type('journal-article') == "article-journal"
- assert crossref_importer.map_release_type('asdf') is None
- assert crossref_importer.map_release_type('book-series') is None
- assert crossref_importer.map_release_type('standard') == 'standard'
+ assert crossref_importer.map_release_type("journal-article") == "article-journal"
+ assert crossref_importer.map_release_type("asdf") is None
+ assert crossref_importer.map_release_type("book-series") is None
+ assert crossref_importer.map_release_type("standard") == "standard"
+
def test_crossref_importer_create(crossref_importer):
crossref_importer.create_containers = True
- with open('tests/files/crossref-works.2018-01-21.badsample.json', 'r') as f:
+ with open("tests/files/crossref-works.2018-01-21.badsample.json", "r") as f:
JsonLinePusher(crossref_importer, f).run()
+
def test_crossref_dict_parse(crossref_importer):
- with open('tests/files/crossref-works.single.json', 'r') as f:
+ with open("tests/files/crossref-works.single.json", "r") as f:
# not a single line
raw = json.loads(f.read())
r = crossref_importer.parse_record(raw)
@@ -78,7 +89,10 @@ def test_crossref_dict_parse(crossref_importer):
JsonLinePusher(crossref_importer, [json.dumps(raw)]).run()
print(r.extra)
- assert r.title == "Renormalized perturbation theory by the moment method for degenerate states: Anharmonic oscillators"
+ assert (
+ r.title
+ == "Renormalized perturbation theory by the moment method for degenerate states: Anharmonic oscillators"
+ )
assert r.publisher == "Wiley-Blackwell"
assert r.release_type == "article-journal"
assert r.release_stage == "published"
@@ -88,22 +102,22 @@ def test_crossref_dict_parse(crossref_importer):
assert r.ext_ids.isbn13 == "978-3-16-148410-0"
assert r.language == "fr"
assert r.subtitle is None
- assert 'subtitle' not in r.extra
- assert 'subtitle' not in r.extra['crossref']
- assert 'funder' not in r.extra
- assert 'funder' not in r.extra['crossref']
+ assert "subtitle" not in r.extra
+ assert "subtitle" not in r.extra["crossref"]
+ assert "funder" not in r.extra
+ assert "funder" not in r.extra["crossref"]
# matched by ISSN, so shouldn't be in there
- #assert extra['container_name'] == "International Journal of Quantum Chemistry"
- assert r.extra['aliases'] == ["some other title"]
- assert r.extra['crossref']['archive'] == ['Portico', 'LOCKSS']
+ # assert extra['container_name'] == "International Journal of Quantum Chemistry"
+ assert r.extra["aliases"] == ["some other title"]
+ assert r.extra["crossref"]["archive"] == ["Portico", "LOCKSS"]
assert len(r.contribs) == 6
assert r.contribs[0].raw_name == "Marcelo D. Radicioni"
assert r.contribs[0].given_name == "Marcelo D."
assert r.contribs[0].surname == "Radicioni"
assert r.contribs[0].index == 0
- assert r.contribs[0].extra['seq'] == "first"
+ assert r.contribs[0].extra["seq"] == "first"
assert r.contribs[1].raw_affiliation == "Some University"
- assert r.contribs[1].extra['more_affiliations'] == ["Some Department"]
+ assert r.contribs[1].extra["more_affiliations"] == ["Some Department"]
assert r.contribs[1].role == "author"
assert r.contribs[4].role == "editor"
assert r.contribs[4].index is None
@@ -115,32 +129,47 @@ def test_crossref_dict_parse(crossref_importer):
assert r.refs[0].year == 1972
assert r.refs[0].locator == "1734"
assert r.refs[0].container_name == "J. Chem. Phys."
- assert r.refs[0].extra == {"volume": "57", "authors": ["Swenson"], "doi": "10.1063/1.1678462", "medium": "DVD"}
- assert r.refs[2].key == 'BIB3'
- assert r.refs[2].extra.get('author') is None
- assert r.refs[2].container_name == "Hypervirial Theorem's, Lecture Notes in Chemistry <3"
- assert r.refs[3].container_name == "Large Order Perturbation Theory and Summation Methods in Quantum Mechanics, Lecture Notes in Chemistry"
+ assert r.refs[0].extra == {
+ "volume": "57",
+ "authors": ["Swenson"],
+ "doi": "10.1063/1.1678462",
+ "medium": "DVD",
+ }
+ assert r.refs[2].key == "BIB3"
+ assert r.refs[2].extra.get("author") is None
+ assert (
+ r.refs[2].container_name == "Hypervirial Theorem's, Lecture Notes in Chemistry <3"
+ )
+ assert (
+ r.refs[3].container_name
+ == "Large Order Perturbation Theory and Summation Methods in Quantum Mechanics, Lecture Notes in Chemistry"
+ )
+
def test_crossref_subtitle(crossref_importer):
"""
Tests new subtitle field, explicitly
"""
- with open('tests/files/crossref-works.single.json', 'r') as f:
+ with open("tests/files/crossref-works.single.json", "r") as f:
# not a single line
raw = json.loads(f.read())
- raw['subtitle'] = ["some bogus subtitle", "blah"]
+ raw["subtitle"] = ["some bogus subtitle", "blah"]
r = crossref_importer.parse_record(raw)
# ensure the API server is ok with format
JsonLinePusher(crossref_importer, [json.dumps(raw)]).run()
print(r.extra)
- assert r.title == "Renormalized perturbation theory by the moment method for degenerate states: Anharmonic oscillators"
+ assert (
+ r.title
+ == "Renormalized perturbation theory by the moment method for degenerate states: Anharmonic oscillators"
+ )
assert r.subtitle == "some bogus subtitle"
- assert 'subtitle' not in r.extra
- assert 'subtitle' not in r.extra['crossref']
+ assert "subtitle" not in r.extra
+ assert "subtitle" not in r.extra["crossref"]
+
def test_stateful_checking(crossref_importer_existing):
- with open('tests/files/crossref-works.single.json', 'r') as f:
+ with open("tests/files/crossref-works.single.json", "r") as f:
# not a single line, a whole document
raw = f.read()
# might not exist yet...