From ba7f9214d2038882952eb50cd4dc5eff4eb0e6ff Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 9 Nov 2021 18:10:35 -0800 Subject: remove deprecated extid sqlite3 lookup table feature from importers This was used during initial bulk imports, but is no longer used and could create serious metadata problems if used accidentially. In retrospect, it also made metadata provenance less transparent, and may have done more harm than good overall. --- python/tests/import_crossref.py | 8 ++------ python/tests/import_datacite.py | 2 -- python/tests/import_jalc.py | 8 ++------ python/tests/import_jstor.py | 8 ++------ python/tests/import_pubmed.py | 2 -- 5 files changed, 6 insertions(+), 22 deletions(-) (limited to 'python/tests') diff --git a/python/tests/import_crossref.py b/python/tests/import_crossref.py index eb931eb1..5f38e73e 100644 --- a/python/tests/import_crossref.py +++ b/python/tests/import_crossref.py @@ -10,17 +10,13 @@ from fatcat_tools.importers import CrossrefImporter, JsonLinePusher @pytest.fixture(scope="function") def crossref_importer(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield CrossrefImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True - ) + yield CrossrefImporter(api, issn_file, bezerk_mode=True) @pytest.fixture(scope="function") def crossref_importer_existing(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield CrossrefImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False - ) + yield CrossrefImporter(api, issn_file, bezerk_mode=False) @pytest.mark.skip( diff --git a/python/tests/import_datacite.py b/python/tests/import_datacite.py index 220dc0f6..b15d14c3 100644 --- a/python/tests/import_datacite.py +++ b/python/tests/import_datacite.py @@ -30,7 +30,6 @@ def datacite_importer(api): yield DataciteImporter( api, issn_file, - extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True, ) @@ -41,7 +40,6 @@ def datacite_importer_existing(api): yield DataciteImporter( api, issn_file, - extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False, ) diff --git a/python/tests/import_jalc.py b/python/tests/import_jalc.py index 4ebc87b4..8281b9a1 100644 --- a/python/tests/import_jalc.py +++ b/python/tests/import_jalc.py @@ -8,17 +8,13 @@ from fatcat_tools.importers import Bs4XmlFilePusher, Bs4XmlLinesPusher, JalcImpo @pytest.fixture(scope="function") def jalc_importer(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield JalcImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True - ) + yield JalcImporter(api, issn_file, bezerk_mode=True) @pytest.fixture(scope="function") def jalc_importer_existing(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield JalcImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False - ) + yield JalcImporter(api, issn_file, bezerk_mode=False) def test_jalc_importer(jalc_importer): diff --git a/python/tests/import_jstor.py b/python/tests/import_jstor.py index 8ad550b3..7e13c8b0 100644 --- a/python/tests/import_jstor.py +++ b/python/tests/import_jstor.py @@ -8,17 +8,13 @@ from fatcat_tools.importers import Bs4XmlFilePusher, JstorImporter @pytest.fixture(scope="function") def jstor_importer(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield JstorImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True - ) + yield JstorImporter(api, issn_file, bezerk_mode=True) @pytest.fixture(scope="function") def jstor_importer_existing(api): with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file: - yield JstorImporter( - api, issn_file, extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False - ) + yield JstorImporter(api, issn_file, bezerk_mode=False) def test_jstor_importer(jstor_importer): diff --git a/python/tests/import_pubmed.py b/python/tests/import_pubmed.py index a5301f29..e783db48 100644 --- a/python/tests/import_pubmed.py +++ b/python/tests/import_pubmed.py @@ -11,7 +11,6 @@ def pubmed_importer(api): yield PubmedImporter( api, issn_file, - extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=True, lookup_refs=True, ) @@ -23,7 +22,6 @@ def pubmed_importer_existing(api): yield PubmedImporter( api, issn_file, - extid_map_file="tests/files/example_map.sqlite3", bezerk_mode=False, lookup_refs=True, ) -- cgit v1.2.3 From 6eaf4f57c1f92b6f4f46adc38e5b39fd30b65d81 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 10 Nov 2021 14:15:23 -0800 Subject: update datacite tests for license slug changes Use datacite-specific wrapper function, and remove a couple non-OA/TDM-limited licenses. --- python/tests/files/datacite/datacite_result_00.json | 3 +-- python/tests/import_datacite.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'python/tests') diff --git a/python/tests/files/datacite/datacite_result_00.json b/python/tests/files/datacite/datacite_result_00.json index 130a46ff..06147cfc 100644 --- a/python/tests/files/datacite/datacite_result_00.json +++ b/python/tests/files/datacite/datacite_result_00.json @@ -87,6 +87,5 @@ "release_type": "article-journal", "release_year": 2019, "title": "Synthesis and Crystal Structure of a Compound with Two Conformational Isomers: N-(2-methylbenzoyl)-N′-(4-nitrophenyl)thiourea", - "volume": "38", - "license_slug": "SPRINGER-TDM" + "volume": "38" } diff --git a/python/tests/import_datacite.py b/python/tests/import_datacite.py index b15d14c3..28884cda 100644 --- a/python/tests/import_datacite.py +++ b/python/tests/import_datacite.py @@ -15,9 +15,9 @@ from fatcat_tools.importers import DataciteImporter, JsonLinePusher from fatcat_tools.importers.datacite import ( clean_doi, contributor_list_contains_contributor, + datacite_lookup_license_slug, find_original_language_title, index_form_to_display_name, - lookup_license_slug, parse_datacite_dates, parse_datacite_titles, ) @@ -463,9 +463,9 @@ def test_lookup_license_slug(): Case("http://creativecommons.org/licenses/by-nd/4.0/legalcode", "CC-BY-ND"), Case("http://creativecommons.org/licenses/by/2.0/uk/legalcode", "CC-BY"), Case("http://creativecommons.org/publicdomain/zero/1.0/legalcode", "CC-0"), - Case("http://doi.wiley.com/10.1002/tdm_license_1.1", "WILEY-TDM-1.1"), + Case("http://doi.wiley.com/10.1002/tdm_license_1.1", None), Case("http://homepage.data-planet.com/terms-use", "SAGE-DATA-PLANET"), - Case("http://www.springer.com/tdm", "SPRINGER-TDM"), + Case("http://www.springer.com/tdm", None), Case( "https://archaeologydataservice.ac.uk/advice/termsOfUseAndAccess.xhtml", "ADS-UK", @@ -477,11 +477,11 @@ def test_lookup_license_slug(): Case("https://www.elsevier.com/tdm/userlicense/1.0", "ELSEVIER-USER-1.0"), Case("https://www.gnu.org/licenses/gpl-3.0.html", "GPL-3.0"), Case("http://rightsstatements.org/page/InC/1.0?language=en", "RS-INC"), - Case("http://onlinelibrary.wiley.com/termsAndConditions", "WILEY"), + Case("http://onlinelibrary.wiley.com/termsAndConditions", None), Case("https://publikationen.bibliothek.kit.edu/kitopen-lizenz", "KIT-OPEN"), Case( "http://journals.sagepub.com/page/policies/text-and-data-mining-license", - "SAGE-TDM", + None, ), Case( "https://creativecommons.org/publicdomain/mark/1.0/deed.de", @@ -506,7 +506,7 @@ def test_lookup_license_slug(): ] for c in cases: - got = lookup_license_slug(c.input) + got = datacite_lookup_license_slug(c.input) assert c.output == got, "{}: got {}, want {}".format(c.input, got, c.output) -- cgit v1.2.3