aboutsummaryrefslogtreecommitdiffstats
path: root/python/tests
diff options
context:
space:
mode:
Diffstat (limited to 'python/tests')
-rw-r--r--python/tests/files/example_doaj_articles.json5
-rw-r--r--python/tests/files/example_ingest.json2
-rw-r--r--python/tests/files/example_ingest_html.json1
-rw-r--r--python/tests/files/example_ingest_xml.json1
-rw-r--r--python/tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json1
-rw-r--r--python/tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json1
-rw-r--r--python/tests/import_doaj.py142
-rw-r--r--python/tests/import_ingest.py68
-rw-r--r--python/tests/transform_elasticsearch.py95
9 files changed, 302 insertions, 14 deletions
diff --git a/python/tests/files/example_doaj_articles.json b/python/tests/files/example_doaj_articles.json
new file mode 100644
index 00000000..018a4800
--- /dev/null
+++ b/python/tests/files/example_doaj_articles.json
@@ -0,0 +1,5 @@
+{"last_updated":"2020-02-04T14:11:44Z","bibjson":{"identifier":[{"id":"0264-1275","type":"pissn"},{"id":"10.1016/j.matdes.2016.06.110","type":"DOI"}],"journal":{"volume":"108","number":"","country":"GB","license":[{"open_access":true,"title":"CC BY-NC-ND","type":"CC BY-NC-ND","url":"https://www.elsevier.com/journals/materials-and-design/0264-1275/open-access-journal"}],"issns":["0264-1275","1873-4197"],"publisher":"Elsevier","language":["EN"],"title":"Materials & Design"},"month":"10","end_page":"617","year":"2016","start_page":"608","subject":[{"code":"TA401-492","scheme":"LCC","term":"Materials of engineering and construction. Mechanics of materials"}],"author":[{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Xinfeng Li"},{"affiliation":"Department of Geosciences, Center for Materials by Design, State University of New York, Stony Brook, NY 11794-2100, USA","name":"Jin Zhang"},{"affiliation":"School of Chemical Engineering & Technology, China University of Mining and Technology, Xuzhou 221116, China","name":"Yanfei Wang"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China","name":"Sicong Shen"},{"affiliation":"State Key Laboratory for Mechanical Behavior of Materials, School of Materials Science and Engineering, Xi'an Jiaotong University, Xi'an 710049, China; Corresponding author.","name":"Xiaolong Song"}],"link":[{"type":"fulltext","url":"http://www.sciencedirect.com/science/article/pii/S0264127516308723"}],"abstract":"The tensile properties and fracture behavior of PH 13-8 Mo steel after subjected to pre-charged hydrogen were investigated by slow strain rate tensile tests. The results suggest that hydrogen slightly increases yield strength, while decreases tensile strength. The susceptibility to hydrogen embrittlement of specimens aged at 650 °C firstly reduces and then increases as the aging time increases, reaching the lowest value at aging time 4 h. This is dominantly attributed to the highest content of austenite. Moreover, hydrogen-induced crack nucleation sites initiate from lath, packet and prior austenite grain boundaries. Crack propagation passes through lath boundaries and walks along packet, prior austenite grain boundaries. Scanning electron microscopy result indicates that hydrogen-charged specimens show quasi-cleavage fracture and intergranular fracture in annular brittle zone while dimple fracture is observed in hydrogen-free specimens. Keywords: Hydrogen embrittlement, PH 13-8 Mo steel, Aging time, Fracture behavior","title":"Effect of hydrogen on tensile properties and fracture behavior of PH 13-8 Mo steel"},"created_date":"2019-06-05T05:25:15Z","id":"e58f08a11ecb495ead55a44ad4f89808"}
+{"last_updated":"2020-02-04T08:06:42Z","bibjson":{"identifier":[{"id":"2072-6694","type":"eissn"},{"id":"10.3390/cancers9080107","type":"doi"}],"journal":{"volume":"9","number":"8","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/cancers/about"}],"issns":["2072-6694"],"publisher":"MDPI AG","language":["EN"],"title":"Cancers"},"month":"8","keywords":["ALK rearrangement, lung cancer, biology, immunohistochemistry, FISH, molecular biology."],"year":"2017","start_page":"107","subject":[{"code":"RC254-282","scheme":"LCC","term":"Neoplasms. Tumors. Oncology. Including cancer and carcinogens"}],"author":[{"affiliation":"Laboratory of Clinical and Experimental Pathology, Pasteur Hospital, 30 avenue de la voie romaine, 06001 Nice cedex 01, France","name":"Paul Hofman"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/2072-6694/9/8/107"}],"abstract":"Patients with advanced-stage non-small cell lung carcinoma (NSCLC) harboring an ALK rearrangement, detected from a tissue sample, can benefit from targeted ALK inhibitor treatment. Several increasingly effective ALK inhibitors are now available for treatment of patients. However, despite an initial favorable response to treatment, in most cases relapse or progression occurs due to resistance mechanisms mainly caused by mutations in the tyrosine kinase domain of ALK. The detection of an ALK rearrangement is pivotal and can be done using different methods, which have variable sensitivity and specificity depending, in particular, on the quality and quantity of the patient’s sample. This review will first highlight briefly some information regarding the pathobiology of an ALK rearrangement and the epidemiology of patients harboring this genomic alteration. The different methods used to detect an ALK rearrangement as well as their advantages and disadvantages will then be examined and algorithms proposed for detection in daily routine practice.","title":"ALK in Non-Small Cell Lung Cancer (NSCLC) Pathobiology, Epidemiology, Detection from Tumor Tissue and Algorithm Diagnosis in a Daily Practice"},"admin":{"seal":true},"created_date":"2018-10-26T07:49:34Z","id":"937c7aa790e048d4ae5f53a2ad71f0dc"}
+{"last_updated":"2020-02-04T13:43:13Z","bibjson":{"identifier":[{"id":"1178-2013","type":"pissn"}],"end_page":"818","keywords":["bioconjugation","biosurfactant","cancer therapy","folic acid receptor","graphene quantum dots","theranostic tool"],"year":"2019","subject":[{"code":"R5-920","scheme":"LCC","term":"Medicine (General)"}],"author":[{"name":"Bansal S"},{"name":"Singh J"},{"name":"Kumari U"},{"name":"Kaur IP"},{"name":"Barnwal RP"},{"name":"Kumar R"},{"name":"Singh S"},{"name":"Singh G"},{"name":"Chatterjee M"}],"link":[{"content_type":"html","type":"fulltext","url":"https://www.dovepress.com/development-of-biosurfactant-based-graphene-quantum-dot-conjugate-as-a-peer-reviewed-article-IJN"}],"abstract":"Smriti Bansal,1 Joga Singh,2 Uma Kumari,3 Indu Pal Kaur,2 Ravi Pratap Barnwal,4 Ravinder Kumar,3 Suman Singh,5 Gurpal Singh,2 Mary Chatterjee1 1Biotechnology Engineering, University Institute of Engineering & Technology, Panjab University, Chandigarh, India; 2Department of Pharmaceutical Sciences, University Institute of Pharmaceutical Sciences, Panjab University, Chandigarh, India; 3Department of Zoology, Panjab University, Chandigarh, India; 4Department of Biophysics, Panjab University, Chandigarh, India; 5Department of Agronomics, Central Scientific Instruments Organisation, Chandigarh, India Background: Biosurfactants are amphipathic molecules of microbial origin that reduce surface and interfacial tension at gas–liquid–solid interfaces. Earlier, the biosurfactant was isolated and characterized in our laboratory from Candida parapsilosis. The property of the biosurfactant is further explored in this study by using quantum dots (QDs) as nanocarrier.Materials and methods: Graphene quantum dots (GQDs) were synthesized by bottom-up approach through pyrolysis of citric acid. GQDs were conjugated with both biosurfactant and folic acid (FA) using carbodiimide chemistry. The prepared GQD bioconjugate was studied for diagnostic and therapeutic effects against cancer cells.Results and discussion: Photoluminescence quantum yield (QY) of plain GQDs was measured as 12.8%. QY for biosurfactant conjugated GQDs and FA-biosurfactant conjugated GQDs was measured as 10.4% and 9.02%, respectively, and it was sufficient for targeting cancer cells. MTT assay showed that more than 90% of cells remained viable at concentration of 1 mg/mL, hence GQDs seemed to be non-toxic to cells. Biosurfactant conjugated GQDs caused 50% reduction in cellular viability within 24 hours. FA conjugation further increased the specificity of bioconjugated GQDs toward tumor cells, which is clearly evident from the drug internalization studies using confocal laser scanning microscopy. A higher amount of drug uptake was observed when bioconjugated GQDs were decorated with FA.Conclusion: The ability of GQD bioconjugate could be used as a theranostic tool for cancer. It is foreseen that in near future cancer can be detected and/or treated at an early stage by utilizing biosurfactant conjugated GQDs. Therefore, the proposed study would provide a stepping stone to improve the life of cancer patients. Keywords: bioconjugation, nanomedicine, nanocarrier, cancer therapy, folic acid receptor, graphene quantum dots","title":"Development of biosurfactant-based graphene quantum dot conjugate as a novel and fluorescent theranostic tool for cancer","journal":{"volume":"Volume 14","country":"GB","license":[{"open_access":true,"title":"CC BY-NC","type":"CC BY-NC","url":"https://www.dovepress.com/author_guidelines.php?content_id=695"}],"issns":["1176-9114","1178-2013"],"publisher":"Dove Medical Press","language":["EN"],"title":"International Journal of Nanomedicine"},"month":"1","start_page":"809"},"created_date":"2019-01-29T18:43:40Z","id":"e0173c80437f4fb88ec4e02e453e13b0"}
+{"last_updated":"2020-02-04T09:46:14Z","bibjson":{"identifier":[{"id":"1424-8220","type":"eissn"},{"id":"10.3390/s18124467","type":"doi"}],"journal":{"volume":"18","number":"12","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/journal/sensors/about"}],"issns":["1424-8220"],"publisher":"MDPI AG","language":["EN"],"title":"Sensors"},"month":"12","keywords":["multilayer sea ice temperature","low temperature","design","performance analysis"],"year":"2018","start_page":"4467","subject":[{"code":"TP1-1185","scheme":"LCC","term":"Chemical technology"}],"author":[{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Guangyu Zuo"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yinke Dou"},{"affiliation":"College of Water Resources Science and Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Xiaomin Chang"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Yan Chen"},{"affiliation":"College of Electrical and Power Engineering, Taiyuan University of Technology, Taiyuan 030024, China","name":"Chunyan Ma"}],"link":[{"content_type":"pdf","type":"fulltext","url":"https://www.mdpi.com/1424-8220/18/12/4467"}],"abstract":"Temperature profiles of sea ice have been recorded more than a few decades. However, few high-precision temperature sensors can complete the observation of temperature profile of sea ice, especially in extreme environments. At present, the most widely used sea ice observation instruments can reach an accuracy of sea ice temperature measurement of 0.1 °C. In this study, a multilayer sea ice temperature sensor is developed with temperature measurement accuracy from −0.0047 °C to 0.0059 °C. The sensor system composition, structure of the thermistor string, and work mode are analyzed. The performance of the sensor system is evaluated from −50 °C to 30 °C. The temperature dependence of the constant current source, the amplification circuit, and the analog-to-digital converter (ADC) circuit are comprehensive tested and quantified. A temperature correction algorithm is designed to correct any deviation in the sensor system. A sea-ice thickness discrimination algorithm is proposed in charge of determining the thickness of sea ice automatically. The sensor system was field tested in Wuliangsuhai, Yellow River on 31 January 2018 and the second reservoir of Fen River, Yellow River on 30 January 2018. The integral practicality of this sensor system is identified and examined. The multilayer sea ice temperature sensor will provide good temperature results of sea ice and maintain stable performance in the low ambient temperature.","title":"Design and Performance Analysis of a Multilayer Sea Ice Temperature Sensor Used in Polar Region"},"admin":{"seal":true},"created_date":"2018-12-18T08:13:29Z","id":"152f83d12b9f477696e681684ba696e7"}
+{"last_updated":"2020-06-02T23:02:32Z","bibjson":{"identifier":[{"id":"10.123/abc","type":"doi"},{"id":"2076-3417","type":"eissn"}],"journal":{"volume":"10","number":"3872","country":"CH","license":[{"open_access":true,"title":"CC BY","type":"CC BY","url":"http://www.mdpi.com/about/openaccess"}],"issns":["2076-3417"],"publisher":"MDPI AG","language":["EN"],"title":"Applied Sciences"},"month":"06","keywords":["Smart parking systems","survey","vehicle routing problem","vehicle detection techniques","routing algorithms"],"year":"2020","start_page":"3872","subject":[{"code":"T","scheme":"LCC","term":"Technology"},{"code":"TA1-2040","scheme":"LCC","term":"Engineering (General). Civil engineering (General)"},{"code":"QH301-705.5","scheme":"LCC","term":"Biology (General)"},{"code":"QC1-999","scheme":"LCC","term":"Physics"},{"code":"QD1-999","scheme":"LCC","term":"Chemistry"}],"author":[{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Mathias Gabriel Diaz Ogás"},{"affiliation":"Institute of Informatics and Applications. University of Girona, 17003 Girona, Spain","name":"Ramon Fabregat"},{"affiliation":"Institute of Computer Science. Faculty of Exact, Physical and Natural Sciences. National University of San Juan, 5400 San Juan, Argentina","name":"Silvana Aciar"}],"link":[{"content_type":"text/html","type":"fulltext","url":"https://www.mdpi.com/2076-3417/10/11/3872"}],"abstract":"The large number of vehicles constantly seeking access to congested areas in cities means that finding a public parking place is often difficult and causes problems for drivers and citizens alike. In this context, strategies that guide vehicles from one point to another, looking for the most optimal path, are needed. Most contributions in the literature are routing strategies that take into account different criteria to select the optimal route required to find a parking space. This paper aims to identify the types of smart parking systems (SPS) that are available today, as well as investigate the kinds of vehicle detection techniques (VDT) they have and the algorithms or other methods they employ, in order to analyze where the development of these systems is at today. To do this, a survey of 274 publications from January 2012 to December 2019 was conducted. The survey considered four principal features: SPS types reported in the literature, the kinds of VDT used in these SPS, the algorithms or methods they implement, and the stage of development at which they are. Based on a search and extraction of results methodology, this work was able to effectively obtain the current state of the research area. In addition, the exhaustive study of the studies analyzed allowed for a discussion to be established concerning the main difficulties, as well as the gaps and open problems detected for the SPS. The results shown in this study may provide a base for future research on the subject.","title":"Survey of Smart Parking Systems"},"admin":{"seal":true},"id":"9cf511bab39445ba9745feb43d7493dd","created_date":"2020-06-03T00:02:28Z"}
diff --git a/python/tests/files/example_ingest.json b/python/tests/files/example_ingest.json
index cea67fa7..a9791587 100644
--- a/python/tests/files/example_ingest.json
+++ b/python/tests/files/example_ingest.json
@@ -1,2 +1,2 @@
-{"file_meta": {"sha1hex": "00242a192acc258bdfdb151943419437f440c313", "md5hex": "f4de91152c7ab9fdc2a128f962faebff", "sha256hex": "ffc1005680cb620eec4c913437dfabbf311b535cfe16cbaeb2faec1f92afc362", "size_bytes": 255629, "mimetype": "application/pdf"}, "request": {"ingest_request_source": "fatcat-changelog", "link_source": "doi", "link_source_id":"10.123/abc","ext_ids": {"doi": "10.123/abc"}}, "terminal": {"terminal_url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable", "terminal_dt": "20170227164644", "terminal_sha1hex": "00242a192acc258bdfdb151943419437f440c313"}, "cdx": { "datetime": "20170227164644", "url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable" }, "grobid": {"status_code": 200 }, "hit": true, "status": "success"}
+{"file_meta": {"sha1hex": "00242a192acc258bdfdb151943419437f440c313", "md5hex": "f4de91152c7ab9fdc2a128f962faebff", "sha256hex": "ffc1005680cb620eec4c913437dfabbf311b535cfe16cbaeb2faec1f92afc362", "size_bytes": 255629, "mimetype": "application/pdf"}, "request": {"ingest_type": "pdf", "ingest_request_source": "fatcat-changelog", "link_source": "doi", "link_source_id":"10.123/abc","ext_ids": {"doi": "10.123/abc"}}, "terminal": {"terminal_url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable", "terminal_dt": "20170227164644", "terminal_sha1hex": "00242a192acc258bdfdb151943419437f440c313"}, "cdx": { "datetime": "20170227164644", "url": "http://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.0020124&type=printable" }, "grobid": {"status_code": 200 }, "hit": true, "status": "success"}
{"request":{"ingest_type":"pdf","ingest_request_source":"fatcat-changelog","base_url":"https://doi.org/10.3917/popav.748.0017","release_stage":"published","fatcat":{"release_ident":"weeqjkvsx5abze2bhithyrx6wu","work_ident":"ujatsk25yrdw5gofubw7nogzgq"},"ext_ids":{"doi":"10.3917/popav.748.0017"},"link_source":"doi","link_source_id":"10.3917/popav.748.0017"},"hit":false,"hops":["https://doi.org/10.3917/popav.748.0017"],"status":"wayback-error","error_message":"replay fetch didn't return X-Archive-Src in headers"}
diff --git a/python/tests/files/example_ingest_html.json b/python/tests/files/example_ingest_html.json
new file mode 100644
index 00000000..6c646814
--- /dev/null
+++ b/python/tests/files/example_ingest_html.json
@@ -0,0 +1 @@
+{"cdx": {"datetime": "20200708025309", "mimetype": "text/html", "sha1b32": "THJFFZJR2VYN2FAR7X7LHFGRU2X5IC2U", "sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "status_code": 200, "surt": "py,una,iics,scielo)/scielo.php?lng=en&nrm=iso&pid=s1683-98032015000200002&script=sci_arttext&tlng=es", "url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es", "warc_csize": 13123, "warc_offset": 77579308, "warc_path": "SCIELO-CRAWL-2020-07-20200707211940442-00279-00347-wbgrp-svc206/SCIELO-CRAWL-2020-07-20200708024511243-00332-13069~wbgrp-svc206.us.archive.org~8443.warc.gz"}, "file_meta": {"md5hex": "515a61845a2f898438e3986e4506da8f", "mimetype": "text/html", "sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "sha256hex": "c4559d548476a325891461b71c796beee717e820d6a00cb8411176ce83a0f23f", "size_bytes": 47442}, "hit": true, "hops": ["http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es"], "html_biblio": {"container_issn": "1683-9803", "container_name": "Pediatr\u00eda (Asunci\u00f3n)", "contrib_names": ["Ruiz Valiente, Syntia Carolina", "Ruiz Ca\u00f1ete, Manuel", "Cohene Velazquez, Bartola"], "doi": "10.18004/ped.2015.agosto.102-107", "first_page": "102", "html_fulltext_url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es", "issue": "2", "last_page": "107", "pdf_fulltext_url": "http://scielo.iics.una.py/pdf/ped/v42n2/v42n2a02.pdf", "publisher": "Sociedad Paraguaya de Pediatr\u00eda", "release_date": "2015-08-06", "title": "Prevalence of malnutrition and eating habits in children under 5 years of age in indigenous communities in Azote'y and Yby Yau, 2011", "volume": "42", "xml_fulltext_url": "http://scielo.iics.una.py/scieloOrg/php/articleXML.php?pid=S1683-98032015000200002&lang=en"}, "html_body": {"agent": "trafilatura/0.5.1", "status": "success", "word_count": 3500}, "html_resources": [{"mimetype": "image/gif", "resource_type": "image", "sha1hex": "4991aa771874daf8cba79be38d18d534f946b5d6", "sha256hex": "5e76fad755b873a439dd5e775684696c547008d45cc901606132e9a1ed970757", "size": 220, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/alpha.gif", "timestamp": "2020-10-31T14:07:30", "url": "http://scielo.iics.una.py/img/en/alpha.gif"}, {"mimetype": "text/plain", "resource_type": "script", "sha1hex": "fd28e342fa1b40b84cc17dc66d22df3bf260170b", "sha256hex": "9cf2e81dd65d5a64200970bbd1cd9497b46b2af232e2fbfb79fef95b070f23d1", "size": 3653, "status_code": 200, "surt": "py,una,iics,scielo)/applications/scielo-org/js/toolbox.js", "timestamp": "2020-10-31T20:14:35", "url": "http://scielo.iics.una.py/applications/scielo-org/js/toolbox.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "c5ea6229ce6a97f2dc2b2e2c8ffac26400dfcd58", "sha256hex": "7fb3d59ea14ab060c2b6cbdd5e63d57e158d6cc9e613ceb05ab1e6ec60d64995", "size": 382, "status_code": 200, "surt": "py,una,iics,scielo)/img/common/iconpermalink.gif", "timestamp": "2020-10-31T20:14:52", "url": "http://scielo.iics.una.py/img/common/iconPermalink.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "fbd3488e6b8cd241605fa2db14ba15e0f037d3a7", "sha256hex": "5492829967d521386bec4323f0d7ef951e9a0b16caa1bcd8e75576dc41bd3b55", "size": 26759, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f1.jpg", "timestamp": "2020-07-08T02:53:11", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f1.jpg"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "9f1833948223109dfaca2c37fbdbacb81002a346", "sha256hex": "f5b08a2022fce73ae04c3b9fe368645a084132a942bb29950bba705ed89e6d91", "size": 35440, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02t1.jpg", "timestamp": "2020-07-08T02:53:18", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02t1.jpg"}, {"mimetype": "image/png", "resource_type": "image", "sha1hex": "0d2d329000cba763e5eec45bd8ee2743393ebd62", "sha256hex": "d964eed5974264b8f107a905b74796cb3d5e60f78da1c500bb547a419538915e", "size": 3091, "status_code": 200, "surt": "py,una,iics,scielo)/img/common/icon-close.png", "timestamp": "2020-10-24T10:17:13", "url": "http://scielo.iics.una.py/img/common/icon-close.png"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "5812dc64389992d7d59d10e57449407778bbd0c0", "sha256hex": "605ce931ded871d924f31765c6bbf778eb8b5194b3396f49638a88331f53dc21", "size": 652, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconxmldocument.gif", "timestamp": "2020-10-24T14:58:51", "url": "http://scielo.iics.una.py/img/en/iconXMLDocument.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "f20584095b9c7d06250140bf7f51f7bd91e2ba08", "sha256hex": "f8292c0c25d5eec546fe16e8a53101b4933adb2e75e58d7335158dc94b2bae91", "size": 239, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/artsrc.gif", "timestamp": "2020-10-29T17:34:52", "url": "http://scielo.iics.una.py/img/en/artsrc.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "09fe461f38958a267695edf5675f668323f754ec", "sha256hex": "d0792cfc52df6414126a541e8cd32ba151d75f87225c63d38a9ddad389b913b3", "size": 229, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/subject.gif", "timestamp": "2020-10-28T12:59:36", "url": "http://scielo.iics.una.py/img/en/subject.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "7b2f78593847928d8f0f8a2068b0cb366501c3e5", "sha256hex": "97dfc989c7af7a0139950696e533fe71c373539091200edba96f151efb045f8d", "size": 181, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/grp1c.gif", "timestamp": "2020-10-24T10:17:25", "url": "http://scielo.iics.una.py/img/en/grp1c.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "a1e6d8818d56678a52a18859b0cf919b8663a5aa", "sha256hex": "2d34923f1bb8e417a4c244ba5be13b7fe52e0dc6dba9dbcdf512a9fb3cb84d91", "size": 27383, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f2.jpg", "timestamp": "2020-07-08T02:53:13", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f2.jpg"}, {"mimetype": "text/plain", "resource_type": "stylesheet", "sha1hex": "3754bfd4a8608ec125c79ccc7b62ead02c323bbc", "sha256hex": "4dc9b9edd3fc1e58d7a1c39c64551ac07530bedf0721323fc2c820a90a7b4a64", "size": 87, "status_code": 200, "surt": "py,una,iics,scielo)/css/screen.css", "timestamp": "2020-10-24T11:51:22", "url": "http://scielo.iics.una.py/css/screen.css"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "818ff217eae41fe796f21e9b56336011d8806de0", "sha256hex": "a3853400c16b0628dd226487d1ad7710f44a2e6ea8de85f2b2a6a34b7334d5b6", "size": 210, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/search.gif", "timestamp": "2020-10-28T12:59:22", "url": "http://scielo.iics.una.py/img/en/search.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "a6300c0530bdc13b1bf75a7f380cef6c1be48cc7", "sha256hex": "aa7fa5a5bedea888ddbb89f20838207eb303323c98c414452f632f96acaccbfe", "size": 660, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconemail.gif", "timestamp": "2020-11-01T12:16:11", "url": "http://scielo.iics.una.py/img/en/iconEmail.gif"}, {"mimetype": "image/png", "resource_type": "image", "sha1hex": "100a6b57582fbf383f96c289c92fbbc9aaa63f06", "sha256hex": "f43d4d35e7ac1e815dc0c8897806e30d928ee62e1aa6ac20f49c649f8b694004", "size": 430, "status_code": 200, "surt": "net,licensebuttons)/l/by/4.0/80x15.png", "timestamp": "2020-07-08T21:51:45", "url": "https://licensebuttons.net/l/by/4.0/80x15.png"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "dbfce64d671bbb03591a297983c81ede279b051d", "sha256hex": "2ef85ef9dd7926099287dd33ab43fc6819b393446e85ddb754897ad457c56282", "size": 244, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/prev.gif", "timestamp": "2020-10-29T01:22:04", "url": "http://scielo.iics.una.py/img/en/prev.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "bf5bed62fc6cc82a8a7e862fceac9ce8ffb12cb8", "sha256hex": "1a90de599f61e3191fec24d504798c372b72ccbc511c3c44d48070a0dddefe25", "size": 262, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconrelatedoff.gif", "timestamp": "2020-10-28T12:59:51", "url": "http://scielo.iics.una.py/img/en/iconRelatedOff.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "4a59b2b1d57a210252311d563eea138afcc7a886", "sha256hex": "f96585d38fb34040d9bd81e83538a7beade916bc2d4456e75d5911181281cb6f", "size": 586, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/e-mailt.gif", "timestamp": "2020-10-31T20:53:28", "url": "http://scielo.iics.una.py/img/en/e-mailt.gif"}, {"mimetype": "image/jpeg", "resource_type": "image", "sha1hex": "d465c5da9dea2a6e10b5d340c5af6af6cc10f3ec", "sha256hex": "ffc1411a8185c8df5ca9c0725fbfab41380706d213ca42c8552f32323b67901d", "size": 33992, "status_code": 200, "surt": "py,una,iics,scielo)/img/revistas/ped/v42n2/2a02f3.jpg", "timestamp": "2020-07-08T02:53:19", "url": "http://scielo.iics.una.py/img/revistas/ped/v42n2/2a02f3.jpg"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "86dbd3881975bc15fef15536e5cbd54bad53271c", "sha256hex": "75cbc76c44915b46c6c44fdeeeabd1bfab774ecf692d37ed8d1b4674f5ee583d", "size": 628, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconpdfdocument.gif", "timestamp": "2020-10-24T11:51:12", "url": "http://scielo.iics.una.py/img/en/iconPDFDocument.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "080333f92aa899e53bcd30a4e734d9b36d7ac7a4", "sha256hex": "e6834ac24d48ec9d75b178de59964eb9fb66e9cff05b439ad247f5af5d5fc1ff", "size": 374, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconreferences.gif", "timestamp": "2020-10-29T05:40:05", "url": "http://scielo.iics.una.py/img/en/iconReferences.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "fe879762fd80c756df0af9c81e3424d651fa1b6a", "sha256hex": "4333f6c0ccd89f3240b6c8bb9b2c109792da6d0513e618c35033e2474981b55d", "size": 578, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/icontranslation.gif", "timestamp": "2020-10-28T22:59:05", "url": "http://scielo.iics.una.py/img/en/iconTranslation.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "8846563b9722db2f3b832c03ad2ee9b6318c1d0e", "sha256hex": "6843f628c71f39631ec5d501f6b62506ae9f8454c0a3cd957f4dc67985c371bb", "size": 219, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/author.gif", "timestamp": "2020-10-24T10:16:50", "url": "http://scielo.iics.una.py/img/en/author.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "db35a7e4171a130d25632d6e1ba9c3806eec1e87", "sha256hex": "bd6496501a92a6ed3c5e8c16ce0af4ac9b4cece3562934010d0878f6ea06ead0", "size": 288, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/iconcitedoff.gif", "timestamp": "2020-10-29T17:34:57", "url": "http://scielo.iics.una.py/img/en/iconCitedOff.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "0d844cf48e3ed3849d7e2deed30fb2e7318107b0", "sha256hex": "f257802855722fed0b2b6936a9aede3a4869fc347e91860a26cc81c1ba9df3a3", "size": 164, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/toc.gif", "timestamp": "2020-10-28T22:59:18", "url": "http://scielo.iics.una.py/img/en/toc.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "33e2d3699619eb6dac7c91c207c748599def84f0", "sha256hex": "d8af84c5c4c10e724a081409b0f0e50eb08b9c2cd3d3e0ee0b33cc9eaa20086c", "size": 193, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/next.gif", "timestamp": "2020-11-01T12:16:02", "url": "http://scielo.iics.una.py/img/en/next.gif"}, {"mimetype": "text/html", "resource_type": "script", "sha1hex": "731cf720a546953efe311566a2d874fae715bfc6", "sha256hex": "0d3602fb417d811e15e1a7bd6725384e5bda874dab0eb7be7ee59cd26d64dbd1", "size": 8231, "status_code": 200, "surt": "py,una,iics,scielo)/article.js", "timestamp": "2020-10-29T05:40:31", "url": "http://scielo.iics.una.py/article.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "756779e5ff89d107b2eb4843cc47dd4b63efc829", "sha256hex": "b52a6dc8cbbf4212790cf57af7489b9dc21c040ae7372c07bb6aa18473098759", "size": 190, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/home.gif", "timestamp": "2020-07-06T21:30:58", "url": "http://scielo.iics.una.py/img/en/home.gif"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "e541555caec87c313c9c1859c4b5913b31f955c5", "sha256hex": "64eeb2c0e97f96d9144aa83d027fb5b9d57d96c74681611c39af99d49b148c6e", "size": 643, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/fulltxt.gif", "timestamp": "2020-10-29T22:47:53", "url": "http://scielo.iics.una.py/img/en/fulltxt.gif"}, {"mimetype": "text/plain", "resource_type": "script", "sha1hex": "65cbff4e9d95d47a6f31d96ab4ea361c1f538a7b", "sha256hex": "e23a2a4e2d7c2b41ebcdd8ffc0679df7140eb7f52e1eebabf827a88182643c59", "size": 72174, "status_code": 200, "surt": "py,una,iics,scielo)/applications/scielo-org/js/jquery-1.4.2.min.js", "timestamp": "2020-07-06T21:17:01", "url": "http://scielo.iics.una.py/applications/scielo-org/js/jquery-1.4.2.min.js"}, {"mimetype": "image/gif", "resource_type": "image", "sha1hex": "20368dd206a00eaf8bb117f98291a30eb0cc8e73", "sha256hex": "534434f1716e29928e0376d0e5dc113808c96d9cedab8675adff7dbf22cb9fd1", "size": 1353, "status_code": 200, "surt": "py,una,iics,scielo)/img/en/fbpelogp.gif", "timestamp": "2020-07-06T21:16:38", "url": "http://scielo.iics.una.py/img/en/fbpelogp.gif"}], "request": {"link_source": "doi", "ingest_request_source": "fatcat-changelog", "base_url": "http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "ext_ids": {"doi": "10.123/abc"}, "fatcat": {"release_ident": null}, "ingest_type": "html"}, "scope": "article-fulltext", "status": "success", "terminal": {"terminal_dt": "20200708025309", "terminal_sha1hex": "99d252e531d570dd1411fdfeb394d1a6afd40b54", "terminal_status_code": 200, "terminal_url": "http://scielo.iics.una.py/scielo.php?script=sci_arttext&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=es"}}
diff --git a/python/tests/files/example_ingest_xml.json b/python/tests/files/example_ingest_xml.json
new file mode 100644
index 00000000..2f525998
--- /dev/null
+++ b/python/tests/files/example_ingest_xml.json
@@ -0,0 +1 @@
+{"cdx": {"datetime": "20200710091403", "mimetype": "text/xml", "sha1b32": "PWMQ2L4RHPJ3NVWC66GIJC36L5FXPOM6", "sha1hex": "7d990d2f913bd3b6d6c2f78c848b7e5f4b77b99e", "status_code": 200, "surt": "py,una,iics,scielo)/scieloorg/php/articlexml.php?lang=en&pid=s1683-98032015000200002", "url": "http://scielo.iics.una.py/scieloOrg/php/articleXML.php?pid=S1683-98032015000200002&lang=en", "warc_csize": 12108, "warc_offset": 94730348, "warc_path": "SCIELO-CRAWL-2020-07-20200710082036515-00773-00843-wbgrp-svc206/SCIELO-CRAWL-2020-07-20200710085423121-00779-13069~wbgrp-svc206.us.archive.org~8443.warc.gz"}, "file_meta": {"md5hex": "cda133a706ce02a07fae8bd8d2694a2a", "mimetype": "application/jats+xml", "sha1hex": "7d990d2f913bd3b6d6c2f78c848b7e5f4b77b99e", "sha256hex": "be982ca211e4debb3f93f36d9f9dc1c80f99a8809eb4c41569b2b9503c27e751", "size_bytes": 49242}, "hit": true, "hops": ["http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "http://scielo.iics.una.py/scieloOrg/php/articleXML.php?pid=S1683-98032015000200002&lang=en"], "request": {"link_source": "doi", "ingest_request_source": "fatcat-changelog","base_url": "http://scielo.iics.una.py/scielo.php?script=sci_abstract&pid=S1683-98032015000200002&lng=en&nrm=iso&tlng=en", "ext_ids": {"doi": "10.123/abc"}, "fatcat": {"release_ident": null}, "ingest_type": "xml"}, "status": "success", "terminal": {"terminal_dt": "20200710091403", "terminal_sha1hex": "7d990d2f913bd3b6d6c2f78c848b7e5f4b77b99e", "terminal_status_code": 200, "terminal_url": "http://scielo.iics.una.py/scieloOrg/php/articleXML.php?pid=S1683-98032015000200002&lang=en"}, "xml_meta": {"status": "success"}}
diff --git a/python/tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json b/python/tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json
new file mode 100644
index 00000000..1c559509
--- /dev/null
+++ b/python/tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json
@@ -0,0 +1 @@
+{"abstracts":[{"sha1":"b2523f13fc2aa730a2e2336f27d448644074e24f","content":"<p>Jakobshavn Isbræ, West Greenland, which holds a 0.6-m sea level volume equivalent, has been speeding up and retreating since the late 1990s. Interpretation of its retreat has been hindered by difficulties in measuring its ice thickness with airborne radar depth sounders. Here, we employ high-resolution, helicopter-borne gravity data from 2012 to reconstruct its bed elevation within 50 km of the ocean margin using a three-dimensional inversion constrained by fjord bathymetry data offshore and a mass conservation algorithm inland. We find the glacier trough to be asymmetric and several 100 m deeper than estimated previously in the lower part. From 1996-2016, the grounding line migrated at 0.6 km/yr from 700 m to 1,100 m depth. Upstream, the bed drops to 1,600 m over 10 km then slowly climbs to 1,200 m depth in 40 km. Jakobshavn Isbræ will continue to retreat along a retrograde slope for decades to come.\n\nAn L., E. Rignot, S.H.P. Elieff, M. Morlighem, R. Millan, J. Mouginot, D.M. Holland, D. Holland, and J. Paden (2017), Bed elevation of Jakobshavn Isbræ, West Greenland, from high-resolution airborne gravity and other data, Geophys. Res. Lett., 44, doi:10.1002/2017GL073245.\n\n</p>","mimetype":"text/html"}],"refs":[],"contribs":[{"raw_name":"Lu An","role":"author","raw_affiliation":"University of California, Irvine"}],"license_slug":"CC-BY","publisher":"UC Irvine","ext_ids":{"doi":"10.7280/d1j37z"},"release_year":2018,"release_type":"dataset","webcaptures":[],"filesets":[{"release_ids":["3mssw2qnlnblbk7oqyv2dafgey"],"urls":[{"url":"https://merritt.cdlib.org/u/ark%3A%2F13030%2Fm5rg0r8q/1","rel":"repo-bundle"},{"url":"https://merritt.cdlib.org/d/ark%3A%2F13030%2Fm5rg0r8q/1/","rel":"repo"},{"url":"dat://77e94744aa5f967e6ed7e3990bfc29f141dbf2c0fff572eb1212b3bd706882f4/files/","rel":"dweb"}],"manifest":[{"path":"JKS_BedElevation_An_etal_2017.nc","size":736484,"md5":"af738fa325833a56bf947622958fd504","sha1":"443f1867b3a56132905e8d611ad03445d8134d3c","sha256":"52438ef0035b391027e989f00208de5c16ab8f9ff619aa7f45e998d6214a452f","extra":{"mimetype":"application/x-netcdf"}}],"state":"active","ident":"ho376wmdanckpp66iwfs7g22ne","revision":"e07ab7b0-bc0e-4da2-9121-542263e84e2d","extra":{"cdl_dash":{"version":1}}}],"files":[],"work_id":"pbf2dmuu5jf4dac2k22gxsjk6y","title":"Jakobshavn Glacier Bed Elevation","state":"active","ident":"3mssw2qnlnblbk7oqyv2dafgey","revision":"23040a75-2aa6-49f2-af3c-a5c12dcceffe","extra":{"ark_id":"ark:/13030/m5rg0r8q","cdl_dash":{"version":1}}} \ No newline at end of file
diff --git a/python/tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json b/python/tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json
new file mode 100644
index 00000000..3bfe8564
--- /dev/null
+++ b/python/tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json
@@ -0,0 +1 @@
+{"abstracts":[],"refs":[],"contribs":[{"index":0,"raw_name":"Catherine C. Marshall","role":"author","extra":{"seq":"first"}}],"language":"en","publisher":"CNRI Acct","issue":"3/4","volume":"14","ext_ids":{"doi":"10.1045/march2008-marshall-pt1"},"release_year":2008,"release_stage":"published","release_type":"article-journal","container_id":"ugbiirfvufgcjkx33r3cmemcuu","webcaptures":[{"release_ids":["mjtqtuyhwfdr7j2c3l36uor7uy"],"timestamp":"2019-01-06T18:58:12Z","original_url":"http://www.dlib.org/dlib/march08/marshall/03marshall-pt1.html","archive_urls":[{"url":"https://web.archive.org/web/","rel":"wayback"}],"cdx":[{"surt":"org,dlib)/dlib/march08/images/spacer00.gif","timestamp":"2019-01-06T19:50:55Z","url":"http://www.dlib.org/dlib/march08/images/spacer00.gif","mimetype":"image/gif","status_code":200,"sha1":"0e75513436e6b01963759f6a88282445ff2e5b3a","sha256":"7455bacb03f7ef04d79010638db14d8434cf7a349914c2ee99eb5d4220338675"},{"surt":"org,dlib)/dlib/march08/marshall/marshall-part1-fig1.png","timestamp":"2019-01-06T19:51:01Z","url":"http://www.dlib.org/dlib/march08/marshall/marshall-part1-fig1.png","mimetype":"image/png","status_code":200,"sha1":"89cee41b938a1d2cdc51688b4be1c72366ae8102","sha256":"d63abfb99c9c48e1e6e3e37bbc5f01c0d37429f0ac0a404ae6aadc1a7d187b60"},{"surt":"org,dlib)/dlib/march08/images/redline00.gif","timestamp":"2019-01-06T19:50:55Z","url":"http://www.dlib.org/dlib/march08/images/redline00.gif","mimetype":"image/gif","status_code":200,"sha1":"3a902e1d6075e37962ab37afc1567819bc3a164e","sha256":"3279d6916807f9e244beb23c91d58cd238509f77a26c06b14314f276b77b9c06"},{"surt":"org,dlib)/dlib/march08/images/commentary00.gif","timestamp":"2019-01-06T19:50:55Z","url":"http://www.dlib.org/dlib/march08/images/commentary00.gif","mimetype":"image/gif","status_code":200,"sha1":"cdbf8804daa2627ef915db725b29cce9eaa9cd68","sha256":"8d8956e992a7f3004ccbbaaebe585ee4c2b1256ad418507d7c33f94b290d0b04"},{"surt":"org,dlib)/dlib/march08/style/main.css","timestamp":"2019-01-06T19:50:55Z","url":"http://www.dlib.org/dlib/march08/style/main.css","mimetype":"text/css","status_code":200,"sha1":"425f00efb41156f03d5c139c1b24acfcbdd611cb","sha256":"ff811660270fc847b5efc3ff9d62967244c924f91a5e4796ac2e6fc8058440ff"},{"surt":"org,dlib)/dlib/march08/marshall/03marshall-pt1.html","timestamp":"2018-12-06T13:16:33Z","url":"http://www.dlib.org/dlib/march08/marshall/03marshall-pt1.html","mimetype":"text/html","status_code":200,"sha1":"8443a044aa1f4571dd1e5561d59150e34eff0dd2","sha256":"0e9c76cdf20db60b93f0d129e5336e5344aae8bd03c5dbd75a5eea8f5d1820da"}],"revision":"6019e2a1-3503-4e91-97ec-5fba3abc70af","ident":"z7uaeatyvfgwdpuxtrdu4okqii","state":"active"}],"filesets":[],"files":[],"container":{"wikidata_qid":"Q5203268","issnl":"1082-9873","publisher":"Corporation for National Research Initiatives","name":"D-Lib Magazine","extra":{"abbrev":"Dlib Mag","country":"us","issne":"1082-9873","road":{"as_of":"2018-01-24"},"szczepanski":{"as_of":"2018"},"urls":["http://www.dlib.org/"]},"revision":"3957936f-d418-4006-b830-71341068121c","ident":"ugbiirfvufgcjkx33r3cmemcuu","state":"active"},"work_id":"kqi27ogvjvcrtnritxwumkebya","title":"Rethinking Personal Digital Archiving, Part 1","state":"active","ident":"mjtqtuyhwfdr7j2c3l36uor7uy","revision":"74270e11-c961-47f7-a682-1f6ad5927205","extra":{"crossref":{"type":"journal-article"},"subtitle":["Four Challenges from the Field"]}}
diff --git a/python/tests/import_doaj.py b/python/tests/import_doaj.py
new file mode 100644
index 00000000..d69aebd7
--- /dev/null
+++ b/python/tests/import_doaj.py
@@ -0,0 +1,142 @@
+
+import json
+import datetime
+
+import pytest
+import fatcat_openapi_client
+
+from fatcat_tools.importers import DoajArticleImporter, JsonLinePusher
+from fatcat_tools.transforms import entity_to_dict
+from fixtures import *
+
+
+@pytest.fixture(scope="function")
+def doaj_importer(api):
+ with open("tests/files/ISSN-to-ISSN-L.snip.txt", "r") as issn_file:
+ yield DoajArticleImporter(api, issn_file, bezerk_mode=True)
+
+def test_doaj_importer(doaj_importer):
+ last_index = doaj_importer.api.get_changelog(limit=1)[0].index
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ doaj_importer.bezerk_mode = True
+ counts = JsonLinePusher(doaj_importer, f).run()
+ assert counts["insert"] == 5
+ assert counts["exists"] == 0
+ assert counts["skip"] == 0
+ success_changelog = doaj_importer.api.get_changelog(limit=1)[0]
+ assert last_index + 1 == success_changelog.index
+
+ # fetch most recent editgroup
+ change = doaj_importer.api.get_changelog_entry(index=last_index + 1)
+ eg = change.editgroup
+ assert eg.description
+ assert "doaj" in eg.description.lower()
+ assert eg.extra["git_rev"]
+ assert "fatcat_tools.DoajArticleImporter" in eg.extra["agent"]
+
+ last_index = doaj_importer.api.get_changelog(limit=1)[0].index
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ doaj_importer.bezerk_mode = False
+ doaj_importer.reset()
+ counts = JsonLinePusher(doaj_importer, f).run()
+ assert counts["insert"] == 0
+ assert counts["exists"] == 5
+ assert counts["skip"] == 0
+ assert last_index == doaj_importer.api.get_changelog(limit=1)[0].index
+
+ # cleanup file entities (so other import tests work)
+ success_editgroup = doaj_importer.api.get_editgroup(success_changelog.editgroup_id)
+ eg = quick_eg(doaj_importer.api)
+ for release_edit in success_editgroup.edits.releases:
+ doaj_importer.api.delete_release(eg.editgroup_id, release_edit.ident)
+ doaj_importer.api.accept_editgroup(eg.editgroup_id)
+
+def test_doaj_importer_existing_doi(doaj_importer):
+ """
+ One of the DOAJ test entities has a dummy DOI (10.123/abc); this test
+ ensures that it isn't clobbered, an then that it gets updated.
+ """
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ doaj_importer.reset()
+ doaj_importer.bezerk_mode = False
+ doaj_importer.do_updates = False
+ counts = JsonLinePusher(doaj_importer, f).run()
+ print(counts)
+ assert counts["insert"] == 4
+ assert counts["exists"] == 1
+ assert counts["skip"] == 0
+ success_changelog = doaj_importer.api.get_changelog(limit=1)[0]
+ success_editgroup = doaj_importer.api.get_editgroup(success_changelog.editgroup_id)
+
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ doaj_importer.reset()
+ doaj_importer.bezerk_mode = False
+ doaj_importer.do_updates = True
+ counts = JsonLinePusher(doaj_importer, f).run()
+ print(counts)
+ assert counts["insert"] == 0
+ assert counts["exists"] == 4
+ assert counts["update"] == 1
+ update_changelog = doaj_importer.api.get_changelog(limit=1)[0]
+ update_editgroup = doaj_importer.api.get_editgroup(update_changelog.editgroup_id)
+
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ doaj_importer.reset()
+ doaj_importer.bezerk_mode = False
+ doaj_importer.do_updates = True
+ counts = JsonLinePusher(doaj_importer, f).run()
+ print(counts)
+ assert counts["insert"] == 0
+ assert counts["exists"] == 5
+ assert counts["update"] == 0
+
+ # cleanup file entities (so other import tests work)
+ eg = quick_eg(doaj_importer.api)
+ for release_edit in success_editgroup.edits.releases:
+ doaj_importer.api.delete_release(eg.editgroup_id, release_edit.ident)
+ for release_edit in update_editgroup.edits.releases:
+ print(release_edit)
+ doaj_importer.api.update_release(
+ eg.editgroup_id,
+ release_edit.ident,
+ ReleaseEntity(
+ revision=release_edit.prev_revision,
+ ext_ids=ReleaseExtIds(),
+ ),
+ )
+ doaj_importer.api.accept_editgroup(eg.editgroup_id)
+
+def test_doaj_dict_parse(doaj_importer):
+ with open("tests/files/example_doaj_articles.json", "r") as f:
+ raw = json.loads(f.readline())
+ r = doaj_importer.parse_record(raw)
+
+ assert r.title == "Effect of hydrogen on tensile properties and fracture behavior of PH 13-8 Mo steel"
+ assert r.publisher == "Elsevier"
+ assert r.release_type == "article-journal"
+ assert r.release_stage == "published"
+ assert r.license_slug == "cc-by-nc-nd"
+ assert r.original_title == None
+ assert r.ext_ids.doi == "10.1016/j.matdes.2016.06.110"
+ assert r.ext_ids.doaj == "e58f08a11ecb495ead55a44ad4f89808"
+ assert r.subtitle == None
+ assert r.release_date == None
+ assert r.release_year == 2016
+ assert r.volume == "108"
+ assert r.number == None
+ assert r.pages == "608-617"
+ assert r.version == None
+ assert r.language == "en"
+ # matched by ISSN, so wouldn't be defined normally
+ assert r.extra['container_name'] == "Materials & Design"
+ assert len(r.abstracts) == 1
+ assert len(r.abstracts[0].content) == 1033
+ assert len(r.contribs) == 5
+ assert r.contribs[0].raw_name == "Xinfeng Li"
+ assert r.contribs[0].given_name == None
+ assert r.contribs[0].surname == None
+ assert not r.refs
+
+ #print(r.extra)
+ assert r.extra['release_month'] == 10
+ assert r.extra['country'] == 'gb'
diff --git a/python/tests/import_ingest.py b/python/tests/import_ingest.py
index 4a46232a..92539f1a 100644
--- a/python/tests/import_ingest.py
+++ b/python/tests/import_ingest.py
@@ -2,7 +2,7 @@
import json
import pytest
-from fatcat_tools.importers import IngestFileResultImporter, JsonLinePusher
+from fatcat_tools.importers import IngestFileResultImporter, IngestWebResultImporter, JsonLinePusher
from fixtures import *
@@ -10,6 +10,10 @@ from fixtures import *
def ingest_importer(api):
yield IngestFileResultImporter(api)
+@pytest.fixture(scope="function")
+def ingest_web_importer(api):
+ yield IngestWebResultImporter(api)
+
# TODO: use API to check that entities actually created...
def test_ingest_importer_basic(ingest_importer):
with open('tests/files/example_ingest.json', 'r') as f:
@@ -41,6 +45,60 @@ def test_ingest_importer(ingest_importer):
assert counts['exists'] == 1
assert counts['skip'] == 1
+def test_ingest_importer_xml(ingest_importer):
+ last_index = ingest_importer.api.get_changelog(limit=1)[0].index
+ with open('tests/files/example_ingest_xml.json', 'r') as f:
+ ingest_importer.bezerk_mode = True
+ counts = JsonLinePusher(ingest_importer, f).run()
+ print(counts)
+ assert counts['insert'] == 1
+ assert counts['exists'] == 0
+ assert counts['skip'] == 0
+
+ # fetch most recent editgroup
+ change = ingest_importer.api.get_changelog_entry(index=last_index+1)
+ eg = change.editgroup
+ assert eg.description
+ assert "crawled from web" in eg.description.lower()
+ assert eg.extra['git_rev']
+ assert "fatcat_tools.IngestFileResultImporter" in eg.extra['agent']
+
+ # re-import should skip
+ with open('tests/files/example_ingest_xml.json', 'r') as f:
+ ingest_importer.reset()
+ ingest_importer.bezerk_mode = False
+ counts = JsonLinePusher(ingest_importer, f).run()
+ assert counts['insert'] == 0
+ assert counts['exists'] == 1
+ assert counts['skip'] == 0
+
+def test_ingest_importer_web(ingest_web_importer):
+ last_index = ingest_web_importer.api.get_changelog(limit=1)[0].index
+ with open('tests/files/example_ingest_html.json', 'r') as f:
+ ingest_web_importer.bezerk_mode = True
+ counts = JsonLinePusher(ingest_web_importer, f).run()
+ print(counts)
+ assert counts['insert'] == 1
+ assert counts['exists'] == 0
+ assert counts['skip'] == 0
+
+ # fetch most recent editgroup
+ change = ingest_web_importer.api.get_changelog_entry(index=last_index+1)
+ eg = change.editgroup
+ assert eg.description
+ assert "crawled from web" in eg.description.lower()
+ assert eg.extra['git_rev']
+ assert "fatcat_tools.IngestWebResultImporter" in eg.extra['agent']
+
+ # re-import should skip
+ with open('tests/files/example_ingest_html.json', 'r') as f:
+ ingest_web_importer.reset()
+ ingest_web_importer.bezerk_mode = False
+ counts = JsonLinePusher(ingest_web_importer, f).run()
+ assert counts['insert'] == 0
+ assert counts['exists'] == 1
+ assert counts['skip'] == 0
+
def test_ingest_importer_stage(ingest_importer, api):
"""
Tests that ingest importer correctly handles release stage matching
@@ -57,7 +115,7 @@ def test_ingest_importer_stage(ingest_importer, api):
with open('tests/files/example_ingest.json', 'r') as f:
raw = json.loads(f.readline())
for row in test_table:
- print(row)
+ #print(row)
# set dummy record stage
eg = quick_eg(api)
@@ -94,6 +152,12 @@ def test_ingest_dict_parse(ingest_importer):
def test_ingest_dict_parse_old(ingest_importer):
with open('tests/files/example_ingest.old.json', 'r') as f:
raw = json.loads(f.readline())
+
+ # ancient ingest requests had no type; skip them
+ f = ingest_importer.parse_record(raw)
+ assert f == None
+ raw['request']['ingest_type'] = 'pdf'
+
f = ingest_importer.parse_record(raw)
assert f.sha1 == "00242a192acc258bdfdb151943419437f440c313"
assert f.md5 == "f4de91152c7ab9fdc2a128f962faebff"
diff --git a/python/tests/transform_elasticsearch.py b/python/tests/transform_elasticsearch.py
index 0d96e139..b5f23e76 100644
--- a/python/tests/transform_elasticsearch.py
+++ b/python/tests/transform_elasticsearch.py
@@ -43,7 +43,7 @@ def test_rich_elasticsearch_convert():
"year_spans": [[1200, 1300]],
},
"jstor": {
- "year_spans": [[1950, 1960], [1980, 2005]],
+ "year_spans": [[1000, 1300], [1950, 1960], [1980, 2005]],
},
},
"sherpa_romeo": {"color": "blue"},
@@ -63,17 +63,23 @@ def test_rich_elasticsearch_convert():
)]
es = release_to_elasticsearch(r)
assert es['release_year'] == r.release_year
- assert es['in_ia'] == True
- assert es['in_jstor'] == False
- assert es['in_ia_sim'] == False
- assert es['in_ia'] == True
- assert es['in_web'] == True
- assert es['in_dweb'] == True
- assert es['is_oa'] == True
- assert es['is_longtail_oa'] == False
+ assert es['file_count'] == 1
+ assert es['fileset_count'] == 0
+ assert es['webcapture_count'] == 0
assert es['ref_count'] == 2
assert es['ref_linked_count'] == 1
+ assert es['preservation'] == "bright"
+ assert es['is_oa'] == True
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == True
+ assert es['in_web'] == True
+ assert es['in_dweb'] == True
+ assert es['in_ia'] == True
+ assert es['in_ia_sim'] == False
+ assert es['in_kbart'] == True
+ assert es['in_jstor'] == True
+
def test_elasticsearch_release_from_json():
r = entity_from_json(open('./tests/files/release_etodop5banbndg3faecnfm6ozi.json', 'r').read(), ReleaseEntity)
es = release_to_elasticsearch(r)
@@ -85,8 +91,59 @@ def test_elasticsearch_release_from_json():
assert es['issue'] == "11"
assert es['volume'] == "118"
assert es['number'] == None
+
+ assert es['preservation'] == "dark"
+ assert es['is_oa'] == False
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == True
+ assert es['in_web'] == False
+ assert es['in_dweb'] == False
+ assert es['in_ia'] == False
assert es['in_ia_sim'] == True
assert es['in_kbart'] == True
+ assert es['in_jstor'] == False
+
+ # this release has a fileset, and no file
+ r = entity_from_json(open('./tests/files/release_3mssw2qnlnblbk7oqyv2dafgey.json', 'r').read(), ReleaseEntity)
+ es = release_to_elasticsearch(r)
+
+ assert es['title'] == "Jakobshavn Glacier Bed Elevation"
+ assert es['ident'] == "3mssw2qnlnblbk7oqyv2dafgey"
+ assert es['file_count'] == 0
+ assert es['fileset_count'] == 1
+ assert es['webcapture_count'] == 0
+
+ assert es['preservation'] == "dark"
+ assert es['is_oa'] == True
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == True
+ assert es['in_web'] == True
+ assert es['in_dweb'] == True
+ assert es['in_ia'] == False
+ assert es['in_ia_sim'] == False
+ assert es['in_kbart'] == False
+ assert es['in_jstor'] == False
+
+ # this release has a web capture, and no file (edited the JSON to remove file)
+ r = entity_from_json(open('./tests/files/release_mjtqtuyhwfdr7j2c3l36uor7uy.json', 'r').read(), ReleaseEntity)
+ es = release_to_elasticsearch(r)
+
+ assert es['title'] == "Rethinking Personal Digital Archiving, Part 1"
+ assert es['ident'] == "mjtqtuyhwfdr7j2c3l36uor7uy"
+ assert es['file_count'] == 0
+ assert es['fileset_count'] == 0
+ assert es['webcapture_count'] == 1
+
+ assert es['preservation'] == "bright"
+ assert es['is_oa'] == True
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == True
+ assert es['in_web'] == True
+ assert es['in_dweb'] == False
+ assert es['in_ia'] == True
+ assert es['in_ia_sim'] == False
+ assert es['in_kbart'] == False
+ assert es['in_jstor'] == False
def test_elasticsearch_container_transform(journal_metadata_importer):
with open('tests/files/journal_metadata.sample.json', 'r') as f:
@@ -164,9 +221,17 @@ def test_elasticsearch_release_kbart_year():
)
es = release_to_elasticsearch(r)
assert es['release_year'] == this_year
+
+ assert es['preservation'] == "none"
+ assert es['is_oa'] == True
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == None
+ assert es['in_web'] == False
+ assert es['in_dweb'] == False
assert es['in_ia'] == False
+ assert es['in_ia_sim'] == False
assert es['in_kbart'] == False
- assert es['preservation'] == "none"
+ assert es['in_jstor'] == False
r.container = ContainerEntity(
name="dummy journal",
@@ -180,6 +245,14 @@ def test_elasticsearch_release_kbart_year():
)
es = release_to_elasticsearch(r)
assert es['release_year'] == this_year
+
+ assert es['preservation'] == "dark"
+ assert es['is_oa'] == True
+ assert es['is_longtail_oa'] == False
+ assert es['is_preserved'] == True
+ assert es['in_web'] == False
+ assert es['in_dweb'] == False
assert es['in_ia'] == False
+ assert es['in_ia_sim'] == False
assert es['in_kbart'] == True
- assert es['preservation'] == "dark"
+ assert es['in_jstor'] == False
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-01 02:33:26 +0000