start implementing ES transform helpers

2 files changed, 20 insertions, 0 deletions

diff --git a/tests/files/release_hsmo6p4smrganpb3fndaj2lon4.json b/tests/files/release_hsmo6p4smrganpb3fndaj2lon4.json
new file mode 100644
index 0000000..6a0c55d
--- /dev/null
+++ b/tests/files/release_hsmo6p4smrganpb3fndaj2lon4.json
@@ -0,0 +1 @@
+{"abstracts":[{"sha1":"bf514b9e0adb8ade4a8d35850a7c28c9d21c1644","content":"<jats:p>Despite growing interest in Open Access (OA) to scholarly literature, there is an unmet need for large-scale, up-to-date, and reproducible studies assessing the prevalence and characteristics of OA. We address this need using oaDOI, an open online service that determines OA status for 67 million articles. We use three samples, each of 100,000 articles, to investigate OA in three populations: (1) all journal articles assigned a Crossref DOI, (2) recent journal articles indexed in Web of Science, and (3) articles viewed by users of Unpaywall, an open-source browser extension that lets users find OA articles using oaDOI. We estimate that at least 28% of the scholarly literature is OA (19M in total) and that this proportion is growing, driven particularly by growth in Gold and Hybrid. The most recent year analyzed (2015) also has the highest percentage of OA (45%). Because of this growth, and the fact that readers disproportionately access newer articles, we find that Unpaywall users encounter OA quite frequently: 47% of articles they view are OA. Notably, the most common mechanism for OA is not Gold, Green, or Hybrid OA, but rather an under-discussed category we dub Bronze: articles made free-to-read on the publisher website, without an explicit Open license. We also examine the citation impact of OA articles, corroborating the so-called open-access citation advantage: accounting for age and discipline, OA articles receive 18% more citations than average, an effect driven primarily by Green and Hybrid OA. We encourage further research using the free oaDOI service, as a way to inform OA policy and practice.</jats:p>","mimetype":"application/xml+jats"}],"refs":[{"index":0,"extra":{"authors":["Anderson"]},"key":"10.7717/peerj.4375/ref-1","year":2017,"title":"When the wolf finally arrives: big deal cancelations in North American Libraries"},{"index":1,"extra":{"authors":["Anderson"]},"key":"10.7717/peerj.4375/ref-2","year":2017,"title":"The forbidden forecast: thinking about open access and library subscriptions"},{"index":2,"extra":{"authors":["Antelman"]},"key":"10.7717/peerj.4375/ref-3","year":2017,"title":"Leveraging the growth of open access in library collection decision making"},{"index":3,"extra":{"authors":["Archambault"]},"key":"10.7717/peerj.4375/ref-4","year":2013,"container_name":"Proportion of open access peer-reviewed papers at the European and world levels–2004–2011"},{"index":4,"extra":{"authors":["Archambault"]},"key":"10.7717/peerj.4375/ref-5","year":2014,"container_name":"Proportion of open access papers published in peer-reviewed journals at the European and world levels–1996–2013"},{"index":5,"extra":{"authors":["Archambault"]},"key":"10.7717/peerj.4375/ref-6","year":2016,"title":"Research impact of paywalled versus open access papers"},{"index":6,"extra":{"authors":["Berg"]},"key":"10.7717/peerj.4375/ref-7","year":2010,"title":"Measuring the scientific output and impact of NIGMS grants"},{"index":7,"extra":{"authors":["Björk"],"doi":"10.1016/j.joi.2016.08.002","volume":"10"},"key":"10.7717/peerj.4375/ref-8","year":2016,"container_name":"Journal of Informetrics","title":"Hybrid open access—a longitudinal study","locator":"919"},{"index":8,"extra":{"authors":["Björk"],"doi":"10.1002/leap.1021","volume":"29"},"key":"10.7717/peerj.4375/ref-9","year":2016,"container_name":"Learned Publishing","title":"The open access movement at a crossroad: are the big publishers and academic social media taking over?","locator":"131"},{"index":9,"extra":{"authors":["Björk"],"doi":"10.1002/leap.1096","volume":"30"},"key":"10.7717/peerj.4375/ref-10","year":2017,"container_name":"Learned Publishing","title":"Gold, green, and black open access","locator":"173"},{"index":10,"extra":{"authors":["Björk"],"doi":"10.1002/asi.22963","volume":"65"},"key":"10.7717/peerj.4375/ref-11","year":2014,"container_name":"Journal of the Association for Information Science and Technology","title":"Anatomy of green open access","locator":"237"},{"index":11,"extra":{"authors":["Björk"],"doi":"10.1371/journal.pone.0011273"},"key":"10.7717/peerj.4375/ref-12","year":2010,"container_name":"PLOS ONE","title":"Open access to the scientific journal literature: situation 2009","locator":"e11273"},{"index":12,"extra":{"authors":["Bohannon"],"doi":"10.1126/science.352.6285.508","issue":"6285","volume":"352"},"key":"10.7717/peerj.4375/ref-13","year":2016,"container_name":"Science","title":"Who's downloading pirated papers? Everyone","locator":"508"},{"index":13,"extra":{"authors":["Boudry"],"doi":"10.1007/s11192-016-2225-6","volume":"110"},"key":"10.7717/peerj.4375/ref-14","year":2017,"container_name":"Scientometrics March","title":"Availability of digital object identifiers in publications archived by PubMed","locator":"1453"},{"index":14,"extra":{"authors":["Chawla"]},"key":"10.7717/peerj.4375/ref-15","year":2017,"container_name":"Science News","title":"Publishers take ResearchGate to court, alleging massive copyright infringement"},{"index":15,"extra":{"authors":["Chen"],"doi":"10.1080/19322909.2013.795426"},"key":"10.7717/peerj.4375/ref-16","year":2013,"container_name":"Journal of Web Librarianship","title":"Journal article retrieval in an age of Open Access: how journal indexes indicate Open Access articles","locator":"243"},{"index":16,"extra":{"authors":["Chen"],"doi":"10.1080/00987913.2016.1182672","volume":"42"},"key":"10.7717/peerj.4375/ref-17","year":2016,"container_name":"Serials Review","title":"Measuring the degrees of openness of scholarly journals with the open access spectrum (OAS) evaluation tool","locator":"108"},{"index":17,"extra":{"authors":["Craig"],"doi":"10.1016/j.joi.2007.04.001"},"key":"10.7717/peerj.4375/ref-18","year":2007,"container_name":"Journal of Informetrics","title":"Do open access articles have greater citation impact?","locator":"239"},{"index":18,"extra":{"authors":["Creative Commons"]},"key":"10.7717/peerj.4375/ref-19","year":2018,"title":"Attribution 4.0 International (CC BY 4.0)"},{"index":19,"extra":{"authors":["Davis"],"doi":"10.1096/fj.11-183988","volume":"25"},"key":"10.7717/peerj.4375/ref-20","year":2011,"container_name":"FASEB Journal","title":"Open access, readership, citations: a randomized controlled trial of scientific journal publishing","locator":"2129"},{"index":20,"extra":{"authors":["Davis"],"doi":"10.3163/1536-5050.99.3.008","volume":"99"},"key":"10.7717/peerj.4375/ref-21","year":2011,"container_name":"Journal of the Medical Library Association","title":"The impact of free access to the scientific literature: a review of recent research","locator":"208"},{"index":21,"extra":{"authors":["Fortney"]},"key":"10.7717/peerj.4375/ref-22","year":2015,"container_name":"Office of Scholarly Communication","title":"A social networking site is not an open access repository"},{"index":22,"extra":{"authors":["Gargouri"],"doi":"10.1371/journal.pone.0013636","issue":"10"},"key":"10.7717/peerj.4375/ref-23","year":2010,"container_name":"PLOS ONE","title":"Self-selected or mandated, open access increases citation impact for higher quality research","locator":"e13636"},{"index":23,"extra":{"authors":["Gargouri"]},"key":"10.7717/peerj.4375/ref-24","year":2012,"title":"Green and gold open access percentages and growth, by discipline"},{"index":24,"extra":{"authors":["Gorraiz"],"doi":"10.1016/j.joi.2015.11.008","volume":"10"},"key":"10.7717/peerj.4375/ref-25","year":2016,"container_name":"Journal of Informetrics","title":"Availability of digital object identifiers (DOIs) in web of science and scopus","locator":"98"},{"index":25,"extra":{"authors":["Greshake"],"doi":"10.12688/f1000research.11366.1"},"key":"10.7717/peerj.4375/ref-26","year":2017,"container_name":"F1000Research","title":"Looking into Pandora's Box: the content of Sci-Hub and its usage [version 1; referees: 2 approved, 2 approved with reservations]"},{"index":26,"extra":{"authors":["Hajjem"]},"key":"10.7717/peerj.4375/ref-27","year":2006,"title":"Ten-year cross-disciplinary comparison of the growth of open access and how it increases research citation impact"},{"index":27,"extra":{"authors":["Harnad"],"doi":"10.1080/00987913.2008.10765150","volume":"34"},"key":"10.7717/peerj.4375/ref-28","year":2008,"container_name":"Serials Review","title":"The access/impact problem and the green and gold roads to open access: an update","locator":"36"},{"index":28,"extra":{"authors":["Himmelstein"],"doi":"10.7287/peerj.preprints.3100v1"},"key":"10.7717/peerj.4375/ref-29","year":2017,"container_name":"PeerJ Preprints","title":"Sci-Hub provides access to nearly all scholarly literature (No. e3100v1)"},{"index":29,"extra":{"authors":["Jamali"],"doi":"10.1007/s11192-017-2291-4","volume":"112"},"key":"10.7717/peerj.4375/ref-30","year":2017,"container_name":"Scientometrics","title":"Copyright compliance and infringement in ResearchGate full-text journal articles","locator":"241"},{"index":30,"extra":{"authors":["Laakso"],"doi":"10.1186/1741-7015-10-124","volume":"10"},"key":"10.7717/peerj.4375/ref-31","year":2012,"container_name":"BMC Medicine","title":"Anatomy of open access publishing: a study of longitudinal development and internal structure"},{"index":31,"extra":{"authors":["Laakso"],"doi":"10.1002/asi.22856","volume":"64"},"key":"10.7717/peerj.4375/ref-32","year":2013,"container_name":"Journal of the American Society for Information Science and Technology","title":"Delayed open access: an overlooked high-impact category of openly available scientific literature","locator":"1323"},{"index":32,"extra":{"authors":["Laakso"],"doi":"10.1371/journal.pone.0020961"},"key":"10.7717/peerj.4375/ref-33","year":2011,"container_name":"PLOS ONE","title":"The development of open access journal publishing from 1993 to 2009","locator":"e20961"},{"index":33,"extra":{"authors":["Matsubayashi"],"doi":"10.3163/1536-5050.97.1.002","volume":"97"},"key":"10.7717/peerj.4375/ref-34","year":2009,"container_name":"Journal of the Medical Library Association","title":"Status of open access in the biomedical field in 2005"},{"index":34,"extra":{"authors":["McCabe"],"doi":"10.1111/ecin.12064","volume":"52"},"key":"10.7717/peerj.4375/ref-35","year":2014,"container_name":"Economic Inquiry","title":"Identifying the effect of open access on citations using a panel of science journals","locator":"1284"},{"index":35,"extra":{"authors":["McKiernan"],"doi":"10.7554/elife.16800"},"key":"10.7717/peerj.4375/ref-36","year":2016,"container_name":"eLife","title":"How open science helps researchers succeed","locator":"e16800"},{"index":36,"extra":{"authors":["Mongeon"],"doi":"10.1007/s11192-015-1765-5","volume":"106"},"key":"10.7717/peerj.4375/ref-37","year":2016,"container_name":"Scientometrics","title":"The journal coverage of Web of Science and Scopus: a comparative analysis","locator":"213"},{"index":37,"extra":{"authors":["Ottaviani"],"doi":"10.1371/journal.pone.0159614","volume":"11"},"key":"10.7717/peerj.4375/ref-38","year":2016,"container_name":"PLOS ONE","title":"The post-embargo open access citation advantage: it exists (probably), it's modest (usually), and the rich get richer (of course)","locator":"e0159614"},{"index":38,"extra":{"authors":["Packer"],"volume":"39"},"key":"10.7717/peerj.4375/ref-39","year":2010,"container_name":"Canadian Journal of Higher Education","title":"The SciELO open access: a gold way from the south","locator":"111"},{"index":39,"extra":{"authors":["PLOS"]},"key":"10.7717/peerj.4375/ref-40","year":2018,"title":"Reviewer guidelines: criteria for publication"},{"index":40,"extra":{"authors":["Schiermeier"],"doi":"10.1038/nature.2016.21223","issue":"7635","volume":"541"},"key":"10.7717/peerj.4375/ref-41","year":2017,"container_name":"Nature News","title":"Scientists in Germany, Peru and Taiwan to lose access to Elsevier journals","locator":"13"},{"index":41,"extra":{"authors":["Smith"]},"key":"10.7717/peerj.4375/ref-42","container_name":"BMC Health Research Policy and System","title":"Knowledge sharing in global health research; the impact, uptake and cost of open access to scholarly literature"},{"index":42,"extra":{"authors":["SPARC Europe"]},"key":"10.7717/peerj.4375/ref-43","year":2015,"title":"The open access citation advantage: list of studies until 2015"},{"index":43,"extra":{"authors":["Suber"]},"key":"10.7717/peerj.4375/ref-44","year":2008,"container_name":"SPARC Open Access Newsletter, 124","title":"Gratis and libre open access"},{"index":44,"extra":{"authors":["Tennant"]},"key":"10.7717/peerj.4375/ref-45","year":2017,"title":"The open access citation advantage"},{"index":45,"extra":{"authors":["Tennant"],"doi":"10.12688/f1000research.8460.3"},"key":"10.7717/peerj.4375/ref-46","year":2016,"container_name":"F1000 Research","title":"The academic, economic and societal impacts of Open Access: an evidence-based review (version 3; referees: 3 approved, 2 approved with reservations)"},{"index":46,"extra":{"authors":["Universitat Konstanz"]},"key":"10.7717/peerj.4375/ref-47","year":2014,"title":"Teurer als die Wissenschaft erlaubt"},{"index":47,"extra":{"authors":["Université de Montréal"]},"key":"10.7717/peerj.4375/ref-48","year":2017,"title":"UdeM Libraries cancel Big Deal subscription to 2231 periodical titles published by Taylor & Francis Group"},{"index":48,"extra":{"authors":["Wagner"],"doi":"10.5062/f4q81b0w","volume":"60"},"key":"10.7717/peerj.4375/ref-49","year":2010,"container_name":"Issues in Science and Technology Librarianship","title":"Open access citation advantage: an annotated bibliography"},{"index":49,"extra":{"authors":["Walker"],"doi":"10.1241/johokanri.41.678","volume":"41"},"key":"10.7717/peerj.4375/ref-50","year":1998,"container_name":"Journal of Information Processing and Management","title":"Free internet access to traditional journals","locator":"678"},{"index":50,"extra":{"authors":["Willinsky"],"volume":"49"},"key":"10.7717/peerj.4375/ref-51","year":2003,"container_name":"Journal of Postgraduate Medicine","title":"The nine flavours of open access scholarly publishing","locator":"263"},{"index":51,"extra":{"authors":["Willinsky"],"volume-title":"The access principle: the case for open access to research and scholarship"},"key":"10.7717/peerj.4375/ref-52","year":2009,"container_name":"The access principle: the case for open access to research and scholarship"}],"contribs":[{"index":0,"raw_name":"Heather Piwowar","role":"author","raw_affiliation":"Impactstory, Sanford, NC, USA","extra":{"seq":"first"}},{"index":1,"raw_name":"Jason Priem","role":"author","raw_affiliation":"Impactstory, Sanford, NC, USA"},{"index":2,"raw_name":"Vincent Larivière","role":"author","raw_affiliation":"École de bibliothéconomie et des sciences de l'information, Université de Montréal, Montréal, QC, Canada","extra":{"more_affiliations":["Observatoire des Sciences et des Technologies (OST), Centre Interuniversitaire de Recherche sur la Science et la Technologie (CIRST), Université du Québec à Montréal, Montréal, QC, Canada"]}},{"index":3,"raw_name":"Juan Pablo Alperin","role":"author","raw_affiliation":"Canadian Institute for Studies in Publishing, Simon Fraser University, Vancouver, BC, Canada","extra":{"more_affiliations":["Public Knowledge Project, Canada"]}},{"index":4,"raw_name":"Lisa Matthias","role":"author","raw_affiliation":"Scholarly Communications Lab, Simon Fraser University, Vancouver, Canada"},{"index":5,"raw_name":"Bree Norlander","role":"author","raw_affiliation":"Information School, University of Washington, Seattle, USA","extra":{"more_affiliations":["FlourishOA, USA"]}},{"index":6,"raw_name":"Ashley Farley","role":"author","raw_affiliation":"Information School, University of Washington, Seattle, USA","extra":{"more_affiliations":["FlourishOA, USA"]}},{"index":7,"raw_name":"Jevin West","role":"author","raw_affiliation":"Information School, University of Washington, Seattle, USA"},{"index":8,"raw_name":"Stefanie Haustein","role":"author","raw_affiliation":"Observatoire des Sciences et des Technologies (OST), Centre Interuniversitaire de Recherche sur la Science et la Technologie (CIRST), Université du Québec à Montréal, Montréal, QC, Canada","extra":{"more_affiliations":["School of Information Studies, University of Ottawa, Ottawa, ON, Canada"]}}],"license_slug":"CC-BY","language":"en","publisher":"PeerJ","pages":"e4375","ext_ids":{"doi":"10.7717/peerj.4375","wikidata_qid":"Q49873702","pmid":"29456894","pmcid":"PMC5815332"},"release_year":2018,"release_date":"2018-02-13","release_stage":"published","release_type":"article-journal","container_id":"eyfkjqp7sva5bbnwatk5zazi7q","webcaptures":[],"filesets":[],"files":[{"release_ids":["bsamgk5hsngejamqdd2naz6fsu","hsmo6p4smrganpb3fndaj2lon4","xqk6bvvwcfgu3ftjjk7jqyhnqa"],"mimetype":"application/pdf","urls":[{"url":"https://web.archive.org/web/20180724215659/https://peerj.com/articles/4375.pdf","rel":"webarchive"},{"url":"https://peerj.com/articles/4375.pdf","rel":"web"}],"sha1":"bca1531b0562c6d72e0c283c1ccb97eb5cb02117","size":2433346,"revision":"2aed7279-0dd2-474e-b66d-5c6882709658","ident":"f76kdq5bxbftnknlyvtmnjtdnq","state":"active"}],"container":{"wikidata_qid":"Q2000010","issnl":"2167-8359","publisher":"PeerJ","name":"PeerJ","extra":{"abbrev":"PeerJ","country":"us","default_license":"CC-BY","doaj":{"archive":["LOCKSS","CLOCKSS","Portico","PMC/Europe PMC/PMC Canada"],"as_of":"2019-07-31","country":"us","crawl-permission":true,"default_license":"CC-BY","issne":"2167-8359","mimetypes":null,"publisher":"PeerJ Inc","seal":true},"issne":"2167-8359","issnp":"2167-8359","kbart":{"clockss":{"year_spans":[[2014,2015]]},"lockss":{"year_spans":[[2013,2013],[2016,2017]]},"portico":{"year_spans":[[2013,2019]]}},"languages":["en"],"road":{"as_of":"2018-01-24"},"sherpa_romeo":{"color":"green"},"urls":["http://www.peerj.com/","http://bibpurl.oclc.org/web/53853"]},"revision":"9c613fec-4801-4ed6-b75f-963846b6ab0f","ident":"eyfkjqp7sva5bbnwatk5zazi7q","state":"active"},"work_id":"fyvohvh7ufcwjcluvjivszmkiq","title":"The state of OA: a large-scale analysis of the prevalence and impact of Open Access articles","state":"active","ident":"hsmo6p4smrganpb3fndaj2lon4","revision":"de98deaf-4720-430a-8a97-1ff244cb602f","extra":{"crossref":{"alternative-id":["10.7717/peerj.4375"],"archive":["CLOCKSS","LOCKSS","Portico"],"license":[{"URL":"http://creativecommons.org/licenses/by/4.0/","content-version":"unspecified","delay-in-days":0,"start":"2018-02-13T00:00:00Z"}],"type":"journal-article"}}}
\ No newline at end of file
diff --git a/tests/test_transform.py b/tests/test_transform.py
new file mode 100644
index 0000000..b7bdb4d
--- /dev/null
+++ b/tests/test_transform.py
@@ -0,0 +1,19 @@
+
+import pytest
+from fatcat_openapi_client import ReleaseEntity
+
+from fatcat_scholar.es_transform import *
+from fatcat_scholar.api_entities import *
+
+def test_es_release_from_release():
+
+    with open('tests/files/release_hsmo6p4smrganpb3fndaj2lon4.json', 'r') as f:
+        release = entity_from_json(f.read(), ReleaseEntity)
+    
+    obj = es_release_from_release(release)
+    d = json.loads(obj.json())
+
+    assert obj.ident == release.ident == "hsmo6p4smrganpb3fndaj2lon4"
+    assert obj.ident == release.ident == d['ident'] == "hsmo6p4smrganpb3fndaj2lon4"
+    assert obj.doi_registrar == "crossref"
+    assert obj.doi_prefix == "10.7717"