Merge branch 'bnewbold-crossref-harvest-test' into 'master'

Basic mocked test for crossref harvester

See merge request webgroup/fatcat!7

2 files changed, 46 insertions, 0 deletions

diff --git a/python/tests/files/crossref_api_works.json b/python/tests/files/crossref_api_works.json
new file mode 100644
index 00000000..c1ffccb3
--- /dev/null
+++ b/python/tests/files/crossref_api_works.json
@@ -0,0 +1 @@
+{"status":"ok","message-type":"work-list","message-version":"1.0.0","message":{"facets":{},"total-results":2473850,"items":[{"indexed":{"date-parts":[[2019,12,4]],"date-time":"2019-12-04T07:47:00Z","timestamp":1575445620667},"reference-count":0,"publisher":"Institute of Electrical and Electronics Engineers (IEEE)","issue":"5","content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["IT Prof."],"published-print":{"date-parts":[[2002,9]]},"DOI":"10.1109\/mitp.2002.1041177","type":"journal-article","created":{"date-parts":[[2002,11,21]],"date-time":"2002-11-21T20:12:04Z","timestamp":1037909524000},"page":"37-41","source":"Crossref","is-referenced-by-count":12,"title":["Inside SSL: accelerating secure transactions"],"prefix":"10.1109","volume":"4","author":[{"given":"W.","family":"Chou","sequence":"first","affiliation":[]}],"member":"263","container-title":["IT Professional"],"language":"en","link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/6294\/22323\/01041177.pdf?arnumber=1041177","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,3,10]],"date-time":"2017-03-10T21:08:49Z","timestamp":1489180129000},"score":1.0,"issued":{"date-parts":[[2002,9]]},"references-count":0,"journal-issue":{"published-print":{"date-parts":[[2002,9]]},"issue":"5"},"URL":"http:\/\/dx.doi.org\/10.1109\/mitp.2002.1041177","ISSN":["1520-9202"],"issn-type":[{"value":"1520-9202","type":"print"}]},{"indexed":{"date-parts":[[2019,12,4]],"date-time":"2019-12-04T07:47:00Z","timestamp":1575445620667},"reference-count":21,"publisher":"IEEE","isbn-type":[{"value":"078039464X","type":"print"}],"content-domain":{"domain":[],"crossmark-restriction":false},"DOI":"10.1109\/infvis.2005.1532150","type":"proceedings-article","created":{"date-parts":[[2005,11,15]],"date-time":"2005-11-15T15:48:27Z","timestamp":1132069707000},"source":"Crossref","is-referenced-by-count":17,"title":["Flow map layout"],"prefix":"10.1109","author":[{"family":"Doantam Phan","sequence":"first","affiliation":[]},{"family":"Ling Xiao","sequence":"additional","affiliation":[]},{"given":"R.","family":"Yeh","sequence":"additional","affiliation":[]},{"given":"P.","family":"Hanrahan","sequence":"additional","affiliation":[]},{"given":"T.","family":"Winograd","sequence":"additional","affiliation":[]}],"member":"263","event":{"name":"IEEE Symposium on Information Visualization, 2005. INFOVIS 2005.","location":"Minneapolis, MN, USA"},"container-title":["IEEE Symposium on Information Visualization, 2005. INFOVIS 2005."],"link":[{"URL":"http:\/\/xplorestaging.ieee.org\/ielx5\/10260\/32681\/01532150.pdf?arnumber=1532150","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2017,3,14]],"date-time":"2017-03-14T16:09:43Z","timestamp":1489507783000},"score":1.0,"issued":{"date-parts":[[null]]},"ISBN":["078039464X"],"references-count":21,"URL":"http:\/\/dx.doi.org\/10.1109\/infvis.2005.1532150"},{"indexed":{"date-parts":[[2019,12,4]],"date-time":"2019-12-04T07:47:00Z","timestamp":1575445620667},"reference-count":45,"publisher":"Springer Science and Business Media LLC","issue":"1-2","license":[{"URL":"http:\/\/www.springer.com\/tdm","start":{"date-parts":[[2009,9,23]],"date-time":"2009-09-23T00:00:00Z","timestamp":1253664000000},"delay-in-days":0,"content-version":"tdm"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":["Math. Program."],"published-print":{"date-parts":[[2011,6]]},"DOI":"10.1007\/s10107-009-0306-5","type":"journal-article","created":{"date-parts":[[2009,9,22]],"date-time":"2009-09-22T12:28:12Z","timestamp":1253622492000},"page":"321-353","source":"Crossref","is-referenced-by-count":426,"title":["Fixed point and Bregman iterative methods for matrix rank minimization"],"prefix":"10.1007","volume":"128","author":[{"given":"Shiqian","family":"Ma","sequence":"first","affiliation":[]},{"given":"Donald","family":"Goldfarb","sequence":"additional","affiliation":[]},{"given":"Lifeng","family":"Chen","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2009,9,23]]},"reference":[{"issue":"Jun","key":"306_CR1","first-page":"1019","volume":"9","author":"F.R. Bach","year":"2008","unstructured":"Bach F.R.: Consistency of trace norm minimization. J. Mach. Learn. Res. 9(Jun), 1019\u20131048 (2008)","journal-title":"J. Mach. Learn. Res."},{"key":"306_CR2","unstructured":"Bertalm\u00edo, M., Sapiro, G., Caselles, V., Ballester, C.: Image inpainting. In: Proceedings of SIGGRAPH 2000, New Orleans, USA (2000)","DOI":"10.1145\/344779.344972","doi-asserted-by":"crossref"},{"key":"306_CR3","author":"J.M. Borwein","year":"2003","unstructured":"Borwein J.M., Lewis A.S.: Convex Analysis and Nonlinear Optimization. Springer, New York (2003)","volume-title":"Convex Analysis and Nonlinear Optimization"},{"key":"306_CR4","doi-asserted-by":"crossref","first-page":"200","DOI":"10.1016\/0041-5553(67)90040-7","volume":"7","author":"L. Bregman","year":"1967","unstructured":"Bregman L.: The relaxation method of finding the common points of convex sets and its application to the solution of problems in convex programming. USSR Comput. Math. Math. Phys. 7, 200\u2013217 (1967)","journal-title":"USSR Comput. Math. Math. Phys."},{"key":"306_CR5","doi-asserted-by":"crossref","first-page":"329","DOI":"10.1007\/s10107-002-0352-8","volume":"95","author":"S. Burer","year":"2003","unstructured":"Burer S., Monteiro R.D.C.: A nonlinear programming algorithm for solving semidefinite programs via low-rank factorization. Math. Program. (Ser. B) 95, 329\u2013357 (2003)","journal-title":"Math. Program. (Ser. B)"},{"issue":"3","key":"306_CR6","doi-asserted-by":"crossref","first-page":"427","DOI":"10.1007\/s10107-004-0564-1","volume":"103","author":"S. Burer","year":"2005","unstructured":"Burer S., Monteiro R.D.C.: Local mimima and convergence in low-rank semidefinite programming. Math. Program. 103(3), 427\u2013444 (2005)","journal-title":"Math. Program."},{"key":"306_CR7","unstructured":"Cai, J., Cand\u00e8s, E.J., Shen, Z.: A singular value thresholding algorithm for matrix completion. Preprint available at http:\/\/arxiv.org\/abs\/0810.3286 (2008)"},{"key":"306_CR8","unstructured":"Cand\u00e8s, E.J., Recht, B.: Exact matrix completion via convex optimization. Found. Comput. Math. (2009)","DOI":"10.1007\/s10208-009-9045-5","doi-asserted-by":"crossref"},{"key":"306_CR9","unstructured":"Cand\u00e8s, E.J., Romberg, J.: \u2113 1-MAGIC: recovery of sparse signals via convex programming. Technical Report, Caltech (2005)"},{"key":"306_CR10","doi-asserted-by":"crossref","first-page":"489","DOI":"10.1109\/TIT.2005.862083","volume":"52","author":"E.J. Cand\u00e8s","year":"2006","unstructured":"Cand\u00e8s E.J., Romberg J., Tao T.: Robust uncertainty principles: exact signal reconstruction from highly incomplete frequency information. IEEE Trans. Inf. Theory 52, 489\u2013509 (2006)","journal-title":"IEEE Trans. Inf. Theory"},{"key":"306_CR11","unstructured":"Cand\u00e8s, E.J., Tao, T.: The power of convex relaxation: near-optimal matrix completion. Preprint available at http:\/\/arxiv.org\/abs\/0903.1476 (2009)"},{"key":"306_CR12","unstructured":"Dai, W., Milenkovic, O.: Subspace pursuit for compressive sensing: closing the gap between performance and complexity. Preprint available at arXiv: 0803.0811 (2008)"},{"key":"306_CR13","doi-asserted-by":"crossref","first-page":"1289","DOI":"10.1109\/TIT.2006.871582","volume":"52","author":"D. Donoho","year":"2006","unstructured":"Donoho D.: Compressed sensing. IEEE Trans. Inf. Theory 52, 1289\u20131306 (2006)","journal-title":"IEEE Trans. Inf. Theory"},{"key":"306_CR14","unstructured":"Donoho, D.L., Tsaig, Y.: Fast solution of \u2113 1-norm minimization problems when the solution may be sparse. Technical Report, Department of Statistics, Stanford University (2006)"},{"key":"306_CR15","unstructured":"Donoho, D., Tsaig, Y., Drori, I., Starck, J.C.: Sparse solution of underdetermined linear equations by stagewise orthogonal matching pursuit. IEEE Trans. Inf. Theory (2006) (submitted)"},{"key":"306_CR16","doi-asserted-by":"crossref","first-page":"158","DOI":"10.1137\/S0097539704442696","volume":"36","author":"P. Drineas","year":"2006","unstructured":"Drineas P., Kannan R., Mahoney M.W.: Fast Monte Carlo algorithms for matrices ii: computing low-rank approximations to a matrix. SIAM J. Comput. 36, 158\u2013183 (2006)","journal-title":"SIAM J. Comput."},{"key":"306_CR17","unstructured":"Fazel, M.: Matrix rank minimization with applications. Ph.D. thesis, Stanford University (2002)"},{"key":"306_CR18","unstructured":"Fazel, M., Hindi, H., Boyd, S.: A rank minimization heuristic with application to minimum order system approximation. In: Proceedings of the American Control Conference, vol. 6, pp. 4734\u20134739 (2001)"},{"key":"306_CR19","unstructured":"Figueiredo, M.A.T., Nowak, R.D., Wright, S.J.: Gradient projection for sparse reconstruction: application to compressed sensing and other inverse problems. IEEE J. Sel. Top. Signal Process. 1(4) (2007)","DOI":"10.1109\/JSTSP.2007.910281","doi-asserted-by":"crossref"},{"key":"306_CR20","unstructured":"Ghaoui, L.E., Gahinet, P.: Rank minimization under LMI constraints: a framework for output feedback problems. In: Proceedings of the European Control Conference (1993)"},{"issue":"2","key":"306_CR21","doi-asserted-by":"crossref","first-page":"133","DOI":"10.1023\/A:1011419012209","volume":"4","author":"K. Goldberg","year":"2001","unstructured":"Goldberg K., Roeder T., Gupta D., Perkins C.: Eigentaste: a constant time collaborative filtering algorithm. Inf. Retr. 4(2), 133\u2013151 (2001)","journal-title":"Inf. Retr."},{"key":"306_CR22","unstructured":"Goldfarb, D., Ma, S.: Convergence of fixed point continuation algorithms for matrix rank minimization. Technical Report, Department of IEOR, Columbia University (2009)"},{"key":"306_CR23","unstructured":"Hale, E.T., Yin, W., Zhang, Y.: A fixed-point continuation method for \u2113 1-regularized minimization with applications to compressed sensing. Technical Report, CAAM TR07-07 (2007)"},{"key":"306_CR24","author":"J.B. Hiriart-Urruty","year":"1993","unstructured":"Hiriart-Urruty J.B., Lemar\u00e9chal C.: Convex Analysis and Minimization Algorithms II: Advanced Theory and Bundle Methods. Springer, New York (1993)","volume-title":"Convex Analysis and Minimization Algorithms II: Advanced Theory and Bundle Methods","DOI":"10.1007\/978-3-662-06409-2","doi-asserted-by":"crossref"},{"key":"306_CR25","author":"R.A. Horn","year":"1985","unstructured":"Horn R.A., Johnson C.R.: Matrix Analysis. Cambridge University Press, Cambridge (1985)","volume-title":"Matrix Analysis","DOI":"10.1017\/CBO9780511810817","doi-asserted-by":"crossref"},{"key":"306_CR26","unstructured":"Keshavan, R.H., Montanari, A., Oh, S.: Matrix completion from a few entries. Preprint available at http:\/\/arxiv.org\/abs\/0901.3150 (2009)","DOI":"10.1109\/ISIT.2009.5205567","doi-asserted-by":"crossref"},{"issue":"1","key":"306_CR27","doi-asserted-by":"crossref","first-page":"606","DOI":"10.1109\/JSTSP.2007.910971","volume":"4","author":"S.J. Kim","year":"2007","unstructured":"Kim S.J., Koh K., Lustig M., Boyd S., Gorinevsky D.: A method for large-scale \u2113 1-regularized least-squares. IEEE J. Sel. Top. Signal Process. 4(1), 606\u2013617 (2007)","journal-title":"IEEE J. Sel. Top. Signal Process."},{"key":"306_CR28","doi-asserted-by":"crossref","first-page":"215","DOI":"10.1007\/BF01200757","volume":"15","author":"N. Linial","year":"1995","unstructured":"Linial N., London E., Rabinovich Y.: The geometry of graphs and some of its algorithmic applications. Combinatorica 15, 215\u2013245 (1995)","journal-title":"Combinatorica"},{"key":"306_CR29","unstructured":"Liu, Z., Vandenberghe, L.: Interior-point method for nuclear norm approximation with application to system identification. Preprint available at http:\/\/www.ee.ucla.edu\/~vandenbe\/publications\/nucnrm.pdf (2008)"},{"issue":"2","key":"306_CR30","doi-asserted-by":"crossref","first-page":"227","DOI":"10.1137\/S0097539792240406","volume":"24","author":"B.K. Natarajan","year":"1995","unstructured":"Natarajan B.K.: Sparse approximation solutions to linear systems. SIAM J. Comput. 24(2), 227\u2013234 (1995)","journal-title":"SIAM J. Comput."},{"issue":"2","key":"306_CR31","doi-asserted-by":"crossref","first-page":"460","DOI":"10.1137\/040605412","volume":"4","author":"S. Osher","year":"2005","unstructured":"Osher S., Burger M., Goldfarb D., Xu J., Yin W.: An iterative regularization method for total varitaion-based image restoration. SIAM MMS 4(2), 460\u2013489 (2005)","journal-title":"SIAM MMS"},{"key":"306_CR32","unstructured":"Recht, B., Fazel, M., Parrilo, P.: Guaranteed minimum rank solutions of matrix equations via nuclear norm minimization. Preprint available at http:\/\/arxiv.org\/abs\/0706.4138 (2007)"},{"key":"306_CR33","unstructured":"Rennie, J.D.M., Srebro, N.: Fast maximum margin matrix factorization for collaborative prediction. In: Proceedings of the International Conference of Machine Learning (2005)","DOI":"10.1145\/1102351.1102441","doi-asserted-by":"crossref"},{"key":"306_CR34","doi-asserted-by":"crossref","first-page":"259","DOI":"10.1016\/0167-2789(92)90242-F","volume":"60","author":"L. Rudin","year":"1992","unstructured":"Rudin L., Osher S., Fatemi E.: Nonlinear total variation based noise removal algorithms. Physica D 60, 259\u2013268 (1992)","journal-title":"Physica D"},{"key":"306_CR35","doi-asserted-by":"crossref","first-page":"3273","DOI":"10.1091\/mbc.9.12.3273","volume":"9","author":"P.T. Spellman","year":"1998","unstructured":"Spellman P.T., Sherlock G., Zhang M.Q., Iyer V.R., Anders K., Eisen M.B., Brown P.O., Botstein D., Futcher B.: Comprehensive identification of cell cycle-regulated genes of the yeast saccharomyces cerevisiae by microarray hybridization. Mol. Biol. Cell 9, 3273\u20133297 (1998)","journal-title":"Mol. Biol. Cell"},{"key":"306_CR36","unstructured":"Srebro, N.: Learning with matrix factorizations. Ph.D. thesis, Massachusetts Institute of Technology (2004)"},{"key":"306_CR37","unstructured":"Srebro, N., Jaakkola, T.: Weighted low-rank approximations. In: Proceedings of the Twentieth International Conference on Machine Learning (ICML-2003) (2003)"},{"issue":"12","key":"306_CR38","doi-asserted-by":"crossref","first-page":"625","DOI":"10.1080\/10556789908805766","volume":"11","author":"J.F. Sturm","year":"1999","unstructured":"Sturm J.F.: Using SeDuMi 1.02, a Matlab toolbox for optimization over symmetric cones. Opt. Methods Softw. 11(12), 625\u2013653 (1999)","journal-title":"Opt. Methods Softw."},{"key":"306_CR39","first-page":"267","volume":"58","author":"R. Tibshirani","year":"1996","unstructured":"Tibshirani R.: Regression shrinkage and selection via the lasso. J. R. Stat. Soc. B 58, 267\u2013288 (1996)","journal-title":"J. R. Stat. Soc. B"},{"key":"306_CR40","doi-asserted-by":"crossref","first-page":"1030","DOI":"10.1109\/TIT.2005.864420","volume":"51","author":"J. Tropp","year":"2006","unstructured":"Tropp J.: Just relax: convex programming methods for identifying sparse signals. IEEE Trans. Inf. Theory 51, 1030\u20131051 (2006)","journal-title":"IEEE Trans. Inf. Theory"},{"issue":"6","key":"306_CR41","doi-asserted-by":"crossref","first-page":"520","DOI":"10.1093\/bioinformatics\/17.6.520","volume":"17","author":"O. Troyanskaya","year":"2001","unstructured":"Troyanskaya O., Cantor M., Sherlock G., Brown P., Hastie T., Tibshirani R., Botstein D., Altman R.B.: Missing value estimation methods for DNA microarrays. Bioinformatics 17(6), 520\u2013525 (2001)","journal-title":"Bioinformatics"},{"key":"306_CR42","doi-asserted-by":"crossref","first-page":"189","DOI":"10.1007\/s10107-002-0347-5","volume":"95","author":"R.H. T\u00fct\u00fcnc\u00fc","year":"2003","unstructured":"T\u00fct\u00fcnc\u00fc R.H., Toh K.C., Todd M.J.: Solving semidefinite-quadratic-linear programs using SDPT3. Math. Program. Ser. B 95, 189\u2013217 (2003)","journal-title":"Math. Program. Ser. B"},{"issue":"2","key":"306_CR43","doi-asserted-by":"crossref","first-page":"890","DOI":"10.1137\/080714488","volume":"31","author":"E. Berg van den","year":"2008","unstructured":"van den Berg E., Friedlander M.P.: Probing the Pareto frontier for basis pursuit solutions. SIAM J. Sci. Comput. 31(2), 890\u2013912 (2008)","journal-title":"SIAM J. Sci. Comput."},{"key":"306_CR44","unstructured":"Wen, Z., Yin, W., Goldfarb, D., Zhang, Y.: A fast algorithm for sparse reconstruction based on shrinkage, subspace optimization and continuation. Technical Report, Department of IEOR, Columbia University (2009)"},{"issue":"1","key":"306_CR45","doi-asserted-by":"crossref","first-page":"143","DOI":"10.1137\/070703983","volume":"1","author":"W. Yin","year":"2008","unstructured":"Yin W., Osher S., Goldfarb D., Darbon J.: Bregman iterative algorithms for \u2113 1-minimization with applications to compressed sensing. SIAM J. Imaging Sci. 1(1), 143\u2013168 (2008)","journal-title":"SIAM J. Imaging Sci."}],"container-title":["Mathematical Programming"],"language":"en","link":[{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10107-009-0306-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/article\/10.1007\/s10107-009-0306-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"http:\/\/link.springer.com\/content\/pdf\/10.1007\/s10107-009-0306-5","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2019,5,29]],"date-time":"2019-05-29T09:50:07Z","timestamp":1559123407000},"score":1.0,"issued":{"date-parts":[[2009,9,23]]},"references-count":45,"journal-issue":{"published-print":{"date-parts":[[2011,6]]},"issue":"1-2"},"alternative-id":["306"],"URL":"http:\/\/dx.doi.org\/10.1007\/s10107-009-0306-5","relation":{"cites":[]},"ISSN":["0025-5610","1436-4646"],"issn-type":[{"value":"0025-5610","type":"print"},{"value":"1436-4646","type":"electronic"}],"subject":["Software","General Mathematics"]}],"items-per-page":3,"query":{"start-index":0,"search-terms":null}}}
\ No newline at end of file
diff --git a/python/tests/harvest_crossref.py b/python/tests/harvest_crossref.py
new file mode 100644
index 00000000..52aa7b81
--- /dev/null
+++ b/python/tests/harvest_crossref.py
@@ -0,0 +1,45 @@
+
+import json
+import pytest
+import datetime
+import responses
+from fatcat_tools.harvest import *
+
+
+@responses.activate
+def test_crossref_harvest_date(mocker):
+
+    # mock out the harvest state object so it doesn't try to actually connect
+    # to Kafka
+    mocker.patch('fatcat_tools.harvest.harvest_common.HarvestState.initialize_from_kafka')
+
+    # mock day request to crossref API
+    with open('tests/files/crossref_api_works.json', 'r') as f:
+        crossref_resp = json.loads(f.readline())
+    responses.add(responses.GET, 'https://api.crossref.org/works',
+        json=crossref_resp, status=200)
+
+    harvester = HarvestCrossrefWorker(
+        kafka_hosts="dummy",
+        produce_topic="dummy-produce-topic",
+        state_topic="dummy-state-topic",
+        contact_email="test@fatcat.wiki",
+    )
+
+    harvester.producer = mocker.Mock()
+
+    harvester.fetch_date(datetime.date(2019, 2, 3))
+
+    assert len(responses.calls) == 1
+
+    # ensure email was included in User-Agent
+    assert "mailto:test@fatcat.wiki" in responses.calls[0].request.headers['User-Agent']
+
+    # check that correct date param was passed as expected
+    assert "filter=from-index-date%3A2019-02-03" in responses.calls[0].request.url
+
+    # check that we published the expected number of DOI objects were published
+    # to the (mock) kafka topic
+    assert harvester.producer.produce.call_count == 3
+    assert harvester.producer.flush.call_count == 1
+    assert harvester.producer.poll.called_once_with(0)