Merge branch 'bnewbold-es-refactor' into 'master'

web ES search refactor

See merge request webgroup/fatcat!68

2 files changed, 119 insertions, 46 deletions

diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py
index 44c7be63..3263f243 100644
--- a/python/tests/fixtures.py
+++ b/python/tests/fixtures.py
@@ -1,9 +1,10 @@
 
 import pytest
 from dotenv import load_dotenv
+import elasticsearch
+
 import fatcat_web
 import fatcat_openapi_client
-
 from fatcat_openapi_client import *
 from fatcat_tools import authenticated_api
 
@@ -13,6 +14,7 @@ def full_app():
     fatcat_web.app.testing = True
     fatcat_web.app.debug = False
     fatcat_web.app.config['WTF_CSRF_ENABLED'] = False
+    fatcat_web.app.es_client = elasticsearch.Elasticsearch("mockbackend")
     return fatcat_web.app
 
 @pytest.fixture
diff --git a/python/tests/web_search.py b/python/tests/web_search.py
index 7647bcf5..55e90d56 100644
--- a/python/tests/web_search.py
+++ b/python/tests/web_search.py
@@ -1,31 +1,36 @@
 
 import json
-import responses
+import pytest
 
+from fatcat_web.search import get_elastic_container_random_releases, get_elastic_container_histogram
+from fatcat_openapi_client.rest import ApiException
 from fixtures import *
 
-@responses.activate
-def test_release_search(app):
+
+def test_release_search(app, mocker):
 
     with open('tests/files/elastic_release_search.json') as f:
         elastic_resp=json.loads(f.read())
 
-    responses.add(responses.GET, 'http://localhost:9200/fatcat_release/_search',
-        json=elastic_resp, status=200)
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp)),
+    ]
 
     rv = app.get('/release/search?q=blood')
     assert rv.status_code == 200
     assert b"Showing" in rv.data
     assert b"Quantum Studies of Acetylene Adsorption on Ice Surface" in rv.data
 
-@responses.activate
-def test_container_search(app):
+def test_container_search(app, mocker):
 
     with open('tests/files/elastic_container_search.json') as f:
         elastic_resp=json.loads(f.read())
 
-    responses.add(responses.GET, 'http://localhost:9200/fatcat_container/_search',
-        json=elastic_resp, status=200)
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp)),
+    ]
 
     rv = app.get('/container/search?q=blood')
     assert rv.status_code == 200
@@ -33,6 +38,20 @@ def test_container_search(app):
     assert b"European Instructional Course Lectures" in rv.data
     assert b"British Editorial Society of Bone and Joint Surger" in rv.data
 
+def test_random_releases(app, mocker):
+
+    with open('tests/files/elastic_release_search.json') as f:
+        elastic_resp=json.loads(f.read())
+
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp)),
+    ]
+
+    resp = get_elastic_container_random_releases("123")
+    assert len(resp) >= 1
+
+
 elastic_resp1 = {
     'timed_out': False,
     'aggregations': {
@@ -60,39 +79,34 @@ elastic_resp3 = {
     'took': 0
 }
 
-@responses.activate
-def test_stats(app):
-
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_release/_search?request_cache=true',
-        json=elastic_resp1.copy(), status=200)
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_release/_search?request_cache=true',
-        json=elastic_resp2.copy(), status=200)
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_container/_search?request_cache=true',
-        json=elastic_resp3.copy(), status=200)
+def test_stats(app, mocker):
+
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp1)),
+        (200, {}, json.dumps(elastic_resp2)),
+        (200, {}, json.dumps(elastic_resp3)),
+    ]
+
     rv = app.get('/stats')
     assert rv.status_code == 200
-    # TODO: robe these responses better
-
-@responses.activate
-def test_stats_json(app):
-
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_release/_search?request_cache=true',
-        json=elastic_resp1.copy(), status=200)
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_release/_search?request_cache=true',
-        json=elastic_resp2.copy(), status=200)
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_container/_search?request_cache=true',
-        json=elastic_resp3.copy(), status=200)
+    assert b"80,578,584" in rv.data
+
+def test_stats_json(app, mocker):
+
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp1)),
+        (200, {}, json.dumps(elastic_resp2)),
+        (200, {}, json.dumps(elastic_resp3)),
+    ]
+
     rv = app.get('/stats.json')
     assert rv.status_code == 200
+    assert rv.json['papers']['in_kbart'] == 51594200
+    assert rv.json['release']['refs_total'] == 8031459
 
-@responses.activate
-def test_container_stats(app):
+def test_container_stats(app, mocker):
 
     elastic_resp = {
         'timed_out': False,
@@ -106,14 +120,71 @@ def test_container_stats(app):
         'took': 50
     }
 
-    responses.add(responses.GET,
-        'http://localhost:9200/fatcat_release/_search?request_cache=true',
-        json=elastic_resp, status=200)
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp)),
+        (200, {}, json.dumps(elastic_resp)),
+    ]
     rv = app.get('/container/issnl/1234-5678/stats.json')
+    #print(rv.json)
+    assert rv.status_code == 200
+
+    rv = app.get('/container/aaaaaaaaaaaaaeiraaaaaaaaam/stats.json')
+    assert rv.status_code == 200
+
+def test_container_coverage(app, mocker):
+
+    elastic_resp1 = {
+        'timed_out': False,
+        'aggregations': {
+            'container_stats': {'buckets': {
+              'is_preserved': {'doc_count': 461939},
+              'in_kbart': {'doc_count': 461939},
+              'in_web': {'doc_count': 2797}}}},
+        'hits': {'total': 461939, 'hits': [], 'max_score': 0.0},
+        '_shards': {'successful': 5, 'total': 5, 'skipped': 0, 'failed': 0},
+        'took': 50
+    }
+
+    elastic_resp2 = {
+        'took': 294,
+        'timed_out': False,
+        '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0},
+        'hits': {'total': 4327, 'max_score': 0.0, 'hits': []},
+        'aggregations': {'year_in_ia': {
+            'after_key': {'year': 2020.0, 'in_ia': True},
+            'buckets': [
+                {'key': {'year': 2004.0, 'in_ia': False}, 'doc_count': 4},
+                {'key': {'year': 2004.0, 'in_ia': True}, 'doc_count': 68},
+                {'key': {'year': 2005.0, 'in_ia': False}, 'doc_count': 26},
+                {'key': {'year': 2005.0, 'in_ia': True}, 'doc_count': 428},
+                {'key': {'year': 2006.0, 'in_ia': False}, 'doc_count': 14},
+                {'key': {'year': 2006.0, 'in_ia': True}, 'doc_count': 487},
+                {'key': {'year': 2007.0, 'in_ia': False}, 'doc_count': 13},
+                {'key': {'year': 2007.0, 'in_ia': True}, 'doc_count': 345},
+            ],
+        }},
+    }
+
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp1)),
+    ]
+
+    rv = app.get('/container/aaaaaaaaaaaaaeiraaaaaaaaam/coverage')
     assert rv.status_code == 200
-    # TODO: probe this response better
 
-# TODO: container stats
-# TODO: container ISSN-L query
-# TODO: release DOI query
-# TODO: release fulltext (filter) query
+    es_raw = mocker.patch('elasticsearch.connection.Urllib3HttpConnection.perform_request')
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp2)),
+    ]
+
+    rv = app.get('/container/aaaaaaaaaaaaaeiraaaaaaaaam/ia_coverage_years.json')
+    assert rv.status_code == 200
+
+    es_raw.side_effect = [
+        (200, {}, json.dumps(elastic_resp2)),
+    ]
+
+    rv = app.get('/container/aaaaaaaaaaaaaeiraaaaaaaaam/ia_coverage_years.svg')
+    assert rv.status_code == 200