aboutsummaryrefslogtreecommitdiffstats
path: root/fatcat_scholar/search.py
blob: f94c403ab39c36a93979c38430d58085eb3db8e7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

"""
Helpers to make elasticsearch queries.
"""

import json
import datetime

import elasticsearch
from elasticsearch_dsl import Search, Q
from dynaconf import settings


def generic_search_execute(search, limit=25, offset=0, deep_page_limit=2000):

    # Sanity checks
    if limit > 100:
        limit = 100
    if offset < 0:
        offset = 0
    if offset > deep_page_limit:
        # Avoid deep paging problem.
        offset = deep_page_limit

    search = search[int(offset):int(offset)+int(limit)]

    try:
        resp = search.execute()
    except elasticsearch.exceptions.RequestError as e:
        # this is a "user" error
        print("elasticsearch 400: " + str(e.info))
        #flash("Search query failed to parse; you might need to use quotes.<p><code>{}: {}</code>".format(e.error, e.info['error']['root_cause'][0]['reason']))
        # XXX: abort(e.status_code)
        raise e
    except elasticsearch.exceptions.TransportError as e:
        # all other errors
        print("elasticsearch non-200 status code: {}".format(e.info))
        # XXX: abort(e.status_code)
        raise e

    # convert from objects to python dicts
    results = []
    for h in resp:
        r = h._d_
        #print(json.dumps(h.meta._d_, indent=2))
        r['_highlights'] = []
        if 'highlight' in dir(h.meta):
            highlights = h.meta.highlight._d_
            for k in highlights:
                r['_highlights'] += highlights[k]
        results.append(r)

    for h in results:
        # Handle surrogate strings that elasticsearch returns sometimes,
        # probably due to mangled data processing in some pipeline.
        # "Crimes against Unicode"; production workaround
        for key in h:
            if type(h[key]) is str:
                h[key] = h[key].encode('utf8', 'ignore').decode('utf8')

    return {
        "count_returned": len(results),
        "count_found": int(resp.hits.total),
        "results": results,
        "offset": offset,
        "limit": limit,
        "deep_page_limit": deep_page_limit,
        "query_time_ms": int(resp.took),
    }

def do_fulltext_search(q, limit=25, offset=0, filter_time=None, filter_type=None):

    # Convert raw DOIs to DOI queries
    if len(q.split()) == 1 and q.startswith("10.") and q.count("/") >= 1:
        q = 'doi:"{}"'.format(q)

    es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_BACKEND)
    search = Search(using=es_client, index=settings.ELASTICSEARCH_FULLTEXT_INDEX)

    # type filters
    if filter_type == "papers":
        search = search.filter("terms", type=[ "article-journal", "paper-conference", "chapter", ])
    elif filter_type == "reports":
        search = search.filter("terms", type=[ "report", "standard", ])
    elif filter_type == "datasets":
        search = search.filter("terms", type=[ "dataset", "software", ])
    elif filter_type == "everything" or filter_type == None:
        pass
    else:
        # XXX: abort(400)
        raise Exception()

    # time filters
    if filter_time == "past_week":
        week_ago_date = str(datetime.date.today() - datetime.timedelta(days=7))
        search = search.filter("range", date=dict(gte=week_ago_date))
    elif filter_time == "this_year":
        search = search.filter("term", year=datetime.date.today().year)
    elif filter_time == "since_2000":
        search = search.filter("range", year=dict(gte=2000))
    elif filter_time == "before_1925":
        search = search.filter("range", year=dict(lt=1925))
    elif filter_time == "all_time" or filter_time == None:
        pass
    else:
        # XXX: abort(400)
        raise Exception()

    search = search.query(
        'query_string',
        query=q,
        default_operator="AND",
        analyze_wildcard=True,
        lenient=True,
        fields=[
            "everything",
            "abstracts_all",
            "fulltext.body",
            "fulltext.annex",
        ],
    )
    search = search.highlight(
        "abstracts_all",
        "fulltext.body",
        "fulltext.annex",
        number_of_fragments=2,
        fragment_size=250,
    )

    resp = generic_search_execute(search, offset=offset)

    resp["query"] = { "q": q }
    return resp