fix abstracts; experiment with search stemming

3 files changed, 36 insertions, 8 deletions

diff --git a/fatcat_scholar/search.py b/fatcat_scholar/search.py
index 4d53667..aeb089d 100644
--- a/fatcat_scholar/search.py
+++ b/fatcat_scholar/search.py
@@ -131,10 +131,12 @@ def do_fulltext_search(query: FulltextQuery, deep_page_limit: int = 2000) -> Ful
         analyze_wildcard=True,
         allow_leading_wildcard=False,
         lenient=True,
+        quote_field_suffix=".exact",
         fields=[
             "title^5",
             "biblio_all^3",
-            "abstracts_all^2",
+            "abstracts.body^2",
+            "fulltext.body",
             "everything",
         ],
     )
@@ -165,7 +167,7 @@ def do_fulltext_search(query: FulltextQuery, deep_page_limit: int = 2000) -> Ful
         negative_boost=0.5,
     )
     search = search.highlight(
-        "abstracts_all",
+        "abstracts.body",
         "fulltext.body",
         "fulltext.annex",
         number_of_fragments=2,
diff --git a/schema/README.md b/schema/README.md
new file mode 100644
index 0000000..3dec14d
--- /dev/null
+++ b/schema/README.md
@@ -0,0 +1,8 @@
+
+fatcat-scholar Elasticsearch Schema Notes
+===========================================
+
+
+### Stemming
+
+https://www.elastic.co/guide/en/elasticsearch/reference/master/mixing-exact-search-with-stemming.html
diff --git a/schema/scholar_fulltext.v01.json b/schema/scholar_fulltext.v01.json
index 48fcc51..23cadf2 100644
--- a/schema/scholar_fulltext.v01.json
+++ b/schema/scholar_fulltext.v01.json
@@ -12,12 +12,24 @@
                     "type": "custom",
                     "tokenizer": "icu_tokenizer",
                     "char_filter": [ "icu_normalizer" ],
-                    "filter": [ "icu_folding" ]
+                    "filter": [ "icu_folding", "stemmer" ]
                 },
                 "textIcuSearch": {
                     "type": "custom",
                     "tokenizer": "icu_tokenizer",
                     "char_filter": [ "icu_normalizer" ],
+                    "filter": [ "icu_folding", "stemmer" ]
+                },
+                "textIcuExact": {
+                    "type": "custom",
+                    "tokenizer": "icu_tokenizer",
+                    "char_filter": [ "icu_normalizer" ],
+                    "filter": [ "icu_folding" ]
+                },
+                "textIcuSearchExact": {
+                    "type": "custom",
+                    "tokenizer": "icu_tokenizer",
+                    "char_filter": [ "icu_normalizer" ],
                     "filter": [ "icu_folding" ]
                 }
             },
@@ -61,7 +73,10 @@
           "dynamic": false,
           "properties": {
             "release_ident":  { "type": "keyword", "normalizer": "default", "doc_values": false },
-            "title":          { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
+            "title":          {
+                "type": "text","index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"],
+                "fields": {"exact": { "type": "text", "analyzer": "textIcuExact", "search_analyzer": "textIcuSearchExact" }}
+            },
             "subtitle":       { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
             "original_title": { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
             "release_date":   { "type": "date" },
@@ -91,7 +106,7 @@
             "mag_id":         { "type": "keyword", "normalizer": "default", "doc_values": false },
             "s2_id":          { "type": "keyword", "normalizer": "default", "doc_values": false },
             "license_slug":   { "type": "keyword", "normalizer": "default" },
-            "publisher":            { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch" },
+            "publisher":            { "type": "text", "index": true, "analyzer": "textIcuExact", "search_analyzer":"textIcuSearchExact" },
             "publisher_type":       { "type": "keyword", "normalizer": "default" },
             "container_name":       { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
             "container_original_name":       { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
@@ -101,8 +116,8 @@
             "issns":                { "type": "keyword", "normalizer": "default" },
             "container_type":       { "type": "keyword", "normalizer": "default" },
             "contrib_count":        { "type": "integer" },
-            "contrib_names":        { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch", "copy_to": ["biblio_all", "everything"] },
-            "affiliations":         { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch" }
+            "contrib_names":        { "type": "text", "index": true, "analyzer": "textIcuExact", "search_analyzer":"textIcuSearchExact", "copy_to": ["biblio_all", "everything"] },
+            "affiliations":         { "type": "text", "index": true, "analyzer": "textIcuExact", "search_analyzer":"textIcuSearchExact" }
           }
         },
 
@@ -192,7 +207,10 @@
         },
 
         "everything":     { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch" },
-        "biblio_all":     { "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch" },
+        "biblio_all":     {
+            "type": "text", "index": true, "analyzer": "textIcu", "search_analyzer":"textIcuSearch",
+            "fields": {"exact": { "type": "text", "analyzer": "textIcuExact", "search_analyzer": "textIcuSearchExact" }}
+        },
 
         "doctype":        { "type": "alias", "path": "doc_type" },