include GROBID-extracted abstracts in search documents

2 files changed, 23 insertions, 10 deletions

diff --git a/fatcat_scholar/schema.py b/fatcat_scholar/schema.py
index cf88011..35cf9a1 100644
--- a/fatcat_scholar/schema.py
+++ b/fatcat_scholar/schema.py
@@ -255,6 +255,14 @@ def contrib_affiliation(contrib: ReleaseContrib) -> Optional[str]:
     return None
 
 
+def es_abstracts_from_grobid(tei_dict: dict) -> List[ScholarAbstract]:
+
+    if tei_dict.get("abstract"):
+        return [ScholarAbstract(lang_code=tei_dict.get("lang"), body=scrub_text(tei_dict["abstract"]))]
+    else:
+        return []
+
+
 def es_abstracts_from_release(release: ReleaseEntity) -> List[ScholarAbstract]:
 
     d = dict()
diff --git a/fatcat_scholar/transform.py b/fatcat_scholar/transform.py
index 7cc0ea5..c219528 100644
--- a/fatcat_scholar/transform.py
+++ b/fatcat_scholar/transform.py
@@ -163,16 +163,15 @@ def _add_file_release_meta(
 
 
 def es_fulltext_from_grobid(
-    tei_xml: str, pdf_meta: Optional[dict], re: ReleaseEntity, fe: FileEntity
+    tei_dict: dict, pdf_meta: Optional[dict], re: ReleaseEntity, fe: FileEntity
 ) -> Optional[ScholarFulltext]:
-    obj = teixml2json(tei_xml)
-    if not obj.get("body"):
+    if not tei_dict.get("body"):
         return None
     ret = ScholarFulltext(
-        lang_code=obj.get("lang"),
-        body=obj.get("body"),
-        acknowledgement=obj.get("acknowledgement"),
-        annex=obj.get("annex"),
+        lang_code=tei_dict.get("lang"),
+        body=tei_dict.get("body"),
+        acknowledgement=tei_dict.get("acknowledgement"),
+        annex=tei_dict.get("annex"),
     )
     return _add_file_release_meta(ret, pdf_meta, re, fe)
 
@@ -219,9 +218,12 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
             r for r in heavy.releases if r.ident == heavy.biblio_release_ident
         ][0]
         biblio = es_biblio_from_release(primary_release)
-
-        # TODO: abstracts from releases also; abstracts_dict; abstracts from GROBID parse
         abstracts = es_abstracts_from_release(primary_release)
+
+        # if no abstract from primary_release, try all the other releases
+        for release in heavy.releases:
+            if not abstracts:
+                abstracts = es_abstracts_from_release(release)
     else:
         raise NotImplementedError(f"doc_type: {heavy.doc_type}")
 
@@ -236,9 +238,12 @@ def transform_heavy(heavy: IntermediateBundle) -> Optional[ScholarDoc]:
             for f in fulltext_release.files
             if f.ident == heavy.grobid_fulltext["file_ident"]
         ][0]
+        tei_dict = teixml2json(heavy.grobid_fulltext["tei_xml"])
         fulltext = es_fulltext_from_grobid(
-            heavy.grobid_fulltext["tei_xml"], heavy.pdf_meta, fulltext_release, fulltext_file
+            tei_dict, heavy.pdf_meta, fulltext_release, fulltext_file
         )
+        if not abstracts:
+            abstracts = es_abstracts_from_grobid(tei_dict)
 
     if not fulltext and heavy.pdftotext_fulltext:
         fulltext_release = [