From 0186b53f819687427a8367d6c5729d10ff25a4be Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 24 Feb 2021 15:23:59 -0800 Subject: fix body size limit --- fatcat_scholar/transform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fatcat_scholar/transform.py') diff --git a/fatcat_scholar/transform.py b/fatcat_scholar/transform.py index bf24da9..ca648b2 100644 --- a/fatcat_scholar/transform.py +++ b/fatcat_scholar/transform.py @@ -23,7 +23,7 @@ def es_fulltext_from_sim(sim: Dict[str, Any]) -> Optional[ScholarFulltext]: issue_item = sim["issue_item"] body = "\n".join([p["raw_text"] for p in sim["page_texts"]]) if body and len(body) > MAX_BODY_CHARS: - body = body[MAX_BODY_CHARS:] + body = body[:MAX_BODY_CHARS] return ScholarFulltext( lang_code=None, # TODO: pub/issue metadata? or langdetect? body=body, @@ -228,7 +228,7 @@ def es_fulltext_from_grobid( return None body = tei_dict.get("body") if body and len(body) > MAX_BODY_CHARS: - body = body[MAX_BODY_CHARS:] + body = body[:MAX_BODY_CHARS] ret = ScholarFulltext( lang_code=tei_dict.get("lang"), body=body, @@ -243,7 +243,7 @@ def es_fulltext_from_pdftotext( ) -> Optional[ScholarFulltext]: if raw_text and len(raw_text) > MAX_BODY_CHARS: - raw_text = raw_text[MAX_BODY_CHARS:] + raw_text = raw_text[:MAX_BODY_CHARS] ret = ScholarFulltext( lang_code=re.language, body=raw_text, acknowledgement=None, annex=None, ) @@ -263,7 +263,7 @@ def es_fulltext_from_html( if body: raw_text = " ".join(body.itertext()) if raw_text and len(raw_text) > MAX_BODY_CHARS: - raw_text = raw_text[MAX_BODY_CHARS:] + raw_text = raw_text[:MAX_BODY_CHARS] else: return None -- cgit v1.2.3