From 807a0d3481f725a4fab81b84f6e7053bde5ef797 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 9 Apr 2020 18:27:47 -0700 Subject: attempt somewhat more robust abstract cleaning Note: there is still a security and robustness issue here in that highlights are marked "safe". Should come up with a better mechanism for escaping/safing. --- fatcat_covid19/transform.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fatcat_covid19/transform.py b/fatcat_covid19/transform.py index 7e1a47f..1d2fa7c 100644 --- a/fatcat_covid19/transform.py +++ b/fatcat_covid19/transform.py @@ -4,6 +4,8 @@ import json import argparse import datetime +from bs4 import BeautifulSoup + from fatcat_covid19.common import * @@ -80,13 +82,8 @@ def fulltext_to_elasticsearch(row, force_bool=True): for a in release['abstracts']: # hack to (partially) clean up common JATS abstract display case - if a.get('mimetype') == 'application/xml+jats': - for tag in ('p', 'b', 'i', 'br', 'jats', 'jats:p', 'jats:title'): - a['content'] = a['content'].replace('<{}>'.format(tag), '') - a['content'] = a['content'].replace(''.format(tag), '') - # ugh, double encoding happens - a['content'] = a['content'].replace('</{}>'.format(tag), '') - a['content'] = a['content'].replace('<{}>'.format(tag), '') + if a.get('mimetype') == 'application/xml+jats' or "