importers: replace newlines in get_text() strings

4 files changed, 25 insertions, 23 deletions

diff --git a/python/fatcat_tools/importers/arxiv.py b/python/fatcat_tools/importers/arxiv.py
index 79b242c4..719592fc 100644
--- a/python/fatcat_tools/importers/arxiv.py
+++ b/python/fatcat_tools/importers/arxiv.py
@@ -118,13 +118,13 @@ class ArxivRawImporter(EntityImporter):
             if not (doi.startswith('10.') and '/' in doi and doi.split('/')[1]):
                 sys.stderr.write("BOGUS DOI: {}\n".format(doi))
                 doi = None
-        title = latex_to_text(metadata.title.get_text())
-        authors = parse_arxiv_authors(metadata.authors.get_text())
+        title = latex_to_text(metadata.title.get_text().replace('\n', ' '))
+        authors = parse_arxiv_authors(metadata.authors.get_text().replace('\n', ' '))
         contribs = [fatcat_openapi_client.ReleaseContrib(index=i, raw_name=a, role='author') for i, a in enumerate(authors)]
 
         lang = "en"     # the vast majority in english
         if metadata.comments and metadata.comments.get_text():
-            comments = metadata.comments.get_text().strip()
+            comments = metadata.comments.get_text().replace('\n', ' ').strip()
             extra_arxiv['comments'] = comments
             if 'in french' in comments.lower():
                 lang = 'fr'
@@ -146,7 +146,7 @@ class ArxivRawImporter(EntityImporter):
 
         number = None
         if metadata.find('journal-ref') and metadata.find('journal-ref').get_text():
-            journal_ref = metadata.find('journal-ref').get_text().strip()
+            journal_ref = metadata.find('journal-ref').get_text().replace('\n', ' ').strip()
             extra_arxiv['journal_ref'] = journal_ref
             if "conf." in journal_ref.lower() or "proc." in journal_ref.lower():
                 release_type = "paper-conference"
diff --git a/python/fatcat_tools/importers/jalc.py b/python/fatcat_tools/importers/jalc.py
index 51760f8a..c2adc0d6 100644
--- a/python/fatcat_tools/importers/jalc.py
+++ b/python/fatcat_tools/importers/jalc.py
@@ -35,13 +35,13 @@ def parse_jalc_persons(raw_persons):
     for raw in raw_persons:
         name = raw.find('name') or None
         if name:
-            name = clean(name.get_text())
+            name = clean(name.get_text().replace('\n', ' '))
         surname = raw.find('familyName') or None
         if surname:
-            surname = clean(surname.get_text())
+            surname = clean(surname.get_text().replace('\n', ' '))
         given_name = raw.find('givenName') or None
         if given_name:
-            given_name = clean(given_name.get_text())
+            given_name = clean(given_name.get_text().replace('\n', ' '))
         lang = 'en'
         if is_cjk(name):
             lang = 'ja'
@@ -163,12 +163,12 @@ class JalcImporter(EntityImporter):
         titles = record.find_all("title")
         if not titles:
             return None
-        title = titles[0].get_text().strip()
+        title = titles[0].get_text().replace('\n', ' ').strip()
         original_title = None
         if title.endswith('.'):
             title = title[:-1]
         if len(titles) > 1:
-            original_title = titles[1].get_text().strip()
+            original_title = titles[1].get_text().replace('\n', ' ').strip()
             if original_title.endswith('.'):
                 original_title = original_title[:-1]
 
@@ -242,7 +242,7 @@ class JalcImporter(EntityImporter):
         container_extra = dict()
 
         if record.publicationName:
-            pubs = [p.get_text().strip() for p in record.find_all("publicationName") if p.get_text()]
+            pubs = [p.get_text().replace('\n', ' ').strip() for p in record.find_all("publicationName") if p.get_text()]
             pubs = [clean(p) for p in pubs if p]
             assert(pubs)
             if len(pubs) > 1 and pubs[0] == pubs[1]:
@@ -255,7 +255,7 @@ class JalcImporter(EntityImporter):
                 container_extra['original_name'] = clean(pubs[1])
 
         if record.publisher:
-            pubs = [p.get_text().strip() for p in record.find_all("publisher") if p.get_text()]
+            pubs = [p.get_text().replace('\n', ' ').strip() for p in record.find_all("publisher") if p.get_text()]
             pubs = [p for p in pubs if p]
             if len(pubs) > 1 and pubs[0] == pubs[1]:
                 pubs = [pubs[0]]
diff --git a/python/fatcat_tools/importers/jstor.py b/python/fatcat_tools/importers/jstor.py
index 184a0bb1..96dbf947 100644
--- a/python/fatcat_tools/importers/jstor.py
+++ b/python/fatcat_tools/importers/jstor.py
@@ -64,12 +64,12 @@ class JstorImporter(EntityImporter):
         release_type = JSTOR_TYPE_MAP.get(article['article-type'])
         title = article_meta.find("article-title")
         if title and title.get_text():
-            title = title.get_text().strip()
+            title = title.get_text().replace('\n', ' ').strip()
         elif title and not title.get_text():
             title = None
 
         if not title and release_type.startswith('review') and article_meta.product.source:
-            title = "Review: {}".format(article_meta.product.source.get_text())
+            title = "Review: {}".format(article_meta.product.source.replace('\n', ' ').get_text())
 
         if not title:
             return None
@@ -96,8 +96,8 @@ class JstorImporter(EntityImporter):
         if journal_ids:
             extra_jstor['journal_ids'] = journal_ids
 
-        journal_title = journal_meta.find("journal-title").get_text()
-        publisher = journal_meta.find("publisher-name").get_text()
+        journal_title = journal_meta.find("journal-title").get_text().replace('\n', ' ')
+        publisher = journal_meta.find("publisher-name").get_text().replace('\n', ' ')
         issn = journal_meta.find("issn")
         if issn:
             issn = issn.string
@@ -141,13 +141,13 @@ class JstorImporter(EntityImporter):
             for c in cgroup.find_all("contrib"):
                 given = c.find("given-names")
                 if given:
-                    given = clean(given.get_text())
+                    given = clean(given.get_text().replace('\n', ' '))
                 surname = c.find("surname")
                 if surname:
-                    surname = clean(surname.get_text())
+                    surname = clean(surname.get_text().replace('\n', ' '))
                 raw_name = c.find("string-name")
                 if raw_name:
-                    raw_name = clean(raw_name.get_text())
+                    raw_name = clean(raw_name.get_text().replace('\n', ' '))
 
                 if not raw_name:
                     if given and surname:
diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py
index 3e9527d4..62bb1ddb 100644
--- a/python/fatcat_tools/importers/pubmed.py
+++ b/python/fatcat_tools/importers/pubmed.py
@@ -394,6 +394,7 @@ class PubmedImporter(EntityImporter):
 
         title = medline.Article.ArticleTitle.get_text() # always present
         if title:
+            title = title.replace('\n', ' ')
             if title.endswith('.'):
                 title = title[:-1]
             # this hides some "special" titles, but the vast majority are
@@ -407,6 +408,7 @@ class PubmedImporter(EntityImporter):
         original_title = medline.Article.find("VernacularTitle", recurse=False)
         if original_title:
             original_title = original_title.get_text() or None
+            original_title = original_title.replace('\n', ' ')
             if original_title and original_title.endswith('.'):
                 original_title = original_title[:-1]
 
@@ -558,15 +560,15 @@ class PubmedImporter(EntityImporter):
                 surname = None
                 raw_name = None
                 if author.ForeName:
-                    given_name = author.ForeName.get_text()
+                    given_name = author.ForeName.get_text().replace('\n', ' ')
                 if author.LastName:
-                    surname = author.LastName.get_text()
+                    surname = author.LastName.get_text().replace('\n', ' ')
                 if given_name and surname:
                     raw_name = "{} {}".format(given_name, surname)
                 elif surname:
                     raw_name = surname
                 if not raw_name and author.CollectiveName and author.CollectiveName.get_text():
-                    raw_name = author.CollectiveName.get_text()
+                    raw_name = author.CollectiveName.get_text().replace('\n', ' ')
                 contrib_extra = dict()
                 orcid = author.find("Identifier", Source="ORCID")
                 if orcid:
@@ -588,9 +590,9 @@ class PubmedImporter(EntityImporter):
                 affiliations = author.find_all("Affiliation")
                 raw_affiliation = None
                 if affiliations:
-                    raw_affiliation = affiliations[0].get_text()
+                    raw_affiliation = affiliations[0].get_text().replace('\n', ' ')
                     if len(affiliations) > 1:
-                        contrib_extra['more_affiliations'] = [ra.get_text() for ra in affiliations[1:]]
+                        contrib_extra['more_affiliations'] = [ra.get_text().replace('\n', ' ') for ra in affiliations[1:]]
                 if author.find("EqualContrib"):
                     # TODO: schema for this?
                     contrib_extra['equal'] = True