summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2020-04-01 12:02:20 -0700
committerBryan Newbold <bnewbold@robocracy.org>2020-04-01 12:02:20 -0700
commitf77a553350238c8ccc9c3bc0edcf47fb9dd067b3 (patch)
tree9cd3fac4da944cd859491d3593a149923948133b
parent6681500eeffe39b7d029a0e0d6b2ed83729f555f (diff)
downloadfatcat-f77a553350238c8ccc9c3bc0edcf47fb9dd067b3.tar.gz
fatcat-f77a553350238c8ccc9c3bc0edcf47fb9dd067b3.zip
importers: replace newlines in get_text() strings
-rw-r--r--python/fatcat_tools/importers/arxiv.py8
-rw-r--r--python/fatcat_tools/importers/jalc.py14
-rw-r--r--python/fatcat_tools/importers/jstor.py14
-rw-r--r--python/fatcat_tools/importers/pubmed.py12
4 files changed, 25 insertions, 23 deletions
diff --git a/python/fatcat_tools/importers/arxiv.py b/python/fatcat_tools/importers/arxiv.py
index 79b242c4..719592fc 100644
--- a/python/fatcat_tools/importers/arxiv.py
+++ b/python/fatcat_tools/importers/arxiv.py
@@ -118,13 +118,13 @@ class ArxivRawImporter(EntityImporter):
if not (doi.startswith('10.') and '/' in doi and doi.split('/')[1]):
sys.stderr.write("BOGUS DOI: {}\n".format(doi))
doi = None
- title = latex_to_text(metadata.title.get_text())
- authors = parse_arxiv_authors(metadata.authors.get_text())
+ title = latex_to_text(metadata.title.get_text().replace('\n', ' '))
+ authors = parse_arxiv_authors(metadata.authors.get_text().replace('\n', ' '))
contribs = [fatcat_openapi_client.ReleaseContrib(index=i, raw_name=a, role='author') for i, a in enumerate(authors)]
lang = "en" # the vast majority in english
if metadata.comments and metadata.comments.get_text():
- comments = metadata.comments.get_text().strip()
+ comments = metadata.comments.get_text().replace('\n', ' ').strip()
extra_arxiv['comments'] = comments
if 'in french' in comments.lower():
lang = 'fr'
@@ -146,7 +146,7 @@ class ArxivRawImporter(EntityImporter):
number = None
if metadata.find('journal-ref') and metadata.find('journal-ref').get_text():
- journal_ref = metadata.find('journal-ref').get_text().strip()
+ journal_ref = metadata.find('journal-ref').get_text().replace('\n', ' ').strip()
extra_arxiv['journal_ref'] = journal_ref
if "conf." in journal_ref.lower() or "proc." in journal_ref.lower():
release_type = "paper-conference"
diff --git a/python/fatcat_tools/importers/jalc.py b/python/fatcat_tools/importers/jalc.py
index 51760f8a..c2adc0d6 100644
--- a/python/fatcat_tools/importers/jalc.py
+++ b/python/fatcat_tools/importers/jalc.py
@@ -35,13 +35,13 @@ def parse_jalc_persons(raw_persons):
for raw in raw_persons:
name = raw.find('name') or None
if name:
- name = clean(name.get_text())
+ name = clean(name.get_text().replace('\n', ' '))
surname = raw.find('familyName') or None
if surname:
- surname = clean(surname.get_text())
+ surname = clean(surname.get_text().replace('\n', ' '))
given_name = raw.find('givenName') or None
if given_name:
- given_name = clean(given_name.get_text())
+ given_name = clean(given_name.get_text().replace('\n', ' '))
lang = 'en'
if is_cjk(name):
lang = 'ja'
@@ -163,12 +163,12 @@ class JalcImporter(EntityImporter):
titles = record.find_all("title")
if not titles:
return None
- title = titles[0].get_text().strip()
+ title = titles[0].get_text().replace('\n', ' ').strip()
original_title = None
if title.endswith('.'):
title = title[:-1]
if len(titles) > 1:
- original_title = titles[1].get_text().strip()
+ original_title = titles[1].get_text().replace('\n', ' ').strip()
if original_title.endswith('.'):
original_title = original_title[:-1]
@@ -242,7 +242,7 @@ class JalcImporter(EntityImporter):
container_extra = dict()
if record.publicationName:
- pubs = [p.get_text().strip() for p in record.find_all("publicationName") if p.get_text()]
+ pubs = [p.get_text().replace('\n', ' ').strip() for p in record.find_all("publicationName") if p.get_text()]
pubs = [clean(p) for p in pubs if p]
assert(pubs)
if len(pubs) > 1 and pubs[0] == pubs[1]:
@@ -255,7 +255,7 @@ class JalcImporter(EntityImporter):
container_extra['original_name'] = clean(pubs[1])
if record.publisher:
- pubs = [p.get_text().strip() for p in record.find_all("publisher") if p.get_text()]
+ pubs = [p.get_text().replace('\n', ' ').strip() for p in record.find_all("publisher") if p.get_text()]
pubs = [p for p in pubs if p]
if len(pubs) > 1 and pubs[0] == pubs[1]:
pubs = [pubs[0]]
diff --git a/python/fatcat_tools/importers/jstor.py b/python/fatcat_tools/importers/jstor.py
index 184a0bb1..96dbf947 100644
--- a/python/fatcat_tools/importers/jstor.py
+++ b/python/fatcat_tools/importers/jstor.py
@@ -64,12 +64,12 @@ class JstorImporter(EntityImporter):
release_type = JSTOR_TYPE_MAP.get(article['article-type'])
title = article_meta.find("article-title")
if title and title.get_text():
- title = title.get_text().strip()
+ title = title.get_text().replace('\n', ' ').strip()
elif title and not title.get_text():
title = None
if not title and release_type.startswith('review') and article_meta.product.source:
- title = "Review: {}".format(article_meta.product.source.get_text())
+ title = "Review: {}".format(article_meta.product.source.replace('\n', ' ').get_text())
if not title:
return None
@@ -96,8 +96,8 @@ class JstorImporter(EntityImporter):
if journal_ids:
extra_jstor['journal_ids'] = journal_ids
- journal_title = journal_meta.find("journal-title").get_text()
- publisher = journal_meta.find("publisher-name").get_text()
+ journal_title = journal_meta.find("journal-title").get_text().replace('\n', ' ')
+ publisher = journal_meta.find("publisher-name").get_text().replace('\n', ' ')
issn = journal_meta.find("issn")
if issn:
issn = issn.string
@@ -141,13 +141,13 @@ class JstorImporter(EntityImporter):
for c in cgroup.find_all("contrib"):
given = c.find("given-names")
if given:
- given = clean(given.get_text())
+ given = clean(given.get_text().replace('\n', ' '))
surname = c.find("surname")
if surname:
- surname = clean(surname.get_text())
+ surname = clean(surname.get_text().replace('\n', ' '))
raw_name = c.find("string-name")
if raw_name:
- raw_name = clean(raw_name.get_text())
+ raw_name = clean(raw_name.get_text().replace('\n', ' '))
if not raw_name:
if given and surname:
diff --git a/python/fatcat_tools/importers/pubmed.py b/python/fatcat_tools/importers/pubmed.py
index 3e9527d4..62bb1ddb 100644
--- a/python/fatcat_tools/importers/pubmed.py
+++ b/python/fatcat_tools/importers/pubmed.py
@@ -394,6 +394,7 @@ class PubmedImporter(EntityImporter):
title = medline.Article.ArticleTitle.get_text() # always present
if title:
+ title = title.replace('\n', ' ')
if title.endswith('.'):
title = title[:-1]
# this hides some "special" titles, but the vast majority are
@@ -407,6 +408,7 @@ class PubmedImporter(EntityImporter):
original_title = medline.Article.find("VernacularTitle", recurse=False)
if original_title:
original_title = original_title.get_text() or None
+ original_title = original_title.replace('\n', ' ')
if original_title and original_title.endswith('.'):
original_title = original_title[:-1]
@@ -558,15 +560,15 @@ class PubmedImporter(EntityImporter):
surname = None
raw_name = None
if author.ForeName:
- given_name = author.ForeName.get_text()
+ given_name = author.ForeName.get_text().replace('\n', ' ')
if author.LastName:
- surname = author.LastName.get_text()
+ surname = author.LastName.get_text().replace('\n', ' ')
if given_name and surname:
raw_name = "{} {}".format(given_name, surname)
elif surname:
raw_name = surname
if not raw_name and author.CollectiveName and author.CollectiveName.get_text():
- raw_name = author.CollectiveName.get_text()
+ raw_name = author.CollectiveName.get_text().replace('\n', ' ')
contrib_extra = dict()
orcid = author.find("Identifier", Source="ORCID")
if orcid:
@@ -588,9 +590,9 @@ class PubmedImporter(EntityImporter):
affiliations = author.find_all("Affiliation")
raw_affiliation = None
if affiliations:
- raw_affiliation = affiliations[0].get_text()
+ raw_affiliation = affiliations[0].get_text().replace('\n', ' ')
if len(affiliations) > 1:
- contrib_extra['more_affiliations'] = [ra.get_text() for ra in affiliations[1:]]
+ contrib_extra['more_affiliations'] = [ra.get_text().replace('\n', ' ') for ra in affiliations[1:]]
if author.find("EqualContrib"):
# TODO: schema for this?
contrib_extra['equal'] = True