aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-02-24 17:38:42 -0800
committerBryan Newbold <bnewbold@archive.org>2020-02-24 17:38:42 -0800
commitbe8f1d134681caaa15485246b65551a67e5bd5a5 (patch)
tree35352f591adb3db736cb5f573d6d39871f7d00cd
parent6a9c1fe64941b4975fa5e52fc3df58b38bf9a0f0 (diff)
downloadsandcrawler-be8f1d134681caaa15485246b65551a67e5bd5a5.tar.gz
sandcrawler-be8f1d134681caaa15485246b65551a67e5bd5a5.zip
ingest: handle missing chemrxvi tag
-rw-r--r--python/sandcrawler/html.py2
1 files changed, 1 insertions, 1 deletions
diff --git a/python/sandcrawler/html.py b/python/sandcrawler/html.py
index e993e74..c76d7a2 100644
--- a/python/sandcrawler/html.py
+++ b/python/sandcrawler/html.py
@@ -303,7 +303,7 @@ def extract_fulltext_url(html_url, html_body):
if "://chemrxiv.org/articles/" in html_url or '.figshare.org/articles/' in html_url:
# <script id="app-data" type="text/json"> [...] </script>
json_tag = soup.find('script', id="app-data", attrs={"type": "text/json"})
- if json_tag.string:
+ if json_tag and json_tag.string:
app_data = json.loads(json_tag.string)
# "exportPdfDownloadUrl": "https://s3-eu-west-1.amazonaws.com/itempdf74155353254prod/10101419/Biradical_Formation_by_Deprotonation_in_Thiazole-Derivatives__The_Hidden_Nature_of_Dasatinib_v1.pdf"
url = app_data.get('article', {}).get('exportPdfDownloadUrl')