aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2022-07-15 14:42:00 -0700
committerBryan Newbold <bnewbold@archive.org>2022-07-15 14:42:00 -0700
commit468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13 (patch)
tree0faf3b83cb15595dc7655867fdd0046aae91b6dd
parentfbc9b8e6edd437392f722112904c3bc1d32ff0e8 (diff)
downloadsandcrawler-468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13.tar.gz
sandcrawler-468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13.zip
html: mangled JSON-in-URL pattern
-rw-r--r--python/sandcrawler/html_metadata.py1
1 files changed, 1 insertions, 0 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index c64d4f3..114d9a1 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -653,6 +653,7 @@ FULLTEXT_URL_PATTERNS_SKIP: List[str] = [
"://onlinelibrary.wiley.com/doi/pdf/",
"://doi.org/",
"://dx.doi.org/",
+ "{'embed': '",
]
FULLTEXT_URL_PREFIX_SKIP: List[str] = [