From 468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 15 Jul 2022 14:42:00 -0700
Subject: html: mangled JSON-in-URL pattern

---
 python/sandcrawler/html_metadata.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'python')

diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index c64d4f3..114d9a1 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -653,6 +653,7 @@ FULLTEXT_URL_PATTERNS_SKIP: List[str] = [
     "://onlinelibrary.wiley.com/doi/pdf/",
     "://doi.org/",
     "://dx.doi.org/",
+    "{'embed': '",
 ]
 
 FULLTEXT_URL_PREFIX_SKIP: List[str] = [
-- 
cgit v1.2.3