diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2022-07-15 14:42:00 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2022-07-15 14:42:00 -0700 |
| commit | 468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13 (patch) | |
| tree | 0faf3b83cb15595dc7655867fdd0046aae91b6dd | |
| parent | fbc9b8e6edd437392f722112904c3bc1d32ff0e8 (diff) | |
| download | sandcrawler-468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13.tar.gz sandcrawler-468a7b6f91c5b3b0a80e3c82c408b8e02ce71e13.zip | |
html: mangled JSON-in-URL pattern
| -rw-r--r-- | python/sandcrawler/html_metadata.py | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py index c64d4f3..114d9a1 100644 --- a/python/sandcrawler/html_metadata.py +++ b/python/sandcrawler/html_metadata.py @@ -653,6 +653,7 @@ FULLTEXT_URL_PATTERNS_SKIP: List[str] = [ "://onlinelibrary.wiley.com/doi/pdf/", "://doi.org/", "://dx.doi.org/", + "{'embed': '", ] FULLTEXT_URL_PREFIX_SKIP: List[str] = [ |