From add030c447fbc2d1a91b0756c395737aa796fef0 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 21 May 2021 16:45:27 -0700
Subject: html ingest: remove whitespace around relative URLs (eg, for d-lib)

---
 python/sandcrawler/html_metadata.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'python')

diff --git a/python/sandcrawler/html_metadata.py b/python/sandcrawler/html_metadata.py
index 1c054fa..1e58778 100644
--- a/python/sandcrawler/html_metadata.py
+++ b/python/sandcrawler/html_metadata.py
@@ -599,7 +599,7 @@ def _extract_generic(doc: HTMLParser, selector: str, attrs: List[str], type_name
                 continue
             url = node.attrs.get(attr)
             if url:
-                resources.append(dict(url=url, type=type_name))
+                resources.append(dict(url=url.strip(), type=type_name))
 
     return resources
 
-- 
cgit v1.2.3