aboutsummaryrefslogtreecommitdiffstats
path: root/python
diff options
context:
space:
mode:
Diffstat (limited to 'python')
-rw-r--r--python/sandcrawler/ia.py4
1 files changed, 3 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py
index 3c1d2f9..096d5d4 100644
--- a/python/sandcrawler/ia.py
+++ b/python/sandcrawler/ia.py
@@ -168,7 +168,9 @@ class CdxApiClient:
'matchType': 'exact',
'limit': -25,
'output': 'json',
- 'collapse': 'timestamp:6',
+ # Collapsing seems efficient, but is complex; would need to include
+ # other filters and status code in filter
+ #'collapse': 'timestamp:6',
'filter': '!mimetype:warc/revisit',
}
if max_age_days: