diff options
Diffstat (limited to 'python/sandcrawler/ia.py')
| -rw-r--r-- | python/sandcrawler/ia.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/python/sandcrawler/ia.py b/python/sandcrawler/ia.py index 3c1d2f9..096d5d4 100644 --- a/python/sandcrawler/ia.py +++ b/python/sandcrawler/ia.py @@ -168,7 +168,9 @@ class CdxApiClient: 'matchType': 'exact', 'limit': -25, 'output': 'json', - 'collapse': 'timestamp:6', + # Collapsing seems efficient, but is complex; would need to include + # other filters and status code in filter + #'collapse': 'timestamp:6', 'filter': '!mimetype:warc/revisit', } if max_age_days: |