diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2021-10-15 17:13:39 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2021-10-15 18:15:29 -0700 |
| commit | 13d56402cf5e01ca8e2306f85dbcc3b3f92a94d7 (patch) | |
| tree | 853a3dc60dcf3bd635be0816ff59b23f0975ae7d /python/sandcrawler/persist.py | |
| parent | a09396caefe709b521e560add5b01c1a5c94cb53 (diff) | |
| download | sandcrawler-13d56402cf5e01ca8e2306f85dbcc3b3f92a94d7.tar.gz sandcrawler-13d56402cf5e01ca8e2306f85dbcc3b3f92a94d7.zip | |
improve fileset ingest integration with file ingest
Diffstat (limited to 'python/sandcrawler/persist.py')
| -rw-r--r-- | python/sandcrawler/persist.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index ee153ab..7fe59f1 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -189,7 +189,7 @@ class PersistIngestFileResultWorker(SandcrawlerWorker): results = [r for r in results if r] requests = [self.request_to_row(raw['request']) for raw in batch if raw.get('request')] - requests = [r for r in requests if r] + requests = [r for r in requests if r and r['ingest_type'] != 'dataset-file'] if requests: resp = self.db.insert_ingest_request(self.cur, requests) |