From 13d56402cf5e01ca8e2306f85dbcc3b3f92a94d7 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 15 Oct 2021 17:13:39 -0700 Subject: improve fileset ingest integration with file ingest --- python/sandcrawler/persist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/sandcrawler/persist.py') diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index ee153ab..7fe59f1 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -189,7 +189,7 @@ class PersistIngestFileResultWorker(SandcrawlerWorker): results = [r for r in results if r] requests = [self.request_to_row(raw['request']) for raw in batch if raw.get('request')] - requests = [r for r in requests if r] + requests = [r for r in requests if r and r['ingest_type'] != 'dataset-file'] if requests: resp = self.db.insert_ingest_request(self.cur, requests) -- cgit v1.2.3