From d6c744002607d8789927cc98fe6c8a6a76da24bd Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 26 Dec 2019 21:19:02 -0800 Subject: filter ingest results to not have key conflicts within batch This handles a corner case with ON CONFLICT ... DO UPDATE where you can't do multiple such updates in the same batch transaction. --- python/sandcrawler/persist.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'python') diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index 3b9cde9..ea54d6b 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -156,8 +156,23 @@ class PersistIngestFileResultWorker(SandcrawlerWorker): if not batch: return [] + # need to ensure that we aren't trying to update same row multiple + # times in same batch (!) results = [self.file_result_to_row(raw) for raw in batch] - results = [r for r in results if r] + results.reverse() + clean_results = [] + result_keys = [] + for r in results: + if not r: + continue + key = (r['ingest_type'], r['base_url']) + if key in result_keys: + self.counts['skip-duplicate-result'] += 1 + continue + result_keys.append(key) + clean_results.append(r) + results = clean_results + requests = [self.request_to_row(raw['request']) for raw in batch if raw.get('request')] requests = [r for r in requests if r] -- cgit v1.2.3