From 50270a9152c8e88e66187ce755920e35c31bd0b5 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 27 Oct 2021 13:41:21 -0700 Subject: fileset: refactor out tables of helpers Having these objects invoked in tables resulted in a whole bunch of objects (including children) getting initialized, which seems like the wrong thing to do. Defer this until the actual ingest fileset worker is initialized. --- python/sandcrawler/fileset_strategies.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'python/sandcrawler/fileset_strategies.py') diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index 4e44d97..9d3bae3 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -294,11 +294,3 @@ class WebFileStrategy(WebFilesetStrategy): super().__init__(**kwargs) self.ingest_strategy = IngestStrategy.WebFile self.success_status = "success-file" - - -FILESET_STRATEGY_HELPER_TABLE = { - IngestStrategy.ArchiveorgFileset: ArchiveorgFilesetStrategy(), - IngestStrategy.ArchiveorgFile: ArchiveorgFileStrategy(), - IngestStrategy.WebFileset: WebFilesetStrategy(), - IngestStrategy.WebFile: WebFileStrategy(), -} -- cgit v1.2.3