From f1bf46cac6c59964c285f03bf93c2856eeb231ef Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 3 Dec 2019 16:38:08 -0800 Subject: tweaks to file ingest importer - allow overriding source filter whitelist (common case for CLI use) - fix editgroup description env variable pass-through --- python/fatcat_tools/importers/ingest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'python/fatcat_tools') diff --git a/python/fatcat_tools/importers/ingest.py b/python/fatcat_tools/importers/ingest.py index bb410b63..7dad13ce 100644 --- a/python/fatcat_tools/importers/ingest.py +++ b/python/fatcat_tools/importers/ingest.py @@ -11,9 +11,9 @@ class IngestFileResultImporter(EntityImporter): def __init__(self, api, require_grobid=True, **kwargs): - eg_desc = kwargs.get('editgroup_description', + eg_desc = kwargs.pop('editgroup_description', "Files crawled from web using sandcrawler ingest tool") - eg_extra = kwargs.get('editgroup_extra', dict()) + eg_extra = kwargs.pop('editgroup_extra', dict()) eg_extra['agent'] = eg_extra.get('agent', 'fatcat_tools.IngestFileResultImporter') super().__init__(api, editgroup_description=eg_desc, @@ -29,7 +29,8 @@ class IngestFileResultImporter(EntityImporter): else: print("NOT checking GROBID success") self.ingest_request_source_whitelist = ['fatcat-changelog'] - #self.ingest_request_source_whitelist = [] + if kwargs.get('skip_source_whitelist', False): + self.ingest_request_source_whitelist = [] def want(self, row): """ -- cgit v1.2.3