summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2020-03-02 16:41:02 -0800
committerBryan Newbold <bnewbold@robocracy.org>2020-03-02 16:41:02 -0800
commitf4cce5a765a9f80f9c5e9c907689c06dc9ebf102 (patch)
treeae2b668c680c4163845e9b06b271f823aa963239
parent3ff5c1f371fb6764df415e816884e1be0577088c (diff)
downloadfatcat-f4cce5a765a9f80f9c5e9c907689c06dc9ebf102.tar.gz
fatcat-f4cce5a765a9f80f9c5e9c907689c06dc9ebf102.zip
add --force-crawl flag to ingest tool
-rwxr-xr-xpython/fatcat_ingest.py5
1 files changed, 5 insertions, 0 deletions
diff --git a/python/fatcat_ingest.py b/python/fatcat_ingest.py
index 6fda74c5..9ba95015 100755
--- a/python/fatcat_ingest.py
+++ b/python/fatcat_ingest.py
@@ -90,6 +90,8 @@ def _run_search_dump(args, search):
)
if not ingest_request:
continue
+ if args.force_recrawl:
+ ingest_request['force_recrawl'] = True
counts['ingest_request'] += 1
if args.dry_run:
continue
@@ -206,6 +208,9 @@ def main():
parser.add_argument('--allow-non-oa',
action='store_true',
help="By default, we limit to OA releases. This removes that filter")
+ parser.add_argument('--force-recrawl',
+ action='store_true',
+ help="Tell ingest worker to skip GWB history lookup and do SPNv2 crawl")
subparsers = parser.add_subparsers()
sub_container = subparsers.add_parser('container',