From e59d1b617d4abd5f002d9e59b6bbaebc9ff30993 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 23 Dec 2019 17:59:10 -0800 Subject: basic shadow importer --- python/fatcat_import.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'python/fatcat_import.py') diff --git a/python/fatcat_import.py b/python/fatcat_import.py index ad4de0e2..1f026edc 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -166,6 +166,11 @@ def run_grobid_metadata(args): bezerk_mode=args.bezerk_mode) LinePusher(fmi, args.tsv_file).run() +def run_shadow_lib(args): + fmi = ShadowLibraryImporter(args.api, + edit_batch_size=100) + JsonLinePusher(fmi, args.json_file).run() + def run_wayback_static(args): api = args.api @@ -473,6 +478,16 @@ def main(): action='store_true', help="don't lookup existing files, just insert (clobbers; only for fast bootstrap)") + sub_shadow_lib = subparsers.add_parser('shadow-lib', + help="create release and file entities based on GROBID PDF metadata extraction") + sub_shadow_lib.set_defaults( + func=run_shadow_lib, + auth_var="FATCAT_API_AUTH_TOKEN", + ) + sub_shadow_lib.add_argument('json_file', + help="JSON file to import from (or stdin)", + default=sys.stdin, type=argparse.FileType('r')) + sub_wayback_static = subparsers.add_parser('wayback-static', help="crude crawl+ingest tool for single-page HTML docs from wayback") sub_wayback_static.set_defaults( -- cgit v1.2.3 From 00754db377df53af18f9c4dddacdeb2e2c559206 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 31 Jan 2020 16:31:57 -0800 Subject: shadow import fixes from QA testing --- python/fatcat_import.py | 2 +- python/fatcat_tools/importers/shadow.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'python/fatcat_import.py') diff --git a/python/fatcat_import.py b/python/fatcat_import.py index 1f026edc..843685aa 100755 --- a/python/fatcat_import.py +++ b/python/fatcat_import.py @@ -482,7 +482,7 @@ def main(): help="create release and file entities based on GROBID PDF metadata extraction") sub_shadow_lib.set_defaults( func=run_shadow_lib, - auth_var="FATCAT_API_AUTH_TOKEN", + auth_var="FATCAT_AUTH_WORKER_SHADOW", ) sub_shadow_lib.add_argument('json_file', help="JSON file to import from (or stdin)", diff --git a/python/fatcat_tools/importers/shadow.py b/python/fatcat_tools/importers/shadow.py index cfe1b1cf..261cf888 100644 --- a/python/fatcat_tools/importers/shadow.py +++ b/python/fatcat_tools/importers/shadow.py @@ -130,6 +130,9 @@ class ShadowLibraryImporter(EntityImporter): if not existing: return True + if not existing.extra: + existing.extra = {} + if existing.extra.get('shadows') and list(fe.extra['shadows'].keys())[0] in existing.extra['shadows']: # already imported from this shadow library; skip self.counts['exists'] += 1 @@ -172,6 +175,9 @@ class ShadowLibraryImporter(EntityImporter): existing.sha1 = existing.sha1 or fe.sha1 existing.sha256 = existing.sha256 or fe.sha256 edit = self.api.update_file(self.get_editgroup_id(), existing.ident, existing) + # add sha1 to non-entity edit row, so we can do more aggressive + # group-level de-dupe + edit.sha1 = existing.sha1 self._edits_inflight.append(edit) self.counts['update'] += 1 return False -- cgit v1.2.3