summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xpython/fatcat_import.py2
-rw-r--r--python/fatcat_tools/importers/shadow.py6
2 files changed, 7 insertions, 1 deletions
diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index 1f026edc..843685aa 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -482,7 +482,7 @@ def main():
help="create release and file entities based on GROBID PDF metadata extraction")
sub_shadow_lib.set_defaults(
func=run_shadow_lib,
- auth_var="FATCAT_API_AUTH_TOKEN",
+ auth_var="FATCAT_AUTH_WORKER_SHADOW",
)
sub_shadow_lib.add_argument('json_file',
help="JSON file to import from (or stdin)",
diff --git a/python/fatcat_tools/importers/shadow.py b/python/fatcat_tools/importers/shadow.py
index cfe1b1cf..261cf888 100644
--- a/python/fatcat_tools/importers/shadow.py
+++ b/python/fatcat_tools/importers/shadow.py
@@ -130,6 +130,9 @@ class ShadowLibraryImporter(EntityImporter):
if not existing:
return True
+ if not existing.extra:
+ existing.extra = {}
+
if existing.extra.get('shadows') and list(fe.extra['shadows'].keys())[0] in existing.extra['shadows']:
# already imported from this shadow library; skip
self.counts['exists'] += 1
@@ -172,6 +175,9 @@ class ShadowLibraryImporter(EntityImporter):
existing.sha1 = existing.sha1 or fe.sha1
existing.sha256 = existing.sha256 or fe.sha256
edit = self.api.update_file(self.get_editgroup_id(), existing.ident, existing)
+ # add sha1 to non-entity edit row, so we can do more aggressive
+ # group-level de-dupe
+ edit.sha1 = existing.sha1
self._edits_inflight.append(edit)
self.counts['update'] += 1
return False