support extids in matched importer

2 files changed, 30 insertions, 3 deletions

diff --git a/python/fatcat_import.py b/python/fatcat_import.py
index d76f706f..8595d16b 100755
--- a/python/fatcat_import.py
+++ b/python/fatcat_import.py
@@ -62,6 +62,7 @@ def run_matched(args):
     fmi = MatchedImporter(args.api,
         edit_batch_size=args.batch_size,
         editgroup_description=args.editgroup_description_override,
+        default_link_rel=args.default_link_rel,
         default_mimetype=args.default_mimetype)
     JsonLinePusher(fmi, args.json_file).run()
 
@@ -267,6 +268,9 @@ def main():
     sub_matched.add_argument('--bezerk-mode',
         action='store_true',
         help="don't lookup existing files, just insert (clobbers; only for fast bootstrap)")
+    sub_matched.add_argument('--default-link-rel',
+        default="web",
+        help="default URL rel for matches (eg, 'publisher', 'web')")
 
     sub_arabesque_match = subparsers.add_parser('arabesque')
     sub_arabesque_match.set_defaults(
diff --git a/python/fatcat_tools/importers/matched.py b/python/fatcat_tools/importers/matched.py
index 04ce4573..3ef617d3 100644
--- a/python/fatcat_tools/importers/matched.py
+++ b/python/fatcat_tools/importers/matched.py
@@ -4,6 +4,8 @@ import json
 import sqlite3
 import itertools
 import fatcat_client
+
+from fatcat_tools.normal import *
 from .common import EntityImporter, clean, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS
 
 
@@ -53,6 +55,10 @@ class MatchedImporter(EntityImporter):
         # lookup dois
         re_list = set()
         for doi in dois:
+            doi = clean_doi(doi)
+            if not doi:
+                self.counts['skip-bad-doi'] += 1
+                return None
             try:
                 re = self.api.lookup_release(doi=doi)
             except fatcat_client.rest.ApiException as err:
@@ -64,12 +70,28 @@ class MatchedImporter(EntityImporter):
                 pass
             else:
                 re_list.add(re.ident)
+
+        # look up other external ids
+        for extid_type in ('arxiv', 'pmid', 'pmcid', 'jstor', 'wikidata_qid', 'core', 'isbn13', 'ark'):
+            extid = obj.get(extid_type)
+            if extid:
+                try:
+                    re = self.api.lookup_release(**{extid_type: extid})
+                except fatcat_client.rest.ApiException as err:
+                    if err.status != 404:
+                        raise err
+                    re = None
+                if re is None:
+                    pass
+                else:
+                    re_list.add(re.ident)
+
         release_ids = list(re_list)
         if len(release_ids) == 0:
-            self.counts['skip-no-doi'] += 1
+            self.counts['skip-no-releases'] += 1
             return None
         if len(release_ids) > SANE_MAX_RELEASES:
-            self.counts['skip-too-many-dois'] += 1
+            self.counts['skip-too-many-releases'] += 1
             return None
 
         # parse URLs and CDX
@@ -142,11 +164,12 @@ class MatchedImporter(EntityImporter):
             return None
         existing.release_ids = list(set(fe.release_ids + existing.release_ids))
         if len(existing.release_ids) > SANE_MAX_RELEASES:
-            self.counts['skip-update-too-many-url'] += 1
+            self.counts['skip-update-too-many-releases'] += 1
             return None
         existing.mimetype = existing.mimetype or fe.mimetype
         existing.size = existing.size or fe.size
         existing.md5 = existing.md5 or fe.md5
+        existing.sha1 = existing.sha1 or fe.sha1
         existing.sha256 = existing.sha256 or fe.sha256
         self.api.update_file(self.get_editgroup_id(), existing.ident, existing)
         self.counts['update'] += 1