release merger: some progress, but also disable (not complete)

1 files changed, 72 insertions, 12 deletions

diff --git a/python/fatcat_tools/mergers/releases.py b/python/fatcat_tools/mergers/releases.py
index fc970057..1f995b00 100644
--- a/python/fatcat_tools/mergers/releases.py
+++ b/python/fatcat_tools/mergers/releases.py
@@ -15,32 +15,83 @@ from .common import EntityMerger
 class ReleaseMerger(EntityMerger):
     """
     Hard merges a set of release entities, redirecting all entities to a single
-    primary release.
+    primary release. This is different from "grouping" multiple releases under
+    a single work.
 
-    Will also redirect works (if appropriate), and re-point {files, filesets,
-    webcaptures} to the new merged release.
+    A "primary" (which the other releases will redirect to) can be provided, or
+    one will be chosen from the set of duplicate releases based on the
+    completeness of metadata and other heuristic factors.
+
+    Releases are some of the most complex entities to merge, because of
+    the complexity of bibliographic metadata and the number of related entities
+    which also need to be updated.
+
+    File, Fileset, and Webcapture entities which currently point to a release
+    which gets redirected will be updated to point at the "primary" release.
+
+    Any Work entities which will end up with no releases pointing at them after
+    the merging will get redirected to the work corresponding to the "primary"
+    release.
+
+    NOTE: the "primary" release will currently (as implemented) *not* get
+    updated with metadata from all the redirected releases
     """
 
     def __init__(self, api: fatcat_openapi_client.ApiClient, **kwargs) -> None:
 
-        eg_desc = kwargs.get("editgroup_description", "Automated merge of release entities")
+        eg_desc = (
+            kwargs.pop("editgroup_description", None) or "Automated merge of release entities"
+        )
         eg_extra = kwargs.get("editgroup_extra", dict())
         eg_extra["agent"] = eg_extra.get("agent", "fatcat_tools.ReleaseMerger")
+        self.dry_run_mode: bool = eg_extra.get("dry_run_mode", False)
         super().__init__(api, editgroup_description=eg_desc, editgroup_extra=eg_extra, **kwargs)
         self.entity_type_name = "release"
 
+    def choose_primary_release(
+        self, entities: List[ReleaseEntity], existing_redirects: Dict[str, List[str]]
+    ) -> str:
+        assert entities and len(entities) >= 2
+
+        # want to sort in descending order, so reverse=True
+        entities = sorted(
+            entities,
+            key=lambda a: (
+                # number of entities already redirected to this one
+                len(existing_redirects[a.ident]),
+                # number of file/fileset/webcapture entities (would need to update)
+                int(len(a.files or []) + len(a.filesets or []) + len(a.webcaptures or [])),
+                # has a strong identifier?
+                bool(a.ext_id.doi or a.ext_id.pmid or a.ext_id.arxiv_id),
+                # has any identifier?
+                bool(a.ext_id),
+                # has basic metadata?
+                bool(a.release_type),
+                bool(a.release_status),
+                bool(a.release_year),
+                bool(a.container_id),
+                # has refs, abstracts, extra stuff?
+                bool(a.refs),
+                bool(a.abstracts),
+            ),
+            reverse=True,
+        )
+        return entities[0].ident
+
     def try_merge(
         self,
         dupe_ids: List[str],
         primary_id: Optional[str] = None,
         evidence: Optional[Dict[str, Any]] = None,
     ) -> int:
-        """
-        XXX: review/refactor; this code is very old
-        """
+
+        # TODO: this code is pretty old and has only been partially refactored.
+        # Needs more testing and review.
+        raise NotImplementedError
 
         updated_entities = 0
         releases = dict()
+        existing_redirects: Dict[str, List[str]] = dict()
         eg_id = self.get_editgroup_id()
 
         all_ids = dupe_ids.copy()
@@ -48,12 +99,16 @@ class ReleaseMerger(EntityMerger):
             all_ids.append(primary_id)
         for ident in all_ids:
             releases[ident] = self.api.get_release(ident, expand="files,filesets,webcaptures")
+            existing_redirects[ident] = self.api.get_release_redirects(ident)
 
         if not primary_id:
-            # XXX:
-            primary_id = dupe_ids[0]
+            primary_id = self.choose_primary_release(
+                list(releases.values()), existing_redirects
+            )
             dupe_ids = [d for d in dupe_ids if d != primary_id]
 
+        assert primary_id not in dupe_ids
+
         primary_work_id = releases[primary_id].work_id
         updated_work_ids = []
         redirected_release_ids = []
@@ -65,6 +120,7 @@ class ReleaseMerger(EntityMerger):
 
             # file redirects
             for e in release.files:
+                assert release.ident in e.release_ids
                 e.release_ids.remove(release.ident)
                 if primary_id not in e.release_ids:
                     e.release_ids.append(primary_id)
@@ -75,6 +131,7 @@ class ReleaseMerger(EntityMerger):
 
             # fileset redirects
             for e in release.filesets:
+                assert release.ident in e.release_ids
                 e.release_ids.remove(release.ident)
                 if primary_id not in e.release_ids:
                     e.release_ids.append(primary_id)
@@ -85,6 +142,7 @@ class ReleaseMerger(EntityMerger):
 
             # webcapture redirects
             for e in release.webcaptures:
+                assert release.ident in e.release_ids
                 e.release_ids.remove(release.ident)
                 if primary_id not in e.release_ids:
                     e.release_ids.append(primary_id)
@@ -93,12 +151,14 @@ class ReleaseMerger(EntityMerger):
                 updated_entities += 1
                 self.counts["updated-webcaptures"] += 1
 
-            # release redirect itself
+            # the release redirect itself
             updated_work_ids.append(release.work_id)
             redirected_release_ids.append(release.ident)
             if not self.dry_run_mode:
                 self.api.update_release(
-                    eg_id, release.ident, ReleaseEntity(redirect=primary_id)
+                    eg_id,
+                    release.ident,
+                    ReleaseEntity(redirect=primary_id, edit_extra=evidence),
                 )
             updated_entities += 1
             self.counts["updated-releases"] += 1
@@ -108,7 +168,7 @@ class ReleaseMerger(EntityMerger):
         updated_work_ids = list(set(updated_work_ids))
         assert primary_work_id not in updated_work_ids
         for work_id in updated_work_ids:
-            work_releases = self.api.get_work_releases(work_id)
+            work_releases = self.api.get_work_releases(work_id, hide="abstracts,refs")
             rids = set([r.ident for r in work_releases])
             if rids.issubset(redirected_release_ids):
                 # all the releases for this work were updated/merged; we should