update elastic for releases when files added

A bunch of remaining TODOs here

1 files changed, 36 insertions, 1 deletions

diff --git a/python/fatcat_tools/workers/changelog.py b/python/fatcat_tools/workers/changelog.py
index 6319d55a..007b2015 100644
--- a/python/fatcat_tools/workers/changelog.py
+++ b/python/fatcat_tools/workers/changelog.py
@@ -94,9 +94,25 @@ class EntityUpdatesWorker(FatcatWorker):
                 cle = json.loads(msg.value.decode('utf-8'))
                 #print(cle)
                 print("processing changelog index {}".format(cle['index']))
+                release_ids = []
+                file_ids = []
+                container_ids = []
                 release_edits = cle['editgroup']['edits']['releases']
                 for re in release_edits:
-                    ident = re['ident']
+                    release_ids.append(re['ident'])
+                file_edits = cle['editgroup']['edits']['files']
+                for e in file_edits:
+                    file_ids.append(e['ident'])
+                    # update release when a file changes
+                    # TODO: fetch old revision as well, and only update
+                    # releases for which list changed
+                    release_ids.extend(e['release_ids'])
+                container_edits = cle['editgroup']['edits']['containers']
+                for e in container_edits:
+                    container_ids.append(e['ident'])
+
+                # TODO: do these fetches in parallel using a thread pool?
+                for ident in set(release_ids):
                     release = self.api.get_release(ident, expand="files,filesets,webcaptures,container")
                     release_dict = self.api.api_client.sanitize_for_serialization(release)
                     producer.produce(
@@ -104,4 +120,23 @@ class EntityUpdatesWorker(FatcatWorker):
                         partition_key=ident.encode('utf-8'),
                         timestamp=None,
                     )
+                if False: # TODO: other topics, producers
+                    for ident in set(file_ids):
+                        file_entity = self.api.get_file(ident, expand=None)
+                        file_dict = self.api.api_client.sanitize_for_serialization(file_entity)
+                        producer.produce(
+                            message=json.dumps(file_dict).encode('utf-8'),
+                            partition_key=ident.encode('utf-8'),
+                            timestamp=None,
+                        )
+                    for ident in set(container_ids):
+                        container = self.api.get_container(ident)
+                        container_dict = self.api.api_client.sanitize_for_serialization(container)
+                        producer.produce(
+                            message=json.dumps(container_dict).encode('utf-8'),
+                            partition_key=ident.encode('utf-8'),
+                            timestamp=None,
+                        )
+                # TODO: track work_ids for all releases updated
+
                 #consumer.commit_offsets()