From efcc947482e9f281bb9c3f3484c29d2c8deffe09 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@robocracy.org>
Date: Wed, 3 Nov 2021 16:44:42 -0700
Subject: cleanups: create a separate JsonLinePusher for cleanup workers
 (distinct base class)

---
 python/fatcat_cleanup.py                 |  3 +--
 python/fatcat_tools/cleanups/__init__.py |  2 +-
 python/fatcat_tools/cleanups/common.py   | 19 ++++++++++++++++++-
 3 files changed, 20 insertions(+), 4 deletions(-)

(limited to 'python')

diff --git a/python/fatcat_cleanup.py b/python/fatcat_cleanup.py
index 8bcc2ea9..b8d0ee4d 100755
--- a/python/fatcat_cleanup.py
+++ b/python/fatcat_cleanup.py
@@ -7,8 +7,7 @@ import sys
 import raven
 
 from fatcat_tools import authenticated_api
-from fatcat_tools.cleanups import FileCleaner
-from fatcat_tools.importers import JsonLinePusher
+from fatcat_tools.cleanups import FileCleaner, JsonLinePusher
 
 # Yep, a global. Gets DSN from `SENTRY_DSN` environment variable
 sentry_client = raven.Client()
diff --git a/python/fatcat_tools/cleanups/__init__.py b/python/fatcat_tools/cleanups/__init__.py
index 0aeec977..d9438f06 100644
--- a/python/fatcat_tools/cleanups/__init__.py
+++ b/python/fatcat_tools/cleanups/__init__.py
@@ -1,2 +1,2 @@
-from .common import EntityCleaner
+from .common import EntityCleaner, JsonLinePusher
 from .files import FileCleaner
diff --git a/python/fatcat_tools/cleanups/common.py b/python/fatcat_tools/cleanups/common.py
index 4e8e49fd..c8ca5800 100644
--- a/python/fatcat_tools/cleanups/common.py
+++ b/python/fatcat_tools/cleanups/common.py
@@ -1,8 +1,9 @@
 import copy
 import json
 import subprocess
+import sys
 from collections import Counter
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Sequence
 
 from fatcat_openapi_client import ApiClient, Editgroup
 
@@ -143,3 +144,19 @@ class EntityCleaner:
 
         assert self._editgroup_id
         return self._editgroup_id
+
+
+class JsonLinePusher:
+    def __init__(self, cleaner: EntityCleaner, json_file: Sequence, **kwargs) -> None:
+        self.cleaner = cleaner
+        self.json_file = json_file
+
+    def run(self) -> Counter:
+        for line in self.json_file:
+            if not line:
+                continue
+            record = json.loads(line)
+            self.cleaner.push_record(record)
+        counts = self.cleaner.finish()
+        print(counts, file=sys.stderr)
+        return counts
-- 
cgit v1.2.3