From efcc947482e9f281bb9c3f3484c29d2c8deffe09 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 3 Nov 2021 16:44:42 -0700 Subject: cleanups: create a separate JsonLinePusher for cleanup workers (distinct base class) --- python/fatcat_tools/cleanups/__init__.py | 2 +- python/fatcat_tools/cleanups/common.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'python/fatcat_tools/cleanups') diff --git a/python/fatcat_tools/cleanups/__init__.py b/python/fatcat_tools/cleanups/__init__.py index 0aeec977..d9438f06 100644 --- a/python/fatcat_tools/cleanups/__init__.py +++ b/python/fatcat_tools/cleanups/__init__.py @@ -1,2 +1,2 @@ -from .common import EntityCleaner +from .common import EntityCleaner, JsonLinePusher from .files import FileCleaner diff --git a/python/fatcat_tools/cleanups/common.py b/python/fatcat_tools/cleanups/common.py index 4e8e49fd..c8ca5800 100644 --- a/python/fatcat_tools/cleanups/common.py +++ b/python/fatcat_tools/cleanups/common.py @@ -1,8 +1,9 @@ import copy import json import subprocess +import sys from collections import Counter -from typing import Any, Dict, List +from typing import Any, Dict, List, Sequence from fatcat_openapi_client import ApiClient, Editgroup @@ -143,3 +144,19 @@ class EntityCleaner: assert self._editgroup_id return self._editgroup_id + + +class JsonLinePusher: + def __init__(self, cleaner: EntityCleaner, json_file: Sequence, **kwargs) -> None: + self.cleaner = cleaner + self.json_file = json_file + + def run(self) -> Counter: + for line in self.json_file: + if not line: + continue + record = json.loads(line) + self.cleaner.push_record(record) + counts = self.cleaner.finish() + print(counts, file=sys.stderr) + return counts -- cgit v1.2.3