diff options
Diffstat (limited to 'python/fatcat_tools/cleanups')
-rw-r--r-- | python/fatcat_tools/cleanups/__init__.py | 2 | ||||
-rw-r--r-- | python/fatcat_tools/cleanups/common.py | 19 |
2 files changed, 19 insertions, 2 deletions
diff --git a/python/fatcat_tools/cleanups/__init__.py b/python/fatcat_tools/cleanups/__init__.py index 0aeec977..d9438f06 100644 --- a/python/fatcat_tools/cleanups/__init__.py +++ b/python/fatcat_tools/cleanups/__init__.py @@ -1,2 +1,2 @@ -from .common import EntityCleaner +from .common import EntityCleaner, JsonLinePusher from .files import FileCleaner diff --git a/python/fatcat_tools/cleanups/common.py b/python/fatcat_tools/cleanups/common.py index 4e8e49fd..c8ca5800 100644 --- a/python/fatcat_tools/cleanups/common.py +++ b/python/fatcat_tools/cleanups/common.py @@ -1,8 +1,9 @@ import copy import json import subprocess +import sys from collections import Counter -from typing import Any, Dict, List +from typing import Any, Dict, List, Sequence from fatcat_openapi_client import ApiClient, Editgroup @@ -143,3 +144,19 @@ class EntityCleaner: assert self._editgroup_id return self._editgroup_id + + +class JsonLinePusher: + def __init__(self, cleaner: EntityCleaner, json_file: Sequence, **kwargs) -> None: + self.cleaner = cleaner + self.json_file = json_file + + def run(self) -> Counter: + for line in self.json_file: + if not line: + continue + record = json.loads(line) + self.cleaner.push_record(record) + counts = self.cleaner.finish() + print(counts, file=sys.stderr) + return counts |