From 4d701f4f2ea99ac95bd4235adef1998f3abdc9f9 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 6 May 2020 18:26:53 -0700 Subject: start a Makefile Move all "index" functions into classes, each in a separate file. Add lots of type annotations. Use dataclass objects to hold database rows. This aspect will need further refactoring to remove "extra" usage, probably by adding database rows to align with DatabaseInfo more closely. --- chocula/common.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 chocula/common.py (limited to 'chocula/common.py') diff --git a/chocula/common.py b/chocula/common.py new file mode 100644 index 0000000..54856c9 --- /dev/null +++ b/chocula/common.py @@ -0,0 +1,35 @@ + +import sys +from typing import Iterable, Optional +from collections import Counter + +from chocula.config import ChoculaConfig +from chocula.database import DirectoryInfo + + +class DirectoryLoader(): + + source_slug: str = "GENERIC" + + def __init__(self, config: ChoculaConfig): + self.config = config + + def open_file(self) -> Iterable: + raise NotImplementedError() + + def parse_record(self, record) -> Optional[DirectoryInfo]: + raise NotImplementedError() + + def index_file(self, db) -> Counter: + print(f"##### Loading {self.source_slug}...", file=sys.stderr) + counts: Counter = Counter() + cur = db.db.cursor() + for record in self.open_file(): + counts['total'] += 1 + info = self.parse_record(record) + if info: + status = db.insert_directory(info, cur=cur) + counts[status] += 1 + cur.close() + db.db.commit() + return counts -- cgit v1.2.3