From 6443fd5756779e2cca986e73e5008a073ef96452 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 27 Jul 2020 15:54:55 -0700 Subject: json: exclude None in output, and sort keys These are both size/performance enhancements. Not including 'None' values will reduce document sizes on-disk and over network, particularly for intermediate objects. Sorting by key should improve compression ratios across multiple documents, both on-disk (gzip) and in elasticsearch itself: https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-disk-usage.html#_put_fields_in_the_same_order_in_documents --- fatcat_scholar/work_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fatcat_scholar/work_pipeline.py') diff --git a/fatcat_scholar/work_pipeline.py b/fatcat_scholar/work_pipeline.py index accbc1d..b2ceaf8 100644 --- a/fatcat_scholar/work_pipeline.py +++ b/fatcat_scholar/work_pipeline.py @@ -355,7 +355,7 @@ class WorkPipeline: continue if batch: ib = self.process_release_list(batch) - print(ib.json()) + print(ib.json(exclude_none=True)) batch_work_id = None batch = [ release, @@ -364,7 +364,7 @@ class WorkPipeline: if batch: ib = self.process_release_list(batch) - print(ib.json()) + print(ib.json(exclude_none=True, sort_keys=True)) def main() -> None: -- cgit v1.2.3