From bf0c3399d3a80b32301a3554971a962478614692 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 24 Sep 2018 18:07:15 -0700 Subject: script for partitioning dumps (needs test) --- extra/partition_dumps/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 extra/partition_dumps/README.md (limited to 'extra/partition_dumps/README.md') diff --git a/extra/partition_dumps/README.md b/extra/partition_dumps/README.md new file mode 100644 index 00000000..2e26a41b --- /dev/null +++ b/extra/partition_dumps/README.md @@ -0,0 +1,25 @@ + +This script is used to "partition" (split up) a complete JSON dump by some key. +For example, split release dump JSON lines into separate files, one per +journal/container. + +Example parititoning a sample by release type: + + cat release_dump_expanded_sample.json | jq .release_type -r > release_dump_expanded_sample.release_type + cat release_dump_expanded_sample.release_type | sort | uniq -c | sort -nr > release_dump_expanded_sample.release_type.counts + cat release_dump_expanded_sample.json | paste release_dump_expanded_sample.release_type - | sort > out + +More production-y example using ISSN-L: + + # will append otherwise + rm -rf ./partitioned + + # it's a pretty huge sort, will need 300+ GB scratch space? this might not scale. + zcat release_dump_expanded.json.gz | jq .container.issnl -r > release_dump_expanded.issnl + zcat release_dump_expanded.json.gz | paste release_dump_expanded.issnl - | sort | ./partition_script.py + + # for verification/stats + cat release_dump_expanded.issnl | sort | uniq -c | sort -nr > release_dump_expanded.issnl.counts + + # cleanup + rm release_dump_expanded.issnl -- cgit v1.2.3