From 5f282a6267182214080ca36bcec4da1755589b46 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@robocracy.org>
Date: Wed, 19 Aug 2020 22:55:05 -0700
Subject: iterate on sitemap generation

---
 extra/sitemap/container_url_lists.sh | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100755 extra/sitemap/container_url_lists.sh

(limited to 'extra/sitemap/container_url_lists.sh')

diff --git a/extra/sitemap/container_url_lists.sh b/extra/sitemap/container_url_lists.sh
new file mode 100755
index 00000000..fcc0f4b6
--- /dev/null
+++ b/extra/sitemap/container_url_lists.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e              # fail on error
+set -u              # fail if variable not set in substitution
+set -o pipefail     # fail if part of a '|' command fails
+
+: ${1?' You you did not supply a date argument'}
+: ${2?' You you did not supply an input file (JSON gzip)'}
+if [ ! -f $2 ] ; then
+  echo "Input file not found: $2" && exit 1;
+fi
+
+# eg, 2020-08-19
+DATE="$1"
+# eg, container_export.json.gz
+EXPORT_FILE_GZ="$2"
+
+zcat $EXPORT_FILE_GZ \
+    | jq .ident -r \
+    | awk '{print "https://fatcat.wiki/container/" $1 }' \
+    | split --lines 20000 - sitemap-containers-$DATE- -d -a 5 --additional-suffix .txt
+
+gzip sitemap-containers-*.txt
-- 
cgit v1.2.3