summaryrefslogtreecommitdiffstats
path: root/extra/sitemap/release_url_lists.sh
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2020-08-19 22:55:05 -0700
committerBryan Newbold <bnewbold@robocracy.org>2020-08-19 22:56:31 -0700
commit5f282a6267182214080ca36bcec4da1755589b46 (patch)
treec6f9bd5a84da9e1c2e53aa6af2931df6f9110e60 /extra/sitemap/release_url_lists.sh
parent88a99387e09c7c43803129e72215ef3f6b4cafc6 (diff)
downloadfatcat-5f282a6267182214080ca36bcec4da1755589b46.tar.gz
fatcat-5f282a6267182214080ca36bcec4da1755589b46.zip
iterate on sitemap generation
Diffstat (limited to 'extra/sitemap/release_url_lists.sh')
-rwxr-xr-xextra/sitemap/release_url_lists.sh29
1 files changed, 29 insertions, 0 deletions
diff --git a/extra/sitemap/release_url_lists.sh b/extra/sitemap/release_url_lists.sh
new file mode 100755
index 00000000..4190011f
--- /dev/null
+++ b/extra/sitemap/release_url_lists.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e # fail on error
+set -u # fail if variable not set in substitution
+set -o pipefail # fail if part of a '|' command fails
+
+: ${1?' You you did not supply a date argument'}
+: ${2?' You you did not supply an input file (JSON gzip)'}
+if [ -f $2 ] ; then
+ echo "Input file not found: $2" && exit 1;
+fi
+
+# eg, 2020-08-19
+DATE = "$1"
+# eg, release_export_expanded.json.gz
+EXPORT_FILE_GZ = "$2"
+
+# filter to fulltext releases only, then filter to only one hit per work
+zcat $EXPORT_FILE_GZ \
+ | rg '"release_ids"' \
+ | rg 'archive.org/' \
+ | rg -v '"stub"' \
+ | jq -r '[.work_id, .ident] | @tsv' \
+ | uniq -w 26 \
+ | cut -f 2 \
+ | awk '{print "https://fatcat.wiki/release/" $1 }' \
+ | split --lines 20000 - sitemap-releases-$DATE- -d -a 5 --additional-suffix .txt
+
+gzip sitemap-releases-*.txt