1 files changed, 16 insertions, 0 deletions
diff --git a/extra/cleanups/check_hashes.sh b/extra/cleanups/check_hashes.sh
new file mode 100755
index 00000000..94102329
--- /dev/null
+++ b/extra/cleanups/check_hashes.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e -u -o pipefail
+
+export LC_ALL=C
+
+HASH_FILE=$1
+
+zcat $HASH_FILE \
+    | awk '{print $3 "\t" $1}' \
+    | rg -v '^\t' \
+    | sort -S 4G \
+    | uniq -d -w 40 \
+    > sha1_ident.dupes.tsv
+
+wc -l sha1_ident.dupes.tsv >> counts.txt