summaryrefslogtreecommitdiffstats
path: root/extra/checks/check_extid.sh
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2019-06-04 17:38:01 -0700
committerBryan Newbold <bnewbold@robocracy.org>2019-06-04 17:38:01 -0700
commit84c38d29fc4c7d0dd2e61a5ee9d57ee0a87a61e3 (patch)
treeb2ce9e75ed06d38216aae039ec12c76e8aaea838 /extra/checks/check_extid.sh
parent4fdb056debb07beb36d9bfd0dd358cd398a61a6e (diff)
downloadfatcat-84c38d29fc4c7d0dd2e61a5ee9d57ee0a87a61e3.tar.gz
fatcat-84c38d29fc4c7d0dd2e61a5ee9d57ee0a87a61e3.zip
QA checks (for hash, extid duplication)
Diffstat (limited to 'extra/checks/check_extid.sh')
-rwxr-xr-xextra/checks/check_extid.sh49
1 files changed, 49 insertions, 0 deletions
diff --git a/extra/checks/check_extid.sh b/extra/checks/check_extid.sh
new file mode 100755
index 00000000..f74f50b6
--- /dev/null
+++ b/extra/checks/check_extid.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e -u -o pipefail
+
+export LC_ALL=C
+
+EXTID_FILE=$1
+
+zcat $EXTID_FILE \
+ | awk '{print $3 "\t" $1}' \
+ | rg -v '^\t' \
+ | sort -S 4G \
+ > doi_ident.tsv
+zcat $EXTID_FILE \
+ | awk '{print $4 "\t" $1}' \
+ | rg -v '^\t' \
+ | sort -S 4G \
+ > pmid_ident.tsv
+zcat $EXTID_FILE \
+ | awk '{print $5 "\t" $1}' \
+ | rg -v '^\t' \
+ | sort -S 4G \
+ > pmcid_ident.tsv
+zcat $EXTID_FILE \
+ | awk '{print $6 "\t" $1}' \
+ | rg -v '^\t' \
+ | sort -S 4G \
+ > wikidata_ident.tsv
+
+# these identifiers aren't fixed-width, so we need to join (sigh)
+cut -f1 doi_ident.tsv \
+ | uniq -d \
+ | join -t$'\t' - doi_ident.tsv \
+ > doi_ident.dupes.tsv
+cut -f1 pmid_ident.tsv \
+ | uniq -d \
+ | join -t$'\t' - pmid_ident.tsv \
+ > pmid_ident.dupes.tsv
+cut -f1 pmcid_ident.tsv \
+ | uniq -d \
+ | join -t$'\t' - pmcid_ident.tsv \
+ > pmcid_ident.dupes.tsv
+cut -f1 wikidata_ident.tsv \
+ | uniq -d \
+ | join -t$'\t' - wikidata_ident.tsv \
+ > wikidata_ident.dupes.tsv
+
+wc -l doi_ident.dupes.tsv pmid_ident.dupes.tsv pmcid_ident.dupes.tsv wikidata_ident.dupes.tsv >> counts.txt
+