aboutsummaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-04-08 15:51:47 -0700
committerBryan Newbold <bnewbold@archive.org>2020-04-08 15:51:51 -0700
commit1552f5dcd8b0abe89f53182c7a495a7b8dc008fc (patch)
tree58c149fca56a814b3d7b46e80f5ce618e7a64380 /bin
parent656da2ece557024c9fc3ca6df9c87caf73741a6c (diff)
downloadfatcat-covid19-1552f5dcd8b0abe89f53182c7a495a7b8dc008fc.tar.gz
fatcat-covid19-1552f5dcd8b0abe89f53182c7a495a7b8dc008fc.zip
helper bash scripts for PDF derivatives
These are lazy (won't run derive process if output already exists), which makes the process much faster
Diffstat (limited to 'bin')
-rwxr-xr-xbin/make_dir_derivatives.sh16
-rwxr-xr-xbin/make_pdf_derivatives.sh22
2 files changed, 38 insertions, 0 deletions
diff --git a/bin/make_dir_derivatives.sh b/bin/make_dir_derivatives.sh
new file mode 100755
index 0000000..c33d75d
--- /dev/null
+++ b/bin/make_dir_derivatives.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e -u -o pipefail
+
+export FULLTEXTDIR=$1
+
+if [ ! -d $FULLTEXTDIR ]; then
+ echo "Directory does not exist: $FULLTEXTDIR"
+ exit -1
+fi
+
+# make directories
+ls $FULLTEXTDIR/pdf/ | parallel mkdir -p $FULLTEXTDIR/pdftotext/{}
+ls $FULLTEXTDIR/pdf/ | parallel mkdir -p $FULLTEXTDIR/thumbnail/{}
+
+fd -I .pdf $FULLTEXTDIR/pdf/ | sed "s/\.pdf//g" | cut -d/ -f3-4 | parallel -j10 ./bin/make_pdf_derivatives.sh $FULLTEXTDIR {}
diff --git a/bin/make_pdf_derivatives.sh b/bin/make_pdf_derivatives.sh
new file mode 100755
index 0000000..8727119
--- /dev/null
+++ b/bin/make_pdf_derivatives.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e -u -o pipefail
+
+export FULLTEXTDIR=$1
+export BLOBPATH=$2
+export PDFPATH="$FULLTEXTDIR/pdf/$BLOBPATH.pdf"
+
+if [[ ! -f $PDFPATH ]]; then
+ echo "PDF does not exist: $PDFPATH"
+ exit -1
+fi
+
+echo "processing: $PDFPATH"
+
+if [[ ! -f "$FULLTEXTDIR/pdftotext/$BLOBPATH.txt" ]]; then
+ pdftotext $PDFPATH $FULLTEXTDIR/pdftotext/$BLOBPATH.txt
+fi
+
+if [[ ! -f "$FULLTEXTDIR/thumbnail/$BLOBPATH.png" ]]; then
+ pdftocairo -png -singlefile -scale-to-x 400 -scale-to-y -1 $PDFPATH $FULLTEXTDIR/thumbnail/$BLOBPATH
+fi