From 92584ec4201ecc27af423cbff7b4bc1573edf175 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 24 Aug 2018 12:05:39 -0700
Subject: rework fetch_hadoop script

Should work on macOS now, and fetches hadoop in addition to pig. Still
requires wget (not installed by default on macOS).
---
 fetch_hadoop.sh | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100755 fetch_hadoop.sh

(limited to 'fetch_hadoop.sh')

diff --git a/fetch_hadoop.sh b/fetch_hadoop.sh
new file mode 100755
index 0000000..633f8fa
--- /dev/null
+++ b/fetch_hadoop.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# This script was originally only for pig scripts; now it can also be used to
+# run scalding code locally (via please)
+
+set -euo pipefail
+
+#PIG_VERSION="0.12.0-cdh5.2.0"
+# Using more recent version to work around snappy classpath problem
+PIG_VERSION="0.17.0"
+HADOOP_VERSION="2.3.0-cdh5.0.1"
+
+mkdir -p pig/deps/
+cd pig/deps/
+
+# Fetch Hadoop Command
+echo https://archive.cloudera.com/cdh5/cdh/5/hadoop-${HADOOP_VERSION}.tar.gz
+#wget -c https://archive.cloudera.com/cdh5/cdh/5/pig-${HADOOP_VERSION}.tar.gz
+#wget -c https://archive.cloudera.com/cdh5/cdh/5/pig-${HADOOP_VERSION}.tar.gz
+wget -c https://archive.org/serve/hadoop_pig_mirror/hadoop-${HADOOP_VERSION}.tar.gz
+echo "Extracting Hadoop (takes a minute)..."
+tar xvf hadoop-${HADOOP_VERSION}.tar.gz > /dev/null
+ln -fs hadoop-${HADOOP_VERSION} hadoop
+
+# Fetch Pig
+#wget -c https://archive.cloudera.com/cdh5/cdh/5/pig-${PIG_VERSION}.tar.gz
+#wget -c http://mirror.metrocast.net/apache/pig/pig-${PIG_VERSION}/pig-${PIG_VERSION}.tar.gz
+wget -c https://archive.org/serve/hadoop_pig_mirror/pig-${PIG_VERSION}.tar.gz
+echo "Extracting Pig (takes a minute)..."
+tar xvf pig-${PIG_VERSION}.tar.gz > /dev/null
+ln -fs pig-${PIG_VERSION} pig
+
+# No 'readlink -f' on macOS
+# https://stackoverflow.com/a/24572274/4682349
+JAVA_HOME=$(perl -MCwd -e 'print Cwd::abs_path shift' /usr/bin/java | sed "s:bin/java::")
+./pig/bin/pig -x local -version
+./hadoop/bin/hadoop version
+
-- 
cgit v1.2.3