From 2f8a538fb658b3de6bf7c9361bd13f2eb7968abb Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Mon, 20 Aug 2018 23:46:36 +0000
Subject: explicit spill and compression settings for ScoreJob

---
 please | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'please')

diff --git a/please b/please
index 1a992f2..0aad5a5 100755
--- a/please
+++ b/please
@@ -126,10 +126,15 @@ def run_matchcrossref(args):
         datetime.strftime(datetime.now(), "%Y-%m-%d-%H%M.%S"))
     # Notes: -D options must come after Tool but before class name
     # https://github.com/twitter/scalding/wiki/Frequently-asked-questions#how-do-i-pass-parameters-to-my-hadoop-job-number-of-reducers--memory-options--etc-
+    # Compression: changed due to errors in production
+    # https://stackoverflow.com/a/11336820/4682349
     cmd = """hadoop jar \
         scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \
         com.twitter.scalding.Tool \
         -Dmapred.reduce.tasks={reducers} \
+        -Dcascading.spill.list.threshold=500000 \
+        -D mapred.output.compress=false \
+        -Dmapred.compress.map.output=true\
         sandcrawler.ScoreJob \
         --hdfs \
         --app.conf.path scalding/ia_cluster.conf \
-- 
cgit v1.2.3