aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-12-18 14:36:55 -0800
committerBryan Newbold <bnewbold@archive.org>2018-12-18 14:36:55 -0800
commite8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca (patch)
tree595b3d306970a13fd8c8546646730bfac22af978
parent6e8305e625f8b033d2697d40ed31ec15368678f9 (diff)
downloadsandcrawler-e8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca.tar.gz
sandcrawler-e8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca.zip
longer match-crossref timeout
-rwxr-xr-xplease5
1 files changed, 3 insertions, 2 deletions
diff --git a/please b/please
index 8b6c040..10c591d 100755
--- a/please
+++ b/please
@@ -189,8 +189,9 @@ def run_matchcrossref(args):
com.twitter.scalding.Tool \
-Dmapred.reduce.tasks={reducers} \
-Dcascading.spill.list.threshold=500000 \
- -D mapred.output.compress=false \
- -Dmapred.compress.map.output=true\
+ -Dmapred.output.compress=false \
+ -Dmapred.compress.map.output=true \
+ -Dmapred.task.timeout=3600000 \
sandcrawler.{jobclass} \
--hdfs \
--app.conf.path scalding/ia_cluster.conf \