diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2018-12-18 14:36:55 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2018-12-18 14:36:55 -0800 |
| commit | e8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca (patch) | |
| tree | 595b3d306970a13fd8c8546646730bfac22af978 | |
| parent | 6e8305e625f8b033d2697d40ed31ec15368678f9 (diff) | |
| download | sandcrawler-e8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca.tar.gz sandcrawler-e8ba7a0bc8d4924f6601b4c82ead58e9f69d8aca.zip | |
longer match-crossref timeout
| -rwxr-xr-x | please | 5 |
1 files changed, 3 insertions, 2 deletions
@@ -189,8 +189,9 @@ def run_matchcrossref(args): com.twitter.scalding.Tool \ -Dmapred.reduce.tasks={reducers} \ -Dcascading.spill.list.threshold=500000 \ - -D mapred.output.compress=false \ - -Dmapred.compress.map.output=true\ + -Dmapred.output.compress=false \ + -Dmapred.compress.map.output=true \ + -Dmapred.task.timeout=3600000 \ sandcrawler.{jobclass} \ --hdfs \ --app.conf.path scalding/ia_cluster.conf \ |