diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-02-26 19:27:26 +0000 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-02-26 19:27:26 +0000 |
| commit | 939ef856b204c29ab6ab4038ff698a402a934c8f (patch) | |
| tree | 745e1e45562daadfe5feb0fcbe73582d8e5eecdb | |
| parent | 0fa705d4e25e329bd976c399acb58cb95d0d424f (diff) | |
| download | sandcrawler-939ef856b204c29ab6ab4038ff698a402a934c8f.tar.gz sandcrawler-939ef856b204c29ab6ab4038ff698a402a934c8f.zip | |
set long timeout on HBaseStatusCountJob
| -rwxr-xr-x | please | 4 |
1 files changed, 3 insertions, 1 deletions
@@ -157,7 +157,9 @@ def run_statuscount(args): datetime.strftime(datetime.now(), "%Y-%m-%d-%H%M.%S")) cmd = """hadoop jar \ scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \ - com.twitter.scalding.Tool sandcrawler.HBaseStatusCountJob \ + com.twitter.scalding.Tool \ + -Dmapred.task.timeout=3600000 \ + sandcrawler.HBaseStatusCountJob \ --hdfs \ --app.conf.path scalding/ia_cluster.conf \ --hbase-table wbgrp-journal-extract-0-{env} \ |