aboutsummaryrefslogtreecommitdiffstats
path: root/please
diff options
context:
space:
mode:
Diffstat (limited to 'please')
-rwxr-xr-xplease22
1 files changed, 22 insertions, 0 deletions
diff --git a/please b/please
index a244b80..119790d 100755
--- a/please
+++ b/please
@@ -116,6 +116,23 @@ def run_statuscount(args):
env=args.env)
subprocess.call(cmd, shell=True)
+def run_sbackfill(args):
+ if args.rebuild:
+ rebuild_scalding()
+ print("Starting scalding backfill job...")
+ cmd = """hadoop jar \
+ scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \
+ com.twitter.scalding.Tool sandcrawler.CdxBackfillJob \
+ --hdfs \
+ --app.conf.path scalding/ia_cluster.conf \
+ --hbase-table wbgrp-journal-extract-0-{env} \
+ --zookeeper-hosts {zookeeper_hosts} \
+ --cdx-input-path {input_cdx}""".format(
+ input_cdx=args.input_cdx,
+ zookeeper_hosts=ZOOKEEPER_HOSTS,
+ env=args.env)
+ subprocess.call(cmd, shell=True)
+
def main():
parser = argparse.ArgumentParser()
@@ -135,6 +152,11 @@ def main():
sub_backfill.add_argument('input_cdx',
help="full HDFS path of CDX file to backfill")
+ sub_sbackfill = subparsers.add_parser('sbackfill')
+ sub_sbackfill.set_defaults(func=run_sbackfill)
+ sub_sbackfill.add_argument('input_cdx',
+ help="full HDFS path of CDX file to backfill")
+
sub_extract = subparsers.add_parser('extract')
sub_extract.set_defaults(func=run_extract)
sub_extract.add_argument('input_cdx',