From d71cc4e6cd7381f5f0596af1ce33c1bc744c8644 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Tue, 24 Jul 2018 18:01:14 -0700
Subject: temporary please option for scala backfill

---
 please | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

(limited to 'please')

diff --git a/please b/please
index a244b80..119790d 100755
--- a/please
+++ b/please
@@ -116,6 +116,23 @@ def run_statuscount(args):
             env=args.env)
     subprocess.call(cmd, shell=True)
 
+def run_sbackfill(args):
+    if args.rebuild:
+        rebuild_scalding()
+    print("Starting scalding backfill job...")
+    cmd = """hadoop jar \
+        scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \
+        com.twitter.scalding.Tool sandcrawler.CdxBackfillJob \
+        --hdfs \
+        --app.conf.path scalding/ia_cluster.conf \
+        --hbase-table wbgrp-journal-extract-0-{env} \
+        --zookeeper-hosts {zookeeper_hosts} \
+        --cdx-input-path {input_cdx}""".format(
+            input_cdx=args.input_cdx,
+            zookeeper_hosts=ZOOKEEPER_HOSTS,
+            env=args.env)
+    subprocess.call(cmd, shell=True)
+
 def main():
     parser = argparse.ArgumentParser()
 
@@ -135,6 +152,11 @@ def main():
     sub_backfill.add_argument('input_cdx',
         help="full HDFS path of CDX file to backfill")
 
+    sub_sbackfill = subparsers.add_parser('sbackfill')
+    sub_sbackfill.set_defaults(func=run_sbackfill)
+    sub_sbackfill.add_argument('input_cdx',
+        help="full HDFS path of CDX file to backfill")
+
     sub_extract = subparsers.add_parser('extract')
     sub_extract.set_defaults(func=run_extract)
     sub_extract.add_argument('input_cdx',
-- 
cgit v1.2.3