diff options
Diffstat (limited to 'please')
| -rwxr-xr-x | please | 24 |
1 files changed, 23 insertions, 1 deletions
@@ -12,7 +12,7 @@ import subprocess from datetime import datetime HDFS_DIR = "hdfs:///user/bnewbold/sandcrawler" -HBASE_HOST = "wbgrp-svc263.us.archive.org" +HBASE_HOST = "wbgrp-svc350.us.archive.org" ZOOKEEPER_HOSTS = "mtrcs-zk1.us.archive.org:2181" GROBID_URI = "http://wbgrp-svc096.us.archive.org:8070" @@ -487,6 +487,23 @@ def run_dumpungrobided(args): env=args.env) subprocess.call(cmd, shell=True) +def run_sbackfill(args): + if args.rebuild: + rebuild_scalding() + print("Starting scalding backfill job...") + cmd = """hadoop jar \ + scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \ + com.twitter.scalding.Tool sandcrawler.CdxBackfillJob \ + --hdfs \ + --app.conf.path scalding/ia_cluster.conf \ + --hbase-table wbgrp-journal-extract-0-{env} \ + --zookeeper-hosts {zookeeper_hosts} \ + --cdx-input-path {input_cdx}""".format( + input_cdx=args.input_cdx, + zookeeper_hosts=ZOOKEEPER_HOSTS, + env=args.env) + subprocess.call(cmd, shell=True) + def main(): parser = argparse.ArgumentParser() @@ -506,6 +523,11 @@ def main(): sub_backfill.add_argument('input_cdx', help="full HDFS path of CDX file to backfill") + sub_sbackfill = subparsers.add_parser('sbackfill') + sub_sbackfill.set_defaults(func=run_sbackfill) + sub_sbackfill.add_argument('input_cdx', + help="full HDFS path of CDX file to backfill") + sub_extract = subparsers.add_parser('extract') sub_extract.set_defaults(func=run_extract) sub_extract.add_argument('input_cdx', |