From bd9c6564a07eb1b5d9c6a25942a198dc3800281b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 21 Aug 2018 17:14:25 -0700 Subject: add please for keysmissingcolumn --- please | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'please') diff --git a/please b/please index b32dd79..a5e7f15 100755 --- a/please +++ b/please @@ -215,6 +215,30 @@ def run_colcount(args): env=args.env) subprocess.call(cmd, shell=True) +def run_keysmissingcol(args): + if args.rebuild: + rebuild_scalding() + print("Starting keysmissingcol job...") + output = "{}/output-{}/{}-keysmissingcol-{}".format( + HDFS_DIR, + args.env, + datetime.strftime(datetime.now(), "%Y-%m-%d-%H%M.%S"), + args.column.replace(":", "_")) + cmd = """hadoop jar \ + scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \ + com.twitter.scalding.Tool sandcrawler.MissingColDumpJob \ + --hdfs \ + --app.conf.path scalding/ia_cluster.conf \ + --hbase-table wbgrp-journal-extract-0-{env} \ + --zookeeper-hosts {zookeeper_hosts} \ + --column {column} \ + --output {output}""".format( + output=output, + zookeeper_hosts=ZOOKEEPER_HOSTS, + column=args.column, + env=args.env) + subprocess.call(cmd, shell=True) + def main(): parser = argparse.ArgumentParser() @@ -264,6 +288,11 @@ def main(): sub_colcount.add_argument('column', help="column name to use in count") + sub_keysmissingcol = subparsers.add_parser('keys-missing-col') + sub_keysmissingcol.set_defaults(func=run_matchcrossref) + sub_keysmissingcol.add_argument('column', + help="column to SCAN for missing keys") + args = parser.parse_args() if not args.__dict__.get("func"): print("tell me what to do! (try --help)") -- cgit v1.2.3 From 9557323addb8ab0b7f84dc404e8ef3d76243f13f Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 22 Aug 2018 03:02:06 +0000 Subject: fixes to please keys-missing-col --- please | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'please') diff --git a/please b/please index a5e7f15..023eeac 100755 --- a/please +++ b/please @@ -226,7 +226,7 @@ def run_keysmissingcol(args): args.column.replace(":", "_")) cmd = """hadoop jar \ scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \ - com.twitter.scalding.Tool sandcrawler.MissingColDumpJob \ + com.twitter.scalding.Tool sandcrawler.MissingColumnDumpJob \ --hdfs \ --app.conf.path scalding/ia_cluster.conf \ --hbase-table wbgrp-journal-extract-0-{env} \ @@ -289,7 +289,7 @@ def main(): help="column name to use in count") sub_keysmissingcol = subparsers.add_parser('keys-missing-col') - sub_keysmissingcol.set_defaults(func=run_matchcrossref) + sub_keysmissingcol.set_defaults(func=run_keysmissingcol) sub_keysmissingcol.add_argument('column', help="column to SCAN for missing keys") -- cgit v1.2.3