From 55a3ca31e0107747d16f95d1c6f900f14c03f5f0 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 19 Aug 2018 19:20:59 -0700 Subject: please support for match-benchmark --- please | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'please') diff --git a/please b/please index c09103f..ce8d34a 100755 --- a/please +++ b/please @@ -144,6 +144,24 @@ def run_matchcrossref(args): crossref_input=args.crossref_input) subprocess.call(cmd, shell=True) +def run_matchbenchmark(args): + if args.rebuild: + rebuild_scalding() + print("Starting matchbenchmark job...") + cmd = """./pig/deps/hadoop/bin/hadoop jar \ + scalding/target/scala-2.11/sandcrawler-assembly-0.2.0-SNAPSHOT.jar \ + com.twitter.scalding.Tool \ + sandcrawler.MatchBenchmarkJob \ + --local \ + --app.conf.path scalding/ia_cluster.conf \ + --left-bibjson {left_bibjson} \ + --right-bibjson {right_bibjson} \ + --output {output}""".format( + output=args.output, + left_bibjson=args.left_bibjson, + right_bibjson=args.right_bibjson) + subprocess.call(cmd, shell=True) + def main(): parser = argparse.ArgumentParser() @@ -182,6 +200,14 @@ def main(): help="number of reducers to run", type=int, default=30) + sub_matchbenchmark = subparsers.add_parser('match-benchmark') + sub_matchbenchmark.set_defaults(func=run_matchbenchmark) + sub_matchbenchmark.add_argument('left_bibjson', + help="First bibjson file") + sub_matchbenchmark.add_argument('right_bibjson', + help="Second bibjson file") + sub_matchbenchmark.add_argument('output', + help="where to write output") args = parser.parse_args() if not args.__dict__.get("func"): -- cgit v1.2.3