diff options
author | Bryan Newbold <bnewbold@archive.org> | 2018-08-24 12:19:09 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2018-08-24 12:19:09 -0700 |
commit | f50d4e081f7994a167c4974ee9d3f6e1f8eae478 (patch) | |
tree | 00cf69ffe345f7766e7477cb9b1f5f7448b4e4fb /scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala | |
parent | 344531eb6a5cdd4ea15e4d82050368c5af0eafee (diff) | |
parent | 5340caad7b39ad29bba77d2a3e486db7a6b1977b (diff) | |
download | sandcrawler-f50d4e081f7994a167c4974ee9d3f6e1f8eae478.tar.gz sandcrawler-f50d4e081f7994a167c4974ee9d3f6e1f8eae478.zip |
Merge branch 'bnewbold-match-quality'
Manually resolved merge conflict in:
please
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala | 29 |
1 files changed, 29 insertions, 0 deletions
diff --git a/scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala b/scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala new file mode 100644 index 0000000..1578258 --- /dev/null +++ b/scalding/src/main/scala/sandcrawler/MatchBenchmarkJob.scala @@ -0,0 +1,29 @@ +package sandcrawler + +import cascading.pipe.Pipe +import com.twitter.scalding.Args +import com.twitter.scalding.TypedPipe +import com.twitter.scalding.TypedTsv +import parallelai.spyglass.base.JobBase + +class MatchBenchmarkJob(args: Args) extends JobBase(args) { + // TODO: Instantiate any subclass of Scorable specified in args. + val sc1 : Scorable = new BibjsonScorable() + val sc2 : Scorable = new BibjsonScorable() + val leftArgs = args + ("bibjson-input" -> List(args("left-bibjson"))) + val rightArgs = args + ("bibjson-input" -> List(args("right-bibjson"))) + val pipe1 : TypedPipe[(String, ReduceFeatures)] = sc1.getInputPipe(leftArgs) + val pipe2 : TypedPipe[(String, ReduceFeatures)] = sc2.getInputPipe(rightArgs) + + pipe1.join(pipe2).map { entry => + val (slug : String, (features1 : ReduceFeatures, features2 : ReduceFeatures)) = entry + new ReduceOutput( + slug, + Scorable.computeSimilarity(features1, features2), + features1.json, + features2.json) + } + //TypedTsv doesn't work over case classes. + .map { entry => (entry.slug, entry.score, entry.json1, entry.json2) } + .write(TypedTsv[(String, Int, String, String)](args("output"))) +} |