aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src/main
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-08-20 23:46:10 +0000
committerBryan Newbold <bnewbold@archive.org>2018-08-20 23:46:10 +0000
commit531c2939c5e267d29fb3f5e3193eb70b39804953 (patch)
tree1176882127170e99ba0e995219f1a66c211138ad /scalding/src/main
parentaf0fa6edf3c21ac38a8ab4e0fb425e5471e6c3b6 (diff)
downloadsandcrawler-531c2939c5e267d29fb3f5e3193eb70b39804953.tar.gz
sandcrawler-531c2939c5e267d29fb3f5e3193eb70b39804953.zip
add a trap to ScoreJob
Diffstat (limited to 'scalding/src/main')
-rw-r--r--scalding/src/main/scala/sandcrawler/ScoreJob.scala13
1 files changed, 8 insertions, 5 deletions
diff --git a/scalding/src/main/scala/sandcrawler/ScoreJob.scala b/scalding/src/main/scala/sandcrawler/ScoreJob.scala
index 75d45e9..28e9132 100644
--- a/scalding/src/main/scala/sandcrawler/ScoreJob.scala
+++ b/scalding/src/main/scala/sandcrawler/ScoreJob.scala
@@ -13,15 +13,18 @@ class ScoreJob(args: Args) extends JobBase(args) {
val pipe1 : TypedPipe[(String, ReduceFeatures)] = sc1.getInputPipe(args)
val pipe2 : TypedPipe[(String, ReduceFeatures)] = sc2.getInputPipe(args)
- pipe1.join(pipe2).map { entry =>
- val (slug : String, (features1 : ReduceFeatures, features2 : ReduceFeatures)) = entry
- new ReduceOutput(
+ pipe1
+ .addTrap(TypedTsv(args("output") + ".trapped"))
+ .join(pipe2)
+ .map { entry =>
+ val (slug : String, (features1 : ReduceFeatures, features2 : ReduceFeatures)) = entry
+ new ReduceOutput(
slug,
Scorable.computeSimilarity(features1, features2),
features1.json,
features2.json)
- }
- //TypedTsv doesn't work over case classes.
+ }
+ //TypedTsv doesn't work over case classes.
.map { entry => (entry.slug, entry.score, entry.json1, entry.json2) }
.write(TypedTsv[(String, Int, String, String)](args("output")))
}