diff options
author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-06 16:38:46 -0700 |
---|---|---|
committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-06 16:38:46 -0700 |
commit | 308b33d889d804380427d2aa112efec77b3e1770 (patch) | |
tree | e0181fd4eb2d311bf45827447afe5d93291f931f /scalding/src/main/scala/sandcrawler/Scorable.scala | |
parent | b1d8a72a5cc469b5139d9a976ccfa9b4b3eea61d (diff) | |
download | sandcrawler-308b33d889d804380427d2aa112efec77b3e1770.tar.gz sandcrawler-308b33d889d804380427d2aa112efec77b3e1770.zip |
New code compiles. Old tests pass. New tests not yet written.
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/Scorable.scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/Scorable.scala | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/scalding/src/main/scala/sandcrawler/Scorable.scala b/scalding/src/main/scala/sandcrawler/Scorable.scala index 8e0c560..89dc835 100644 --- a/scalding/src/main/scala/sandcrawler/Scorable.scala +++ b/scalding/src/main/scala/sandcrawler/Scorable.scala @@ -1,6 +1,9 @@ +package sandcrawler + import scala.math import scala.util.parsing.json.JSON +import cascading.flow.FlowDef import com.twitter.scalding._ import com.twitter.scalding.typed.TDsl._ @@ -9,9 +12,9 @@ case class ReduceFeatures(json : String) case class ReduceOutput(val score : Int, json1 : String, json2 : String) abstract class Scorable { - def getInputPipe(args : Args) : TypedPipe[(String, ReduceFeatures)] = + def getInputPipe(args : Args, flowDef : FlowDef, mode : Mode) : TypedPipe[(String, ReduceFeatures)] = { - getFeaturesPipe(args) + getFeaturesPipe(args)(flowDef, mode) .filter { entry => Scorable.isValidSlug(entry.slug) } .groupBy { case MapFeatures(key, slug, json) => slug } .map { tuple => @@ -21,7 +24,7 @@ abstract class Scorable { } // abstract method - def getFeaturesPipe(args : Args) : TypedPipe[MapFeatures] + def getFeaturesPipe(args : Args)(implicit flowDef : FlowDef, mode : Mode) : TypedPipe[MapFeatures] } object Scorable { |