aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-05-21 12:09:25 -0700
committerBryan Newbold <bnewbold@archive.org>2018-05-21 12:09:27 -0700
commit174e5ec2766dca81e57ee455219b54a689b41e61 (patch)
tree12f6ee14b01e5b547308f9782d4871ff94c56f40
parentca8c5dedea7907cf26ec63309f4e99030e644351 (diff)
downloadsandcrawler-174e5ec2766dca81e57ee455219b54a689b41e61.tar.gz
sandcrawler-174e5ec2766dca81e57ee455219b54a689b41e61.zip
WordCount -> WordCountJob
Also use the exact file from scalding repo
-rw-r--r--scald-mvp/README.md2
-rw-r--r--scald-mvp/src/main/scala/example/WordCount.scala12
-rw-r--r--scald-mvp/src/main/scala/example/WordCountJob.scala12
3 files changed, 13 insertions, 13 deletions
diff --git a/scald-mvp/README.md b/scald-mvp/README.md
index dfa2bca..6c7ec76 100644
--- a/scald-mvp/README.md
+++ b/scald-mvp/README.md
@@ -25,4 +25,4 @@ running on my laptop:
later, using hadop command instead:
- hadoop jar scald-mvp-assembly-0.1.0-SNAPSHOT.jar com.twitter.scalding.Tool example.WordCount --hdfs --input hdfs:///user/bnewbold/dummy.txt --output hdfs:///user/bnewbold/test_scalding_out2
+ hadoop jar scald-mvp-assembly-0.1.0-SNAPSHOT.jar com.twitter.scalding.Tool example.WordCountJob --hdfs --input hdfs:///user/bnewbold/dummy.txt --output hdfs:///user/bnewbold/test_scalding_out3
diff --git a/scald-mvp/src/main/scala/example/WordCount.scala b/scald-mvp/src/main/scala/example/WordCount.scala
deleted file mode 100644
index 0de6ae0..0000000
--- a/scald-mvp/src/main/scala/example/WordCount.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-
-package example
-
-import com.twitter.scalding._
-
-class WordCount(args : Args) extends Job(args) {
- TypedPipe.from(TextLine(args("input")))
- .flatMap { line => line.split("""\s+""") }
- .groupBy { word => word }
- .size
- .write(TypedTsv(args("output")))
-}
diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala
new file mode 100644
index 0000000..83a8dd0
--- /dev/null
+++ b/scald-mvp/src/main/scala/example/WordCountJob.scala
@@ -0,0 +1,12 @@
+package com.twitter.scalding.examples
+
+import com.twitter.scalding._
+
+class WordCountJob(args: Args) extends Job(args) {
+ TypedPipe.from(TextLine(args("input")))
+ .flatMap { line => line.split("\\s+") }
+ .map { word => (word, 1L) }
+ .sumByKey
+ // The compiler will enforce the type coming out of the sumByKey is the same as the type we have for our sink
+ .write(TypedTsv[(String, Long)](args("output")))
+}