diff options
author | Bryan Newbold <bnewbold@archive.org> | 2018-05-21 12:09:25 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2018-05-21 12:09:27 -0700 |
commit | 174e5ec2766dca81e57ee455219b54a689b41e61 (patch) | |
tree | 12f6ee14b01e5b547308f9782d4871ff94c56f40 /scald-mvp | |
parent | ca8c5dedea7907cf26ec63309f4e99030e644351 (diff) | |
download | sandcrawler-174e5ec2766dca81e57ee455219b54a689b41e61.tar.gz sandcrawler-174e5ec2766dca81e57ee455219b54a689b41e61.zip |
WordCount -> WordCountJob
Also use the exact file from scalding repo
Diffstat (limited to 'scald-mvp')
-rw-r--r-- | scald-mvp/README.md | 2 | ||||
-rw-r--r-- | scald-mvp/src/main/scala/example/WordCount.scala | 12 | ||||
-rw-r--r-- | scald-mvp/src/main/scala/example/WordCountJob.scala | 12 |
3 files changed, 13 insertions, 13 deletions
diff --git a/scald-mvp/README.md b/scald-mvp/README.md index dfa2bca..6c7ec76 100644 --- a/scald-mvp/README.md +++ b/scald-mvp/README.md @@ -25,4 +25,4 @@ running on my laptop: later, using hadop command instead: - hadoop jar scald-mvp-assembly-0.1.0-SNAPSHOT.jar com.twitter.scalding.Tool example.WordCount --hdfs --input hdfs:///user/bnewbold/dummy.txt --output hdfs:///user/bnewbold/test_scalding_out2 + hadoop jar scald-mvp-assembly-0.1.0-SNAPSHOT.jar com.twitter.scalding.Tool example.WordCountJob --hdfs --input hdfs:///user/bnewbold/dummy.txt --output hdfs:///user/bnewbold/test_scalding_out3 diff --git a/scald-mvp/src/main/scala/example/WordCount.scala b/scald-mvp/src/main/scala/example/WordCount.scala deleted file mode 100644 index 0de6ae0..0000000 --- a/scald-mvp/src/main/scala/example/WordCount.scala +++ /dev/null @@ -1,12 +0,0 @@ - -package example - -import com.twitter.scalding._ - -class WordCount(args : Args) extends Job(args) { - TypedPipe.from(TextLine(args("input"))) - .flatMap { line => line.split("""\s+""") } - .groupBy { word => word } - .size - .write(TypedTsv(args("output"))) -} diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala new file mode 100644 index 0000000..83a8dd0 --- /dev/null +++ b/scald-mvp/src/main/scala/example/WordCountJob.scala @@ -0,0 +1,12 @@ +package com.twitter.scalding.examples + +import com.twitter.scalding._ + +class WordCountJob(args: Args) extends Job(args) { + TypedPipe.from(TextLine(args("input"))) + .flatMap { line => line.split("\\s+") } + .map { word => (word, 1L) } + .sumByKey + // The compiler will enforce the type coming out of the sumByKey is the same as the type we have for our sink + .write(TypedTsv[(String, Long)](args("output"))) +} |