diff options
| -rw-r--r-- | scald-mvp/build.sbt | 7 | ||||
| -rw-r--r-- | scald-mvp/src/main/scala/example/WordCountJob.scala | 2 | ||||
| -rw-r--r-- | scald-mvp/src/test/scala/example/WordCountTest.scala | 36 |
3 files changed, 43 insertions, 2 deletions
diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt index 5e3d9f7..1db04b9 100644 --- a/scald-mvp/build.sbt +++ b/scald-mvp/build.sbt @@ -1,5 +1,8 @@ import Dependencies._ +val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1 +val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1 + lazy val root = (project in file(".")). settings( inThisBuild(List( @@ -12,7 +15,9 @@ lazy val root = (project in file(".")). libraryDependencies += scalaTest % Test, libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6", libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2", - libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0", + libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", // cargo-culted from twitter/scalding's build.sbt // hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952 diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala index 83a8dd0..0e63fed 100644 --- a/scald-mvp/src/main/scala/example/WordCountJob.scala +++ b/scald-mvp/src/main/scala/example/WordCountJob.scala @@ -1,4 +1,4 @@ -package com.twitter.scalding.examples +package example import com.twitter.scalding._ diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala new file mode 100644 index 0000000..c42770f --- /dev/null +++ b/scald-mvp/src/test/scala/example/WordCountTest.scala @@ -0,0 +1,36 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.scalding + +import org.scalatest.{ Matchers, WordSpec } + +class WordCountTest extends WordSpec with Matchers { + "A WordCount job" should { + JobTest(new example.WordCountJob(_)) + .arg("input", "inputFile") + .arg("output", "outputFile") + .source(TextLine("inputFile"), List((0, "hack hack hack and hack"))) + .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer => + val outMap = outputBuffer.toMap + "count words correctly" in { + outMap("hack") shouldBe 4 + outMap("and") shouldBe 1 + } + } + .run + .finish() + } +} |