diff options
Diffstat (limited to 'scald-mvp')
| -rw-r--r-- | scald-mvp/build.sbt | 7 | ||||
| -rw-r--r-- | scald-mvp/src/main/scala/example/WordCountJob.scala | 2 | ||||
| -rw-r--r-- | scald-mvp/src/test/scala/example/WordCountTest.scala | 36 | 
3 files changed, 43 insertions, 2 deletions
diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt index 5e3d9f7..1db04b9 100644 --- a/scald-mvp/build.sbt +++ b/scald-mvp/build.sbt @@ -1,5 +1,8 @@  import Dependencies._ +val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1 +val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1 +  lazy val root = (project in file(".")).    settings(      inThisBuild(List( @@ -12,7 +15,9 @@ lazy val root = (project in file(".")).      libraryDependencies += scalaTest % Test,      libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6",      libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2", -    libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0", +    libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion, +    libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion, +    libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests",      // cargo-culted from twitter/scalding's build.sbt      // hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952 diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala index 83a8dd0..0e63fed 100644 --- a/scald-mvp/src/main/scala/example/WordCountJob.scala +++ b/scald-mvp/src/main/scala/example/WordCountJob.scala @@ -1,4 +1,4 @@ -package com.twitter.scalding.examples +package example  import com.twitter.scalding._ diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala new file mode 100644 index 0000000..c42770f --- /dev/null +++ b/scald-mvp/src/test/scala/example/WordCountTest.scala @@ -0,0 +1,36 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.scalding + +import org.scalatest.{ Matchers, WordSpec } + +class WordCountTest extends WordSpec with Matchers { +  "A WordCount job" should { +    JobTest(new example.WordCountJob(_)) +      .arg("input", "inputFile") +      .arg("output", "outputFile") +      .source(TextLine("inputFile"), List((0, "hack hack hack and hack"))) +      .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer => +        val outMap = outputBuffer.toMap +        "count words correctly" in { +          outMap("hack") shouldBe 4 +          outMap("and") shouldBe 1 +        } +      } +      .run +      .finish() +  } +}  | 
