aboutsummaryrefslogtreecommitdiffstats
path: root/scald-mvp
diff options
context:
space:
mode:
Diffstat (limited to 'scald-mvp')
-rw-r--r--scald-mvp/build.sbt7
-rw-r--r--scald-mvp/src/main/scala/example/WordCountJob.scala2
-rw-r--r--scald-mvp/src/test/scala/example/WordCountTest.scala36
3 files changed, 43 insertions, 2 deletions
diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt
index 5e3d9f7..1db04b9 100644
--- a/scald-mvp/build.sbt
+++ b/scald-mvp/build.sbt
@@ -1,5 +1,8 @@
import Dependencies._
+val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1
+val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1
+
lazy val root = (project in file(".")).
settings(
inThisBuild(List(
@@ -12,7 +15,9 @@ lazy val root = (project in file(".")).
libraryDependencies += scalaTest % Test,
libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6",
libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2",
- libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0",
+ libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion,
+ libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion,
+ libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests",
// cargo-culted from twitter/scalding's build.sbt
// hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952
diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala
index 83a8dd0..0e63fed 100644
--- a/scald-mvp/src/main/scala/example/WordCountJob.scala
+++ b/scald-mvp/src/main/scala/example/WordCountJob.scala
@@ -1,4 +1,4 @@
-package com.twitter.scalding.examples
+package example
import com.twitter.scalding._
diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala
new file mode 100644
index 0000000..c42770f
--- /dev/null
+++ b/scald-mvp/src/test/scala/example/WordCountTest.scala
@@ -0,0 +1,36 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package com.twitter.scalding
+
+import org.scalatest.{ Matchers, WordSpec }
+
+class WordCountTest extends WordSpec with Matchers {
+ "A WordCount job" should {
+ JobTest(new example.WordCountJob(_))
+ .arg("input", "inputFile")
+ .arg("output", "outputFile")
+ .source(TextLine("inputFile"), List((0, "hack hack hack and hack")))
+ .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer =>
+ val outMap = outputBuffer.toMap
+ "count words correctly" in {
+ outMap("hack") shouldBe 4
+ outMap("and") shouldBe 1
+ }
+ }
+ .run
+ .finish()
+ }
+}