From e517b2dd05d29e36b62b3ddfff4537cb980928f0 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Mon, 21 May 2018 12:25:13 -0700
Subject: fix WordCountJob package; tests; hadoop version

When copying from upstream scalding, forgot to change the path/namespace
of the WordCountJob.

Production IA cluster is actually running Hadoop 2.5, not 2.6 (I keep
forgetting).

Pull in more dependencies so test runs (copied from scalding repo, only
changed the namespace of the job)
---
 scald-mvp/build.sbt                                |  7 ++++-
 .../src/main/scala/example/WordCountJob.scala      |  2 +-
 .../src/test/scala/example/WordCountTest.scala     | 36 ++++++++++++++++++++++
 3 files changed, 43 insertions(+), 2 deletions(-)
 create mode 100644 scald-mvp/src/test/scala/example/WordCountTest.scala

(limited to 'scald-mvp')

diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt
index 5e3d9f7..1db04b9 100644
--- a/scald-mvp/build.sbt
+++ b/scald-mvp/build.sbt
@@ -1,5 +1,8 @@
 import Dependencies._
 
+val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1
+val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1
+
 lazy val root = (project in file(".")).
   settings(
     inThisBuild(List(
@@ -12,7 +15,9 @@ lazy val root = (project in file(".")).
     libraryDependencies += scalaTest % Test,
     libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6",
     libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2",
-    libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0",
+    libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion,
+    libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion,
+    libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests",
 
     // cargo-culted from twitter/scalding's build.sbt
     // hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952
diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala
index 83a8dd0..0e63fed 100644
--- a/scald-mvp/src/main/scala/example/WordCountJob.scala
+++ b/scald-mvp/src/main/scala/example/WordCountJob.scala
@@ -1,4 +1,4 @@
-package com.twitter.scalding.examples
+package example
 
 import com.twitter.scalding._
 
diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala
new file mode 100644
index 0000000..c42770f
--- /dev/null
+++ b/scald-mvp/src/test/scala/example/WordCountTest.scala
@@ -0,0 +1,36 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package com.twitter.scalding
+
+import org.scalatest.{ Matchers, WordSpec }
+
+class WordCountTest extends WordSpec with Matchers {
+  "A WordCount job" should {
+    JobTest(new example.WordCountJob(_))
+      .arg("input", "inputFile")
+      .arg("output", "outputFile")
+      .source(TextLine("inputFile"), List((0, "hack hack hack and hack")))
+      .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer =>
+        val outMap = outputBuffer.toMap
+        "count words correctly" in {
+          outMap("hack") shouldBe 4
+          outMap("and") shouldBe 1
+        }
+      }
+      .run
+      .finish()
+  }
+}
-- 
cgit v1.2.3