aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-05-21 12:25:13 -0700
committerBryan Newbold <bnewbold@archive.org>2018-05-21 12:25:16 -0700
commite517b2dd05d29e36b62b3ddfff4537cb980928f0 (patch)
tree5bfe1a5c60472c8ae28771626cee31184b3d6e68
parent174e5ec2766dca81e57ee455219b54a689b41e61 (diff)
downloadsandcrawler-e517b2dd05d29e36b62b3ddfff4537cb980928f0.tar.gz
sandcrawler-e517b2dd05d29e36b62b3ddfff4537cb980928f0.zip
fix WordCountJob package; tests; hadoop version
When copying from upstream scalding, forgot to change the path/namespace of the WordCountJob. Production IA cluster is actually running Hadoop 2.5, not 2.6 (I keep forgetting). Pull in more dependencies so test runs (copied from scalding repo, only changed the namespace of the job)
-rw-r--r--scald-mvp/build.sbt7
-rw-r--r--scald-mvp/src/main/scala/example/WordCountJob.scala2
-rw-r--r--scald-mvp/src/test/scala/example/WordCountTest.scala36
3 files changed, 43 insertions, 2 deletions
diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt
index 5e3d9f7..1db04b9 100644
--- a/scald-mvp/build.sbt
+++ b/scald-mvp/build.sbt
@@ -1,5 +1,8 @@
import Dependencies._
+val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1
+val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1
+
lazy val root = (project in file(".")).
settings(
inThisBuild(List(
@@ -12,7 +15,9 @@ lazy val root = (project in file(".")).
libraryDependencies += scalaTest % Test,
libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6",
libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2",
- libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0",
+ libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion,
+ libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion,
+ libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests",
// cargo-culted from twitter/scalding's build.sbt
// hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952
diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala
index 83a8dd0..0e63fed 100644
--- a/scald-mvp/src/main/scala/example/WordCountJob.scala
+++ b/scald-mvp/src/main/scala/example/WordCountJob.scala
@@ -1,4 +1,4 @@
-package com.twitter.scalding.examples
+package example
import com.twitter.scalding._
diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala
new file mode 100644
index 0000000..c42770f
--- /dev/null
+++ b/scald-mvp/src/test/scala/example/WordCountTest.scala
@@ -0,0 +1,36 @@
+/*
+Copyright 2012 Twitter, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package com.twitter.scalding
+
+import org.scalatest.{ Matchers, WordSpec }
+
+class WordCountTest extends WordSpec with Matchers {
+ "A WordCount job" should {
+ JobTest(new example.WordCountJob(_))
+ .arg("input", "inputFile")
+ .arg("output", "outputFile")
+ .source(TextLine("inputFile"), List((0, "hack hack hack and hack")))
+ .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer =>
+ val outMap = outputBuffer.toMap
+ "count words correctly" in {
+ outMap("hack") shouldBe 4
+ outMap("and") shouldBe 1
+ }
+ }
+ .run
+ .finish()
+ }
+}