diff options
author | Bryan Newbold <bnewbold@archive.org> | 2018-05-21 12:25:13 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2018-05-21 12:25:16 -0700 |
commit | e517b2dd05d29e36b62b3ddfff4537cb980928f0 (patch) | |
tree | 5bfe1a5c60472c8ae28771626cee31184b3d6e68 | |
parent | 174e5ec2766dca81e57ee455219b54a689b41e61 (diff) | |
download | sandcrawler-e517b2dd05d29e36b62b3ddfff4537cb980928f0.tar.gz sandcrawler-e517b2dd05d29e36b62b3ddfff4537cb980928f0.zip |
fix WordCountJob package; tests; hadoop version
When copying from upstream scalding, forgot to change the path/namespace
of the WordCountJob.
Production IA cluster is actually running Hadoop 2.5, not 2.6 (I keep
forgetting).
Pull in more dependencies so test runs (copied from scalding repo, only
changed the namespace of the job)
-rw-r--r-- | scald-mvp/build.sbt | 7 | ||||
-rw-r--r-- | scald-mvp/src/main/scala/example/WordCountJob.scala | 2 | ||||
-rw-r--r-- | scald-mvp/src/test/scala/example/WordCountTest.scala | 36 |
3 files changed, 43 insertions, 2 deletions
diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt index 5e3d9f7..1db04b9 100644 --- a/scald-mvp/build.sbt +++ b/scald-mvp/build.sbt @@ -1,5 +1,8 @@ import Dependencies._ +val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1 +val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1 + lazy val root = (project in file(".")). settings( inThisBuild(List( @@ -12,7 +15,9 @@ lazy val root = (project in file(".")). libraryDependencies += scalaTest % Test, libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6", libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2", - libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0", + libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", // cargo-culted from twitter/scalding's build.sbt // hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952 diff --git a/scald-mvp/src/main/scala/example/WordCountJob.scala b/scald-mvp/src/main/scala/example/WordCountJob.scala index 83a8dd0..0e63fed 100644 --- a/scald-mvp/src/main/scala/example/WordCountJob.scala +++ b/scald-mvp/src/main/scala/example/WordCountJob.scala @@ -1,4 +1,4 @@ -package com.twitter.scalding.examples +package example import com.twitter.scalding._ diff --git a/scald-mvp/src/test/scala/example/WordCountTest.scala b/scald-mvp/src/test/scala/example/WordCountTest.scala new file mode 100644 index 0000000..c42770f --- /dev/null +++ b/scald-mvp/src/test/scala/example/WordCountTest.scala @@ -0,0 +1,36 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.scalding + +import org.scalatest.{ Matchers, WordSpec } + +class WordCountTest extends WordSpec with Matchers { + "A WordCount job" should { + JobTest(new example.WordCountJob(_)) + .arg("input", "inputFile") + .arg("output", "outputFile") + .source(TextLine("inputFile"), List((0, "hack hack hack and hack"))) + .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer => + val outMap = outputBuffer.toMap + "count words correctly" in { + outMap("hack") shouldBe 4 + outMap("and") shouldBe 1 + } + } + .run + .finish() + } +} |