From e517b2dd05d29e36b62b3ddfff4537cb980928f0 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 21 May 2018 12:25:13 -0700 Subject: fix WordCountJob package; tests; hadoop version When copying from upstream scalding, forgot to change the path/namespace of the WordCountJob. Production IA cluster is actually running Hadoop 2.5, not 2.6 (I keep forgetting). Pull in more dependencies so test runs (copied from scalding repo, only changed the namespace of the job) --- scald-mvp/build.sbt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'scald-mvp/build.sbt') diff --git a/scald-mvp/build.sbt b/scald-mvp/build.sbt index 5e3d9f7..1db04b9 100644 --- a/scald-mvp/build.sbt +++ b/scald-mvp/build.sbt @@ -1,5 +1,8 @@ import Dependencies._ +val hadoopVersion = "2.5.0" // IA cluster 2018-05-21: 2.5.0-cdh5.3.1 +val hbaseVersion = "0.98.6" // IA cluster 2018-05-21: 0.98.6-cdh5.3.1 + lazy val root = (project in file(".")). settings( inThisBuild(List( @@ -12,7 +15,9 @@ lazy val root = (project in file(".")). libraryDependencies += scalaTest % Test, libraryDependencies += "org.scala-lang" % "scala-library" % "2.11.6", libraryDependencies += "com.twitter" % "scalding-core_2.11" % "0.17.2", - libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.6.0", + libraryDependencies += "org.apache.hadoop" % "hadoop-common" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion, + libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-jobclient" % hadoopVersion classifier "tests", // cargo-culted from twitter/scalding's build.sbt // hint via https://stackoverflow.com/questions/23280494/sbt-assembly-error-deduplicate-different-file-contents-found-in-the-following#23280952 -- cgit v1.2.3