diff options
Diffstat (limited to 'scalding/src')
6 files changed, 242 insertions, 0 deletions
diff --git a/scalding/src/main/scala/example/SimpleHBaseSourceExample.scala b/scalding/src/main/scala/example/SimpleHBaseSourceExample.scala new file mode 100644 index 0000000..fe2a120 --- /dev/null +++ b/scalding/src/main/scala/example/SimpleHBaseSourceExample.scala @@ -0,0 +1,36 @@ +package example + +import com.twitter.scalding.{Tsv, Args} +import parallelai.spyglass.base.JobBase +import org.apache.log4j.{Level, Logger} +import parallelai.spyglass.hbase.{HBasePipeConversions, HBaseSource} +import parallelai.spyglass.hbase.HBaseConstants.SourceMode +import cascading.tuple.Fields +import cascading.property.AppProps +import java.util.Properties + +/** + * Simple example of HBaseSource usage + */ +class SimpleHBaseSourceExample(args: Args) extends JobBase(args) with HBasePipeConversions { + + val isDebug: Boolean = args("debug").toBoolean + + if (isDebug) Logger.getRootLogger.setLevel(Level.DEBUG) + + val output = args("output") + + val hbs = new HBaseSource( + "table_name", + //"quorum_name:2181", + "mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?) + new Fields("key"), + List("column_family"), + List(new Fields("column_name1", "column_name2")), + sourceMode = SourceMode.GET_LIST, keyList = List("1", "2", "3")) + .read + .debug + .fromBytesWritable(new Fields("key", "column_name1", "column_name2")) + .write(Tsv(output format "get_list")) + + } diff --git a/scalding/src/main/scala/example/WordCountJob.scala b/scalding/src/main/scala/example/WordCountJob.scala new file mode 100644 index 0000000..0e63fed --- /dev/null +++ b/scalding/src/main/scala/example/WordCountJob.scala @@ -0,0 +1,12 @@ +package example + +import com.twitter.scalding._ + +class WordCountJob(args: Args) extends Job(args) { + TypedPipe.from(TextLine(args("input"))) + .flatMap { line => line.split("\\s+") } + .map { word => (word, 1L) } + .sumByKey + // The compiler will enforce the type coming out of the sumByKey is the same as the type we have for our sink + .write(TypedTsv[(String, Long)](args("output"))) +} diff --git a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala new file mode 100644 index 0000000..5df6b2e --- /dev/null +++ b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala @@ -0,0 +1,41 @@ +package sandcrawler + +import com.twitter.scalding._ +import parallelai.spyglass.base.JobBase +import parallelai.spyglass.hbase.{HBaseSource, HBasePipeConversions, HBaseConstants} +import parallelai.spyglass.hbase.HBaseConstants.SourceMode +import cascading.tuple.Fields +import cascading.property.AppProps +import java.util.Properties + + +class HBaseRowCountJob(args: Args) extends JobBase(args) with HBasePipeConversions { + + + // For now doesn't actually count, just dumps a "word count" + + val output = args("output") + + val hbs = new HBaseSource( + //"table_name", + //"quorum_name:2181", + "wbgrp-journal-extract-0-qa", // HBase Table Name + "mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?) + new Fields("key"), + List("file"), + List(new Fields("size", "mimetype")), + sourceMode = SourceMode.GET_LIST, keyList = List("sha1:K2DKSSVTXWPRMFDTWSTCQW3RVWRIOV3Q", "sha1:C3YNNEGH5WAG5ZAAXWAEBNXJWT6CZ3WU")) + .read + .debug + .fromBytesWritable(new Fields("key")) + .write(Tsv(output format "get_list")) + + /* + List("column_family"), + sourceMode = SourceMode.SCAN_ALL) + .read + .debug + .fromBytesWritable(new Fields("key")) + .write(Tsv(output format "get_list")) + */ +} diff --git a/scalding/src/test/scala/example/SimpleHBaseSourceExampleTest.scala b/scalding/src/test/scala/example/SimpleHBaseSourceExampleTest.scala new file mode 100644 index 0000000..cf068c1 --- /dev/null +++ b/scalding/src/test/scala/example/SimpleHBaseSourceExampleTest.scala @@ -0,0 +1,58 @@ +package example + +import org.junit.runner.RunWith +import com.twitter.scalding.{JobTest, TupleConversions} +import org.scalatest.FunSpec +import org.scalatest.junit.JUnitRunner +import org.slf4j.LoggerFactory +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import cascading.tuple.{Tuple, Fields} +import org.apache.hadoop.hbase.util.Bytes +import scala._ +import com.twitter.scalding.Tsv +import parallelai.spyglass.hbase.HBaseSource +import parallelai.spyglass.hbase.HBaseConstants.SourceMode + +/** + * Example of how to define tests for HBaseSource + */ +@RunWith(classOf[JUnitRunner]) +class SimpleHBaseSourceExampleTest extends FunSpec with TupleConversions { + + val output = "/tmp/testOutput" + + val log = LoggerFactory.getLogger(this.getClass.getName) + + val sampleData = List( + List("1", "kk1", "pp1"), + List("2", "kk2", "pp2"), + List("3", "kk3", "pp3") + ) + + JobTest("example.SimpleHBaseSourceExample") + .arg("test", "") + .arg("app.conf.path", "app.conf") + .arg("output", output) + .arg("debug", "true") + .source[Tuple]( + new HBaseSource( + "table_name", + "mtrcs-zk1.us.archive.org:2181", + new Fields("key"), + List("column_family"), + List(new Fields("column_name1", "column_name2")), + sourceMode = SourceMode.GET_LIST, keyList = List("1", "2", "3")), + sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*))) + .sink[Tuple](Tsv(output format "get_list")) { + outputBuffer => + log.debug("Output => " + outputBuffer) + + it("should return the test data provided.") { + println("outputBuffer.size => " + outputBuffer.size) + assert(outputBuffer.size === 3) + } + } + .run + .finish + +} diff --git a/scalding/src/test/scala/example/WordCountTest.scala b/scalding/src/test/scala/example/WordCountTest.scala new file mode 100644 index 0000000..c42770f --- /dev/null +++ b/scalding/src/test/scala/example/WordCountTest.scala @@ -0,0 +1,36 @@ +/* +Copyright 2012 Twitter, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.twitter.scalding + +import org.scalatest.{ Matchers, WordSpec } + +class WordCountTest extends WordSpec with Matchers { + "A WordCount job" should { + JobTest(new example.WordCountJob(_)) + .arg("input", "inputFile") + .arg("output", "outputFile") + .source(TextLine("inputFile"), List((0, "hack hack hack and hack"))) + .sink[(String, Int)](TypedTsv[(String, Long)]("outputFile")){ outputBuffer => + val outMap = outputBuffer.toMap + "count words correctly" in { + outMap("hack") shouldBe 4 + outMap("and") shouldBe 1 + } + } + .run + .finish() + } +} diff --git a/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala new file mode 100644 index 0000000..598f45d --- /dev/null +++ b/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala @@ -0,0 +1,59 @@ +package example + +import org.junit.runner.RunWith +import com.twitter.scalding.{JobTest, TupleConversions} +import org.scalatest.FunSpec +import org.scalatest.junit.JUnitRunner +import org.slf4j.LoggerFactory +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import cascading.tuple.{Tuple, Fields} +import org.apache.hadoop.hbase.util.Bytes +import scala._ +import com.twitter.scalding.Tsv +import parallelai.spyglass.hbase.HBaseSource +import parallelai.spyglass.hbase.HBaseConstants.SourceMode + +/** + * Example of how to define tests for HBaseSource + */ +@RunWith(classOf[JUnitRunner]) +class HBaseRowCountTest extends FunSpec with TupleConversions { + + val output = "/tmp/testOutput" + + val log = LoggerFactory.getLogger(this.getClass.getName) + + val sampleData = List( + List("sha1:K2DKSSVTXWPRMFDTWSTCQW3RVWRIOV3Q", "a", "b"), + List("sha1:C3YNNEGH5WAG5ZAAXWAEBNXJWT6CZ3WU", "a", "b") + ) + + JobTest("sandcrawler.HBaseRowCountJob") + .arg("test", "") + .arg("app.conf.path", "app.conf") + .arg("output", output) + .arg("debug", "true") + .source[Tuple]( + new HBaseSource( + //"table_name", + //"quorum_name:2181", + "wbgrp-journal-extract-0-qa", + "mtrcs-zk1.us.archive.org:2181", + new Fields("key"), + List("file"), + List(new Fields("size", "mimetype")), + sourceMode = SourceMode.GET_LIST, keyList = List("sha1:K2DKSSVTXWPRMFDTWSTCQW3RVWRIOV3Q", "sha1:C3YNNEGH5WAG5ZAAXWAEBNXJWT6CZ3WU")), + sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*))) + .sink[Tuple](Tsv(output format "get_list")) { + outputBuffer => + log.debug("Output => " + outputBuffer) + + it("should return the test data provided.") { + println("outputBuffer.size => " + outputBuffer.size) + assert(outputBuffer.size === 2) + } + } + .run + .finish + +} |