diff options
Diffstat (limited to 'scald-mvp/src/test')
| -rw-r--r-- | scald-mvp/src/test/scala/example/SimpleHBaseSourceExampleTest.scala | 2 | ||||
| -rw-r--r-- | scald-mvp/src/test/scala/sandcrawler/HBaseRowCountTest.scala | 55 | 
2 files changed, 56 insertions, 1 deletions
| diff --git a/scald-mvp/src/test/scala/example/SimpleHBaseSourceExampleTest.scala b/scald-mvp/src/test/scala/example/SimpleHBaseSourceExampleTest.scala index d416af8..41c297d 100644 --- a/scald-mvp/src/test/scala/example/SimpleHBaseSourceExampleTest.scala +++ b/scald-mvp/src/test/scala/example/SimpleHBaseSourceExampleTest.scala @@ -29,7 +29,7 @@ class SimpleHBaseSourceExampleTest extends FunSpec with TupleConversions {      List("3", "kk3", "pp3")    ) -  JobTest("parallelai.spyglass.hbase.example.SimpleHBaseSourceExample") +  JobTest("example.SimpleHBaseSourceExample")      .arg("test", "")      .arg("app.conf.path", "app.conf")      .arg("output", output) diff --git a/scald-mvp/src/test/scala/sandcrawler/HBaseRowCountTest.scala b/scald-mvp/src/test/scala/sandcrawler/HBaseRowCountTest.scala new file mode 100644 index 0000000..c46b0fc --- /dev/null +++ b/scald-mvp/src/test/scala/sandcrawler/HBaseRowCountTest.scala @@ -0,0 +1,55 @@ +package example + +import org.junit.runner.RunWith +import com.twitter.scalding.{JobTest, TupleConversions} +import org.scalatest.FunSpec +import org.scalatest.junit.JUnitRunner +import org.slf4j.LoggerFactory +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import cascading.tuple.{Tuple, Fields} +import org.apache.hadoop.hbase.util.Bytes +import scala._ +import com.twitter.scalding.Tsv +import parallelai.spyglass.hbase.HBaseSource +import parallelai.spyglass.hbase.HBaseConstants.SourceMode + +/** + * Example of how to define tests for HBaseSource + */ +@RunWith(classOf[JUnitRunner]) +class HBaseRowCountTest extends FunSpec with TupleConversions { + +  val output = "/tmp/testOutput" + +  val log = LoggerFactory.getLogger(this.getClass.getName) + +  val sampleData = List( +    List("sha1:K2DKSSVTXWPRMFDTWSTCQW3RVWRIOV3Q"), +    List("sha1:C3YNNEGH5WAG5ZAAXWAEBNXJWT6CZ3WU") +  ) + +  JobTest("sandcrawler.HBaseRowCountJob") +    .arg("test", "") +    .arg("app.conf.path", "app.conf") +    .arg("output", output) +    .arg("debug", "true") +    .source[Tuple]( +    new HBaseSource( +      "table_name", +      "quorum_name:2181", +      new Fields("key"), +      sourceMode = SourceMode.GET_LIST, keyList = List("sha1:K2DKSSVTXWPRMFDTWSTCQW3RVWRIOV3Q", "sha1:C3YNNEGH5WAG5ZAAXWAEBNXJWT6CZ3WU", "3")), +    sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*))) +    .sink[Tuple](Tsv(output format "get_list")) { +      outputBuffer => +        log.debug("Output => " + outputBuffer) + +        it("should return the test data provided.") { +          println("outputBuffer.size => " + outputBuffer.size) +          assert(outputBuffer.size === 3) +        } +    } +    .run +    .finish + +} | 
