From 83af28c8aa3ba707340b752dd7b619c92d1c5067 Mon Sep 17 00:00:00 2001 From: Antonios Chalkiopoulos Date: Thu, 22 May 2014 22:40:14 +0100 Subject: Adding a full example --- README.md | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2f63d0f..bdf7797 100644 --- a/README.md +++ b/README.md @@ -23,17 +23,17 @@ Building To use SpyGlass as a dependency use the following repository - + conjars.org http://conjars.org/repo - + For Scalding 0.10.0 use : - + parallelai parallelai.spyglass 2.10_0.10_4.3 @@ -382,3 +382,104 @@ e.g. .write(TextLine("saltTesting/ScanRangeNoSalt01")) .groupAll(group => group.toList[List[List[String]]]('testData -> 'testData)) +9. Complete example +=================== + +If using Maven - create a pom.xml + + + + 4.0.0 + + mygroup + myartifact + 1.0-SNAPSHOT + myname + + + + conjars.org + http://conjars.org/repo + + + + + + parallelai + parallelai.spyglass + + 2.10_0.10_4.3 + + + + + + + org.scala-tools + maven-scala-plugin + 2.15.2 + + + scala-compile + + compile + testCompile + + + + -make:transitive + -dependencyfile + ${project.build.directory}/.scala_dependencies + + + + + + + + + + +A single dependency brings in Scalding 0.10.0 (in this example) + + import com.twitter.scalding.{TextLine, Job, Args} + import parallelai.spyglass.hbase.{HBaseSource, HBasePipeConversions} + import cascading.tuple.Fields + import parallelai.spyglass.hbase.HBaseConstants.SourceMode + + /** + * Before executing this example, have a working HBase + * (see https://github.com/kijiproject/kiji-bento for a standalone zero-configuration HBase micro-cluster) + * , and + * + * $ hbase shell + * + * hbase(main):003:0> create 'spyglass.hbase.test1' , 'data' + * hbase(main):006:0> put 'spyglass.hbase.test1' , 'row1' , 'data:column1' , 'value1' + * hbase(main):007:0> put 'spyglass.hbase.test1' , 'row2' , 'data:column1' , 'value2' + * hbase(main):008:0> put 'spyglass.hbase.test1' , 'row3' , 'data:column1' , 'value3' + * hbase(main):009:0> scan 'spyglass.hbase.test1' + * + */ + class HBaseTest(args: Args) extends Job(args) with HBasePipeConversions { + + val SCHEMA = List('key, 'column1) + val tableName = "spyglass.hbase.test1" + val hbaseHost = "localhost:2181" + + val data = new HBaseSource( + tableName, + hbaseHost, + SCHEMA.head, + SCHEMA.tail.map((x: Symbol) => "data"), + SCHEMA.tail.map((x: Symbol) => new Fields(x.name)), + sourceMode = SourceMode.SCAN_ALL) + .read + .fromBytesWritable(SCHEMA) + .debug + .write(TextLine("test_hbase")) + + } + -- cgit v1.2.3