aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorcra14 <chandan.rajah2@bskyb.com>2013-09-04 14:07:19 +0100
committercra14 <chandan.rajah2@bskyb.com>2013-09-04 14:07:19 +0100
commit95f28b46c9edf2257c6fbfbb4b116785c1bca6c2 (patch)
tree61cc19e471ecb933cb4dfe08f5495bb6284f0104
parentee3d7244992ea365743bab3bf2ca82da284512ba (diff)
downloadSpyGlass-95f28b46c9edf2257c6fbfbb4b116785c1bca6c2.tar.gz
SpyGlass-95f28b46c9edf2257c6fbfbb4b116785c1bca6c2.zip
Cosmetic changes
-rw-r--r--README.md96
1 files changed, 48 insertions, 48 deletions
diff --git a/README.md b/README.md
index 6df2fb8..6cb6e4d 100644
--- a/README.md
+++ b/README.md
@@ -263,16 +263,16 @@ e.g.
val hbaseOut = new HBaseRawSource("MY_RESULTS", "hbase-local", Array("out-family"), writeNulls=false, sinkMode = SinkMode.REPLACE)
hbaseSource.read
- .mapTo(('rowkey, 'row) -> ('rowkey, "different_family:col1", 'col2)) {
- x: (ImmutableBytesWritable, Result) =>
- {
- val (rowkey, row) = x
- val col1Times2 = Bytes.toInt(row.getValue(Bytes.toBytes("col-family"), Bytes.toBytes("col1"))) * 2;
- val col2 = row.getValue(Bytes.toBytes("col-family"), Bytes.toBytes("col2"));
- (rowkey, col1Times2, col2)
- }
+ .mapTo(('rowkey, 'row) -> ('rowkey, "different_family:col1", 'col2)) {
+ x: (ImmutableBytesWritable, Result) =>
+ {
+ val (rowkey, row) = x
+ val col1Times2 = Bytes.toInt(row.getValue(Bytes.toBytes("col-family"), Bytes.toBytes("col1"))) * 2;
+ val col2 = row.getValue(Bytes.toBytes("col-family"), Bytes.toBytes("col2"));
+ (rowkey, col1Times2, col2)
}
- .write(hbaseOut)
+ }
+ .write(hbaseOut)
6. Jdbc Tap and Source
===========================
@@ -280,31 +280,31 @@ e.g.
To be added soon.
e.g.
- val jdbcSourceRead = new JDBCSource(
- "TABLE_01",
- "com.mysql.jdbc.Driver",
- "jdbc:mysql://localhost:3306/sky_db?zeroDateTimeBehavior=convertToNull",
- "root",
- "password",
- List("ID", "TEST_COLUMN1", "TEST_COLUMN2", "TEST_COLUMN3"),
- List("bigint(20)", "varchar(45)", "varchar(45)", "bigint(20)"),
- List("id"),
- new Fields("key", "column1", "column2", "column3"),
- null, null, null
- )
-
- val jdbcSourceWrite = new JDBCSource(
- "TABLE_01",
- "com.mysql.jdbc.Driver",
- "jdbc:mysql://localhost:3306/sky_db?zeroDateTimeBehavior=convertToNull",
- "root",
- "password",
- List("ID", "TEST_COLUMN1", "TEST_COLUMN2", "TEST_COLUMN3"),
- List("bigint(20)", "varchar(45)", "varchar(45)", "bigint(20)"),
- List("id"),
- new Fields("key", "column1", "column2", "column3"),
- null, null, null
- )
+ val jdbcSourceRead = new JDBCSource(
+ "TABLE_01",
+ "com.mysql.jdbc.Driver",
+ "jdbc:mysql://localhost:3306/sky_db?zeroDateTimeBehavior=convertToNull",
+ "root",
+ "password",
+ List("ID", "TEST_COLUMN1", "TEST_COLUMN2", "TEST_COLUMN3"),
+ List("bigint(20)", "varchar(45)", "varchar(45)", "bigint(20)"),
+ List("id"),
+ new Fields("key", "column1", "column2", "column3"),
+ null, null, null
+ )
+
+ val jdbcSourceWrite = new JDBCSource(
+ "TABLE_01",
+ "com.mysql.jdbc.Driver",
+ "jdbc:mysql://localhost:3306/sky_db?zeroDateTimeBehavior=convertToNull",
+ "root",
+ "password",
+ List("ID", "TEST_COLUMN1", "TEST_COLUMN2", "TEST_COLUMN3"),
+ List("bigint(20)", "varchar(45)", "varchar(45)", "bigint(20)"),
+ List("id"),
+ new Fields("key", "column1", "column2", "column3"),
+ null, null, null
+ )
7. HBase Delete Functionality
@@ -316,10 +316,10 @@ The feature can be enable by using the parameter sinkMode = SinkMode.REPLACE in
You can use sinkMode = SinkMode.UPDATE to add to the HBaseTable as usual.
e.g.
- val eraser = toIBW(input, TABLE_SCHEMA)
- .write(new HBaseSource( "_TEST.SALT.01", quorum, 'key,
- TABLE_SCHEMA.tail.map((x: Symbol) => "data"),
- TABLE_SCHEMA.tail.map((x: Symbol) => new Fields(x.name)), sinkMode = SinkMode.REPLACE ))
+ val eraser = toIBW(input, TABLE_SCHEMA)
+ .write(new HBaseSource( "_TEST.SALT.01", quorum, 'key,
+ TABLE_SCHEMA.tail.map((x: Symbol) => "data"),
+ TABLE_SCHEMA.tail.map((x: Symbol) => new Fields(x.name)), sinkMode = SinkMode.REPLACE ))
All rows with the key will be deleted. This includes all versions too.
@@ -332,13 +332,13 @@ This feature can be activated by using inputSplitType = SplitType.REGIONAL
You can use inputSplitType = SplitType.GRANULAR to use the previous functionality as is.
e.g.
- val hbase04 = new HBaseSource( "_TEST.SALT.01", quorum, 'key,
- TABLE_SCHEMA.tail.map((x: Symbol) => "data"),
- TABLE_SCHEMA.tail.map((x: Symbol) => new Fields(x.name)),
- sourceMode = SourceMode.SCAN_RANGE, startKey = sttKeyP, stopKey = stpKeyP,
- inputSplitType = splitType).read
- .fromBytesWritable(TABLE_SCHEMA )
- .map(('key, 'salted, 'unsalted) -> 'testData) {x: (String, String, String) => List(x._1, x._2, x._3)}
- .project('testData)
- .write(TextLine("saltTesting/ScanRangeNoSalt01"))
- .groupAll(group => group.toList[List[List[String]]]('testData -> 'testData)) \ No newline at end of file
+ val hbase04 = new HBaseSource( "_TEST.SALT.01", quorum, 'key,
+ TABLE_SCHEMA.tail.map((x: Symbol) => "data"),
+ TABLE_SCHEMA.tail.map((x: Symbol) => new Fields(x.name)),
+ sourceMode = SourceMode.SCAN_RANGE, startKey = sttKeyP, stopKey = stpKeyP,
+ inputSplitType = splitType).read
+ .fromBytesWritable(TABLE_SCHEMA )
+ .map(('key, 'salted, 'unsalted) -> 'testData) {x: (String, String, String) => List(x._1, x._2, x._3)}
+ .project('testData)
+ .write(TextLine("saltTesting/ScanRangeNoSalt01"))
+ .groupAll(group => group.toList[List[List[String]]]('testData -> 'testData)) \ No newline at end of file