aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/java/parallelai/spyglass
diff options
context:
space:
mode:
authorChandan Rajah <crajah@parallelai.com>2013-08-21 14:41:54 +0100
committerChandan Rajah <crajah@parallelai.com>2013-08-21 14:41:54 +0100
commit147a423b345ea365c22af48727c83ea4f31b948c (patch)
tree1bcc9de60ab8faebd5c2979eabac0a7accc1f7f9 /src/main/java/parallelai/spyglass
parent466e739d71f426422881986cf018643b7876acf5 (diff)
downloadSpyGlass-147a423b345ea365c22af48727c83ea4f31b948c.tar.gz
SpyGlass-147a423b345ea365c22af48727c83ea4f31b948c.zip
Added minor echanges to code and more testing
Diffstat (limited to 'src/main/java/parallelai/spyglass')
-rw-r--r--src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java4
-rw-r--r--src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java2
-rw-r--r--src/main/java/parallelai/spyglass/hbase/HBaseSalter.java11
3 files changed, 14 insertions, 3 deletions
diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java b/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java
index c145eb0..3c62f82 100644
--- a/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java
+++ b/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java
@@ -52,8 +52,8 @@ FileOutputFormat<ImmutableBytesWritable, Put> implements JobConfigurable {
LOG.error(e);
throw e;
}
- // TODO: Should Autoflush be set to true ????
- table.setAutoFlush(false);
+ // TODO: Should Autoflush be set to true ???? - DONE
+ table.setAutoFlush(true);
HBaseRecordWriter recordWriter = new HBaseRecordWriter(table);
recordWriter.setSinkMode(sinkMode);
return recordWriter;
diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java b/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java
index 832ce95..7b62c88 100644
--- a/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java
+++ b/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java
@@ -278,7 +278,7 @@ public class HBaseRawScheme extends Scheme<JobConf, RecordReader, OutputCollecto
public void copyValue(Result oldValue, Result newValue) {
if (null != oldValue && null != newValue) {
-// oldValue.copyFrom(newValue);
+ oldValue.copyFrom(newValue);
}
}
diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java b/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java
index e3f5dc9..6766458 100644
--- a/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java
+++ b/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java
@@ -171,17 +171,28 @@ public class HBaseSalter {
}
SortedSet<Byte> subSet = prefixSet.subSet(startPrefix, true, stopPrefix, true);
+
+ LOG.info("".format("Prefix subset (%s)", subSet));
return getAllKeys(originalKey, subSet.toArray(new Byte[]{}));
}
public static byte[][] getAllKeys(byte[] originalKey, Byte [] prefixArray) {
+ LOG.info("".format("getAllKeys: OKEY (%s) PARRAY (%s)",
+ Bytes.toString(originalKey), prefixArray ));
+
byte[][] keys = new byte[prefixArray.length][];
for (byte i = 0; i < prefixArray.length; i++) {
keys[i] = Bytes.add(new byte[] {prefixArray[i].byteValue()}, Bytes.add( Bytes.toBytes("_"), originalKey));
}
+ for(int i = 0; i < keys.length; i ++) {
+ for(int j = 0; j < keys[i].length; j++) {
+ LOG.info("" + i + " : " + j + " : " + keys[i][j]);
+ }
+ }
+
return keys;
}