diff options
| author | Chandan Rajah <crajah@parallelai.com> | 2013-08-21 14:41:54 +0100 |
|---|---|---|
| committer | Chandan Rajah <crajah@parallelai.com> | 2013-08-21 14:41:54 +0100 |
| commit | 147a423b345ea365c22af48727c83ea4f31b948c (patch) | |
| tree | 1bcc9de60ab8faebd5c2979eabac0a7accc1f7f9 /src/main/java/parallelai/spyglass | |
| parent | 466e739d71f426422881986cf018643b7876acf5 (diff) | |
| download | SpyGlass-147a423b345ea365c22af48727c83ea4f31b948c.tar.gz SpyGlass-147a423b345ea365c22af48727c83ea4f31b948c.zip | |
Added minor echanges to code and more testing
Diffstat (limited to 'src/main/java/parallelai/spyglass')
3 files changed, 14 insertions, 3 deletions
diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java b/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java index c145eb0..3c62f82 100644 --- a/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java +++ b/src/main/java/parallelai/spyglass/hbase/HBaseOutputFormat.java @@ -52,8 +52,8 @@ FileOutputFormat<ImmutableBytesWritable, Put> implements JobConfigurable { LOG.error(e); throw e; } - // TODO: Should Autoflush be set to true ???? - table.setAutoFlush(false); + // TODO: Should Autoflush be set to true ???? - DONE + table.setAutoFlush(true); HBaseRecordWriter recordWriter = new HBaseRecordWriter(table); recordWriter.setSinkMode(sinkMode); return recordWriter; diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java b/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java index 832ce95..7b62c88 100644 --- a/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java +++ b/src/main/java/parallelai/spyglass/hbase/HBaseRawScheme.java @@ -278,7 +278,7 @@ public class HBaseRawScheme extends Scheme<JobConf, RecordReader, OutputCollecto public void copyValue(Result oldValue, Result newValue) { if (null != oldValue && null != newValue) { -// oldValue.copyFrom(newValue); + oldValue.copyFrom(newValue); } } diff --git a/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java b/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java index e3f5dc9..6766458 100644 --- a/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java +++ b/src/main/java/parallelai/spyglass/hbase/HBaseSalter.java @@ -171,17 +171,28 @@ public class HBaseSalter { } SortedSet<Byte> subSet = prefixSet.subSet(startPrefix, true, stopPrefix, true); + + LOG.info("".format("Prefix subset (%s)", subSet)); return getAllKeys(originalKey, subSet.toArray(new Byte[]{})); } public static byte[][] getAllKeys(byte[] originalKey, Byte [] prefixArray) { + LOG.info("".format("getAllKeys: OKEY (%s) PARRAY (%s)", + Bytes.toString(originalKey), prefixArray )); + byte[][] keys = new byte[prefixArray.length][]; for (byte i = 0; i < prefixArray.length; i++) { keys[i] = Bytes.add(new byte[] {prefixArray[i].byteValue()}, Bytes.add( Bytes.toBytes("_"), originalKey)); } + for(int i = 0; i < keys.length; i ++) { + for(int j = 0; j < keys[i].length; j++) { + LOG.info("" + i + " : " + j + " : " + keys[i][j]); + } + } + return keys; } |