From 5521b1b520a550373369da8b9cbd36148e071115 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 13 Sep 2018 21:11:57 -0700 Subject: new simple file metadata dump script --- .../main/scala/sandcrawler/DumpFileMetaJob.scala | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 scalding/src/main/scala/sandcrawler/DumpFileMetaJob.scala (limited to 'scalding') diff --git a/scalding/src/main/scala/sandcrawler/DumpFileMetaJob.scala b/scalding/src/main/scala/sandcrawler/DumpFileMetaJob.scala new file mode 100644 index 0000000..b3734f0 --- /dev/null +++ b/scalding/src/main/scala/sandcrawler/DumpFileMetaJob.scala @@ -0,0 +1,36 @@ +package sandcrawler + +import java.util.Properties + +import cascading.property.AppProps +import cascading.tuple.Fields +import com.twitter.scalding._ +import com.twitter.scalding.typed.TDsl._ +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import org.apache.hadoop.hbase.util.Bytes +import parallelai.spyglass.base.JobBase +import parallelai.spyglass.hbase.HBaseConstants.SourceMode +import parallelai.spyglass.hbase.HBasePipeConversions +import parallelai.spyglass.hbase.HBaseSource + +// Dumps all the info needed to insert a file entity in Fatcat. Useful for +// joining. +class DumpFileMetaJob(args: Args) extends JobBase(args) with HBasePipeConversions { + + val metaPipe : TypedPipe[(String, String, String, Long)] = HBaseBuilder.build(args("hbase-table"), + args("zookeeper-hosts"), + List("file:cdx", "file:mime", "file:size"), + SourceMode.SCAN_ALL) + .read + .toTypedPipe[(ImmutableBytesWritable,ImmutableBytesWritable,ImmutableBytesWritable,ImmutableBytesWritable)](new Fields("key", "cdx", "mime", "size")) + .filter { case (_, cdx, mime, size) => cdx != null && mime != null && size != null } + .map { case (key, cdx, mime, size) => + (Bytes.toString(key.copyBytes()), + Bytes.toString(cdx.copyBytes()), + Bytes.toString(mime.copyBytes()), + Bytes.toLong(size.copyBytes())) + }; + + metaPipe.write(TypedTsv[(String,String,String,Long)](args("output"))) + +} -- cgit v1.2.3