Now ignores grobid entries with status other than 200.

1 files changed, 7 insertions, 3 deletions

diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
index 5ba7d58..c319fe6 100644
--- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
@@ -11,6 +11,8 @@ import parallelai.spyglass.hbase.HBasePipeConversions
 import parallelai.spyglass.hbase.HBaseSource
 
 class GrobidScorable extends Scorable with HBasePipeConversions {
+  val StatusOK = 200
+
   def getSource(args : Args) : Source = {
     // TODO: Generalize args so there can be multiple grobid pipes in one job.
     GrobidScorable.getHBaseSource(args("hbase-table"), args("zookeeper-hosts"))
@@ -19,15 +21,17 @@ class GrobidScorable extends Scorable with HBasePipeConversions {
   def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = {
     getSource(args)
       .read
-      .fromBytesWritable(new Fields("key", "tei_json"))
-      .toTypedPipe[(String, String)](new Fields("key", "tei_json"))
+      .fromBytesWritable(new Fields("key", "tei_json", "status_code"))
+      .toTypedPipe[(String, String, Int)](new Fields("key", "tei_json", "status_code"))
+      // TODO: Should I combine next two stages for efficiency?
+      .collect { case (key, json, StatusOK) => (key, json) }
       .map { entry : (String, String) => GrobidScorable.jsonToMapFeatures(entry._1, entry._2) }
   }
 }
 
 object GrobidScorable {
   def getHBaseSource(table : String, host : String) : HBaseSource = {
-    HBaseBuilder.build(table, host, List("grobid0:tei_json"), SourceMode.SCAN_ALL)
+    HBaseBuilder.build(table, host, List("grobid0:tei_json", "grobid0:status_code"), SourceMode.SCAN_ALL)
   }
 
   def jsonToMapFeatures(key : String, json : String) : MapFeatures = {