diff options
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/ScorableFeatures.scala')
| -rw-r--r-- | scalding/src/main/scala/sandcrawler/ScorableFeatures.scala | 13 | 
1 files changed, 8 insertions, 5 deletions
diff --git a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala index 241db79..b56f102 100644 --- a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala +++ b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala @@ -35,9 +35,9 @@ class ScorableFeatures private(title : String, authors : List[Any] = List(), yea      JSONObject(toMap).toString    } -  def toSlug() : String = { +  def toSlug() : Option[String] = {      if (title == null) { -      Scorable.NoSlug +      None      } else {        val unaccented = StringUtilities.removeAccents(title)        // Remove punctuation @@ -45,10 +45,13 @@ class ScorableFeatures private(title : String, authors : List[Any] = List(), yea        if (slug.isEmpty            || slug == null            || (ScorableFeatures.SlugBlacklist contains slug) -          || (slug.length < ScorableFeatures.MinSlugLength)) Scorable.NoSlug else slug +          || (slug.length < ScorableFeatures.MinSlugLength)) None else Some(slug)      }    } -  def toMapFeatures : MapFeatures = -    MapFeatures(toSlug, toString) +  def toMapFeatures : Option[MapFeatures] = +    toSlug match { +      case None => None +      case Some(slug) => Some(MapFeatures(slug, toString)) +    }  }  | 
