scalding/src/main/scala/sandcrawler/ScorableFeatures.scala


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

package sandcrawler

import scala.util.parsing.json.JSONObject

// Contains features needed to make slug and to score (in combination
// with a second ScorableFeatures).
class ScorableFeatures(title : String, year: Int = 0, doi : String = "", sha1: String = "") {
  def toMap() : Map[String, Any] = {
    Map("title" -> title, "year" -> year, "doi" -> doi, "sha1" -> sha1)
  }

  override def toString() : String = {
    JSONObject(toMap()).toString
  }

  def toSlug() : String = {
    if (title == null) {
      Scorable.NoSlug
    } else {
      val unaccented = StringUtilities.removeAccents(title)
      // Remove punctuation after splitting on colon.
      val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase())).replaceAll("\\s", "")
      if (slug.isEmpty || slug == null) Scorable.NoSlug else slug
    }
  }

  def toMapFeatures = {
    MapFeatures(toSlug, toString)
  }
}