aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src/main
diff options
context:
space:
mode:
Diffstat (limited to 'scalding/src/main')
-rw-r--r--scalding/src/main/scala/sandcrawler/CrossrefScorable.scala7
-rw-r--r--scalding/src/main/scala/sandcrawler/GrobidScorable.scala6
-rw-r--r--scalding/src/main/scala/sandcrawler/Scorable.scala30
-rw-r--r--scalding/src/main/scala/sandcrawler/ScorableFeatures.scala30
4 files changed, 33 insertions, 40 deletions
diff --git a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
index 4558ee6..4897b1c 100644
--- a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
@@ -34,11 +34,8 @@ object CrossrefScorable {
if (titles.isEmpty || titles == null || doi.isEmpty || doi == null) {
new MapFeatures(Scorable.NoSlug, json)
} else {
- val title = titles(0)
- val map2 = Scorable.toScorableMap(title=title, doi=doi)
- new MapFeatures(
- Scorable.mapToSlug(map2),
- JSONObject(map2).toString)
+ val sf : ScorableFeatures = new ScorableFeatures(title=titles(0), doi=doi)
+ new MapFeatures(sf.toSlug, sf.toString)
}
} else {
new MapFeatures(Scorable.NoSlug, json)
diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
index 94b3494..5ba7d58 100644
--- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
@@ -35,11 +35,7 @@ object GrobidScorable {
case None => MapFeatures(Scorable.NoSlug, json)
case Some(map) => {
if (map contains "title") {
- val map2 = Scorable.toScorableMap(Scorable.getString(map, "title"),
- sha1=key)
- new MapFeatures(
- Scorable.mapToSlug(map2),
- JSONObject(map2).toString)
+ new ScorableFeatures(Scorable.getString(map, "title"), sha1=key).toMapFeatures
} else {
MapFeatures(Scorable.NoSlug, json)
}
diff --git a/scalding/src/main/scala/sandcrawler/Scorable.scala b/scalding/src/main/scala/sandcrawler/Scorable.scala
index 717b2d5..9b9c633 100644
--- a/scalding/src/main/scala/sandcrawler/Scorable.scala
+++ b/scalding/src/main/scala/sandcrawler/Scorable.scala
@@ -36,21 +36,6 @@ object Scorable {
slug != NoSlug
}
- // NOTE: I could go all out and make ScorableMap a type.
- // TODO: Require year. Other features will get added here.
- def toScorableMap(title : String, year : Int = 0, doi : String = "", sha1 : String = "") : Map[String, Any] = {
- Map("title" -> title, "year" -> year, "doi" -> doi, "sha1" -> sha1)
- }
-
- def toScorableJson(title : String, year : Int, doi : String = "", sha1 : String = "") : String = {
- JSONObject(toScorableMap(title=title, year=year, doi=doi, sha1=sha1)).toString
- }
-
- // TODO: Score on more fields than "title".
- def isScorableMap(map : Map[String, Any]) : Boolean = {
- map.contains("title")
- }
-
def jsonToMap(json : String) : Option[Map[String, Any]] = {
// https://stackoverflow.com/a/32717262/631051
val jsonObject = JSON.parseFull(json)
@@ -61,21 +46,6 @@ object Scorable {
}
}
- // Map should have been produced by toScorableMap.
- // This guarantees it will have all of the fields needed to compute
- // the ultimate score, which are a superset of those needed for a slug.
- def mapToSlug(map : Map[String, Any]) : String = {
- val title = getString(map, "title")
- if (title == null) {
- NoSlug
- } else {
- val unaccented = StringUtilities.removeAccents(title)
- // Remove punctuation after splitting on colon.
- val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase())).replaceAll("\\s", "")
- if (slug.isEmpty || slug == null) NoSlug else slug
- }
- }
-
def getStringOption(optionalMap : Option[Map[String, Any]], key : String) : Option[String] = {
optionalMap match {
case None => None
diff --git a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
new file mode 100644
index 0000000..5d6dea0
--- /dev/null
+++ b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
@@ -0,0 +1,30 @@
+package sandcrawler
+
+import scala.util.parsing.json.JSONObject
+
+// Contains features needed to make slug and to score (in combination
+// with a second ScorableFeatures).
+class ScorableFeatures(title : String, year: Int = 0, doi : String = "", sha1: String = "") {
+ def toMap() : Map[String, Any] = {
+ Map("title" -> title, "year" -> year, "doi" -> doi, "sha1" -> sha1)
+ }
+
+ override def toString() : String = {
+ JSONObject(toMap()).toString
+ }
+
+ def toSlug() : String = {
+ if (title == null) {
+ Scorable.NoSlug
+ } else {
+ val unaccented = StringUtilities.removeAccents(title)
+ // Remove punctuation after splitting on colon.
+ val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase())).replaceAll("\\s", "")
+ if (slug.isEmpty || slug == null) Scorable.NoSlug else slug
+ }
+ }
+
+ def toMapFeatures = {
+ MapFeatures(toSlug, toString)
+ }
+}