Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F80340779
nervisjson.scala
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Sat, Aug 31, 06:06
Size
2 KB
Mime Type
text/x-c++
Expires
Mon, Sep 2, 06:06 (1 d, 23 h)
Engine
blob
Format
Raw Data
Handle
20346444
Attached To
R1473 warcbase
nervisjson.scala
View Options
import org.apache.spark.rdd.RDD
import org.warcbase.spark.matchbox.NER3Classifier
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
class JsonUtil extends Serializable {
val mapper = new ObjectMapper() with ScalaObjectMapper
mapper.registerModule(DefaultScalaModule)
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
def toJson(value: Map[Symbol, Any]): String = {
toJson(value map { case (k,v) => k.name -> v})
}
def toJson(value: Any): String = {
mapper.writeValueAsString(value)
}
def fromJson[T](json: String)(implicit m : Manifest[T]): T = {
mapper.readValue[T](json)
}
}
def combineKeyCountLists (l1: List[(String, Int)], l2: List[(String, Int)]): List[(String, Int)] = {
(l1 ++ l2).groupBy(_._1 ).map {
case (key, tuples) => (key, tuples.map( _._2).sum)
}.toList
}
sc.addFile("/cliphomes/jrwiebe/english.all.3class.distsim.crf.ser.gz")
val out = sc.textFile("hdfs:///user/jrwiebe/cpp.greenparty-arc-plaintext/")
.mapPartitions(iter => {
NER3Classifier.apply("english.all.3class.distsim.crf.ser.gz")
iter.map(line => {
val ind1 = line.indexOf(",")
val ind2 = line.indexOf(",", ind1 + 1)
(line.substring(1, ind1),
line.substring(ind1 + 1, ind2),
line.substring(ind2 + 1, line.length - 1))
})
.map(r => {
val classifiedJson = NER3Classifier.classify(r._3)
val jUtl = new JsonUtil
val classifiedMap = jUtl.fromJson[Map[String,List[String]]](classifiedJson)
val classifiedMapCountTuples: Map[String, List[(String, Int)]] = classifiedMap.map {
case (nerType, entityList) => (nerType, entityList.groupBy(identity).mapValues(_.size).toList)
}
((r._1, r._2), classifiedMapCountTuples)
})
})
.reduceByKey( (a, b) => (a ++ b).keySet.map(r => (r, combineKeyCountLists(a(r), b(r)))).toMap)
.mapPartitions(iter => {
val jUtl = new JsonUtil
iter.map(r => {
(jUtl.toJson(r))
})
})
.saveAsTextFile("nerjson/")
Event Timeline
Log In to Comment