diff --git a/commons/util/src/main/scala/net/shrine/json/CaseClasses.scala b/commons/util/src/main/scala/net/shrine/json/CaseClasses.scala index f1d531966..a6d4dab8f 100644 --- a/commons/util/src/main/scala/net/shrine/json/CaseClasses.scala +++ b/commons/util/src/main/scala/net/shrine/json/CaseClasses.scala @@ -1,17 +1,133 @@ package net.shrine.json import java.util.UUID +import net.shrine.problem.ProblemDigest +import rapture.json.{Json, JsonBuffer} + +import scala.xml.Node + // The Adapter, Topic, and User entities don't have to // carry arbitrary json, so they can just be case classes case class Adapter(name: String, id: UUID) case class Topic(name: String, description: String, id: UUID) case class User(userName: String, domain: String, id: UUID) case class Breakdown(category: String, results: List[BreakdownProperty]) case class BreakdownProperty(name: String, count: Int) case class NoiseTerms(clamp: Int, sigma: Int, rounding: Int) + +case class NestedQuery(queryId: UUID, + topic: Topic, + user: User, + startTime: Long, + i2b2QueryText: Node, + extraXml: Node, + queryResults: List[NestedQueryResult]) + +sealed trait NestedQueryResult { + val adapter : Adapter + val status : String + val resultId: UUID + val queryId : UUID +} + +case class NSuccessResult(resultId: UUID, + queryId: UUID, + adapter: Adapter, + count: Int, + noiseTerms: NoiseTerms, + i2b2Mapping: Node, + flags: List[String], + breakdowns: List[Breakdown]) + extends NestedQueryResult { + override val status: String = Statuses.success +} + +object NSuccessResult { + + import Statuses._ + + def apply(json: Json): Option[NSuccessResult] = + if (eq(json.status, success) && json.is[NSuccessResult]) + Some(json.as[NSuccessResult]) + else None +} + +case class NPendingResult(resultId: UUID, queryId: UUID, adapter: Adapter) + extends NestedQueryResult { + override val status: String = Statuses.pending +} + +object NPendingResult { + + import Statuses._ + + def apply(json: Json): Option[NPendingResult] = + if (eq(json.status, pending) && json.is[NPendingResult]) + Some(json.as[NPendingResult]) + else None +} + +case class NFailureResult(resultId: UUID, + queryId: UUID, + adapter: Adapter, + problemDigest: ProblemDigest) + extends NestedQueryResult { + override val status: String = Statuses.failure +} + +object NFailureResult { + + import Statuses._ + + def apply(json: Json): Option[NFailureResult] = + if (eq(json.status, failure) && json.is[NFailureResult]) + Some(json.as[NFailureResult]) + else None +} + +object Statuses { + val success = "success" + val pending = "pending" + val failure = "failure" + def eq(json: Json, string: String) = + json.is[String] && json.as[String].toLowerCase == string +} + +case class JsonQuery(json: Json, nestedQuery: NestedQuery) { + val normalized:Seq[(Query, QueryResult, User, Topic, Adapter)] = { + val qb = toBuffer(json) + val qrs = qb.queryResults.as[List[Json]].map(rj => { + val rb = toBuffer(rj) + val a = rb.adapter.as[Adapter] + rb.adapterId = a.id + rb -= "adapter" + (rb.as[QueryResult], a) + }) + val user = nestedQuery.user + val topic = nestedQuery.topic + qb.userId = user.id + qb.topicId = topic.id + qb.queryResults = nestedQuery.queryResults.map(_.resultId) + qb -= "user" + qb -= "topic" + qb -= "queryResults" + val query = qb.as[Query] + + qrs.map{ + case (result, adapter) => (query, result, user, topic, adapter) + } + } +} + +object JsonQuery { + def apply(json: Json): Option[JsonQuery] = + if (json.is[NestedQuery]) + Some(JsonQuery(json, json.as[NestedQuery])) + else None +} diff --git a/commons/util/src/main/scala/net/shrine/json/QueryResult.scala b/commons/util/src/main/scala/net/shrine/json/QueryResult.scala index db5937117..a872ad7ed 100644 --- a/commons/util/src/main/scala/net/shrine/json/QueryResult.scala +++ b/commons/util/src/main/scala/net/shrine/json/QueryResult.scala @@ -1,115 +1,115 @@ package net.shrine.json import java.util.UUID import net.shrine.problem.ProblemDigest import rapture.json._ import scala.util.Try import scala.util.hashing.Hashing.default import scala.xml.Node /** * @author ty * @since 2/1/17 * A Query Result can either be a Success, Failure, or Pending result * Upon creation, it looks at the status field of the json to determine * which to create. Defines structural equality and unapply methods for each * while still being backed by the dynamic json ast */ object QueryResult { def apply(json: Json): Option[QueryResult] = SuccessResult(json) .orElse(PendingResult(json)) .orElse(FailureResult(json)) } sealed trait QueryResult { val json: Json val status: String = json.status.as[String] val resultId: UUID = json.resultId.as[UUID] val adapterId: UUID = json.adapterId.as[UUID] val queryId: UUID = json.queryId.as[UUID] } // <--=== Success Result ===--> // final class SuccessResult(val json: Json) extends QueryResult { val count: Int = json.count.as[Int] val noiseTerms: NoiseTerms = json.noiseTerms.as[NoiseTerms] val i2b2Mapping: Node = json.i2b2Mapping.as[Node] // TODO: Once we figure out what flags are, replace them with a concrete type val flags: List[String] = json.flags.as[List[String]] val breakdowns: List[Breakdown] = json.breakdowns.as[List[Breakdown]] // This allows us to define structural equality on the fields themselves private[SuccessResult] val fields = (resultId, adapterId, count, noiseTerms, i2b2Mapping, flags, breakdowns) override def equals(that: scala.Any): Boolean = that match { case sr: SuccessResult => sr.fields == fields case _ => false } override def hashCode(): Int = default.hash(fields) override def toString: String = s"SuccessResult$fields" } object SuccessResult { def unapply(arg: SuccessResult) = Some(arg.fields) def apply(json: Json): Option[SuccessResult] = if (JsonCompare.==(json.status, "success")) // This is shorter than doing a .is for every field. Try(new SuccessResult(json)).toOption else None } // <--=== Pending Result ===--> // final class PendingResult(val json: Json) extends QueryResult { override def equals(that: scala.Any): Boolean = that.isInstanceOf[PendingResult] override def toString: String = "PendingResult()" } object PendingResult { def apply(json: Json): Option[PendingResult] = - if (JsonCompare.==(json.status, "status")) + if (JsonCompare.==(json.status, "pending")) Try(new PendingResult(json)).toOption else None } // <--=== Failure Result ===--> // final class FailureResult(val json: Json) extends QueryResult { val problemDigest: ProblemDigest = json.problemDigest.as[ProblemDigest] override def equals(that: scala.Any): Boolean = that match { case fr: FailureResult => fr.problemDigest == problemDigest case _ => false } override def hashCode(): Int = problemDigest.hashCode override def toString: String = s"FailureResult($problemDigest)" } object FailureResult { def apply(json: Json): Option[FailureResult] = if (JsonCompare.==(json.status, "failure")) Try(new FailureResult(json)).toOption else None def unapply(arg: FailureResult): Option[ProblemDigest] = Some(arg.problemDigest) } // Just a private helper since I was using this three times private object JsonCompare { def ==(json: Json, string: String): Boolean = json.is[String] && json.as[String].toLowerCase == string } diff --git a/commons/util/src/main/scala/net/shrine/json/package.scala b/commons/util/src/main/scala/net/shrine/json/package.scala index 5ef83c134..60dcd51d4 100644 --- a/commons/util/src/main/scala/net/shrine/json/package.scala +++ b/commons/util/src/main/scala/net/shrine/json/package.scala @@ -1,39 +1,66 @@ package net.shrine import java.util.UUID import rapture.json._ import jsonBackends.jawn._ import scala.xml.{Node, NodeSeq, XML} /** * @author ty * @since 2/1/17 * Defining an extractor gives you the .is and .as methods on Json objects. * .is reports true if the extract completes the extraction, and returns false * if an exception is thrown. Easy pattern is to call .get on an option. * See https://github.com/propensive/rapture/blob/dev/doc/json.md for rapture docs, * the website is slightly out of date */ package object json { + def toBuffer(json: Json): JsonBuffer = + JsonBuffer.construct(json.$root.copy(), Vector()) implicit val node: Extractor[Node, Json] = Json.extractor[String].map(XML.loadString) implicit val nodeSeq: Extractor[NodeSeq, Json] = Json.extractor[String].map(XML.loadString) implicit val uuid: Extractor[UUID, Json] = Json.extractor[String].map(UUID.fromString) implicit val query: Extractor[Query, Json] = Json.extractor[Json].map(new Query(_)) implicit val successResult: Extractor[SuccessResult, Json] = Json.extractor[Json].map(new SuccessResult(_)) implicit val failureResult: Extractor[FailureResult, Json] = Json.extractor[Json].map(new FailureResult(_)) implicit val pendingResult: Extractor[PendingResult, Json] = Json.extractor[Json].map(new PendingResult(_)) implicit val queryResult: Extractor[QueryResult, Json] = successResult .orElse(failureResult) .orElse(pendingResult) - implicit val uuidSerializer = Json.serializer[Json].contramap[UUID](uuid => Json(uuid.toString)) + implicit val nSuccessResult: Extractor[NSuccessResult, Json] = + Json.extractor[Json].map(NSuccessResult(_).get) + implicit val nFailureResult: Extractor[NFailureResult, Json] = + Json.extractor[Json].map(NFailureResult(_).get) + implicit val nPendingResult: Extractor[NPendingResult, Json] = + Json.extractor[Json].map(NPendingResult(_).get) + implicit val nestedQueryResult: Extractor[NestedQueryResult, Json] = + nSuccessResult + .orElse(nFailureResult) + .orElse(nPendingResult) + implicit val nestedQuery: Extractor[NestedQuery, Json] = + Json + .extractor[Json] + .map( + js => + NestedQuery( + js.queryID.as[UUID], + js.topic.as[Topic], + js.user.as[User], + js.startTime.as[Long], + js.i2b2QueryText.as[Node], + js.extraXml.as[Node], + js.queryResults.as[List[NestedQueryResult]] + )) + implicit val uuidSerializer: AnyRef with Serializer[UUID, Json] = + Json.serializer[Json].contramap[UUID](uuid => Json(uuid.toString)) } diff --git a/commons/util/src/test/resources/test.db b/commons/util/src/test/resources/test.db deleted file mode 100644 index 067fb6277..000000000 Binary files a/commons/util/src/test/resources/test.db and /dev/null differ diff --git a/commons/util/src/test/scala/net/shrine/json/StorageDemo.scala b/commons/util/src/test/scala/net/shrine/json/StorageDemo.scala index f86c6c06f..e1f1bc98a 100644 --- a/commons/util/src/test/scala/net/shrine/json/StorageDemo.scala +++ b/commons/util/src/test/scala/net/shrine/json/StorageDemo.scala @@ -1,323 +1,322 @@ package net.shrine.json import java.io.File import java.nio.charset.Charset import java.util.UUID import com.typesafe.config.ConfigFactory import jawn.Parser.parseFromString import rapture.json._ import rapture.json.jsonBackends.jawn._ import slick.dbio.Effect.Write import slick.driver.{H2Driver, JdbcProfile, MySQLDriver, SQLiteDriver} import slick.jdbc.JdbcBackend.Database import net.shrine.json.{Query => ShrineQuery} import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ /** * @by ty * @since 2/22/17 */ class StorageDemo {} /** * @author ty */ object Storage { def main(args: Array[String]): Unit = { val uid = UUID.randomUUID() val startTime = System.currentTimeMillis() val i2b2Xml = suchi2b2, wow val extraXml = lots of extra val noiseTerms = NoiseTerms(10, 11, 12) val resultCount = 1000 val j = Json(Topic("hey", "hey", uid)) val breakdown = Breakdown( "gender", List(BreakdownProperty("male", 70), BreakdownProperty("female", 30))) val flags = List("hey", "what's", "that", "sound") val queryJson = json"""{ "queryId": $uid, "topicId": $uid, "userId": $uid, "startTime": $startTime, "i2b2QueryText": ${i2b2Xml.toString}, "extraXml": ${extraXml.toString}, "queryResults": [ $uid ] } """ val queryResultJson = json"""{ "status": "success", "resultId": $uid, "adapterId": $uid, "queryId": $uid, "count": $resultCount, "noiseTerms": { "sigma": ${noiseTerms.sigma}, "clamp": ${noiseTerms.clamp}, "rounding": ${noiseTerms.rounding} }, "i2b2Mapping": ${i2b2Xml.toString}, "flags": $flags, "breakdowns": [ { "category": "gender", "results": [ { "name": "male", "count": 70 }, { "name": "female", "count": 30 } ] } ] } """ val queryResults = (0 to 10).map(i => QueryResult(queryResultJson.copy(_.resultId = UUID.randomUUID())) .get) val query = Query(queryJson.copy(_.queryResults = queryResults.map(_.resultId))).get val user = User("user", "domain", uid) val topic = Topic("topic", "domain", uid) val adapter = Adapter("adapter", uid) val sqliteDB = Database.forConfig("sqlite", ConfigFactory.load("shrine.conf")) val h2DB = Database.forConfig("h2", ConfigFactory.load("shrine.conf")) val sqliteDAO = DAO(SQLiteDriver) val h2DAO = DAO(H2Driver) // Await.result(sqliteDB.run(setup), 10.seconds) Seq((sqliteDAO, sqliteDB), (h2DAO, h2DB)).foreach(daodb => { val (dao, db) = daodb import dao.profile.api._ val insertQueries = queryResults.map(queryResult => dao.SlickQueries.insert(query, user, topic, adapter, queryResult) match { case Left(s) => throw new IllegalArgumentException(s) case Right(dbQuery) => dbQuery }) val acts = dao.SlickQueries.create .andThen(DBIO.sequence(insertQueries)) .andThen(dao.queryRunsQuery.result) .andThen(sqliteDAO.SlickQueries.selectJsonForQuery(uid)) val results = Await.result(db.run(acts), 10.seconds) - println(results) + println(results.get.nestedQuery) }) // println(h2DAO.SlickQueries.create.statements.mkString(";\n")) // println(DAO(MySQLDriver).SlickQueries.create.statements.mkString(";\n")) } } case class DAO(profile: JdbcProfile) { import profile.api._ val charset = "UTF-8" def bytesToJson(bytes: Array[Byte]): Json = { Json(jawn.Parser.parseFromString(new String(bytes, charset)).get) } def jsonToBytes(json: Json): Array[Byte] = { json.toBareString.getBytes(charset) } class Queries(tag: Tag) extends Table[ShrineQuery](tag, "QUERIES") { def queryId = column[UUID]("query_id", O.PrimaryKey) def queryDate = column[Long]("query_epoch") def queryJson = column[Array[Byte]]("query_json") def * = (queryId, queryDate, queryJson) <> ( (row: (UUID, Long, Array[Byte])) => bytesToJson(row._3).as[ShrineQuery], (query: ShrineQuery) => Some((query.queryId, query.startTime, jsonToBytes(query.json))) ) } val queries = TableQuery[Queries] class QueryResults(tag: Tag) extends Table[QueryResult](tag, "QUERYRESULTS") { def queryResultId = column[UUID]("query_result_id", O.PrimaryKey) def queryResultJson = column[Array[Byte]]("query_result_json") def * = (queryResultId, queryResultJson) <> ((row: (UUID, Array[Byte])) => bytesToJson(row._2).as[QueryResult], (result: QueryResult) => Some((result.resultId, jsonToBytes(result.json)))) } val queryResults = TableQuery[QueryResults] class Users(tag: Tag) extends Table[User](tag, "USERS") { def userId = column[UUID]("user_id", O.PrimaryKey) def userJson = column[Array[Byte]]("user_json") def * = { (userId, userJson) <> ((row: (UUID, Array[Byte])) => bytesToJson(row._2).as[User], (user: User) => Some((user.id, jsonToBytes(Json(user))))) } } val users = TableQuery[Users] class Topics(tag: Tag) extends Table[Topic](tag, "TOPICS") { def topicId = column[UUID]("topic_id", O.PrimaryKey) def topicJson = column[Array[Byte]]("topic_json") def * = (topicId, topicJson) <> ((row: (UUID, Array[Byte])) => bytesToJson(row._2).as[Topic], (topic: Topic) => Some((topic.id, jsonToBytes(Json(topic))))) } val topics = TableQuery[Topics] class Adapters(tag: Tag) extends Table[Adapter](tag, "ADAPTERS") { def adapterId = column[UUID]("adapter_id", O.PrimaryKey) def adapterJson = column[Array[Byte]]("adapter_json") def * = (adapterId, adapterJson) <> ((row: (UUID, Array[Byte])) => bytesToJson(row._2).as[Adapter], (adapter: Adapter) => Some((adapter.id, jsonToBytes(Json(adapter))))) } val adapters = TableQuery[Adapters] class QueryRuns(tag: Tag) extends Table[(Int, UUID, UUID, UUID, UUID, UUID)](tag, "QUERYRUNS") { def queryRunId = column[Int]("query_run_id", O.PrimaryKey, O.AutoInc) def queryId = column[UUID]("query_id") def queryResultId = column[UUID]("query_result_id") def userId = column[UUID]("user_id") def topicId = column[UUID]("topic_id") def adapterId = column[UUID]("adapter_id") def * = (queryRunId, queryId, queryResultId, userId, topicId, adapterId) def queryFk = foreignKey("query_fk", queryId, queries)( _.queryId, onUpdate = ForeignKeyAction.NoAction, onDelete = ForeignKeyAction.NoAction) def queryResultFk = foreignKey("query_result_fk", queryResultId, queryResults)( _.queryResultId, onUpdate = ForeignKeyAction.NoAction, onDelete = ForeignKeyAction.NoAction) def userFk = foreignKey("user_fk", userId, users)( _.userId, onUpdate = ForeignKeyAction.NoAction, onDelete = ForeignKeyAction.NoAction) def topicFk = foreignKey("topic_fk", topicId, topics)( _.topicId, onUpdate = ForeignKeyAction.NoAction, onDelete = ForeignKeyAction.NoAction) def adapterFk = foreignKey("adapter_fk", adapterId, adapters)( _.adapterId, onUpdate = ForeignKeyAction.NoAction, onDelete = ForeignKeyAction.NoAction) } val queryRunsQuery = TableQuery[QueryRuns] object SlickQueries { def selectAllForQuery(queryId: UUID) = for { queryRun <- queryRunsQuery.filter(_.queryId === queryId) q <- queries if q.queryId === queryRun.queryId r <- queryResults if r.queryResultId === queryRun.queryResultId u <- users if u.userId === queryRun.userId t <- topics if t.topicId === queryRun.topicId a <- adapters if a.adapterId === queryRun.adapterId } yield(q, r, u, t, a) // Generates the json for a given query Id def selectJsonForQuery(queryId: UUID) = { val allResults = selectAllForQuery(queryId) allResults.result.map(t => { - t.headOption.map{ + t.headOption.flatMap{ case (query, _, user, topic, _) => - val queryJson = query.json - val jb = JsonBuffer.construct(queryJson.$root.copy(), Vector()) + val jb = toBuffer(query.json) jb -= "userId" jb -= "topicId" jb.user = user jb.topic = topic jb.queryResults = t.map{ case (_, queryResult, _, _, adapter) => - val jb2 = JsonBuffer.construct(queryResult.json.$root.copy(), Vector()) + val jb2 = toBuffer(queryResult.json) jb2 -= "adapterId" jb2.adapter = adapter jb2 } - Json(jb) - }.getOrElse(json"""{}""") + JsonQuery(Json(jb)) + } }) } def insert(query: ShrineQuery, user: User, topic: Topic, adapter: Adapter, result: QueryResult) : Either[String, DBIOAction[Unit, NoStream, Write]] = { if (result.queryId != query.queryId) Left("The result's queryId does not match the query's queryId") else if (query.topicId != topic.id) Left("The query's topicId does not match the topic's id") else if (query.userId != user.id) Left("The query's userId does not match the user's id") else if (result.adapterId != adapter.id) Left("The query result's adapterId does not match the adapter's id") else Right( DBIO.seq( // Have to upsert here, as these items may already be in the table queries.insertOrUpdate(query), queryResults.insertOrUpdate(result), users.insertOrUpdate(user), topics.insertOrUpdate(topic), adapters.insertOrUpdate(adapter), queryRunsQuery += (0, query.queryId, result.resultId, user.id, topic.id, adapter.id) )) } def schema = queries.schema ++ users.schema ++ topics.schema ++ adapters.schema ++ queryResults.schema ++ queryRunsQuery.schema def create = schema.create } } diff --git a/integration/src/test/scala/net/shrine/integration/JsonDemos.scala b/integration/src/test/scala/net/shrine/integration/JsonDemos.scala index 5d8c78338..ae56c4e81 100644 --- a/integration/src/test/scala/net/shrine/integration/JsonDemos.scala +++ b/integration/src/test/scala/net/shrine/integration/JsonDemos.scala @@ -1,88 +1,89 @@ package net.shrine.integration import java.io.InputStream import jawn.{AsyncParser, Parser, ast} import jawn.ast.JValue import rapture.json._ import jsonBackends.jawn._ import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{FlatSpec, Matchers} import scala.collection.mutable.{Map => MMap} import scala.util.Try /** * @by ty * @since 1/24/17 */ @RunWith(classOf[JUnitRunner]) class JsonDemos extends FlatSpec with Matchers { // The async parser will spit back values from an array as it parses them. Can also do whitespace // separated json objects val parser: AsyncParser[JValue] = ast.JParser.async(mode = AsyncParser.UnwrapArray) // Rapture provides the json""" syntax, but the actual parsing is handled by jawn itself // The ast that's produced is also a jawn AST object // One annoyance is that intellji tries to add the stripMargin thing, which json""" doesn't handle val exampleJson = json""" {"a":{"a":{"a":{"a":"a"}},"b":{"b":{"b":"b"}}},"b":{"b":{"b":{"b":"b"}}}} """ // This is the api jawn normally exposes to build an AST object val jawnJson = ast.JObject( MMap( "a" -> ast.JObject( MMap("a" -> ast.JObject( MMap("a" -> ast.JObject(MMap("a" -> ast.JString("a"))))), "b" -> ast.JObject( MMap("b" -> ast.JObject(MMap("b" -> ast.JString("b"))))))), "b" -> ast.JObject(MMap("b" -> ast.JObject( MMap("b" -> ast.JObject(MMap("b" -> ast.JString("b"))))))) )) val person = Person("Ty", "Coghlan") val account = Account(person, -100000000.87) // Student loans man val accountJson = Json(account) val accountJsonLiteral = json""" {"person": {"first": "Ty", "last": "Coghlan"}, "money": -100000000.87} """ val extraStuffAccount = accountJson ++ json"""{"overdue":true, "randomArray": [0, 1, 2, 3, {"surprise": "boo"}]}""" "The simple, synchronous jawn parser" should "parse the example string" in { Parser.parseFromString(exampleJson.toBareString)(jawn.ast.JawnFacade) shouldBe Try( jawnJson) } "The async jawn parser" should "parse the file stream and return values as they appear" in { val stream: InputStream = getClass.getResourceAsStream("/test_data.json") val lines = scala.io.Source .fromInputStream(stream) .getLines .flatMap(l => parser.absorb(l).right.get) .toSeq lines.foreach(_.valueType shouldBe "object") lines.length shouldBe 24 } "Case class serialization" should "be painless" in { accountJson shouldBe accountJsonLiteral accountJson.as[Account] shouldBe account // Can also serialize json that has "extra" extraStuffAccount.as[Account] shouldBe account // Or if you just want the values themselves extraStuffAccount.overdue shouldBe Json(true) extraStuffAccount.randomArray(4).surprise.as[String] shouldBe "boo" extraStuffAccount match { case json""" {"person": $p, "overdue": false} """ => fail("Did not match overdue as expected") + // Note that you don't need money for the match case json""" {"person": $p, "overdue": true } """ => person shouldBe p.as[Person] case _ => fail("Did not match as expected") } } } case class Account(person: Person, money: Double) case class Person(first: String, last: String)