diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/templates/paginator-template.html b/apps/dashboard-app/src/main/js/src/app/diagnostic/templates/paginator-template.html
index 0247a4134..4b77d4dc5 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/templates/paginator-template.html
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/templates/paginator-template.html
@@ -1,9 +1,9 @@
-
+
\ No newline at end of file
diff --git a/apps/dashboard-app/src/main/resources/reference.conf b/apps/dashboard-app/src/main/resources/reference.conf
index f0740198c..6495dfe82 100644
--- a/apps/dashboard-app/src/main/resources/reference.conf
+++ b/apps/dashboard-app/src/main/resources/reference.conf
@@ -1,85 +1,85 @@
shrine {
problem {
problemHandler = "net.shrine.problem.DatabaseProblemHandler$"
}
dashboard {
gruntWatch = false //false for production, true for mvn tomcat7:run . Allows the client javascript and html files to be loaded via gruntWatch .
happyBaseUrl = "https://localhost:6443/shrine/rest/happy"
statusBaseUrl = "https://localhost:6443/shrine/rest/internalstatus"
remoteDashboard {
protocol = "https://"
port = ":6443"
pathPrefix = "shrine-dashboard/fromDashboard"
}
database {
dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
- jndiDataSourceName = "java:comp/env/jdbc/adapterAuditDB" //or leave out for tests
+ jndiDataSourceName = "java:comp/env/jdbc/problemDB" //or leave out for tests
slickProfileClassName = "slick.driver.MySQLDriver$" // Can be
// slick.driver.H2Driver$
// slick.driver.MySQLDriver$
// slick.driver.PostgresDriver$
// slick.driver.SQLServerDriver$
// slick.driver.JdbcDriver$
// freeslick.OracleProfile$
// freeslick.MSSQLServerProfile$
//
// (Yes, with the $ on the end)
// For testing without JNDI
// testDataSource {
//typical test settings for unit tests
//driverClassName = "org.h2.Driver"
//url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests
//url = "jdbc:h2:~/stewardTest.h2" //H2 embedded on disk at ~/test
// }
createTablesOnStart = false //for testing with H2 in memory, when not running unit tests. Set to false normally
}
}
pmEndpoint {
url = "http://changeme.com/i2b2/services/PMService/getServices" //"http://services.i2b2.org/i2b2/services/PMService/getServices"
acceptAllCerts = true
timeout {
seconds = 10
}
}
authenticate {
realm = "SHRINE Steward API"
usersource
{
type = "PmUserSource" //Must be ConfigUserSource (for isolated testing) or PmUserSource (for everything else)
domain = "set shrine.authenticate.usersource.domain to the PM authentication domain in dashboard.conf" //"i2b2demo"
}
}
// If the pmEndpoint acceptAllCerts = false then you need to supply a keystore
// Or if you would like dashboard-to-dashboard comms to work.
// keystore {
// file = "shrine.keystore"
// password = "chiptesting"
// privateKeyAlias = "test-cert"
// keyStoreType = "JKS"
// caCertAliases = [carra ca]
// }
}
//todo typesafe config precedence seems to do the right thing, but I haven't found the rules that say this reference.conf should override others
akka {
loglevel = INFO
// log-config-on-start = on
loggers = ["akka.event.slf4j.Slf4jLogger"]
// logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
// Toggles whether the threads created by this ActorSystem should be daemons or not
daemonic = on
}
spray.servlet {
boot-class = "net.shrine.dashboard.Boot"
request-timeout = 30s
}
diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
index 224da53c7..839cc1c5a 100644
--- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
+++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
@@ -1,20 +1,20 @@
package net.shrine.dashboard
import akka.actor.{ActorSystem, Props}
import net.shrine.problem.Problems
import spray.servlet.WebBoot
// this class is instantiated by the servlet initializer
// it needs to have a default constructor and implement
// the spray.servlet.WebBoot trait
class Boot extends WebBoot {
- val warmUp = Problems.DatabaseConnector.warmup
+ val warmUp:Unit = Problems.warmUp()
// we need an ActorSystem to host our application in
val system = ActorSystem("DashboardActors",DashboardConfigSource.config)
// the service actor replies to incoming HttpRequests
val serviceActor = system.actorOf(Props[DashboardServiceActor])
}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
index f5028a65a..c718f35d7 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
@@ -1,23 +1,23 @@
package net.shrine.steward
import akka.actor.{ActorSystem, Props}
import net.shrine.log.Loggable
import net.shrine.steward.db.StewardDatabase
import spray.servlet.WebBoot
// this class is instantiated by the servlet initializer
// it needs to have a default constructor and implement
// the spray.servlet.WebBoot trait
class Boot extends WebBoot with Loggable {
info(s"StewardActors akka daemonic config is ${StewardConfigSource.config.getString("akka.daemonic")}")
- val warmUp = StewardDatabase.db.warmUp
+ val warmUp:Unit = StewardDatabase.warmUp()
// we need an ActorSystem to host our application in
val system = ActorSystem("StewardActors",StewardConfigSource.config)
// the service actor replies to incoming HttpRequests
val serviceActor = system.actorOf(Props[StewardServiceActor])
}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
index 7d161711e..2f05f346e 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
@@ -1,638 +1,639 @@
package net.shrine.steward.db
import java.sql.SQLException
import java.util.concurrent.atomic.AtomicInteger
import javax.sql.DataSource
import com.typesafe.config.Config
import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole}
import net.shrine.i2b2.protocol.pm.User
import net.shrine.log.Loggable
-import net.shrine.slick.TestableDataSourceCreator
+import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator}
import net.shrine.steward.{CreateTopicsMode, StewardConfigSource}
import slick.dbio.Effect.Read
import slick.driver.JdbcProfile
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.{Duration, DurationInt}
import scala.concurrent.{Await, Future, blocking}
import scala.language.postfixOps
import scala.util.Try
/**
* Database access code for the data steward service.
*
* I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace.
*
* @author dwalend
* @since 1.19
*/
case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable {
import schemaDef._
import jdbcProfile.api._
val database = Database.forDataSource(dataSource)
def createTables() = schemaDef.createTables(database)
def dropTables() = schemaDef.dropTables(database)
def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = {
val future: Future[R] = database.run(action)
blocking {
Await.result(future, 10 seconds)
}
}
def warmUp = {
- dbRun(allUserQuery.size.result)
+ dbRun(allUserQuery.size.result)dbRun(allUserQuery.size.result)
}
def selectUsers:Seq[UserRecord] = {
dbRun(allUserQuery.result)
}
// todo use whenever a shrine query is logged
def upsertUser(user:User):Unit = {
val userRecord = UserRecord(user)
dbRun(allUserQuery.insertOrUpdate(userRecord))
}
def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = {
val createInState = StewardConfigSource.createTopicsInState
val now = System.currentTimeMillis()
val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState)
val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now)
dbRun(for{
_ <- allTopicQuery += topicRecord
_ <- allUserTopicQuery += userTopicRecord
} yield topicRecord)
}
def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try {
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option =>
val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId))
if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy)
if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId)
val updatedTopic = oldTopicRecord.copy(name = topicRequest.name,
description = topicRequest.description,
changedBy = user.username,
changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).flatMap{_ =>
outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic)
}
}
)
}
def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = {
require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id")
val (count,topics,userNamesToOutboundUsers) = dbRun(
for{
count <- topicCountQuery(parameters).length.result
topics <- topicSelectQuery(parameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count, topics,userNamesToOutboundUsers))
ResearchersTopics(parameters.researcherIdOption.get,
count,
parameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
//treat as private (currently used in test)
def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = {
dbRun(topicSelectQuery(queryParameters).result)
}
def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = {
val (count,topics,userNamesToOutboundUsers) = dbRun{
for{
count <- topicCountQuery(queryParameters).length.result
topics <- topicSelectQuery(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,topics,userNamesToOutboundUsers)
}
StewardsTopics(count,
queryParameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val countFilter = topicCountQuery(queryParameters)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "id" => x.id
case "name" => x.name
case "description" => x.description
case "createdBy" => x.createdBy
case "createDate" => x.createDate
case "state" => x.state
case "changedBy" => x.changedBy
case "changeDate" => x.changeDate
})))
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery
val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId))
val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate))
maxDateFilter
}
def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = {
val noTopicRecord:Option[TopicRecord] = None
val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord)
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap(
_.fold(noOpDBIO){ originalTopic =>
val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).map(_ => Option(updatedTopic))
}
))
}
def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = {
dbRun(for{
totalTopics <- topicCountQuery(queryParameters).length.result
topicsPerStateName <- topicCountsPerState(queryParameters).result
} yield TopicsPerState(totalTopics,topicsPerStateName))
}
private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = {
val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state)
groupedByState.map{case (state,result) => (state,result.length)}
}
def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = {
//todo upsertUser(user) when the info is available from the PM
val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful {
if (StewardConfigSource.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved
else TopicState.createTopicsModeRequiresTopic
}
val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None}
val (state,topicName) = dbRun(for{
state <- topicId.fold(noOpDBIOForState)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map(
_.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state)))
)
topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption
)
_ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state)
} yield (state,topicName))
val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match {
case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name))
case (None,None) => None
case (Some(id),None) =>
if(state == TopicState.unknownForUser) None
else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery")
}
(state,topicIdAndName)
}
def selectQueryHistory(queryParameters: QueryParameters,topicParameter:Option[TopicId]):QueryHistory = {
val (count,shrineQueries,topics,userNamesToOutboundUsers) = dbRun(for {
count <- shrineQueryCountQuery(queryParameters,topicParameter).length.result
shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result
topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,shrineQueries,topics,userNamesToOutboundUsers))
val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap
def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = {
val topic = topicIdsToTopics.get(queryRecord.topicId)
val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers))
val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId)
//todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward
val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId))
queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser)
}
val result = QueryHistory(count,queryParameters.skipOption.getOrElse(0),shrineQueries.map(toOutboundShrineQuery))
result
}
private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = {
allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap)
}
private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val countQuery = shrineQueryCountQuery(queryParameters,topicParameter)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(limitFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countQuery) {
case "topicName" =>
val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id)
joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1)
case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "stewardId" => x.stewardId
case "externalId" => x.externalId
case "researcherId" => x.researcherId
case "name" => x.name
case "topic" => x.topicId
case "queryContents" => x.queryContents
case "stewardResponse" => x.stewardResponse
case "date" => x.date
}))
}
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable
val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId))
val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId))
//todo this is probably a binary Approved/Not approved
val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate))
maxDateFilter
}
def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = {
val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for {
totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result
queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set])
} yield (totalQueries,queriesPerUser,userNamesToOutboundUsers))
val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2))
QueriesPerUser(totalQueries,queriesPerOutboundUser)
}
private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = {
val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None)
val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId)
groupedByResearcher.map{case (researcher,result) => (researcher,result.length)}
}
lazy val nextTopicId:AtomicInteger = new AtomicInteger({
dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1
})
}
/**
* Separate class to support schema generation without actually connecting to the database.
*
* @param jdbcProfile Database profile to use for the schema
*/
case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable {
import jdbcProfile.api._
def ddlForAllTables = {
allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema
}
//to get the schema, use the REPL
//println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n"))
def createTables(database:Database) = {
try {
val future = database.run(ddlForAllTables.create)
Await.result(future,10 seconds)
} catch {
//I'd prefer to check and create schema only if absent. No way to do that with Oracle.
case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x)
}
}
def dropTables(database:Database) = {
val future = database.run(ddlForAllTables.drop)
//Really wait forever for the cleanup
Await.result(future,Duration.Inf)
}
class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") {
def userName = column[UserName]("userName",O.PrimaryKey)
def fullName = column[String]("fullName")
def isSteward = column[Boolean]("isSteward")
def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply)
}
class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") {
def id = column[TopicId]("id")
def name = column[String]("name")
def description = column[String]("description")
def createdBy = column[UserName]("createdBy")
def createDate = column[Date]("createDate")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def idIndex = index("idIndex",id,unique = false)
def topicNameIndex = index("topicNameIndex",name,unique = false)
def createdByIndex = index("createdByIndex",createdBy,unique = false)
def createDateIndex = index("createDateIndex",createDate,unique = false)
def stateIndex = index("stateIndex",state,unique = false)
def changedByIndex = index("changedByIndex",changedBy,unique = false)
def changeDateIndex = index("changeDateIndex",changeDate,unique = false)
def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply)
def fromRow = (fromParams _).tupled
def fromParams(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
stateName:String,
changedBy:UserName,
changeDate:Date): TopicRecord = {
TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(topicRecord: TopicRecord) =
Some((topicRecord.id,
topicRecord.name,
topicRecord.description,
topicRecord.createdBy,
topicRecord.createDate,
topicRecord.state.name,
topicRecord.changedBy,
topicRecord.changeDate
))
}
class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") {
def researcher = column[UserName]("researcher")
def topicId = column[TopicId]("topicId")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true)
def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow)
def fromRow = (fromParams _).tupled
def fromParams(researcher:UserName,
topicId:TopicId,
stateName:String,
changedBy:UserName,
changeDate:Date): UserTopicRecord = {
UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] =
Some((userTopicRecord.researcher,
userTopicRecord.topicId,
userTopicRecord.state.name,
userTopicRecord.changedBy,
userTopicRecord.changeDate
))
}
class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") {
def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc)
def externalId = column[ExternalQueryId]("id")
def name = column[String]("name")
def researcherId = column[UserName]("researcher")
def topicId = column[Option[TopicId]]("topic")
def queryContents = column[QueryContents]("queryContents")
def stewardResponse = column[String]("stewardResponse")
def date = column[Date]("date")
def externalIdIndex = index("externalIdIndex",externalId,unique = false)
def queryNameIndex = index("queryNameIndex",name,unique = false)
def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false)
def topicIdIndex = index("topicIdIndex",topicId,unique = false)
def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false)
def dateIndex = index("dateIndex",date,unique = false)
def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow)
def fromRow = (fromParams _).tupled
def fromParams(stewardId:Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:String,
date:Date): ShrineQueryRecord = {
ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date)
}
def toRow(queryRecord: ShrineQueryRecord):Option[(
Option[StewardQueryId],
ExternalQueryId,
String,
UserName,
Option[TopicId],
QueryContents,
String,
Date
)] =
Some((queryRecord.stewardId,
queryRecord.externalId,
queryRecord.name,
queryRecord.userId,
queryRecord.topicId,
queryRecord.queryContents,
queryRecord.stewardResponse.name,
queryRecord.date)
)
}
val allUserQuery = TableQuery[UserTable]
val allTopicQuery = TableQuery[TopicTable]
val allQueryTable = TableQuery[QueryTable]
val allUserTopicQuery = TableQuery[UserTopicTable]
val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for(
topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists
) yield topic
}
object StewardSchema {
val allConfig:Config = StewardConfigSource.config
val config:Config = allConfig.getConfig("shrine.steward.database")
val slickProfileClassName = config.getString("slickProfileClassName")
val slickProfile:JdbcProfile = StewardConfigSource.objectForName(slickProfileClassName)
val schema = StewardSchema(slickProfile)
}
-object StewardDatabase {
+object StewardDatabase extends NeedsWarmUp {
val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config)
val db = StewardDatabase(StewardSchema.schema,dataSource)
val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart")
if(createTablesOnStart) StewardDatabase.db.createTables()
+ override def warmUp() = StewardDatabase.db.warmUp
}
//API help
sealed case class SortOrder(name:String){
import slick.lifted.ColumnOrdered
def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = {
if(this == SortOrder.ascending) column.asc
else column.desc
}
}
object SortOrder {
val ascending = SortOrder("ascending")
val descending = SortOrder("descending")
val sortOrders = Seq(ascending,descending)
val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap
def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_))
}
case class QueryParameters(researcherIdOption:Option[UserName] = None,
stateOption:Option[TopicState] = None,
skipOption:Option[Int] = None,
limitOption:Option[Int] = None,
sortByOption:Option[String] = None,
sortOrder:SortOrder = SortOrder.ascending,
minDate:Option[Date] = None,
maxDate:Option[Date] = None
)
//DAO case classes, exposed for testing only
case class ShrineQueryRecord(stewardId: Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:TopicState,
date:Date) {
def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = {
OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,scala.xml.XML.loadString(queryContents),stewardResponse.name,date)
}
}
object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) {
def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = {
ShrineQueryRecord(
None,
shrineQuery.externalId,
shrineQuery.name,
userId,
topicId,
shrineQuery.queryContents,
stewardResponse,
System.currentTimeMillis())
}
}
case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) {
lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole)
else Set(researcherRole))
}
object UserRecord extends ((UserName,String,Boolean) => UserRecord) {
def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true")))
}
case class TopicRecord(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState,
changedBy:UserName,
changeDate:Date) {
def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = {
OutboundTopic(id.get,
name,
description,
userNamesToOutboundUsers(createdBy),
createDate,
state.name,
userNamesToOutboundUsers(changedBy),
changeDate)
}
}
object TopicRecord {
def apply(id:Option[TopicId],
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState
):TopicRecord = TopicRecord(id,
name,
description,
createdBy,
createDate,
state,
createdBy,
createDate)
}
case class UserTopicRecord(researcher:UserName,
topicId:TopicId,
state:TopicState,
changedBy:UserName,
changeDate:Date)
case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId")
case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}")
case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.")
\ No newline at end of file
diff --git a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
index ff5c8752f..7e19ab104 100644
--- a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
+++ b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
@@ -1,192 +1,194 @@
package net.shrine.problem
import java.util.concurrent.TimeoutException
import javax.sql.DataSource
import com.typesafe.config.Config
-import net.shrine.slick.{CouldNotRunDbIoActionException, TestableDataSourceCreator}
+import net.shrine.slick.{CouldNotRunDbIoActionException, NeedsWarmUp, TestableDataSourceCreator}
import slick.dbio.SuccessAction
import slick.driver.JdbcProfile
import slick.jdbc.meta.MTable
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.util.control.NonFatal
import scala.xml.XML
/**
* Problems database object, defines the PROBLEMS table schema and related queries,
* as well as all interactions with the database.
* @author ty
* @since 07/16
*/
-object Problems {
+object Problems extends NeedsWarmUp {
val config:Config = ProblemConfigSource.config.getConfig("shrine.dashboard.database")
val slickProfile:JdbcProfile = ProblemConfigSource.getObject("slickProfileClassName", config)
import slickProfile.api._
val dataSource: DataSource = TestableDataSourceCreator.dataSource(config)
lazy val db = {
val db = Database.forDataSource(dataSource)
val createTables: String = "createTablesOnStart"
if (config.hasPath(createTables) && config.getBoolean(createTables)) {
val duration = FiniteDuration(3, SECONDS)
Await.ready(db.run(IOActions.createIfNotExists), duration)
val testValues: String = "createTestValuesOnStart"
if (config.hasPath(testValues) && config.getBoolean(testValues)) {
def dumb(id: Int) = ProblemDigest(s"codec($id)", s"stamp($id)", s"sum($id)", s"desc($id)", {id} , id)
val dummyValues: Seq[ProblemDigest] = Seq(0, 1, 2, 3, 4, 5, 6).map(dumb)
Await.ready(db.run(Queries ++= dummyValues), duration)
}
}
db
}
+
+ def warmUp = DatabaseConnector.runBlocking(Queries.length.result)
+
/**
* The Problems Table. This is the table schema.
*/
class ProblemsT(tag: Tag) extends Table[ProblemDigest](tag, Queries.tableName) {
def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
def codec = column[String]("codec")
def stampText = column[String]("stampText")
def summary = column[String]("summary")
def description = column[String]("description")
def xml = column[String]("detailsXml")
def epoch= column[Long]("epoch")
// projection between table row and problem digest
def * = (id, codec, stampText, summary, description, xml, epoch) <> (rowToProblem, problemToRow)
def idx = index("idx_epoch", epoch, unique=false)
/**
* Converts a table row into a ProblemDigest.
* @param args the table row, represented as a five-tuple string
* @return the corresponding ProblemDigest
*/
def rowToProblem(args: (Int, String, String, String, String, String, Long)): ProblemDigest = args match {
case (id, codec, stampText, summary, description, detailsXml, epoch) =>
ProblemDigest(codec, stampText, summary, description, XML.loadString(detailsXml), epoch)
}
/**
* Converts a ProblemDigest into an Option of a table row. For now there is no failure
* condition, ie a ProblemDigest can always be a table row, but this is a place for
* possible future error handling
* @param problem the ProblemDigest to convert
* @return an Option of a table row.
*/
def problemToRow(problem: ProblemDigest): Option[(Int, String, String, String, String, String, Long)] = problem match {
case ProblemDigest(codec, stampText, summary, description, detailsXml, epoch) =>
// 7 is ignored on insert and replaced with an auto incremented id
Some((7, codec, stampText, summary, description, detailsXml.toString, epoch))
}
}
/**
* Queries related to the Problems table.
*/
object Queries extends TableQuery(new ProblemsT(_)) {
/**
* The table name
*/
val tableName = "problems"
/**
* Equivalent to Select * from Problems;
*/
val selectAll = this
/**
* Selects all the details xml sorted by the problem's time stamp.
*/
val selectDetails = this.map(_.xml)
/**
* Sorts the problems in descending order
*/
val descending = this.sortBy(_.epoch.desc)
/**
* Selects the last N problems, after the offset
*/
def lastNProblems(n: Int, offset: Int = 0) = this.descending.drop(offset).take(n)
}
/**
* DBIO Actions. These are pre-defined IO actions that may be useful.
* Using it to centralize the location of DBIOs.
*/
object IOActions {
val problems = Queries
val tableExists = MTable.getTables(problems.tableName).map(_.nonEmpty)
val createIfNotExists = tableExists.flatMap(
if (_) SuccessAction(NoOperation) else problems.schema.create)
val dropIfExists = tableExists.flatMap(
if (_) problems.schema.drop else SuccessAction(NoOperation))
val resetTable = createIfNotExists >> problems.selectAll.delete
val selectAll = problems.result
def sizeAndProblemDigest(n: Int, offset: Int = 0) = problems.lastNProblems(n, offset).result.zip(problems.size.result)
def findIndexOfDate(date: Long) = (problems.size - problems.filter(_.epoch <= date).size).result
}
/**
* Entry point for interacting with the database. Runs IO actions.
*/
object DatabaseConnector {
val IO = IOActions
/**
* Executes a series of IO actions as a single transactions
*/
def executeTransaction(actions: DBIOAction[_, NoStream, _]*): Future[Unit] = {
db.run(DBIO.seq(actions:_*).transactionally)
}
/**
* Executes a series of IO actions as a single transaction, synchronous
*/
def executeTransactionBlocking(actions: DBIOAction[_, NoStream, _]*)(implicit timeout: Duration): Unit = {
try {
Await.ready(this.executeTransaction(actions: _*), timeout)
} catch {
// TODO: Handle this better
case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx)
case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x)
}
}
/**
* Straight copy of db.run
*/
def run[R](dbio: DBIOAction[R, NoStream, _]): Future[R] = {
db.run(dbio)
}
/**
* Synchronized copy of db.run
*/
def runBlocking[R](dbio: DBIOAction[R, NoStream, _], timeout: Duration = new FiniteDuration(15, SECONDS)): R = {
try {
Await.result(this.run(dbio), timeout)
} catch {
case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx)
case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x)
}
}
def insertProblem(problem: ProblemDigest, timeout: Duration = new FiniteDuration(15, SECONDS)) = {
runBlocking(Queries += problem, timeout)
}
- def warmup = runBlocking(Queries.length.result)
}
}
// For SuccessAction, just a no_op.
case object NoOperation
\ No newline at end of file
diff --git a/commons/util/src/main/scala/net/shrine/slick/NeedsWarmUp.scala b/commons/util/src/main/scala/net/shrine/slick/NeedsWarmUp.scala
new file mode 100644
index 000000000..ab91c0560
--- /dev/null
+++ b/commons/util/src/main/scala/net/shrine/slick/NeedsWarmUp.scala
@@ -0,0 +1,9 @@
+package net.shrine.slick
+
+/**
+ * Designates initialization heavy objects that should be warmed up in order
+ * to improve user experience and front end performance
+ */
+trait NeedsWarmUp {
+ def warmUp(): Unit
+}
diff --git a/qep/service/src/main/scala/net/shrine/qep/audit/QepAuditDb.scala b/qep/service/src/main/scala/net/shrine/qep/audit/QepAuditDb.scala
index 4be710634..734a97e4d 100644
--- a/qep/service/src/main/scala/net/shrine/qep/audit/QepAuditDb.scala
+++ b/qep/service/src/main/scala/net/shrine/qep/audit/QepAuditDb.scala
@@ -1,180 +1,186 @@
package net.shrine.qep.audit
import java.sql.SQLException
import javax.sql.DataSource
import com.typesafe.config.Config
import net.shrine.audit.{NetworkQueryId, QueryName, QueryTopicId, QueryTopicName, ShrineNodeId, Time, UserName}
import net.shrine.log.Loggable
import net.shrine.protocol.RunQueryRequest
import net.shrine.qep.QepConfigSource
-import net.shrine.slick.TestableDataSourceCreator
+import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator}
import slick.driver.JdbcProfile
import scala.concurrent.duration.{Duration, DurationInt}
import scala.concurrent.{Await, Future, blocking}
import scala.language.postfixOps
/**
* DB code for the QEP audit metrics.
*
* @author david
* @since 8/18/15
*/
case class QepAuditDb(schemaDef:QepAuditSchema,dataSource: DataSource) extends Loggable {
import schemaDef._
import jdbcProfile.api._
val database = Database.forDataSource(dataSource)
+ def warmUp = dbRun(allQepQueryQuery.size.result)
+
def createTables() = schemaDef.createTables(database)
def dropTables() = schemaDef.dropTables(database)
def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = {
val future: Future[R] = database.run(action)
blocking {
Await.result(future, 10 seconds)
}
}
def insertQepQuery(runQueryRequest:RunQueryRequest,commonName:String):Unit = {
debug(s"insertQepQuery $runQueryRequest")
insertQepQuery(QepQueryAuditData.fromRunQueryRequest(runQueryRequest,commonName))
}
def insertQepQuery(qepQueryAuditData: QepQueryAuditData):Unit = {
dbRun(allQepQueryQuery += qepQueryAuditData)
}
def selectAllQepQueries:Seq[QepQueryAuditData] = {
dbRun(allQepQueryQuery.result)
}
}
-object QepAuditDb extends Loggable {
+object QepAuditDb extends Loggable with NeedsWarmUp {
val dataSource:DataSource = TestableDataSourceCreator.dataSource(QepAuditSchema.config)
val db = QepAuditDb(QepAuditSchema.schema,dataSource)
val createTablesOnStart = QepAuditSchema.config.getBoolean("createTablesOnStart")
if(createTablesOnStart) QepAuditDb.db.createTables()
+ def warmUp() = db.warmUp
+ // Todo, call this higher up
+ warmUp()
+
}
/**
* Separate class to support schema generation without actually connecting to the database.
*
* @param jdbcProfile Database profile to use for the schema
*/
case class QepAuditSchema(jdbcProfile: JdbcProfile) extends Loggable {
import jdbcProfile.api._
def ddlForAllTables = {
allQepQueryQuery.schema
}
//to get the schema, use the REPL
//println(QepAuditSchema.schema.ddlForAllTables.createStatements.mkString(";\n"))
def createTables(database:Database) = {
try {
val future = database.run(ddlForAllTables.create)
Await.result(future,10 seconds)
} catch {
//I'd prefer to check and create schema only if absent. No way to do that with Oracle.
case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x)
}
}
def dropTables(database:Database) = {
val future = database.run(ddlForAllTables.drop)
//Really wait forever for the cleanup
Await.result(future,Duration.Inf)
}
class QueriesSent(tag:Tag) extends Table[QepQueryAuditData](tag,"queriesSent") {
def shrineNodeId = column[ShrineNodeId]("shrineNodeId")
def userName = column[UserName]("userName")
def networkQueryId = column[NetworkQueryId]("networkQueryId")
def queryName = column[QueryName]("queryName")
def queryTopicId = column[Option[QueryTopicId]]("queryTopicId")
def queryTopicName = column[Option[QueryTopicName]]("queryTopicName")
def timeQuerySent = column[Time]("timeQuerySent")
def * = (shrineNodeId,userName,networkQueryId,queryName,queryTopicId,queryTopicName,timeQuerySent) <> (QepQueryAuditData.tupled,QepQueryAuditData.unapply)
}
val allQepQueryQuery = TableQuery[QueriesSent]
}
object QepAuditSchema {
val allConfig:Config = QepConfigSource.config
val config:Config = allConfig.getConfig("shrine.queryEntryPoint.audit.database")
val slickProfileClassName = config.getString("slickProfileClassName")
val slickProfile:JdbcProfile = QepConfigSource.objectForName(slickProfileClassName)
val schema = QepAuditSchema(slickProfile)
}
/**
* Container for QEP audit data for ACT metrics
*
* @author david
* @since 8/17/15
*/
case class QepQueryAuditData(
shrineNodeId:ShrineNodeId,
userName:UserName,
networkQueryId:NetworkQueryId,
queryName:QueryName,
queryTopicId:Option[QueryTopicId],
queryTopicName:Option[QueryTopicName],
timeQuerySent:Time
) {}
object QepQueryAuditData extends ((
ShrineNodeId,
UserName,
NetworkQueryId,
QueryName,
Option[QueryTopicId],
Option[QueryTopicName],
Time
) => QepQueryAuditData) {
def apply(
shrineNodeId:String,
userName:String,
networkQueryId:Long,
queryName:String,
queryTopicId:Option[String],
queryTopicName: Option[QueryTopicName]
):QepQueryAuditData = QepQueryAuditData(
shrineNodeId,
userName,
networkQueryId,
queryName,
queryTopicId,
queryTopicName,
System.currentTimeMillis()
)
def fromRunQueryRequest(request:RunQueryRequest,commonName:String):QepQueryAuditData = {
QepQueryAuditData(
commonName,
request.authn.username,
request.networkQueryId,
request.queryDefinition.name,
request.topicId,
request.topicName
)
}
}
\ No newline at end of file