diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
index 0f776cf44..224da53c7 100644
--- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
+++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala
@@ -1,17 +1,20 @@
package net.shrine.dashboard
-import akka.actor.{Props, ActorSystem}
+import akka.actor.{ActorSystem, Props}
+import net.shrine.problem.Problems
import spray.servlet.WebBoot
// this class is instantiated by the servlet initializer
// it needs to have a default constructor and implement
// the spray.servlet.WebBoot trait
class Boot extends WebBoot {
+ val warmUp = Problems.DatabaseConnector.warmup
+
// we need an ActorSystem to host our application in
val system = ActorSystem("DashboardActors",DashboardConfigSource.config)
// the service actor replies to incoming HttpRequests
val serviceActor = system.actorOf(Props[DashboardServiceActor])
}
\ No newline at end of file
diff --git a/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala b/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala
index 59f159f44..bd5f820b5 100644
--- a/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala
+++ b/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala
@@ -1,105 +1,98 @@
package net.shrine.proxy
-import java.io.BufferedReader
-import java.io.File
-import java.io.FileReader
-import java.io.IOException
-import java.util.ArrayList
-import java.util.List
import net.shrine.log.Loggable
import scala.xml.XML
import scala.xml.NodeSeq
import net.shrine.client.JerseyHttpClient
import net.shrine.client.HttpResponse
import net.shrine.crypto.TrustParam.AcceptAllCerts
-import scala.util.control.NonFatal
import net.shrine.client.HttpClient
/**
* [ Author ]
*
* @author Clint Gilbert
* @author Ricardo Delima
* @author Andrew McMurry
* @author Britt Fitch
*
* Date: Apr 1, 2008
* Harvard Medical School Center for BioMedical Informatics
* @link http://cbmi.med.harvard.edu
*
* NB: In the previous version of this class, the black list had no effect on the result of calling
* isAllawableDomain (now isAllowableUrl). This behavior is preserved here. -Clint
*
*/
trait ShrineProxy {
def isAllowableUrl(redirectURL: String): Boolean
def redirect(request: NodeSeq): HttpResponse
}
object DefaultShrineProxy extends Loggable {
private[proxy] def loadWhiteList: Set[String] = loadList("whitelist")
private[proxy] def loadBlackList: Set[String] = loadList("blacklist")
private def loadList(listname: String): Set[String] = {
val confFile = getClass.getClassLoader.getResource("shrine-proxy-acl.xml").getFile
val confXml = XML.loadFile(confFile)
(confXml \\ "lists" \ listname \ "host").map(_.text.trim).toSet
}
val jerseyHttpClient: HttpClient = {
import scala.concurrent.duration._
//TODO: Make timeout configurable?
JerseyHttpClient(AcceptAllCerts, Duration.Inf)
}
}
final class DefaultShrineProxy(val whiteList: Set[String], val blackList: Set[String], val httpClient: HttpClient) extends ShrineProxy with Loggable {
def this() = this(DefaultShrineProxy.loadWhiteList, DefaultShrineProxy.loadBlackList, DefaultShrineProxy.jerseyHttpClient)
import DefaultShrineProxy._
whiteList.foreach(entry => info(s"Whitelist entry: $entry"))
blackList.foreach(entry => info(s"Blacklist entry: $entry"))
info("Loaded access control lists.")
override def isAllowableUrl(redirectURL: String): Boolean = whiteList.exists(redirectURL.startsWith) && !blackList.exists(redirectURL.startsWith)
/**
* Redirect to a URL embedded within the I2B2 message
*
* @param request a chunk of xml with a element, containing the url to redirect to.
* @return the String result of accessing the url embedded in the passed request xml.
* @throws ShrineMessageFormatException bad input XML
*/
override def redirect(request: NodeSeq): HttpResponse = {
val redirectUrl = (request \\ "redirect_url").headOption.map(_.text.trim).getOrElse {
error("Error parsing redirect_url tag")
throw new ShrineMessageFormatException("Error parsing redirect_url tag")
}
if (redirectUrl == null || redirectUrl.isEmpty) {
error("Detected missing redirect_url tag")
throw new ShrineMessageFormatException("ShrineAdapter detected missing redirect_url tag")
}
//if redirectURL is not in the white list, do not proceed.
if (!isAllowableUrl(redirectUrl)) {
throw new ShrineMessageFormatException(s"redirectURL not in white list or is in black list: $redirectUrl")
}
debug(s"Proxy redirecting to $redirectUrl")
httpClient.post(request.toString, redirectUrl)
}
}
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
index ed225fd98..f5028a65a 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala
@@ -1,20 +1,23 @@
package net.shrine.steward
-import akka.actor.{Props, ActorSystem}
+import akka.actor.{ActorSystem, Props}
import net.shrine.log.Loggable
+import net.shrine.steward.db.StewardDatabase
import spray.servlet.WebBoot
// this class is instantiated by the servlet initializer
// it needs to have a default constructor and implement
// the spray.servlet.WebBoot trait
class Boot extends WebBoot with Loggable {
info(s"StewardActors akka daemonic config is ${StewardConfigSource.config.getString("akka.daemonic")}")
+ val warmUp = StewardDatabase.db.warmUp
+
// we need an ActorSystem to host our application in
val system = ActorSystem("StewardActors",StewardConfigSource.config)
// the service actor replies to incoming HttpRequests
val serviceActor = system.actorOf(Props[StewardServiceActor])
}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
index 660e1e440..7d161711e 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
@@ -1,634 +1,638 @@
package net.shrine.steward.db
import java.sql.SQLException
import java.util.concurrent.atomic.AtomicInteger
import javax.sql.DataSource
import com.typesafe.config.Config
import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole}
import net.shrine.i2b2.protocol.pm.User
import net.shrine.log.Loggable
import net.shrine.slick.TestableDataSourceCreator
import net.shrine.steward.{CreateTopicsMode, StewardConfigSource}
import slick.dbio.Effect.Read
import slick.driver.JdbcProfile
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.{Duration, DurationInt}
import scala.concurrent.{Await, Future, blocking}
import scala.language.postfixOps
import scala.util.Try
/**
* Database access code for the data steward service.
*
* I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace.
*
* @author dwalend
* @since 1.19
*/
case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable {
import schemaDef._
import jdbcProfile.api._
val database = Database.forDataSource(dataSource)
def createTables() = schemaDef.createTables(database)
def dropTables() = schemaDef.dropTables(database)
def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = {
val future: Future[R] = database.run(action)
blocking {
Await.result(future, 10 seconds)
}
}
+ def warmUp = {
+ dbRun(allUserQuery.size.result)
+ }
+
def selectUsers:Seq[UserRecord] = {
dbRun(allUserQuery.result)
}
// todo use whenever a shrine query is logged
def upsertUser(user:User):Unit = {
val userRecord = UserRecord(user)
dbRun(allUserQuery.insertOrUpdate(userRecord))
}
def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = {
val createInState = StewardConfigSource.createTopicsInState
val now = System.currentTimeMillis()
val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState)
val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now)
dbRun(for{
_ <- allTopicQuery += topicRecord
_ <- allUserTopicQuery += userTopicRecord
} yield topicRecord)
}
def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try {
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option =>
val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId))
if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy)
if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId)
val updatedTopic = oldTopicRecord.copy(name = topicRequest.name,
description = topicRequest.description,
changedBy = user.username,
changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).flatMap{_ =>
outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic)
}
}
)
}
def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = {
require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id")
val (count,topics,userNamesToOutboundUsers) = dbRun(
for{
count <- topicCountQuery(parameters).length.result
topics <- topicSelectQuery(parameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count, topics,userNamesToOutboundUsers))
ResearchersTopics(parameters.researcherIdOption.get,
count,
parameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
//treat as private (currently used in test)
def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = {
dbRun(topicSelectQuery(queryParameters).result)
}
def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = {
val (count,topics,userNamesToOutboundUsers) = dbRun{
for{
count <- topicCountQuery(queryParameters).length.result
topics <- topicSelectQuery(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,topics,userNamesToOutboundUsers)
}
StewardsTopics(count,
queryParameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val countFilter = topicCountQuery(queryParameters)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "id" => x.id
case "name" => x.name
case "description" => x.description
case "createdBy" => x.createdBy
case "createDate" => x.createDate
case "state" => x.state
case "changedBy" => x.changedBy
case "changeDate" => x.changeDate
})))
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery
val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId))
val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate))
maxDateFilter
}
def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = {
val noTopicRecord:Option[TopicRecord] = None
val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord)
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap(
_.fold(noOpDBIO){ originalTopic =>
val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).map(_ => Option(updatedTopic))
}
))
}
def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = {
dbRun(for{
totalTopics <- topicCountQuery(queryParameters).length.result
topicsPerStateName <- topicCountsPerState(queryParameters).result
} yield TopicsPerState(totalTopics,topicsPerStateName))
}
private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = {
val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state)
groupedByState.map{case (state,result) => (state,result.length)}
}
def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = {
//todo upsertUser(user) when the info is available from the PM
val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful {
if (StewardConfigSource.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved
else TopicState.createTopicsModeRequiresTopic
}
val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None}
val (state,topicName) = dbRun(for{
state <- topicId.fold(noOpDBIOForState)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map(
_.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state)))
)
topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption
)
_ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state)
} yield (state,topicName))
val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match {
case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name))
case (None,None) => None
case (Some(id),None) =>
if(state == TopicState.unknownForUser) None
else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery")
}
(state,topicIdAndName)
}
def selectQueryHistory(queryParameters: QueryParameters,topicParameter:Option[TopicId]):QueryHistory = {
val (count,shrineQueries,topics,userNamesToOutboundUsers) = dbRun(for {
count <- shrineQueryCountQuery(queryParameters,topicParameter).length.result
shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result
topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,shrineQueries,topics,userNamesToOutboundUsers))
val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap
def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = {
val topic = topicIdsToTopics.get(queryRecord.topicId)
val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers))
val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId)
//todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward
val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId))
queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser)
}
val result = QueryHistory(count,queryParameters.skipOption.getOrElse(0),shrineQueries.map(toOutboundShrineQuery))
result
}
private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = {
allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap)
}
private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val countQuery = shrineQueryCountQuery(queryParameters,topicParameter)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(limitFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countQuery) {
case "topicName" =>
val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id)
joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1)
case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "stewardId" => x.stewardId
case "externalId" => x.externalId
case "researcherId" => x.researcherId
case "name" => x.name
case "topic" => x.topicId
case "queryContents" => x.queryContents
case "stewardResponse" => x.stewardResponse
case "date" => x.date
}))
}
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable
val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId))
val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId))
//todo this is probably a binary Approved/Not approved
val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate))
maxDateFilter
}
def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = {
val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for {
totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result
queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set])
} yield (totalQueries,queriesPerUser,userNamesToOutboundUsers))
val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2))
QueriesPerUser(totalQueries,queriesPerOutboundUser)
}
private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = {
val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None)
val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId)
groupedByResearcher.map{case (researcher,result) => (researcher,result.length)}
}
lazy val nextTopicId:AtomicInteger = new AtomicInteger({
dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1
})
}
/**
* Separate class to support schema generation without actually connecting to the database.
*
* @param jdbcProfile Database profile to use for the schema
*/
case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable {
import jdbcProfile.api._
def ddlForAllTables = {
allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema
}
//to get the schema, use the REPL
//println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n"))
def createTables(database:Database) = {
try {
val future = database.run(ddlForAllTables.create)
Await.result(future,10 seconds)
} catch {
//I'd prefer to check and create schema only if absent. No way to do that with Oracle.
case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x)
}
}
def dropTables(database:Database) = {
val future = database.run(ddlForAllTables.drop)
//Really wait forever for the cleanup
Await.result(future,Duration.Inf)
}
class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") {
def userName = column[UserName]("userName",O.PrimaryKey)
def fullName = column[String]("fullName")
def isSteward = column[Boolean]("isSteward")
def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply)
}
class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") {
def id = column[TopicId]("id")
def name = column[String]("name")
def description = column[String]("description")
def createdBy = column[UserName]("createdBy")
def createDate = column[Date]("createDate")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def idIndex = index("idIndex",id,unique = false)
def topicNameIndex = index("topicNameIndex",name,unique = false)
def createdByIndex = index("createdByIndex",createdBy,unique = false)
def createDateIndex = index("createDateIndex",createDate,unique = false)
def stateIndex = index("stateIndex",state,unique = false)
def changedByIndex = index("changedByIndex",changedBy,unique = false)
def changeDateIndex = index("changeDateIndex",changeDate,unique = false)
def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply)
def fromRow = (fromParams _).tupled
def fromParams(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
stateName:String,
changedBy:UserName,
changeDate:Date): TopicRecord = {
TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(topicRecord: TopicRecord) =
Some((topicRecord.id,
topicRecord.name,
topicRecord.description,
topicRecord.createdBy,
topicRecord.createDate,
topicRecord.state.name,
topicRecord.changedBy,
topicRecord.changeDate
))
}
class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") {
def researcher = column[UserName]("researcher")
def topicId = column[TopicId]("topicId")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true)
def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow)
def fromRow = (fromParams _).tupled
def fromParams(researcher:UserName,
topicId:TopicId,
stateName:String,
changedBy:UserName,
changeDate:Date): UserTopicRecord = {
UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] =
Some((userTopicRecord.researcher,
userTopicRecord.topicId,
userTopicRecord.state.name,
userTopicRecord.changedBy,
userTopicRecord.changeDate
))
}
class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") {
def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc)
def externalId = column[ExternalQueryId]("id")
def name = column[String]("name")
def researcherId = column[UserName]("researcher")
def topicId = column[Option[TopicId]]("topic")
def queryContents = column[QueryContents]("queryContents")
def stewardResponse = column[String]("stewardResponse")
def date = column[Date]("date")
def externalIdIndex = index("externalIdIndex",externalId,unique = false)
def queryNameIndex = index("queryNameIndex",name,unique = false)
def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false)
def topicIdIndex = index("topicIdIndex",topicId,unique = false)
def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false)
def dateIndex = index("dateIndex",date,unique = false)
def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow)
def fromRow = (fromParams _).tupled
def fromParams(stewardId:Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:String,
date:Date): ShrineQueryRecord = {
ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date)
}
def toRow(queryRecord: ShrineQueryRecord):Option[(
Option[StewardQueryId],
ExternalQueryId,
String,
UserName,
Option[TopicId],
QueryContents,
String,
Date
)] =
Some((queryRecord.stewardId,
queryRecord.externalId,
queryRecord.name,
queryRecord.userId,
queryRecord.topicId,
queryRecord.queryContents,
queryRecord.stewardResponse.name,
queryRecord.date)
)
}
val allUserQuery = TableQuery[UserTable]
val allTopicQuery = TableQuery[TopicTable]
val allQueryTable = TableQuery[QueryTable]
val allUserTopicQuery = TableQuery[UserTopicTable]
val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for(
topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists
) yield topic
}
object StewardSchema {
val allConfig:Config = StewardConfigSource.config
val config:Config = allConfig.getConfig("shrine.steward.database")
val slickProfileClassName = config.getString("slickProfileClassName")
val slickProfile:JdbcProfile = StewardConfigSource.objectForName(slickProfileClassName)
val schema = StewardSchema(slickProfile)
}
object StewardDatabase {
val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config)
val db = StewardDatabase(StewardSchema.schema,dataSource)
val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart")
if(createTablesOnStart) StewardDatabase.db.createTables()
}
//API help
sealed case class SortOrder(name:String){
import slick.lifted.ColumnOrdered
def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = {
if(this == SortOrder.ascending) column.asc
else column.desc
}
}
object SortOrder {
val ascending = SortOrder("ascending")
val descending = SortOrder("descending")
val sortOrders = Seq(ascending,descending)
val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap
def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_))
}
case class QueryParameters(researcherIdOption:Option[UserName] = None,
stateOption:Option[TopicState] = None,
skipOption:Option[Int] = None,
limitOption:Option[Int] = None,
sortByOption:Option[String] = None,
sortOrder:SortOrder = SortOrder.ascending,
minDate:Option[Date] = None,
maxDate:Option[Date] = None
)
//DAO case classes, exposed for testing only
case class ShrineQueryRecord(stewardId: Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:TopicState,
date:Date) {
def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = {
OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,scala.xml.XML.loadString(queryContents),stewardResponse.name,date)
}
}
object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) {
def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = {
ShrineQueryRecord(
None,
shrineQuery.externalId,
shrineQuery.name,
userId,
topicId,
shrineQuery.queryContents,
stewardResponse,
System.currentTimeMillis())
}
}
case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) {
lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole)
else Set(researcherRole))
}
object UserRecord extends ((UserName,String,Boolean) => UserRecord) {
def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true")))
}
case class TopicRecord(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState,
changedBy:UserName,
changeDate:Date) {
def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = {
OutboundTopic(id.get,
name,
description,
userNamesToOutboundUsers(createdBy),
createDate,
state.name,
userNamesToOutboundUsers(changedBy),
changeDate)
}
}
object TopicRecord {
def apply(id:Option[TopicId],
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState
):TopicRecord = TopicRecord(id,
name,
description,
createdBy,
createDate,
state,
createdBy,
createDate)
}
case class UserTopicRecord(researcher:UserName,
topicId:TopicId,
state:TopicState,
changedBy:UserName,
changeDate:Date)
case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId")
case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}")
case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.")
\ No newline at end of file
diff --git a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
index ca22812c1..ff5c8752f 100644
--- a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
+++ b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala
@@ -1,190 +1,192 @@
package net.shrine.problem
import java.util.concurrent.TimeoutException
import javax.sql.DataSource
import com.typesafe.config.Config
import net.shrine.slick.{CouldNotRunDbIoActionException, TestableDataSourceCreator}
import slick.dbio.SuccessAction
import slick.driver.JdbcProfile
import slick.jdbc.meta.MTable
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.util.control.NonFatal
import scala.xml.XML
/**
* Problems database object, defines the PROBLEMS table schema and related queries,
* as well as all interactions with the database.
* @author ty
* @since 07/16
*/
object Problems {
val config:Config = ProblemConfigSource.config.getConfig("shrine.dashboard.database")
val slickProfile:JdbcProfile = ProblemConfigSource.getObject("slickProfileClassName", config)
import slickProfile.api._
val dataSource: DataSource = TestableDataSourceCreator.dataSource(config)
lazy val db = {
val db = Database.forDataSource(dataSource)
val createTables: String = "createTablesOnStart"
if (config.hasPath(createTables) && config.getBoolean(createTables)) {
val duration = FiniteDuration(3, SECONDS)
Await.ready(db.run(IOActions.createIfNotExists), duration)
val testValues: String = "createTestValuesOnStart"
if (config.hasPath(testValues) && config.getBoolean(testValues)) {
def dumb(id: Int) = ProblemDigest(s"codec($id)", s"stamp($id)", s"sum($id)", s"desc($id)", {id} , id)
val dummyValues: Seq[ProblemDigest] = Seq(0, 1, 2, 3, 4, 5, 6).map(dumb)
Await.ready(db.run(Queries ++= dummyValues), duration)
}
}
db
}
/**
* The Problems Table. This is the table schema.
*/
class ProblemsT(tag: Tag) extends Table[ProblemDigest](tag, Queries.tableName) {
def id = column[Int]("id", O.PrimaryKey, O.AutoInc)
def codec = column[String]("codec")
def stampText = column[String]("stampText")
def summary = column[String]("summary")
def description = column[String]("description")
def xml = column[String]("detailsXml")
def epoch= column[Long]("epoch")
// projection between table row and problem digest
def * = (id, codec, stampText, summary, description, xml, epoch) <> (rowToProblem, problemToRow)
def idx = index("idx_epoch", epoch, unique=false)
/**
* Converts a table row into a ProblemDigest.
* @param args the table row, represented as a five-tuple string
* @return the corresponding ProblemDigest
*/
def rowToProblem(args: (Int, String, String, String, String, String, Long)): ProblemDigest = args match {
case (id, codec, stampText, summary, description, detailsXml, epoch) =>
ProblemDigest(codec, stampText, summary, description, XML.loadString(detailsXml), epoch)
}
/**
* Converts a ProblemDigest into an Option of a table row. For now there is no failure
* condition, ie a ProblemDigest can always be a table row, but this is a place for
* possible future error handling
* @param problem the ProblemDigest to convert
* @return an Option of a table row.
*/
def problemToRow(problem: ProblemDigest): Option[(Int, String, String, String, String, String, Long)] = problem match {
case ProblemDigest(codec, stampText, summary, description, detailsXml, epoch) =>
// 7 is ignored on insert and replaced with an auto incremented id
Some((7, codec, stampText, summary, description, detailsXml.toString, epoch))
}
}
/**
* Queries related to the Problems table.
*/
object Queries extends TableQuery(new ProblemsT(_)) {
/**
* The table name
*/
val tableName = "problems"
/**
* Equivalent to Select * from Problems;
*/
val selectAll = this
/**
* Selects all the details xml sorted by the problem's time stamp.
*/
val selectDetails = this.map(_.xml)
/**
* Sorts the problems in descending order
*/
val descending = this.sortBy(_.epoch.desc)
/**
* Selects the last N problems, after the offset
*/
def lastNProblems(n: Int, offset: Int = 0) = this.descending.drop(offset).take(n)
}
/**
* DBIO Actions. These are pre-defined IO actions that may be useful.
* Using it to centralize the location of DBIOs.
*/
object IOActions {
val problems = Queries
val tableExists = MTable.getTables(problems.tableName).map(_.nonEmpty)
val createIfNotExists = tableExists.flatMap(
if (_) SuccessAction(NoOperation) else problems.schema.create)
val dropIfExists = tableExists.flatMap(
if (_) problems.schema.drop else SuccessAction(NoOperation))
val resetTable = createIfNotExists >> problems.selectAll.delete
val selectAll = problems.result
def sizeAndProblemDigest(n: Int, offset: Int = 0) = problems.lastNProblems(n, offset).result.zip(problems.size.result)
def findIndexOfDate(date: Long) = (problems.size - problems.filter(_.epoch <= date).size).result
}
/**
* Entry point for interacting with the database. Runs IO actions.
*/
object DatabaseConnector {
val IO = IOActions
/**
* Executes a series of IO actions as a single transactions
*/
def executeTransaction(actions: DBIOAction[_, NoStream, _]*): Future[Unit] = {
db.run(DBIO.seq(actions:_*).transactionally)
}
/**
* Executes a series of IO actions as a single transaction, synchronous
*/
def executeTransactionBlocking(actions: DBIOAction[_, NoStream, _]*)(implicit timeout: Duration): Unit = {
try {
Await.ready(this.executeTransaction(actions: _*), timeout)
} catch {
// TODO: Handle this better
case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx)
case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x)
}
}
/**
* Straight copy of db.run
*/
def run[R](dbio: DBIOAction[R, NoStream, _]): Future[R] = {
db.run(dbio)
}
/**
* Synchronized copy of db.run
*/
def runBlocking[R](dbio: DBIOAction[R, NoStream, _], timeout: Duration = new FiniteDuration(15, SECONDS)): R = {
try {
Await.result(this.run(dbio), timeout)
} catch {
case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx)
case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x)
}
}
def insertProblem(problem: ProblemDigest, timeout: Duration = new FiniteDuration(15, SECONDS)) = {
runBlocking(Queries += problem, timeout)
}
+
+ def warmup = runBlocking(Queries.length.result)
}
}
// For SuccessAction, just a no_op.
case object NoOperation
\ No newline at end of file