diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala index 0f776cf44..224da53c7 100644 --- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala +++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/Boot.scala @@ -1,17 +1,20 @@ package net.shrine.dashboard -import akka.actor.{Props, ActorSystem} +import akka.actor.{ActorSystem, Props} +import net.shrine.problem.Problems import spray.servlet.WebBoot // this class is instantiated by the servlet initializer // it needs to have a default constructor and implement // the spray.servlet.WebBoot trait class Boot extends WebBoot { + val warmUp = Problems.DatabaseConnector.warmup + // we need an ActorSystem to host our application in val system = ActorSystem("DashboardActors",DashboardConfigSource.config) // the service actor replies to incoming HttpRequests val serviceActor = system.actorOf(Props[DashboardServiceActor]) } \ No newline at end of file diff --git a/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala b/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala index 59f159f44..bd5f820b5 100644 --- a/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala +++ b/apps/proxy/src/main/scala/net/shrine/proxy/ShrineProxy.scala @@ -1,105 +1,98 @@ package net.shrine.proxy -import java.io.BufferedReader -import java.io.File -import java.io.FileReader -import java.io.IOException -import java.util.ArrayList -import java.util.List import net.shrine.log.Loggable import scala.xml.XML import scala.xml.NodeSeq import net.shrine.client.JerseyHttpClient import net.shrine.client.HttpResponse import net.shrine.crypto.TrustParam.AcceptAllCerts -import scala.util.control.NonFatal import net.shrine.client.HttpClient /** * [ Author ] * * @author Clint Gilbert * @author Ricardo Delima * @author Andrew McMurry * @author Britt Fitch *

* Date: Apr 1, 2008 * Harvard Medical School Center for BioMedical Informatics * @link http://cbmi.med.harvard.edu *

* NB: In the previous version of this class, the black list had no effect on the result of calling * isAllawableDomain (now isAllowableUrl). This behavior is preserved here. -Clint * */ trait ShrineProxy { def isAllowableUrl(redirectURL: String): Boolean def redirect(request: NodeSeq): HttpResponse } object DefaultShrineProxy extends Loggable { private[proxy] def loadWhiteList: Set[String] = loadList("whitelist") private[proxy] def loadBlackList: Set[String] = loadList("blacklist") private def loadList(listname: String): Set[String] = { val confFile = getClass.getClassLoader.getResource("shrine-proxy-acl.xml").getFile val confXml = XML.loadFile(confFile) (confXml \\ "lists" \ listname \ "host").map(_.text.trim).toSet } val jerseyHttpClient: HttpClient = { import scala.concurrent.duration._ //TODO: Make timeout configurable? JerseyHttpClient(AcceptAllCerts, Duration.Inf) } } final class DefaultShrineProxy(val whiteList: Set[String], val blackList: Set[String], val httpClient: HttpClient) extends ShrineProxy with Loggable { def this() = this(DefaultShrineProxy.loadWhiteList, DefaultShrineProxy.loadBlackList, DefaultShrineProxy.jerseyHttpClient) import DefaultShrineProxy._ whiteList.foreach(entry => info(s"Whitelist entry: $entry")) blackList.foreach(entry => info(s"Blacklist entry: $entry")) info("Loaded access control lists.") override def isAllowableUrl(redirectURL: String): Boolean = whiteList.exists(redirectURL.startsWith) && !blackList.exists(redirectURL.startsWith) /** * Redirect to a URL embedded within the I2B2 message * * @param request a chunk of xml with a element, containing the url to redirect to. * @return the String result of accessing the url embedded in the passed request xml. * @throws ShrineMessageFormatException bad input XML */ override def redirect(request: NodeSeq): HttpResponse = { val redirectUrl = (request \\ "redirect_url").headOption.map(_.text.trim).getOrElse { error("Error parsing redirect_url tag") throw new ShrineMessageFormatException("Error parsing redirect_url tag") } if (redirectUrl == null || redirectUrl.isEmpty) { error("Detected missing redirect_url tag") throw new ShrineMessageFormatException("ShrineAdapter detected missing redirect_url tag") } //if redirectURL is not in the white list, do not proceed. if (!isAllowableUrl(redirectUrl)) { throw new ShrineMessageFormatException(s"redirectURL not in white list or is in black list: $redirectUrl") } debug(s"Proxy redirecting to $redirectUrl") httpClient.post(request.toString, redirectUrl) } } diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala index ed225fd98..f5028a65a 100644 --- a/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala +++ b/apps/steward-app/src/main/scala/net/shrine/steward/Boot.scala @@ -1,20 +1,23 @@ package net.shrine.steward -import akka.actor.{Props, ActorSystem} +import akka.actor.{ActorSystem, Props} import net.shrine.log.Loggable +import net.shrine.steward.db.StewardDatabase import spray.servlet.WebBoot // this class is instantiated by the servlet initializer // it needs to have a default constructor and implement // the spray.servlet.WebBoot trait class Boot extends WebBoot with Loggable { info(s"StewardActors akka daemonic config is ${StewardConfigSource.config.getString("akka.daemonic")}") + val warmUp = StewardDatabase.db.warmUp + // we need an ActorSystem to host our application in val system = ActorSystem("StewardActors",StewardConfigSource.config) // the service actor replies to incoming HttpRequests val serviceActor = system.actorOf(Props[StewardServiceActor]) } \ No newline at end of file diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala index 660e1e440..7d161711e 100644 --- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala +++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala @@ -1,634 +1,638 @@ package net.shrine.steward.db import java.sql.SQLException import java.util.concurrent.atomic.AtomicInteger import javax.sql.DataSource import com.typesafe.config.Config import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole} import net.shrine.i2b2.protocol.pm.User import net.shrine.log.Loggable import net.shrine.slick.TestableDataSourceCreator import net.shrine.steward.{CreateTopicsMode, StewardConfigSource} import slick.dbio.Effect.Read import slick.driver.JdbcProfile import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, Future, blocking} import scala.language.postfixOps import scala.util.Try /** * Database access code for the data steward service. * * I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace. * * @author dwalend * @since 1.19 */ case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable { import schemaDef._ import jdbcProfile.api._ val database = Database.forDataSource(dataSource) def createTables() = schemaDef.createTables(database) def dropTables() = schemaDef.dropTables(database) def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = { val future: Future[R] = database.run(action) blocking { Await.result(future, 10 seconds) } } + def warmUp = { + dbRun(allUserQuery.size.result) + } + def selectUsers:Seq[UserRecord] = { dbRun(allUserQuery.result) } // todo use whenever a shrine query is logged def upsertUser(user:User):Unit = { val userRecord = UserRecord(user) dbRun(allUserQuery.insertOrUpdate(userRecord)) } def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = { val createInState = StewardConfigSource.createTopicsInState val now = System.currentTimeMillis() val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState) val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now) dbRun(for{ _ <- allTopicQuery += topicRecord _ <- allUserTopicQuery += userTopicRecord } yield topicRecord) } def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try { dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option => val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId)) if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy) if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId) val updatedTopic = oldTopicRecord.copy(name = topicRequest.name, description = topicRequest.description, changedBy = user.username, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).flatMap{_ => outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic) } } ) } def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = { require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id") val (count,topics,userNamesToOutboundUsers) = dbRun( for{ count <- topicCountQuery(parameters).length.result topics <- topicSelectQuery(parameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count, topics,userNamesToOutboundUsers)) ResearchersTopics(parameters.researcherIdOption.get, count, parameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } //treat as private (currently used in test) def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = { dbRun(topicSelectQuery(queryParameters).result) } def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = { val (count,topics,userNamesToOutboundUsers) = dbRun{ for{ count <- topicCountQuery(queryParameters).length.result topics <- topicSelectQuery(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count,topics,userNamesToOutboundUsers) } StewardsTopics(count, queryParameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val countFilter = topicCountQuery(queryParameters) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(countFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countFilter)( columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "id" => x.id case "name" => x.name case "description" => x.description case "createdBy" => x.createdBy case "createDate" => x.createDate case "state" => x.state case "changedBy" => x.changedBy case "changeDate" => x.changeDate }))) val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId)) val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate)) maxDateFilter } def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = { val noTopicRecord:Option[TopicRecord] = None val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord) dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap( _.fold(noOpDBIO){ originalTopic => val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).map(_ => Option(updatedTopic)) } )) } def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = { dbRun(for{ totalTopics <- topicCountQuery(queryParameters).length.result topicsPerStateName <- topicCountsPerState(queryParameters).result } yield TopicsPerState(totalTopics,topicsPerStateName)) } private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = { val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state) groupedByState.map{case (state,result) => (state,result.length)} } def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = { //todo upsertUser(user) when the info is available from the PM val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful { if (StewardConfigSource.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved else TopicState.createTopicsModeRequiresTopic } val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None} val (state,topicName) = dbRun(for{ state <- topicId.fold(noOpDBIOForState)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map( _.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state))) ) topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption ) _ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state) } yield (state,topicName)) val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match { case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name)) case (None,None) => None case (Some(id),None) => if(state == TopicState.unknownForUser) None else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery") } (state,topicIdAndName) } def selectQueryHistory(queryParameters: QueryParameters,topicParameter:Option[TopicId]):QueryHistory = { val (count,shrineQueries,topics,userNamesToOutboundUsers) = dbRun(for { count <- shrineQueryCountQuery(queryParameters,topicParameter).length.result shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count,shrineQueries,topics,userNamesToOutboundUsers)) val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = { val topic = topicIdsToTopics.get(queryRecord.topicId) val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers)) val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId) //todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId)) queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser) } val result = QueryHistory(count,queryParameters.skipOption.getOrElse(0),shrineQueries.map(toOutboundShrineQuery)) result } private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = { allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap) } private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val countQuery = shrineQueryCountQuery(queryParameters,topicParameter) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(limitFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countQuery) { case "topicName" => val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id) joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1) case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "stewardId" => x.stewardId case "externalId" => x.externalId case "researcherId" => x.researcherId case "name" => x.name case "topic" => x.topicId case "queryContents" => x.queryContents case "stewardResponse" => x.stewardResponse case "date" => x.date })) } val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId)) val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId)) //todo this is probably a binary Approved/Not approved val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate)) maxDateFilter } def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = { val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for { totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set]) } yield (totalQueries,queriesPerUser,userNamesToOutboundUsers)) val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2)) QueriesPerUser(totalQueries,queriesPerOutboundUser) } private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = { val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None) val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId) groupedByResearcher.map{case (researcher,result) => (researcher,result.length)} } lazy val nextTopicId:AtomicInteger = new AtomicInteger({ dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1 }) } /** * Separate class to support schema generation without actually connecting to the database. * * @param jdbcProfile Database profile to use for the schema */ case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable { import jdbcProfile.api._ def ddlForAllTables = { allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema } //to get the schema, use the REPL //println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n")) def createTables(database:Database) = { try { val future = database.run(ddlForAllTables.create) Await.result(future,10 seconds) } catch { //I'd prefer to check and create schema only if absent. No way to do that with Oracle. case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x) } } def dropTables(database:Database) = { val future = database.run(ddlForAllTables.drop) //Really wait forever for the cleanup Await.result(future,Duration.Inf) } class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") { def userName = column[UserName]("userName",O.PrimaryKey) def fullName = column[String]("fullName") def isSteward = column[Boolean]("isSteward") def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply) } class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") { def id = column[TopicId]("id") def name = column[String]("name") def description = column[String]("description") def createdBy = column[UserName]("createdBy") def createDate = column[Date]("createDate") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def idIndex = index("idIndex",id,unique = false) def topicNameIndex = index("topicNameIndex",name,unique = false) def createdByIndex = index("createdByIndex",createdBy,unique = false) def createDateIndex = index("createDateIndex",createDate,unique = false) def stateIndex = index("stateIndex",state,unique = false) def changedByIndex = index("changedByIndex",changedBy,unique = false) def changeDateIndex = index("changeDateIndex",changeDate,unique = false) def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply) def fromRow = (fromParams _).tupled def fromParams(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, stateName:String, changedBy:UserName, changeDate:Date): TopicRecord = { TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(topicRecord: TopicRecord) = Some((topicRecord.id, topicRecord.name, topicRecord.description, topicRecord.createdBy, topicRecord.createDate, topicRecord.state.name, topicRecord.changedBy, topicRecord.changeDate )) } class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") { def researcher = column[UserName]("researcher") def topicId = column[TopicId]("topicId") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true) def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow) def fromRow = (fromParams _).tupled def fromParams(researcher:UserName, topicId:TopicId, stateName:String, changedBy:UserName, changeDate:Date): UserTopicRecord = { UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] = Some((userTopicRecord.researcher, userTopicRecord.topicId, userTopicRecord.state.name, userTopicRecord.changedBy, userTopicRecord.changeDate )) } class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") { def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc) def externalId = column[ExternalQueryId]("id") def name = column[String]("name") def researcherId = column[UserName]("researcher") def topicId = column[Option[TopicId]]("topic") def queryContents = column[QueryContents]("queryContents") def stewardResponse = column[String]("stewardResponse") def date = column[Date]("date") def externalIdIndex = index("externalIdIndex",externalId,unique = false) def queryNameIndex = index("queryNameIndex",name,unique = false) def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false) def topicIdIndex = index("topicIdIndex",topicId,unique = false) def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false) def dateIndex = index("dateIndex",date,unique = false) def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow) def fromRow = (fromParams _).tupled def fromParams(stewardId:Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:String, date:Date): ShrineQueryRecord = { ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date) } def toRow(queryRecord: ShrineQueryRecord):Option[( Option[StewardQueryId], ExternalQueryId, String, UserName, Option[TopicId], QueryContents, String, Date )] = Some((queryRecord.stewardId, queryRecord.externalId, queryRecord.name, queryRecord.userId, queryRecord.topicId, queryRecord.queryContents, queryRecord.stewardResponse.name, queryRecord.date) ) } val allUserQuery = TableQuery[UserTable] val allTopicQuery = TableQuery[TopicTable] val allQueryTable = TableQuery[QueryTable] val allUserTopicQuery = TableQuery[UserTopicTable] val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for( topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists ) yield topic } object StewardSchema { val allConfig:Config = StewardConfigSource.config val config:Config = allConfig.getConfig("shrine.steward.database") val slickProfileClassName = config.getString("slickProfileClassName") val slickProfile:JdbcProfile = StewardConfigSource.objectForName(slickProfileClassName) val schema = StewardSchema(slickProfile) } object StewardDatabase { val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config) val db = StewardDatabase(StewardSchema.schema,dataSource) val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart") if(createTablesOnStart) StewardDatabase.db.createTables() } //API help sealed case class SortOrder(name:String){ import slick.lifted.ColumnOrdered def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = { if(this == SortOrder.ascending) column.asc else column.desc } } object SortOrder { val ascending = SortOrder("ascending") val descending = SortOrder("descending") val sortOrders = Seq(ascending,descending) val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_)) } case class QueryParameters(researcherIdOption:Option[UserName] = None, stateOption:Option[TopicState] = None, skipOption:Option[Int] = None, limitOption:Option[Int] = None, sortByOption:Option[String] = None, sortOrder:SortOrder = SortOrder.ascending, minDate:Option[Date] = None, maxDate:Option[Date] = None ) //DAO case classes, exposed for testing only case class ShrineQueryRecord(stewardId: Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:TopicState, date:Date) { def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = { OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,scala.xml.XML.loadString(queryContents),stewardResponse.name,date) } } object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) { def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = { ShrineQueryRecord( None, shrineQuery.externalId, shrineQuery.name, userId, topicId, shrineQuery.queryContents, stewardResponse, System.currentTimeMillis()) } } case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) { lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole) else Set(researcherRole)) } object UserRecord extends ((UserName,String,Boolean) => UserRecord) { def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true"))) } case class TopicRecord(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState, changedBy:UserName, changeDate:Date) { def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = { OutboundTopic(id.get, name, description, userNamesToOutboundUsers(createdBy), createDate, state.name, userNamesToOutboundUsers(changedBy), changeDate) } } object TopicRecord { def apply(id:Option[TopicId], name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState ):TopicRecord = TopicRecord(id, name, description, createdBy, createDate, state, createdBy, createDate) } case class UserTopicRecord(researcher:UserName, topicId:TopicId, state:TopicState, changedBy:UserName, changeDate:Date) case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId") case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}") case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.") \ No newline at end of file diff --git a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala index ca22812c1..ff5c8752f 100644 --- a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala +++ b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala @@ -1,190 +1,192 @@ package net.shrine.problem import java.util.concurrent.TimeoutException import javax.sql.DataSource import com.typesafe.config.Config import net.shrine.slick.{CouldNotRunDbIoActionException, TestableDataSourceCreator} import slick.dbio.SuccessAction import slick.driver.JdbcProfile import slick.jdbc.meta.MTable import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.concurrent.{Await, Future} import scala.util.control.NonFatal import scala.xml.XML /** * Problems database object, defines the PROBLEMS table schema and related queries, * as well as all interactions with the database. * @author ty * @since 07/16 */ object Problems { val config:Config = ProblemConfigSource.config.getConfig("shrine.dashboard.database") val slickProfile:JdbcProfile = ProblemConfigSource.getObject("slickProfileClassName", config) import slickProfile.api._ val dataSource: DataSource = TestableDataSourceCreator.dataSource(config) lazy val db = { val db = Database.forDataSource(dataSource) val createTables: String = "createTablesOnStart" if (config.hasPath(createTables) && config.getBoolean(createTables)) { val duration = FiniteDuration(3, SECONDS) Await.ready(db.run(IOActions.createIfNotExists), duration) val testValues: String = "createTestValuesOnStart" if (config.hasPath(testValues) && config.getBoolean(testValues)) { def dumb(id: Int) = ProblemDigest(s"codec($id)", s"stamp($id)", s"sum($id)", s"desc($id)",

{id}
, id) val dummyValues: Seq[ProblemDigest] = Seq(0, 1, 2, 3, 4, 5, 6).map(dumb) Await.ready(db.run(Queries ++= dummyValues), duration) } } db } /** * The Problems Table. This is the table schema. */ class ProblemsT(tag: Tag) extends Table[ProblemDigest](tag, Queries.tableName) { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def codec = column[String]("codec") def stampText = column[String]("stampText") def summary = column[String]("summary") def description = column[String]("description") def xml = column[String]("detailsXml") def epoch= column[Long]("epoch") // projection between table row and problem digest def * = (id, codec, stampText, summary, description, xml, epoch) <> (rowToProblem, problemToRow) def idx = index("idx_epoch", epoch, unique=false) /** * Converts a table row into a ProblemDigest. * @param args the table row, represented as a five-tuple string * @return the corresponding ProblemDigest */ def rowToProblem(args: (Int, String, String, String, String, String, Long)): ProblemDigest = args match { case (id, codec, stampText, summary, description, detailsXml, epoch) => ProblemDigest(codec, stampText, summary, description, XML.loadString(detailsXml), epoch) } /** * Converts a ProblemDigest into an Option of a table row. For now there is no failure * condition, ie a ProblemDigest can always be a table row, but this is a place for * possible future error handling * @param problem the ProblemDigest to convert * @return an Option of a table row. */ def problemToRow(problem: ProblemDigest): Option[(Int, String, String, String, String, String, Long)] = problem match { case ProblemDigest(codec, stampText, summary, description, detailsXml, epoch) => // 7 is ignored on insert and replaced with an auto incremented id Some((7, codec, stampText, summary, description, detailsXml.toString, epoch)) } } /** * Queries related to the Problems table. */ object Queries extends TableQuery(new ProblemsT(_)) { /** * The table name */ val tableName = "problems" /** * Equivalent to Select * from Problems; */ val selectAll = this /** * Selects all the details xml sorted by the problem's time stamp. */ val selectDetails = this.map(_.xml) /** * Sorts the problems in descending order */ val descending = this.sortBy(_.epoch.desc) /** * Selects the last N problems, after the offset */ def lastNProblems(n: Int, offset: Int = 0) = this.descending.drop(offset).take(n) } /** * DBIO Actions. These are pre-defined IO actions that may be useful. * Using it to centralize the location of DBIOs. */ object IOActions { val problems = Queries val tableExists = MTable.getTables(problems.tableName).map(_.nonEmpty) val createIfNotExists = tableExists.flatMap( if (_) SuccessAction(NoOperation) else problems.schema.create) val dropIfExists = tableExists.flatMap( if (_) problems.schema.drop else SuccessAction(NoOperation)) val resetTable = createIfNotExists >> problems.selectAll.delete val selectAll = problems.result def sizeAndProblemDigest(n: Int, offset: Int = 0) = problems.lastNProblems(n, offset).result.zip(problems.size.result) def findIndexOfDate(date: Long) = (problems.size - problems.filter(_.epoch <= date).size).result } /** * Entry point for interacting with the database. Runs IO actions. */ object DatabaseConnector { val IO = IOActions /** * Executes a series of IO actions as a single transactions */ def executeTransaction(actions: DBIOAction[_, NoStream, _]*): Future[Unit] = { db.run(DBIO.seq(actions:_*).transactionally) } /** * Executes a series of IO actions as a single transaction, synchronous */ def executeTransactionBlocking(actions: DBIOAction[_, NoStream, _]*)(implicit timeout: Duration): Unit = { try { Await.ready(this.executeTransaction(actions: _*), timeout) } catch { // TODO: Handle this better case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx) case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x) } } /** * Straight copy of db.run */ def run[R](dbio: DBIOAction[R, NoStream, _]): Future[R] = { db.run(dbio) } /** * Synchronized copy of db.run */ def runBlocking[R](dbio: DBIOAction[R, NoStream, _], timeout: Duration = new FiniteDuration(15, SECONDS)): R = { try { Await.result(this.run(dbio), timeout) } catch { case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx) case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x) } } def insertProblem(problem: ProblemDigest, timeout: Duration = new FiniteDuration(15, SECONDS)) = { runBlocking(Queries += problem, timeout) } + + def warmup = runBlocking(Queries.length.result) } } // For SuccessAction, just a no_op. case object NoOperation \ No newline at end of file