diff --git a/apps/steward-app/pom.xml b/apps/steward-app/pom.xml
index 0a3f77e30..16f918d44 100644
--- a/apps/steward-app/pom.xml
+++ b/apps/steward-app/pom.xml
@@ -1,172 +1,177 @@
shrine-base
net.shrine
1.22.2.0-SNAPSHOT
../../pom.xml
4.0.0
steward-app
Steward App
jar
src/main/scala
src/test/scala
net.alchim31.maven
scala-maven-plugin
com.github.eirslett
frontend-maven-plugin
1.0
src/main/js
install node and npm
install-node-and-npm
v6.2.2
3.10.3
npm install
npm
generate-resources
bower install
bower
gulp test and build
gulp
--no-color
io.spray
spray-routing_2.11
${spray-version}
io.spray
spray-servlet_2.11
${spray-version}
io.spray
spray-util_2.11
${spray-version}
io.spray
spray-testkit_2.11
${spray-version}
test
com.typesafe.akka
akka-actor_2.11
${akka-version}
com.typesafe.akka
akka-slf4j_2.11
${akka-version}
com.typesafe.akka
akka-testkit_2.11
${akka-testkit-version}
test
org.json4s
json4s-native_2.11
${json4s-version}
com.typesafe.slick
slick_2.11
${slick-version}
org.suecarter
freeslick_2.11
${freeslick-version}
org.slf4j
slf4j-log4j12
${slf4j-version}
com.h2database
h2
${h2-version}
test
net.shrine
shrine-protocol
${project.version}
+
+ net.shrine
+ shrine-email
+ ${project.version}
+
net.shrine
shrine-data-commons
${project.version}
net.shrine
shrine-client
${project.version}
net.shrine
shrine-crypto
${project.version}
test-jar
test
net.shrine
shrine-auth
${project.version}
mysql
mysql-connector-java
${mysql-version}
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
index 60110c7d9..fb96164eb 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
@@ -1,638 +1,751 @@
package net.shrine.steward.db
import java.sql.SQLException
import java.util.concurrent.atomic.AtomicInteger
import javax.sql.DataSource
import com.typesafe.config.Config
-import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole}
+import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearcherToAudit, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole}
import net.shrine.i2b2.protocol.pm.User
import net.shrine.log.Loggable
import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator}
import net.shrine.steward.{CreateTopicsMode, StewardConfigSource}
import slick.dbio.Effect.Read
import slick.driver.JdbcProfile
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.{Duration, DurationInt}
import scala.concurrent.{Await, Future, blocking}
import scala.language.postfixOps
import scala.util.Try
/**
* Database access code for the data steward service.
*
* I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace.
*
* @author dwalend
* @since 1.19
*/
case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable {
import schemaDef._
import jdbcProfile.api._
val database = Database.forDataSource(dataSource)
def createTables() = schemaDef.createTables(database)
def dropTables() = schemaDef.dropTables(database)
def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = {
val future: Future[R] = database.run(action)
blocking {
Await.result(future, 10 seconds)
}
}
def warmUp = {
dbRun(allUserQuery.size.result)
}
def selectUsers:Seq[UserRecord] = {
dbRun(allUserQuery.result)
}
// todo use whenever a shrine query is logged
def upsertUser(user:User):Unit = {
val userRecord = UserRecord(user)
dbRun(allUserQuery.insertOrUpdate(userRecord))
}
def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = {
val createInState = StewardConfigSource.createTopicsInState
val now = System.currentTimeMillis()
val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState)
val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now)
dbRun(for{
_ <- allTopicQuery += topicRecord
_ <- allUserTopicQuery += userTopicRecord
} yield topicRecord)
}
def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try {
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option =>
val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId))
if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy)
if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId)
val updatedTopic = oldTopicRecord.copy(name = topicRequest.name,
description = topicRequest.description,
changedBy = user.username,
changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).flatMap{_ =>
outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic)
}
}
)
}
def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = {
require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id")
val (count,topics,userNamesToOutboundUsers) = dbRun(
for{
count <- topicCountQuery(parameters).length.result
topics <- topicSelectQuery(parameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count, topics,userNamesToOutboundUsers))
ResearchersTopics(parameters.researcherIdOption.get,
count,
parameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
//treat as private (currently used in test)
def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = {
dbRun(topicSelectQuery(queryParameters).result)
}
def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = {
val (count,topics,userNamesToOutboundUsers) = dbRun{
for{
count <- topicCountQuery(queryParameters).length.result
topics <- topicSelectQuery(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,topics,userNamesToOutboundUsers)
}
StewardsTopics(count,
queryParameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val countFilter = topicCountQuery(queryParameters)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "id" => x.id
case "name" => x.name
case "description" => x.description
case "createdBy" => x.createdBy
case "createDate" => x.createDate
case "state" => x.state
case "changedBy" => x.changedBy
case "changeDate" => x.changeDate
})))
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery
val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId))
val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate))
maxDateFilter
}
def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = {
val noTopicRecord:Option[TopicRecord] = None
val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord)
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap(
_.fold(noOpDBIO){ originalTopic =>
val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).map(_ => Option(updatedTopic))
}
))
}
def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = {
dbRun(for{
totalTopics <- topicCountQuery(queryParameters).length.result
topicsPerStateName <- topicCountsPerState(queryParameters).result
} yield TopicsPerState(totalTopics,topicsPerStateName))
}
private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = {
val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state)
groupedByState.map{case (state,result) => (state,result.length)}
}
def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = {
//todo upsertUser(user) when the info is available from the PM
val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful {
if (StewardConfigSource.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved
else TopicState.createTopicsModeRequiresTopic
}
val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None}
val (state,topicName) = dbRun(for{
state <- topicId.fold(noOpDBIOForState)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map(
_.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state)))
)
topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption
)
_ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state)
} yield (state,topicName))
val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match {
case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name))
case (None,None) => None
case (Some(id),None) =>
if(state == TopicState.unknownForUser) None
else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery")
+ case (None,Some(name)) =>
+ if(state == TopicState.unknownForUser) None
+ else throw new IllegalStateException(s"How did you get here for $userId with no topic id but a topic name of $name and $state for $shrineQuery")
}
(state,topicIdAndName)
}
def selectQueryHistory(queryParameters: QueryParameters,topicParameter:Option[TopicId]):QueryHistory = {
val (count,shrineQueries,topics,userNamesToOutboundUsers) = dbRun(for {
count <- shrineQueryCountQuery(queryParameters,topicParameter).length.result
shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result
topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,shrineQueries,topics,userNamesToOutboundUsers))
val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap
def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = {
val topic = topicIdsToTopics.get(queryRecord.topicId)
val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers))
val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId)
//todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward
val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId))
queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser)
}
val result = QueryHistory(count,queryParameters.skipOption.getOrElse(0),shrineQueries.map(toOutboundShrineQuery))
result
}
private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = {
allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap)
}
private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val countQuery = shrineQueryCountQuery(queryParameters,topicParameter)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(limitFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countQuery) {
case "topicName" =>
val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id)
joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1)
case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "stewardId" => x.stewardId
case "externalId" => x.externalId
case "researcherId" => x.researcherId
case "name" => x.name
case "topic" => x.topicId
case "queryContents" => x.queryContents
case "stewardResponse" => x.stewardResponse
case "date" => x.date
}))
}
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable
val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId))
val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId))
//todo this is probably a binary Approved/Not approved
val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate))
maxDateFilter
}
def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = {
val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for {
totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result
queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set])
} yield (totalQueries,queriesPerUser,userNamesToOutboundUsers))
val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2))
QueriesPerUser(totalQueries,queriesPerOutboundUser)
}
private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = {
val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None)
val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId)
groupedByResearcher.map{case (researcher,result) => (researcher,result.length)}
}
lazy val nextTopicId:AtomicInteger = new AtomicInteger({
dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1
})
+
+ def selectAllAuditRequests: Seq[UserAuditRecord] = {
+ dbRun(allUserAudits.result)
+ }
+
+ def selectMostRecentAuditRequests: Seq[UserAuditRecord] = {
+ dbRun(mostRecentUserAudits.result)
+ }
+
+ def selectResearchersToAudit(maxQueryCountBetweenAudits:Int,minTimeBetweenAudits:Duration):Seq[ResearcherToAudit] = {
+
+ val now = System.currentTimeMillis()
+ println(s"now $now minTimeBetweenAudits $minTimeBetweenAudits maxQueryCountBetweenAudits $maxQueryCountBetweenAudits")
+
+
+ //todo one round with the db instead of O(researchers)
+
+ //for each researcher
+ //horizon = if the researcher has had an audit
+ // date of last audit
+ // else if no audit yet
+ // date of first query
+ val researchersToHorizons: Map[UserName, Date] = dbRun(for{
+ dateOfFirstQuery: Seq[(UserName, Date)] <- leastRecentUserQuery.map(record => record.researcherId -> record.date).result
+ mostRecentAudit: Seq[(UserName, Date)] <- mostRecentUserAudits.map(record => record.researcher -> record.changeDate).result
+ } yield {
+ dateOfFirstQuery.toMap ++ mostRecentAudit.toMap
+ })
+
+ println(s"researchersToHorizons $researchersToHorizons")
+
+ val researchersToHorizonsAndCounts = researchersToHorizons.map{ researcherDate =>
+
+ val queryParameters = QueryParameters(researcherIdOption = Some(researcherDate._1),
+ minDate = Some(researcherDate._2))
+
+ val count:Int = dbRun(shrineQueryCountQuery(queryParameters,None).length.result)
+ (researcherDate._1,(researcherDate._2,count))
+ }
+
+ println(s"researchersToHorizonsAndCounts $researchersToHorizonsAndCounts")
+
+ //audit if oldest query within the horizon is >= 30 days in the past and the researcher has run at least one query since.
+ //todo configure
+ val oldestAllowed = System.currentTimeMillis() - minTimeBetweenAudits.toMillis
+ val timeBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 > 0 && x._2._1 <= oldestAllowed)
+
+ println(s"timeBasedAudit $timeBasedAudit")
+
+ //audit if the researcher has run >= 30 queries since horizon?
+ //todo configure
+ val queryBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 >= maxQueryCountBetweenAudits)
+
+ println(s"queryBasedAudit $queryBasedAudit")
+
+ val toAudit = timeBasedAudit ++ queryBasedAudit
+
+ val namesToOutboundUsers: Map[UserName, OutboundUser] = dbRun(outboundUsersForNamesAction(toAudit.keySet))
+
+ toAudit.map(x => ResearcherToAudit(namesToOutboundUsers(x._1),x._2._2,x._2._1,now)).to[Seq]
+ }
+
+ def logAuditRequests(auditRequests:Seq[ResearcherToAudit],now:Date) {
+ dbRun{
+ allUserAudits ++= auditRequests.map(x => UserAuditRecord(researcher = x.researcher.userName,
+ queryCount = x.count,
+ changeDate = now
+ ))
+ }
+ }
+
}
/**
* Separate class to support schema generation without actually connecting to the database.
*
* @param jdbcProfile Database profile to use for the schema
*/
case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable {
import jdbcProfile.api._
def ddlForAllTables = {
- allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema
+ allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema ++ allUserAudits.schema
}
//to get the schema, use the REPL
//println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n"))
def createTables(database:Database) = {
try {
val future = database.run(ddlForAllTables.create)
Await.result(future,10 seconds)
} catch {
//I'd prefer to check and create schema only if absent. No way to do that with Oracle.
case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x)
}
}
def dropTables(database:Database) = {
val future = database.run(ddlForAllTables.drop)
//Really wait forever for the cleanup
Await.result(future,Duration.Inf)
}
class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") {
def userName = column[UserName]("userName",O.PrimaryKey)
def fullName = column[String]("fullName")
def isSteward = column[Boolean]("isSteward")
def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply)
}
class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") {
def id = column[TopicId]("id")
def name = column[String]("name")
def description = column[String]("description")
def createdBy = column[UserName]("createdBy")
def createDate = column[Date]("createDate")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def idIndex = index("idIndex",id,unique = false)
def topicNameIndex = index("topicNameIndex",name,unique = false)
def createdByIndex = index("createdByIndex",createdBy,unique = false)
def createDateIndex = index("createDateIndex",createDate,unique = false)
def stateIndex = index("stateIndex",state,unique = false)
def changedByIndex = index("changedByIndex",changedBy,unique = false)
def changeDateIndex = index("changeDateIndex",changeDate,unique = false)
def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply)
def fromRow = (fromParams _).tupled
def fromParams(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
stateName:String,
changedBy:UserName,
changeDate:Date): TopicRecord = {
TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(topicRecord: TopicRecord) =
Some((topicRecord.id,
topicRecord.name,
topicRecord.description,
topicRecord.createdBy,
topicRecord.createDate,
topicRecord.state.name,
topicRecord.changedBy,
topicRecord.changeDate
))
}
class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") {
def researcher = column[UserName]("researcher")
def topicId = column[TopicId]("topicId")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true)
def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow)
def fromRow = (fromParams _).tupled
def fromParams(researcher:UserName,
topicId:TopicId,
stateName:String,
changedBy:UserName,
changeDate:Date): UserTopicRecord = {
UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] =
Some((userTopicRecord.researcher,
userTopicRecord.topicId,
userTopicRecord.state.name,
userTopicRecord.changedBy,
userTopicRecord.changeDate
))
+ }
+
+ class UserAuditTable(tag:Tag) extends Table[UserAuditRecord](tag,"userAudit") {
+ def researcher = column[UserName]("researcher")
+ def queryCount = column[Int]("queryCount")
+ def changeDate = column[Date]("changeDate")
+
+ def * = (researcher, queryCount, changeDate) <> (fromRow, toRow)
+ def fromRow = (fromParams _).tupled
+
+ def fromParams(researcher:UserName,
+ queryCount:Int,
+ changeDate:Date): UserAuditRecord = {
+ UserAuditRecord(researcher,queryCount, changeDate)
+ }
+
+ def toRow(record: UserAuditRecord):Option[(UserName,Int,Date)] =
+ Some((record.researcher,
+ record.queryCount,
+ record.changeDate
+ ))
}
class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") {
def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc)
def externalId = column[ExternalQueryId]("id")
def name = column[String]("name")
def researcherId = column[UserName]("researcher")
def topicId = column[Option[TopicId]]("topic")
def queryContents = column[QueryContents]("queryContents")
def stewardResponse = column[String]("stewardResponse")
def date = column[Date]("date")
def externalIdIndex = index("externalIdIndex",externalId,unique = false)
def queryNameIndex = index("queryNameIndex",name,unique = false)
def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false)
def topicIdIndex = index("topicIdIndex",topicId,unique = false)
def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false)
def dateIndex = index("dateIndex",date,unique = false)
def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow)
def fromRow = (fromParams _).tupled
def fromParams(stewardId:Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:String,
date:Date): ShrineQueryRecord = {
ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date)
}
def toRow(queryRecord: ShrineQueryRecord):Option[(
Option[StewardQueryId],
ExternalQueryId,
String,
UserName,
Option[TopicId],
QueryContents,
String,
Date
)] =
Some((queryRecord.stewardId,
queryRecord.externalId,
queryRecord.name,
queryRecord.userId,
queryRecord.topicId,
queryRecord.queryContents,
queryRecord.stewardResponse.name,
queryRecord.date)
)
}
val allUserQuery = TableQuery[UserTable]
val allTopicQuery = TableQuery[TopicTable]
val allQueryTable = TableQuery[QueryTable]
val allUserTopicQuery = TableQuery[UserTopicTable]
+ val allUserAudits = TableQuery[UserAuditTable]
val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for(
topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists
) yield topic
+ val mostRecentUserAudits: Query[UserAuditTable, UserAuditRecord, Seq] = for(
+ record <- allUserAudits if !allUserAudits.filter(_.researcher === record.researcher).filter(_.changeDate > record.changeDate).exists
+ ) yield record
+
+ val leastRecentUserQuery: Query[QueryTable, ShrineQueryRecord, Seq] = for(
+ record <- allQueryTable if !allQueryTable.filter(_.researcherId === record.researcherId).filter(_.date < record.date).exists
+ ) yield record
+
}
object StewardSchema {
val allConfig:Config = StewardConfigSource.config
val config:Config = allConfig.getConfig("shrine.steward.database")
val slickProfile:JdbcProfile = StewardConfigSource.getObject("slickProfileClassName", config)
val schema = StewardSchema(slickProfile)
}
object StewardDatabase extends NeedsWarmUp {
val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config)
val db = StewardDatabase(StewardSchema.schema,dataSource)
val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart")
if(createTablesOnStart) StewardDatabase.db.createTables()
override def warmUp() = StewardDatabase.db.warmUp
}
//API help
sealed case class SortOrder(name:String){
import slick.lifted.ColumnOrdered
def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = {
if(this == SortOrder.ascending) column.asc
else column.desc
}
}
object SortOrder {
val ascending = SortOrder("ascending")
val descending = SortOrder("descending")
val sortOrders = Seq(ascending,descending)
val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap
def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_))
}
case class QueryParameters(researcherIdOption:Option[UserName] = None,
stateOption:Option[TopicState] = None,
skipOption:Option[Int] = None,
limitOption:Option[Int] = None,
sortByOption:Option[String] = None,
sortOrder:SortOrder = SortOrder.ascending,
minDate:Option[Date] = None,
maxDate:Option[Date] = None
)
//DAO case classes, exposed for testing only
case class ShrineQueryRecord(stewardId: Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:TopicState,
date:Date) {
def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = {
OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,queryContents,stewardResponse.name,date)
}
}
object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) {
def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = {
ShrineQueryRecord(
None,
shrineQuery.externalId,
shrineQuery.name,
userId,
topicId,
shrineQuery.queryContents,
stewardResponse,
System.currentTimeMillis())
}
}
case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) {
lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole)
else Set(researcherRole))
}
object UserRecord extends ((UserName,String,Boolean) => UserRecord) {
def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true")))
}
case class TopicRecord(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState,
changedBy:UserName,
changeDate:Date) {
def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = {
OutboundTopic(id.get,
name,
description,
userNamesToOutboundUsers(createdBy),
createDate,
state.name,
userNamesToOutboundUsers(changedBy),
changeDate)
}
}
object TopicRecord {
def apply(id:Option[TopicId],
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState
):TopicRecord = TopicRecord(id,
name,
description,
createdBy,
createDate,
state,
createdBy,
createDate)
}
case class UserTopicRecord(researcher:UserName,
topicId:TopicId,
state:TopicState,
changedBy:UserName,
changeDate:Date)
+case class UserAuditRecord(researcher:UserName,
+ queryCount:Int,
+ changeDate:Date) {
+ def sameExceptForTimes(userAuditRecord: UserAuditRecord):Boolean = {
+ (researcher == userAuditRecord.researcher) &&
+ (queryCount == userAuditRecord.queryCount)
+ }
+}
+
case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId")
case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}")
case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.")
\ No newline at end of file
diff --git a/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala b/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala
index 56aae5824..e95d6988d 100644
--- a/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala
+++ b/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala
@@ -1,1380 +1,1380 @@
package net.shrine.steward
import net.shrine.authorization.steward._
import net.shrine.i2b2.protocol.pm.User
import net.shrine.protocol.Credential
import net.shrine.steward.db.{QueryParameters, StewardDatabase, UserRecord}
import org.json4s.native.JsonMethods.parse
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{BeforeAndAfterEach, FlatSpec, Suite}
import spray.http.BasicHttpCredentials
import spray.routing.MalformedRequestContentRejection
import spray.testkit.ScalatestRouteTest
import spray.http.StatusCodes._
import scala.xml.{Elem, NodeSeq}
@RunWith(classOf[JUnitRunner])
class StewardServiceTest extends FlatSpec with ScalatestRouteTest with TestWithDatabase with StewardService {
def actorRefFactory = system
import scala.concurrent.duration._
implicit val routeTestTimeout = RouteTestTimeout(10 seconds)
val researcherUserName = "ben"
val researcherFullName = researcherUserName
val stewardUserName = "dave"
val stewardFullName = stewardUserName
/**
* to run these tests with I2B2
* add a user named qep, to be the qep
* add a Boolean parameter for qep, qep, true
* add a user named ben, to be a researcher
* add a user named dave, to be the data steward
* add a Boolean parameter for dave, DataSteward, true
* add all three users to the i2b2 project
*/
val stewardCredentials = BasicHttpCredentials(stewardUserName,"kablam")
val researcherCredentials = BasicHttpCredentials(researcherUserName,"kapow")
val qepCredentials = BasicHttpCredentials("qep","trustme")
val badCredentials = BasicHttpCredentials("qep","wrongPassword")
val researcherUser = User(
- fullName = researcherUserName,
- username = researcherFullName,
+ fullName = researcherFullName,
+ username = researcherUserName,
domain = "domain",
credential = new Credential("ben's password",false),
params = Map(),
rolesByProject = Map()
)
val stewardUser = User(
- fullName = stewardUserName,
- username = stewardFullName,
+ fullName = stewardFullName,
+ username = stewardUserName,
domain = "domain",
credential = new Credential("dave's password",false),
params = Map(stewardRole -> "true"),
rolesByProject = Map()
)
val researcherOutboundUser = OutboundUser.createFromUser(researcherUser)
val stewardOutboundUser = OutboundUser.createFromUser(stewardUser)
val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",researcherOutboundUser,0L,TopicState.pending.name,researcherOutboundUser,0L)
val forbiddenTopicId = 0
private val queryContent: QueryContents = "18-34 years old@18:31:51\\\\SHRINE\\SHRINE\\Demographics\\Age\\18-34 years old\\"
val diffs = List.empty
val getItBack = scala.xml.XML.loadString(queryContent).toString
"StewardService" should "return an OK and the correct createTopicsMode name" in {
StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.Pending.name,s"${CreateTopicsMode.Pending.name} test") {
{
Get(s"/about/createTopicsMode") ~>
route ~> check {
assertResult(OK)(status)
val createTopicsModeName = new String(body.data.toByteArray)
assertResult(s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName)
assertResult(s""""${CreateTopicsMode.Pending.name}"""")(createTopicsModeName)
}
}
}
StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.Approved.name,s"${CreateTopicsMode.Approved.name} test") {
Get(s"/about/createTopicsMode") ~>
route ~> check {
assertResult(OK)(status)
val createTopicsModeName = new String(body.data.toByteArray)
assertResult( s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName)
assertResult( s""""${CreateTopicsMode.Approved.name}"""")(createTopicsModeName)
}
}
StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") {
Get(s"/about/createTopicsMode") ~>
route ~> check {
assertResult(OK)(status)
val createTopicsModeName = new String(body.data.toByteArray)
assertResult( s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName)
assertResult( s""""${CreateTopicsMode.TopicsIgnoredJustLog.name}"""")(createTopicsModeName)
}
}
}
"StewardService" should "return an OK and a valid outbound user for a user/whoami request" in {
Get(s"/user/whoami") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(OK)(status)
val userJson = new String(body.data.toByteArray)
val outboundUser = parse(userJson).extract[OutboundUser]
assertResult(researcherOutboundUser)(outboundUser)
}
}
"StewardService" should "return a 200 for a user/whoami request with bad credentials, with a body of 'AuthenticationFailed'" in {
// "StewardService" should "return an TemporaryRedirect for a user/whoami request with bad credentials" in {
Get(s"/user/whoami") ~>
addCredentials(badCredentials) ~>
sealRoute(route) ~> check {
assertResult(OK)(status)
assertResult(""""AuthenticationFailed"""")(new String(body.data.toByteArray))
}
}
/* todo
"StewardService" should "return a 200 for a user/whoami request with bad credentials, with a body of 'AuthenticationFailed' even wiht a back-tick" in {
// "StewardService" should "return an TemporaryRedirect for a user/whoami request with bad credentials" in {
val badCredentials = BasicHttpCredentials("o`brien","wrongPassword")
Get(s"/user/whoami") ~>
addCredentials(badCredentials) ~>
sealRoute(route) ~> check {
assertResult(OK)(status)
assertResult(""""AuthenticationFailed"""")(new String(body.data.toByteArray))
}
}
*/
"StewardService" should "return an OK for an approved request" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
val topicInDb = StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
val dbFound = StewardDatabase.db.changeTopicState(uncontroversialTopic.id,TopicState.approved,stewardUserName)
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(1,"test query","crazy syntax")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(OK)(status)
}
}
"StewardService" should "complain about bad http credentials from the QEP" in {
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(2,"test query","too bad about your password")) ~>
addCredentials(badCredentials) ~>
sealRoute(route) ~> check {
assertResult(Unauthorized)(status)
}
}
"StewardService" should "complain about unexpected json" in {
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}","""{"field":"not in ShrineQuery"}""") ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(false)(handled)
assertResult(true)(rejection.isInstanceOf[MalformedRequestContentRejection])
}
}
"StewardService" should "return a rejection for an unacceptable request" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.rejected,stewardUserName)
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(3,"test query","too bad about your topic")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(UnavailableForLegalReasons)(status)
}
}
"StewardService" should "return a rejection for a pending topic" in {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(4,"test query","topic still pending")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(UnavailableForLegalReasons)(status)
}
}
"StewardService" should "return an UnprocessableEntity for an unknown topic" in {
Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${forbiddenTopicId}",InboundShrineQuery(5,"test query","no one knows about your topic")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(UnprocessableEntity)(status)
}
}
"StewardService" should " return an UnprocessableEntity for query requests with no topic" in {
Post(s"/qep/requestQueryAccess/user/${researcherUserName}",InboundShrineQuery(5,"test query","Not even using a topic")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(UnprocessableEntity)(status)
}
}
"StewardService" should " accept query requests with no topic in 'just log and approve everything' mode " in {
StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") {
Post(s"/qep/requestQueryAccess/user/${researcherUserName}", InboundShrineQuery(5, "test query", "Not even using a topic")) ~>
addCredentials(qepCredentials) ~>
route ~> check {
assertResult(OK)(status)
}
}
}
"StewardService" should "return approved topics for the qep" in {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/qep/approvedTopics/user/${researcherUserName}") ~>
addCredentials(qepCredentials) ~>
route ~> check {
status === OK
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics) === true
}
}
"StewardService" should "return the list of a researcher's Topics as Json" in {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/researcher/topics") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics))
}
}
"StewardService" should "return the list of a researcher's Topics as Json for various states" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/researcher/topics?state=Pending") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics))
assertResult(OK)(status)
}
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
Get(s"/researcher/topics?state=Approved") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(
OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.approved.name,stewardOutboundUser,0))).sameExceptForTimes(topics))
assertResult(OK)(status)
}
StewardDatabase.db.changeTopicState(1,TopicState.rejected,stewardUserName)
Get(s"/researcher/topics?state=Rejected") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(
OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.rejected.name,stewardOutboundUser,0))).sameExceptForTimes(topics))
assertResult(OK)(status)
}
}
"DataStewardService" should "reject nonsense topic request states" in {
Get(s"/researcher/topics?state=nonsense") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(UnprocessableEntity)(status)
}
}
"StewardService" should "return the list of a researcher's Topics as Json with skip and limit set" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createFiveTopics()
Get(s"/researcher/topics?skip=0&limit=2") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(2)(topics.topics.size)
assertResult(OK)(status)
}
Get(s"/researcher/topics?skip=2&limit=2") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(2)(topics.topics.size)
assertResult(OK)(status)
}
Get(s"/researcher/topics?skip=4&limit=2") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(1)(topics.topics.size)
assertResult(OK)(status)
}
}
"StewardService" should "return the list of a researcher's Topics as Json with different sorting and ordering options" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createFiveTopics()
Get(s"/researcher/topics?sortBy=id") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(5)(topics.topics.size)
assertResult(OK)(status)
assertResult(topics.topics.sortBy(_.id))(topics.topics)
}
Get(s"/researcher/topics?sortBy=name") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(5)(topics.topics.size)
assertResult(OK)(status)
assertResult(topics.topics.sortBy(_.createdBy.userName))(topics.topics)
}
Get(s"/researcher/topics?sortBy=name&sortDirection=descending") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(5)(topics.topics.size)
assertResult(OK)(status)
assertResult(topics.topics.sortBy(_.name).reverse)(topics.topics)
}
Get(s"/researcher/topics?sortBy=name&sortDirection=ascending") ~>
addCredentials(researcherCredentials) ~>
route ~> check {
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[ResearchersTopics]
assertResult(5)(topics.topics.size)
assertResult(OK)(status)
assertResult(topics.topics.sortBy(_.createdBy.userName))(topics.topics)
}
}
"StewardService" should "return the list of a researcher's query history as Json" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
Get(s"/researcher/queryHistory?asJson=false") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
val history: QueryHistory = QueryHistory(1, 0, List(
OutboundShrineQuery(1, 0, "test query", researcherOutboundUser, Some(
OutboundTopic(1, uncontroversialTopic.name, uncontroversialTopic.description, researcherOutboundUser, 0, TopicState.approved.name, stewardOutboundUser, 0)
), getItBack, TopicState.approved.name, 0)
))
assertResult(diffs)(history.differencesExceptTimes(queries))
}
}
"StewardService" should "return the list of a researcher's query history as Json, with query bodies as Json" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
Get(s"/researcher/queryHistory?asJson=true") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesString = new String(body.data.toByteArray)
val queriesJson = parse(queriesString).extract[QueryHistoryWithJson]
assertResult(diffs)(QueryHistory(1,0,List(
OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(
OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.approved.name,stewardOutboundUser,0)
),getItBack,TopicState.approved.name,0)
)).differencesExceptTimes(queriesJson.convertToXml))
}
}
"StewardService" should "return the list of a researcher's query history as Json, filtered by state" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("Forbidden topic","No way is the data steward going for this"))
StewardDatabase.db.changeTopicState(2,TopicState.rejected,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(1,"forbidden query",queryContent))
Get(s"/researcher/queryHistory?state=Approved") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(diffs)(
QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(OutboundTopic(
1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,
TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0)))
.differencesExceptTimes(queries))
}
}
"StewardService" should "return the list of a researcher's query history as Json, with skip and limit parameters" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/researcher/queryHistory?") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?skip=0&limit=5") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(5)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?skip=1&limit=4") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(4)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?skip=4&limit=5") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(2)(queries.queryRecords.size)
}
}
"StewardService" should "return the list of a researcher's query history as Json, using parameters for sorting" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/researcher/queryHistory?sortBy=externalId") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords)
}
Get(s"/researcher/queryHistory?sortBy=externalId&sortDirection=ascending") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords)
}
Get(s"/researcher/queryHistory?sortBy=externalId&sortDirection=descending") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.externalId).reverse)(queries.queryRecords)
}
Get(s"/researcher/queryHistory?sortBy=stewardResponse") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.stewardResponse))(queries.queryRecords)
}
Get(s"/researcher/queryHistory?sortBy=name") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.name))(queries.queryRecords)
}
}
"StewardService" should "return the list of a researcher's query history, using parameters for sorting, skip, and limit" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
val sixQueries:Seq[OutboundShrineQuery] = {
Get(s"/researcher/queryHistory") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries:QueryHistory = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords)
queries.queryRecords
}
}
Get(s"/researcher/queryHistory?skip=0&limit=3&sortBy=name") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
assertResult(queries.queryRecords)(sixQueries.sortBy(_.name).take(3))
}
Get(s"/researcher/queryHistory?skip=2&limit=3&sortBy=name") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
assertResult(queries.queryRecords)(sixQueries.sortBy(_.name).slice(2, 5))
}
}
"StewardService" should "return the list of a researcher's query history as Json, with filtered by topic" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/researcher/queryHistory") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory/topic/1") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory/topic/1?skip=1&limit=2") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(2)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory/topic/2") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
}
}
"StewardService" should "return the list of a researcher's query history as Json, filtered by date" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
val startTime = System.currentTimeMillis()
Thread.sleep(10)
createSixQueries()
Thread.sleep(10)
val finishTime = System.currentTimeMillis()
Get(s"/researcher/queryHistory") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?minDate=$startTime&maxDate=$finishTime") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?minDate=$startTime&maxDate=$startTime") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(0)(queries.queryRecords.size)
}
Get(s"/researcher/queryHistory?minDate=$finishTime&maxDate=$finishTime") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(0)(queries.queryRecords.size)
}
}
"StewardService" should "return the list of a researcher's query history as Json, even including an unknown topic id" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query for unknown topic",queryContent))
Get(s"/researcher/queryHistory") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(diffs)(QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query for unknown topic",researcherOutboundUser,None,getItBack,TopicState.unknownForUser.name,0))).differencesExceptTimes(queries))
}
}
"StewardService" should "return the list of a researcher's query history as Json, sorted by topic name" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/researcher/queryHistory?sortBy=topicName") ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
// assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords)
}
}
"StewardService" should "return counts of topics, total and by state" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createFiveTopics()
Get(s"/steward/statistics/topicsPerState") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val statisticsJson = new String(body.data.toByteArray)
val statistics = parse(statisticsJson).extract[TopicsPerState]
assertResult(5)(statistics.total)
assertResult(TopicsPerState(5,Seq((TopicState.pending.name,5))))(statistics)
}
}
"StewardService" should "injest a researcher's request to study a topic" in {
Post(s"/researcher/requestTopicAccess",InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(Accepted)(status)
assertResult(1)(StewardDatabase.db.selectUsers.size)
assertResult(UserRecord(researcherUserName,researcherUserName,false))(StewardDatabase.db.selectUsers.head)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(uncontroversialTopic.description)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(TopicState.pending)(topic.state)
}
}
"StewardService" should "injest a second request from a researcher request to study a new topic" in {
val secondTopicName = "Liver"
val secondTopicDescription = "Liver Study"
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Post(s"/researcher/requestTopicAccess",InboundTopicRequest(secondTopicName,secondTopicDescription)) ~>
addCredentials(researcherCredentials) ~> route ~> check {
assertResult(Accepted)(status)
assertResult(1)(StewardDatabase.db.selectUsers.size)
assertResult(UserRecord(researcherUserName,researcherUserName,false))(StewardDatabase.db.selectUsers.head)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(2)(topics.size)
val firstTopic = topics(0)
assertResult(uncontroversialTopic.name)(firstTopic.name)
assertResult(uncontroversialTopic.description)(firstTopic.description)
assertResult(researcherUserName)(firstTopic.createdBy)
assertResult(TopicState.pending)(firstTopic.state)
val secondTopic = topics(1)
assertResult(secondTopicName)(secondTopic.name)
assertResult(secondTopicDescription)(secondTopic.description)
assertResult(researcherUserName)(secondTopic.createdBy)
assertResult(TopicState.pending)(secondTopic.state)
}
}
"StewardService" should " place new topics in the Approved state in auto-approve mode" in {
StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") {
Post(s"/researcher/requestTopicAccess",InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(Accepted)(status)
assertResult(1)(StewardDatabase.db.selectUsers.size)
assertResult(UserRecord(researcherUserName, researcherUserName, false))(StewardDatabase.db.selectUsers.head)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(uncontroversialTopic.description)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(TopicState.approved)(topic.state)
}
}
}
"StewardService" should "injest a researcher's request to edit a topic" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
val updatedDescription = "Really you should accept this"
Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(Accepted)(status)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(updatedDescription)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(researcherUserName)(topic.changedBy)
assertResult(TopicState.pending)(topic.state)
}
}
"StewardService" should "reject a researcher's request to edit a topic that does not exist" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
val updatedDescription = "Really you should accept this"
Post(s"/researcher/editTopicRequest/2",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(NotFound)(status)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(uncontroversialTopic.description)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(researcherUserName)(topic.changedBy)
assertResult(TopicState.pending)(topic.state)
}
}
"StewardService" should "reject a researcher's request to edit an approved topic" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
val updatedDescription = "Really you should accept this"
Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~>
addCredentials(researcherCredentials) ~>
route ~> check {
assertResult(Forbidden)(status)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(uncontroversialTopic.description)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(stewardUserName)(topic.changedBy)
assertResult(TopicState.approved)(topic.state)
}
}
"StewardService" should "reject an attempt to edit a topic owned by a different user" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
val updatedDescription = "Really you should accept this"
Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(Forbidden)(status)
val topics = StewardDatabase.db.selectTopics(QueryParameters())
assertResult(1)(topics.size)
val topic = topics.head
assertResult(uncontroversialTopic.name)(topic.name)
assertResult(uncontroversialTopic.description)(topic.description)
assertResult(researcherUserName)(topic.createdBy)
assertResult(TopicState.pending)(topic.state)
}
}
"StewardService" should "return the full history of queries as Json" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
Get(s"/steward/queryHistory") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(diffs) (
QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(
OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,
TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0)))
.differencesExceptTimes(queries))
}
}
"StewardService" should "return the history of queries for a specific user as Json" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
Get(s"/steward/queryHistory/user/${researcherUserName}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(diffs)(
QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(OutboundTopic(
1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,
TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0)))
.differencesExceptTimes(queries))
}
}
"StewardService" should "return the query history as Json, filtered by topic" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/steward/queryHistory") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/steward/queryHistory/user/ben") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(6)(queries.queryRecords.size)
}
Get(s"/steward/queryHistory/topic/1") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
}
Get(s"/steward/queryHistory/user/ben/topic/1") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
}
Get(s"/steward/queryHistory/topic/1?skip=1&limit=2") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(2)(queries.queryRecords.size)
}
Get(s"/steward/queryHistory/topic/2") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val queriesJson = new String(body.data.toByteArray)
val queries = parse(queriesJson).extract[QueryHistory]
assertResult(3)(queries.queryRecords.size)
}
}
"StewardService" should "return counts of queries, total and by user" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createSixQueries()
Get(s"/steward/statistics/queriesPerUser") ~>
addCredentials(stewardCredentials) ~> route ~> check {
assertResult(OK)(status)
val statisticsJson = new String(body.data.toByteArray)
val statistics = parse(statisticsJson).extract[QueriesPerUser]
assertResult(6)(statistics.total)
assertResult(QueriesPerUser(6,Seq((researcherOutboundUser,6))))(statistics)
}
}
"StewardService" should "return the topics for a specific user as Json" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/steward/topics/user/${researcherUserName}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics]
assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics))
}
}
"StewardService" should "return the topics for a specific user as Json, given skip and limit parameters" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
createFiveTopics()
Get(s"/steward/topics/user/${researcherUserName}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics]
assertResult(5)(topics.topics.size)
}
Get(s"/steward/topics/user/${researcherUserName}?skip=0&limit=3") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics]
assertResult(3)(topics.topics.size)
}
Get(s"/steward/topics/user/${researcherUserName}?skip=2&limit=3") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics]
assertResult(3)(topics.topics.size)
}
Get(s"/steward/topics/user/${researcherUserName}?skip=3&limit=4") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics]
assertResult(2)(topics.topics.size)
}
}
"StewardService" should "return all of the topics" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/steward/topics") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[StewardsTopics]
assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics))
}
}
"StewardService" should "return all of the pending topics" in {
StewardDatabase.db.upsertUser(researcherUser)
StewardDatabase.db.upsertUser(stewardUser)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Get(s"/steward/topics?state=Pending") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
val topicsJson = new String(body.data.toByteArray)
val topics = parse(topicsJson).extract[StewardsTopics]
assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics))
}
}
"StewardService" should "approve a researcher's request to study a topic" in {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Post(s"/steward/approveTopic/topic/${uncontroversialTopic.id}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
}
}
"StewardService" should "reject a researcher's request to study a topic" in {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
Post(s"/steward/rejectTopic/topic/${uncontroversialTopic.id}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(OK)(status)
}
}
"A steward's attempt to approve or reject a topic that doesn't exist" should "report an error" in {
Post(s"/steward/approveTopic/topic/${uncontroversialTopic.id}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(UnprocessableEntity)(status)
}
Post(s"/steward/rejectTopic/topic/${uncontroversialTopic.id}") ~>
addCredentials(stewardCredentials) ~>
route ~> check {
assertResult(UnprocessableEntity)(status)
}
}
"StewardService" should "redirect several urls to client/index.html" in {
Get() ~>
route ~> check {
status === PermanentRedirect
header("Location") === "client/index.html"
}
Get("/") ~>
route ~> check {
status === PermanentRedirect
header("Location") === "client/index.html"
}
Get("/index.html") ~>
route ~> check {
status === PermanentRedirect
header("Location") === "client/index.html"
}
Get("/client") ~>
route ~> check {
status === PermanentRedirect
header("Location") === "client/index.html"
}
Get("/client/") ~>
route ~> check {
status === PermanentRedirect
header("Location") === "client/index.html"
}
}
/*
//CORS won't be turned on in the real server, so this test won't normally pass.
"DataStewardService" should "support a CORS OPTIONS request" in {
Options(s"/steward/topics") ~>
//No credentials for Options, so no addCredentials(testCredentials) ~>
route ~> check {
assertResult(OK)(status)
}
}
*/
/* For some reason, the test engine isn't finding the static resources. I suspect it is testing with raw .class files ,
not an actual .war or .jar. Works on the actual server.
"StewardService" should "serve up static files from resources/client" in {
Get("/steward/client/test.txt") ~> route ~> check {
assertResult(OK)(status)
assertResult("Test file")(body.asString)
}
}
*/
def createFiveTopics(): Unit ={
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("slightlyControversial","who cares?"))
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("controversial","controversial"))
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("moderatelyControversial","more controversial than that"))
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("veryControversial","Just say no"))
}
def createSixQueries() = {
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("Forbidden topic","No way is the data steward going for this"))
StewardDatabase.db.changeTopicState(2,TopicState.rejected,stewardUserName)
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"A test query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(1,"B forbidden query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2," C test query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(3,"4 forbidden query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(4,"7 test query",queryContent))
StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(5,"% forbidden query",queryContent))
}
}
trait TestWithDatabase extends BeforeAndAfterEach { this:Suite =>
override def beforeEach() = {
StewardDatabase.db.createTables()
}
override def afterEach() = {
StewardDatabase.db.nextTopicId.set(1)
StewardDatabase.db.dropTables()
}
}
/*
object OutboundTopic {
val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",OutboundUser.someOutboundResearcher,0L,TopicState.pending.name,OutboundUser.someOutboundResearcher,0L)
val forbiddenTopicId = 0
}
object OutboundShrineQuery extends ((
StewardQueryId,
ExternalQueryId,
String,
OutboundUser,
Option[OutboundTopic],
QueryContents,
TopicStateName,
Date) => OutboundShrineQuery) {
val notSureAboutQueryContents:QueryContents = "Appropriate query contents"
val someQueryRecord = OutboundShrineQuery(-5,-2,"Kidney Query",OutboundUser.someOutboundResearcher,Some(uncontroversialTopic),OutboundShrineQuery.notSureAboutQueryContents,TopicState.approved.name,System.currentTimeMillis())
}
*/
\ No newline at end of file
diff --git a/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala b/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala
new file mode 100644
index 000000000..3e7e99909
--- /dev/null
+++ b/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala
@@ -0,0 +1,238 @@
+package net.shrine.steward.db
+
+import net.shrine.authorization.steward.{InboundShrineQuery, InboundTopicRequest, OutboundTopic, OutboundUser, QueryContents, ResearcherToAudit, TopicState}
+import net.shrine.i2b2.protocol.pm.User
+import net.shrine.protocol.Credential
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{BeforeAndAfterEach, FlatSpec, Suite}
+
+import scala.concurrent.duration._
+
+/**
+ * Tests of data steward db functions not excercised via the StewardService
+ *
+ * @author david
+ * @since 1.22
+ */
+
+@RunWith(classOf[JUnitRunner])
+class StewardServiceTest extends FlatSpec with TestWithDatabase {
+
+ val researcherUserName = "ben"
+ val researcherFullName = "Ben here before"
+
+ val researcherUser = User(
+ fullName = researcherFullName,
+ username = researcherUserName,
+ domain = "domain",
+ credential = new Credential("ben's password",false),
+ params = Map(),
+ rolesByProject = Map()
+ )
+
+ val researcherOutboundUser = OutboundUser.createFromUser(researcherUser)
+
+ "The database" should "record an audit request" in {
+
+ val now = System.currentTimeMillis()
+ val expectedAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now))
+ val expectedAuditRecords = expectedAuditRequests.map(x => UserAuditRecord(x.researcher.userName,x.count,now))
+
+ StewardDatabase.db.logAuditRequests(expectedAuditRequests,now)
+ val actual: Seq[UserAuditRecord] = StewardDatabase.db.selectAllAuditRequests
+
+ assertResult(expectedAuditRecords)(actual)
+ }
+
+ "The database" should "supply the most recent audit request" in {
+
+ val now = System.currentTimeMillis()
+ val firstAuditRecords: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-5000,now-3000))
+ val secondAuditRecords: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now))
+
+ val expectedAuditRecords = secondAuditRecords.map(x => UserAuditRecord(x.researcher.userName,x.count,now))
+
+ StewardDatabase.db.logAuditRequests(firstAuditRecords,now)
+ val actual: Seq[UserAuditRecord] = StewardDatabase.db.selectMostRecentAuditRequests
+
+ assertResult(expectedAuditRecords)(actual)
+ }
+
+ val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",researcherOutboundUser,0L,TopicState.pending.name,researcherOutboundUser,0L)
+ val stewardUserName = "dave"
+ val queryContent: QueryContents = "18-34 years old@18:31:51\\\\SHRINE\\SHRINE\\Demographics\\Age\\18-34 years old\\"
+
+ "The database" should "recommend an audit for a user who has never been audited and has an unaudited query far enough in the past" in {
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
+ val now = System.currentTimeMillis()
+
+
+ Thread.sleep(20)
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,10 milliseconds)
+
+ val expected = Seq(ResearcherToAudit(researcherOutboundUser,1,now - 20,now))
+
+ assertResult(expected.size)(audits.size)
+ assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2)))
+ }
+
+ "The database" should "not recommend an audit for a user who has never been audited and has an unaudited query not far enough in the past" in {
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
+ val now = System.currentTimeMillis()
+
+
+ Thread.sleep(20)
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days)
+
+ assertResult(Seq.empty)(audits)
+ }
+
+ "The database" should "recommend an audit for a user who has been audited and has an unaudited query far enough in the past" in {
+
+ val now = System.currentTimeMillis()
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000))
+ StewardDatabase.db.logAuditRequests(oldAuditRequests,now)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
+
+ Thread.sleep(20)
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,10 milliseconds)
+
+ val expected = Seq(ResearcherToAudit(researcherOutboundUser,1,now - 20,now))
+
+ assertResult(expected.size)(audits.size)
+ assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2)))
+ }
+
+ "The database" should "not recommend an audit for a user who has been audited and has an unaudited query not far enough in the past" in {
+
+ val now = System.currentTimeMillis()
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000))
+ StewardDatabase.db.logAuditRequests(oldAuditRequests,now)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent))
+
+ Thread.sleep(20)
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days)
+
+ assertResult(Seq.empty)(audits)
+ }
+
+ "The database" should "recommend an audit for a user who has never been audited and has run many queries" in {
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent))
+ val now = System.currentTimeMillis()
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(3,30 days)
+
+ val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now))
+
+ assertResult(expected.size)(audits.size)
+ assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2)))
+ }
+
+ "The database" should "not recommend an audit for a user who has never been audited and has not run many queries" in {
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent))
+ val now = System.currentTimeMillis()
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days)
+
+ assertResult(Seq.empty)(audits)
+ }
+
+ "The database" should "recommend an audit for a user who has been audited and has run many queries" in {
+
+ val now = System.currentTimeMillis()
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000))
+ StewardDatabase.db.logAuditRequests(oldAuditRequests,now)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent))
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(3,30 days)
+
+ val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now))
+
+ assertResult(expected.size)(audits.size)
+ assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2)))
+ }
+
+ "The database" should "not recommend an audit for a user who has been audited and has not run many queries" in {
+
+ val now = System.currentTimeMillis()
+
+ StewardDatabase.db.upsertUser(researcherUser)
+
+ val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000))
+ StewardDatabase.db.logAuditRequests(oldAuditRequests,now)
+
+ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description))
+ StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName)
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent))
+ StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent))
+
+ val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days)
+
+ val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now))
+
+ assertResult(Seq.empty)(audits)
+ }
+}
+
+trait TestWithDatabase extends BeforeAndAfterEach { this:Suite =>
+
+ override def beforeEach() = {
+ StewardDatabase.db.createTables()
+ }
+
+ override def afterEach() = {
+ StewardDatabase.db.nextTopicId.set(1)
+ StewardDatabase.db.dropTables()
+ }
+
+}
diff --git a/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala b/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala
index ae273766d..5ff90092d 100644
--- a/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala
+++ b/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala
@@ -1,256 +1,264 @@
package net.shrine.authorization.steward
import java.lang.reflect.Field
import net.shrine.i2b2.protocol.pm.User
import net.shrine.log.Loggable
import net.shrine.protocol.Credential
import net.shrine.serialization.NodeSeqSerializer
import org.json4s.{DefaultFormats, Formats}
import spray.http.{StatusCode, StatusCodes}
import spray.httpx.Json4sSupport
import scala.util.{Failure, Try}
import scala.xml.NodeSeq
/**
* Data model for the data steward.
*
* @author dwalend
* @since 1.19
*/
//http response json
case class OutboundTopic(id:TopicId,
name:String,
description:String,
createdBy:OutboundUser,
createDate:Date,
state:TopicStateName,
changedBy:OutboundUser,
changeDate:Date
) {
def differences(other:OutboundTopic):Seq[(String,Any,Any)] = {
if (this == other) List()
else {
val fields = getClass.getDeclaredFields
val names = fields.map(_.getName)
def getFromField(field:Field,thing:OutboundTopic):Any = {
field.setAccessible(true)
field.get(thing)
}
val thisUnapplied = fields.map(getFromField(_,this))
val otherUnapplied = fields.map(getFromField(_,other))
val tuples = names.zip(thisUnapplied.zip(otherUnapplied))
def difference(name:String,one:Any,other:Any):Option[(String,Any,Any)] = {
if(one == other) None
else {
Some((name,one,other))
}
}
tuples.map(x => difference(x._1,x._2._1,x._2._2)).to[Seq].flatten
}
}
def differencesExceptTimes(other:OutboundTopic):Seq[(String,Any,Any)] = {
differences(other).filterNot(x => x._1 == "createDate").filterNot(x => x._1 == "changeDate")
}
}
case class OutboundUser(userName:UserName,fullName:String,roles:Set[Role] = Set(researcherRole)) {}
object OutboundUser extends Loggable {
def createResearcher(userName:UserName,fullName:String) = OutboundUser(userName,fullName)
def createSteward(userName:UserName,fullName:String) = OutboundUser(userName,fullName,Set(researcherRole,stewardRole))
def createFromUser(user:User) = if( user.params.toList.contains((stewardRole,"true"))) createSteward(user.username,user.fullName)
else createResearcher(user.username,user.fullName)
/** If the user is unknown but has a user id, best effort is to guess this user is a researcher with userName as their full name*/
def createUnknownUser(userName:UserName) = {
info(s"Creating an OutboundUser for unknown userName $userName")
createResearcher(userName,userName)
}
}
sealed case class TopicState(name:TopicStateName,statusCode:StatusCode,message:String)
//todo split out TopicResponse from TopicState
object TopicState {
val pending = TopicState("Pending",StatusCodes.UnavailableForLegalReasons,"Topic pending data steward's approval")
val approved = TopicState("Approved",StatusCodes.OK,"OK")
val rejected = TopicState("Rejected",StatusCodes.UnavailableForLegalReasons,"Topic rejected by data steward")
val unknownForUser = TopicState("Unknown For User",StatusCodes.UnprocessableEntity,"Topic unknown for user")
val createTopicsModeRequiresTopic = TopicState("Required But Not Supplied",StatusCodes.UnprocessableEntity,"Topic required but not supplied")
val namesToStates = Seq(pending,approved,rejected,unknownForUser,createTopicsModeRequiresTopic).map(x => x.name -> x).toMap
def stateForStringOption(stringOption: Option[TopicStateName]): Try[Option[TopicState]] = {
val notPresent: Try[Option[TopicState]] = Try(None)
stringOption.fold(notPresent)((string: TopicStateName) => {
val notInTheMap: Try[Option[TopicState]] = Failure(new IllegalArgumentException(s"No value for $string in $namesToStates"))
namesToStates.get(string).fold(notInTheMap)(x => Try(Some(x)))
})
}
}
case class OutboundShrineQueryWithJson(
stewardId:StewardQueryId,
externalId:ExternalQueryId,
name:String,
user:OutboundUser,
topic:Option[OutboundTopic],
queryContents: NodeSeq,
stewardResponse:TopicStateName,
date:Date)
extends Json4sSupport
{
override implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer
def convertToXml:OutboundShrineQuery = {
OutboundShrineQuery(stewardId, externalId, name, user, topic, queryContents.toString, stewardResponse, date)
}
}
case class OutboundShrineQuery(
stewardId:StewardQueryId,
externalId:ExternalQueryId,
name:String,
user:OutboundUser,
topic:Option[OutboundTopic],
queryContents: QueryContents,
stewardResponse:TopicStateName,
date:Date) {
def differences(other:OutboundShrineQuery):Seq[(String,Any,Any)] = {
if (this == other) List()
else {
val fields = getClass.getDeclaredFields
val names = fields.map(_.getName)
def getFromField(field: Field, thing: OutboundShrineQuery): Any = {
field.setAccessible(true)
field.get(thing)
}
val thisUnapplied = fields.map(getFromField(_, this))
val otherUnapplied = fields.map(getFromField(_, other))
val tuples = names.zip(thisUnapplied.zip(otherUnapplied))
def difference(name: String, one: Any, other: Any): Option[(String, Any, Any)] = {
// TODO: Remove this horrible string equality hack.
if (one == other)
None
else {
Some((name, one, other))
}
}
tuples.map(x => difference(x._1, x._2._1, x._2._2)).to[Seq].flatten
}
}
def convertToJson:OutboundShrineQueryWithJson = {
OutboundShrineQueryWithJson(stewardId, externalId, name, user, topic, scala.xml.XML.loadString(queryContents), stewardResponse, date)
}
def differencesExceptTimes(other:OutboundShrineQuery):Seq[(String,Any,Any)] = {
val diffWithoutTimes = differences(other).filterNot(x => x._1 == "date").filterNot(x => x._1 == "topic")
val topicDiffs:Seq[(String,Any,Any)] = (topic,other.topic) match {
case (None,None) => Seq.empty[(String,Any,Any)]
case (Some(thisTopic),Some(otherTopic)) => thisTopic.differencesExceptTimes(otherTopic)
case _ => Seq(("topic",this.topic,other.topic))
}
diffWithoutTimes ++ topicDiffs
}
}
case class QueriesPerUser(total:Int,queriesPerUser:Seq[(OutboundUser,Int)]) //todo rename QueriesPerResearcher
case class TopicsPerState(total:Int,topicsPerState:Seq[(TopicStateName,Int)])
case class ResearchersTopics(userId:UserName,totalCount:Int,skipped:Int,topics:Seq[OutboundTopic]) {
def sameExceptForTimes(researchersTopics: ResearchersTopics):Boolean = {
(totalCount == researchersTopics.totalCount) &&
(skipped == researchersTopics.skipped) &&
(userId == researchersTopics.userId) &&
(topics.size == researchersTopics.topics.size) &&
topics.zip(researchersTopics.topics).forall(x => x._1.id == x._2.id)
}
}
case class StewardsTopics(totalCount:Int,skipped:Int,topics:Seq[OutboundTopic]) {
def sameExceptForTimes(stewardsTopics: StewardsTopics):Boolean = {
(totalCount == stewardsTopics.totalCount) &&
(skipped == stewardsTopics.skipped) &&
(topics.size == stewardsTopics.topics.size) &&
topics.zip(stewardsTopics.topics).forall(x => x._1.id == x._2.id)
}
}
case class QueryHistory(totalCount:Int,skipped:Int,queryRecords:Seq[OutboundShrineQuery]) {
def sameExceptForTimes(queryResponse: QueryHistory):Boolean = {
(totalCount == queryResponse.totalCount) &&
(skipped == queryResponse.skipped) &&
(queryRecords.size == queryResponse.queryRecords.size) &&
queryRecords.zip(queryResponse.queryRecords).forall(x => x._1.differencesExceptTimes(x._2).isEmpty)
}
def differences(other:QueryHistory):Seq[(String,Any,Any)] = {
if (this == other) List()
else {
val fields = getClass.getDeclaredFields
val names = fields.map(_.getName)
def getFromField(field:Field,thing:QueryHistory):Any = {
field.setAccessible(true)
field.get(thing)
}
val thisUnapplied = fields.map(getFromField(_,this))
val otherUnapplied = fields.map(getFromField(_,other))
val tuples = names.zip(thisUnapplied.zip(otherUnapplied))
def difference(name:String,one:Any,other:Any):Option[(String,Any,Any)] = {
if(one == other) None
else {
Some((name,one,other))
}
}
tuples.map(x => difference(x._1,x._2._1,x._2._2)).to[Seq].flatten
}
}
def differencesExceptTimes(other:QueryHistory):Seq[(String,Any,Any)] = {
val normalDiffs:Seq[(String,Any,Any)] = differences(other).filterNot(x => x._1 == "queryRecords")
val timeDiffs:Seq[(String,Any,Any)] = queryRecords.zip(other.queryRecords).flatMap(x => x._1.differencesExceptTimes(x._2))
normalDiffs ++ timeDiffs
}
def convertToJson = QueryHistoryWithJson(totalCount, skipped, queryRecords.map(_.convertToJson))
}
case class QueryHistoryWithJson(totalCount:Int,skipped:Int,queryRecords:Seq[OutboundShrineQueryWithJson]) extends Json4sSupport {
implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer
def convertToXml = QueryHistory(totalCount, skipped, queryRecords.map(_.convertToXml))
}
case class TopicIdAndName(id:String,name:String)
+case class ResearcherToAudit(researcher:OutboundUser, count:Int, leastRecentQueryDate:Date, currentAuditDate:Date) {
+ def sameExceptForTimes(audit: ResearcherToAudit): Boolean = {
+ (researcher == audit.researcher) &&
+ (count == audit.count)
+ }
+}
+
+
//http request Json
case class InboundShrineQuery(
externalId:ExternalQueryId,
name:String,
queryContents: QueryContents)
case class InboundTopicRequest(name:String,description:String)
\ No newline at end of file
diff --git a/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala b/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala
index 53f20e83a..6ef6284ff 100644
--- a/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala
+++ b/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala
@@ -1,12 +1,14 @@
package net.shrine.crypto
import net.shrine.protocol.BroadcastMessage
import scala.concurrent.duration.Duration
/**
* @author clint
* @date Nov 27, 2013
*/
+
+//todo delete this interface . Only one thing implements it.
trait Verifier {
def verifySig(message: BroadcastMessage, maxSignatureAge: Duration): Boolean
}
\ No newline at end of file