diff --git a/apps/steward-app/pom.xml b/apps/steward-app/pom.xml index 0a3f77e30..16f918d44 100644 --- a/apps/steward-app/pom.xml +++ b/apps/steward-app/pom.xml @@ -1,172 +1,177 @@ shrine-base net.shrine 1.22.2.0-SNAPSHOT ../../pom.xml 4.0.0 steward-app Steward App jar src/main/scala src/test/scala net.alchim31.maven scala-maven-plugin com.github.eirslett frontend-maven-plugin 1.0 src/main/js install node and npm install-node-and-npm v6.2.2 3.10.3 npm install npm generate-resources bower install bower gulp test and build gulp --no-color io.spray spray-routing_2.11 ${spray-version} io.spray spray-servlet_2.11 ${spray-version} io.spray spray-util_2.11 ${spray-version} io.spray spray-testkit_2.11 ${spray-version} test com.typesafe.akka akka-actor_2.11 ${akka-version} com.typesafe.akka akka-slf4j_2.11 ${akka-version} com.typesafe.akka akka-testkit_2.11 ${akka-testkit-version} test org.json4s json4s-native_2.11 ${json4s-version} com.typesafe.slick slick_2.11 ${slick-version} org.suecarter freeslick_2.11 ${freeslick-version} org.slf4j slf4j-log4j12 ${slf4j-version} com.h2database h2 ${h2-version} test net.shrine shrine-protocol ${project.version} + + net.shrine + shrine-email + ${project.version} + net.shrine shrine-data-commons ${project.version} net.shrine shrine-client ${project.version} net.shrine shrine-crypto ${project.version} test-jar test net.shrine shrine-auth ${project.version} mysql mysql-connector-java ${mysql-version} diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala index 60110c7d9..fb96164eb 100644 --- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala +++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala @@ -1,638 +1,751 @@ package net.shrine.steward.db import java.sql.SQLException import java.util.concurrent.atomic.AtomicInteger import javax.sql.DataSource import com.typesafe.config.Config -import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole} +import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearcherToAudit, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole} import net.shrine.i2b2.protocol.pm.User import net.shrine.log.Loggable import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator} import net.shrine.steward.{CreateTopicsMode, StewardConfigSource} import slick.dbio.Effect.Read import slick.driver.JdbcProfile import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, Future, blocking} import scala.language.postfixOps import scala.util.Try /** * Database access code for the data steward service. * * I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace. * * @author dwalend * @since 1.19 */ case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable { import schemaDef._ import jdbcProfile.api._ val database = Database.forDataSource(dataSource) def createTables() = schemaDef.createTables(database) def dropTables() = schemaDef.dropTables(database) def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = { val future: Future[R] = database.run(action) blocking { Await.result(future, 10 seconds) } } def warmUp = { dbRun(allUserQuery.size.result) } def selectUsers:Seq[UserRecord] = { dbRun(allUserQuery.result) } // todo use whenever a shrine query is logged def upsertUser(user:User):Unit = { val userRecord = UserRecord(user) dbRun(allUserQuery.insertOrUpdate(userRecord)) } def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = { val createInState = StewardConfigSource.createTopicsInState val now = System.currentTimeMillis() val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState) val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now) dbRun(for{ _ <- allTopicQuery += topicRecord _ <- allUserTopicQuery += userTopicRecord } yield topicRecord) } def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try { dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option => val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId)) if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy) if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId) val updatedTopic = oldTopicRecord.copy(name = topicRequest.name, description = topicRequest.description, changedBy = user.username, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).flatMap{_ => outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic) } } ) } def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = { require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id") val (count,topics,userNamesToOutboundUsers) = dbRun( for{ count <- topicCountQuery(parameters).length.result topics <- topicSelectQuery(parameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count, topics,userNamesToOutboundUsers)) ResearchersTopics(parameters.researcherIdOption.get, count, parameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } //treat as private (currently used in test) def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = { dbRun(topicSelectQuery(queryParameters).result) } def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = { val (count,topics,userNamesToOutboundUsers) = dbRun{ for{ count <- topicCountQuery(queryParameters).length.result topics <- topicSelectQuery(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count,topics,userNamesToOutboundUsers) } StewardsTopics(count, queryParameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val countFilter = topicCountQuery(queryParameters) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(countFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countFilter)( columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "id" => x.id case "name" => x.name case "description" => x.description case "createdBy" => x.createdBy case "createDate" => x.createDate case "state" => x.state case "changedBy" => x.changedBy case "changeDate" => x.changeDate }))) val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId)) val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate)) maxDateFilter } def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = { val noTopicRecord:Option[TopicRecord] = None val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord) dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap( _.fold(noOpDBIO){ originalTopic => val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).map(_ => Option(updatedTopic)) } )) } def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = { dbRun(for{ totalTopics <- topicCountQuery(queryParameters).length.result topicsPerStateName <- topicCountsPerState(queryParameters).result } yield TopicsPerState(totalTopics,topicsPerStateName)) } private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = { val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state) groupedByState.map{case (state,result) => (state,result.length)} } def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = { //todo upsertUser(user) when the info is available from the PM val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful { if (StewardConfigSource.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved else TopicState.createTopicsModeRequiresTopic } val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None} val (state,topicName) = dbRun(for{ state <- topicId.fold(noOpDBIOForState)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map( _.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state))) ) topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption ) _ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state) } yield (state,topicName)) val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match { case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name)) case (None,None) => None case (Some(id),None) => if(state == TopicState.unknownForUser) None else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery") + case (None,Some(name)) => + if(state == TopicState.unknownForUser) None + else throw new IllegalStateException(s"How did you get here for $userId with no topic id but a topic name of $name and $state for $shrineQuery") } (state,topicIdAndName) } def selectQueryHistory(queryParameters: QueryParameters,topicParameter:Option[TopicId]):QueryHistory = { val (count,shrineQueries,topics,userNamesToOutboundUsers) = dbRun(for { count <- shrineQueryCountQuery(queryParameters,topicParameter).length.result shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count,shrineQueries,topics,userNamesToOutboundUsers)) val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = { val topic = topicIdsToTopics.get(queryRecord.topicId) val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers)) val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId) //todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId)) queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser) } val result = QueryHistory(count,queryParameters.skipOption.getOrElse(0),shrineQueries.map(toOutboundShrineQuery)) result } private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = { allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap) } private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val countQuery = shrineQueryCountQuery(queryParameters,topicParameter) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(limitFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countQuery) { case "topicName" => val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id) joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1) case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "stewardId" => x.stewardId case "externalId" => x.externalId case "researcherId" => x.researcherId case "name" => x.name case "topic" => x.topicId case "queryContents" => x.queryContents case "stewardResponse" => x.stewardResponse case "date" => x.date })) } val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId)) val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId)) //todo this is probably a binary Approved/Not approved val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate)) maxDateFilter } def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = { val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for { totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set]) } yield (totalQueries,queriesPerUser,userNamesToOutboundUsers)) val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2)) QueriesPerUser(totalQueries,queriesPerOutboundUser) } private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = { val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None) val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId) groupedByResearcher.map{case (researcher,result) => (researcher,result.length)} } lazy val nextTopicId:AtomicInteger = new AtomicInteger({ dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1 }) + + def selectAllAuditRequests: Seq[UserAuditRecord] = { + dbRun(allUserAudits.result) + } + + def selectMostRecentAuditRequests: Seq[UserAuditRecord] = { + dbRun(mostRecentUserAudits.result) + } + + def selectResearchersToAudit(maxQueryCountBetweenAudits:Int,minTimeBetweenAudits:Duration):Seq[ResearcherToAudit] = { + + val now = System.currentTimeMillis() + println(s"now $now minTimeBetweenAudits $minTimeBetweenAudits maxQueryCountBetweenAudits $maxQueryCountBetweenAudits") + + + //todo one round with the db instead of O(researchers) + + //for each researcher + //horizon = if the researcher has had an audit + // date of last audit + // else if no audit yet + // date of first query + val researchersToHorizons: Map[UserName, Date] = dbRun(for{ + dateOfFirstQuery: Seq[(UserName, Date)] <- leastRecentUserQuery.map(record => record.researcherId -> record.date).result + mostRecentAudit: Seq[(UserName, Date)] <- mostRecentUserAudits.map(record => record.researcher -> record.changeDate).result + } yield { + dateOfFirstQuery.toMap ++ mostRecentAudit.toMap + }) + + println(s"researchersToHorizons $researchersToHorizons") + + val researchersToHorizonsAndCounts = researchersToHorizons.map{ researcherDate => + + val queryParameters = QueryParameters(researcherIdOption = Some(researcherDate._1), + minDate = Some(researcherDate._2)) + + val count:Int = dbRun(shrineQueryCountQuery(queryParameters,None).length.result) + (researcherDate._1,(researcherDate._2,count)) + } + + println(s"researchersToHorizonsAndCounts $researchersToHorizonsAndCounts") + + //audit if oldest query within the horizon is >= 30 days in the past and the researcher has run at least one query since. + //todo configure + val oldestAllowed = System.currentTimeMillis() - minTimeBetweenAudits.toMillis + val timeBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 > 0 && x._2._1 <= oldestAllowed) + + println(s"timeBasedAudit $timeBasedAudit") + + //audit if the researcher has run >= 30 queries since horizon? + //todo configure + val queryBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 >= maxQueryCountBetweenAudits) + + println(s"queryBasedAudit $queryBasedAudit") + + val toAudit = timeBasedAudit ++ queryBasedAudit + + val namesToOutboundUsers: Map[UserName, OutboundUser] = dbRun(outboundUsersForNamesAction(toAudit.keySet)) + + toAudit.map(x => ResearcherToAudit(namesToOutboundUsers(x._1),x._2._2,x._2._1,now)).to[Seq] + } + + def logAuditRequests(auditRequests:Seq[ResearcherToAudit],now:Date) { + dbRun{ + allUserAudits ++= auditRequests.map(x => UserAuditRecord(researcher = x.researcher.userName, + queryCount = x.count, + changeDate = now + )) + } + } + } /** * Separate class to support schema generation without actually connecting to the database. * * @param jdbcProfile Database profile to use for the schema */ case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable { import jdbcProfile.api._ def ddlForAllTables = { - allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema + allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema ++ allUserAudits.schema } //to get the schema, use the REPL //println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n")) def createTables(database:Database) = { try { val future = database.run(ddlForAllTables.create) Await.result(future,10 seconds) } catch { //I'd prefer to check and create schema only if absent. No way to do that with Oracle. case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x) } } def dropTables(database:Database) = { val future = database.run(ddlForAllTables.drop) //Really wait forever for the cleanup Await.result(future,Duration.Inf) } class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") { def userName = column[UserName]("userName",O.PrimaryKey) def fullName = column[String]("fullName") def isSteward = column[Boolean]("isSteward") def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply) } class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") { def id = column[TopicId]("id") def name = column[String]("name") def description = column[String]("description") def createdBy = column[UserName]("createdBy") def createDate = column[Date]("createDate") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def idIndex = index("idIndex",id,unique = false) def topicNameIndex = index("topicNameIndex",name,unique = false) def createdByIndex = index("createdByIndex",createdBy,unique = false) def createDateIndex = index("createDateIndex",createDate,unique = false) def stateIndex = index("stateIndex",state,unique = false) def changedByIndex = index("changedByIndex",changedBy,unique = false) def changeDateIndex = index("changeDateIndex",changeDate,unique = false) def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply) def fromRow = (fromParams _).tupled def fromParams(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, stateName:String, changedBy:UserName, changeDate:Date): TopicRecord = { TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(topicRecord: TopicRecord) = Some((topicRecord.id, topicRecord.name, topicRecord.description, topicRecord.createdBy, topicRecord.createDate, topicRecord.state.name, topicRecord.changedBy, topicRecord.changeDate )) } class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") { def researcher = column[UserName]("researcher") def topicId = column[TopicId]("topicId") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true) def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow) def fromRow = (fromParams _).tupled def fromParams(researcher:UserName, topicId:TopicId, stateName:String, changedBy:UserName, changeDate:Date): UserTopicRecord = { UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] = Some((userTopicRecord.researcher, userTopicRecord.topicId, userTopicRecord.state.name, userTopicRecord.changedBy, userTopicRecord.changeDate )) + } + + class UserAuditTable(tag:Tag) extends Table[UserAuditRecord](tag,"userAudit") { + def researcher = column[UserName]("researcher") + def queryCount = column[Int]("queryCount") + def changeDate = column[Date]("changeDate") + + def * = (researcher, queryCount, changeDate) <> (fromRow, toRow) + def fromRow = (fromParams _).tupled + + def fromParams(researcher:UserName, + queryCount:Int, + changeDate:Date): UserAuditRecord = { + UserAuditRecord(researcher,queryCount, changeDate) + } + + def toRow(record: UserAuditRecord):Option[(UserName,Int,Date)] = + Some((record.researcher, + record.queryCount, + record.changeDate + )) } class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") { def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc) def externalId = column[ExternalQueryId]("id") def name = column[String]("name") def researcherId = column[UserName]("researcher") def topicId = column[Option[TopicId]]("topic") def queryContents = column[QueryContents]("queryContents") def stewardResponse = column[String]("stewardResponse") def date = column[Date]("date") def externalIdIndex = index("externalIdIndex",externalId,unique = false) def queryNameIndex = index("queryNameIndex",name,unique = false) def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false) def topicIdIndex = index("topicIdIndex",topicId,unique = false) def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false) def dateIndex = index("dateIndex",date,unique = false) def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow) def fromRow = (fromParams _).tupled def fromParams(stewardId:Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:String, date:Date): ShrineQueryRecord = { ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date) } def toRow(queryRecord: ShrineQueryRecord):Option[( Option[StewardQueryId], ExternalQueryId, String, UserName, Option[TopicId], QueryContents, String, Date )] = Some((queryRecord.stewardId, queryRecord.externalId, queryRecord.name, queryRecord.userId, queryRecord.topicId, queryRecord.queryContents, queryRecord.stewardResponse.name, queryRecord.date) ) } val allUserQuery = TableQuery[UserTable] val allTopicQuery = TableQuery[TopicTable] val allQueryTable = TableQuery[QueryTable] val allUserTopicQuery = TableQuery[UserTopicTable] + val allUserAudits = TableQuery[UserAuditTable] val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for( topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists ) yield topic + val mostRecentUserAudits: Query[UserAuditTable, UserAuditRecord, Seq] = for( + record <- allUserAudits if !allUserAudits.filter(_.researcher === record.researcher).filter(_.changeDate > record.changeDate).exists + ) yield record + + val leastRecentUserQuery: Query[QueryTable, ShrineQueryRecord, Seq] = for( + record <- allQueryTable if !allQueryTable.filter(_.researcherId === record.researcherId).filter(_.date < record.date).exists + ) yield record + } object StewardSchema { val allConfig:Config = StewardConfigSource.config val config:Config = allConfig.getConfig("shrine.steward.database") val slickProfile:JdbcProfile = StewardConfigSource.getObject("slickProfileClassName", config) val schema = StewardSchema(slickProfile) } object StewardDatabase extends NeedsWarmUp { val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config) val db = StewardDatabase(StewardSchema.schema,dataSource) val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart") if(createTablesOnStart) StewardDatabase.db.createTables() override def warmUp() = StewardDatabase.db.warmUp } //API help sealed case class SortOrder(name:String){ import slick.lifted.ColumnOrdered def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = { if(this == SortOrder.ascending) column.asc else column.desc } } object SortOrder { val ascending = SortOrder("ascending") val descending = SortOrder("descending") val sortOrders = Seq(ascending,descending) val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_)) } case class QueryParameters(researcherIdOption:Option[UserName] = None, stateOption:Option[TopicState] = None, skipOption:Option[Int] = None, limitOption:Option[Int] = None, sortByOption:Option[String] = None, sortOrder:SortOrder = SortOrder.ascending, minDate:Option[Date] = None, maxDate:Option[Date] = None ) //DAO case classes, exposed for testing only case class ShrineQueryRecord(stewardId: Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:TopicState, date:Date) { def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = { OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,queryContents,stewardResponse.name,date) } } object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) { def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = { ShrineQueryRecord( None, shrineQuery.externalId, shrineQuery.name, userId, topicId, shrineQuery.queryContents, stewardResponse, System.currentTimeMillis()) } } case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) { lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole) else Set(researcherRole)) } object UserRecord extends ((UserName,String,Boolean) => UserRecord) { def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true"))) } case class TopicRecord(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState, changedBy:UserName, changeDate:Date) { def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = { OutboundTopic(id.get, name, description, userNamesToOutboundUsers(createdBy), createDate, state.name, userNamesToOutboundUsers(changedBy), changeDate) } } object TopicRecord { def apply(id:Option[TopicId], name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState ):TopicRecord = TopicRecord(id, name, description, createdBy, createDate, state, createdBy, createDate) } case class UserTopicRecord(researcher:UserName, topicId:TopicId, state:TopicState, changedBy:UserName, changeDate:Date) +case class UserAuditRecord(researcher:UserName, + queryCount:Int, + changeDate:Date) { + def sameExceptForTimes(userAuditRecord: UserAuditRecord):Boolean = { + (researcher == userAuditRecord.researcher) && + (queryCount == userAuditRecord.queryCount) + } +} + case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId") case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}") case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.") \ No newline at end of file diff --git a/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala b/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala index 56aae5824..e95d6988d 100644 --- a/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala +++ b/apps/steward-app/src/test/scala/net/shrine/steward/StewardServiceSpec.scala @@ -1,1380 +1,1380 @@ package net.shrine.steward import net.shrine.authorization.steward._ import net.shrine.i2b2.protocol.pm.User import net.shrine.protocol.Credential import net.shrine.steward.db.{QueryParameters, StewardDatabase, UserRecord} import org.json4s.native.JsonMethods.parse import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfterEach, FlatSpec, Suite} import spray.http.BasicHttpCredentials import spray.routing.MalformedRequestContentRejection import spray.testkit.ScalatestRouteTest import spray.http.StatusCodes._ import scala.xml.{Elem, NodeSeq} @RunWith(classOf[JUnitRunner]) class StewardServiceTest extends FlatSpec with ScalatestRouteTest with TestWithDatabase with StewardService { def actorRefFactory = system import scala.concurrent.duration._ implicit val routeTestTimeout = RouteTestTimeout(10 seconds) val researcherUserName = "ben" val researcherFullName = researcherUserName val stewardUserName = "dave" val stewardFullName = stewardUserName /** * to run these tests with I2B2 * add a user named qep, to be the qep * add a Boolean parameter for qep, qep, true * add a user named ben, to be a researcher * add a user named dave, to be the data steward * add a Boolean parameter for dave, DataSteward, true * add all three users to the i2b2 project */ val stewardCredentials = BasicHttpCredentials(stewardUserName,"kablam") val researcherCredentials = BasicHttpCredentials(researcherUserName,"kapow") val qepCredentials = BasicHttpCredentials("qep","trustme") val badCredentials = BasicHttpCredentials("qep","wrongPassword") val researcherUser = User( - fullName = researcherUserName, - username = researcherFullName, + fullName = researcherFullName, + username = researcherUserName, domain = "domain", credential = new Credential("ben's password",false), params = Map(), rolesByProject = Map() ) val stewardUser = User( - fullName = stewardUserName, - username = stewardFullName, + fullName = stewardFullName, + username = stewardUserName, domain = "domain", credential = new Credential("dave's password",false), params = Map(stewardRole -> "true"), rolesByProject = Map() ) val researcherOutboundUser = OutboundUser.createFromUser(researcherUser) val stewardOutboundUser = OutboundUser.createFromUser(stewardUser) val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",researcherOutboundUser,0L,TopicState.pending.name,researcherOutboundUser,0L) val forbiddenTopicId = 0 private val queryContent: QueryContents = "18-34 years old@18:31:51\\\\SHRINE\\SHRINE\\Demographics\\Age\\18-34 years old\\" val diffs = List.empty val getItBack = scala.xml.XML.loadString(queryContent).toString "StewardService" should "return an OK and the correct createTopicsMode name" in { StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.Pending.name,s"${CreateTopicsMode.Pending.name} test") { { Get(s"/about/createTopicsMode") ~> route ~> check { assertResult(OK)(status) val createTopicsModeName = new String(body.data.toByteArray) assertResult(s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName) assertResult(s""""${CreateTopicsMode.Pending.name}"""")(createTopicsModeName) } } } StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.Approved.name,s"${CreateTopicsMode.Approved.name} test") { Get(s"/about/createTopicsMode") ~> route ~> check { assertResult(OK)(status) val createTopicsModeName = new String(body.data.toByteArray) assertResult( s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName) assertResult( s""""${CreateTopicsMode.Approved.name}"""")(createTopicsModeName) } } StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") { Get(s"/about/createTopicsMode") ~> route ~> check { assertResult(OK)(status) val createTopicsModeName = new String(body.data.toByteArray) assertResult( s""""${StewardConfigSource.createTopicsInState.name}"""")(createTopicsModeName) assertResult( s""""${CreateTopicsMode.TopicsIgnoredJustLog.name}"""")(createTopicsModeName) } } } "StewardService" should "return an OK and a valid outbound user for a user/whoami request" in { Get(s"/user/whoami") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val userJson = new String(body.data.toByteArray) val outboundUser = parse(userJson).extract[OutboundUser] assertResult(researcherOutboundUser)(outboundUser) } } "StewardService" should "return a 200 for a user/whoami request with bad credentials, with a body of 'AuthenticationFailed'" in { // "StewardService" should "return an TemporaryRedirect for a user/whoami request with bad credentials" in { Get(s"/user/whoami") ~> addCredentials(badCredentials) ~> sealRoute(route) ~> check { assertResult(OK)(status) assertResult(""""AuthenticationFailed"""")(new String(body.data.toByteArray)) } } /* todo "StewardService" should "return a 200 for a user/whoami request with bad credentials, with a body of 'AuthenticationFailed' even wiht a back-tick" in { // "StewardService" should "return an TemporaryRedirect for a user/whoami request with bad credentials" in { val badCredentials = BasicHttpCredentials("o`brien","wrongPassword") Get(s"/user/whoami") ~> addCredentials(badCredentials) ~> sealRoute(route) ~> check { assertResult(OK)(status) assertResult(""""AuthenticationFailed"""")(new String(body.data.toByteArray)) } } */ "StewardService" should "return an OK for an approved request" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) val topicInDb = StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) val dbFound = StewardDatabase.db.changeTopicState(uncontroversialTopic.id,TopicState.approved,stewardUserName) Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(1,"test query","crazy syntax")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(OK)(status) } } "StewardService" should "complain about bad http credentials from the QEP" in { Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(2,"test query","too bad about your password")) ~> addCredentials(badCredentials) ~> sealRoute(route) ~> check { assertResult(Unauthorized)(status) } } "StewardService" should "complain about unexpected json" in { Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}","""{"field":"not in ShrineQuery"}""") ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(false)(handled) assertResult(true)(rejection.isInstanceOf[MalformedRequestContentRejection]) } } "StewardService" should "return a rejection for an unacceptable request" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.rejected,stewardUserName) Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(3,"test query","too bad about your topic")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(UnavailableForLegalReasons)(status) } } "StewardService" should "return a rejection for a pending topic" in { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${uncontroversialTopic.id}",InboundShrineQuery(4,"test query","topic still pending")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(UnavailableForLegalReasons)(status) } } "StewardService" should "return an UnprocessableEntity for an unknown topic" in { Post(s"/qep/requestQueryAccess/user/${researcherUserName}/topic/${forbiddenTopicId}",InboundShrineQuery(5,"test query","no one knows about your topic")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(UnprocessableEntity)(status) } } "StewardService" should " return an UnprocessableEntity for query requests with no topic" in { Post(s"/qep/requestQueryAccess/user/${researcherUserName}",InboundShrineQuery(5,"test query","Not even using a topic")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(UnprocessableEntity)(status) } } "StewardService" should " accept query requests with no topic in 'just log and approve everything' mode " in { StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") { Post(s"/qep/requestQueryAccess/user/${researcherUserName}", InboundShrineQuery(5, "test query", "Not even using a topic")) ~> addCredentials(qepCredentials) ~> route ~> check { assertResult(OK)(status) } } } "StewardService" should "return approved topics for the qep" in { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/qep/approvedTopics/user/${researcherUserName}") ~> addCredentials(qepCredentials) ~> route ~> check { status === OK val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics) === true } } "StewardService" should "return the list of a researcher's Topics as Json" in { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/researcher/topics") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics)) } } "StewardService" should "return the list of a researcher's Topics as Json for various states" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/researcher/topics?state=Pending") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics)) assertResult(OK)(status) } StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) Get(s"/researcher/topics?state=Approved") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq( OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.approved.name,stewardOutboundUser,0))).sameExceptForTimes(topics)) assertResult(OK)(status) } StewardDatabase.db.changeTopicState(1,TopicState.rejected,stewardUserName) Get(s"/researcher/topics?state=Rejected") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(true)(ResearchersTopics(researcherUserName,1,0,Seq( OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.rejected.name,stewardOutboundUser,0))).sameExceptForTimes(topics)) assertResult(OK)(status) } } "DataStewardService" should "reject nonsense topic request states" in { Get(s"/researcher/topics?state=nonsense") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(UnprocessableEntity)(status) } } "StewardService" should "return the list of a researcher's Topics as Json with skip and limit set" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createFiveTopics() Get(s"/researcher/topics?skip=0&limit=2") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(2)(topics.topics.size) assertResult(OK)(status) } Get(s"/researcher/topics?skip=2&limit=2") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(2)(topics.topics.size) assertResult(OK)(status) } Get(s"/researcher/topics?skip=4&limit=2") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(1)(topics.topics.size) assertResult(OK)(status) } } "StewardService" should "return the list of a researcher's Topics as Json with different sorting and ordering options" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createFiveTopics() Get(s"/researcher/topics?sortBy=id") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(5)(topics.topics.size) assertResult(OK)(status) assertResult(topics.topics.sortBy(_.id))(topics.topics) } Get(s"/researcher/topics?sortBy=name") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(5)(topics.topics.size) assertResult(OK)(status) assertResult(topics.topics.sortBy(_.createdBy.userName))(topics.topics) } Get(s"/researcher/topics?sortBy=name&sortDirection=descending") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(5)(topics.topics.size) assertResult(OK)(status) assertResult(topics.topics.sortBy(_.name).reverse)(topics.topics) } Get(s"/researcher/topics?sortBy=name&sortDirection=ascending") ~> addCredentials(researcherCredentials) ~> route ~> check { val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[ResearchersTopics] assertResult(5)(topics.topics.size) assertResult(OK)(status) assertResult(topics.topics.sortBy(_.createdBy.userName))(topics.topics) } } "StewardService" should "return the list of a researcher's query history as Json" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) Get(s"/researcher/queryHistory?asJson=false") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] val history: QueryHistory = QueryHistory(1, 0, List( OutboundShrineQuery(1, 0, "test query", researcherOutboundUser, Some( OutboundTopic(1, uncontroversialTopic.name, uncontroversialTopic.description, researcherOutboundUser, 0, TopicState.approved.name, stewardOutboundUser, 0) ), getItBack, TopicState.approved.name, 0) )) assertResult(diffs)(history.differencesExceptTimes(queries)) } } "StewardService" should "return the list of a researcher's query history as Json, with query bodies as Json" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) Get(s"/researcher/queryHistory?asJson=true") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesString = new String(body.data.toByteArray) val queriesJson = parse(queriesString).extract[QueryHistoryWithJson] assertResult(diffs)(QueryHistory(1,0,List( OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some( OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0,TopicState.approved.name,stewardOutboundUser,0) ),getItBack,TopicState.approved.name,0) )).differencesExceptTimes(queriesJson.convertToXml)) } } "StewardService" should "return the list of a researcher's query history as Json, filtered by state" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("Forbidden topic","No way is the data steward going for this")) StewardDatabase.db.changeTopicState(2,TopicState.rejected,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(1,"forbidden query",queryContent)) Get(s"/researcher/queryHistory?state=Approved") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(diffs)( QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(OutboundTopic( 1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0, TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0))) .differencesExceptTimes(queries)) } } "StewardService" should "return the list of a researcher's query history as Json, with skip and limit parameters" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/researcher/queryHistory?") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?skip=0&limit=5") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(5)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?skip=1&limit=4") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(4)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?skip=4&limit=5") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(2)(queries.queryRecords.size) } } "StewardService" should "return the list of a researcher's query history as Json, using parameters for sorting" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/researcher/queryHistory?sortBy=externalId") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords) } Get(s"/researcher/queryHistory?sortBy=externalId&sortDirection=ascending") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords) } Get(s"/researcher/queryHistory?sortBy=externalId&sortDirection=descending") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.externalId).reverse)(queries.queryRecords) } Get(s"/researcher/queryHistory?sortBy=stewardResponse") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.stewardResponse))(queries.queryRecords) } Get(s"/researcher/queryHistory?sortBy=name") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.name))(queries.queryRecords) } } "StewardService" should "return the list of a researcher's query history, using parameters for sorting, skip, and limit" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() val sixQueries:Seq[OutboundShrineQuery] = { Get(s"/researcher/queryHistory") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries:QueryHistory = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords) queries.queryRecords } } Get(s"/researcher/queryHistory?skip=0&limit=3&sortBy=name") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) assertResult(queries.queryRecords)(sixQueries.sortBy(_.name).take(3)) } Get(s"/researcher/queryHistory?skip=2&limit=3&sortBy=name") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) assertResult(queries.queryRecords)(sixQueries.sortBy(_.name).slice(2, 5)) } } "StewardService" should "return the list of a researcher's query history as Json, with filtered by topic" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/researcher/queryHistory") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/researcher/queryHistory/topic/1") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) } Get(s"/researcher/queryHistory/topic/1?skip=1&limit=2") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(2)(queries.queryRecords.size) } Get(s"/researcher/queryHistory/topic/2") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) } } "StewardService" should "return the list of a researcher's query history as Json, filtered by date" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) val startTime = System.currentTimeMillis() Thread.sleep(10) createSixQueries() Thread.sleep(10) val finishTime = System.currentTimeMillis() Get(s"/researcher/queryHistory") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?minDate=$startTime&maxDate=$finishTime") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?minDate=$startTime&maxDate=$startTime") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(0)(queries.queryRecords.size) } Get(s"/researcher/queryHistory?minDate=$finishTime&maxDate=$finishTime") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(0)(queries.queryRecords.size) } } "StewardService" should "return the list of a researcher's query history as Json, even including an unknown topic id" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query for unknown topic",queryContent)) Get(s"/researcher/queryHistory") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(diffs)(QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query for unknown topic",researcherOutboundUser,None,getItBack,TopicState.unknownForUser.name,0))).differencesExceptTimes(queries)) } } "StewardService" should "return the list of a researcher's query history as Json, sorted by topic name" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/researcher/queryHistory?sortBy=topicName") ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) // assertResult(queries.queryRecords.sortBy(_.externalId))(queries.queryRecords) } } "StewardService" should "return counts of topics, total and by state" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createFiveTopics() Get(s"/steward/statistics/topicsPerState") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val statisticsJson = new String(body.data.toByteArray) val statistics = parse(statisticsJson).extract[TopicsPerState] assertResult(5)(statistics.total) assertResult(TopicsPerState(5,Seq((TopicState.pending.name,5))))(statistics) } } "StewardService" should "injest a researcher's request to study a topic" in { Post(s"/researcher/requestTopicAccess",InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(Accepted)(status) assertResult(1)(StewardDatabase.db.selectUsers.size) assertResult(UserRecord(researcherUserName,researcherUserName,false))(StewardDatabase.db.selectUsers.head) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(uncontroversialTopic.description)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(TopicState.pending)(topic.state) } } "StewardService" should "injest a second request from a researcher request to study a new topic" in { val secondTopicName = "Liver" val secondTopicDescription = "Liver Study" StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Post(s"/researcher/requestTopicAccess",InboundTopicRequest(secondTopicName,secondTopicDescription)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(Accepted)(status) assertResult(1)(StewardDatabase.db.selectUsers.size) assertResult(UserRecord(researcherUserName,researcherUserName,false))(StewardDatabase.db.selectUsers.head) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(2)(topics.size) val firstTopic = topics(0) assertResult(uncontroversialTopic.name)(firstTopic.name) assertResult(uncontroversialTopic.description)(firstTopic.description) assertResult(researcherUserName)(firstTopic.createdBy) assertResult(TopicState.pending)(firstTopic.state) val secondTopic = topics(1) assertResult(secondTopicName)(secondTopic.name) assertResult(secondTopicDescription)(secondTopic.description) assertResult(researcherUserName)(secondTopic.createdBy) assertResult(TopicState.pending)(secondTopic.state) } } "StewardService" should " place new topics in the Approved state in auto-approve mode" in { StewardConfigSource.configForBlock(StewardConfigSource.createTopicsModeConfigKey,CreateTopicsMode.TopicsIgnoredJustLog.name,s"${CreateTopicsMode.TopicsIgnoredJustLog.name} test") { Post(s"/researcher/requestTopicAccess",InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(Accepted)(status) assertResult(1)(StewardDatabase.db.selectUsers.size) assertResult(UserRecord(researcherUserName, researcherUserName, false))(StewardDatabase.db.selectUsers.head) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(uncontroversialTopic.description)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(TopicState.approved)(topic.state) } } } "StewardService" should "injest a researcher's request to edit a topic" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) val updatedDescription = "Really you should accept this" Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(Accepted)(status) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(updatedDescription)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(researcherUserName)(topic.changedBy) assertResult(TopicState.pending)(topic.state) } } "StewardService" should "reject a researcher's request to edit a topic that does not exist" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) val updatedDescription = "Really you should accept this" Post(s"/researcher/editTopicRequest/2",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(NotFound)(status) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(uncontroversialTopic.description)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(researcherUserName)(topic.changedBy) assertResult(TopicState.pending)(topic.state) } } "StewardService" should "reject a researcher's request to edit an approved topic" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) val updatedDescription = "Really you should accept this" Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~> addCredentials(researcherCredentials) ~> route ~> check { assertResult(Forbidden)(status) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(uncontroversialTopic.description)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(stewardUserName)(topic.changedBy) assertResult(TopicState.approved)(topic.state) } } "StewardService" should "reject an attempt to edit a topic owned by a different user" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) val updatedDescription = "Really you should accept this" Post(s"/researcher/editTopicRequest/1",InboundTopicRequest(uncontroversialTopic.name,updatedDescription)) ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(Forbidden)(status) val topics = StewardDatabase.db.selectTopics(QueryParameters()) assertResult(1)(topics.size) val topic = topics.head assertResult(uncontroversialTopic.name)(topic.name) assertResult(uncontroversialTopic.description)(topic.description) assertResult(researcherUserName)(topic.createdBy) assertResult(TopicState.pending)(topic.state) } } "StewardService" should "return the full history of queries as Json" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) Get(s"/steward/queryHistory") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(diffs) ( QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some( OutboundTopic(1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0, TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0))) .differencesExceptTimes(queries)) } } "StewardService" should "return the history of queries for a specific user as Json" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) Get(s"/steward/queryHistory/user/${researcherUserName}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(diffs)( QueryHistory(1,0,List(OutboundShrineQuery(1,0,"test query",researcherOutboundUser,Some(OutboundTopic( 1,uncontroversialTopic.name,uncontroversialTopic.description,researcherOutboundUser,0, TopicState.approved.name,stewardOutboundUser,0)),getItBack,TopicState.approved.name,0))) .differencesExceptTimes(queries)) } } "StewardService" should "return the query history as Json, filtered by topic" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/steward/queryHistory") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/steward/queryHistory/user/ben") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(6)(queries.queryRecords.size) } Get(s"/steward/queryHistory/topic/1") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) } Get(s"/steward/queryHistory/user/ben/topic/1") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) } Get(s"/steward/queryHistory/topic/1?skip=1&limit=2") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(2)(queries.queryRecords.size) } Get(s"/steward/queryHistory/topic/2") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val queriesJson = new String(body.data.toByteArray) val queries = parse(queriesJson).extract[QueryHistory] assertResult(3)(queries.queryRecords.size) } } "StewardService" should "return counts of queries, total and by user" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createSixQueries() Get(s"/steward/statistics/queriesPerUser") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val statisticsJson = new String(body.data.toByteArray) val statistics = parse(statisticsJson).extract[QueriesPerUser] assertResult(6)(statistics.total) assertResult(QueriesPerUser(6,Seq((researcherOutboundUser,6))))(statistics) } } "StewardService" should "return the topics for a specific user as Json" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/steward/topics/user/${researcherUserName}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics] assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics)) } } "StewardService" should "return the topics for a specific user as Json, given skip and limit parameters" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) createFiveTopics() Get(s"/steward/topics/user/${researcherUserName}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics] assertResult(5)(topics.topics.size) } Get(s"/steward/topics/user/${researcherUserName}?skip=0&limit=3") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics] assertResult(3)(topics.topics.size) } Get(s"/steward/topics/user/${researcherUserName}?skip=2&limit=3") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics] assertResult(3)(topics.topics.size) } Get(s"/steward/topics/user/${researcherUserName}?skip=3&limit=4") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics:StewardsTopics = parse(topicsJson).extract[StewardsTopics] assertResult(2)(topics.topics.size) } } "StewardService" should "return all of the topics" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/steward/topics") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[StewardsTopics] assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics)) } } "StewardService" should "return all of the pending topics" in { StewardDatabase.db.upsertUser(researcherUser) StewardDatabase.db.upsertUser(stewardUser) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Get(s"/steward/topics?state=Pending") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) val topicsJson = new String(body.data.toByteArray) val topics = parse(topicsJson).extract[StewardsTopics] assertResult(true)(StewardsTopics(1,0,Seq(uncontroversialTopic)).sameExceptForTimes(topics)) } } "StewardService" should "approve a researcher's request to study a topic" in { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Post(s"/steward/approveTopic/topic/${uncontroversialTopic.id}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) } } "StewardService" should "reject a researcher's request to study a topic" in { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) Post(s"/steward/rejectTopic/topic/${uncontroversialTopic.id}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(OK)(status) } } "A steward's attempt to approve or reject a topic that doesn't exist" should "report an error" in { Post(s"/steward/approveTopic/topic/${uncontroversialTopic.id}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(UnprocessableEntity)(status) } Post(s"/steward/rejectTopic/topic/${uncontroversialTopic.id}") ~> addCredentials(stewardCredentials) ~> route ~> check { assertResult(UnprocessableEntity)(status) } } "StewardService" should "redirect several urls to client/index.html" in { Get() ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/index.html") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/client") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/client/") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } } /* //CORS won't be turned on in the real server, so this test won't normally pass. "DataStewardService" should "support a CORS OPTIONS request" in { Options(s"/steward/topics") ~> //No credentials for Options, so no addCredentials(testCredentials) ~> route ~> check { assertResult(OK)(status) } } */ /* For some reason, the test engine isn't finding the static resources. I suspect it is testing with raw .class files , not an actual .war or .jar. Works on the actual server. "StewardService" should "serve up static files from resources/client" in { Get("/steward/client/test.txt") ~> route ~> check { assertResult(OK)(status) assertResult("Test file")(body.asString) } } */ def createFiveTopics(): Unit ={ StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("slightlyControversial","who cares?")) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("controversial","controversial")) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("moderatelyControversial","more controversial than that")) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("veryControversial","Just say no")) } def createSixQueries() = { StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest("Forbidden topic","No way is the data steward going for this")) StewardDatabase.db.changeTopicState(2,TopicState.rejected,stewardUserName) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"A test query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(1,"B forbidden query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2," C test query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(3,"4 forbidden query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(4,"7 test query",queryContent)) StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(2),InboundShrineQuery(5,"% forbidden query",queryContent)) } } trait TestWithDatabase extends BeforeAndAfterEach { this:Suite => override def beforeEach() = { StewardDatabase.db.createTables() } override def afterEach() = { StewardDatabase.db.nextTopicId.set(1) StewardDatabase.db.dropTables() } } /* object OutboundTopic { val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",OutboundUser.someOutboundResearcher,0L,TopicState.pending.name,OutboundUser.someOutboundResearcher,0L) val forbiddenTopicId = 0 } object OutboundShrineQuery extends (( StewardQueryId, ExternalQueryId, String, OutboundUser, Option[OutboundTopic], QueryContents, TopicStateName, Date) => OutboundShrineQuery) { val notSureAboutQueryContents:QueryContents = "Appropriate query contents" val someQueryRecord = OutboundShrineQuery(-5,-2,"Kidney Query",OutboundUser.someOutboundResearcher,Some(uncontroversialTopic),OutboundShrineQuery.notSureAboutQueryContents,TopicState.approved.name,System.currentTimeMillis()) } */ \ No newline at end of file diff --git a/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala b/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala new file mode 100644 index 000000000..3e7e99909 --- /dev/null +++ b/apps/steward-app/src/test/scala/net/shrine/steward/db/StewardDbTest.scala @@ -0,0 +1,238 @@ +package net.shrine.steward.db + +import net.shrine.authorization.steward.{InboundShrineQuery, InboundTopicRequest, OutboundTopic, OutboundUser, QueryContents, ResearcherToAudit, TopicState} +import net.shrine.i2b2.protocol.pm.User +import net.shrine.protocol.Credential +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Suite} + +import scala.concurrent.duration._ + +/** + * Tests of data steward db functions not excercised via the StewardService + * + * @author david + * @since 1.22 + */ + +@RunWith(classOf[JUnitRunner]) +class StewardServiceTest extends FlatSpec with TestWithDatabase { + + val researcherUserName = "ben" + val researcherFullName = "Ben here before" + + val researcherUser = User( + fullName = researcherFullName, + username = researcherUserName, + domain = "domain", + credential = new Credential("ben's password",false), + params = Map(), + rolesByProject = Map() + ) + + val researcherOutboundUser = OutboundUser.createFromUser(researcherUser) + + "The database" should "record an audit request" in { + + val now = System.currentTimeMillis() + val expectedAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now)) + val expectedAuditRecords = expectedAuditRequests.map(x => UserAuditRecord(x.researcher.userName,x.count,now)) + + StewardDatabase.db.logAuditRequests(expectedAuditRequests,now) + val actual: Seq[UserAuditRecord] = StewardDatabase.db.selectAllAuditRequests + + assertResult(expectedAuditRecords)(actual) + } + + "The database" should "supply the most recent audit request" in { + + val now = System.currentTimeMillis() + val firstAuditRecords: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-5000,now-3000)) + val secondAuditRecords: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now)) + + val expectedAuditRecords = secondAuditRecords.map(x => UserAuditRecord(x.researcher.userName,x.count,now)) + + StewardDatabase.db.logAuditRequests(firstAuditRecords,now) + val actual: Seq[UserAuditRecord] = StewardDatabase.db.selectMostRecentAuditRequests + + assertResult(expectedAuditRecords)(actual) + } + + val uncontroversialTopic = OutboundTopic(1,"UncontroversialKidneys","Study kidneys without controversy",researcherOutboundUser,0L,TopicState.pending.name,researcherOutboundUser,0L) + val stewardUserName = "dave" + val queryContent: QueryContents = "18-34 years old@18:31:51\\\\SHRINE\\SHRINE\\Demographics\\Age\\18-34 years old\\" + + "The database" should "recommend an audit for a user who has never been audited and has an unaudited query far enough in the past" in { + + StewardDatabase.db.upsertUser(researcherUser) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) + val now = System.currentTimeMillis() + + + Thread.sleep(20) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,10 milliseconds) + + val expected = Seq(ResearcherToAudit(researcherOutboundUser,1,now - 20,now)) + + assertResult(expected.size)(audits.size) + assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2))) + } + + "The database" should "not recommend an audit for a user who has never been audited and has an unaudited query not far enough in the past" in { + + StewardDatabase.db.upsertUser(researcherUser) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) + val now = System.currentTimeMillis() + + + Thread.sleep(20) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days) + + assertResult(Seq.empty)(audits) + } + + "The database" should "recommend an audit for a user who has been audited and has an unaudited query far enough in the past" in { + + val now = System.currentTimeMillis() + + StewardDatabase.db.upsertUser(researcherUser) + + val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000)) + StewardDatabase.db.logAuditRequests(oldAuditRequests,now) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) + + Thread.sleep(20) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,10 milliseconds) + + val expected = Seq(ResearcherToAudit(researcherOutboundUser,1,now - 20,now)) + + assertResult(expected.size)(audits.size) + assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2))) + } + + "The database" should "not recommend an audit for a user who has been audited and has an unaudited query not far enough in the past" in { + + val now = System.currentTimeMillis() + + StewardDatabase.db.upsertUser(researcherUser) + + val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000)) + StewardDatabase.db.logAuditRequests(oldAuditRequests,now) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query",queryContent)) + + Thread.sleep(20) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days) + + assertResult(Seq.empty)(audits) + } + + "The database" should "recommend an audit for a user who has never been audited and has run many queries" in { + + StewardDatabase.db.upsertUser(researcherUser) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent)) + val now = System.currentTimeMillis() + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(3,30 days) + + val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now)) + + assertResult(expected.size)(audits.size) + assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2))) + } + + "The database" should "not recommend an audit for a user who has never been audited and has not run many queries" in { + + StewardDatabase.db.upsertUser(researcherUser) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent)) + val now = System.currentTimeMillis() + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days) + + assertResult(Seq.empty)(audits) + } + + "The database" should "recommend an audit for a user who has been audited and has run many queries" in { + + val now = System.currentTimeMillis() + + StewardDatabase.db.upsertUser(researcherUser) + + val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000)) + StewardDatabase.db.logAuditRequests(oldAuditRequests,now) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent)) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(3,30 days) + + val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now)) + + assertResult(expected.size)(audits.size) + assertResult(true)(expected.zip(audits).forall(x => x._1.sameExceptForTimes(x._2))) + } + + "The database" should "not recommend an audit for a user who has been audited and has not run many queries" in { + + val now = System.currentTimeMillis() + + StewardDatabase.db.upsertUser(researcherUser) + + val oldAuditRequests: Seq[ResearcherToAudit] = Seq(ResearcherToAudit(researcherOutboundUser,31,now-2000,now-1000)) + StewardDatabase.db.logAuditRequests(oldAuditRequests,now) + + StewardDatabase.db.createRequestForTopicAccess(researcherUser,InboundTopicRequest(uncontroversialTopic.name,uncontroversialTopic.description)) + StewardDatabase.db.changeTopicState(1,TopicState.approved,stewardUserName) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(0,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(1,"test query1",queryContent)) + StewardDatabase.db.logAndCheckQuery(researcherUserName,Some(1),InboundShrineQuery(2,"test query1",queryContent)) + + val audits: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(30,30 days) + + val expected = Seq(ResearcherToAudit(researcherOutboundUser,3,now - 20,now)) + + assertResult(Seq.empty)(audits) + } +} + +trait TestWithDatabase extends BeforeAndAfterEach { this:Suite => + + override def beforeEach() = { + StewardDatabase.db.createTables() + } + + override def afterEach() = { + StewardDatabase.db.nextTopicId.set(1) + StewardDatabase.db.dropTables() + } + +} diff --git a/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala b/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala index ae273766d..5ff90092d 100644 --- a/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala +++ b/commons/auth/src/main/scala/net/shrine/authorization/steward/StewardModel.scala @@ -1,256 +1,264 @@ package net.shrine.authorization.steward import java.lang.reflect.Field import net.shrine.i2b2.protocol.pm.User import net.shrine.log.Loggable import net.shrine.protocol.Credential import net.shrine.serialization.NodeSeqSerializer import org.json4s.{DefaultFormats, Formats} import spray.http.{StatusCode, StatusCodes} import spray.httpx.Json4sSupport import scala.util.{Failure, Try} import scala.xml.NodeSeq /** * Data model for the data steward. * * @author dwalend * @since 1.19 */ //http response json case class OutboundTopic(id:TopicId, name:String, description:String, createdBy:OutboundUser, createDate:Date, state:TopicStateName, changedBy:OutboundUser, changeDate:Date ) { def differences(other:OutboundTopic):Seq[(String,Any,Any)] = { if (this == other) List() else { val fields = getClass.getDeclaredFields val names = fields.map(_.getName) def getFromField(field:Field,thing:OutboundTopic):Any = { field.setAccessible(true) field.get(thing) } val thisUnapplied = fields.map(getFromField(_,this)) val otherUnapplied = fields.map(getFromField(_,other)) val tuples = names.zip(thisUnapplied.zip(otherUnapplied)) def difference(name:String,one:Any,other:Any):Option[(String,Any,Any)] = { if(one == other) None else { Some((name,one,other)) } } tuples.map(x => difference(x._1,x._2._1,x._2._2)).to[Seq].flatten } } def differencesExceptTimes(other:OutboundTopic):Seq[(String,Any,Any)] = { differences(other).filterNot(x => x._1 == "createDate").filterNot(x => x._1 == "changeDate") } } case class OutboundUser(userName:UserName,fullName:String,roles:Set[Role] = Set(researcherRole)) {} object OutboundUser extends Loggable { def createResearcher(userName:UserName,fullName:String) = OutboundUser(userName,fullName) def createSteward(userName:UserName,fullName:String) = OutboundUser(userName,fullName,Set(researcherRole,stewardRole)) def createFromUser(user:User) = if( user.params.toList.contains((stewardRole,"true"))) createSteward(user.username,user.fullName) else createResearcher(user.username,user.fullName) /** If the user is unknown but has a user id, best effort is to guess this user is a researcher with userName as their full name*/ def createUnknownUser(userName:UserName) = { info(s"Creating an OutboundUser for unknown userName $userName") createResearcher(userName,userName) } } sealed case class TopicState(name:TopicStateName,statusCode:StatusCode,message:String) //todo split out TopicResponse from TopicState object TopicState { val pending = TopicState("Pending",StatusCodes.UnavailableForLegalReasons,"Topic pending data steward's approval") val approved = TopicState("Approved",StatusCodes.OK,"OK") val rejected = TopicState("Rejected",StatusCodes.UnavailableForLegalReasons,"Topic rejected by data steward") val unknownForUser = TopicState("Unknown For User",StatusCodes.UnprocessableEntity,"Topic unknown for user") val createTopicsModeRequiresTopic = TopicState("Required But Not Supplied",StatusCodes.UnprocessableEntity,"Topic required but not supplied") val namesToStates = Seq(pending,approved,rejected,unknownForUser,createTopicsModeRequiresTopic).map(x => x.name -> x).toMap def stateForStringOption(stringOption: Option[TopicStateName]): Try[Option[TopicState]] = { val notPresent: Try[Option[TopicState]] = Try(None) stringOption.fold(notPresent)((string: TopicStateName) => { val notInTheMap: Try[Option[TopicState]] = Failure(new IllegalArgumentException(s"No value for $string in $namesToStates")) namesToStates.get(string).fold(notInTheMap)(x => Try(Some(x))) }) } } case class OutboundShrineQueryWithJson( stewardId:StewardQueryId, externalId:ExternalQueryId, name:String, user:OutboundUser, topic:Option[OutboundTopic], queryContents: NodeSeq, stewardResponse:TopicStateName, date:Date) extends Json4sSupport { override implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer def convertToXml:OutboundShrineQuery = { OutboundShrineQuery(stewardId, externalId, name, user, topic, queryContents.toString, stewardResponse, date) } } case class OutboundShrineQuery( stewardId:StewardQueryId, externalId:ExternalQueryId, name:String, user:OutboundUser, topic:Option[OutboundTopic], queryContents: QueryContents, stewardResponse:TopicStateName, date:Date) { def differences(other:OutboundShrineQuery):Seq[(String,Any,Any)] = { if (this == other) List() else { val fields = getClass.getDeclaredFields val names = fields.map(_.getName) def getFromField(field: Field, thing: OutboundShrineQuery): Any = { field.setAccessible(true) field.get(thing) } val thisUnapplied = fields.map(getFromField(_, this)) val otherUnapplied = fields.map(getFromField(_, other)) val tuples = names.zip(thisUnapplied.zip(otherUnapplied)) def difference(name: String, one: Any, other: Any): Option[(String, Any, Any)] = { // TODO: Remove this horrible string equality hack. if (one == other) None else { Some((name, one, other)) } } tuples.map(x => difference(x._1, x._2._1, x._2._2)).to[Seq].flatten } } def convertToJson:OutboundShrineQueryWithJson = { OutboundShrineQueryWithJson(stewardId, externalId, name, user, topic, scala.xml.XML.loadString(queryContents), stewardResponse, date) } def differencesExceptTimes(other:OutboundShrineQuery):Seq[(String,Any,Any)] = { val diffWithoutTimes = differences(other).filterNot(x => x._1 == "date").filterNot(x => x._1 == "topic") val topicDiffs:Seq[(String,Any,Any)] = (topic,other.topic) match { case (None,None) => Seq.empty[(String,Any,Any)] case (Some(thisTopic),Some(otherTopic)) => thisTopic.differencesExceptTimes(otherTopic) case _ => Seq(("topic",this.topic,other.topic)) } diffWithoutTimes ++ topicDiffs } } case class QueriesPerUser(total:Int,queriesPerUser:Seq[(OutboundUser,Int)]) //todo rename QueriesPerResearcher case class TopicsPerState(total:Int,topicsPerState:Seq[(TopicStateName,Int)]) case class ResearchersTopics(userId:UserName,totalCount:Int,skipped:Int,topics:Seq[OutboundTopic]) { def sameExceptForTimes(researchersTopics: ResearchersTopics):Boolean = { (totalCount == researchersTopics.totalCount) && (skipped == researchersTopics.skipped) && (userId == researchersTopics.userId) && (topics.size == researchersTopics.topics.size) && topics.zip(researchersTopics.topics).forall(x => x._1.id == x._2.id) } } case class StewardsTopics(totalCount:Int,skipped:Int,topics:Seq[OutboundTopic]) { def sameExceptForTimes(stewardsTopics: StewardsTopics):Boolean = { (totalCount == stewardsTopics.totalCount) && (skipped == stewardsTopics.skipped) && (topics.size == stewardsTopics.topics.size) && topics.zip(stewardsTopics.topics).forall(x => x._1.id == x._2.id) } } case class QueryHistory(totalCount:Int,skipped:Int,queryRecords:Seq[OutboundShrineQuery]) { def sameExceptForTimes(queryResponse: QueryHistory):Boolean = { (totalCount == queryResponse.totalCount) && (skipped == queryResponse.skipped) && (queryRecords.size == queryResponse.queryRecords.size) && queryRecords.zip(queryResponse.queryRecords).forall(x => x._1.differencesExceptTimes(x._2).isEmpty) } def differences(other:QueryHistory):Seq[(String,Any,Any)] = { if (this == other) List() else { val fields = getClass.getDeclaredFields val names = fields.map(_.getName) def getFromField(field:Field,thing:QueryHistory):Any = { field.setAccessible(true) field.get(thing) } val thisUnapplied = fields.map(getFromField(_,this)) val otherUnapplied = fields.map(getFromField(_,other)) val tuples = names.zip(thisUnapplied.zip(otherUnapplied)) def difference(name:String,one:Any,other:Any):Option[(String,Any,Any)] = { if(one == other) None else { Some((name,one,other)) } } tuples.map(x => difference(x._1,x._2._1,x._2._2)).to[Seq].flatten } } def differencesExceptTimes(other:QueryHistory):Seq[(String,Any,Any)] = { val normalDiffs:Seq[(String,Any,Any)] = differences(other).filterNot(x => x._1 == "queryRecords") val timeDiffs:Seq[(String,Any,Any)] = queryRecords.zip(other.queryRecords).flatMap(x => x._1.differencesExceptTimes(x._2)) normalDiffs ++ timeDiffs } def convertToJson = QueryHistoryWithJson(totalCount, skipped, queryRecords.map(_.convertToJson)) } case class QueryHistoryWithJson(totalCount:Int,skipped:Int,queryRecords:Seq[OutboundShrineQueryWithJson]) extends Json4sSupport { implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer def convertToXml = QueryHistory(totalCount, skipped, queryRecords.map(_.convertToXml)) } case class TopicIdAndName(id:String,name:String) +case class ResearcherToAudit(researcher:OutboundUser, count:Int, leastRecentQueryDate:Date, currentAuditDate:Date) { + def sameExceptForTimes(audit: ResearcherToAudit): Boolean = { + (researcher == audit.researcher) && + (count == audit.count) + } +} + + //http request Json case class InboundShrineQuery( externalId:ExternalQueryId, name:String, queryContents: QueryContents) case class InboundTopicRequest(name:String,description:String) \ No newline at end of file diff --git a/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala b/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala index 53f20e83a..6ef6284ff 100644 --- a/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala +++ b/commons/crypto/src/main/scala/net/shrine/crypto/Verifier.scala @@ -1,12 +1,14 @@ package net.shrine.crypto import net.shrine.protocol.BroadcastMessage import scala.concurrent.duration.Duration /** * @author clint * @date Nov 27, 2013 */ + +//todo delete this interface . Only one thing implements it. trait Verifier { def verifySig(message: BroadcastMessage, maxSignatureAge: Duration): Boolean } \ No newline at end of file