diff --git a/apps/shrine-app/pom.xml b/apps/shrine-app/pom.xml
index 5f61c9b29..36e3cf7f0 100644
--- a/apps/shrine-app/pom.xml
+++ b/apps/shrine-app/pom.xml
@@ -1,108 +1,114 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>shrine-base</artifactId>
<groupId>net.shrine</groupId>
<version>1.22.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>shrine-app</artifactId>
<name>SHRINE App</name>
<packaging>jar</packaging>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-test-commons</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-config</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-hms-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-qep</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-broadcaster-aggregator</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-broadcaster-service</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-adapter-service</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-crypto</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-ont-support</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-crypto</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-protocol</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-config</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-broadcaster-aggregator</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.shrine</groupId>
<artifactId>shrine-client</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>${h2-version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/apps/shrine-app/src/main/resources/reference.conf b/apps/shrine-app/src/main/resources/reference.conf
index d5f8cf46f..b2cd24847 100644
--- a/apps/shrine-app/src/main/resources/reference.conf
+++ b/apps/shrine-app/src/main/resources/reference.conf
@@ -1,15 +1,22 @@
shrine {
status {
//permittedHostOfOrigin = "localhost" //If absent then get the host name via java.net.InetAddress.getLocalHost.getHostName . Override to control
}
+
+ squerylDataSource {
+ database {
+ dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+ jndiDataSourceName = "java:comp/env/jdbc/shrineDB" //or leave out for tests
+ }
+ }
}
akka {
loglevel = INFO
// log-config-on-start = on
loggers = ["akka.event.slf4j.Slf4jLogger"]
// logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
// Toggles whether the threads created by this ActorSystem should be daemons or not
daemonic = on
}
\ No newline at end of file
diff --git a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala
index d4c7a7f38..8cfdb8795 100644
--- a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala
+++ b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala
@@ -1,344 +1,358 @@
package net.shrine.status
import java.io.File
import javax.ws.rs.{GET, Path, Produces, WebApplicationException}
import javax.ws.rs.core.{MediaType, Response}
import com.sun.jersey.spi.container.{ContainerRequest, ContainerRequestFilter}
import com.typesafe.config.{Config => TsConfig}
import net.shrine.authorization.StewardQueryAuthorizationService
import net.shrine.broadcaster.{Broadcaster, NodeHandle}
import net.shrine.client.PosterOntClient
import net.shrine.wiring.ShrineOrchestrator
import org.json4s.{DefaultFormats, Formats}
import org.json4s.native.Serialization
-import net.shrine.log.Loggable
+import net.shrine.log.{Log, Loggable}
import scala.collection.JavaConverters._
import scala.collection.immutable.{Map, Seq, Set}
import net.shrine.config.ConfigExtensions
import net.shrine.crypto.SigningCertStrategy
import net.shrine.protocol.query.{OccuranceLimited, QueryDefinition, Term}
import net.shrine.protocol.{AuthenticationInfo, BroadcastMessage, Credential, Failure, Result, ResultOutputType, RunQueryRequest, Timeout}
import net.shrine.util.Versions
import scala.concurrent.Await
import scala.util.Try
+import scala.util.control.NonFatal
/**
* A subservice that shares internal state of the shrine servlet.
*
* @author david
* @since 12/2/15
*/
@Path("/internalstatus")
@Produces(Array(MediaType.APPLICATION_JSON))
case class StatusJaxrs(shrineConfig:TsConfig) extends Loggable {
implicit def json4sFormats: Formats = DefaultFormats
@GET
@Path("version")
def version: String = {
val version = Version("changeMe")
val versionString = Serialization.write(version)
versionString
}
@GET
@Path("config")
def config: String = {
//todo probably better to reach out and grab the config from ManuallyWiredShrineJaxrsResources once it is a singleton
Serialization.write(Json4sConfig(shrineConfig))
}
@GET
@Path("summary")
def summary: String = {
val summary = Summary()
Serialization.write(summary)
}
@GET
@Path("i2b2")
def i2b2: String = {
val i2b2 = I2b2()
Serialization.write(i2b2)
}
@GET
@Path("optionalParts")
def optionalParts: String = {
val optionalParts = OptionalParts()
Serialization.write(optionalParts)
}
@GET
@Path("hub")
def hub: String = {
val hub = Hub()
Serialization.write(hub)
}
@GET
@Path("adapter")
def adapter: String = {
val adapter = Adapter()
Serialization.write(adapter)
}
@GET
@Path("qep")
def qep: String = {
val qep = Qep()
Serialization.write(qep)
}
}
case class I2b2(pmUrl:String,
crcUrl:Option[String],
ontUrl:String,
i2b2Domain:String,
username:String,
crcProject:String,
ontProject:String)
object I2b2 {
def apply(): I2b2 = new I2b2(
pmUrl = ShrineOrchestrator.pmPoster.url,
crcUrl = ShrineOrchestrator.adapterComponents.map(_.i2b2AdminService.crcUrl),
ontUrl = "", //todo
i2b2Domain = ShrineOrchestrator.crcHiveCredentials.domain,
username = ShrineOrchestrator.crcHiveCredentials.username,
crcProject = ShrineOrchestrator.crcHiveCredentials.projectId,
ontProject = ShrineOrchestrator.ontologyMetadata.client match {
case client: PosterOntClient => client.hiveCredentials.projectId
case _ => ""
}
)
}
case class DownstreamNode(name:String, url:String)
case class Qep(
maxQueryWaitTimeMillis:Long,
create:Boolean,
attachSigningCert:Boolean,
authorizationType:String,
includeAggregateResults:Boolean,
authenticationType:String
)
object Qep{
val key = "shrine.queryEntryPoint."
def apply():Qep = new Qep(
maxQueryWaitTimeMillis = ShrineOrchestrator.queryEntryPointComponents.fold(0L)(_.i2b2Service.queryTimeout.toMicros),
create = ShrineOrchestrator.queryEntryPointComponents.isDefined,
attachSigningCert = ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.i2b2Service.broadcastAndAggregationService.attachSigningCert),
authorizationType = ShrineOrchestrator.queryEntryPointComponents.fold("")(_.i2b2Service.authorizationService.getClass.getSimpleName),
includeAggregateResults = ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.i2b2Service.includeAggregateResult),
authenticationType = ShrineOrchestrator.queryEntryPointComponents.fold("")(_.i2b2Service.authenticator.getClass.getName)
)
}
object DownstreamNodes {
def get():Seq[DownstreamNode] = {
ShrineOrchestrator.hubComponents.fold(Seq.empty[DownstreamNode])(_.broadcastDestinations.map(DownstreamNode(_)).to[Seq])
}
}
object DownstreamNode {
def apply(nodeHandle: NodeHandle): DownstreamNode = new DownstreamNode(
nodeHandle.nodeId.name,
nodeHandle.client.url.map(_.toString).getOrElse("not applicable"))
}
case class Adapter(crcEndpointUrl:String,
setSizeObfuscation:Boolean,
adapterLockoutAttemptsThreshold:Int,
adapterMappingsFilename:String)
object Adapter{
def apply():Adapter = {
val crcEndpointUrl = ShrineOrchestrator.adapterComponents.fold("")(_.i2b2AdminService.crcUrl)
val setSizeObfuscation = ShrineOrchestrator.adapterComponents.fold(false)(_.i2b2AdminService.obfuscate)
val adapterLockoutAttemptsThreshold = ShrineOrchestrator.adapterComponents.fold(0)(_.i2b2AdminService.adapterLockoutAttemptsThreshold)
val adapterMappingsFileName = ShrineOrchestrator.adapterComponents.fold("")(_.adapterMappings.source)
Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileName)
}
}
case class Hub(shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible
create:Boolean,
downstreamNodes:Seq[DownstreamNode])
object Hub{
def apply():Hub = {
val shouldQuerySelf = false
val create = ShrineOrchestrator.hubComponents.isDefined
val downstreamNodes = DownstreamNodes.get()
Hub(shouldQuerySelf, create, downstreamNodes)
}
}
case class OptionalParts(isHub:Boolean,
stewardEnabled:Boolean,
shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible
downstreamNodes:Seq[DownstreamNode])
object OptionalParts {
def apply(): OptionalParts = {
OptionalParts(
ShrineOrchestrator.hubComponents.isDefined,
ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.shrineService.authorizationService.isInstanceOf[StewardQueryAuthorizationService]),
shouldQuerySelf = false,
DownstreamNodes.get()
)
}
}
case class Summary(
isHub:Boolean,
shrineVersion:String,
shrineBuildDate:String,
ontologyVersion:String,
ontologyTerm:String,
adapterMappingsFileName:Option[String],
adapterMappingsDate:Option[Long],
adapterOk:Boolean,
keystoreOk:Boolean,
hubOk:Boolean,
qepOk:Boolean
)
object Summary {
val term = Term(ShrineOrchestrator.shrineConfig.getString("networkStatusQuery"))
def runQueryRequest: BroadcastMessage = {
val domain = "happy"
val username = "happy"
val networkAuthn = AuthenticationInfo(domain, username, Credential("", isToken = false))
val queryDefinition = QueryDefinition("TestQuery", OccuranceLimited(1, term))
import scala.concurrent.duration._
val req = RunQueryRequest(
"happyProject",
3.minutes,
networkAuthn,
None,
None,
Set(ResultOutputType.PATIENT_COUNT_XML),
queryDefinition)
ShrineOrchestrator.signerVerifier.sign(BroadcastMessage(req.networkQueryId, networkAuthn, req), SigningCertStrategy.Attach)
}
def apply(): Summary = {
val message = runQueryRequest
val adapterOk: Boolean = ShrineOrchestrator.adapterService.fold(true) { adapterService =>
import scala.concurrent.duration._
val start = System.currentTimeMillis
val resultAttempt: Try[Result] = Try(adapterService.handleRequest(message))
val end = System.currentTimeMillis
val elapsed = (end - start).milliseconds
resultAttempt match {
case scala.util.Failure(cause) => false
case scala.util.Success(response) => true
}
}
val hubOk = ShrineOrchestrator.hubComponents.fold(true){ hubComponents =>
val maxQueryWaitTime = hubComponents.broadcasterMultiplexerService.maxQueryWaitTime
val broadcaster: Broadcaster = hubComponents.broadcasterMultiplexerService.broadcaster
val message = runQueryRequest
- val multiplexer = broadcaster.broadcast(message)
- val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq
- val failures = responses.collect { case f: Failure => f }
- val timeouts = responses.collect { case t: Timeout => t }
- val validResults = responses.collect { case r: Result => r }
-
- failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size )
+ val triedMultiplexer = Try(broadcaster.broadcast(message))
+ //todo just use fold()() in scala 2.12
+ triedMultiplexer.toOption.fold(false) { multiplexer =>
+ val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq
+ val failures = responses.collect { case f: Failure => f }
+ val timeouts = responses.collect { case t: Timeout => t }
+ val validResults = responses.collect { case r: Result => r }
+
+ failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size)
+ }
}
val adapterMappingsFileName = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.source)
val adapterMappingsVersion = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.version) //todo use this?
val noDate:Option[Long] = None
val adapterMappingsDate:Option[Long] = adapterMappingsFileName.fold(noDate){ fileName =>
val file:File = new File(fileName)
if(file.exists) Some(file.lastModified())
else None
}
+ val ontologyVersion = try {
+ ShrineOrchestrator.ontologyMetadata.ontologyVersion
+ }
+ catch {
+ case NonFatal(x) =>
+ Log.info("Problem while getting ontology version",x)
+ x.getMessage
+ }
+
Summary(
isHub = ShrineOrchestrator.hubComponents.isDefined,
shrineVersion = Versions.version,
shrineBuildDate = Versions.buildDate,
- ontologyVersion = ShrineOrchestrator.ontologyMetadata.ontologyVersion,
+ //todo in scala 2.12, do better
+ ontologyVersion = ontologyVersion,
ontologyTerm = term.value,
adapterMappingsFileName = adapterMappingsFileName,
adapterMappingsDate = adapterMappingsDate,
adapterOk = adapterOk,
keystoreOk = true, //todo something for this
hubOk = hubOk,
qepOk = true //todo something for this
)
}
}
case class Version(version:String)
//todo SortedMap when possible
case class Json4sConfig(keyValues:Map[String,String])
object Json4sConfig{
def isPassword(key:String):Boolean = {
if(key.toLowerCase.contains("password")) true
else false
}
def apply(config:TsConfig):Json4sConfig = {
val entries: Set[(String, String)] = config.entrySet.asScala.to[Set].map(x => (x.getKey,x.getValue.render())).filterNot(x => isPassword(x._1))
val sortedMap: Map[String, String] = entries.toMap
Json4sConfig(sortedMap)
}
}
class PermittedHostOnly extends ContainerRequestFilter {
//todo generalize for happy, too
//todo for tomcat 8 see https://jersey.java.net/documentation/latest/filters-and-interceptors.html for a cleaner version
//shell code from http://stackoverflow.com/questions/17143514/how-to-add-custom-response-and-abort-request-in-jersey-1-11-filters
//how to apply in http://stackoverflow.com/questions/4358213/how-does-one-intercept-a-request-during-the-jersey-lifecycle
override def filter(requestContext: ContainerRequest): ContainerRequest = {
val hostOfOrigin = requestContext.getBaseUri.getHost
val shrineConfig:TsConfig = ShrineOrchestrator.config
val permittedHostOfOrigin:String = shrineConfig.getOption("shrine.status.permittedHostOfOrigin",_.getString).getOrElse("localhost")
val path = requestContext.getPath
//happy and internalstatus API calls must come from the same host as tomcat is running on (hopefully the dashboard servlet).
// todo access to the happy service permitted for SHRINE 1.21 per SHRINE-1366
// restrict access to happy service when database work resumes as part of SHRINE-
// if ((path.contains("happy") || path.contains("internalstatus")) && (hostOfOrigin != permittedHostOfOrigin)) {
if (path.contains("internalstatus") && (hostOfOrigin != permittedHostOfOrigin)) {
val response = Response.status(Response.Status.UNAUTHORIZED).entity(s"Only available from $permittedHostOfOrigin, not $hostOfOrigin, controlled by shrine.status.permittedHostOfOrigin in shrine.conf").build()
throw new WebApplicationException(response)
}
else requestContext
}
}
\ No newline at end of file
diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala
deleted file mode 100644
index 6ed87de25..000000000
--- a/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package net.shrine.wiring
-
-import javax.naming.InitialContext
-import scala.util.Try
-
-/**
- * @author clint
- * @date Jan 15, 2014
- */
-object Jndi {
- def apply[T](name: String): Try[T] = Try {
- (new InitialContext).lookup(name).asInstanceOf[T]
- }
-}
\ No newline at end of file
diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala
index c610d5815..9218e49a9 100644
--- a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala
+++ b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala
@@ -1,159 +1,159 @@
package net.shrine.wiring
import javax.sql.DataSource
import com.typesafe.config.{Config, ConfigFactory}
import net.shrine.adapter.AdapterComponents
import net.shrine.adapter.dao.AdapterDao
import net.shrine.adapter.service.{AdapterRequestHandler, AdapterResource, AdapterService, I2b2AdminResource, I2b2AdminService}
import net.shrine.broadcaster.dao.HubDao
import net.shrine.broadcaster.dao.squeryl.SquerylHubDao
import net.shrine.broadcaster.service.HubComponents
import net.shrine.client.{EndpointConfig, JerseyHttpClient, OntClient, Poster, PosterOntClient}
import net.shrine.config.ConfigExtensions
import net.shrine.config.mappings.AdapterMappings
import net.shrine.crypto.{DefaultSignerVerifier, KeyStoreCertCollection, KeyStoreDescriptorParser, TrustParam}
import net.shrine.dao.squeryl.{DataSourceSquerylInitializer, SquerylDbAdapterSelecter, SquerylInitializer}
import net.shrine.happy.{HappyShrineResource, HappyShrineService}
import net.shrine.log.Loggable
import net.shrine.ont.data.OntClientOntologyMetadata
import net.shrine.protocol.{HiveCredentials, NodeId, ResultOutputType, ResultOutputTypes}
-
import net.shrine.qep.{I2b2BroadcastResource, QueryEntryPointComponents, ShrineResource}
+import net.shrine.slick.TestableDataSourceCreator
import net.shrine.status.StatusJaxrs
import org.squeryl.internals.DatabaseAdapter
/**
* @author clint
* @since Jan 14, 2014
*
* Application wiring for Shrine.
*/
object ShrineOrchestrator extends ShrineJaxrsResources with Loggable {
override def resources: Iterable[AnyRef] = {
Seq(happyResource,statusJaxrs) ++
shrineResource ++
i2b2BroadcastResource ++
adapterResource ++
i2b2AdminResource ++
hubComponents.map(_.broadcasterMultiplexerResource)
}
//todo another pass to put things only used in one place into that place's apply(Config)
//Load config from file on the classpath called "shrine.conf"
lazy val config: Config = ConfigFactory.load("shrine")
val shrineConfig = config.getConfig("shrine")
protected lazy val nodeId: NodeId = NodeId(shrineConfig.getString("humanReadableNodeName"))
//TODO: Don't assume keystore lives on the filesystem, could come from classpath, etc
lazy val keyStoreDescriptor = shrineConfig.getConfigured("keystore",KeyStoreDescriptorParser(_))
- lazy val certCollection: KeyStoreCertCollection = KeyStoreCertCollection.fromFile(keyStoreDescriptor)
+ lazy val certCollection: KeyStoreCertCollection = KeyStoreCertCollection.fromFileRecoverWithClassPath(keyStoreDescriptor)
protected lazy val keystoreTrustParam: TrustParam = TrustParam.SomeKeyStore(certCollection)
//todo used by the adapterServide and happyShrineService, but not by the QEP. maybe each can have its own signerVerivier
lazy val signerVerifier: DefaultSignerVerifier = new DefaultSignerVerifier(certCollection)
- protected lazy val dataSource: DataSource = Jndi("java:comp/env/jdbc/shrineDB").get
+ protected lazy val dataSource: DataSource = TestableDataSourceCreator.dataSource(shrineConfig.getConfig("squerylDataSource.database"))
protected lazy val squerylAdapter: DatabaseAdapter = SquerylDbAdapterSelecter.determineAdapter(shrineConfig.getString("shrineDatabaseType"))
protected lazy val squerylInitializer: SquerylInitializer = new DataSourceSquerylInitializer(dataSource, squerylAdapter)
//todo it'd be better for the adapter and qep to each have its own connection to the pm cell.
private lazy val pmEndpoint: EndpointConfig = shrineConfig.getConfigured("pmEndpoint", EndpointConfig(_))
lazy val pmPoster: Poster = Poster(certCollection,pmEndpoint)
protected lazy val breakdownTypes: Set[ResultOutputType] = shrineConfig.getOptionConfigured("breakdownResultOutputTypes", ResultOutputTypes.fromConfig).getOrElse(Set.empty)
//todo why does the qep need a HubDao ?
protected lazy val hubDao: HubDao = new SquerylHubDao(squerylInitializer, new net.shrine.broadcaster.dao.squeryl.tables.Tables)
//todo really should be part of the adapter config, but is out in shrine's part of the name space
lazy val crcHiveCredentials: HiveCredentials = shrineConfig.getConfigured("hiveCredentials", HiveCredentials(_, HiveCredentials.CRC))
val adapterComponents:Option[AdapterComponents] = shrineConfig.getOptionConfiguredIf("adapter", AdapterComponents(
_,
certCollection,
squerylInitializer,
breakdownTypes,
crcHiveCredentials,
signerVerifier,
pmPoster,
nodeId
))
//todo maybe just break demeter too use this
lazy val adapterService: Option[AdapterService] = adapterComponents.map(_.adapterService)
//todo maybe just break demeter too use this
lazy val i2b2AdminService: Option[I2b2AdminService] = adapterComponents.map(_.i2b2AdminService)
//todo this is only used by happy
lazy val adapterDao: Option[AdapterDao] = adapterComponents.map(_.adapterDao)
//todo this is only used by happy
lazy val adapterMappings: Option[AdapterMappings] = adapterComponents.map(_.adapterMappings)
val shouldQuerySelf = "hub.shouldQuerySelf"
lazy val localAdapterServiceOption: Option[AdapterRequestHandler] = if(shrineConfig.getOption(shouldQuerySelf,_.getBoolean).getOrElse(false)) { //todo give this a default value (of false) in the reference.conf for the Hub, and make it part of the Hub's apply(config)
require(adapterService.isDefined, s"Self-querying requested because shrine.$shouldQuerySelf is true, but this node is not configured to have an adapter")
adapterService
}
else None //todo eventually make this just another downstream node accessed via loopback
val hubConfig = shrineConfig.getConfig("hub")
lazy val hubComponents: Option[HubComponents] = shrineConfig.getOptionConfiguredIf("hub",HubComponents(_,
keystoreTrustParam,
nodeId,
localAdapterServiceOption,
breakdownTypes,
hubDao
))
//todo anything that requires qepConfig should be inside QueryEntryPointComponents's apply
protected lazy val qepConfig = shrineConfig.getConfig("queryEntryPoint")
lazy val queryEntryPointComponents:Option[QueryEntryPointComponents] = shrineConfig.getOptionConfiguredIf("queryEntryPoint",QueryEntryPointComponents(_,
certCollection,
breakdownTypes,
hubComponents.map(_.broadcastDestinations),
hubDao, //todo the QEP should not need the hub dao
squerylInitializer, //todo could really have its own
pmPoster //todo could really have its own
))
protected lazy val pmUrlString: String = pmEndpoint.url.toString
private lazy val ontEndpoint: EndpointConfig = shrineConfig.getConfigured("ontEndpoint", EndpointConfig(_))
protected lazy val ontPoster: Poster = Poster(certCollection,ontEndpoint)
//todo only used by happy outside of here
lazy val ontologyMetadata: OntClientOntologyMetadata = {
import scala.concurrent.duration._
//TODO: XXX: Un-hard-code max wait time param
val ontClient: OntClient = new PosterOntClient(shrineConfig.getConfigured("hiveCredentials", HiveCredentials(_, HiveCredentials.ONT)), 1.minute, ontPoster)
new OntClientOntologyMetadata(ontClient)
}
protected lazy val happyResource: HappyShrineResource = new HappyShrineResource(HappyShrineService)
protected lazy val statusJaxrs: StatusJaxrs = StatusJaxrs(config)
protected lazy val shrineResource: Option[ShrineResource] = queryEntryPointComponents.map(x => ShrineResource(x.shrineService))
protected lazy val i2b2BroadcastResource: Option[I2b2BroadcastResource] = queryEntryPointComponents.map(x => new I2b2BroadcastResource(x.i2b2Service,breakdownTypes))
protected lazy val adapterResource: Option[AdapterResource] = adapterService.map(AdapterResource(_))
protected lazy val i2b2AdminResource: Option[I2b2AdminResource] = i2b2AdminService.map(I2b2AdminResource(_, breakdownTypes))
def poster(keystoreCertCollection: KeyStoreCertCollection)(endpoint: EndpointConfig): Poster = {
val httpClient = JerseyHttpClient(keystoreCertCollection, endpoint)
Poster(endpoint.url.toString, httpClient)
}
}
diff --git a/apps/shrine-app/src/test/resources/shrine.conf b/apps/shrine-app/src/test/resources/shrine.conf
new file mode 100644
index 000000000..26cf640ca
--- /dev/null
+++ b/apps/shrine-app/src/test/resources/shrine.conf
@@ -0,0 +1,68 @@
+shrine {
+
+ keystore {
+ file = "shrine.keystore"
+ password = "chiptesting"
+ privateKeyAlias = "test-cert"
+ keyStoreType = "JKS"
+ caCertAliases = [carra ca]
+ }
+
+ queryEntryPoint {
+ audit {
+ collectQepAudit = false
+
+ database {
+ slickProfileClassName = "slick.driver.H2Driver$"
+ createTablesOnStart = true //for testing with H2 in memory, when not running unit tests. Set to false normally
+
+ dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+
+ testDataSource {
+ driverClassName = "org.h2.Driver"
+ url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace
+ }
+ }
+ }
+ }
+
+ adapter {
+ create = true
+ audit {
+ collectQepAudit = false
+
+ database {
+ slickProfileClassName = "slick.driver.H2Driver$"
+ createTablesOnStart = true //for testing with H2 in memory, when not running unit tests. Set to false normally
+
+ dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+
+ testDataSource {
+ driverClassName = "org.h2.Driver"
+ url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace
+ }
+ }
+ }
+ }
+
+ squerylDataSource {
+ database {
+ dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+ testDataSource {
+ driverClassName = "org.h2.Driver"
+ url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace
+ }
+ }
+ }
+// squerylDataSource {
+// database {
+// dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+//
+// testDataSource {
+// driverClassName = "org.h2.Driver"
+// url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace
+// }
+// }
+// }
+
+}
\ No newline at end of file
diff --git a/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala b/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala
index 4b7c7ff40..aaa0e7e35 100644
--- a/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala
+++ b/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala
@@ -1,63 +1,63 @@
package net.shrine.status
import com.typesafe.config.ConfigFactory
import net.shrine.util.ShouldMatchersForJUnit
import org.json4s.{DefaultFormats, Formats}
import org.junit.Test
import org.json4s.native.Serialization
import scala.collection.immutable.Map
/**
* Tests for StatusJaxrs
*
* @author david
* @since 12/2/15
*/
class StatusJaxrsTest extends ShouldMatchersForJUnit {
implicit def json4sFormats: Formats = DefaultFormats
val expectedConfig = ConfigFactory.load("shrine")
val statusJaxrs = StatusJaxrs(expectedConfig)
@Test
def testVersion() = {
val versionString = statusJaxrs.version
val version = Serialization.read[Version](versionString)
version should equal(Version("changeMe"))
}
@Test
def testConfig() = {
val expectedJson4sConfig = Json4sConfig(expectedConfig)
val configString = statusJaxrs.config
val config = Serialization.read[Json4sConfig](configString)
config should equal(expectedJson4sConfig)
val passwordKeys = config.keyValues.filter(x => Json4sConfig.isPassword(x._1))
passwordKeys should equal(Map.empty[String,String])
}
- /* todo need some way to start Shrine for this to work
@Test
def testSummary() = {
-// val expectedJson4sConfig = Json4sConfig(expectedConfig)
val summaryString = statusJaxrs.summary
println(summaryString)
val summary = Serialization.read[Summary](summaryString)
-// config should equal(expectedJson4sConfig)
-
- println(summary)
-
+ summary.isHub should be (true)
+ summary.adapterMappingsFileName.isDefined should be (true)
+ summary.adapterMappingsDate.isEmpty should be (true)
+ summary.adapterOk should be (true)
+ summary.keystoreOk should be (true)
+ summary.hubOk should be (false)
+ summary.qepOk should be (true)
}
- */
}
diff --git a/apps/war/src/main/resources/reference.conf b/apps/war/src/main/resources/reference.conf
index d5f8cf46f..84c38b8dc 100644
--- a/apps/war/src/main/resources/reference.conf
+++ b/apps/war/src/main/resources/reference.conf
@@ -1,15 +1,21 @@
shrine {
status {
//permittedHostOfOrigin = "localhost" //If absent then get the host name via java.net.InetAddress.getLocalHost.getHostName . Override to control
}
+ squerylDataSource{
+ database {
+ dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else
+ jndiDataSourceName = "java:comp/env/jdbc/shrineDB" //or leave out for tests
+ }
+ }
}
akka {
loglevel = INFO
// log-config-on-start = on
loggers = ["akka.event.slf4j.Slf4jLogger"]
// logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
// Toggles whether the threads created by this ActorSystem should be daemons or not
daemonic = on
}
\ No newline at end of file
diff --git a/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala b/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala
index 6dc1f4045..f36b2d96e 100644
--- a/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala
+++ b/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala
@@ -1,15 +1,15 @@
package net.shrine.broadcaster
import net.shrine.protocol.SingleNodeResult
-import scala.concurrent.duration.Duration
+
import scala.concurrent.Future
/**
* @author clint
- * @date Nov 15, 2013
+ * @since Nov 15, 2013
*/
trait Multiplexer {
def processResponse(response: SingleNodeResult): Unit
def responses: Future[Iterable[SingleNodeResult]]
}
\ No newline at end of file
diff --git a/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala b/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala
index 7342249fc..ca8f92f88 100644
--- a/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala
+++ b/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala
@@ -1,337 +1,337 @@
package net.shrine.integration
import java.net.URL
import net.shrine.log.Loggable
import scala.concurrent.Future
import scala.concurrent.duration.DurationInt
import org.junit.Test
import net.shrine.util.ShouldMatchersForJUnit
import net.shrine.adapter.AdapterMap
import net.shrine.adapter.DeleteQueryAdapter
import net.shrine.adapter.client.AdapterClient
import net.shrine.adapter.dao.squeryl.AbstractSquerylAdapterTest
import net.shrine.adapter.service.AdapterRequestHandler
import net.shrine.adapter.service.AdapterService
import net.shrine.broadcaster.AdapterClientBroadcaster
import net.shrine.broadcaster.NodeHandle
import net.shrine.crypto.DefaultSignerVerifier
import net.shrine.crypto.TestKeystore
import net.shrine.protocol.{HiveCredentials, AuthenticationInfo, BroadcastMessage, Credential, DeleteQueryRequest, DeleteQueryResponse, NodeId, Result, RunQueryRequest, CertId, RequestType, FlagQueryRequest, FlagQueryResponse, RawCrcRunQueryResponse, ResultOutputType, QueryResult, RunQueryResponse, AggregatedRunQueryResponse, UnFlagQueryRequest, UnFlagQueryResponse, DefaultBreakdownResultOutputTypes}
import net.shrine.qep.QepService
import net.shrine.broadcaster.SigningBroadcastAndAggregationService
import net.shrine.broadcaster.InJvmBroadcasterClient
import net.shrine.adapter.FlagQueryAdapter
import net.shrine.protocol.query.Term
import net.shrine.adapter.RunQueryAdapter
import net.shrine.client.Poster
import net.shrine.client.HttpClient
import net.shrine.client.HttpResponse
import net.shrine.adapter.translators.QueryDefinitionTranslator
import net.shrine.adapter.translators.ExpressionTranslator
import net.shrine.util.XmlDateHelper
import net.shrine.adapter.ReadQueryResultAdapter
import net.shrine.protocol.query.QueryDefinition
import net.shrine.adapter.UnFlagQueryAdapter
import net.shrine.crypto.SigningCertStrategy
/**
* @author clint
* @since Nov 27, 2013
*
- * An in-JVM simulation of a Shrine network with one hub and 4 doanstream adapters.
+ * An in-JVM simulation of a Shrine network with one hub and 4 downstream adapters.
*
* The hub and adapters are wired up with mock AdapterClients that do in-JVM communication via method calls
* instead of remotely.
*
* The adapters are configured to respond with valid results for DeleteQueryRequests
* only. Other requests could be handled, but that would not provide benefit to offset the effort of wiring
* up more and more-complex Adapters.
*
* The test network is queried, and the final result, as well as the state of each adapter, is inspected to
* ensure that the right messages were sent between elements of the system.
*
*/
final class NetworkSimulationTest extends AbstractSquerylAdapterTest with ShouldMatchersForJUnit {
private val certCollection = TestKeystore.certCollection
private lazy val myCertId: CertId = certCollection.myCertId.get
private lazy val signerVerifier = new DefaultSignerVerifier(certCollection)
private val domain = "test-domain"
private val username = "test-username"
private val password = "test-password"
import NetworkSimulationTest._
import scala.concurrent.duration._
private def deleteQueryAdapter: DeleteQueryAdapter = new DeleteQueryAdapter(dao)
private def flagQueryAdapter: FlagQueryAdapter = new FlagQueryAdapter(dao)
private def unFlagQueryAdapter: UnFlagQueryAdapter = new UnFlagQueryAdapter(dao)
private def mockPoster = Poster("http://example.com", new HttpClient {
override def post(input: String, url: String): HttpResponse = ???
})
private val hiveCredentials = HiveCredentials("d", "u", "pwd", "pid")
private def queuesQueriesRunQueryAdapter: RunQueryAdapter = {
val translator = new QueryDefinitionTranslator(new ExpressionTranslator(Map("n1" -> Set("l1"))))
RunQueryAdapter(
mockPoster,
dao,
hiveCredentials,
translator,
10000,
doObfuscation = false,
runQueriesImmediately = false,
DefaultBreakdownResultOutputTypes.toSet,
collectAdapterAudit = false
)
}
private def immediatelyRunsQueriesRunQueryAdapter(setSize: Long): RunQueryAdapter = {
val mockCrcPoster = Poster("http://example.com", new HttpClient {
override def post(input: String, url: String): HttpResponse = {
val req = RunQueryRequest.fromI2b2String(DefaultBreakdownResultOutputTypes.toSet)(input).get
val now = XmlDateHelper.now
val queryResult = QueryResult(1L, 42L, Some(ResultOutputType.PATIENT_COUNT_XML), setSize, Some(now), Some(now), Some("desc"), QueryResult.StatusType.Finished, Some("status"))
val mockCrcXml = RawCrcRunQueryResponse(req.networkQueryId, XmlDateHelper.now, req.authn.username, req.projectId, req.queryDefinition, 42L, Map(ResultOutputType.PATIENT_COUNT_XML -> Seq(queryResult))).toI2b2String
HttpResponse.ok(mockCrcXml)
}
})
queuesQueriesRunQueryAdapter.copy(poster = mockCrcPoster, runQueriesImmediately = true)
}
private def readQueryResultAdapter(setSize: Long): ReadQueryResultAdapter = {
new ReadQueryResultAdapter(
mockPoster,
hiveCredentials,
dao,
doObfuscation = false,
DefaultBreakdownResultOutputTypes.toSet,
collectAdapterAudit = false
)
}
private lazy val adaptersByNodeId: Seq[(NodeId, MockAdapterRequestHandler)] = {
import NodeName._
import RequestType.{ MasterDeleteRequest => MasterDeleteRequestRT, FlagQueryRequest => FlagQueryRequestRT, QueryDefinitionRequest => RunQueryRT, GetQueryResult => ReadQueryResultRT, UnFlagQueryRequest => UnFlagQueryRequestRT }
(for {
(childName, setSize) <- Seq((A, 1L), (B, 2L), (C, 3L), (D, 4L))
} yield {
val nodeId = NodeId(childName.name)
val maxSignatureAge = 1.hour
val adapterMap = AdapterMap(Map(
MasterDeleteRequestRT -> deleteQueryAdapter,
FlagQueryRequestRT -> flagQueryAdapter,
UnFlagQueryRequestRT -> unFlagQueryAdapter,
RunQueryRT -> queuesQueriesRunQueryAdapter,
ReadQueryResultRT -> readQueryResultAdapter(setSize)))
nodeId -> MockAdapterRequestHandler(new AdapterService(nodeId, signerVerifier, maxSignatureAge, adapterMap))
})
}
private lazy val shrineService: QepService = {
val destinations: Set[NodeHandle] = {
(for {
(nodeId, adapterRequestHandler) <- adaptersByNodeId
} yield {
NodeHandle(nodeId, MockAdapterClient(nodeId, adapterRequestHandler))
}).toSet
}
QepService(
"example.com",
MockAuditDao,
MockAuthenticator,
MockQueryAuthorizationService,
true,
SigningBroadcastAndAggregationService(InJvmBroadcasterClient(AdapterClientBroadcaster(destinations, MockHubDao)), signerVerifier, SigningCertStrategy.Attach),
1.hour,
DefaultBreakdownResultOutputTypes.toSet,
false)
}
@Test
def testSimulatedNetwork = afterCreatingTables {
val authn = AuthenticationInfo(domain, username, Credential(password, false))
val masterId = 12345L
import scala.concurrent.duration._
val req = DeleteQueryRequest("some-project-id", 1.second, authn, masterId)
val resp = shrineService.deleteQuery(req, true)
for {
(nodeId, mockAdapter) <- adaptersByNodeId
} {
mockAdapter.lastMessage.networkAuthn.domain should equal(authn.domain)
mockAdapter.lastMessage.networkAuthn.username should equal(authn.username)
mockAdapter.lastMessage.request should equal(req)
mockAdapter.lastResult.response should equal(DeleteQueryResponse(masterId))
}
resp should equal(DeleteQueryResponse(masterId))
}
@Test
def testQueueQuery = afterCreatingTables {
val authn = AuthenticationInfo(domain, username, Credential(password, false))
val topicId = "askldjlkas"
val topicName = "Topic Name"
val queryName = "lsadj3028940"
import scala.concurrent.duration._
val runQueryReq = RunQueryRequest("some-project-id", 1.second, authn, 12345L, Some(topicId), Some(topicName), Set(ResultOutputType.PATIENT_COUNT_XML), QueryDefinition(queryName, Term("n1")))
val aggregatedRunQueryResp = shrineService.runQuery(runQueryReq, true).asInstanceOf[AggregatedRunQueryResponse]
var broadcastMessageId: Option[Long] = None
//Broadcast the original run query request; all nodes should queue the query
for {
(nodeId, mockAdapter) <- adaptersByNodeId
} {
broadcastMessageId = Option(mockAdapter.lastMessage.requestId)
mockAdapter.lastMessage.networkAuthn.domain should equal(authn.domain)
mockAdapter.lastMessage.networkAuthn.username should equal(authn.username)
val lastReq = mockAdapter.lastMessage.request.asInstanceOf[RunQueryRequest]
lastReq.authn should equal(runQueryReq.authn)
lastReq.requestType should equal(runQueryReq.requestType)
lastReq.waitTime should equal(runQueryReq.waitTime)
//todo what to do with this check? lastReq.networkQueryId should equal(mockAdapter.lastMessage.requestId)
lastReq.outputTypes should equal(runQueryReq.outputTypes)
lastReq.projectId should equal(runQueryReq.projectId)
lastReq.queryDefinition should equal(runQueryReq.queryDefinition)
lastReq.topicId should equal(runQueryReq.topicId)
val runQueryResp = mockAdapter.lastResult.response.asInstanceOf[RunQueryResponse]
runQueryResp.queryId should equal(-1L)
runQueryResp.singleNodeResult.statusType should equal(QueryResult.StatusType.Held)
runQueryResp.singleNodeResult.setSize should equal(-1L)
}
aggregatedRunQueryResp.queryId should equal(broadcastMessageId.get)
aggregatedRunQueryResp.results.map(_.setSize) should equal(Seq(-1L, -1L, -1L, -1L, -4L))
}
@Test
def testFlagQuery = afterCreatingTables {
val authn = AuthenticationInfo(domain, username, Credential(password, false))
val masterId = 12345L
import scala.concurrent.duration._
val networkQueryId = 9999L
val name = "some query"
val expr = Term("foo")
val fooQuery = QueryDefinition(name,expr)
dao.insertQuery(masterId.toString, networkQueryId, authn, fooQuery, isFlagged = false, hasBeenRun = true, flagMessage = None)
dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(false)
dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(None)
val req = FlagQueryRequest("some-project-id", 1.second, authn, networkQueryId, Some("foo"))
val resp = shrineService.flagQuery(req, true)
resp should equal(FlagQueryResponse)
dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(true)
dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(Some("foo"))
}
@Test
def testUnFlagQuery = afterCreatingTables {
val authn = AuthenticationInfo(domain, username, Credential(password, false))
val masterId = 12345L
import scala.concurrent.duration._
val networkQueryId = 9999L
val flagMsg = Some("foo")
val name = "some query"
val expr = Term("foo")
val fooQuery = QueryDefinition(name,expr)
dao.insertQuery(masterId.toString, networkQueryId, authn, fooQuery, isFlagged = true, hasBeenRun = true, flagMessage = flagMsg)
dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(true)
dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(flagMsg)
val req = UnFlagQueryRequest("some-project-id", 1.second, authn, networkQueryId)
val resp = shrineService.unFlagQuery(req, true)
resp should equal(UnFlagQueryResponse)
dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(false)
dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(None)
}
}
object NetworkSimulationTest {
private final case class MockAdapterClient(nodeId: NodeId, adapter: AdapterRequestHandler) extends AdapterClient with Loggable {
import scala.concurrent.ExecutionContext.Implicits.global
override def query(message: BroadcastMessage): Future[Result] = Future.successful {
debug(s"Invoking Adapter $nodeId with $message")
val result = adapter.handleRequest(message)
debug(s"Got result from $nodeId: $result")
result
}
override def url: Option[URL] = ???
}
private final case class MockAdapterRequestHandler(delegate: AdapterRequestHandler) extends AdapterRequestHandler {
@volatile var lastMessage: BroadcastMessage = _
@volatile var lastResult: Result = _
override def handleRequest(request: BroadcastMessage): Result = {
lastMessage = request
val result = delegate.handleRequest(request)
lastResult = result
result
}
}
}