diff --git a/apps/shrine-app/pom.xml b/apps/shrine-app/pom.xml index 5f61c9b29..36e3cf7f0 100644 --- a/apps/shrine-app/pom.xml +++ b/apps/shrine-app/pom.xml @@ -1,108 +1,114 @@ shrine-base net.shrine 1.22.0-SNAPSHOT ../../pom.xml 4.0.0 shrine-app SHRINE App jar src/main/scala src/test/scala net.alchim31.maven scala-maven-plugin net.shrine shrine-test-commons ${project.version} test-jar test net.shrine shrine-config ${project.version} net.shrine shrine-hms-core ${project.version} net.shrine shrine-qep ${project.version} net.shrine shrine-broadcaster-aggregator ${project.version} net.shrine shrine-broadcaster-service ${project.version} net.shrine shrine-adapter-service ${project.version} net.shrine shrine-crypto ${project.version} net.shrine shrine-ont-support ${project.version} net.shrine shrine-crypto ${project.version} test-jar test net.shrine shrine-protocol ${project.version} test-jar test net.shrine shrine-config ${project.version} test-jar test net.shrine shrine-broadcaster-aggregator ${project.version} test-jar test net.shrine shrine-client ${project.version} test-jar test + + com.h2database + h2 + ${h2-version} + test + diff --git a/apps/shrine-app/src/main/resources/reference.conf b/apps/shrine-app/src/main/resources/reference.conf index d5f8cf46f..b2cd24847 100644 --- a/apps/shrine-app/src/main/resources/reference.conf +++ b/apps/shrine-app/src/main/resources/reference.conf @@ -1,15 +1,22 @@ shrine { status { //permittedHostOfOrigin = "localhost" //If absent then get the host name via java.net.InetAddress.getLocalHost.getHostName . Override to control } + + squerylDataSource { + database { + dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else + jndiDataSourceName = "java:comp/env/jdbc/shrineDB" //or leave out for tests + } + } } akka { loglevel = INFO // log-config-on-start = on loggers = ["akka.event.slf4j.Slf4jLogger"] // logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" // Toggles whether the threads created by this ActorSystem should be daemons or not daemonic = on } \ No newline at end of file diff --git a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala index d4c7a7f38..8cfdb8795 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala @@ -1,344 +1,358 @@ package net.shrine.status import java.io.File import javax.ws.rs.{GET, Path, Produces, WebApplicationException} import javax.ws.rs.core.{MediaType, Response} import com.sun.jersey.spi.container.{ContainerRequest, ContainerRequestFilter} import com.typesafe.config.{Config => TsConfig} import net.shrine.authorization.StewardQueryAuthorizationService import net.shrine.broadcaster.{Broadcaster, NodeHandle} import net.shrine.client.PosterOntClient import net.shrine.wiring.ShrineOrchestrator import org.json4s.{DefaultFormats, Formats} import org.json4s.native.Serialization -import net.shrine.log.Loggable +import net.shrine.log.{Log, Loggable} import scala.collection.JavaConverters._ import scala.collection.immutable.{Map, Seq, Set} import net.shrine.config.ConfigExtensions import net.shrine.crypto.SigningCertStrategy import net.shrine.protocol.query.{OccuranceLimited, QueryDefinition, Term} import net.shrine.protocol.{AuthenticationInfo, BroadcastMessage, Credential, Failure, Result, ResultOutputType, RunQueryRequest, Timeout} import net.shrine.util.Versions import scala.concurrent.Await import scala.util.Try +import scala.util.control.NonFatal /** * A subservice that shares internal state of the shrine servlet. * * @author david * @since 12/2/15 */ @Path("/internalstatus") @Produces(Array(MediaType.APPLICATION_JSON)) case class StatusJaxrs(shrineConfig:TsConfig) extends Loggable { implicit def json4sFormats: Formats = DefaultFormats @GET @Path("version") def version: String = { val version = Version("changeMe") val versionString = Serialization.write(version) versionString } @GET @Path("config") def config: String = { //todo probably better to reach out and grab the config from ManuallyWiredShrineJaxrsResources once it is a singleton Serialization.write(Json4sConfig(shrineConfig)) } @GET @Path("summary") def summary: String = { val summary = Summary() Serialization.write(summary) } @GET @Path("i2b2") def i2b2: String = { val i2b2 = I2b2() Serialization.write(i2b2) } @GET @Path("optionalParts") def optionalParts: String = { val optionalParts = OptionalParts() Serialization.write(optionalParts) } @GET @Path("hub") def hub: String = { val hub = Hub() Serialization.write(hub) } @GET @Path("adapter") def adapter: String = { val adapter = Adapter() Serialization.write(adapter) } @GET @Path("qep") def qep: String = { val qep = Qep() Serialization.write(qep) } } case class I2b2(pmUrl:String, crcUrl:Option[String], ontUrl:String, i2b2Domain:String, username:String, crcProject:String, ontProject:String) object I2b2 { def apply(): I2b2 = new I2b2( pmUrl = ShrineOrchestrator.pmPoster.url, crcUrl = ShrineOrchestrator.adapterComponents.map(_.i2b2AdminService.crcUrl), ontUrl = "", //todo i2b2Domain = ShrineOrchestrator.crcHiveCredentials.domain, username = ShrineOrchestrator.crcHiveCredentials.username, crcProject = ShrineOrchestrator.crcHiveCredentials.projectId, ontProject = ShrineOrchestrator.ontologyMetadata.client match { case client: PosterOntClient => client.hiveCredentials.projectId case _ => "" } ) } case class DownstreamNode(name:String, url:String) case class Qep( maxQueryWaitTimeMillis:Long, create:Boolean, attachSigningCert:Boolean, authorizationType:String, includeAggregateResults:Boolean, authenticationType:String ) object Qep{ val key = "shrine.queryEntryPoint." def apply():Qep = new Qep( maxQueryWaitTimeMillis = ShrineOrchestrator.queryEntryPointComponents.fold(0L)(_.i2b2Service.queryTimeout.toMicros), create = ShrineOrchestrator.queryEntryPointComponents.isDefined, attachSigningCert = ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.i2b2Service.broadcastAndAggregationService.attachSigningCert), authorizationType = ShrineOrchestrator.queryEntryPointComponents.fold("")(_.i2b2Service.authorizationService.getClass.getSimpleName), includeAggregateResults = ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.i2b2Service.includeAggregateResult), authenticationType = ShrineOrchestrator.queryEntryPointComponents.fold("")(_.i2b2Service.authenticator.getClass.getName) ) } object DownstreamNodes { def get():Seq[DownstreamNode] = { ShrineOrchestrator.hubComponents.fold(Seq.empty[DownstreamNode])(_.broadcastDestinations.map(DownstreamNode(_)).to[Seq]) } } object DownstreamNode { def apply(nodeHandle: NodeHandle): DownstreamNode = new DownstreamNode( nodeHandle.nodeId.name, nodeHandle.client.url.map(_.toString).getOrElse("not applicable")) } case class Adapter(crcEndpointUrl:String, setSizeObfuscation:Boolean, adapterLockoutAttemptsThreshold:Int, adapterMappingsFilename:String) object Adapter{ def apply():Adapter = { val crcEndpointUrl = ShrineOrchestrator.adapterComponents.fold("")(_.i2b2AdminService.crcUrl) val setSizeObfuscation = ShrineOrchestrator.adapterComponents.fold(false)(_.i2b2AdminService.obfuscate) val adapterLockoutAttemptsThreshold = ShrineOrchestrator.adapterComponents.fold(0)(_.i2b2AdminService.adapterLockoutAttemptsThreshold) val adapterMappingsFileName = ShrineOrchestrator.adapterComponents.fold("")(_.adapterMappings.source) Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileName) } } case class Hub(shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible create:Boolean, downstreamNodes:Seq[DownstreamNode]) object Hub{ def apply():Hub = { val shouldQuerySelf = false val create = ShrineOrchestrator.hubComponents.isDefined val downstreamNodes = DownstreamNodes.get() Hub(shouldQuerySelf, create, downstreamNodes) } } case class OptionalParts(isHub:Boolean, stewardEnabled:Boolean, shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible downstreamNodes:Seq[DownstreamNode]) object OptionalParts { def apply(): OptionalParts = { OptionalParts( ShrineOrchestrator.hubComponents.isDefined, ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.shrineService.authorizationService.isInstanceOf[StewardQueryAuthorizationService]), shouldQuerySelf = false, DownstreamNodes.get() ) } } case class Summary( isHub:Boolean, shrineVersion:String, shrineBuildDate:String, ontologyVersion:String, ontologyTerm:String, adapterMappingsFileName:Option[String], adapterMappingsDate:Option[Long], adapterOk:Boolean, keystoreOk:Boolean, hubOk:Boolean, qepOk:Boolean ) object Summary { val term = Term(ShrineOrchestrator.shrineConfig.getString("networkStatusQuery")) def runQueryRequest: BroadcastMessage = { val domain = "happy" val username = "happy" val networkAuthn = AuthenticationInfo(domain, username, Credential("", isToken = false)) val queryDefinition = QueryDefinition("TestQuery", OccuranceLimited(1, term)) import scala.concurrent.duration._ val req = RunQueryRequest( "happyProject", 3.minutes, networkAuthn, None, None, Set(ResultOutputType.PATIENT_COUNT_XML), queryDefinition) ShrineOrchestrator.signerVerifier.sign(BroadcastMessage(req.networkQueryId, networkAuthn, req), SigningCertStrategy.Attach) } def apply(): Summary = { val message = runQueryRequest val adapterOk: Boolean = ShrineOrchestrator.adapterService.fold(true) { adapterService => import scala.concurrent.duration._ val start = System.currentTimeMillis val resultAttempt: Try[Result] = Try(adapterService.handleRequest(message)) val end = System.currentTimeMillis val elapsed = (end - start).milliseconds resultAttempt match { case scala.util.Failure(cause) => false case scala.util.Success(response) => true } } val hubOk = ShrineOrchestrator.hubComponents.fold(true){ hubComponents => val maxQueryWaitTime = hubComponents.broadcasterMultiplexerService.maxQueryWaitTime val broadcaster: Broadcaster = hubComponents.broadcasterMultiplexerService.broadcaster val message = runQueryRequest - val multiplexer = broadcaster.broadcast(message) - val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq - val failures = responses.collect { case f: Failure => f } - val timeouts = responses.collect { case t: Timeout => t } - val validResults = responses.collect { case r: Result => r } - - failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size ) + val triedMultiplexer = Try(broadcaster.broadcast(message)) + //todo just use fold()() in scala 2.12 + triedMultiplexer.toOption.fold(false) { multiplexer => + val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq + val failures = responses.collect { case f: Failure => f } + val timeouts = responses.collect { case t: Timeout => t } + val validResults = responses.collect { case r: Result => r } + + failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size) + } } val adapterMappingsFileName = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.source) val adapterMappingsVersion = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.version) //todo use this? val noDate:Option[Long] = None val adapterMappingsDate:Option[Long] = adapterMappingsFileName.fold(noDate){ fileName => val file:File = new File(fileName) if(file.exists) Some(file.lastModified()) else None } + val ontologyVersion = try { + ShrineOrchestrator.ontologyMetadata.ontologyVersion + } + catch { + case NonFatal(x) => + Log.info("Problem while getting ontology version",x) + x.getMessage + } + Summary( isHub = ShrineOrchestrator.hubComponents.isDefined, shrineVersion = Versions.version, shrineBuildDate = Versions.buildDate, - ontologyVersion = ShrineOrchestrator.ontologyMetadata.ontologyVersion, + //todo in scala 2.12, do better + ontologyVersion = ontologyVersion, ontologyTerm = term.value, adapterMappingsFileName = adapterMappingsFileName, adapterMappingsDate = adapterMappingsDate, adapterOk = adapterOk, keystoreOk = true, //todo something for this hubOk = hubOk, qepOk = true //todo something for this ) } } case class Version(version:String) //todo SortedMap when possible case class Json4sConfig(keyValues:Map[String,String]) object Json4sConfig{ def isPassword(key:String):Boolean = { if(key.toLowerCase.contains("password")) true else false } def apply(config:TsConfig):Json4sConfig = { val entries: Set[(String, String)] = config.entrySet.asScala.to[Set].map(x => (x.getKey,x.getValue.render())).filterNot(x => isPassword(x._1)) val sortedMap: Map[String, String] = entries.toMap Json4sConfig(sortedMap) } } class PermittedHostOnly extends ContainerRequestFilter { //todo generalize for happy, too //todo for tomcat 8 see https://jersey.java.net/documentation/latest/filters-and-interceptors.html for a cleaner version //shell code from http://stackoverflow.com/questions/17143514/how-to-add-custom-response-and-abort-request-in-jersey-1-11-filters //how to apply in http://stackoverflow.com/questions/4358213/how-does-one-intercept-a-request-during-the-jersey-lifecycle override def filter(requestContext: ContainerRequest): ContainerRequest = { val hostOfOrigin = requestContext.getBaseUri.getHost val shrineConfig:TsConfig = ShrineOrchestrator.config val permittedHostOfOrigin:String = shrineConfig.getOption("shrine.status.permittedHostOfOrigin",_.getString).getOrElse("localhost") val path = requestContext.getPath //happy and internalstatus API calls must come from the same host as tomcat is running on (hopefully the dashboard servlet). // todo access to the happy service permitted for SHRINE 1.21 per SHRINE-1366 // restrict access to happy service when database work resumes as part of SHRINE- // if ((path.contains("happy") || path.contains("internalstatus")) && (hostOfOrigin != permittedHostOfOrigin)) { if (path.contains("internalstatus") && (hostOfOrigin != permittedHostOfOrigin)) { val response = Response.status(Response.Status.UNAUTHORIZED).entity(s"Only available from $permittedHostOfOrigin, not $hostOfOrigin, controlled by shrine.status.permittedHostOfOrigin in shrine.conf").build() throw new WebApplicationException(response) } else requestContext } } \ No newline at end of file diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala deleted file mode 100644 index 6ed87de25..000000000 --- a/apps/shrine-app/src/main/scala/net/shrine/wiring/Jndi.scala +++ /dev/null @@ -1,14 +0,0 @@ -package net.shrine.wiring - -import javax.naming.InitialContext -import scala.util.Try - -/** - * @author clint - * @date Jan 15, 2014 - */ -object Jndi { - def apply[T](name: String): Try[T] = Try { - (new InitialContext).lookup(name).asInstanceOf[T] - } -} \ No newline at end of file diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala index c610d5815..9218e49a9 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala @@ -1,159 +1,159 @@ package net.shrine.wiring import javax.sql.DataSource import com.typesafe.config.{Config, ConfigFactory} import net.shrine.adapter.AdapterComponents import net.shrine.adapter.dao.AdapterDao import net.shrine.adapter.service.{AdapterRequestHandler, AdapterResource, AdapterService, I2b2AdminResource, I2b2AdminService} import net.shrine.broadcaster.dao.HubDao import net.shrine.broadcaster.dao.squeryl.SquerylHubDao import net.shrine.broadcaster.service.HubComponents import net.shrine.client.{EndpointConfig, JerseyHttpClient, OntClient, Poster, PosterOntClient} import net.shrine.config.ConfigExtensions import net.shrine.config.mappings.AdapterMappings import net.shrine.crypto.{DefaultSignerVerifier, KeyStoreCertCollection, KeyStoreDescriptorParser, TrustParam} import net.shrine.dao.squeryl.{DataSourceSquerylInitializer, SquerylDbAdapterSelecter, SquerylInitializer} import net.shrine.happy.{HappyShrineResource, HappyShrineService} import net.shrine.log.Loggable import net.shrine.ont.data.OntClientOntologyMetadata import net.shrine.protocol.{HiveCredentials, NodeId, ResultOutputType, ResultOutputTypes} - import net.shrine.qep.{I2b2BroadcastResource, QueryEntryPointComponents, ShrineResource} +import net.shrine.slick.TestableDataSourceCreator import net.shrine.status.StatusJaxrs import org.squeryl.internals.DatabaseAdapter /** * @author clint * @since Jan 14, 2014 * * Application wiring for Shrine. */ object ShrineOrchestrator extends ShrineJaxrsResources with Loggable { override def resources: Iterable[AnyRef] = { Seq(happyResource,statusJaxrs) ++ shrineResource ++ i2b2BroadcastResource ++ adapterResource ++ i2b2AdminResource ++ hubComponents.map(_.broadcasterMultiplexerResource) } //todo another pass to put things only used in one place into that place's apply(Config) //Load config from file on the classpath called "shrine.conf" lazy val config: Config = ConfigFactory.load("shrine") val shrineConfig = config.getConfig("shrine") protected lazy val nodeId: NodeId = NodeId(shrineConfig.getString("humanReadableNodeName")) //TODO: Don't assume keystore lives on the filesystem, could come from classpath, etc lazy val keyStoreDescriptor = shrineConfig.getConfigured("keystore",KeyStoreDescriptorParser(_)) - lazy val certCollection: KeyStoreCertCollection = KeyStoreCertCollection.fromFile(keyStoreDescriptor) + lazy val certCollection: KeyStoreCertCollection = KeyStoreCertCollection.fromFileRecoverWithClassPath(keyStoreDescriptor) protected lazy val keystoreTrustParam: TrustParam = TrustParam.SomeKeyStore(certCollection) //todo used by the adapterServide and happyShrineService, but not by the QEP. maybe each can have its own signerVerivier lazy val signerVerifier: DefaultSignerVerifier = new DefaultSignerVerifier(certCollection) - protected lazy val dataSource: DataSource = Jndi("java:comp/env/jdbc/shrineDB").get + protected lazy val dataSource: DataSource = TestableDataSourceCreator.dataSource(shrineConfig.getConfig("squerylDataSource.database")) protected lazy val squerylAdapter: DatabaseAdapter = SquerylDbAdapterSelecter.determineAdapter(shrineConfig.getString("shrineDatabaseType")) protected lazy val squerylInitializer: SquerylInitializer = new DataSourceSquerylInitializer(dataSource, squerylAdapter) //todo it'd be better for the adapter and qep to each have its own connection to the pm cell. private lazy val pmEndpoint: EndpointConfig = shrineConfig.getConfigured("pmEndpoint", EndpointConfig(_)) lazy val pmPoster: Poster = Poster(certCollection,pmEndpoint) protected lazy val breakdownTypes: Set[ResultOutputType] = shrineConfig.getOptionConfigured("breakdownResultOutputTypes", ResultOutputTypes.fromConfig).getOrElse(Set.empty) //todo why does the qep need a HubDao ? protected lazy val hubDao: HubDao = new SquerylHubDao(squerylInitializer, new net.shrine.broadcaster.dao.squeryl.tables.Tables) //todo really should be part of the adapter config, but is out in shrine's part of the name space lazy val crcHiveCredentials: HiveCredentials = shrineConfig.getConfigured("hiveCredentials", HiveCredentials(_, HiveCredentials.CRC)) val adapterComponents:Option[AdapterComponents] = shrineConfig.getOptionConfiguredIf("adapter", AdapterComponents( _, certCollection, squerylInitializer, breakdownTypes, crcHiveCredentials, signerVerifier, pmPoster, nodeId )) //todo maybe just break demeter too use this lazy val adapterService: Option[AdapterService] = adapterComponents.map(_.adapterService) //todo maybe just break demeter too use this lazy val i2b2AdminService: Option[I2b2AdminService] = adapterComponents.map(_.i2b2AdminService) //todo this is only used by happy lazy val adapterDao: Option[AdapterDao] = adapterComponents.map(_.adapterDao) //todo this is only used by happy lazy val adapterMappings: Option[AdapterMappings] = adapterComponents.map(_.adapterMappings) val shouldQuerySelf = "hub.shouldQuerySelf" lazy val localAdapterServiceOption: Option[AdapterRequestHandler] = if(shrineConfig.getOption(shouldQuerySelf,_.getBoolean).getOrElse(false)) { //todo give this a default value (of false) in the reference.conf for the Hub, and make it part of the Hub's apply(config) require(adapterService.isDefined, s"Self-querying requested because shrine.$shouldQuerySelf is true, but this node is not configured to have an adapter") adapterService } else None //todo eventually make this just another downstream node accessed via loopback val hubConfig = shrineConfig.getConfig("hub") lazy val hubComponents: Option[HubComponents] = shrineConfig.getOptionConfiguredIf("hub",HubComponents(_, keystoreTrustParam, nodeId, localAdapterServiceOption, breakdownTypes, hubDao )) //todo anything that requires qepConfig should be inside QueryEntryPointComponents's apply protected lazy val qepConfig = shrineConfig.getConfig("queryEntryPoint") lazy val queryEntryPointComponents:Option[QueryEntryPointComponents] = shrineConfig.getOptionConfiguredIf("queryEntryPoint",QueryEntryPointComponents(_, certCollection, breakdownTypes, hubComponents.map(_.broadcastDestinations), hubDao, //todo the QEP should not need the hub dao squerylInitializer, //todo could really have its own pmPoster //todo could really have its own )) protected lazy val pmUrlString: String = pmEndpoint.url.toString private lazy val ontEndpoint: EndpointConfig = shrineConfig.getConfigured("ontEndpoint", EndpointConfig(_)) protected lazy val ontPoster: Poster = Poster(certCollection,ontEndpoint) //todo only used by happy outside of here lazy val ontologyMetadata: OntClientOntologyMetadata = { import scala.concurrent.duration._ //TODO: XXX: Un-hard-code max wait time param val ontClient: OntClient = new PosterOntClient(shrineConfig.getConfigured("hiveCredentials", HiveCredentials(_, HiveCredentials.ONT)), 1.minute, ontPoster) new OntClientOntologyMetadata(ontClient) } protected lazy val happyResource: HappyShrineResource = new HappyShrineResource(HappyShrineService) protected lazy val statusJaxrs: StatusJaxrs = StatusJaxrs(config) protected lazy val shrineResource: Option[ShrineResource] = queryEntryPointComponents.map(x => ShrineResource(x.shrineService)) protected lazy val i2b2BroadcastResource: Option[I2b2BroadcastResource] = queryEntryPointComponents.map(x => new I2b2BroadcastResource(x.i2b2Service,breakdownTypes)) protected lazy val adapterResource: Option[AdapterResource] = adapterService.map(AdapterResource(_)) protected lazy val i2b2AdminResource: Option[I2b2AdminResource] = i2b2AdminService.map(I2b2AdminResource(_, breakdownTypes)) def poster(keystoreCertCollection: KeyStoreCertCollection)(endpoint: EndpointConfig): Poster = { val httpClient = JerseyHttpClient(keystoreCertCollection, endpoint) Poster(endpoint.url.toString, httpClient) } } diff --git a/apps/shrine-app/src/test/resources/shrine.conf b/apps/shrine-app/src/test/resources/shrine.conf new file mode 100644 index 000000000..26cf640ca --- /dev/null +++ b/apps/shrine-app/src/test/resources/shrine.conf @@ -0,0 +1,68 @@ +shrine { + + keystore { + file = "shrine.keystore" + password = "chiptesting" + privateKeyAlias = "test-cert" + keyStoreType = "JKS" + caCertAliases = [carra ca] + } + + queryEntryPoint { + audit { + collectQepAudit = false + + database { + slickProfileClassName = "slick.driver.H2Driver$" + createTablesOnStart = true //for testing with H2 in memory, when not running unit tests. Set to false normally + + dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else + + testDataSource { + driverClassName = "org.h2.Driver" + url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace + } + } + } + } + + adapter { + create = true + audit { + collectQepAudit = false + + database { + slickProfileClassName = "slick.driver.H2Driver$" + createTablesOnStart = true //for testing with H2 in memory, when not running unit tests. Set to false normally + + dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else + + testDataSource { + driverClassName = "org.h2.Driver" + url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace + } + } + } + } + + squerylDataSource { + database { + dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else + testDataSource { + driverClassName = "org.h2.Driver" + url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace + } + } + } +// squerylDataSource { +// database { +// dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else +// +// testDataSource { +// driverClassName = "org.h2.Driver" +// url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace +// } +// } +// } + +} \ No newline at end of file diff --git a/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala b/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala index 4b7c7ff40..aaa0e7e35 100644 --- a/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala +++ b/apps/shrine-app/src/test/scala/net/shrine/status/StatusJaxrsTest.scala @@ -1,63 +1,63 @@ package net.shrine.status import com.typesafe.config.ConfigFactory import net.shrine.util.ShouldMatchersForJUnit import org.json4s.{DefaultFormats, Formats} import org.junit.Test import org.json4s.native.Serialization import scala.collection.immutable.Map /** * Tests for StatusJaxrs * * @author david * @since 12/2/15 */ class StatusJaxrsTest extends ShouldMatchersForJUnit { implicit def json4sFormats: Formats = DefaultFormats val expectedConfig = ConfigFactory.load("shrine") val statusJaxrs = StatusJaxrs(expectedConfig) @Test def testVersion() = { val versionString = statusJaxrs.version val version = Serialization.read[Version](versionString) version should equal(Version("changeMe")) } @Test def testConfig() = { val expectedJson4sConfig = Json4sConfig(expectedConfig) val configString = statusJaxrs.config val config = Serialization.read[Json4sConfig](configString) config should equal(expectedJson4sConfig) val passwordKeys = config.keyValues.filter(x => Json4sConfig.isPassword(x._1)) passwordKeys should equal(Map.empty[String,String]) } - /* todo need some way to start Shrine for this to work @Test def testSummary() = { -// val expectedJson4sConfig = Json4sConfig(expectedConfig) val summaryString = statusJaxrs.summary println(summaryString) val summary = Serialization.read[Summary](summaryString) -// config should equal(expectedJson4sConfig) - - println(summary) - + summary.isHub should be (true) + summary.adapterMappingsFileName.isDefined should be (true) + summary.adapterMappingsDate.isEmpty should be (true) + summary.adapterOk should be (true) + summary.keystoreOk should be (true) + summary.hubOk should be (false) + summary.qepOk should be (true) } - */ } diff --git a/apps/war/src/main/resources/reference.conf b/apps/war/src/main/resources/reference.conf index d5f8cf46f..84c38b8dc 100644 --- a/apps/war/src/main/resources/reference.conf +++ b/apps/war/src/main/resources/reference.conf @@ -1,15 +1,21 @@ shrine { status { //permittedHostOfOrigin = "localhost" //If absent then get the host name via java.net.InetAddress.getLocalHost.getHostName . Override to control } + squerylDataSource{ + database { + dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else + jndiDataSourceName = "java:comp/env/jdbc/shrineDB" //or leave out for tests + } + } } akka { loglevel = INFO // log-config-on-start = on loggers = ["akka.event.slf4j.Slf4jLogger"] // logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" // Toggles whether the threads created by this ActorSystem should be daemons or not daemonic = on } \ No newline at end of file diff --git a/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala b/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala index 6dc1f4045..f36b2d96e 100644 --- a/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala +++ b/hub/broadcaster-aggregator/src/main/scala/net/shrine/broadcaster/Multiplexer.scala @@ -1,15 +1,15 @@ package net.shrine.broadcaster import net.shrine.protocol.SingleNodeResult -import scala.concurrent.duration.Duration + import scala.concurrent.Future /** * @author clint - * @date Nov 15, 2013 + * @since Nov 15, 2013 */ trait Multiplexer { def processResponse(response: SingleNodeResult): Unit def responses: Future[Iterable[SingleNodeResult]] } \ No newline at end of file diff --git a/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala b/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala index 7342249fc..ca8f92f88 100644 --- a/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala +++ b/integration/src/test/scala/net/shrine/integration/NetworkSimulationTest.scala @@ -1,337 +1,337 @@ package net.shrine.integration import java.net.URL import net.shrine.log.Loggable import scala.concurrent.Future import scala.concurrent.duration.DurationInt import org.junit.Test import net.shrine.util.ShouldMatchersForJUnit import net.shrine.adapter.AdapterMap import net.shrine.adapter.DeleteQueryAdapter import net.shrine.adapter.client.AdapterClient import net.shrine.adapter.dao.squeryl.AbstractSquerylAdapterTest import net.shrine.adapter.service.AdapterRequestHandler import net.shrine.adapter.service.AdapterService import net.shrine.broadcaster.AdapterClientBroadcaster import net.shrine.broadcaster.NodeHandle import net.shrine.crypto.DefaultSignerVerifier import net.shrine.crypto.TestKeystore import net.shrine.protocol.{HiveCredentials, AuthenticationInfo, BroadcastMessage, Credential, DeleteQueryRequest, DeleteQueryResponse, NodeId, Result, RunQueryRequest, CertId, RequestType, FlagQueryRequest, FlagQueryResponse, RawCrcRunQueryResponse, ResultOutputType, QueryResult, RunQueryResponse, AggregatedRunQueryResponse, UnFlagQueryRequest, UnFlagQueryResponse, DefaultBreakdownResultOutputTypes} import net.shrine.qep.QepService import net.shrine.broadcaster.SigningBroadcastAndAggregationService import net.shrine.broadcaster.InJvmBroadcasterClient import net.shrine.adapter.FlagQueryAdapter import net.shrine.protocol.query.Term import net.shrine.adapter.RunQueryAdapter import net.shrine.client.Poster import net.shrine.client.HttpClient import net.shrine.client.HttpResponse import net.shrine.adapter.translators.QueryDefinitionTranslator import net.shrine.adapter.translators.ExpressionTranslator import net.shrine.util.XmlDateHelper import net.shrine.adapter.ReadQueryResultAdapter import net.shrine.protocol.query.QueryDefinition import net.shrine.adapter.UnFlagQueryAdapter import net.shrine.crypto.SigningCertStrategy /** * @author clint * @since Nov 27, 2013 * - * An in-JVM simulation of a Shrine network with one hub and 4 doanstream adapters. + * An in-JVM simulation of a Shrine network with one hub and 4 downstream adapters. * * The hub and adapters are wired up with mock AdapterClients that do in-JVM communication via method calls * instead of remotely. * * The adapters are configured to respond with valid results for DeleteQueryRequests * only. Other requests could be handled, but that would not provide benefit to offset the effort of wiring * up more and more-complex Adapters. * * The test network is queried, and the final result, as well as the state of each adapter, is inspected to * ensure that the right messages were sent between elements of the system. * */ final class NetworkSimulationTest extends AbstractSquerylAdapterTest with ShouldMatchersForJUnit { private val certCollection = TestKeystore.certCollection private lazy val myCertId: CertId = certCollection.myCertId.get private lazy val signerVerifier = new DefaultSignerVerifier(certCollection) private val domain = "test-domain" private val username = "test-username" private val password = "test-password" import NetworkSimulationTest._ import scala.concurrent.duration._ private def deleteQueryAdapter: DeleteQueryAdapter = new DeleteQueryAdapter(dao) private def flagQueryAdapter: FlagQueryAdapter = new FlagQueryAdapter(dao) private def unFlagQueryAdapter: UnFlagQueryAdapter = new UnFlagQueryAdapter(dao) private def mockPoster = Poster("http://example.com", new HttpClient { override def post(input: String, url: String): HttpResponse = ??? }) private val hiveCredentials = HiveCredentials("d", "u", "pwd", "pid") private def queuesQueriesRunQueryAdapter: RunQueryAdapter = { val translator = new QueryDefinitionTranslator(new ExpressionTranslator(Map("n1" -> Set("l1")))) RunQueryAdapter( mockPoster, dao, hiveCredentials, translator, 10000, doObfuscation = false, runQueriesImmediately = false, DefaultBreakdownResultOutputTypes.toSet, collectAdapterAudit = false ) } private def immediatelyRunsQueriesRunQueryAdapter(setSize: Long): RunQueryAdapter = { val mockCrcPoster = Poster("http://example.com", new HttpClient { override def post(input: String, url: String): HttpResponse = { val req = RunQueryRequest.fromI2b2String(DefaultBreakdownResultOutputTypes.toSet)(input).get val now = XmlDateHelper.now val queryResult = QueryResult(1L, 42L, Some(ResultOutputType.PATIENT_COUNT_XML), setSize, Some(now), Some(now), Some("desc"), QueryResult.StatusType.Finished, Some("status")) val mockCrcXml = RawCrcRunQueryResponse(req.networkQueryId, XmlDateHelper.now, req.authn.username, req.projectId, req.queryDefinition, 42L, Map(ResultOutputType.PATIENT_COUNT_XML -> Seq(queryResult))).toI2b2String HttpResponse.ok(mockCrcXml) } }) queuesQueriesRunQueryAdapter.copy(poster = mockCrcPoster, runQueriesImmediately = true) } private def readQueryResultAdapter(setSize: Long): ReadQueryResultAdapter = { new ReadQueryResultAdapter( mockPoster, hiveCredentials, dao, doObfuscation = false, DefaultBreakdownResultOutputTypes.toSet, collectAdapterAudit = false ) } private lazy val adaptersByNodeId: Seq[(NodeId, MockAdapterRequestHandler)] = { import NodeName._ import RequestType.{ MasterDeleteRequest => MasterDeleteRequestRT, FlagQueryRequest => FlagQueryRequestRT, QueryDefinitionRequest => RunQueryRT, GetQueryResult => ReadQueryResultRT, UnFlagQueryRequest => UnFlagQueryRequestRT } (for { (childName, setSize) <- Seq((A, 1L), (B, 2L), (C, 3L), (D, 4L)) } yield { val nodeId = NodeId(childName.name) val maxSignatureAge = 1.hour val adapterMap = AdapterMap(Map( MasterDeleteRequestRT -> deleteQueryAdapter, FlagQueryRequestRT -> flagQueryAdapter, UnFlagQueryRequestRT -> unFlagQueryAdapter, RunQueryRT -> queuesQueriesRunQueryAdapter, ReadQueryResultRT -> readQueryResultAdapter(setSize))) nodeId -> MockAdapterRequestHandler(new AdapterService(nodeId, signerVerifier, maxSignatureAge, adapterMap)) }) } private lazy val shrineService: QepService = { val destinations: Set[NodeHandle] = { (for { (nodeId, adapterRequestHandler) <- adaptersByNodeId } yield { NodeHandle(nodeId, MockAdapterClient(nodeId, adapterRequestHandler)) }).toSet } QepService( "example.com", MockAuditDao, MockAuthenticator, MockQueryAuthorizationService, true, SigningBroadcastAndAggregationService(InJvmBroadcasterClient(AdapterClientBroadcaster(destinations, MockHubDao)), signerVerifier, SigningCertStrategy.Attach), 1.hour, DefaultBreakdownResultOutputTypes.toSet, false) } @Test def testSimulatedNetwork = afterCreatingTables { val authn = AuthenticationInfo(domain, username, Credential(password, false)) val masterId = 12345L import scala.concurrent.duration._ val req = DeleteQueryRequest("some-project-id", 1.second, authn, masterId) val resp = shrineService.deleteQuery(req, true) for { (nodeId, mockAdapter) <- adaptersByNodeId } { mockAdapter.lastMessage.networkAuthn.domain should equal(authn.domain) mockAdapter.lastMessage.networkAuthn.username should equal(authn.username) mockAdapter.lastMessage.request should equal(req) mockAdapter.lastResult.response should equal(DeleteQueryResponse(masterId)) } resp should equal(DeleteQueryResponse(masterId)) } @Test def testQueueQuery = afterCreatingTables { val authn = AuthenticationInfo(domain, username, Credential(password, false)) val topicId = "askldjlkas" val topicName = "Topic Name" val queryName = "lsadj3028940" import scala.concurrent.duration._ val runQueryReq = RunQueryRequest("some-project-id", 1.second, authn, 12345L, Some(topicId), Some(topicName), Set(ResultOutputType.PATIENT_COUNT_XML), QueryDefinition(queryName, Term("n1"))) val aggregatedRunQueryResp = shrineService.runQuery(runQueryReq, true).asInstanceOf[AggregatedRunQueryResponse] var broadcastMessageId: Option[Long] = None //Broadcast the original run query request; all nodes should queue the query for { (nodeId, mockAdapter) <- adaptersByNodeId } { broadcastMessageId = Option(mockAdapter.lastMessage.requestId) mockAdapter.lastMessage.networkAuthn.domain should equal(authn.domain) mockAdapter.lastMessage.networkAuthn.username should equal(authn.username) val lastReq = mockAdapter.lastMessage.request.asInstanceOf[RunQueryRequest] lastReq.authn should equal(runQueryReq.authn) lastReq.requestType should equal(runQueryReq.requestType) lastReq.waitTime should equal(runQueryReq.waitTime) //todo what to do with this check? lastReq.networkQueryId should equal(mockAdapter.lastMessage.requestId) lastReq.outputTypes should equal(runQueryReq.outputTypes) lastReq.projectId should equal(runQueryReq.projectId) lastReq.queryDefinition should equal(runQueryReq.queryDefinition) lastReq.topicId should equal(runQueryReq.topicId) val runQueryResp = mockAdapter.lastResult.response.asInstanceOf[RunQueryResponse] runQueryResp.queryId should equal(-1L) runQueryResp.singleNodeResult.statusType should equal(QueryResult.StatusType.Held) runQueryResp.singleNodeResult.setSize should equal(-1L) } aggregatedRunQueryResp.queryId should equal(broadcastMessageId.get) aggregatedRunQueryResp.results.map(_.setSize) should equal(Seq(-1L, -1L, -1L, -1L, -4L)) } @Test def testFlagQuery = afterCreatingTables { val authn = AuthenticationInfo(domain, username, Credential(password, false)) val masterId = 12345L import scala.concurrent.duration._ val networkQueryId = 9999L val name = "some query" val expr = Term("foo") val fooQuery = QueryDefinition(name,expr) dao.insertQuery(masterId.toString, networkQueryId, authn, fooQuery, isFlagged = false, hasBeenRun = true, flagMessage = None) dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(false) dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(None) val req = FlagQueryRequest("some-project-id", 1.second, authn, networkQueryId, Some("foo")) val resp = shrineService.flagQuery(req, true) resp should equal(FlagQueryResponse) dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(true) dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(Some("foo")) } @Test def testUnFlagQuery = afterCreatingTables { val authn = AuthenticationInfo(domain, username, Credential(password, false)) val masterId = 12345L import scala.concurrent.duration._ val networkQueryId = 9999L val flagMsg = Some("foo") val name = "some query" val expr = Term("foo") val fooQuery = QueryDefinition(name,expr) dao.insertQuery(masterId.toString, networkQueryId, authn, fooQuery, isFlagged = true, hasBeenRun = true, flagMessage = flagMsg) dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(true) dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(flagMsg) val req = UnFlagQueryRequest("some-project-id", 1.second, authn, networkQueryId) val resp = shrineService.unFlagQuery(req, true) resp should equal(UnFlagQueryResponse) dao.findQueryByNetworkId(networkQueryId).get.isFlagged should be(false) dao.findQueryByNetworkId(networkQueryId).get.flagMessage should be(None) } } object NetworkSimulationTest { private final case class MockAdapterClient(nodeId: NodeId, adapter: AdapterRequestHandler) extends AdapterClient with Loggable { import scala.concurrent.ExecutionContext.Implicits.global override def query(message: BroadcastMessage): Future[Result] = Future.successful { debug(s"Invoking Adapter $nodeId with $message") val result = adapter.handleRequest(message) debug(s"Got result from $nodeId: $result") result } override def url: Option[URL] = ??? } private final case class MockAdapterRequestHandler(delegate: AdapterRequestHandler) extends AdapterRequestHandler { @volatile var lastMessage: BroadcastMessage = _ @volatile var lastResult: Result = _ override def handleRequest(request: BroadcastMessage): Result = { lastMessage = request val result = delegate.handleRequest(request) lastResult = result result } } }