diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineConfig.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineConfig.scala index 93533f3a5..6a5badb7e 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineConfig.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineConfig.scala @@ -1,49 +1,48 @@ package net.shrine.wiring import com.typesafe.config.Config import net.shrine.adapter.service.AdapterConfig import net.shrine.broadcaster.HubConfig import net.shrine.config.{ConfigExtensions, Keys} import net.shrine.protocol.{ResultOutputTypes, HiveCredentials, ResultOutputType} import net.shrine.qep.QepConfig /** * @author clint * @since Feb 6, 2013 */ final case class ShrineConfig( adapterConfig: Option[AdapterConfig], hubConfig: Option[HubConfig], queryEntryPointConfig: Option[QepConfig], crcHiveCredentials: HiveCredentials, ontHiveCredentials: HiveCredentials, - adapterStatusQuery: String, - breakdownResultOutputTypes: Set[ResultOutputType]) { + adapterStatusQuery: String + ) { //NB: Preparing for the possible case where we'd need distinct credentials for talking to the PM def pmHiveCredentials: HiveCredentials = crcHiveCredentials } object ShrineConfig { def apply(config: Config): ShrineConfig = { val configForShrine = config.getConfig("shrine") import Keys._ def getOptionConfiguredIf[T](key:String,constructor: Config => T):Option[T] = { if(configForShrine.getBoolean(s"$key.create")) configForShrine.getOptionConfigured(key,constructor) else None } ShrineConfig( adapterConfig = getOptionConfiguredIf(adapter, AdapterConfig(_)), hubConfig = getOptionConfiguredIf(hub, HubConfig(_)), queryEntryPointConfig = getOptionConfiguredIf(queryEntryPoint, QepConfig(_)), crcHiveCredentials = configForShrine.getConfigured(hiveCredentials, HiveCredentials(_, HiveCredentials.CRC)), ontHiveCredentials = configForShrine.getConfigured(hiveCredentials, HiveCredentials(_, HiveCredentials.ONT)), - adapterStatusQuery = configForShrine.getString(networkStatusQuery), - breakdownResultOutputTypes = configForShrine.getOptionConfigured(breakdownResultOutputTypes, ResultOutputTypes.fromConfig).getOrElse(Set.empty) + adapterStatusQuery = configForShrine.getString(networkStatusQuery) ) } } diff --git a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala index 72b256ee6..557a154f9 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/wiring/ShrineOrchestrator.scala @@ -1,347 +1,347 @@ package net.shrine.wiring import javax.sql.DataSource import com.typesafe.config.{Config, ConfigFactory} import net.shrine.adapter.dao.squeryl.tables.{Tables => AdapterTables} import net.shrine.adapter.dao.squeryl.{SquerylAdapterDao, SquerylI2b2AdminDao} import net.shrine.adapter.dao.{AdapterDao, I2b2AdminDao} import net.shrine.adapter.service.{AdapterRequestHandler, AdapterResource, AdapterService, I2b2AdminResource, I2b2AdminService} import net.shrine.adapter.translators.{ExpressionTranslator, QueryDefinitionTranslator} import net.shrine.adapter.{Adapter, AdapterMap, DeleteQueryAdapter, FlagQueryAdapter, ReadInstanceResultsAdapter, ReadPreviousQueriesAdapter, ReadQueryDefinitionAdapter, ReadQueryResultAdapter, ReadTranslatedQueryDefinitionAdapter, RenameQueryAdapter, RunQueryAdapter, UnFlagQueryAdapter} import net.shrine.authentication.Authenticator import net.shrine.authorization.QueryAuthorizationService import net.shrine.broadcaster.dao.HubDao import net.shrine.broadcaster.dao.squeryl.SquerylHubDao import net.shrine.broadcaster.service.{BroadcasterMultiplexerResource, BroadcasterMultiplexerService} import net.shrine.broadcaster.{AdapterClientBroadcaster, BroadcastAndAggregationService, BroadcasterClient, InJvmBroadcasterClient, NodeHandle, PosterBroadcasterClient, SigningBroadcastAndAggregationService} import net.shrine.client.{EndpointConfig, HttpClient, JerseyHttpClient, OntClient, Poster, PosterOntClient} import net.shrine.config.ConfigExtensions import net.shrine.config.Keys._ import net.shrine.config.mappings.{AdapterMappings, AdapterMappingsSource, ClasspathFormatDetectingAdapterMappingsSource} import net.shrine.crypto.{DefaultSignerVerifier, KeyStoreCertCollection, KeyStoreDescriptorParser, TrustParam} import net.shrine.dao.squeryl.{DataSourceSquerylInitializer, SquerylDbAdapterSelecter, SquerylInitializer} import net.shrine.happy.{HappyShrineResource, HappyShrineService} import net.shrine.log.Loggable import net.shrine.ont.data.{OntClientOntologyMetadata, OntologyMetadata} -import net.shrine.protocol.{NodeId, RequestType, ResultOutputType} +import net.shrine.protocol.{ResultOutputTypes, NodeId, RequestType, ResultOutputType} import net.shrine.qep.dao.AuditDao import net.shrine.qep.dao.squeryl.SquerylAuditDao import net.shrine.qep.dao.squeryl.tables.{Tables => HubTables} import net.shrine.qep.{I2b2BroadcastResource, I2b2QepService, QepService, ShrineResource} import net.shrine.status.StatusJaxrs import org.squeryl.internals.DatabaseAdapter /** * @author clint * @since Jan 14, 2014 * * Application wiring for Shrine, in the base, non-HMS case. All vals are protecetd, so they may be accessed, * in subclasses without ballooning this class's public API, and lazy, to work around init-order surprises when * overriding vals declared inline. See * * https://stackoverflow.com/questions/15762650/scala-override-val-in-class-inheritance * * among other links mentioning val overrides, early initializers, etc. -Clint */ object ShrineOrchestrator extends ShrineJaxrsResources with Loggable { import NodeHandleSource.makeNodeHandles override def resources: Iterable[AnyRef] = { Seq(happyResource,statusJaxrs) ++ shrineResource ++ i2b2BroadcastResource ++ adapterResource ++ i2b2AdminResource ++ broadcasterMultiplexerResource } //Load config from file on the classpath called "shrine.conf" lazy val config: Config = ConfigFactory.load("shrine") protected lazy val shrineConfigurationBall: ShrineConfig = ShrineConfig(config) val shrineConfig = config.getConfig("shrine") protected lazy val nodeId: NodeId = NodeId(shrineConfig.getString("humanReadableNodeName")) //TODO: Don't assume keystore lives on the filesystem, could come from classpath, etc protected lazy val keyStoreDescriptor = shrineConfig.getConfigured("keystore",KeyStoreDescriptorParser(_)) protected lazy val shrineCertCollection: KeyStoreCertCollection = KeyStoreCertCollection.fromFile(keyStoreDescriptor) protected lazy val keystoreTrustParam: TrustParam = TrustParam.SomeKeyStore(shrineCertCollection) protected lazy val signerVerifier: DefaultSignerVerifier = new DefaultSignerVerifier(shrineCertCollection) protected lazy val dataSource: DataSource = Jndi("java:comp/env/jdbc/shrineDB").get protected lazy val squerylAdapter: DatabaseAdapter = SquerylDbAdapterSelecter.determineAdapter(shrineConfig.getString("shrineDatabaseType")) protected lazy val squerylInitializer: SquerylInitializer = new DataSourceSquerylInitializer(dataSource, squerylAdapter) private def makePoster = poster(shrineCertCollection) _ private lazy val pmEndpoint: EndpointConfig = shrineConfig.getConfigured("pmEndpoint", EndpointConfig(_)) protected lazy val pmPoster: Poster = makePoster(pmEndpoint) private lazy val ontEndpoint: EndpointConfig = shrineConfig.getConfigured("ontEndpoint", EndpointConfig(_)) protected lazy val ontPoster: Poster = makePoster(ontEndpoint) - protected lazy val breakdownTypes: Set[ResultOutputType] = shrineConfigurationBall.breakdownResultOutputTypes + protected lazy val breakdownTypes: Set[ResultOutputType] = shrineConfig.getOptionConfigured("breakdownResultOutputTypes", ResultOutputTypes.fromConfig).getOrElse(Set.empty) protected lazy val hubDao: HubDao = new SquerylHubDao(squerylInitializer, new net.shrine.broadcaster.dao.squeryl.tables.Tables) //todo move as much of this block as possible to the adapter project protected lazy val (adapterService, i2b2AdminService, adapterDao, adapterMappings) = adapterComponentsToTuple(shrineConfigurationBall.adapterConfig.map { adapterConfig => val crcEndpoint: EndpointConfig = adapterConfig.crcEndpoint val crcPoster: Poster = makePoster(crcEndpoint) val squerylAdapterTables: AdapterTables = new AdapterTables val adapterDao: AdapterDao = new SquerylAdapterDao(squerylInitializer, squerylAdapterTables)(breakdownTypes) //NB: Is i2b2HiveCredentials.projectId the right project id to use? val i2b2AdminDao: I2b2AdminDao = new SquerylI2b2AdminDao(shrineConfigurationBall.crcHiveCredentials.projectId, squerylInitializer, squerylAdapterTables) val adapterMappingsSource: AdapterMappingsSource = ClasspathFormatDetectingAdapterMappingsSource(adapterConfig.adapterMappingsFileName) //NB: Fail fast val adapterMappings: AdapterMappings = adapterMappingsSource.load.get val expressionTranslator: ExpressionTranslator = ExpressionTranslator(adapterMappings) val queryDefinitionTranslator: QueryDefinitionTranslator = new QueryDefinitionTranslator(expressionTranslator) val doObfuscation = adapterConfig.setSizeObfuscation val runQueryAdapter = new RunQueryAdapter( crcPoster, adapterDao, shrineConfigurationBall.crcHiveCredentials, queryDefinitionTranslator, adapterConfig.adapterLockoutAttemptsThreshold, doObfuscation, adapterConfig.immediatelyRunIncomingQueries, breakdownTypes, collectAdapterAudit = adapterConfig.collectAdapterAudit ) val readInstanceResultsAdapter: Adapter = new ReadInstanceResultsAdapter( crcPoster, shrineConfigurationBall.crcHiveCredentials, adapterDao, doObfuscation, breakdownTypes, collectAdapterAudit = adapterConfig.collectAdapterAudit ) val readQueryResultAdapter: Adapter = new ReadQueryResultAdapter( crcPoster, shrineConfigurationBall.crcHiveCredentials, adapterDao, doObfuscation, breakdownTypes, collectAdapterAudit = adapterConfig.collectAdapterAudit ) val readPreviousQueriesAdapter: Adapter = new ReadPreviousQueriesAdapter(adapterDao) val deleteQueryAdapter: Adapter = new DeleteQueryAdapter(adapterDao) val renameQueryAdapter: Adapter = new RenameQueryAdapter(adapterDao) val readQueryDefinitionAdapter: Adapter = new ReadQueryDefinitionAdapter(adapterDao) val readTranslatedQueryDefinitionAdapter: Adapter = new ReadTranslatedQueryDefinitionAdapter(nodeId, queryDefinitionTranslator) val flagQueryAdapter: Adapter = new FlagQueryAdapter(adapterDao) val unFlagQueryAdapter: Adapter = new UnFlagQueryAdapter(adapterDao) val adapterMap = AdapterMap(Map( RequestType.QueryDefinitionRequest -> runQueryAdapter, RequestType.GetRequestXml -> readQueryDefinitionAdapter, RequestType.UserRequest -> readPreviousQueriesAdapter, RequestType.InstanceRequest -> readInstanceResultsAdapter, RequestType.MasterDeleteRequest -> deleteQueryAdapter, RequestType.MasterRenameRequest -> renameQueryAdapter, RequestType.GetQueryResult -> readQueryResultAdapter, RequestType.ReadTranslatedQueryDefinitionRequest -> readTranslatedQueryDefinitionAdapter, RequestType.FlagQueryRequest -> flagQueryAdapter, RequestType.UnFlagQueryRequest -> unFlagQueryAdapter)) AdapterComponents( new AdapterService(nodeId, signerVerifier, adapterConfig.maxSignatureAge, adapterMap), new I2b2AdminService(adapterDao, i2b2AdminDao, pmPoster, runQueryAdapter), adapterDao, adapterMappings) }) private lazy val localAdapterServiceOption: Option[AdapterRequestHandler] = shrineConfigurationBall.hubConfig.flatMap(hubConfig => makeAdapterServiceOption(hubConfig.shouldQuerySelf, adapterService)) private lazy val broadcastDestinations: Option[Set[NodeHandle]] = shrineConfigurationBall.hubConfig.map(hubConfig => makeNodeHandles(keystoreTrustParam, hubConfig.maxQueryWaitTime, hubConfig.downstreamNodes, nodeId, localAdapterServiceOption, breakdownTypes)) protected lazy val (shrineService, i2b2Service, auditDao) = queryEntryPointComponentsToTuple(shrineConfigurationBall.queryEntryPointConfig.map { queryEntryPointConfig => val broadcasterClient: BroadcasterClient = { if(queryEntryPointConfig.broadcasterIsLocal) { //If broadcaster is local, we need a hub config //TODO: Enforce this when unmarshalling configs require(broadcastDestinations.isDefined, s"The QEP's config implied a local hub (no broadcasterServiceEndpoint), but either no downstream nodes were configured, the hub was not configured, or the hub's configuration specified not to create it.") val broadcaster: AdapterClientBroadcaster = AdapterClientBroadcaster(broadcastDestinations.get, hubDao) InJvmBroadcasterClient(broadcaster) } else { //if broadcaster is remote, we need an endpoint //TODO: Enforce this when unmarshalling configs require(queryEntryPointConfig.broadcasterServiceEndpoint.isDefined, "Non-local broadcaster requested, but no URL for the remote broadcaster is specified") PosterBroadcasterClient(makePoster(queryEntryPointConfig.broadcasterServiceEndpoint.get), breakdownTypes) } } val commonName:String = shrineCertCollection.myCommonName.getOrElse{ val hostname = java.net.InetAddress.getLocalHost.getHostName warn(s"No common name available from ${shrineCertCollection.descriptor}. Using $hostname instead.") hostname } val broadcastService: BroadcastAndAggregationService = SigningBroadcastAndAggregationService(broadcasterClient, signerVerifier, queryEntryPointConfig.signingCertStrategy) val auditDao: AuditDao = new SquerylAuditDao(squerylInitializer, new HubTables) val authenticationType = queryEntryPointConfig.authenticationType val authorizationType = queryEntryPointConfig.authorizationType val authenticator: Authenticator = AuthStrategy.determineAuthenticator(authenticationType, pmPoster) val authorizationService: QueryAuthorizationService = AuthStrategy.determineQueryAuthorizationService(authorizationType, shrineConfigurationBall, authenticator) debug(s"authorizationService set to $authorizationService") QueryEntryPointComponents( QepService( commonName, auditDao, authenticator, authorizationService, queryEntryPointConfig.includeAggregateResults, broadcastService, queryEntryPointConfig.maxQueryWaitTime, breakdownTypes, queryEntryPointConfig.collectQepAudit ), I2b2QepService( commonName, auditDao, authenticator, authorizationService, queryEntryPointConfig.includeAggregateResults, broadcastService, queryEntryPointConfig.maxQueryWaitTime, breakdownTypes, queryEntryPointConfig.collectQepAudit ), auditDao) }) private lazy val broadcasterOption = unpackHubComponents { for { hubConfig <- shrineConfigurationBall.hubConfig } yield { require(broadcastDestinations.isDefined, "This node is configured to be a hub, but no downstream nodes are defined") HubComponents(AdapterClientBroadcaster(broadcastDestinations.get, hubDao)) } } protected lazy val broadcasterMultiplexerService = { for { broadcaster <- broadcasterOption hubConfig <- shrineConfigurationBall.hubConfig } yield { BroadcasterMultiplexerService(broadcaster, hubConfig.maxQueryWaitTime) } } protected lazy val pmUrlString: String = pmEndpoint.url.toString protected lazy val ontologyMetadata: OntologyMetadata = { import scala.concurrent.duration._ //TODO: XXX: Un-hard-code max wait time param val ontClient: OntClient = new PosterOntClient(shrineConfigurationBall.ontHiveCredentials, 1.minute, ontPoster) new OntClientOntologyMetadata(ontClient) } //TODO: Don't assume we're an adapter with an AdapterMappings (don't call .get) protected lazy val happyService: HappyShrineService = { new HappyShrineService( config = config, keystoreDescriptor = keyStoreDescriptor, shrineConfigObject = shrineConfigurationBall, certCollection = shrineCertCollection, signer = signerVerifier, pmPoster = pmPoster, ontologyMetadata = ontologyMetadata, adapterMappings = adapterMappings, auditDaoOption = auditDao, adapterDaoOption = adapterDao, broadcasterOption = broadcasterOption, adapterOption = adapterService ) } protected lazy val happyResource: HappyShrineResource = new HappyShrineResource(happyService) protected lazy val statusJaxrs: StatusJaxrs = StatusJaxrs(config) protected lazy val shrineResource: Option[ShrineResource] = shrineService.map(ShrineResource(_)) protected lazy val i2b2BroadcastResource: Option[I2b2BroadcastResource] = i2b2Service.map(new I2b2BroadcastResource(_, breakdownTypes)) protected lazy val adapterResource: Option[AdapterResource] = adapterService.map(AdapterResource(_)) protected lazy val i2b2AdminResource: Option[I2b2AdminResource] = i2b2AdminService.map(I2b2AdminResource(_, breakdownTypes)) protected lazy val broadcasterMultiplexerResource: Option[BroadcasterMultiplexerResource] = broadcasterMultiplexerService.map(BroadcasterMultiplexerResource(_)) def makeAdapterServiceOption(isQueryable: Boolean, adapterRequestHandler: Option[AdapterRequestHandler]): Option[AdapterRequestHandler] = { if (isQueryable) { require(adapterRequestHandler.isDefined, "Self-querying requested, but this node is not configured to be an adapter") adapterRequestHandler } else { None } } def makeHttpClient(keystoreCertCollection: KeyStoreCertCollection, endpoint: EndpointConfig): HttpClient = { import TrustParam.{AcceptAllCerts, SomeKeyStore} val trustParam = if (endpoint.acceptAllCerts) AcceptAllCerts else SomeKeyStore(keystoreCertCollection) JerseyHttpClient(trustParam, endpoint.timeout) } private final case class AdapterComponents(adapterService: AdapterService, i2b2AdminService: I2b2AdminService, adapterDao: AdapterDao, adapterMappings: AdapterMappings) private final case class QueryEntryPointComponents(shrineService: QepService, i2b2Service: I2b2QepService, auditDao: AuditDao) private final case class HubComponents(broadcaster: AdapterClientBroadcaster) //TODO: TEST private def adapterComponentsToTuple(option: Option[AdapterComponents]): (Option[AdapterService], Option[I2b2AdminService], Option[AdapterDao], Option[AdapterMappings]) = option match { case None => (None, None, None, None) case Some(AdapterComponents(a, b, c, d)) => (Option(a), Option(b), Option(c), Option(d)) } //TODO: TEST private def queryEntryPointComponentsToTuple(option: Option[QueryEntryPointComponents]): (Option[QepService], Option[I2b2QepService], Option[AuditDao]) = option match { case None => (None, None, None) case Some(QueryEntryPointComponents(a, b, c)) => (Option(a), Option(b), Option(c)) } //TODO: TEST private def unpackHubComponents(option: Option[HubComponents]): Option[AdapterClientBroadcaster] = option.map(_.broadcaster) def poster(keystoreCertCollection: KeyStoreCertCollection)(endpoint: EndpointConfig): Poster = { val httpClient = makeHttpClient(keystoreCertCollection, endpoint) Poster(endpoint.url.toString, httpClient) } } diff --git a/apps/shrine-app/src/test/scala/net/shrine/wiring/AuthStrategyTest.scala b/apps/shrine-app/src/test/scala/net/shrine/wiring/AuthStrategyTest.scala index 337d366ed..711b5c56f 100644 --- a/apps/shrine-app/src/test/scala/net/shrine/wiring/AuthStrategyTest.scala +++ b/apps/shrine-app/src/test/scala/net/shrine/wiring/AuthStrategyTest.scala @@ -1,108 +1,107 @@ package net.shrine.wiring import net.shrine.protocol.CredentialConfig import net.shrine.util.ShouldMatchersForJUnit import org.junit.Test import net.shrine.client.{EndpointConfig, Poster} import net.shrine.authentication.{AuthenticationType, PmAuthenticator, Authenticator} import net.shrine.hms.authentication.EcommonsPmAuthenticator import net.shrine.qep.{QepConfig, AllowsAllAuthenticator} import net.shrine.authorization.{AuthorizationType, AllowsAllAuthorizationService} import net.shrine.hms.authorization.HmsDataStewardAuthorizationService import java.net.URL import net.shrine.hms.authorization.JerseySheriffClient import net.shrine.crypto.SigningCertStrategy /** * @author clint * @since Jul 2, 2014 */ final class AuthStrategyTest extends ShouldMatchersForJUnit { private[this] val pmPoster = Poster("http://example.com", null) @Test def testDefaultDetermineAuthenticator(): Unit = { import AuthenticationType._ intercept[Exception] { AuthStrategy.determineAuthenticator(null, pmPoster) } { val authenticator = AuthStrategy.determineAuthenticator(Pm, pmPoster).asInstanceOf[PmAuthenticator] authenticator.pmPoster should be(pmPoster) } { val authenticator = AuthStrategy.determineAuthenticator(Ecommons, pmPoster).asInstanceOf[EcommonsPmAuthenticator] authenticator.pmPoster should be(pmPoster) } { val authenticator = AuthStrategy.determineAuthenticator(NoAuthentication, pmPoster) authenticator should be(AllowsAllAuthenticator) } } @Test def testDefaultDetermineAuthorizationService(): Unit = { import AuthorizationType._ intercept[Exception] { AuthStrategy.determineQueryAuthorizationService(null, null, null) } { val authService = AuthStrategy.determineQueryAuthorizationService(NoAuthorization, null, null) authService should be(AllowsAllAuthorizationService) } { val authenticationType = AuthenticationType.Ecommons val authorizationType = HmsSteward import scala.concurrent.duration._ val sheriffUrl = "http://example.com/sheriff" val sheriffCredentials = CredentialConfig(None, "u", "p") val shrineConfig = ShrineConfig( None, //adapter config None, //hub config Some(QepConfig( authenticationType, authorizationType, Some(EndpointConfig(new URL(sheriffUrl), acceptAllCerts = false, 42.minutes)), Some(sheriffCredentials), None, includeAggregateResults = false, 1.minute, None, SigningCertStrategy.Attach, collectQepAudit = false)), null, //hiveCredentials null, //more hiveCredentials - null, //adapterStatusQuery - Set.empty + null //adapterStatusQuery ) //breakdown types val authenticator: Authenticator = AllowsAllAuthenticator val authService = AuthStrategy.determineQueryAuthorizationService(HmsSteward, shrineConfig, authenticator).asInstanceOf[HmsDataStewardAuthorizationService] authService.authenticator should be(authenticator) //noinspection ScalaUnnecessaryParentheses authService.sheriffClient should not be(null) val jerseySheriffClient = authService.sheriffClient.asInstanceOf[JerseySheriffClient] jerseySheriffClient.sheriffUrl should equal(sheriffUrl) jerseySheriffClient.sheriffUsername should equal(sheriffCredentials.username) jerseySheriffClient.sheriffPassword should equal(sheriffCredentials.password) } } } \ No newline at end of file diff --git a/apps/shrine-app/src/test/scala/net/shrine/wiring/ShrineConfigTest.scala b/apps/shrine-app/src/test/scala/net/shrine/wiring/ShrineConfigTest.scala index 36b573ac1..b9332fe37 100644 --- a/apps/shrine-app/src/test/scala/net/shrine/wiring/ShrineConfigTest.scala +++ b/apps/shrine-app/src/test/scala/net/shrine/wiring/ShrineConfigTest.scala @@ -1,116 +1,111 @@ package net.shrine.wiring import com.typesafe.config.ConfigFactory import net.shrine.authentication.AuthenticationType import net.shrine.authorization.AuthorizationType import net.shrine.broadcaster.NodeListParserTest import net.shrine.client.EndpointConfigTest import net.shrine.crypto.SigningCertStrategy -import net.shrine.protocol.TestResultOutputTypes import net.shrine.util.ShouldMatchersForJUnit import org.junit.Test /** * @author clint * @since Feb 6, 2013 */ final class ShrineConfigTest extends ShouldMatchersForJUnit { private def shrineConfig(baseFileName: String, loadBreakdownsFile: Boolean = true) = { val baseConfig = ConfigFactory.load(baseFileName) val breakdownConfig = ConfigFactory.load("breakdowns") val config = if(loadBreakdownsFile) baseConfig.withFallback(breakdownConfig) else baseConfig ShrineConfig(config) } import scala.concurrent.duration._ @Test def testApply() { import NodeListParserTest.node import EndpointConfigTest.endpoint val conf = shrineConfig("shrine") conf.adapterConfig.get.crcEndpoint should equal(endpoint("http://services.i2b2.org/i2b2/rest/QueryToolService/")) conf.queryEntryPointConfig.get.authenticationType should be(AuthenticationType.Ecommons) conf.queryEntryPointConfig.get.authorizationType should be(AuthorizationType.HmsSteward) conf.queryEntryPointConfig.get.sheriffEndpoint.get should equal(endpoint("http://localhost:8080/shrine-hms-authorization/queryAuthorization")) conf.queryEntryPointConfig.get.sheriffCredentials.get.domain should be(None) conf.queryEntryPointConfig.get.sheriffCredentials.get.username should be("sheriffUsername") conf.queryEntryPointConfig.get.sheriffCredentials.get.password should be("sheriffPassword") conf.crcHiveCredentials should equal(conf.pmHiveCredentials) conf.crcHiveCredentials.domain should equal("HarvardDemo") conf.crcHiveCredentials.username should equal("demo") conf.crcHiveCredentials.password should equal("demouser") conf.crcHiveCredentials.projectId should equal("Demo") conf.ontHiveCredentials.domain should equal("HarvardDemo") conf.ontHiveCredentials.username should equal("demo") conf.ontHiveCredentials.password should equal("demouser") conf.ontHiveCredentials.projectId should equal("SHRINE") conf.ontHiveCredentials.domain should equal("HarvardDemo") conf.ontHiveCredentials.username should equal("demo") conf.ontHiveCredentials.password should equal("demouser") conf.ontHiveCredentials.projectId should equal("SHRINE") conf.queryEntryPointConfig.get.broadcasterIsLocal should be(false) conf.queryEntryPointConfig.get.broadcasterServiceEndpoint.get should equal(endpoint("http://example.com/shrine/rest/broadcaster/broadcast")) conf.queryEntryPointConfig.get.maxQueryWaitTime should equal(5.minutes) conf.queryEntryPointConfig.get.signingCertStrategy should equal(SigningCertStrategy.Attach) conf.adapterConfig.get.setSizeObfuscation should equal(true) conf.queryEntryPointConfig.get.includeAggregateResults should equal(false) conf.adapterConfig.get.adapterLockoutAttemptsThreshold should equal(10) conf.hubConfig.get.maxQueryWaitTime should equal(4.5.minutes) conf.adapterConfig.get.maxSignatureAge should equal(5.minutes) conf.adapterStatusQuery should equal("""\\SHRINE\SHRINE\Diagnoses\Mental Illness\Disorders usually diagnosed in infancy, childhood, or adolescence\Pervasive developmental disorders\Infantile autism, current or active state\""") conf.adapterConfig.get.adapterMappingsFileName should equal("AdapterMappings.xml") conf.hubConfig.get.downstreamNodes.toSet should equal { Set( node("some hospital", "http://example.com/foo"), node("CHB", "http://example.com/chb"), node("PHS", "http://example.com/phs")) } conf.adapterConfig.get.immediatelyRunIncomingQueries should be(false) - - conf.breakdownResultOutputTypes should equal(TestResultOutputTypes.values) } @Test def testApplyOptionalFields() { val conf = shrineConfig("shrine-no-optional-configs", loadBreakdownsFile = false) conf.adapterConfig should be(None) conf.hubConfig should be(None) conf.queryEntryPointConfig should be(None) - conf.breakdownResultOutputTypes should be(Set.empty) } @Test def testApplySomeOptionalFields() { val conf = shrineConfig("shrine-some-optional-props") conf.queryEntryPointConfig.get.authenticationType should be(AuthenticationType.Pm) conf.queryEntryPointConfig.get.authorizationType should be(AuthorizationType.NoAuthorization) conf.queryEntryPointConfig.get.signingCertStrategy should be(SigningCertStrategy.DontAttach) - conf.breakdownResultOutputTypes should equal(TestResultOutputTypes.values) } } \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/Keys.scala b/commons/config/src/main/scala/net/shrine/config/Keys.scala index 625bef34c..e55757010 100644 --- a/commons/config/src/main/scala/net/shrine/config/Keys.scala +++ b/commons/config/src/main/scala/net/shrine/config/Keys.scala @@ -1,40 +1,39 @@ package net.shrine.config /** * @author clint * @since Jan 17, 2014 * * Keys for Shrine */ //todo distribute to where they are used once the rest of config is cleaned up object Keys { val crcEndpoint = "crcEndpoint" val sheriffEndpoint = "sheriffEndpoint" val sheriffCredentials = "sheriffCredentials" val shrineSteward = "shrineSteward" val hiveCredentials = "hiveCredentials" val ontProjectId = "ontProjectId" val crcProjectId = "crcProjectId" val setSizeObfuscation = "setSizeObfuscation" val isAdapter = "isAdapter" val isBroadcaster = "isBroadcaster" val includeAggregateResults = "includeAggregateResults" val adapterLockoutAttemptsThreshold = "adapterLockoutAttemptsThreshold" val maxQueryWaitTime = "maxQueryWaitTime" val networkStatusQuery = "networkStatusQuery" val adapterMappingsFileName = "adapterMappingsFileName" val adapterMappingsFileType = "adapterMappingsFileType" val downstreamNodes = "downstreamNodes" val maxSignatureAge = "maxSignatureAge" val shouldQuerySelf = "shouldQuerySelf" val adapter = "adapter" val hub = "hub" val queryEntryPoint = "queryEntryPoint" val broadcasterIsLocal = "broadcasterIsLocal" val broadcasterServiceEndpoint = "broadcasterServiceEndpoint" val immediatelyRunIncomingQueries = "immediatelyRunIncomingQueries" val authenticationType = "authenticationType" val authorizationType = "authorizationType" - val breakdownResultOutputTypes = "breakdownResultOutputTypes" val attachSigningCert = "attachSigningCert" } diff --git a/qep/service/src/main/scala/net/shrine/qep/queries/QepQueryDb.scala b/qep/service/src/main/scala/net/shrine/qep/queries/QepQueryDb.scala index 2606aba51..734eeebf2 100644 --- a/qep/service/src/main/scala/net/shrine/qep/queries/QepQueryDb.scala +++ b/qep/service/src/main/scala/net/shrine/qep/queries/QepQueryDb.scala @@ -1,536 +1,536 @@ package net.shrine.qep.queries import java.sql.SQLException import java.util.concurrent.TimeoutException import javax.sql.DataSource import com.typesafe.config.Config import net.shrine.audit.{NetworkQueryId, QueryName, Time, UserName} import net.shrine.log.Loggable import net.shrine.problem.{AbstractProblem, ProblemSources, ProblemDigest} import net.shrine.protocol.{ResultOutputTypes, DeleteQueryRequest, RenameQueryRequest, I2b2ResultEnvelope, QueryResult, ResultOutputType, DefaultBreakdownResultOutputTypes, UnFlagQueryRequest, FlagQueryRequest, QueryMaster, ReadPreviousQueriesRequest, ReadPreviousQueriesResponse, RunQueryRequest} import net.shrine.qep.QepConfigSource import net.shrine.slick.TestableDataSourceCreator import net.shrine.util.XmlDateHelper import slick.driver.JdbcProfile import scala.collection.immutable.Iterable import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, Future, blocking} import scala.language.postfixOps import scala.concurrent.ExecutionContext.Implicits.global import scala.util.control.NonFatal import scala.xml.XML /** * DB code for the QEP's query instances and query results. * * @author david * @since 1/19/16 */ case class QepQueryDb(schemaDef:QepQuerySchema,dataSource: DataSource,timeout:Duration) extends Loggable { import schemaDef._ import jdbcProfile.api._ val database = Database.forDataSource(dataSource) def createTables() = schemaDef.createTables(database) def dropTables() = schemaDef.dropTables(database) def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = { val future: Future[R] = database.run(action) try { blocking { Await.result(future, timeout) } } catch { case tx:TimeoutException => throw CouldNotRunDbIoActionException(dataSource,tx) case NonFatal(x) => throw CouldNotRunDbIoActionException(dataSource,x) } } def insertQepQuery(runQueryRequest: RunQueryRequest):Unit = { debug(s"insertQepQuery $runQueryRequest") insertQepQuery(QepQuery(runQueryRequest)) } def insertQepQuery(qepQuery: QepQuery):Unit = { dbRun(allQepQueryQuery += qepQuery) } def selectAllQepQueries:Seq[QepQuery] = { dbRun(mostRecentVisibleQepQueries.result) } def selectPreviousQueries(request: ReadPreviousQueriesRequest):ReadPreviousQueriesResponse = { val previousQueries: Seq[QepQuery] = selectPreviousQueriesByUserAndDomain(request.authn.username,request.authn.domain,request.fetchSize) val flags:Map[NetworkQueryId,QepQueryFlag] = selectMostRecentQepQueryFlagsFor(previousQueries.map(_.networkId).to[Set]) val queriesAndFlags = previousQueries.map(x => (x,flags.get(x.networkId))) ReadPreviousQueriesResponse(queriesAndFlags.map(x => x._1.toQueryMaster(x._2))) } def selectPreviousQueriesByUserAndDomain(userName: UserName, domain: String, limit:Int):Seq[QepQuery] = { dbRun(mostRecentVisibleQepQueries.filter(_.userName === userName).filter(_.userDomain === domain).sortBy(x => x.changeDate.desc).take(limit).result) } def renamePreviousQuery(request:RenameQueryRequest):Unit = { val networkQueryId = request.networkQueryId dbRun( for { queryResults <- mostRecentVisibleQepQueries.filter(_.networkId === networkQueryId).result _ <- allQepQueryQuery ++= queryResults.map(_.copy(queryName = request.queryName,changeDate = System.currentTimeMillis())) } yield queryResults ) } def markDeleted(request:DeleteQueryRequest):Unit = { val networkQueryId = request.networkQueryId dbRun( for { queryResults <- mostRecentVisibleQepQueries.filter(_.networkId === networkQueryId).result _ <- allQepQueryQuery ++= queryResults.map(_.copy(deleted = true,changeDate = System.currentTimeMillis())) } yield queryResults ) } def insertQepQueryFlag(flagQueryRequest: FlagQueryRequest):Unit = { insertQepQueryFlag(QepQueryFlag(flagQueryRequest)) } def insertQepQueryFlag(unflagQueryRequest: UnFlagQueryRequest):Unit = { insertQepQueryFlag(QepQueryFlag(unflagQueryRequest)) } def insertQepQueryFlag(qepQueryFlag: QepQueryFlag):Unit = { dbRun(allQepQueryFlags += qepQueryFlag) } def selectMostRecentQepQueryFlagsFor(networkIds:Set[NetworkQueryId]):Map[NetworkQueryId,QepQueryFlag] = { val flags:Seq[QepQueryFlag] = dbRun(mostRecentQueryFlags.filter(_.networkId inSet networkIds).result) flags.map(x => x.networkQueryId -> x).toMap } def insertQepResultRow(qepQueryRow:QueryResultRow) = { dbRun(allQueryResultRows += qepQueryRow) } def insertQueryResult(networkQueryId:NetworkQueryId,result:QueryResult) = { val adapterNode = result.description.getOrElse(throw new IllegalStateException("description is empty, does not have an adapter node")) val queryResultRow = QueryResultRow(networkQueryId,result) val breakdowns: Iterable[QepQueryBreakdownResultsRow] = result.breakdowns.flatMap(QepQueryBreakdownResultsRow.breakdownRowsFor(networkQueryId,adapterNode,result.resultId,_)) val problem: Seq[QepProblemDigestRow] = result.problemDigest.map(p => QepProblemDigestRow(networkQueryId,adapterNode,p.codec,p.stampText,p.summary,p.description,p.detailsXml.toString,System.currentTimeMillis())).to[Seq] dbRun( for { _ <- allQueryResultRows += queryResultRow _ <- allBreakdownResultsRows ++= breakdowns _ <- allProblemDigestRows ++= problem } yield () ) } def selectMostRecentQepResultRowsFor(networkId:NetworkQueryId): Seq[QueryResultRow] = { dbRun(mostRecentQueryResultRows.filter(_.networkQueryId === networkId).result) } def selectMostRecentQepResultsFor(networkId:NetworkQueryId): Seq[QueryResult] = { val (queryResults, breakdowns,problems) = dbRun( for { queryResults <- mostRecentQueryResultRows.filter(_.networkQueryId === networkId).result breakdowns <- mostRecentBreakdownResultsRows.filter(_.networkQueryId === networkId).result problems <- mostRecentProblemDigestRows.filter(_.networkQueryId === networkId).result } yield (queryResults, breakdowns, problems) ) val resultIdsToI2b2ResultEnvelopes: Map[Long, Map[ResultOutputType, I2b2ResultEnvelope]] = breakdowns.groupBy(_.resultId).map(rIdToB => rIdToB._1 -> QepQueryBreakdownResultsRow.resultEnvelopesFrom(rIdToB._2)) def seqOfOneProblemRowToProblemDigest(problemSeq:Seq[QepProblemDigestRow]):ProblemDigest = { if(problemSeq.size == 1) problemSeq.head.toProblemDigest else throw new IllegalStateException(s"problemSeq size was not 1. $problemSeq") } val adapterNodesToProblemDigests: Map[String, ProblemDigest] = problems.groupBy(_.adapterNode).map(nodeToProblem => nodeToProblem._1 -> seqOfOneProblemRowToProblemDigest(nodeToProblem._2) ) queryResults.map(r => r.toQueryResult( resultIdsToI2b2ResultEnvelopes.getOrElse(r.resultId,Map.empty), adapterNodesToProblemDigests.get(r.adapterNode) )) } def insertQueryBreakdown(breakdownResultsRow:QepQueryBreakdownResultsRow) = { dbRun(allBreakdownResultsRows += breakdownResultsRow) } def selectAllBreakdownResultsRows: Seq[QepQueryBreakdownResultsRow] = { dbRun(allBreakdownResultsRows.result) } } object QepQueryDb extends Loggable { val dataSource:DataSource = TestableDataSourceCreator.dataSource(QepQuerySchema.config) val timeout = QepQuerySchema.config.getInt("timeout") seconds val db = QepQueryDb(QepQuerySchema.schema,dataSource,timeout) val createTablesOnStart = QepQuerySchema.config.getBoolean("createTablesOnStart") if(createTablesOnStart) QepQueryDb.db.createTables() } /** * Separate class to support schema generation without actually connecting to the database. * * @param jdbcProfile Database profile to use for the schema */ case class QepQuerySchema(jdbcProfile: JdbcProfile,moreBreakdowns: Set[ResultOutputType]) extends Loggable { import jdbcProfile.api._ def ddlForAllTables: jdbcProfile.DDL = { allQepQueryQuery.schema ++ allQepQueryFlags.schema ++ allQueryResultRows.schema ++ allBreakdownResultsRows.schema ++ allProblemDigestRows.schema } //to get the schema, use the REPL //println(QepQuerySchema.schema.ddlForAllTables.createStatements.mkString(";\n")) def createTables(database:Database) = { try { val future = database.run(ddlForAllTables.create) Await.result(future,10 seconds) } catch { //I'd prefer to check and create schema only if absent. No way to do that with Oracle. case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x) } } def dropTables(database:Database) = { val future = database.run(ddlForAllTables.drop) //Really wait forever for the cleanup Await.result(future,Duration.Inf) } class QepQueries(tag:Tag) extends Table[QepQuery](tag,"previousQueries") { def networkId = column[NetworkQueryId]("networkId") def userName = column[UserName]("userName") def userDomain = column[String]("domain") def queryName = column[QueryName]("queryName") def expression = column[Option[String]]("expression") def dateCreated = column[Time]("dateCreated") def deleted = column[Boolean]("deleted") def queryXml = column[String]("queryXml") def changeDate = column[Long]("changeDate") def * = (networkId,userName,userDomain,queryName,expression,dateCreated,deleted,queryXml,changeDate) <> (QepQuery.tupled,QepQuery.unapply) } val allQepQueryQuery = TableQuery[QepQueries] val mostRecentQepQueryQuery: Query[QepQueries, QepQuery, Seq] = for( queries <- allQepQueryQuery if !allQepQueryQuery.filter(_.networkId === queries.networkId).filter(_.changeDate > queries.changeDate).exists ) yield queries val mostRecentVisibleQepQueries = mostRecentQepQueryQuery.filter(_.deleted === false) class QepQueryFlags(tag:Tag) extends Table[QepQueryFlag](tag,"queryFlags") { def networkId = column[NetworkQueryId]("networkId") def flagged = column[Boolean]("flagged") def flagMessage = column[String]("flagMessage") def changeDate = column[Long]("changeDate") def * = (networkId,flagged,flagMessage,changeDate) <> (QepQueryFlag.tupled,QepQueryFlag.unapply) } val allQepQueryFlags = TableQuery[QepQueryFlags] val mostRecentQueryFlags: Query[QepQueryFlags, QepQueryFlag, Seq] = for( queryFlags <- allQepQueryFlags if !allQepQueryFlags.filter(_.networkId === queryFlags.networkId).filter(_.changeDate > queryFlags.changeDate).exists ) yield queryFlags val qepQueryResultTypes = DefaultBreakdownResultOutputTypes.toSet ++ ResultOutputType.values ++ moreBreakdowns val stringsToQueryResultTypes: Map[String, ResultOutputType] = qepQueryResultTypes.map(x => (x.name,x)).toMap val queryResultTypesToString: Map[ResultOutputType, String] = stringsToQueryResultTypes.map(_.swap) implicit val qepQueryResultTypesColumnType = MappedColumnType.base[ResultOutputType,String] ({ (resultType: ResultOutputType) => queryResultTypesToString(resultType) },{ (string: String) => stringsToQueryResultTypes(string) }) implicit val queryStatusColumnType = MappedColumnType.base[QueryResult.StatusType,String] ({ statusType => statusType.name },{ name => QueryResult.StatusType.valueOf(name).getOrElse(throw new IllegalStateException(s"$name is not one of ${QueryResult.StatusType.values.map(_.name).mkString(", ")}")) }) class QepQueryResults(tag:Tag) extends Table[QueryResultRow](tag,"queryResults") { def resultId = column[Long]("resultId") def networkQueryId = column[NetworkQueryId]("networkQueryId") def instanceId = column[Long]("instanceId") def adapterNode = column[String]("adapterNode") def resultType = column[Option[ResultOutputType]]("resultType") def size = column[Long]("size") def startDate = column[Option[Long]]("startDate") def endDate = column[Option[Long]]("endDate") def status = column[QueryResult.StatusType]("status") def statusMessage = column[Option[String]]("statusMessage") def changeDate = column[Long]("changeDate") def * = (resultId,networkQueryId,instanceId,adapterNode,resultType,size,startDate,endDate,status,statusMessage,changeDate) <> (QueryResultRow.tupled,QueryResultRow.unapply) } val allQueryResultRows = TableQuery[QepQueryResults] //Most recent query result rows for each queryId from each adapter val mostRecentQueryResultRows: Query[QepQueryResults, QueryResultRow, Seq] = for( queryResultRows <- allQueryResultRows if !allQueryResultRows.filter(_.networkQueryId === queryResultRows.networkQueryId).filter(_.adapterNode === queryResultRows.adapterNode).filter(_.changeDate > queryResultRows.changeDate).exists ) yield queryResultRows class QepQueryBreakdownResults(tag:Tag) extends Table[QepQueryBreakdownResultsRow](tag,"queryBreakdownResults") { def networkQueryId = column[NetworkQueryId]("networkQueryId") def adapterNode = column[String]("adapterNode") def resultId = column[Long]("resultId") def resultType = column[ResultOutputType]("resultType") def dataKey = column[String]("dataKey") def value = column[Long]("value") def changeDate = column[Long]("changeDate") def * = (networkQueryId,adapterNode,resultId,resultType,dataKey,value,changeDate) <> (QepQueryBreakdownResultsRow.tupled,QepQueryBreakdownResultsRow.unapply) } val allBreakdownResultsRows = TableQuery[QepQueryBreakdownResults] //Most recent query result rows for each queryId from each adapter val mostRecentBreakdownResultsRows: Query[QepQueryBreakdownResults, QepQueryBreakdownResultsRow, Seq] = for( breakdownResultsRows <- allBreakdownResultsRows if !allBreakdownResultsRows.filter(_.networkQueryId === breakdownResultsRows.networkQueryId).filter(_.adapterNode === breakdownResultsRows.adapterNode).filter(_.resultId === breakdownResultsRows.resultId).filter(_.changeDate > breakdownResultsRows.changeDate).exists ) yield breakdownResultsRows /* case class ProblemDigest(codec: String, stampText: String, summary: String, description: String, detailsXml: NodeSeq) extends XmlMarshaller { */ class QepResultProblemDigests(tag:Tag) extends Table [QepProblemDigestRow](tag,"queryResultProblemDigests") { def networkQueryId = column[NetworkQueryId]("networkQueryId") def adapterNode = column[String]("adapterNode") def codec = column[String]("codec") def stamp = column[String]("stamp") def summary = column[String]("summary") def description = column[String]("description") def details = column[String]("details") def changeDate = column[Long]("changeDate") def * = (networkQueryId,adapterNode,codec,stamp,summary,description,details,changeDate) <> (QepProblemDigestRow.tupled,QepProblemDigestRow.unapply) } val allProblemDigestRows = TableQuery[QepResultProblemDigests] val mostRecentProblemDigestRows: Query[QepResultProblemDigests, QepProblemDigestRow, Seq] = for( problemDigests <- allProblemDigestRows if !allProblemDigestRows.filter(_.networkQueryId === problemDigests.networkQueryId).filter(_.adapterNode === problemDigests.adapterNode).filter(_.changeDate > problemDigests.changeDate).exists ) yield problemDigests } object QepQuerySchema { val allConfig:Config = QepConfigSource.config val config:Config = allConfig.getConfig("shrine.queryEntryPoint.audit.database") val slickProfileClassName = config.getString("slickProfileClassName") val slickProfile:JdbcProfile = QepConfigSource.objectForName(slickProfileClassName) import net.shrine.config.{ConfigExtensions, Keys} - val moreBreakdowns: Set[ResultOutputType] = config.getOptionConfigured(Keys.breakdownResultOutputTypes,ResultOutputTypes.fromConfig).getOrElse(Set.empty) + val moreBreakdowns: Set[ResultOutputType] = config.getOptionConfigured("breakdownResultOutputTypes",ResultOutputTypes.fromConfig).getOrElse(Set.empty) val schema = QepQuerySchema(slickProfile,moreBreakdowns) } case class QepQuery( networkId:NetworkQueryId, userName: UserName, userDomain: String, queryName: QueryName, expression: Option[String], dateCreated: Time, deleted: Boolean, queryXml: String, changeDate: Time ){ def toQueryMaster(qepQueryFlag:Option[QepQueryFlag]):QueryMaster = { QueryMaster( queryMasterId = networkId.toString, networkQueryId = networkId, name = queryName, userId = userName, groupId = userDomain, createDate = XmlDateHelper.toXmlGregorianCalendar(dateCreated), flagged = qepQueryFlag.map(_.flagged), flagMessage = qepQueryFlag.map(_.flagMessage) ) } } object QepQuery extends ((NetworkQueryId,UserName,String,QueryName,Option[String],Time,Boolean,String,Time) => QepQuery) { def apply(runQueryRequest: RunQueryRequest):QepQuery = { new QepQuery( networkId = runQueryRequest.networkQueryId, userName = runQueryRequest.authn.username, userDomain = runQueryRequest.authn.domain, queryName = runQueryRequest.queryDefinition.name, expression = runQueryRequest.queryDefinition.expr.map(_.toString), dateCreated = System.currentTimeMillis(), deleted = false, queryXml = runQueryRequest.toXmlString, changeDate = System.currentTimeMillis() ) } } case class QepQueryFlag( networkQueryId: NetworkQueryId, flagged:Boolean, flagMessage:String, changeDate:Long ) object QepQueryFlag extends ((NetworkQueryId,Boolean,String,Long) => QepQueryFlag) { def apply(flagQueryRequest: FlagQueryRequest):QepQueryFlag = { QepQueryFlag( networkQueryId = flagQueryRequest.networkQueryId, flagged = true, flagMessage = flagQueryRequest.message.getOrElse(""), changeDate = System.currentTimeMillis() ) } def apply(unflagQueryRequest: UnFlagQueryRequest):QepQueryFlag = { QepQueryFlag( networkQueryId = unflagQueryRequest.networkQueryId, flagged = false, flagMessage = "", changeDate = System.currentTimeMillis() ) } } case class QueryResultRow( resultId:Long, networkQueryId:NetworkQueryId, instanceId:Long, adapterNode:String, resultType:Option[ResultOutputType], size:Long, startDate:Option[Long], endDate:Option[Long], status:QueryResult.StatusType, statusMessage:Option[String], changeDate:Long ) { def toQueryResult(breakdowns:Map[ResultOutputType,I2b2ResultEnvelope],problemDigest:Option[ProblemDigest]) = QueryResult( resultId = resultId, instanceId = instanceId, resultType = resultType, setSize = size, startDate = startDate.map(XmlDateHelper.toXmlGregorianCalendar), endDate = endDate.map(XmlDateHelper.toXmlGregorianCalendar), description = Some(adapterNode), statusType = status, statusMessage = statusMessage, breakdowns = breakdowns, problemDigest = problemDigest ) } object QueryResultRow extends ((Long,NetworkQueryId,Long,String,Option[ResultOutputType],Long,Option[Long],Option[Long],QueryResult.StatusType,Option[String],Long) => QueryResultRow) { def apply(networkQueryId:NetworkQueryId,result:QueryResult):QueryResultRow = { new QueryResultRow( resultId = result.resultId, networkQueryId = networkQueryId, instanceId = result.instanceId, adapterNode = result.description.getOrElse(s"$result has None in its description field, not a name of an adapter node."), resultType = result.resultType, size = result.setSize, startDate = result.startDate.map(_.toGregorianCalendar.getTimeInMillis), endDate = result.endDate.map(_.toGregorianCalendar.getTimeInMillis), status = result.statusType, statusMessage = result.statusMessage, changeDate = System.currentTimeMillis() ) } } case class QepQueryBreakdownResultsRow( networkQueryId: NetworkQueryId, adapterNode:String, resultId:Long, resultType: ResultOutputType, dataKey:String, value:Long, changeDate:Long ) object QepQueryBreakdownResultsRow extends ((NetworkQueryId,String,Long,ResultOutputType,String,Long,Long) => QepQueryBreakdownResultsRow){ def breakdownRowsFor(networkQueryId:NetworkQueryId, adapterNode:String, resultId:Long, breakdown:(ResultOutputType,I2b2ResultEnvelope)): Iterable[QepQueryBreakdownResultsRow] = { breakdown._2.data.map(b => QepQueryBreakdownResultsRow(networkQueryId,adapterNode,resultId,breakdown._1,b._1,b._2,System.currentTimeMillis())) } def resultEnvelopesFrom(breakdowns:Seq[QepQueryBreakdownResultsRow]): Map[ResultOutputType, I2b2ResultEnvelope] = { def resultEnvelopeFrom(resultType:ResultOutputType,breakdowns:Seq[QepQueryBreakdownResultsRow]):I2b2ResultEnvelope = { val data = breakdowns.map(b => b.dataKey -> b.value).toMap I2b2ResultEnvelope(resultType,data) } breakdowns.groupBy(_.resultType).map(r => r._1 -> resultEnvelopeFrom(r._1,r._2)) } } case class QepProblemDigestRow( networkQueryId: NetworkQueryId, adapterNode: String, codec: String, stampText: String, summary: String, description: String, details: String, changeDate:Long ){ def toProblemDigest = { ProblemDigest( codec, stampText, summary, description, if(!details.isEmpty) XML.loadString(details) else
) } } case class CouldNotRunDbIoActionException(dataSource: DataSource, exception: Throwable) extends RuntimeException(exception) { override def getMessage:String = s"Could not use the database defined by $dataSource due to ${exception.getLocalizedMessage}" } case class QepDatabaseProblem(x:Exception) extends AbstractProblem(ProblemSources.Qep){ override val summary = "A problem encountered while using a database." override val throwable = Some(x) override val description = x.getMessage } \ No newline at end of file diff --git a/tools/batch-querier/src/main/scala/net/shrine/utilities/batchquerier/BatchQuerierConfig.scala b/tools/batch-querier/src/main/scala/net/shrine/utilities/batchquerier/BatchQuerierConfig.scala index f686c798f..94a4d859e 100644 --- a/tools/batch-querier/src/main/scala/net/shrine/utilities/batchquerier/BatchQuerierConfig.scala +++ b/tools/batch-querier/src/main/scala/net/shrine/utilities/batchquerier/BatchQuerierConfig.scala @@ -1,79 +1,79 @@ package net.shrine.utilities.batchquerier import java.io.File import net.shrine.protocol.{ResultOutputTypes, AuthenticationInfo, Credential, ResultOutputType} import com.typesafe.config.Config import com.typesafe.config.ConfigException import net.shrine.utilities.scallop.{Keys => BaseKeys} import scala.util.Try import net.shrine.config.Keys /** * @author clint * @since Sep 6, 2013 */ final case class BatchQuerierConfig( expressionFile: File, shrineUrl: String, authorization: AuthenticationInfo, outputFile: File, projectId: String, topicId: String, queriesPerTerm: Int, breakdownTypes: Set[ResultOutputType]) object BatchQuerierConfig { def apply(config: Config): BatchQuerierConfig = { def file(path: String) = new File(path) def getAuthInfo(subConfig: Config): AuthenticationInfo = { import BaseKeys._ def string(path: String) = subConfig.getString(path) def requirePath(path: String) = if (!subConfig.hasPath(path)) throw new ConfigException.Missing(s"Expected to find '$path' in $subConfig") requirePath(domain) requirePath(username) requirePath(password) AuthenticationInfo(string(domain), string(username), Credential(string(password), false)) } def string(path: String) = config.getString(path) def intOption(path: String) = if(config.hasPath(path)) Option(config.getInt(path)) else None import BatchQuerierConfigKeys._ BatchQuerierConfig( file(string(inputFile)), string(shrineUrl), getAuthInfo(config.getConfig(credentials)), file(string(outputFile)), string(projectId), string(topicId), intOption(queriesPerTerm).getOrElse(Defaults.queriesPerTerm), Try(ResultOutputTypes.fromConfig(config.getConfig(breakdownResultOutputTypes))).getOrElse(Set.empty)) } object Defaults { val queriesPerTerm = 3 } object BatchQuerierConfigKeys { private val base = "batch" private def subKey(k: String) = BaseKeys.subKey(base)(k) val inputFile = subKey("inputFile") val outputFile = subKey("outputFile") val shrineUrl = subKey("shrineUrl") val credentials = BaseKeys.credentials(base) val projectId = subKey("projectId") val topicId = subKey("topicId") val queriesPerTerm = subKey("queriesPerTerm") - val breakdownResultOutputTypes = Keys.breakdownResultOutputTypes + val breakdownResultOutputTypes = "breakdownResultOutputTypes" } } \ No newline at end of file diff --git a/tools/scanner/src/main/scala/net/shrine/utilities/scanner/ScannerConfig.scala b/tools/scanner/src/main/scala/net/shrine/utilities/scanner/ScannerConfig.scala index cc9eafce8..cc76e1730 100644 --- a/tools/scanner/src/main/scala/net/shrine/utilities/scanner/ScannerConfig.scala +++ b/tools/scanner/src/main/scala/net/shrine/utilities/scanner/ScannerConfig.scala @@ -1,96 +1,96 @@ package net.shrine.utilities.scanner import scala.concurrent.duration.Duration import net.shrine.protocol.{ResultOutputTypes, AuthenticationInfo, Credential, ResultOutputType} import com.typesafe.config.Config import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigException import scala.util.Try import net.shrine.utilities.scallop.{Keys => BaseKeys} import net.shrine.crypto.{KeyStoreDescriptorParser, KeyStoreDescriptor} import net.shrine.config.Keys /** * @author clint * @since Mar 6, 2013 */ final case class ScannerConfig( adapterMappingsFile: String, ontologySqlFile: String, reScanTimeout: Duration, maxTimeToWaitForResults: Duration, shrineUrl: String, projectId: String, authorization: AuthenticationInfo, outputFile: String, keystoreDescriptor: KeyStoreDescriptor, pmUrl: String, breakdownTypes: Set[ResultOutputType]) object ScannerConfig { private[scanner] def getDuration(keyName: String, subConfig: Config): Duration = { import scala.concurrent.duration._ import ScannerConfigKeys._ import net.shrine.utilities.scallop.Keys._ if (subConfig.hasPath(milliseconds)) { subConfig.getLong(milliseconds).milliseconds } else if (subConfig.hasPath(seconds)) { subConfig.getLong(seconds).seconds } else if (subConfig.hasPath(minutes)) { subConfig.getLong(minutes).minutes } else { throw new ConfigException.Missing(s"Expected to find one of $keyName.{${milliseconds}, ${seconds}, ${minutes}} at subConfig $subConfig") } } def apply(config: Config): ScannerConfig = { def getAuthInfo(subConfig: Config): AuthenticationInfo = { import BaseKeys._ def requirePath(path: String) = if (!subConfig.hasPath(path)) throw new ConfigException.Missing(s"Expected to find '$path' in $subConfig") requirePath(domain) requirePath(username) requirePath(password) AuthenticationInfo(subConfig.getString(domain), subConfig.getString(username), Credential(subConfig.getString(password), false)) } import ScannerConfigKeys._ def string(k: String) = config.getString(k) def duration(k: String) = getDuration(k, config.getConfig(k)) def authInfo(k: String) = getAuthInfo(config.getConfig(k)) ScannerConfig( string(adapterMappingsFile), string(ontologySqlFile), duration(reScanTimeout), duration(maxTimeToWaitForResults), string(shrineUrl), string(projectId), authInfo(credentials), Try(string(outputFile)).getOrElse(FileNameSource.nextOutputFileName), KeyStoreDescriptorParser(config.getConfig(keystore)), string(pmUrl), Try(ResultOutputTypes.fromConfig(config.getConfig(breakdownResultOutputTypes))).getOrElse(Set.empty)) } object ScannerConfigKeys { val base = "scanner" private def subKey(k: String) = BaseKeys.subKey(base)(k) val keystore = subKey("keystore") val adapterMappingsFile = subKey("adapterMappingsFile") val ontologySqlFile = subKey("ontologySqlFile") val reScanTimeout = subKey("reScanTimeout") val maxTimeToWaitForResults = subKey("maxTimeToWaitForResults") val shrineUrl = subKey("shrineUrl") val pmUrl = subKey("pmUrl") val projectId = subKey("projectId") val credentials = BaseKeys.credentials(base) - val breakdownResultOutputTypes = subKey(Keys.breakdownResultOutputTypes) + val breakdownResultOutputTypes = subKey("breakdownResultOutputTypes") val outputFile = subKey("outputFile") } }