diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala index 264c35b7e..425b0ff05 100644 --- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala +++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala @@ -1,345 +1,345 @@ package net.shrine.dashboard -import akka.actor.{ActorSystem, Actor} +import akka.actor.Actor import akka.event.Logging import net.shrine.authentication.UserAuthenticator import net.shrine.authorization.steward.OutboundUser import net.shrine.dashboard.jwtauth.ShrineJwtAuthenticator import net.shrine.i2b2.protocol.pm.User import net.shrine.dashboard.httpclient.HttpClientDirectives.{forwardUnmatchedPath,requestUriThenRoute} import net.shrine.log.Loggable import shapeless.HNil import spray.http.{Uri, HttpResponse, HttpRequest, StatusCodes} import spray.httpx.Json4sSupport import spray.routing.directives.LogEntry import spray.routing.{AuthenticationFailedRejection, Rejected, RouteConcatenation, Directive0, Route, HttpService} import org.json4s.{DefaultFormats, Formats} import scala.concurrent.ExecutionContext.Implicits.global // we don't implement our route structure directly in the service actor because // we want to be able to test it independently, without having to spin up an actor class DashboardServiceActor extends Actor with DashboardService { // the HttpService trait defines only one abstract member, which // connects the services environment to the enclosing actor or test def actorRefFactory = context // this actor only runs our route, but you could add // other things here, like request stream processing // or timeout handling def receive = runRoute(route) } // this trait defines our service behavior independently from the service actor trait DashboardService extends HttpService with Json4sSupport with Loggable { implicit def json4sFormats: Formats = DefaultFormats val userAuthenticator = UserAuthenticator(DashboardConfigSource.config) //don't need to do anything special for unauthorized users, but they do need access to a static form. lazy val route:Route = gruntWatchCorsSupport{ redirectToIndex ~ staticResources ~ makeTrouble ~ about ~ authenticatedInBrowser ~ authenticatedDashboard } // logs just the request method, uri and response at info level def logEntryForRequestResponse(req: HttpRequest): Any => Option[LogEntry] = { case res: HttpResponse => Some(LogEntry(s"\n Request: $req\n Response: $res", Logging.InfoLevel)) case _ => None // other kind of responses } // logs just the request method, uri and response status at info level def logEntryForRequest(req: HttpRequest): Any => Option[LogEntry] = { case res: HttpResponse => Some(LogEntry(s"\n Request: $req\n Response status: ${res.status}", Logging.InfoLevel)) case _ => None // other kind of responses } //pathPrefixTest shields the QEP code from the redirect. def authenticatedInBrowser: Route = pathPrefixTest("user"|"admin"|"toDashboard") { logRequestResponse(logEntryForRequestResponse _) { //logging is controlled by Akka's config, slf4j, and log4j config reportIfFailedToAuthenticate { authenticate(userAuthenticator.basicUserAuthenticator) { user => pathPrefix("user") { userRoute(user) } ~ pathPrefix("admin") { adminRoute(user) } ~ pathPrefix("toDashboard") { toDashboardRoute(user) } } } } } val reportIfFailedToAuthenticate = routeRouteResponse { case Rejected(List(AuthenticationFailedRejection(_,_))) => complete("AuthenticationFailed") } def authenticatedDashboard:Route = pathPrefix("fromDashboard") { logRequestResponse(logEntryForRequestResponse _) { //logging is controlled by Akka's config, slf4j, and log4j config get { //all remote dashboard calls are gets. authenticate(ShrineJwtAuthenticator.authenticate) { user => adminRoute(user) } } } } def makeTrouble = pathPrefix("makeTrouble") { complete(throw new IllegalStateException("fake trouble")) } lazy val redirectToIndex = pathEnd { redirect("shrine-dashboard/client/index.html", StatusCodes.PermanentRedirect) //todo pick up "shrine-dashboard" programatically } ~ ( path("index.html") | pathSingleSlash) { redirect("client/index.html", StatusCodes.PermanentRedirect) } lazy val staticResources = pathPrefix("client") { pathEnd { redirect("client/index.html", StatusCodes.PermanentRedirect) } ~ pathSingleSlash { redirect("index.html", StatusCodes.PermanentRedirect) } ~ { getFromResourceDirectory("client") } } lazy val about = pathPrefix("about") { complete("Nothing here yet") //todo } def userRoute(user:User):Route = get { pathPrefix("whoami") { complete(OutboundUser.createFromUser(user)) } } //todo is this an admin? Does it matter? def adminRoute(user:User):Route = get { pathPrefix("happy") { val happyBaseUrl: String = DashboardConfigSource.config.getString("shrine.dashboard.happyBaseUrl") forwardUnmatchedPath(happyBaseUrl) } ~ pathPrefix("messWithHappyVersion") { val happyBaseUrl: String = DashboardConfigSource.config.getString("shrine.dashboard.happyBaseUrl") def pullClasspathFromConfig(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { val result = httpResponse.entity.asString ctx.complete(s"Got '$result' from $uri") } } requestUriThenRoute(happyBaseUrl+"/version",pullClasspathFromConfig) } ~ pathPrefix("ping") {complete("pong")}~ pathPrefix("status"){statusRoute(user)} } //Manually test this by running a curl command //curl -k -w "\n%{response_code}\n" -u dave:kablam "https://shrine-dev1.catalyst:6443/shrine-dashboard/toDashboard/shrine-dev2.catalyst/shrine-dashboard/fromDashboard/ping" def toDashboardRoute(user:User):Route = get { pathPrefix(Segment) { dnsName => val remoteDashboardProtocol = DashboardConfigSource.config.getString("shrine.dashboard.remoteDashboard.protocol") val remoteDashboardPort = DashboardConfigSource.config.getString("shrine.dashboard.remoteDashboard.port") val remoteDashboardPathPrefix = DashboardConfigSource.config.getString("shrine.dashboard.remoteDashboard.pathPrefix") val baseUrl = s"$remoteDashboardProtocol$dnsName$remoteDashboardPort/$remoteDashboardPathPrefix" forwardUnmatchedPath(baseUrl,Some(ShrineJwtAuthenticator.createOAuthCredentials(user))) } } def statusRoute(user:User):Route = get { pathPrefix("config"){getConfig}~ pathPrefix("classpath"){getClasspath} } lazy val getConfig:Route = { val statusBaseUrl: String = DashboardConfigSource.config.getString("shrine" + ".dashboard.statusBaseUrl") forwardUnmatchedPath(statusBaseUrl) } lazy val getClasspath:Route = { val statusBaseUrl: String = DashboardConfigSource.config.getString("shrine" + ".dashboard" + ".statusBaseUrl") def pullClasspathFromConfig(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { import org.json4s.native.JsonMethods.parse val result = httpResponse.entity.asString val config = parse(result).extract[net.shrine.status.protocol.Config].keyValues.filterKeys(_.toLowerCase.startsWith("shrine")) val shrineConfig = ShrineConfig(config) //for((k, v) <- config.filterKeys(_.startsWith("shrine.queryEntryPoint"))) //println (k + "<--" + v) val audit = Audit(config) println(audit) ctx.complete(shrineConfig) } } requestUriThenRoute(statusBaseUrl,pullClasspathFromConfig) } } case class ShrineConfig(isHub:Boolean, hub:Hub, pmEndpoint:Endpoint, ontEndpoint:Endpoint, hiveCredentials: HiveCredentials) object ShrineConfig{ def apply(configMap:Map[String, String]):ShrineConfig = { val hub = Hub(configMap) val isHub = hub.create val pmEndpoint = Endpoint(configMap, "pm") val ontEndpoint = Endpoint(configMap, "ont") val hiveCredentials = HiveCredentials(configMap) ShrineConfig(isHub, hub, pmEndpoint, ontEndpoint, hiveCredentials) } } case class Endpoint(acceptAllCerts:Boolean, url:String, timeoutSeconds:Int) object Endpoint{ def apply(configMap:Map[String, String], endpointType:String):Endpoint = { val prefix = "shrine." + endpointType.toLowerCase + "Endpoint." val acceptAllCerts = configMap.getOrElse(prefix + "acceptAllCerts", "") == "true" val url = configMap.getOrElse(prefix + "url", "") val timeoutSeconds = configMap.getOrElse(prefix + "timeout.seconds", "").toInt Endpoint(acceptAllCerts, url, timeoutSeconds) } } case class HiveCredentials(domain:String, username:String, password:String, crcProjectId:String, ontProjectId:String) object HiveCredentials{ def apply(configMap:Map[String, String]):HiveCredentials = { val key = "shrine.hiveCredentials." val domain = configMap.getOrElse(key + "domain", "") val username = configMap.getOrElse(key + "username", "") val password = "REDACTED" val crcProjectId = configMap.getOrElse(key + "crcProjectId", "") val ontProjectId = configMap.getOrElse(key + "ontProjectId", "") HiveCredentials(domain, username, password, crcProjectId, ontProjectId) } } // -- hub only -- // case class Hub(shouldQuerySelf:Boolean, create:Boolean, downstreamNodes:Map[String,String]) object Hub{ def apply(configMap:Map[String,String]):Hub = { //@todo: use missing istead of "" for 'Else' val shouldQuerySelf = configMap.getOrElse("shrine.hub.shouldQuerySelf", "") == "true" val create = configMap.getOrElse("shrine.hub.create", "") == "true" val downstreamNodes = for((k,v) <- configMap.filterKeys(_.toLowerCase.startsWith ("shrine.hub.downstreamnodes"))) yield (k.split('.').last, v) Hub(shouldQuerySelf, create, downstreamNodes) } } // -- if needed -- // case class TimeoutInfo (timeUnit:String, description:String) case class DatabaseInfo(createTablesOnStart:Boolean, dataSourceFrom:String, jndiDataSourceName:String, slickProfileClassName:String) case class Audit(database:DatabaseInfo, collectQepAudit:Boolean) object Audit{ def apply(configMap:Map[String, String]):Audit = { val key = "shrine.queryEntryPoint.audit." val createTablesOnStart = configMap.getOrElse(key + "database.createTablesOnStart", "") == "true" val dataSourceFrom = configMap.getOrElse(key + "database.dataSourceFrom", "") val jndiDataSourceName = configMap.getOrElse(key + "database.jndiDataSourceName", "") val slickProfileClassName = configMap.getOrElse(key + "database.slickProfileClassName", "") val collectQepAudit = configMap.getOrElse(key + "collectQepAudit", "") == "true" val database = DatabaseInfo(createTablesOnStart, dataSourceFrom, jndiDataSourceName, slickProfileClassName) Audit(database, collectQepAudit) } } case class QEP(maxQueryWaitTimeMinutes:Int, create:Boolean,attachSigningCert:Boolean, authorizationType:String, includeAggregateResults:Boolean, authenticationType:String, audit:Audit) object QEP{ def apply(configMap:Map[String, String]):QEP = { val key = "shrine.queryEntryPoint." val sheriffUsername = configMap.getOrElse(key + "sheriffCredentials.username", "") val maxQueryWaitTimeMinutes = configMap.getOrElse(key + "maxQueryWaitTime.minutes", "").toInt val create = configMap.getOrElse(key + "create", "") == "true" val attachSigningCert = configMap.getOrElse(key + "attachSigningCert", "") == "true" val authorizationType = configMap.getOrElse(key + "authorizationType", "") val includeAggregateResults = configMap.getOrElse(key + "includeAggregateResults", "") == "true" val authenticationType = configMap.getOrElse(key + "authenticationType", "") val audit = Audit(configMap) val sheriffServiceEndpoint = Endpoint(configMap, "queryEntryPoint.sheriff") val broadcasterServiceEndpoint = Endpoint(configMap, "queryEntryPoint.broadcasterService") QEP(maxQueryWaitTimeMinutes, create, attachSigningCert, authorizationType, includeAggregateResults, authenticationType, audit) } } /* val shrineConfig = keyValues.filterKeys(_.toLowerCase.startsWith("shrine")) def getHub = { val shouldQuerySelf = shrineConfig.getOrElse("shrine.hub.shouldQuerySelf", "") val create = shrineConfig.getOrElse("shrine.hub.create", "") val downstreamNodes = for((k,v) <- shrineConfig.filterKeys(_.toLowerCase.startsWith ("shrine" + ".hub.downstreamnodes"))) yield (k.split('.').last, v) case class Hub(shouldQuerySelf:String, create:String, downstreamNodes:Map[String, String]) Hub(shouldQuerySelf, create, downstreamNodes) } */ //adapted from https://gist.github.com/joseraya/176821d856b43b1cfe19 object gruntWatchCorsSupport extends Directive0 with RouteConcatenation { import spray.http.HttpHeaders.{`Access-Control-Allow-Methods`, `Access-Control-Max-Age`, `Access-Control-Allow-Headers`,`Access-Control-Allow-Origin`} import spray.routing.directives.RespondWithDirectives.respondWithHeaders import spray.routing.directives.MethodDirectives.options import spray.routing.directives.RouteDirectives.complete import spray.http.HttpMethods.{OPTIONS,GET,POST} import spray.http.AllOrigins private val allowOriginHeader = `Access-Control-Allow-Origin`(AllOrigins) private val optionsCorsHeaders = List( `Access-Control-Allow-Headers`("Origin, X-Requested-With, Content-Type, Accept, Accept-Encoding, Accept-Language, Host, Referer, User-Agent, Authorization"), `Access-Control-Max-Age`(1728000)) //20 days val gruntWatch:Boolean = DashboardConfigSource.config.getBoolean("shrine.dashboard.gruntWatch") override def happly(f: (HNil) => Route): Route = { if(gruntWatch) { options { respondWithHeaders(`Access-Control-Allow-Methods`(OPTIONS, GET, POST) :: allowOriginHeader :: optionsCorsHeaders){ complete(StatusCodes.OK) } } ~ f(HNil) } else f(HNil) } } diff --git a/qep/service/src/main/scala/net/shrine/qep/AbstractQepService.scala b/qep/service/src/main/scala/net/shrine/qep/AbstractQepService.scala index 5057366c5..bf4c5467a 100644 --- a/qep/service/src/main/scala/net/shrine/qep/AbstractQepService.scala +++ b/qep/service/src/main/scala/net/shrine/qep/AbstractQepService.scala @@ -1,210 +1,215 @@ package net.shrine.qep import net.shrine.log.Loggable import net.shrine.qep.audit.QepAuditDb import net.shrine.qep.dao.AuditDao import net.shrine.authentication.Authenticator import net.shrine.authorization.QueryAuthorizationService import net.shrine.broadcaster.BroadcastAndAggregationService import net.shrine.qep.queries.QepQueryDb import scala.concurrent.duration.Duration import net.shrine.util.XmlDateHelper import scala.concurrent.Future import scala.concurrent.Await import net.shrine.protocol.{ReadPreviousQueriesResponse, RunQueryRequest, BaseShrineRequest, AuthenticationInfo, Credential, BaseShrineResponse, ReadQueryInstancesRequest, QueryInstance, ReadQueryInstancesResponse, ReadQueryDefinitionRequest, DeleteQueryRequest, ReadApprovedQueryTopicsRequest, ReadInstanceResultsRequest, ReadPreviousQueriesRequest, RenameQueryRequest, ReadPdoRequest, FlagQueryRequest, UnFlagQueryRequest, ReadResultOutputTypesRequest, ReadResultOutputTypesResponse, ResultOutputType} import net.shrine.authorization.AuthorizationResult.{Authorized, NotAuthorized} import net.shrine.authentication.AuthenticationResult import net.shrine.authentication.NotAuthenticatedException import net.shrine.aggregation.RunQueryAggregator import net.shrine.aggregation.Aggregators import net.shrine.aggregation.Aggregator import net.shrine.aggregation.ReadQueryDefinitionAggregator import net.shrine.aggregation.DeleteQueryAggregator import net.shrine.aggregation.ReadPdoResponseAggregator import net.shrine.aggregation.RenameQueryAggregator import net.shrine.aggregation.ReadInstanceResultsAggregator import net.shrine.aggregation.FlagQueryAggregator import net.shrine.aggregation.UnFlagQueryAggregator /** * @author clint * @since Feb 19, 2014 */ trait AbstractQepService[BaseResp <: BaseShrineResponse] extends Loggable { val commonName:String val auditDao: AuditDao val authenticator: Authenticator val authorizationService: QueryAuthorizationService val includeAggregateResult: Boolean val broadcastAndAggregationService: BroadcastAndAggregationService val queryTimeout: Duration val breakdownTypes: Set[ResultOutputType] val collectQepAudit:Boolean protected def doReadResultOutputTypes(request: ReadResultOutputTypesRequest): BaseResp = { info(s"doReadResultOutputTypes($request)") authenticateAndThen(request) { authResult => val resultOutputTypes = ResultOutputType.nonErrorTypes ++ breakdownTypes //TODO: XXX: HACK: Would like to remove the cast ReadResultOutputTypesResponse(resultOutputTypes).asInstanceOf[BaseResp] } } protected def doFlagQuery(request: FlagQueryRequest, shouldBroadcast: Boolean = true): BaseResp = { - info(s"Flag request is $request") - QepQueryDb.db.insertQepQueryFlag(request) doBroadcastQuery(request, new FlagQueryAggregator, shouldBroadcast) } protected def doUnFlagQuery(request: UnFlagQueryRequest, shouldBroadcast: Boolean = true): BaseResp = { QepQueryDb.db.insertQepQueryFlag(request) doBroadcastQuery(request, new UnFlagQueryAggregator, shouldBroadcast) } protected def doRunQuery(request: RunQueryRequest, shouldBroadcast: Boolean): BaseResp = { info(s"doRunQuery($request,$shouldBroadcast) with $runQueryAggregatorFor") //store the query in the qep's database doBroadcastQuery(request, runQueryAggregatorFor(request), shouldBroadcast) } protected def doReadQueryDefinition(request: ReadQueryDefinitionRequest, shouldBroadcast: Boolean): BaseResp = { + info(s"doReadQueryDefinition($request,$shouldBroadcast)") + doBroadcastQuery(request, new ReadQueryDefinitionAggregator, shouldBroadcast) } protected def doReadPdo(request: ReadPdoRequest, shouldBroadcast: Boolean): BaseResp = { + info(s"doReadPdo($request,$shouldBroadcast)") doBroadcastQuery(request, new ReadPdoResponseAggregator, shouldBroadcast) } protected def doReadInstanceResults(request: ReadInstanceResultsRequest, shouldBroadcast: Boolean): BaseResp = { + info(s"doReadInstanceResults($request,$shouldBroadcast)") doBroadcastQuery(request, new ReadInstanceResultsAggregator(request.shrineNetworkQueryId, false), shouldBroadcast) } protected def doReadQueryInstances(request: ReadQueryInstancesRequest, shouldBroadcast: Boolean): BaseResp = { - info(s"doReadQueryInstances($request)") + info(s"doReadQueryInstances($request,$shouldBroadcast)") authenticateAndThen(request) { authResult => val now = XmlDateHelper.now val networkQueryId = request.queryId val username = request.authn.username val groupId = request.projectId //NB: Return a dummy response, with a dummy QueryInstance containing the network (Shrine) id of the query we'd like //to get "instances" for. This allows the legacy web client to formulate a request for query results that Shrine //can understand, while meeting the conversational requirements of the legacy web client. val instance = QueryInstance(networkQueryId.toString, networkQueryId.toString, username, groupId, now, now) //TODO: XXX: HACK: Would like to remove the cast //NB: Munge in username from authentication result ReadQueryInstancesResponse(networkQueryId, authResult.username, groupId, Seq(instance)).asInstanceOf[BaseResp] } } protected def doReadPreviousQueries(request: ReadPreviousQueriesRequest, shouldBroadcast: Boolean): ReadPreviousQueriesResponse = { + info(s"doReadPreviousQueries($request,$shouldBroadcast)") //pull results from the local database. QepQueryDb.db.selectPreviousQueries(request) } protected def doRenameQuery(request: RenameQueryRequest, shouldBroadcast: Boolean): BaseResp = { + info(s"doRenameQuery($request,$shouldBroadcast)") doBroadcastQuery(request, new RenameQueryAggregator, shouldBroadcast) } protected def doDeleteQuery(request: DeleteQueryRequest, shouldBroadcast: Boolean): BaseResp = { + info(s"doDeleteQuery($request,$shouldBroadcast)") doBroadcastQuery(request, new DeleteQueryAggregator, shouldBroadcast) } protected def doReadApprovedQueryTopics(request: ReadApprovedQueryTopicsRequest, shouldBroadcast: Boolean): BaseResp = authenticateAndThen(request) { _ => - info(s"doReadApprovedQueryTopics($request)") + info(s"doReadApprovedQueryTopics($request,$shouldBroadcast)") //TODO: XXX: HACK: Would like to remove the cast authorizationService.readApprovedEntries(request) match { case Left(errorResponse) => errorResponse.asInstanceOf[BaseResp] case Right(validResponse) => validResponse.asInstanceOf[BaseResp] } } import broadcastAndAggregationService.sendAndAggregate protected def doBroadcastQuery(request: BaseShrineRequest, aggregator: Aggregator, shouldBroadcast: Boolean): BaseResp = { authenticateAndThen(request) { authResult => debug(s"doBroadcastQuery($request) authResult is $authResult") //NB: Use credentials obtained from Authenticator (oddly, we authenticate with one set of credentials and are "logged in" under (possibly!) another //When making BroadcastMessages val networkAuthn = AuthenticationInfo(authResult.domain, authResult.username, Credential("", isToken = false)) //NB: Only audit RunQueryRequests request match { case runQueryRequest: RunQueryRequest => // inject modified, authorized runQueryRequest auditAuthorizeAndThen(runQueryRequest) { authorizedRequest => debug(s"doBroadcastQuery authorizedRequest is $authorizedRequest") // tuck the ACT audit metrics data into a database here if (collectQepAudit) QepAuditDb.db.insertQepQuery(authorizedRequest,commonName) QepQueryDb.db.insertQepQuery(authorizedRequest) doSynchronousQuery(networkAuthn,authorizedRequest,aggregator,shouldBroadcast) } case _ => doSynchronousQuery(networkAuthn,request,aggregator,shouldBroadcast) } } } private def doSynchronousQuery(networkAuthn: AuthenticationInfo,request: BaseShrineRequest, aggregator: Aggregator, shouldBroadcast: Boolean) = { info(s"doSynchronousQuery($request) started") val response = waitFor(sendAndAggregate(networkAuthn, request, aggregator, shouldBroadcast)).asInstanceOf[BaseResp] info(s"doSynchronousQuery($request) completed with response $response") response } private[qep] val runQueryAggregatorFor: RunQueryRequest => RunQueryAggregator = Aggregators.forRunQueryRequest(includeAggregateResult) protected def waitFor[R](futureResponse: Future[R]): R = { XmlDateHelper.time("Waiting for aggregated results")(debug(_)) { Await.result(futureResponse, queryTimeout) } } private[qep] def auditAuthorizeAndThen[T](request: RunQueryRequest)(body: (RunQueryRequest => T)): T = { auditTransactionally(request) { debug(s"auditAuthorizeAndThen($request) with $authorizationService") val authorizedRequest = authorizationService.authorizeRunQueryRequest(request) match { case na: NotAuthorized => throw na.toException case authorized: Authorized => request.copy(topicName = authorized.topicIdAndName.map(x => x._2)) } body(authorizedRequest) } } private[qep] def auditTransactionally[T](request: RunQueryRequest)(body: => T): T = { try { body } finally { auditDao.addAuditEntry( request.projectId, request.authn.domain, request.authn.username, request.queryDefinition.toI2b2String, //TODO: Use i2b2 format Still? request.topicId) } } import AuthenticationResult._ private[qep] def authenticateAndThen[T](request: BaseShrineRequest)(f: Authenticated => T): T = { val AuthenticationInfo(domain, username, _) = request.authn val authResult = authenticator.authenticate(request.authn) authResult match { case a: Authenticated => f(a) case NotAuthenticated(_, _, reason) => throw new NotAuthenticatedException(s"User $domain:$username could not be authenticated: $reason") } } } \ No newline at end of file