diff --git a/adapter/adapter-service/src/main/sql/mssql.ddl b/adapter/adapter-service/src/main/sql/mssql.ddl index 1af337b5f..7a9875173 100644 --- a/adapter/adapter-service/src/main/sql/mssql.ddl +++ b/adapter/adapter-service/src/main/sql/mssql.ddl @@ -1,85 +1,88 @@ /* Audit db tables in adapterAuditDB */ create table "queriesReceived" ("shrineNodeId" VARCHAR(MAX) NOT NULL,"userName" VARCHAR(MAX) NOT NULL,"networkQueryId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"topicId" VARCHAR(MAX),"topicName" VARCHAR(MAX),"timeQuerySent" BIGINT NOT NULL,"timeReceived" BIGINT NOT NULL); create table "executionsStarted" ("networkQueryId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeExecutionStarted" BIGINT NOT NULL); create table "executionsCompleted" ("networkQueryId" BIGINT NOT NULL,"replyId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeExecutionCompleted" BIGINT NOT NULL); create table "resultsSent" ("networkQueryId" BIGINT NOT NULL,"replyId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeResultsSent" BIGINT NOT NULL); /* Working tables in shrine_query_history */ +create database shrine_query_history; +use shrine_query_history; + create table SHRINE_QUERY( [id] [int] not null IDENTITY(1,1), local_id [varchar](255) not null, network_id bigint not null, username [varchar](255) not null, domain [varchar](255) not null, query_name [varchar](255) not null, query_expression text, date_created datetime default current_timestamp, has_been_run bit not null default 0, flagged bit not null default 0, flag_message [varchar](255) null, constraint query_id_pk primary key clustered (id asc), query_xml text -) -CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_network_id] ON [dbo].[SHRINE_QUERY] ([network_id] ASC) -CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_local_id] ON [dbo].[SHRINE_QUERY] ([local_id] ASC) -CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_username_domain] ON [dbo].[SHRINE_QUERY] (username, domain ASC) +); +CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_network_id] ON [dbo].[SHRINE_QUERY] ([network_id] ASC); +CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_local_id] ON [dbo].[SHRINE_QUERY] ([local_id] ASC); +CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_username_domain] ON [dbo].[SHRINE_QUERY] (username, domain ASC); -alter table SHRINE_QUERY alter column flag_message [varchar](MAX) +alter table SHRINE_QUERY alter column flag_message [varchar](MAX); create table QUERY_RESULT( id int not null identity(1,1), local_id varchar(255) not null, query_id int not null, [type] varchar(255) not null check ([type] in ('PATIENTSET','PATIENT_COUNT_XML','PATIENT_AGE_COUNT_XML','PATIENT_RACE_COUNT_XML','PATIENT_VITALSTATUS_COUNT_XML','PATIENT_GENDER_COUNT_XML','ERROR')), [status] varchar(50) not null check ([status] in ('FINISHED', 'ERROR', 'PROCESSING', 'QUEUED')), time_elapsed int null, last_updated datetime default current_timestamp, constraint QUERY_RESULT_id_pk primary key(id), constraint fk_QUERY_RESULT_query_id foreign key (query_id) references SHRINE_QUERY (id) on delete cascade -) +); create table ERROR_RESULT( id int not null identity(1,1), result_id int not null, message varchar(255) not null, constraint ERROR_RESULT_id_pk primary key(id), constraint fk_ERROR_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade -) -alter table ERROR_RESULT add column 'CODEC' varchar not null default "Pre-1.20 Error" -alter table ERROR_RESULT add column 'SUMMARY' text not null default "Pre-1.20 Error" -alter table ERROR_RESULT add column 'DESCRIPTION' text not null default "Pre-1.20 Error" -alter table ERROR_RESULT add column 'DETAILS' text not null default "Pre-1.20 Error" +); +alter table ERROR_RESULT add CODEC text not null default 'Pre-1.20 Error'; +alter table ERROR_RESULT add SUMMARY text not null default 'Pre-1.20 Error'; +alter table ERROR_RESULT add DESCRIPTION text not null default 'Pre-1.20 Error'; +alter table ERROR_RESULT add DETAILS text not null default 'Pre-1.20 Error'; create table COUNT_RESULT( id int not null identity(1,1), result_id int not null, original_count int not null, obfuscated_count int not null, date_created datetime default current_timestamp, constraint COUNT_RESULT_id_pk primary key(id), constraint fk_COUNT_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade -) +); create table BREAKDOWN_RESULT( id int not null identity(1,1), result_id int not null, data_key varchar(255) not null, original_value int not null, obfuscated_value int not null, constraint BREAKDOWN_RESULT_id_pk primary key(id), constraint fk_BREAKDOWN_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade -) +); create table PRIVILEGED_USER( id int not null identity(1,1), username varchar(255) not null, domain varchar(255) not null, threshold int not null, override_date timestamp null, constraint priviliged_user_pk primary key(id), constraint ix_PRIVILEGED_USER_username_domain unique (username, domain) -) +); create table "problems" ("id" INTEGER NOT NULL PRIMARY KEY IDENTITY, "codec" VARCHAR(254) NOT NULL,"stampText" VARCHAR(500) NOT NULL,"summary" VARCHAR(254) NOT NULL,"description" VARCHAR(MAX) NOT NULL,"detailsXml" VARCHAR(MAX) NOT NULL,"epoch" BIGINT NOT NULL); create index "idx_epoch" on "problems" ("epoch"); diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js index ef1f52aac..e027520b5 100644 --- a/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js +++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js @@ -1,247 +1,247 @@ (function (){ 'use strict'; // -- angular module -- // angular.module('shrine-tools') .factory('DiagnosticModel', DiagnosticModel); DiagnosticModel.$inject = ['$http', '$q', 'UrlGetter', '$location']; function DiagnosticModel (h, q, urlGetter, $location) { var toDashboard = {url:''}; var cache = {}; // used solely for remote dashboard persistence var m = {}; m.remoteSiteStatuses = []; m.siteAlias = ''; // -- private const -- // var Config = { AdapterEndpoint: 'admin/status/adapter', ConfigEndpoint: 'admin/status/config', HubEndpoint: 'admin/status/hub', I2B2Endpoint: 'admin/status/i2b2', KeystoreEndpoint: 'admin/status/keystore', OptionsEndpoint: 'admin/status/optionalParts', ProblemEndpoint: 'admin/status/problems', QepEndpoint: 'admin/status/qep', SummaryEndpoint: 'admin/status/summary' }; // -- public -- // return { getAdapter: getJsonMaker(Config.AdapterEndpoint, 'adapter'), getConfig: getJsonMaker(Config.ConfigEndpoint, 'config', parseConfig), getHub: getJsonMaker(Config.HubEndpoint, 'hub'), getI2B2: getJsonMaker(Config.I2B2Endpoint, 'i2b2'), getKeystore: getJsonMaker(Config.KeystoreEndpoint, 'keystore', storeRemoteSites), getOptionalParts: getJsonMaker(Config.OptionsEndpoint, 'optionalParts'), getProblems: getProblemsMaker(), getQep: getJsonMaker(Config.QepEndpoint, 'qep'), getSummary: getJsonMaker(Config.SummaryEndpoint, 'summary'), safeLogout: safeLogout, clearCache: clearCache, map: map, formatDate: formatDate, cache: cache, toDashboard: toDashboard, m: m }; function map(func, list) { var result = []; for(var i = 0; i < list.length; i++) { result.push(func(list[i])) } return result; } function formatDate(dateObject) { return [dateObject.getUTCFullYear(), "-", pad2(dateObject.getUTCMonth() + 1), "-", - pad2(dateObject.getUTCDay()), " ", + pad2(dateObject.getUTCDate()), " ", pad2(dateObject.getUTCHours()), ":", pad2(dateObject.getUTCMinutes()), ":", pad2(dateObject.getUTCSeconds())].join(""); } function pad2(stringLikeThing) { // Does javascript provide a string format thing? Would love to write %02d here. var stringed = "" + stringLikeThing; if (stringed.length > 2) { return stringed; } return ("00" + stringed).slice(-2); } /** * Clears the current remote dashboard before logging out. */ function safeLogout() { clearCache(); toDashboard.url = ''; m.siteAlias = ''; $location.path('/login'); } function clearCache() { for (var member in cache) { if(cache.hasOwnProperty(member)) delete cache[member]; } } /** * Method for Handling a failed rest call. * @param failedResult * @returns {*} */ function onFail(failedResult) { return q.reject(failedResult); } /*** * Method for handling a successful rest call. Simply caches it and returns it. * @param result * @param cacheKey * @returns {*} */ function parseJsonResult(result, cacheKey) { cache[cacheKey] = result.data; return result.data; } /** * Still cache and return the result, however, save the RemoteSites outside of the cache, * as we don't want these values to change between cache resets (which occur when switching sites) * @param result * @param cacheKey */ function storeRemoteSites(result, cacheKey) { cache[cacheKey] = result.data; if (m.remoteSiteStatuses.length == 0) { m.remoteSiteStatuses = result.data.remoteSiteStatuses; } return result.data } /** * Parses the json config map and turns it into a nested json object * @param json the flat config map * @param cacheKey a unique identifier for the function */ function parseConfig (json, cacheKey) { var configMap = json.data.configMap; var processed = preProcessJson(configMap); cache[cacheKey] = processed; return processed; } // IE11 doesn't support string includes // This only searchers for characters, not arbitrary strings function stringIncludes(haystack, needle) { var arr = haystack.split(""); for (var i = 0; i < arr.length; i++) { if (arr[i] == needle) { return true; } } return false; } // "explodes" and merges the flag config map. // e.g., {"key.foo": 10, "key.baz": 5} -> {"key": {"foo": 10, "baz": 5}} function preProcessJson (object) { var result = {}; for (var key in object) { if (object.hasOwnProperty(key)) { if (!stringIncludes(key, ".")) { result[key] = object[key] } else { var split = key.split("."); var prev = result; for (var i = 0; i < split.length; i++) { var cur = split[i]; if (!(cur in prev)) { prev[cur] = {} } if (i == split.length - 1) { prev[cur] = object[key]; } else { prev = prev[cur] } } } } } return result; } /** * There's a lot going on here. Essentially, this is a function factory that allows one to * define backend calls just through the path. It also implements a simple caching * strategy. * Essentially the get function only needs to be called once, and from then on it will spit * back a cached promise. This lets you write the code and not care whether it's cached or * not, but also get the caching performance anyways. For this function to work, the * resolver function has to take in the http response and the cache key to set, and make * sure that it caches what it returns (see parseJsonResult or parseConfig). * @param endpoint * @param cacheKey * @param resolverDefault * @returns {Function} */ function getJsonMaker(endpoint, cacheKey, resolverDefault) { var resolver = (typeof resolverDefault !== 'undefined')? function (response) { return resolverDefault(response, cacheKey) }: function (response) { return parseJsonResult(response, cacheKey) }; return function() { var cachedValue = cache[cacheKey]; if (cachedValue === undefined) { var url = urlGetter(endpoint, undefined, toDashboard.url); return h.get(url) .then(resolver, onFail) } else { return q(function(resolver) { resolver(cachedValue)}); } } } function getProblemsMaker() { // Caches the last offset and page size to hold onto it between different views var prevOffset = 0; var prevN = 20; /** * ProblemEndpoint: 'admin/status/problems', * @returns {*} */ return function(offset, n, epoch) { if (offset != null) { prevOffset = offset; } else { offset = prevOffset; } if (n != null) { prevN = n; } else { n = prevN; } var epochString = epoch && isFinite(epoch) ? '&epoch=' + epoch : ''; var url = urlGetter( Config.ProblemEndpoint + '?offset=' + offset + '&n=' + n + epochString, undefined, toDashboard.url ); return h.get(url) .then(parseJsonResult, onFail); } } } })(); diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js index 7c97bb946..ccdbe9d71 100644 --- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js +++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js @@ -1,92 +1,92 @@ /** * Controller for the Remote Dashboards panel. * Parses the keystore get call, and has to handle * some tricky caching logic with the model backend * (namely we don't want to reset the dashboard links themselves * between remoteDashboard visits) */ (function () { 'use strict'; // -- register controller with angular -- // angular.module('shrine-tools') .controller('DashboardController', DashboardController); /** * * @type {string[]} */ DashboardController.$inject = ['$app', '$log', '$location']; function DashboardController ($app, $log, $location) { var vm = this; var map = $app.model.map; vm.keyStoreError = false; init(); /** * */ function init () { $app.model.getKeystore() .then(setDashboard, handleFailure); } function handleFailure (failure) { vm.keyStoreError = failure; } /** * * @param keystore */ function setDashboard (keystore) { var modelStatuses = $app.model.m.remoteSiteStatuses; var tempList = []; for (var i = 0; i < modelStatuses.length; i++) { var abbreviatedEntry = modelStatuses[i]; if (abbreviatedEntry.url != "") // ignore self tempList.push(abbreviatedEntry) } - vm.otherDashboards = [['Hub', '']].concat(map(entryToPair, tempList)); + vm.otherDashboards = [['Hub', '', true]].concat(map(entryToPair, tempList)); vm.otherDashboards.sort(comparator); vm.clearCache = clearCache; vm.switchDashboard = switchDashboard; } /** * Lexicographic sort where Hub is always first. I'm sure there's a more * golf-way of writing this. */ function comparator(first, second) { if (first[0] == 'Hub') { return -2; } else if (second[0] == 'Hub') { return 2; } else { var less = first[0].toLowerCase() < second[0].toLowerCase(); var eq = first[0].toLowerCase() == second[0].toLowerCase(); return less? -1: eq? 0 : 1 } } //todo remove duplication with header.js function switchDashboard(url, alias) { $app.model.toDashboard.url = url; $app.model.m.siteAlias = alias == 'Hub'? '': alias; clearCache(); $location.url("/diagnostic/summary"); } function clearCache() { $app.model.clearCache(); } function entryToPair(entry){ - return [entry.siteAlias, entry.url]; + return [entry.siteAlias, entry.url, entry.theyHaveMine && entry.haveTheirs && !entry.timeOutError]; } } })(); diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html index a30ca468b..5bd8c8af0 100644 --- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html +++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html @@ -1,17 +1,19 @@
+ flex-flow: column wrap; + float: left"> -
-
\ No newline at end of file diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js index 6faf3d130..6e86e4d77 100644 --- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js +++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js @@ -1,71 +1,81 @@ (function () { 'use strict'; // -- register controller with shrine-tools module angular.module('shrine-tools') .controller('SummaryController', SummaryController); /** * Summary Controller. * */ - SummaryController.$inject = ['$app', '$sce', '$log'] - function SummaryController ($app, $sce, $log) { + SummaryController.$inject = ['$app', '$sce', '$log', '$timeout']; + function SummaryController ($app, $sce, $log, $timeout) { var vm = this; var unknown = 'UNKNOWN'; vm.summaryError = false; vm.i2b2Error = false; + vm.loading = true; $app.model.reloadSummary = init; init(); /** * */ function init() { + vm.loading = true; + var fifteenSeconds = 15*1000; $app.model.getSummary() .then(setSummary, handleSummaryFailure); $app.model.getI2B2() .then(setI2B2, handleI2B2Failure); + + $timeout(setTimeoutError, fifteenSeconds); + } + + function setTimeoutError() { + vm.loading = false; } function handleSummaryFailure(failure) { vm.summaryError = failure; } function handleI2B2Failure(failure) { vm.i2b2Error = failure; } function formatDate(maybeEpoch) { if (!(maybeEpoch && isFinite(maybeEpoch))) { return unknown; } else { var d = new Date(maybeEpoch); return $app.model.formatDate(d); } } /** * * @param summary */ function setSummary(summary) { - vm.summary = summary; + vm.loading = false; + vm.summary = summary; if (vm.summary.adapterMappingsFileName === undefined) { vm.summary.adapterMappingsFileName = unknown; } else if (vm.summary.adapterMappingsDate === undefined) { vm.summary.adapterMappingsDate = unknown; } else { vm.summary.adapterMappingsDate = formatDate(vm.summary.adapterMappingsDate); } return this; } function setI2B2(i2b2) { vm.ontProject = i2b2.ontProject; } } })(); diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html index 1985b7071..fe1eb4bb6 100644 --- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html +++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html @@ -1,121 +1,134 @@
Version Info
This site is running SHRINE {{vm.summary.shrineVersion}} built on
This site is currently using ontology version
Based on concept term:
This site is currently using for mappings, last edited on
Error: click here for more details. OK
Error: click here for more details. OK
Error: click here for more details. OK
Error: click here for more details. OK
Fatal SHRINE Error
+
+
+

Timeout or Error while calling Summary, please check the Problems tab if available

+
+

Summary Error

+ {{vm.summaryError}} +
+
+

i2b2 Error

+ {{vm.i2b2Error}} +
+
+
\ No newline at end of file diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala index 36f83c3f9..d69f27359 100644 --- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala +++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/DashboardService.scala @@ -1,527 +1,528 @@ package net.shrine.dashboard import akka.actor.Actor import akka.event.Logging import net.shrine.authentication.UserAuthenticator import net.shrine.authorization.steward.OutboundUser import net.shrine.config.ConfigExtensions import net.shrine.crypto.{BouncyKeyStoreCollection, KeyStoreDescriptorParser, UtilHasher} import net.shrine.dashboard.httpclient.HttpClientDirectives.{forwardUnmatchedPath, requestUriThenRoute} import net.shrine.dashboard.jwtauth.ShrineJwtAuthenticator import net.shrine.i2b2.protocol.pm.User import net.shrine.log.Loggable import net.shrine.problem.{ProblemDigest, Problems} import net.shrine.serialization.NodeSeqSerializer import net.shrine.source.ConfigSource import net.shrine.spray._ import net.shrine.status.protocol.{Config => StatusProtocolConfig} import net.shrine.util.SingleHubModel import org.json4s.native.JsonMethods.{parse => json4sParse} import org.json4s.{DefaultFormats, Formats} import shapeless.HNil import spray.http.{HttpRequest, HttpResponse, StatusCodes, Uri} import spray.httpx.Json4sSupport import spray.routing._ import spray.routing.directives.LogEntry import scala.collection.immutable.Iterable import scala.concurrent.ExecutionContext.Implicits.global /** * Mixes the DashboardService trait with an Akka Actor to provide the actual service. */ class DashboardServiceActor extends Actor with DashboardService { // the HttpService trait defines only one abstract member, which // connects the services environment to the enclosing actor or test def actorRefFactory = context // this actor only runs our route, but you could add // other things here, like request stream processing // or timeout handling def receive = runRoute(route) } /** * A web service that provides the Dashboard endpoints. It is a trait to support testing independent of Akka. */ trait DashboardService extends HttpService with Loggable { val userAuthenticator = UserAuthenticator(ConfigSource.config) //don't need to do anything special for unauthorized users, but they do need access to a static form. lazy val route:Route = gruntWatchCorsSupport { redirectToIndex ~ staticResources ~ makeTrouble ~ about ~ authenticatedInBrowser ~ authenticatedDashboard ~ post { // Chicken and egg problem; Can't check status of certs validation between sites if you need valid certs to exchange messages pathPrefix("status") pathPrefix("verifySignature") verifySignature } } /** logs the request method, uri and response at info level */ def logEntryForRequestResponse(req: HttpRequest): Any => Option[LogEntry] = { case res: HttpResponse => Some(LogEntry(s"\n Request: $req\n Response: $res", Logging.InfoLevel)) case _ => None // other kind of responses } /** logs just the request method, uri and response status at info level */ def logEntryForRequest(req: HttpRequest): Any => Option[LogEntry] = { case res: HttpResponse => Some(LogEntry(s"\n Request: $req\n Response status: ${res.status}", Logging.InfoLevel)) case _ => None // other kind of responses } def authenticatedInBrowser: Route = pathPrefixTest("user"|"admin"|"toDashboard") { logRequestResponse(logEntryForRequestResponse _) { //logging is controlled by Akka's config, slf4j, and log4j config reportIfFailedToAuthenticate { authenticate(userAuthenticator.basicUserAuthenticator) { user => pathPrefix("user") { userRoute(user) } ~ pathPrefix("admin") { adminRoute(user) } ~ pathPrefix("toDashboard") { toDashboardRoute(user) } } } } } val reportIfFailedToAuthenticate = routeRouteResponse { case Rejected(List(AuthenticationFailedRejection(_,_))) => complete("AuthenticationFailed") } def authenticatedDashboard:Route = pathPrefix("fromDashboard") { logRequestResponse(logEntryForRequestResponse _) { //logging is controlled by Akka's config, slf4j, and log4j config get { //all remote dashboard calls are gets. authenticate(ShrineJwtAuthenticator.authenticate) { user => info(s"Sucessfully authenticated user `$user`") adminRoute(user) } } } } def makeTrouble = pathPrefix("makeTrouble") { complete(throw new IllegalStateException("fake trouble")) } lazy val redirectToIndex = pathEnd { redirect("shrine-dashboard/client/index.html", StatusCodes.PermanentRedirect) //todo pick up "shrine-dashboard" programatically } ~ ( path("index.html") | pathSingleSlash) { redirect("client/index.html", StatusCodes.PermanentRedirect) } lazy val staticResources = pathPrefix("client") { pathEnd { redirect("client/index.html", StatusCodes.PermanentRedirect) } ~ pathSingleSlash { redirect("index.html", StatusCodes.PermanentRedirect) } ~ { getFromResourceDirectory("client") } } lazy val about = pathPrefix("about") { complete("Nothing here yet") //todo } def userRoute(user:User):Route = get { pathPrefix("whoami") { complete(OutboundUser.createFromUser(user)) } } //todo check that this an admin. def adminRoute(user:User):Route = get { pathPrefix("happy") { val happyBaseUrl: String = ConfigSource.config.getString("shrine.dashboard.happyBaseUrl") forwardUnmatchedPath(happyBaseUrl) } ~ pathPrefix("messWithHappyVersion") { //todo is this used? val happyBaseUrl: String = ConfigSource.config.getString("shrine.dashboard.happyBaseUrl") def pullClasspathFromConfig(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { val result = httpResponse.entity.asString ctx.complete(s"Got '$result' from $uri") } } requestUriThenRoute(happyBaseUrl+"/version",pullClasspathFromConfig) } ~ pathPrefix("ping") { complete("pong") }~ pathPrefix("status") { statusRoute(user) } } //Manually test this by running a curl command //curl -k -w "\n%{response_code}\n" -u dave:kablam "https://shrine-dev1.catalyst:6443/shrine-dashboard/toDashboard/shrine-dev2.catalyst/ping" /** * Forward a request from this dashboard to a remote dashboard */ def toDashboardRoute(user:User):Route = get { pathPrefix(Segment) { dnsName => import scala.collection.JavaConversions._ val urlToParse: String = KeyStoreInfo.keyStoreDescriptor.trustModel match { case SingleHubModel(false) => ConfigSource.config.getString("shrine.queryEntryPoint.broadcasterServiceEndpoint.url") case _ => ConfigSource.config.getObject("shrine.hub.downstreamNodes").values.head.unwrapped.toString } val remoteDashboardPort = urlToParse.split(':')(2).split('/')(0) // TODO: Do ports vary between sites? val remoteDashboardProtocol = urlToParse.split("://")(0) val remoteDashboardPathPrefix = "shrine-dashboard/fromDashboard" // I don't think this needs to be configurable val baseUrl = s"$remoteDashboardProtocol://$dnsName:$remoteDashboardPort/$remoteDashboardPathPrefix" + info(s"toDashboardRoute: BaseURL: $baseUrl") forwardUnmatchedPath(baseUrl,Some(ShrineJwtAuthenticator.createOAuthCredentials(user, dnsName))) } } def statusRoute(user:User):Route = get { val( adapter , hub , i2b2 , keystore , optionalParts , qep , summary ) = ("adapter", "hub", "i2b2", "keystore", "optionalParts", "qep", "summary") pathPrefix("classpath") { getClasspath }~ pathPrefix("config") { getConfig }~ pathPrefix("problems") { getProblems }~ pathPrefix(adapter) { getFromSubService(adapter) }~ pathPrefix(hub) { getFromSubService(hub) }~ pathPrefix(i2b2) { getFromSubService(i2b2) }~ pathPrefix(keystore) { getFromSubService(keystore) }~ pathPrefix(optionalParts) { getFromSubService(optionalParts) }~ pathPrefix(qep) { getFromSubService(qep) }~ pathPrefix(summary) { getFromSubService(summary) } } val statusBaseUrl = ConfigSource.config.getString("shrine.dashboard.statusBaseUrl") // TODO: Move this over to Status API? lazy val verifySignature:Route = { formField("sha256".as[String].?) { sha256: Option[String] => val response = sha256.map(s => KeyStoreInfo.hasher.handleSig(s)) implicit val format = ShaResponse.json4sFormats response match { case None => complete(StatusCodes.BadRequest) case Some(sh@ShaResponse(ShaResponse.badFormat, _)) => complete(StatusCodes.BadRequest -> sh) case Some(sh@ShaResponse(_, false)) => complete(StatusCodes.NotFound -> sh) case Some(sh@ShaResponse(_, true)) => complete(StatusCodes.OK -> sh) } } } lazy val getConfig:Route = { def completeConfigRoute(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { val config = ParsedConfig(httpResponse.entity.asString) ctx.complete( ShrineConfig(config) ) } } requestUriThenRoute(statusBaseUrl + "/config", completeConfigRoute) } lazy val getClasspath:Route = { def pullClasspathFromConfig(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { val result = httpResponse.entity.asString val shrineConfig = ShrineConfig(ParsedConfig(result)) ctx.complete(shrineConfig) } } requestUriThenRoute(statusBaseUrl + "/config",pullClasspathFromConfig) } def getFromSubService(key: String):Route = { requestUriThenRoute(s"$statusBaseUrl/$key") } // table based view, can see N problems at a time. Front end sends how many problems that they want // to skip, and it will take N the 'nearest N' ie with n = 20, 0-19 -> 20, 20-39 -> 20-40 lazy val getProblems:Route = { def floorMod(x: Int, y: Int) = { x - (x % y) } val db = Problems.DatabaseConnector // Intellij loudly complains if you use parameters instead of chained parameter calls. // ¯\_(ツ)_/¯ parameter("offset".as[Int].?(0)) {(offsetPreMod:Int) => { parameter("n".as[Int].?(20)) {(nPreMax:Int) => parameter("epoch".as[Long].?) {(epoch:Option[Long]) => val n = Math.max(0, nPreMax) val moddedOffset = floorMod(Math.max(0, offsetPreMod), n) val query = for { result <- db.IO.sizeAndProblemDigest(n, moddedOffset) } yield (result._2, floorMod(Math.max(0, moddedOffset), n), n, result._1) val query2 = for { dateOffset <- db.IO.findIndexOfDate(epoch.getOrElse(0)) moddedOffset = floorMod(dateOffset, n) result <- db.IO.sizeAndProblemDigest(n, moddedOffset) } yield (result._2, moddedOffset, n, result._1) val queryReal = if (epoch.isEmpty) query else query2 val tupled = db.runBlocking(queryReal) val response: ProblemResponse = ProblemResponse(tupled._1, tupled._2, tupled._3, tupled._4) implicit val formats = response.json4sMarshaller complete(response) }}}} } } case class ProblemResponse(size: Int, offset: Int, n: Int, problems: Seq[ProblemDigest]) extends Json4sSupport { override implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer } object KeyStoreInfo { val config = ConfigSource.config val keyStoreDescriptor = KeyStoreDescriptorParser( config.getConfig("shrine.keystore"), config.getConfigOrEmpty("shrine.hub"), config.getConfigOrEmpty("shrine.queryEntryPoint")) val certCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(keyStoreDescriptor) val hasher = UtilHasher(certCollection) } /** * Centralized parsing logic for map of shrine.conf * the class literal `T.class` in Java. */ //todo most of this info should come directly from the status service in Shrine, not from reading the config case class ParsedConfig(configMap:Map[String, String]){ private val trueVal = "true" private val rootKey = "shrine" def isHub = getOrElse(rootKey + ".hub.create", "") .toLowerCase == trueVal def stewardEnabled = configMap.keySet .contains(rootKey + ".queryEntryPoint.shrineSteward") def shouldQuerySelf = getOrElse(rootKey + ".hub.shouldQuerySelf", "") .toLowerCase == trueVal def fromJsonString(jsonString:String): String = jsonString.split("\"").mkString("") def get(key:String): Option[String] = configMap.get(key).map(fromJsonString) def getOrElse(key:String, elseVal:String = ""): String = get(key).getOrElse(elseVal) } object ParsedConfig { def apply(jsonString:String):ParsedConfig = { implicit def json4sFormats: Formats = DefaultFormats ParsedConfig(json4sParse(jsonString).extract[StatusProtocolConfig].keyValues)//.filterKeys(_.toLowerCase.startsWith("shrine"))) } } case class DownstreamNode(name:String, url:String) object DownstreamNode { def create(configMap:Map[String,String]):Iterable[DownstreamNode] = { for ((k, v) <- configMap.filterKeys(_.toLowerCase.startsWith ("shrine.hub.downstreamnodes"))) yield DownstreamNode(k.split('.').last,v.split("\"").mkString("")) } } //todo replace with the actual config, scrubbed of passwords case class ShrineConfig(isHub:Boolean, hub:Hub, pmEndpoint:Endpoint, ontEndpoint:Endpoint, hiveCredentials: HiveCredentials, adapter: Adapter, queryEntryPoint:QEP, networkStatusQuery:String, configMap:Map[String, String] ) extends DefaultJsonSupport object ShrineConfig extends DefaultJsonSupport { def apply(config:ParsedConfig):ShrineConfig = { val hub = Hub(config) val isHub = config.isHub val pmEndpoint = Endpoint("pm",config) val ontEndpoint = Endpoint("ont",config) val hiveCredentials = HiveCredentials(config) val adapter = Adapter(config) val queryEntryPoint = QEP(config) val networkStatusQuery = config.configMap("shrine.networkStatusQuery") ShrineConfig(isHub, hub, pmEndpoint, ontEndpoint, hiveCredentials, adapter, queryEntryPoint, networkStatusQuery, config.configMap) } } case class Endpoint(acceptAllCerts:Boolean, url:String, timeoutSeconds:Int) object Endpoint{ def apply(endpointType:String,parsedConfig:ParsedConfig):Endpoint = { val prefix = "shrine." + endpointType.toLowerCase + "Endpoint." val acceptAllCerts = parsedConfig.configMap.getOrElse(prefix + "acceptAllCerts", "") == "true" val url = parsedConfig.configMap.getOrElse(prefix + "url","") val timeoutSeconds = parsedConfig.configMap.getOrElse(prefix + "timeout.seconds", "0").toInt Endpoint(acceptAllCerts, url, timeoutSeconds) } } case class HiveCredentials(domain:String, username:String, password:String, crcProjectId:String, ontProjectId:String) object HiveCredentials{ def apply(parsedConfig:ParsedConfig):HiveCredentials = { val key = "shrine.hiveCredentials." val domain = parsedConfig.configMap.getOrElse(key + "domain","") val username = parsedConfig.configMap.getOrElse(key + "username","") val password = "REDACTED" val crcProjectId = parsedConfig.configMap.getOrElse(key + "crcProjectId","") val ontProjectId = parsedConfig.configMap.getOrElse(key + "ontProjectId","") HiveCredentials(domain, username, password, crcProjectId, ontProjectId) } } // -- hub only -- // //todo delete when the Dashboard front end can use the status service's hub method case class Hub(shouldQuerySelf:Boolean, create:Boolean, downstreamNodes:Iterable[DownstreamNode]) object Hub{ def apply(parsedConfig:ParsedConfig):Hub = { val shouldQuerySelf = parsedConfig.shouldQuerySelf val create = parsedConfig.isHub val downstreamNodes = DownstreamNode.create(parsedConfig.configMap) Hub(shouldQuerySelf, create, downstreamNodes) } } // -- adapter info -- // case class Adapter(crcEndpointUrl:String, setSizeObfuscation:Boolean, adapterLockoutAttemptsThreshold:Int, adapterMappingsFilename:String) object Adapter{ def apply(parsedConfig:ParsedConfig):Adapter = { val key = "shrine.adapter." val crcEndpointUrl = parsedConfig.configMap.getOrElse(key + "crcEndpoint.url","") val setSizeObfuscation = parsedConfig.configMap.getOrElse(key + "setSizeObfuscation","").toLowerCase == "true" val adapterLockoutAttemptsThreshold = parsedConfig.configMap.getOrElse(key + "adapterLockoutAttemptsThreshold", "0").toInt val adapterMappingsFileName = parsedConfig.configMap.getOrElse(key + "adapterMappingsFileName","") Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileName) } } case class Steward(qepUserName:String, stewardBaseUrl:String) object Steward { def apply (parsedConfig:ParsedConfig):Steward = { val key = "shrine.queryEntryPoint.shrineSteward." val qepUserName = parsedConfig.configMap.getOrElse(key + "qepUserName","") val stewardBaseUrl = parsedConfig.configMap.getOrElse(key + "stewardBaseUrl","") Steward(qepUserName, stewardBaseUrl) } } // -- if needed -- // case class TimeoutInfo (timeUnit:String, description:String) case class DatabaseInfo(createTablesOnStart:Boolean, dataSourceFrom:String, jndiDataSourceName:String, slickProfileClassName:String) case class Audit(database:DatabaseInfo, collectQepAudit:Boolean) object Audit{ def apply(parsedConfig:ParsedConfig):Audit = { val key = "shrine.queryEntryPoint.audit." val createTablesOnStart = parsedConfig.configMap.getOrElse(key + "database.createTablesOnStart","") == "true" val dataSourceFrom = parsedConfig.configMap.getOrElse(key + "database.dataSourceFrom","") val jndiDataSourceName = parsedConfig.configMap.getOrElse(key + "database.jndiDataSourceName","") val slickProfileClassName = parsedConfig.configMap.getOrElse(key + "database.slickProfileClassName","") val collectQepAudit = parsedConfig.configMap.getOrElse(key + "collectQepAudit","") == "true" val database = DatabaseInfo(createTablesOnStart, dataSourceFrom, jndiDataSourceName, slickProfileClassName) Audit(database, collectQepAudit) } } case class QEP( maxQueryWaitTimeMinutes:Int, create:Boolean, attachSigningCert:Boolean, authorizationType:String, includeAggregateResults:Boolean, authenticationType:String, audit:Audit, shrineSteward:Steward, broadcasterServiceEndpointUrl:Option[String] ) object QEP{ val key = "shrine.queryEntryPoint." def apply(parsedConfig:ParsedConfig):QEP = QEP( maxQueryWaitTimeMinutes = parsedConfig.configMap.getOrElse(key + "maxQueryWaitTime.minutes", "0").toInt, create = parsedConfig.configMap.getOrElse(key + "create","") == "true", attachSigningCert = parsedConfig.configMap.getOrElse(key + "attachSigningCert","") == "true", authorizationType = parsedConfig.configMap.getOrElse(key + "authorizationType",""), includeAggregateResults = parsedConfig.configMap.getOrElse(key + "includeAggregateResults","") == "true", authenticationType = parsedConfig.configMap.getOrElse(key + "authenticationType", ""), audit = Audit(parsedConfig), shrineSteward = Steward(parsedConfig), broadcasterServiceEndpointUrl = parsedConfig.configMap.get(key + "broadcasterServiceEndpoint.url") ) } //adapted from https://gist.github.com/joseraya/176821d856b43b1cfe19 object gruntWatchCorsSupport extends Directive0 with RouteConcatenation { import spray.http.AllOrigins import spray.http.HttpHeaders.{`Access-Control-Allow-Headers`, `Access-Control-Allow-Methods`, `Access-Control-Allow-Origin`, `Access-Control-Max-Age`} import spray.http.HttpMethods.{GET, OPTIONS, POST} import spray.routing.directives.MethodDirectives.options import spray.routing.directives.RespondWithDirectives.respondWithHeaders import spray.routing.directives.RouteDirectives.complete private val allowOriginHeader = `Access-Control-Allow-Origin`(AllOrigins) private val optionsCorsHeaders = List( `Access-Control-Allow-Headers`("Origin, X-Requested-With, Content-Type, Accept, Accept-Encoding, Accept-Language, Host, Referer, User-Agent, Authorization"), `Access-Control-Max-Age`(1728000)) //20 days val gruntWatch:Boolean = ConfigSource.config.getBoolean("shrine.dashboard.gruntWatch") override def happly(f: (HNil) => Route): Route = { if(gruntWatch) { options { respondWithHeaders(`Access-Control-Allow-Methods`(OPTIONS, GET, POST) :: allowOriginHeader :: optionsCorsHeaders){ complete(StatusCodes.OK) } } ~ f(HNil) } else f(HNil) } } \ No newline at end of file diff --git a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/httpclient/HttpClientDirectives.scala b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/httpclient/HttpClientDirectives.scala index da1755a57..e40f2e7cf 100644 --- a/apps/dashboard-app/src/main/scala/net/shrine/dashboard/httpclient/HttpClientDirectives.scala +++ b/apps/dashboard-app/src/main/scala/net/shrine/dashboard/httpclient/HttpClientDirectives.scala @@ -1,206 +1,207 @@ package net.shrine.dashboard.httpclient import java.io.InputStream import java.security.cert.X509Certificate import javax.net.ssl.{SSLContext, X509TrustManager} import net.shrine.log.Loggable import spray.can.Http import akka.io.IO import akka.actor.{ActorRef, ActorSystem} import spray.can.Http.{ConnectionAttemptFailedException, HostConnectorSetup} import spray.http.{HttpCredentials, HttpEntity, HttpHeader, HttpHeaders, HttpRequest, HttpResponse, StatusCodes, Uri} import spray.io.ClientSSLEngineProvider import spray.routing.{RequestContext, Route} import akka.pattern.ask import net.shrine.source.ConfigSource import scala.concurrent.{Await, Future, TimeoutException, blocking} import scala.concurrent.duration.DurationInt import scala.language.postfixOps import scala.concurrent.ExecutionContext.Implicits.global import scala.util.control.NonFatal /** * From https://github.com/bthuillier/spray/commit/d31fc1b5e1415e1b908fe7d1f01f364a727e2593 with extra bits from http://www.cakesolutions.net/teamblogs/http-proxy-with-spray . * Replace when Spray has its own version. * * @author david * @since 9/14/15 */ trait HttpClientDirectives extends Loggable { implicit val system = ActorSystem("dashboardServer",ConfigSource.config) /** * Proxy the request to the specified base uri appended with the unmatched path. * */ def forwardUnmatchedPath(baseUri: Uri,maybeCredentials:Option[HttpCredentials] = None): Route = { def completeWithEntityAsString(httpResponse:HttpResponse,uri:Uri):Route = { ctx => { ctx.complete(httpResponse.entity.asString) } } requestWithUnmatchedPath(baseUri,completeWithEntityAsString,maybeCredentials) } /** * Make the request to the specified base uri appended with the unmatched path, then use the returned entity (as a string) to complete the route. * */ def requestWithUnmatchedPath(baseUri:Uri, route:(HttpResponse,Uri) => Route,maybeCredentials:Option[HttpCredentials] = None): Route = { ctx => { - val resourceUri = baseUri.withPath(baseUri.path.++(ctx.unmatchedPath)) + + val resourceUri = baseUri.withPath(baseUri.path.++(ctx.unmatchedPath)).withQuery(ctx.request.uri.query) requestUriThenRoute(resourceUri,route,maybeCredentials)(ctx) } } /** * Just pass the result through */ def passThrough(httpResponse: HttpResponse,uri: Uri):Route = ctx => ctx.complete(httpResponse.entity.asString) /** * proxy the request to the specified uri with the unmatched path, then use the returned entity (as a string) to complete the route. * */ def requestUriThenRoute( resourceUri:Uri, route:(HttpResponse,Uri) => Route = passThrough, maybeCredentials:Option[HttpCredentials] = None ): Route = { ctx => { val httpResponse = httpResponseForUri(resourceUri,ctx,maybeCredentials) info(s"Got $httpResponse for $resourceUri") handleCommonErrorsOrRoute(route)(httpResponse,resourceUri)(ctx) } } private def httpResponseForUri(resourceUri:Uri,ctx: RequestContext,maybeCredentials:Option[HttpCredentials] = None):HttpResponse = { info(s"Requesting $resourceUri") if(resourceUri.scheme == "classpath") ClasspathResourceHttpClient.loadFromResource(resourceUri.path.toString()) else { val basicRequest = HttpRequest(ctx.request.method,resourceUri) val request = maybeCredentials.fold(basicRequest){ (credentials: HttpCredentials) => val headers: List[HttpHeader] = basicRequest.headers :+ HttpHeaders.Authorization(credentials) basicRequest.copy(headers = headers) } HttpClient.webApiCall(request) } } def handleCommonErrorsOrRoute(route:(HttpResponse,Uri) => Route)(httpResponse: HttpResponse,uri:Uri): Route = { ctx => { if(httpResponse.status != StatusCodes.OK) { //todo create and report a problem val ctxCopy: RequestContext = ctx.withHttpResponseMapped(_.copy(status = httpResponse.status)) ctxCopy.complete(s"$uri replied with $httpResponse") } else route(httpResponse,uri)(ctx) } } } object HttpClientDirectives extends HttpClientDirectives /** * A simple HttpClient to use inside the HttpDirectives */ object HttpClient extends Loggable { //todo hand back a Try, Failures with custom exceptions instead of a crappy response def webApiCall(request:HttpRequest)(implicit system: ActorSystem): HttpResponse = { val transport: ActorRef = IO(Http)(system) debug(s"Requesting $request uri is ${request.uri} path is ${request.uri.path}") blocking { val future:Future[HttpResponse] = for { Http.HostConnectorInfo(connector, _) <- transport.ask(createConnector(request))(10 seconds) //todo make this timeout configurable response <- connector.ask(request)(10 seconds).mapTo[HttpResponse] //todo make this timeout configurable } yield response try { Await.result(future, 10 seconds) //todo make this timeout configurable } catch { case x:TimeoutException => HttpResponse(status = StatusCodes.RequestTimeout,entity = HttpEntity(s"${request.uri} timed out after 10 seconds. ${x.getMessage}")) //todo is there a better message? What comes up in real life? case x:ConnectionAttemptFailedException => { //no web service is there to respond info(s"${request.uri} failed with ${x.getMessage}",x) HttpResponse(status = StatusCodes.NotFound,entity = HttpEntity(s"${request.uri} failed with ${x.getMessage}")) } case NonFatal(x) => { info(s"${request.uri} failed with ${x.getMessage}",x) HttpResponse(status = StatusCodes.InternalServerError,entity = HttpEntity(s"${request.uri} failed with ${x.getMessage}")) } } } } //from https://github.com/TimothyKlim/spray-ssl-poc/blob/master/src/main/scala/Main.scala //trust all SSL contexts. We just want encrypted comms. implicit val trustfulSslContext: SSLContext = { class IgnoreX509TrustManager extends X509TrustManager { def checkClientTrusted(chain: Array[X509Certificate], authType: String) {} def checkServerTrusted(chain: Array[X509Certificate], authType: String) {} def getAcceptedIssuers = null } val context = SSLContext.getInstance("TLS") context.init(null, Array(new IgnoreX509TrustManager), null) info("trustfulSslContex initialized") context } implicit val clientSSLEngineProvider = //todo lookup this constructor ClientSSLEngineProvider { _ => val engine = trustfulSslContext.createSSLEngine() engine.setUseClientMode(true) engine } def createConnector(request: HttpRequest) = { val connector = new HostConnectorSetup(host = request.uri.authority.host.toString, port = request.uri.effectivePort, sslEncryption = request.uri.scheme == "https", defaultHeaders = request.headers) connector } } /** * For testing, get an HttpResponse for a classpath resource */ object ClasspathResourceHttpClient extends Loggable { def loadFromResource(resourceName:String):HttpResponse = { blocking { val cleanResourceName = if (resourceName.startsWith ("/") ) resourceName.drop(1) else resourceName val classLoader = getClass.getClassLoader try { val is: InputStream = classLoader.getResourceAsStream (cleanResourceName) val string:String = scala.io.Source.fromInputStream (is).mkString HttpResponse(entity = HttpEntity(string)) } catch{ case NonFatal(x) => { info(s"Could not load $resourceName",x) HttpResponse(status = StatusCodes.NotFound,entity = HttpEntity(s"Could not load $resourceName due to ${x.getMessage}")) } } } } } \ No newline at end of file diff --git a/apps/dashboard-app/src/test/scala/net/shrine/dashboard/DashboardServiceTest.scala b/apps/dashboard-app/src/test/scala/net/shrine/dashboard/DashboardServiceTest.scala index 584f9eecb..8f73f9119 100644 --- a/apps/dashboard-app/src/test/scala/net/shrine/dashboard/DashboardServiceTest.scala +++ b/apps/dashboard-app/src/test/scala/net/shrine/dashboard/DashboardServiceTest.scala @@ -1,434 +1,434 @@ package net.shrine.dashboard import java.security.PrivateKey import java.util.Date import io.jsonwebtoken.impl.TextCodec import io.jsonwebtoken.{Jwts, SignatureAlgorithm} import net.shrine.authorization.steward.OutboundUser import net.shrine.config.ConfigExtensions import net.shrine.crypto.{BouncyKeyStoreCollection, KeyStoreDescriptorParser} import net.shrine.dashboard.jwtauth.ShrineJwtAuthenticator import net.shrine.i2b2.protocol.pm.User import net.shrine.protocol.Credential import net.shrine.source.ConfigSource import net.shrine.spray.ShaResponse import org.json4s.native.JsonMethods.parse import org.junit.runner.RunWith import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner import spray.http.StatusCodes.{NotFound, OK, PermanentRedirect, Unauthorized} import spray.http.{BasicHttpCredentials, FormData, OAuth2BearerToken, StatusCodes} import spray.testkit.ScalatestRouteTest import scala.language.postfixOps @RunWith(classOf[JUnitRunner]) class DashboardServiceTest extends FlatSpec with ScalatestRouteTest with DashboardService { def actorRefFactory = system import scala.concurrent.duration._ implicit val routeTestTimeout = RouteTestTimeout(10 seconds) val adminUserName = "keith" val adminFullName = adminUserName /** * to run these tests with I2B2 * add a user named keith, to be the admin * add a Boolean parameter for keith, Admin, true * add all this user to the i2b2 project */ val adminCredentials = BasicHttpCredentials(adminUserName,"shh!") val brokenCredentials = BasicHttpCredentials(adminUserName,"wrong password") val adminUser = User( fullName = adminUserName, username = adminFullName, domain = "domain", credential = new Credential("admin's password",false), params = Map(), rolesByProject = Map() ) val adminOutboundUser = OutboundUser.createFromUser(adminUser) "DashboardService" should "return an OK and a valid outbound user for a user/whoami request" in { Get(s"/user/whoami") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) implicit val formats = OutboundUser.json4sFormats val userJson = new String(body.data.toByteArray) val outboundUser = parse(userJson).extract[OutboundUser] assertResult(adminOutboundUser)(outboundUser) } } "DashboardService" should "return an OK and a valid outbound user for a user/whoami request and an '' " in { Get(s"/user/whoami") ~> addCredentials(brokenCredentials) ~> route ~> check { assertResult(OK)(status) val response = new String(body.data.toByteArray) assertResult("""AuthenticationFailed""")(response) } } "DashboardService" should "redirect several urls to client/index.html" in { Get() ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/index.html") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/client") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } Get("/client/") ~> route ~> check { status === PermanentRedirect header("Location") === "client/index.html" } } - "DashboardService" should "return an OK and the right version string for an admin/happy/all test" in { + "DashboardService" should "return an OK and the right version string for an admin/happy/all?extra=true test" in { - Get(s"/admin/happy/all") ~> + Get(s"/admin/happy/all?extra=true") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val allString = new String(body.data.toByteArray) //todo test it to see if it's right } } "DashboardService" should "return an OK and mess with the right version string for an admin/messWithHappyVersion test" in { Get(s"/admin/messWithHappyVersion") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val versionString = new String(body.data.toByteArray) //todo test it to see if it's right } } "DashboardService" should "return an OK for admin/status/config" in { Get(s"/admin/status/config") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val configString = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/classpath" in { Get(s"/admin/status/classpath") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val classpathString = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/optionalParts" in { Get(s"/admin/status/optionalParts") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val options = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/summary" in { Get(s"/admin/status/summary") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val summary = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/adapter" in { Get(s"/admin/status/adapter") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val adapter = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/hub" in { Get(s"/admin/status/hub") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val hub = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/i2b2" in { Get(s"/admin/status/i2b2") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val i2b2 = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/keystore" in { Get(s"/admin/status/keystore") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val keystore = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/qep" in { Get(s"/admin/status/qep") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val qep = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/problems" in { Get("/admin/status/problems") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val problems = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/problems with queries" in { Get("/admin/status/problems?offset=2&n=1") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val problems = new String(body.data.toByteArray) } } "DashboardService" should "return an OK for admin/status/problems with queries and an epoch filter" in { Get("/admin/status/problems?offset=2&n=3&epoch=3") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) val problems = new String(body.data.toByteArray) } } "DashboardService" should "return a BadRequest for admin/status/signature with no signature parameter" in { Post("/status/verifySignature") ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(StatusCodes.BadRequest)(status) } } "DashboardService" should "return a BadRequest for admin/status/signature with a malformatted signature parameter" in { Post("/status/verifySignature", FormData(Seq("sha256" -> "foo"))) ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(StatusCodes.BadRequest)(status) implicit val formats = ShaResponse.json4sFormats assertResult(ShaResponse(ShaResponse.badFormat, false))(parse(new String(body.data.toByteArray)).extract[ShaResponse]) } } "DashboardService" should "return a NotFound for admin/status/signature with a correctly formatted parameter that is not in the keystore" in { Post("/status/verifySignature", FormData(Seq("sha256" -> "00:00:00:00:00:00:00:7C:4B:FD:8D:A8:0A:C7:A4:AA:13:3E:22:B3:57:A7:C6:B0:95:15:1B:22:C0:E5:15:9A"))) ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(NotFound)(status) implicit val formats = ShaResponse.json4sFormats assertResult(ShaResponse("0E:5D:D1:10:68:2B:63:F4:66:E2:50:41:EA:13:AF:1A:F9:99:DB:40:6A:F7:EE:39:F2:1A:0D:51:7A:44:09:7A", false)) ( parse(new String(body.data.toByteArray)).extract[ShaResponse]) } } "DashboardService" should "return an OK for admin/status/signature with a valid sha256 hash" in { val post = Post("/status/verifySignature", FormData(Seq("sha256" -> "0E:5D:D1:10:68:2B:63:F4:66:E2:50:41:EA:13:AF:1A:F9:99:DB:40:6A:F7:EE:39:F2:1A:0D:51:7A:44:09:7A"))) post ~> addCredentials(adminCredentials) ~> route ~> check { assertResult(OK)(status) implicit val formats = ShaResponse.json4sFormats assertResult(ShaResponse("0E:5D:D1:10:68:2B:63:F4:66:E2:50:41:EA:13:AF:1A:F9:99:DB:40:6A:F7:EE:39:F2:1A:0D:51:7A:44:09:7A", true))( parse(new String(body.data.toByteArray)).extract[ShaResponse] ) } } val dashboardCredentials = BasicHttpCredentials(adminUserName,"shh!") "DashboardService" should "return an OK and pong for fromDashboard/ping" in { Get(s"/fromDashboard/ping") ~> addCredentials(ShrineJwtAuthenticator.createOAuthCredentials(adminUser, "")) ~> route ~> check { assertResult(OK)(status) val string = new String(body.data.toByteArray) assertResult("pong")(string) } } "DashboardService" should "reject a fromDashboard/ping with an expired jwts header" in { val config = ConfigSource.config val shrineCertCollection: BouncyKeyStoreCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(KeyStoreDescriptorParser( config.getConfig("shrine.keystore"), config.getConfigOrEmpty("shrine.hub"), config.getConfigOrEmpty("shrine.queryEntryPoint"))) val base64Cert = new String(TextCodec.BASE64URL.encode(shrineCertCollection.myEntry.cert.getEncoded)) val key: PrivateKey = shrineCertCollection.myEntry.privateKey.get val expiration: Date = new Date(System.currentTimeMillis() - 300 * 1000) //bad for 5 minutes val jwtsString = Jwts.builder(). setHeaderParam("kid", base64Cert). setSubject(java.net.InetAddress.getLocalHost.getHostName). setExpiration(expiration). signWith(SignatureAlgorithm.RS512, key). compact() Get(s"/fromDashboard/ping") ~> addCredentials(OAuth2BearerToken(jwtsString)) ~> sealRoute(route) ~> check { assertResult(Unauthorized)(status) } } "DashboardService" should "reject a fromDashboard/ping with no subject" in { val config = ConfigSource.config val shrineCertCollection: BouncyKeyStoreCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(KeyStoreDescriptorParser( config.getConfig("shrine.keystore"), config.getConfigOrEmpty("shrine.hub"), config.getConfigOrEmpty("shrine.queryEntryPoint"))) val base64Cert = new String(TextCodec.BASE64URL.encode(shrineCertCollection.myEntry.cert.getEncoded)) val key: PrivateKey = shrineCertCollection.myEntry.privateKey.get val expiration: Date = new Date(System.currentTimeMillis() + 30 * 1000) val jwtsString = Jwts.builder(). setHeaderParam("kid", base64Cert). setExpiration(expiration). signWith(SignatureAlgorithm.RS512, key). compact() Get(s"/fromDashboard/ping") ~> addCredentials(OAuth2BearerToken(jwtsString)) ~> sealRoute(route) ~> check { assertResult(Unauthorized)(status) } } "DashboardService" should "reject a fromDashboard/ping with no Authorization header" in { Get(s"/fromDashboard/ping") ~> sealRoute(route) ~> check { assertResult(Unauthorized)(status) } } "DashboardService" should "reject a fromDashboard/ping with an Authorization header for the wrong authorization spec" in { Get(s"/fromDashboard/ping") ~> addCredentials(adminCredentials) ~> sealRoute(route) ~> check { assertResult(Unauthorized)(status) } } "DashboardService" should "not find a bogus web service to talk to" in { Get(s"/toDashboard/bogus.harvard.edu/ping") ~> addCredentials(adminCredentials) ~> sealRoute(route) ~> check { val string = new String(body.data.toByteArray) assertResult(NotFound)(status) } } } diff --git a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala index 0a28b2310..2d6bb0f4b 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala @@ -1,601 +1,601 @@ package net.shrine.status import java.security.cert.X509Certificate import javax.net.ssl.{KeyManager, SSLContext, X509TrustManager} import javax.ws.rs.core.{MediaType, Response} import javax.ws.rs.{GET, Path, Produces, WebApplicationException} import akka.actor.ActorSystem import akka.io.IO import akka.util.Timeout import com.sun.jersey.spi.container.{ContainerRequest, ContainerRequestFilter} import com.typesafe.config.{ConfigFactory, Config => TsConfig} import net.shrine.authorization.{QueryAuthorizationService, StewardQueryAuthorizationService} import net.shrine.broadcaster._ import net.shrine.client.PosterOntClient import net.shrine.config.ConfigExtensions import net.shrine.crypto._ import net.shrine.log.{Log, Loggable} import net.shrine.ont.data.OntClientOntologyMetadata import net.shrine.problem.{AbstractProblem, ProblemSources} import net.shrine.protocol._ import net.shrine.protocol.query.{OccuranceLimited, QueryDefinition, Term} import net.shrine.serialization.NodeSeqSerializer import net.shrine.spray._ import net.shrine.util.{PeerToPeerModel, SingleHubModel, Versions} import net.shrine.wiring.ShrineOrchestrator import org.json4s.native.Serialization import org.json4s.{DefaultFormats, Formats} import spray.can.Http import spray.can.Http.{HostConnectorInfo, HostConnectorSetup} import spray.client.pipelining._ import spray.http.{ContentType, HttpCharsets, HttpEntity, HttpRequest, HttpResponse, MediaTypes} import spray.io.{ClientSSLEngineProvider, PipelineContext, SSLContextProvider} import scala.collection.JavaConverters._ import scala.collection.immutable.{Map, Seq, Set} import scala.concurrent.duration.FiniteDuration import scala.concurrent.{Await, Future, duration} import scala.util.control.NonFatal import scala.util.{Failure, Success, Try} /** * A subservice that shares internal state of the shrine servlet. * * @author david * @since 12/2/15 */ @Path("/internalstatus") @Produces(Array(MediaType.APPLICATION_JSON)) case class StatusJaxrs(shrineConfig: TsConfig) extends Loggable { implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer @GET @Path("version") def version: String = { val version = Version("changeMe") val versionString = Serialization.write(version) versionString } @GET @Path("config") def config: String = { //todo probably better to reach out and grab the config from ManuallyWiredShrineJaxrsResources once it is a singleton Serialization.write(Json4sConfig(shrineConfig)) } @GET @Path("summary") def summary: String = { val summary = Summary() Serialization.write(summary) } @GET @Path("i2b2") def i2b2: String = { val i2b2 = I2b2() Serialization.write(i2b2) } @GET @Path("optionalParts") def optionalParts: String = { val optionalParts = OptionalParts() Serialization.write(optionalParts) } @GET @Path("hub") def hub: String = { val hub = Hub() Serialization.write(hub) } @GET @Path("adapter") def adapter: String = { val adapter = Adapter() Serialization.write(adapter) } @GET @Path("qep") def qep: String = { val qep = Qep() Serialization.write(qep) } @GET @Path("keystore") def keystore: String = { Serialization.write(KeyStoreReport()) } } /* todo fill in later when you take the time to get the right parts in place SHRINE-1529 case class KeyStoreEntryReport( alias:String, commonName:String, md5Signature:String ) */ case class SiteStatus(siteAlias: String, theyHaveMine: Boolean, haveTheirs: Boolean, url: String, timeOutError: Boolean = false) extends DefaultJsonSupport case class AbbreviatedKeyStoreEntry(alias: String, cn: String, md5: String) extends DefaultJsonSupport case class KeyStoreReport( fileName: String, password: String = "REDACTED", privateKeyAlias: Option[String], owner: Option[String], issuer: Option[String], expires: Long, md5Signature: String, sha256Signature: String, caTrustedAlias: Option[String], caTrustedSignature: Option[String], remoteSiteStatuses: Seq[SiteStatus], isHub: Boolean, abbreviatedEntries: Seq[AbbreviatedKeyStoreEntry] // keyStoreContents:List[KeyStoreEntryReport] //todo SHRINE-1529 ) //todo build new API for the dashboard to use to check signatures object KeyStoreReport { def apply(): KeyStoreReport = { val keyStoreDescriptor: KeyStoreDescriptor = ShrineOrchestrator.keyStoreDescriptor val certCollection: BouncyKeyStoreCollection = ShrineOrchestrator.certCollection val maybeCaEntry: Option[KeyStoreEntry] = certCollection match { case DownStreamCertCollection(_, caEntry, _) => Some(caEntry) case HubCertCollection(_, caEntry, _) => Some(caEntry) case px: PeerCertCollection => None } val siteStatusesPreZip = ShaVerificationService(certCollection.remoteSites.toList) val siteStatuses = siteStatusesPreZip.zipWithIndex def sortFormat(input: String): Option[String] = { if (input.isEmpty) None else { def isLong(str: String) = str.split('=').headOption.getOrElse(str).length > 2 // Just an ugly sort for formatting purposes. I want The long key last, and otherwise just // Sort them lexicographically. Some(input.split(", ").sortBy(a => (isLong(a), a)).mkString(", ")) } } lazy val blockForSiteStatuses = siteStatuses.map(fut => Try(Await.result(fut._1, new FiniteDuration(5, duration.SECONDS))) match { case Success(Some(status)) => status case Success(None) => Log.warn("There was an issue with the verifySignature endpoint, check that we have network connectivity") SiteStatus(certCollection.remoteSites(fut._2).alias, false, false, "", true) case Failure(exc) => Log.warn("We timed out while trying to connect to the verifySignature endpoint, please check network connectivity") SiteStatus(certCollection.remoteSites(fut._2).alias, false, false, "", true) }) new KeyStoreReport( fileName = keyStoreDescriptor.file, privateKeyAlias = keyStoreDescriptor.privateKeyAlias, owner = sortFormat(certCollection.myEntry.cert.getSubjectDN.getName), issuer = sortFormat(certCollection.myEntry.cert.getIssuerDN.getName), expires = certCollection.myEntry.cert.getNotAfter.getTime, md5Signature = UtilHasher.encodeCert(certCollection.myEntry.cert, "MD5"), sha256Signature = UtilHasher.encodeCert(certCollection.myEntry.cert, "SHA-256"), - //todo sha1 signature if needed caTrustedAlias = maybeCaEntry.map(_.aliases.first), caTrustedSignature = maybeCaEntry.map(entry => UtilHasher.encodeCert(entry.cert, "MD5")), remoteSiteStatuses = blockForSiteStatuses, isHub = keyStoreDescriptor.trustModel == SingleHubModel(true), abbreviatedEntries = certCollection.allEntries.map(entry => AbbreviatedKeyStoreEntry( entry.aliases.first, entry.commonName.getOrElse("No common name"), UtilHasher.encodeCert(entry.cert, "MD5"))).toList // keyStoreContents = certCollection.caCerts.zipWithIndex.map((cert: ((Principal, X509Certificate), Int)) => KeyStoreEntryReport(keyStoreDescriptor.caCertAliases(cert._2),cert._1._1.getName,toMd5(cert._1._2))).to[List] ) } } case class I2b2(pmUrl: String, crcUrl: Option[String], ontUrl: String, i2b2Domain: String, username: String, crcProject: String, ontProject: String) object I2b2 { def apply(): I2b2 = new I2b2( pmUrl = ShrineOrchestrator.pmPoster.url, crcUrl = ShrineOrchestrator.adapterComponents.map(_.i2b2AdminService.crcUrl), ontUrl = ShrineOrchestrator.ontEndpoint.url.toString, i2b2Domain = ShrineOrchestrator.crcHiveCredentials.domain, username = ShrineOrchestrator.crcHiveCredentials.username, crcProject = ShrineOrchestrator.crcHiveCredentials.projectId, ontProject = ShrineOrchestrator.ontologyMetadata.client match { case client: PosterOntClient => client.hiveCredentials.projectId case _ => "" } ) } case class DownstreamNode(name: String, url: String) // Replaces StewardQueryAuthorizationService so that we never transmit a password case class Steward(stewardBaseUrl: String, qepUsername: String, password: String = "REDACTED") case class Qep( maxQueryWaitTimeMillis: Long, create: Boolean, attachSigningCert: Boolean, authorizationType: String, includeAggregateResults: Boolean, authenticationType: String, steward: Option[Steward], broadcasterUrl: Option[String], trustModel: String, trustModelIsHub: Boolean ) object Qep { val key = "shrine.queryEntryPoint." import ShrineOrchestrator.queryEntryPointComponents def apply(): Qep = new Qep( maxQueryWaitTimeMillis = queryEntryPointComponents.fold(0L)(_.i2b2Service.queryTimeout.toMicros), create = queryEntryPointComponents.isDefined, + //todo: delete attatchSingingCert attachSigningCert = queryEntryPointComponents.fold(false)(_.i2b2Service.broadcastAndAggregationService.attachSigningCert), authorizationType = queryEntryPointComponents.fold("")(_.i2b2Service.authorizationService.getClass.getSimpleName), includeAggregateResults = queryEntryPointComponents.fold(false)(_.i2b2Service.includeAggregateResult), authenticationType = queryEntryPointComponents.fold("")(_.i2b2Service.authenticator.getClass.getSimpleName), steward = queryEntryPointComponents.flatMap(qec => checkStewardAuthorization(qec.shrineService.authorizationService)), broadcasterUrl = queryEntryPointComponents.flatMap(_.shrineService.broadcastAndAggregationService.broadcasterUrl.map(_.toString)), trustModel = ShrineOrchestrator.keyStoreDescriptor.trustModel.description, trustModelIsHub = ShrineOrchestrator.keyStoreDescriptor.trustModel match { case sh: SingleHubModel => true case PeerToPeerModel => false }) def checkStewardAuthorization(auth: QueryAuthorizationService): Option[Steward] = auth match { case sa: StewardQueryAuthorizationService => Some(Steward(sa.stewardBaseUrl.toString, sa.qepUserName)) case _ => None } } object DownstreamNodes { def get(): Seq[DownstreamNode] = { ShrineOrchestrator.hubComponents.fold(Seq.empty[DownstreamNode])(_.broadcastDestinations.map(DownstreamNode(_)).to[Seq]) } } object DownstreamNode { def apply(nodeHandle: NodeHandle): DownstreamNode = nodeHandle.client.url.fold(new DownstreamNode("self", "not applicable"))(url => new DownstreamNode(nodeHandle.nodeId.name, url.toString)) } case class Adapter(crcEndpointUrl: String, setSizeObfuscation: Boolean, adapterLockoutAttemptsThreshold: Int, adapterMappingsFilename: Option[String], adapterMappingsDate: Option[Long] ) object Adapter { def apply(): Adapter = { val crcEndpointUrl = ShrineOrchestrator.adapterComponents.fold("")(_.i2b2AdminService.crcUrl) val setSizeObfuscation = ShrineOrchestrator.adapterComponents.fold(false)(_.i2b2AdminService.obfuscate) val adapterLockoutAttemptsThreshold = ShrineOrchestrator.adapterComponents.fold(0)(_.i2b2AdminService.adapterLockoutAttemptsThreshold) val adapterMappingsFileName = mappingFileInfo.map(_._1) val adapterMappingsFileDate = mappingFileInfo.map(_._2) Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileName, adapterMappingsFileDate) } def mappingFileInfo: Option[(String, Long, String)] = ShrineOrchestrator.adapterComponents.map(ac => (ac.adapterMappings.source, ac.lastModified, ac.adapterMappings.version)) } case class Hub(shouldQuerySelf: Boolean, //todo don't use this field any more. Drop it when possible create: Boolean, downstreamNodes: Seq[DownstreamNode]) object Hub { def apply(): Hub = { val shouldQuerySelf = false val create = ShrineOrchestrator.hubComponents.isDefined val downstreamNodes = DownstreamNodes.get() Hub(shouldQuerySelf, create, downstreamNodes) } } case class OptionalParts(isHub: Boolean, stewardEnabled: Boolean, shouldQuerySelf: Boolean, //todo don't use this field any more. Drop it when possible downstreamNodes: Seq[DownstreamNode]) object OptionalParts { def apply(): OptionalParts = { OptionalParts( ShrineOrchestrator.hubComponents.isDefined, ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.shrineService.authorizationService.isInstanceOf[StewardQueryAuthorizationService]), shouldQuerySelf = false, DownstreamNodes.get() ) } } case class Summary( isHub: Boolean, shrineVersion: String, shrineBuildDate: String, ontologyVersion: String, ontologyVersionTerm: String, ontologyTerm: String, queryResult: Option[SingleNodeResult], adapterMappingsFileName: Option[String], adapterMappingsDate: Option[Long], adapterOk: Boolean, keystoreOk: Boolean, hubOk: Boolean, qepOk: Boolean ) object Summary { val term = Term(ShrineOrchestrator.shrineConfig.getString("networkStatusQuery")) def runQueryRequest: BroadcastMessage = { val domain = "happy" val username = "happy" val networkAuthn = AuthenticationInfo(domain, username, Credential("", isToken = false)) val queryDefinition = QueryDefinition("TestQuery", OccuranceLimited(1, term)) import scala.concurrent.duration._ val req = RunQueryRequest( "happyProject", 3.minutes, networkAuthn, None, None, Set(ResultOutputType.PATIENT_COUNT_XML), queryDefinition) ShrineOrchestrator.signerVerifier.sign(BroadcastMessage(req.networkQueryId, networkAuthn, req), SigningCertStrategy.Attach) } def apply(): Summary = { val message = runQueryRequest val queryResult: Option[SingleNodeResult] = ShrineOrchestrator.adapterService.map { adapterService => import scala.concurrent.duration._ val start = System.currentTimeMillis val resultAttempt: Try[Result] = Try(adapterService.handleRequest(message)) val end = System.currentTimeMillis val elapsed = (end - start).milliseconds resultAttempt match { case scala.util.Success(result) => result case scala.util.Failure(throwable) => FailureResult(NodeId("Local"), throwable) } } val adapterOk = queryResult.fold(true) { case r: Result => true case f: FailureResult => false } val hubOk = ShrineOrchestrator.hubComponents.fold(true) { hubComponents => val maxQueryWaitTime = hubComponents.broadcasterMultiplexerService.maxQueryWaitTime val broadcaster: Broadcaster = hubComponents.broadcasterMultiplexerService.broadcaster val message = runQueryRequest val triedMultiplexer = Try(broadcaster.broadcast(message)) //todo just use fold()() in scala 2.12 triedMultiplexer.toOption.fold(false) { multiplexer => val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq val failures = responses.collect { case f: FailureResult => f } val timeouts = responses.collect { case t: Timeout => t } val validResults = responses.collect { case r: Result => r } failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size) } } val adapterMappingInfo = Adapter.mappingFileInfo val ontologyVersion = try { ShrineOrchestrator.ontologyMetadata.ontologyVersion } catch { case NonFatal(x) => Log.info("Problem while getting ontology version", x) s"Unavailable due to: ${x.getMessage}" // TODO: Investigate whether a Fatal exception is being thrown } Summary( isHub = ShrineOrchestrator.hubComponents.isDefined, shrineVersion = Versions.version, shrineBuildDate = Versions.buildDate, //todo in scala 2.12, do better ontologyVersion = ontologyVersion, ontologyVersionTerm = OntClientOntologyMetadata.versionContainerTerm, ontologyTerm = term.value, queryResult = queryResult, adapterMappingsFileName = adapterMappingInfo.map(_._1), adapterMappingsDate = adapterMappingInfo.map(_._2), adapterOk = adapterOk, keystoreOk = true, //todo something for this hubOk = hubOk, qepOk = true //todo something for this ) } } case class Version(version: String) //todo SortedMap when possible case class Json4sConfig(keyValues: Map[String, String]) object Json4sConfig { def isPassword(key: String): Boolean = { if (key.toLowerCase.contains("password")) true else false } def apply(config: TsConfig): Json4sConfig = { val entries: Set[(String, String)] = config.entrySet.asScala.to[Set].map(x => (x.getKey, x.getValue.render())).filterNot(x => isPassword(x._1)) val sortedMap: Map[String, String] = entries.toMap Json4sConfig(sortedMap) } } class PermittedHostOnly extends ContainerRequestFilter { //todo generalize for happy, too //todo for tomcat 8 see https://jersey.java.net/documentation/latest/filters-and-interceptors.html for a cleaner version //shell code from http://stackoverflow.com/questions/17143514/how-to-add-custom-response-and-abort-request-in-jersey-1-11-filters //how to apply in http://stackoverflow.com/questions/4358213/how-does-one-intercept-a-request-during-the-jersey-lifecycle override def filter(requestContext: ContainerRequest): ContainerRequest = { val hostOfOrigin = requestContext.getBaseUri.getHost val shrineConfig: TsConfig = ShrineOrchestrator.config val permittedHostOfOrigin: String = shrineConfig.getOption("shrine.status.permittedHostOfOrigin", _.getString).getOrElse("localhost") val path = requestContext.getPath //happy and internalstatus API calls must come from the same host as tomcat is running on (hopefully the dashboard servlet). // todo access to the happy service permitted for SHRINE 1.21 per SHRINE-1366 // restrict access to happy service when database work resumes as part of SHRINE- // if ((path.contains("happy") || path.contains("internalstatus")) && (hostOfOrigin != permittedHostOfOrigin)) { if (path.contains("internalstatus") && (hostOfOrigin != permittedHostOfOrigin)) { val response = Response.status(Response.Status.UNAUTHORIZED).entity(s"Only available from $permittedHostOfOrigin, not $hostOfOrigin, controlled by shrine.status.permittedHostOfOrigin in shrine.conf").build() throw new WebApplicationException(response) } else requestContext } } object ShaVerificationService extends Loggable with DefaultJsonSupport { //todo: remove duplication with StewardQueryAuthorizationService import akka.pattern.ask import org.json4s.native.JsonMethods.parseOpt import system.dispatcher // execution context for futures implicit val system = ActorSystem("AuthorizationServiceActors", ConfigFactory.load("shrine")) //todo use shrine's config val certCollection = ShrineOrchestrator.certCollection def sendHttpRequest(httpRequest: HttpRequest): Future[HttpResponse] = { implicit val timeout: Timeout = Timeout.durationToTimeout(new FiniteDuration(5, duration.SECONDS)) //5 seconds implicit def json4sFormats: Formats = DefaultFormats implicit def trustfulSslContext: SSLContext = { object BlindFaithX509TrustManager extends X509TrustManager { def checkClientTrusted(chain: Array[X509Certificate], authType: String) = info(s"Client asked BlindFaithX509TrustManager to check $chain for $authType") def checkServerTrusted(chain: Array[X509Certificate], authType: String) = info(s"Server asked BlindFaithX509TrustManager to check $chain for $authType") def getAcceptedIssuers = Array[X509Certificate]() } val context = SSLContext.getInstance("TLS") context.init(Array[KeyManager](), Array(BlindFaithX509TrustManager), null) context } implicit def trustfulSslContextProvider: SSLContextProvider = { SSLContextProvider.forContext(trustfulSslContext) } class CustomClientSSLEngineProvider extends ClientSSLEngineProvider { def apply(pc: PipelineContext) = ClientSSLEngineProvider.default(trustfulSslContextProvider).apply(pc) } implicit def sslEngineProvider: ClientSSLEngineProvider = new CustomClientSSLEngineProvider val responseFuture: Future[HttpResponse] = for { HostConnectorInfo(hostConnector, _) <- { val hostConnectorSetup = new HostConnectorSetup(httpRequest.uri.authority.host.address, httpRequest.uri.authority.port, sslEncryption = httpRequest.uri.scheme == "https")( sslEngineProvider = sslEngineProvider) IO(Http) ask hostConnectorSetup } response <- sendReceive(hostConnector).apply(httpRequest) _ <- hostConnector ask Http.CloseAll } yield response responseFuture } type MaybeSiteStatus = Future[Option[SiteStatus]] def apply(sites: Seq[RemoteSite]): Seq[MaybeSiteStatus] = sites.map(curl) def curl(site: RemoteSite): MaybeSiteStatus = { val shaEntry = certCollection match { case HubCertCollection(_, caEntry, _) => caEntry case PeerCertCollection(my, _, _) => my case DownStreamCertCollection(_, caEntry, _) => caEntry } val sha256 = UtilHasher.encodeCert(shaEntry.cert, "SHA-256") implicit val formats = org.json4s.DefaultFormats val request = Post(s"https://${site.url}:${site.port}/shrine-dashboard/status/verifySignature") .withEntity( // For some reason, FormData isn't producing the correct HTTP call, so we do it manually HttpEntity.apply( ContentType( MediaTypes.`application/x-www-form-urlencoded`, HttpCharsets.`UTF-8`), s"sha256=$sha256")) for {response <- sendHttpRequest(request) rawResponse = new String(response.entity.data.toByteArray) status = parseOpt(rawResponse).fold(handleError(rawResponse))(_.extractOpt[ShaResponse] match { case Some(ShaResponse(ShaResponse.badFormat, false)) => error(s"Somehow, this client is sending an incorrectly formatted SHA256 signature to the dashboard. Offending sig: $sha256") None - case Some(ShaResponse(sha, true)) => Some(SiteStatus(site.alias, theyHaveMine = true, haveTheirs = doWeHaveCert(sha), site.url)) - case Some(ShaResponse(sha, false)) => Some(SiteStatus(site.alias, theyHaveMine = false, haveTheirs = doWeHaveCert(sha), site.url)) + case Some(ShaResponse(sha, true)) => Some(SiteStatus(site.alias, theyHaveMine = true, haveTheirs = doWeHaveCert(sha), site.url)) + case Some(ShaResponse(sha, false)) => Some(SiteStatus(site.alias, theyHaveMine = false, haveTheirs = doWeHaveCert(sha), site.url)) case None => InvalidVerifySignatureResponse(rawResponse) None })} yield status } def doWeHaveCert(sha256: String): Boolean = UtilHasher(certCollection).handleSig(sha256).found def handleError(response: String): Option[SiteStatus] = { InvalidVerifySignatureResponse(response) None } } case class InvalidVerifySignatureResponse(response: String) extends AbstractProblem(ProblemSources.ShrineApp) { override def summary: String = "The client for handling certificate diagnostic across Dashboards in the Status Service received an invalid response from shrine-dashboard/admin/status/verifySignature" override def description: String = s"See details for incorrect response:" override def throwable: Option[Throwable] = Some(InvalidResponseException(response)) } case class InvalidResponseException(response: String) extends IllegalStateException { override def getMessage: String = s"Invalid response `$response`" } diff --git a/apps/steward-app/src/main/js/app/assets/css/shrine.css b/apps/steward-app/src/main/js/app/assets/css/shrine.css index 58a8ef545..8fd6793af 100644 --- a/apps/steward-app/src/main/js/app/assets/css/shrine.css +++ b/apps/steward-app/src/main/js/app/assets/css/shrine.css @@ -1,1146 +1,1149 @@ @charset "UTF-8"; /* import timeline.scss */ @font-face { font-family: 'Nexa'; src: url("../fnt/nexa/nexa-light-webfont.eot"); src: url("../fnt/nexa/nexa-light-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/nexa/nexa-light-webfont.woff2") format("woff2"), url("../fnt/nexa/nexa-light-webfont.woff") format("woff"), url("../fnt/nexa/nexa-light-webfont.ttf") format("truetype"), url("../fnt/nexa/nexa-light-webfont.svg#nexa_lightregular") format("svg"); font-weight: normal; font-style: normal; } @font-face { font-family: 'Nexa Bold'; src: url("../fnt/nexa/nexa-bold-webfont.eot"); src: url("../fnt/nexa/nexa-bold-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/nexa/nexa-bold-webfont.woff2") format("woff2"), url("../fnt/nexa/nexa-bold-webfont.woff") format("woff"), url("../fnt/nexa/nexa-bold-webfont.ttf") format("truetype"), url("../fnt/nexa/nexa-bold-webfont.svg#nexa_boldregular") format("svg"); font-weight: normal; font-style: normal; } @font-face { font-family: 'Open Sans Regular'; src: url("../fnt/open-sans/opensans-regular-webfont.eot"); src: url("../fnt/open-sans/opensans-regular-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/open-sans/opensans-regular-webfont.woff2") format("woff2"), url("../fnt/open-sans/opensans-regular-webfont.woff") format("woff"), url("../fnt/open-sans/opensans-regular-webfont.ttf") format("truetype"), url("../fnt/open-sans/opensans-regular-webfont.svg#open_sansregular") format("svg"); font-weight: normal; font-style: normal; } @font-face { font-family: 'Open Sans Semibold'; src: url("../fnt/open-sans/opensans-semibold-webfont.eot"); src: url("../fnt/open-sans/opensans-semibold-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/open-sans/opensans-semibold-webfont.woff2") format("woff2"), url("../fnt/open-sans/opensans-semibold-webfont.woff") format("woff"), url("../fnt/open-sans/opensans-semibold-webfont.ttf") format("truetype"), url("../fnt/open-sans/opensans-semibold-webfont.svg#open_sanssemibold") format("svg"); font-weight: normal; font-style: normal; } @font-face { font-family: 'Roboto Bold'; src: url("../fnt/roboto/roboto-bold-webfont.eot"); src: url("../fnt/roboto/roboto-bold-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/roboto/roboto-bold-webfont.woff2") format("woff2"), url("../fnt/roboto/roboto-bold-webfont.woff") format("woff"), url("../fnt/roboto/roboto-bold-webfont.ttf") format("truetype"), url("../fnt/roboto/roboto-bold-webfont.svg#robotobold") format("svg"); font-weight: normal; font-style: normal; } @font-face { font-family: 'Roboto Regular'; src: url("../fnt/roboto/roboto-regular-webfont.eot"); src: url("../fnt/roboto/roboto-regular-webfont.eot?#iefix") format("embedded-opentype"), url("../fnt/roboto/roboto-regular-webfont.woff2") format("woff2"), url("../fnt/roboto/roboto-regular-webfont.woff") format("woff"), url("../fnt/roboto/roboto-regular-webfont.ttf") format("truetype"), url("../fnt/roboto/roboto-regular-webfont.svg#robotoregular") format("svg"); font-weight: normal; font-style: normal; } .main-app { background-image: url("../img/Background_Shrine.jpg"); background-size: cover; background-color: transparent; background-attachment: fixed; width: 100%; height: 100%; width: calc(100vw); height: calc(100vh); min-width: 100%; min-height: 100%; } .shrine-navbar { background-color: rgba(255, 255, 255, 0.62); border-color: transparent; font-family: "Nexa"; color: #5d5d5d; min-height: 60px; width: 100%; height: 4.8em; } .shrine-brand { float: left; padding: 22px 15px; font-size: 30px; line-height: 30px; height: 30px; } .shrine-brand strong { font-family: "Nexa Bold"; color: #2c5566; } .shrine-navbar .shrine-institution-logo { background-image: url("/static/logo.png"); background-size: contain; background-color: transparent; background-color: rgba(255, 255, 255, 0.1); background-repeat: no-repeat; background-position: right top; margin-right: 2%; margin-top: 5px; width: 4em; height: 4em; max-height: 4em; max-width: 4em; } .shrine-button { cursor: pointer !important; background-color: transparent; border: none; } .shrine-button span { position: relative; bottom: 5px; } .shrine-btn-default { margin-right: 6px; border: none; } .shrine-btn-on { padding: 12px 12px; border-radius: 0; font-family: "Roboto Bold" !important; color: #FFFFFF !important; background: linear-gradient(rgba(4, 141, 190, 0.8), rgba(2, 89, 120, 0.8)), url("../img/bckg_diagonal_lines_no_border.png") !important; } .shrine-btn-off { padding: 6px 12px !important; border-radius: 4px !important; font-family: "Open Sans Semibold" !important; background-color: #8896A4 !important; color: #FFFFFF !important; } .shrine-on, .shrine-on a { font-family: "Roboto Bold" !important; color: #FFFFFF !important; background: linear-gradient(#048DBE, #025978) !important; } .shrine-off { font-family: "Roboto Regular" !important; color: #2C5566 !important; background-color: #ECEEEE !important; } .shrine-button.disabled, .shrine-button[disabled] { cursor: default !important; opacity: 0.2 !important; } .shrine-copy-bold { font-family: "Nexa Bold"; color: #64818e; } .shrine-copy { font-family: "Nexa"; color: #000000; } .row { margin-right: 0; margin-left: 0; } td.error, span.error { color: red; } td.error a, td.error a:hover, span.error a, span.error a:hover { color: inherit; text-decoration: underline !important; cursor: pointer; } td.ok, span.ok { color: green; } .form-group span { font-family: "Open Sans Semibold"; color: #2c5566; } fieldset button { color: #2E5366; } fieldset button:hover, form a:hover { color: #008CBA; text-decoration: none; cursor: pointer; } form a { font-family: "Open Sans Regular"; color: #647d8d; text-decoration: none; } footer img { margin-left: 10px; margin-top: 2px; } footer { background-color: rgba(50, 62, 74, 0.48); position: fixed; bottom: 0; left: 0; width: 100%; height: 83px; min-height: 83px; max-width: 100%; } table { background-image: url("../img/bckg_diagonal_lines_no_border.png"); border: 1px solid #CCD8DF; } .table tr > td:first-child { width: 20%; min-width: 140px; } .table tr > td.thin-col { width: 6%; min-width: 35px; } .table-striped > tbody > tr:nth-of-type(odd) { background-color: #EFF6F9; } .table-striped > tbody > tr:nth-of-type(even) { background-color: #FFFFFF; } thead tr { border: 1px solid #CCD8DF; } td { border-right: 1px solid #CCD8DF; overflow: hidden; max-width: 450px; word-break: break-all; } thead tr td, thead tr td label, tfoot tr td span { font-family: "Open Sans Semibold"; color: #003153; } td a, td a:hover { text-decoration: none !important; cursor: pointer; font-family: "Open Sans Semibold"; color: #003153; } .shrine-panel { background-image: url("../img/bckg_diagonal_lines.png"); background-size: 100% 100%; padding-right: 20px; padding-left: 20px; padding-top: 30px; padding-bottom: 30px; } /*! * Start Bootstrap - SB Admin 2 Bootstrap Admin Theme (http://startbootstrap.com) * Code licensed under the Apache License v2.0. * For details, see http://www.apache.org/licenses/LICENSE-2.0. */ body { background-image: url("../img/Background_Shrine.jpg"); background-repeat: no-repeat; background-size: 100% 100%; } #wrapper { width: 100%; padding-bottom: 83px; /* clearance space for footer at bottom of long page */ } .login-wrapper { margin-top: 51px; margin-right: 20px; } #page-wrapper { background-color: transparent; margin-top: 54px; margin-right: 20px; } .navbar-top-links li { display: inline-block; } .navbar-top-links li:last-child { margin-right: 15px; } .navbar-top-links li a { padding: 15px; min-height: 50px; font-family: "Open Sans Semibold"; color: #2c5566; } .navbar-top-links .dropdown-menu li { font-family: "Open Sans Semibold"; color: #2c5566; display: block; } .navbar-top-links .dropdown-menu li:last-child { margin-right: 0; } .navbar-top-links .dropdown-menu li a { padding: 3px 20px; min-height: 0; } .navbar-top-links .dropdown-menu li a div { white-space: normal; } .navbar-top-links .dropdown-messages, .navbar-top-links .dropdown-tasks, .navbar-top-links .dropdown-alerts { width: 310px; min-width: 0; } .navbar-top-links .dropdown-messages { margin-left: 5px; } .navbar-top-links .dropdown-tasks { margin-left: -59px; } .navbar-top-links .dropdown-alerts { margin-left: -123px; } .navbar-top-links { right: 0; left: auto; } .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; } .sidebar .sidebar-search { padding: 15px; } .sidebar ul li { border-bottom: 1px solid #e7e7e7; } .sidebar ul li a.active { background-color: #eee; } .sidebar .arrow { float: right; } .sidebar .fa.arrow:before { content: "\f104"; } .sidebar .active > a > .fa.arrow:before { content: "\f107"; } .sidebar .nav-second-level li, .sidebar .nav-third-level li { border-bottom: 0 !important; } .sidebar .nav-second-level li a { padding-left: 37px; } .sidebar .nav-third-level li a { padding-left: 52px; } @media (min-width: 768px) { .sidebar { z-index: 1; margin-top: 51px; } .navbar-top-links .dropdown-messages, .navbar-top-links .dropdown-tasks, .navbar-top-links .dropdown-alerts { margin-left: auto; } } .btn-outline { color: inherit; background-color: transparent; transition: all .5s; } .btn-primary.btn-outline { color: #428bca; } .btn-success.btn-outline { color: #5cb85c; } .btn-info.btn-outline { color: #5bc0de; } .btn-warning.btn-outline { color: #f0ad4e; } .btn-danger.btn-outline { color: #d9534f; } .btn-primary.btn-outline:hover, .btn-success.btn-outline:hover, .btn-info.btn-outline:hover, .btn-warning.btn-outline:hover, .btn-danger.btn-outline:hover { color: #fff; } .chat { margin: 0; padding: 0; list-style: none; } .chat li { margin-bottom: 10px; padding-bottom: 5px; border-bottom: 1px dotted #999; } .chat li.left .chat-body { margin-left: 60px; } .chat li.right .chat-body { margin-right: 60px; } .chat li .chat-body p { margin: 0; } .panel .slidedown .glyphicon, .chat .glyphicon { margin-right: 5px; } .chat-panel .panel-body { height: 350px; overflow-y: scroll; } .login-panel { margin-top: 25%; } .flot-chart { display: block; height: 400px; } .flot-chart-content { width: 100%; height: 100%; } .dataTables_wrapper { position: relative; clear: both; } table.dataTable thead .sorting, table.dataTable thead .sorting_asc, table.dataTable thead .sorting_desc, table.dataTable thead .sorting_asc_disabled, table.dataTable thead .sorting_desc_disabled { background: 0 0; } table.dataTable thead .sorting_asc:after { content: "\f0de"; float: right; font-family: fontawesome; } table.dataTable thead .sorting_desc:after { content: "\f0dd"; float: right; font-family: fontawesome; } table.dataTable thead .sorting:after { content: "\f0dc"; float: right; font-family: fontawesome; color: rgba(50, 50, 50, 0.5); } .btn-circle { width: 30px; height: 30px; padding: 6px 0; border-radius: 15px; text-align: center; font-size: 12px; line-height: 1.428571429; } .btn-circle.btn-lg { width: 50px; height: 50px; padding: 10px 16px; border-radius: 25px; font-size: 18px; line-height: 1.33; } .btn-circle.btn-xl { width: 70px; height: 70px; padding: 10px 16px; border-radius: 35px; font-size: 24px; line-height: 1.33; } .show-grid [class^=col-] { padding-top: 10px; padding-bottom: 10px; border: 1px solid #ddd; background-color: #eee !important; } .show-grid { margin: 15px 0; } .huge { font-size: 40px; } .panel-green { border-color: #5cb85c; } .panel-green .panel-heading { border-color: #5cb85c; color: #fff; background-color: #5cb85c; } .panel-green a { color: #5cb85c; } .panel-green a:hover { color: #3d8b3d; } .panel-red { border-color: #d9534f; } .panel-red .panel-heading { border-color: #d9534f; color: #fff; background-color: #d9534f; } .panel-red a { color: #d9534f; } .panel-red a:hover { color: #b52b27; } .panel-yellow { border-color: #f0ad4e; } .panel-yellow .panel-heading { border-color: #f0ad4e; color: #fff; background-color: #f0ad4e; } .panel-yellow a { color: #f0ad4e; } .panel-yellow a:hover { color: #df8a13; } .modal-content { border: none; } .shrine-modal { background-color: white; border: 1px solid #2c5566; font-family: "Open Sans Semibold"; color: #2e5366; padding: 15px; } .shrine-modal form div.col-sm-12 { border: 1px solid rgba(2, 89, 120, 0.8); } .shrine-modal input, .shrine-modal textarea { border-radius: 0px; border: 1px solid #2c5566; } .shrine-modal span { font-family: "Nexa Bold"; color: #2e5366; } .shrine-modal span:hover { font-family: "Nexa Bold"; color: #008CBA; } .shrine-modal button { background-color: white; border: none; font-family: "Nexa Bold"; color: #2e5366; } .shrine-modal button span { position: relative; bottom: 6px; } .shrine-modal button:hover, .btn-success { font-family: "Nexa Bold"; color: #008CBA; background-color: transparent; border: none; } .shrine-login { margin-top: 8%; margin-left: 1%; } .shrine-content { overflow: auto; } /*Fix for resizeable text area.*/ textarea { resize: none; } @media (min-width: 768px) { .shrine-content { padding: 0; } } .shrine-calendar-input { margin-right: 1px; max-width: 50%; } i.shrine-close { float: right; margin-top: -40px; margin-right: -40px; cursor: pointer; color: #fff; border: 2px solid #C8CED1; border-radius: 30px; background: #8896a4; font-size: 31px; font-weight: normal; display: inline-block; line-height: 0px; padding: 11px 3px; font-style: normal; } i.shrine-close:hover { background: #008cba; } .shrine-close:before { content: "×"; } .timeline { position: relative; padding: 20px 0 20px; list-style: none; } .timeline:before { content: " "; position: absolute; top: 0; bottom: 0; left: 50%; width: 3px; margin-left: -1.5px; background-color: #eeeeee; } .timeline > li { position: relative; margin-bottom: 20px; } .timeline > li:before, .timeline > li:after { content: " "; display: table; } .timeline > li:after { clear: both; } .timeline > li:before, .timeline > li:after { content: " "; display: table; } .timeline > li:after { clear: both; } .timeline > li > .timeline-panel { float: left; position: relative; width: 46%; padding: 20px; border: 1px solid #d4d4d4; border-radius: 2px; -webkit-box-shadow: 0 1px 6px rgba(0, 0, 0, 0.175); box-shadow: 0 1px 6px rgba(0, 0, 0, 0.175); } .timeline > li > .timeline-panel:before { content: " "; display: inline-block; position: absolute; top: 26px; right: -15px; border-top: 15px solid transparent; border-right: 0 solid #ccc; border-bottom: 15px solid transparent; border-left: 15px solid #ccc; } .timeline > li > .timeline-panel:after { content: " "; display: inline-block; position: absolute; top: 27px; right: -14px; border-top: 14px solid transparent; border-right: 0 solid #fff; border-bottom: 14px solid transparent; border-left: 14px solid #fff; } .timeline > li > .timeline-badge { z-index: 100; position: absolute; top: 16px; left: 50%; width: 50px; height: 50px; margin-left: -25px; border-radius: 50% 50% 50% 50%; text-align: center; font-size: 1.4em; line-height: 50px; color: #fff; background-color: #999999; } .timeline > li.timeline-inverted > .timeline-panel { float: right; } .timeline > li.timeline-inverted > .timeline-panel:before { right: auto; left: -15px; border-right-width: 15px; border-left-width: 0; } .timeline > li.timeline-inverted > .timeline-panel:after { right: auto; left: -14px; border-right-width: 14px; border-left-width: 0; } .timeline-badge.primary { background-color: #2e6da4 !important; } .timeline-badge.success { background-color: #3f903f !important; } .timeline-badge.warning { background-color: #f0ad4e !important; } .timeline-badge.danger { background-color: #d9534f !important; } .timeline-badge.info { background-color: #5bc0de !important; } .timeline-title { margin-top: 0; color: inherit; } .timeline-body > p, .timeline-body > ul { margin-bottom: 0; } .timeline-body > p + p { margin-top: 5px; } @media (max-width: 767px) { ul.timeline:before { left: 40px; } ul.timeline > li > .timeline-panel { width: calc(100% - 90px); width: -moz-calc(100% - 90px); width: -webkit-calc(100% - 90px); } ul.timeline > li > .timeline-badge { top: 16px; left: 15px; margin-left: 0; } ul.timeline > li > .timeline-panel { float: right; } ul.timeline > li > .timeline-panel:before { right: auto; left: -15px; border-right-width: 15px; border-left-width: 0; } ul.timeline > li > .timeline-panel:after { right: auto; left: -14px; border-right-width: 14px; border-left-width: 0; } } /* -- stats-graph styles -- */ .stats-minimized, .stats-graph-minimized, .stats-ontology-minimized { cursor: pointer; opacity: .7; } .stats-minimized:hover, .stats-graph-minimized:hover, .stats-ontology-minimized:hover { border: 1px solid white; opacity: 1; } .stats-graph-minimized { transform: translate(60%, -20%) scale(0.1, 0.25); } .stats-ontology-minimized { transform: translate(60%, 0%) scale(0.1, 0.25); } .stats-stage-base, .stats-graph-container, .stats-ontology-container { padding-bottom: 0px; padding-top: 0px; color: white; margin: 0 auto; width: 100%; } .stats-graph-container { border-bottom: none; /*border-left: 1px solid #125c8a;;*/ overflow: auto; padding-top: .6em; /*background: linear-gradient(#048DBE, #025978) !important;*/ } .stats-ontology-container { overflow: auto; height: 100vh; } .stats-graph-container .bar-bg { background-color: white; height: 30px; margin: 0 auto 10px auto; line-height: 30px; font-size: 16px; font-family: Roboto, sans-serif; font-weight: 100; color: white; padding: 0 0 0 10px; position: relative; } .stats-graph-container .bar-bg .user { position: absolute; z-index: 4; left: .2em; } .stats-graph-container .bar-bg .bar { opacity: .8; z-index: 1; background-color: #125c8a; border: 1px solid #1c8ed6; height: 30px; transition: 0.7s; width: 100%; display: block; animation: stats-bar-before 1 1.8s; position: absolute; top: 0; left: 0; z-index: 2; padding-right: 8px; text-align: right; } .stats-graph-container .bar-bg .bar:hover { background: rgba(0, 0, 0, 0.5); cursor: pointer; } @keyframes stats-bar-before { 0% { width: 0px; } 100% { width: 100%; } } .digest ul { top: 15rem; left: 5rem; float: left; clear: left; margin: .25em; padding: 0; } .digest ul::before { display: none; } .digest ul li { display: block; position: relative; float: left; clear: both; right: auto; padding-left: 1em; width: auto; text-align: center; color: white; animation: showterm .25s; } .digest ul li a { display: block; position: relative; float: left; z-index: 4; margin: .25em; padding: .25em; background: white; color: #C0C0C0; border: 2px solid #94a0b4; border-radius: .5rem; transition: all .75s; font-weight: 300; font-family: sans-serif; } .digest .expandable { position: absolute; z-index: 4; top: 0; left: -.5em; margin: .65em; padding: 0; width: .8em; height: .8em; text-align: center; line-height: .6em; font-size: 1em; background-color: white; color: #C0C0C0; border: 1px solid #94a0b4; border-radius: .25rem; } .digest > ul { position: relative; font-family: "Georgia"; } .digest > ul:before { left: .5em; } .digest > ul:after { display: none; } @keyframes showterm { 0% { top: -.5rem; opacity: 0; } 100% { top: 0rem; opacity: 1; } } ul:after { color: blue; border: solid gray 1px; border-radius: .1em; } ul > li > a { color: blue; background: white; } .digest li span.expandable:hover, .digest li span.expandable:hover + a, .digest li span.expandable:hover + a + ul li a, .digest li a:hover, .digest li a:hover + ul li a { background: rgba(50, 130, 136, 0) !important; color: #fff; cursor: pointer; z-index: 1000; text-decoration: none; } .digest li a:hover + ul li::after, .digest li a:hover + ul li::before, .digest li a:hover + ul::before, .digest li a:hover + ul ul::before { border-color: #99C794 !important; } /* color settings */ a { font-family: "Roboto", sans-serif; font-size: 14px; font-weight: 400; line-height: 1.5em; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; color: #000000; } .red { background: rgba(158, 56, 68, 0.5) !important; border: 1px solid #9e3844 !important; } .dark-red { background: rgba(236, 95, 103, 0.5) !important; border: 1px solid #EC5F67 !important; } .dark-blue { background: rgba(53, 105, 151, 0.75) !important; border: 1px solid #6699CC !important; } .blue { background: rgba(102, 153, 204, 0.25) !important; border: 1px solid #6699CC !important; } .purple { background: rgba(197, 148, 197, 0.5) !important; border: 1px solid #C594C0 !important; } .ont-hidden { margin: auto; position: fixed; top: 100px; left: 0; right: 0; width: 50%; max-width: 630px; min-width: 320px; opacity: 0; height: auto; z-index: 2000; visibility: hidden; -webkit-backface-visibility: hidden; -moz-backface-visibility: hidden; backface-visibility: hidden; } .ont-overlay { position: fixed; width: 100%; height: 100%; visibility: visible; top: 0; left: 0; z-index: 2000; opacity: 1; background: rgba(0, 0, 0, 0.9); -webkit-transition: all 0.3s; -moz-transition: all 0.3s; transition: all 0.1s; } button.ont-close { display: block; background: transparent; color: #c0c0c0; right: 2%; top: 2%; position: absolute; border-radius: 9px; font-size: 20px; cursor: pointer; border: none; } button.ont-close:hover { color: #fff; } .tooltip-toggle { cursor: pointer; position: relative; } .tooltip-toggle::before { border: 1px solid rgba(255, 255, 255, 0.8); position: absolute; top: -5rem; left: -3rem; background-color: rgba(0, 0, 0, 0.8); border-radius: 5px; color: #fff; content: attr(aria-label); padding: 1rem; text-transform: none; -webkit-transition: all 2s; transition: all 2s; width: 160px; } .tooltip-toggle::after { position: absolute; top: -.8rem; left: 5rem; border-left: 5px solid transparent; border-right: 5px solid transparent; border-top: 5px solid white; content: " "; font-size: 0; line-height: 0; margin-left: -5px; width: 0; } .tooltip-toggle::before, .tooltip-toggle::after { color: #efefef; font-family: monospace; font-size: 16px; opacity: 0; pointer-events: none; text-align: center; } .tooltip-toggle:focus::before, .tooltip-toggle:focus::after, .tooltip-toggle:hover::before, .tooltip-toggle:hover::after { opacity: 1; -webkit-transition: all 2s; transition: all 2s; } .topic-dropdown { z-index: 20000; margin: 4rem; left: 0%; color: rgba(0, 0, 0, 0.87); font-family: "Roboto", sans-serif; font-size: 14px; font-weight: 300; line-height: 1.5em; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; position: fixed; } .topic-dropdown div.title { text-transform: uppercase; font-weight: 500; font-size: 1.3em; color: #63BCF8; } .topic-dropdown .tdd-btn { text-align: left; max-width: 1000px; outline: 0; /* takes care of blue outline */ display: -webkit-inline-box; display: -webkit-inline-flex; display: -ms-inline-flexbox; display: inline-flex; -webkit-box-align: center; -webkit-align-items: center; -ms-flex-align: center; align-items: center; -webkit-box-pack: justify; -webkit-justify-content: space-between; -ms-flex-pack: justify; justify-content: space-between; background: rgba(0, 0, 0, 0.75); border: 1px solid #63BCF8 !important; min-width: 400px; border: 0; border-radius: 4px; box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); box-sizing: border-box; padding: 16px 20px; color: #FFFFFF; font-size: 12px; font-weight: 300; letter-spacing: 1.2px; text-transform: uppercase; overflow: hidden; cursor: pointer; } .topic-dropdown .tdd-btn:hover { cursor: pointer; } - .topic-dropdown .tdd-btn .tdd-list { + .topic-dropdown .tdd-list { + position: absolute; + top: 100%; + left: 0px; + background: rgba(0, 0, 0, 0.75); + border: 1px solid #63BCF8 !important; + width: 100%; + border-radius: 4px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); + text-align: left; + opacity: 0; + visibility: hidden; + -webkit-transition: 0.3s ease; + transition: 0.3s ease; + max-height: 75vh; + overflow: auto; } + .topic-dropdown .tdd-list a { + display: block; + border-bottom: 1px solid rgba(0, 0, 0, 0.05); + padding: 16px 0; + color: inherit; + text-decoration: none; + background: none !important; } + .topic-dropdown .tdd-list:before { + content: ''; position: absolute; - top: 100%; - left: 0px; - background: rgba(0, 0, 0, 0.75); - border: 1px solid #63BCF8 !important; - width: 100%; - border-radius: 4px; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); - text-align: left; - opacity: 0; - visibility: hidden; - -webkit-transition: 0.3s ease; - transition: 0.3s ease; } - .topic-dropdown .tdd-btn .tdd-list a { - display: block; - border-bottom: 1px solid rgba(0, 0, 0, 0.05); - padding: 16px 0; - color: inherit; - text-decoration: none; - background: none !important; } - .topic-dropdown .tdd-btn .tdd-list:before { - content: ''; - position: absolute; - top: -6px; - left: 20px; - width: 0; - height: 0; - box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); - border-top: 6px solid #63BCF8; - border-right: 6px solid #63BCF8; - border-bottom: 6px solid transparent; - border-left: 6px solid transparent; - -webkit-transform: rotate(-45deg); - transform: rotate(-45deg); - mix-blend-mode: multiple; } - .topic-dropdown .tdd-btn .tdd-list li { - z-index: 100; - position: relative; - padding: 0 20px; } - .topic-dropdown .tdd-btn .tdd-list li.active { - color: #5380F7; } - .topic-dropdown .tdd-btn .tdd-list li:first-child { - border-radius: 4px 4px 0 0; } - .topic-dropdown .tdd-btn .tdd-list li:last-child { - border-radius: 0 0 4px 4px; } - .topic-dropdown .tdd-btn .tdd-list li:last-child a { - border-bottom: 0; } - .topic-dropdown .tdd-btn:focus .tdd-list, .topic-dropdown .tdd-btn:active .tdd-list { - -webkit-transform: translate(0, 20px); - transform: translate(0, 20px); - opacity: 1; - visibility: visible; } + top: -6px; + left: 20px; + width: 0; + height: 0; + box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); + border-top: 6px solid #63BCF8; + border-right: 6px solid #63BCF8; + border-bottom: 6px solid transparent; + border-left: 6px solid transparent; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + mix-blend-mode: multiple; } + .topic-dropdown .tdd-list li { + z-index: 100; + position: relative; + padding: 0 20px; + color: white; } + .topic-dropdown .tdd-list li.active { + color: #5380F7; } + .topic-dropdown .tdd-list li:first-child { + border-radius: 4px 4px 0 0; } + .topic-dropdown .tdd-list li:last-child { + border-radius: 0 0 4px 4px; } + .topic-dropdown .tdd-list li:last-child a { + border-bottom: 0; } + .topic-dropdown .tdd-list-open { + -webkit-transform: translate(0, 20px); + transform: translate(0, 20px); + opacity: 1 !important; + visibility: visible !important; } .dropdown-arrow { content: ''; opacity: .8; position: absolute; right: 10px; top: 43px; width: 0; height: 0; box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); border-top: 4px solid #63BCF8; border-right: 4px solid #63BCF8; border-bottom: 4px solid transparent; border-left: 4px solid transparent; -webkit-transform: rotate(-45deg); transform: rotate(135deg); } diff --git a/apps/steward-app/src/main/js/app/assets/css/topic-dropdown.css b/apps/steward-app/src/main/js/app/assets/css/topic-dropdown.css index 5c821998a..d20bde020 100644 --- a/apps/steward-app/src/main/js/app/assets/css/topic-dropdown.css +++ b/apps/steward-app/src/main/js/app/assets/css/topic-dropdown.css @@ -1,104 +1,107 @@ .topic-dropdown { z-index: 20000; margin: 4rem; left: 0%; color: rgba(0, 0, 0, 0.87); font-family: "Roboto", sans-serif; font-size: 14px; font-weight: 300; line-height: 1.5em; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; position: fixed; } .topic-dropdown div.title { text-transform: uppercase; font-weight: 500; font-size: 1.3em; color: #63BCF8; } .topic-dropdown .tdd-btn { text-align: left; max-width: 1000px; outline: 0; /* takes care of blue outline */ display: -webkit-inline-box; display: -webkit-inline-flex; display: -ms-inline-flexbox; display: inline-flex; -webkit-box-align: center; -webkit-align-items: center; -ms-flex-align: center; align-items: center; -webkit-box-pack: justify; -webkit-justify-content: space-between; -ms-flex-pack: justify; justify-content: space-between; background: rgba(0, 0, 0, 0.75); border: 1px solid #63BCF8 !important; min-width: 400px; border: 0; border-radius: 4px; box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); box-sizing: border-box; padding: 16px 20px; color: #FFFFFF; font-size: 12px; font-weight: 300; letter-spacing: 1.2px; text-transform: uppercase; overflow: hidden; cursor: pointer; } .topic-dropdown .tdd-btn:hover { cursor: pointer; } - .topic-dropdown .tdd-btn .tdd-list { + .topic-dropdown .tdd-list { + position: absolute; + top: 100%; + left: 0px; + background: rgba(0, 0, 0, 0.75); + border: 1px solid #63BCF8 !important; + width: 100%; + border-radius: 4px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); + text-align: left; + opacity: 0; + visibility: hidden; + -webkit-transition: 0.3s ease; + transition: 0.3s ease; + max-height: 75vh; + overflow: auto; } + .topic-dropdown .tdd-list a { + display: block; + border-bottom: 1px solid rgba(0, 0, 0, 0.05); + padding: 16px 0; + color: inherit; + text-decoration: none; + background: none !important; } + .topic-dropdown .tdd-list:before { + content: ''; position: absolute; - top: 100%; - left: 0px; - background: rgba(0, 0, 0, 0.75); - border: 1px solid #63BCF8 !important; - width: 100%; - border-radius: 4px; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); - text-align: left; - opacity: 0; - visibility: hidden; - -webkit-transition: 0.3s ease; - transition: 0.3s ease; } - .topic-dropdown .tdd-btn .tdd-list a { - display: block; - border-bottom: 1px solid rgba(0, 0, 0, 0.05); - padding: 16px 0; - color: inherit; - text-decoration: none; - background: none !important; } - .topic-dropdown .tdd-btn .tdd-list:before { - content: ''; - position: absolute; - top: -6px; - left: 20px; - width: 0; - height: 0; - box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); - border-top: 6px solid #63BCF8; - border-right: 6px solid #63BCF8; - border-bottom: 6px solid transparent; - border-left: 6px solid transparent; - -webkit-transform: rotate(-45deg); - transform: rotate(-45deg); - mix-blend-mode: multiple; } - .topic-dropdown .tdd-btn .tdd-list li { - z-index: 100; - position: relative; - padding: 0 20px; } - .topic-dropdown .tdd-btn .tdd-list li.active { - color: #5380F7; } - .topic-dropdown .tdd-btn .tdd-list li:first-child { - border-radius: 4px 4px 0 0; } - .topic-dropdown .tdd-btn .tdd-list li:last-child { - border-radius: 0 0 4px 4px; } - .topic-dropdown .tdd-btn .tdd-list li:last-child a { - border-bottom: 0; } - .topic-dropdown .tdd-btn:focus .tdd-list, .topic-dropdown .tdd-btn:active .tdd-list { - -webkit-transform: translate(0, 20px); - transform: translate(0, 20px); - opacity: 1; - visibility: visible; } + top: -6px; + left: 20px; + width: 0; + height: 0; + box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); + border-top: 6px solid #63BCF8; + border-right: 6px solid #63BCF8; + border-bottom: 6px solid transparent; + border-left: 6px solid transparent; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + mix-blend-mode: multiple; } + .topic-dropdown .tdd-list li { + z-index: 100; + position: relative; + padding: 0 20px; + color: white; } + .topic-dropdown .tdd-list li.active { + color: #5380F7; } + .topic-dropdown .tdd-list li:first-child { + border-radius: 4px 4px 0 0; } + .topic-dropdown .tdd-list li:last-child { + border-radius: 0 0 4px 4px; } + .topic-dropdown .tdd-list li:last-child a { + border-bottom: 0; } + .topic-dropdown .tdd-list-open { + -webkit-transform: translate(0, 20px); + transform: translate(0, 20px); + opacity: 1 !important; + visibility: visible !important; } diff --git a/apps/steward-app/src/main/js/app/assets/img/SHRINE-Data-Steward-Help.pdf b/apps/steward-app/src/main/js/app/assets/img/SHRINE-Data-Steward-Help.pdf index e1762f9b5..dd3b93af3 100644 Binary files a/apps/steward-app/src/main/js/app/assets/img/SHRINE-Data-Steward-Help.pdf and b/apps/steward-app/src/main/js/app/assets/img/SHRINE-Data-Steward-Help.pdf differ diff --git a/apps/steward-app/src/main/js/app/client/common/date.service.js b/apps/steward-app/src/main/js/app/client/common/date.service.js index 537d7fe77..61a31cb3f 100644 --- a/apps/steward-app/src/main/js/app/client/common/date.service.js +++ b/apps/steward-app/src/main/js/app/client/common/date.service.js @@ -1,99 +1,100 @@ (function () { 'use strict'; /** * Date methods. */ angular .module('shrine.common') .factory('DateService', DateService); function DateService() { return { utcToDateString: utcToDateString, timestampToUtc: timestampToUtc, utcToMMDDYYYY: utcToMMDDYYYY, utcToTimeStamp: utcToTimeStamp }; /** * Convert UTC to date string. * deprecated. * @todo: delete. * @param utc * @returns {string} */ function utcToDateString(utc) { return new Date(utc).toDateString(); } + function timestampToUtc(timestamp) { return Date.parse(timestamp); } function utcToMMDDYYYY(utc) { //format data. var date = new Date(utc), format = 'MM/DD/YYYY', dd, mm, yyyy; //parse values from date dd = date.getDate(); mm = date.getMonth() + 1; yyyy = date.getFullYear(); format = format.replace(/DD/, dd) .replace(/MM/, mm) .replace(/YYYY/, yyyy); return format; } /** * Get a timestamp from a universal time code. * @param utc universal time code. * @returns {string} */ function utcToTimeStamp(utc) { //format data. var date = new Date(utc), format = 'MM/DD/YYYY HH:MN:SS', dd, mm, yyyy, mn, hh, ss, time, postfix; //parse values from date dd = date.getDate(); mm = date.getMonth() + 1; yyyy = date.getFullYear(); //parse values for time. hh = date.getHours(); mn = date.getMinutes(); ss = date.getSeconds(); //format date. dd = (dd < 10) ? '0' + dd : dd; mm = (mm < 10) ? '0' + mm : mm; //format time. postfix = (hh > 12) ? ' PM' : ' AM'; hh = ((hh < 10) ? '0' + hh : hh) % 12; mn = (mn < 10) ? '0' + mn : mn; ss = (ss < 10) ? '0' + ss : ss; format = format.replace(/DD/, dd) .replace(/MM/, mm) .replace(/YYYY/, yyyy) .replace(/HH/, hh) .replace(/MN/, mn) .replace(/SS/, ss); format += postfix; return format; } } })(); diff --git a/apps/steward-app/src/main/js/app/client/directives/history.tpl.html b/apps/steward-app/src/main/js/app/client/directives/history.tpl.html index 2ffbb56d1..ed0274352 100644 --- a/apps/steward-app/src/main/js/app/client/directives/history.tpl.html +++ b/apps/steward-app/src/main/js/app/client/directives/history.tpl.html @@ -1,80 +1,80 @@
ID
  • Query Topic
  • Username
  • Query Text
  • Status
  • Date
  • {{query.stewardId}} {{query.topic.name}} {{query.user.userName}} {{query.name}} {{query.stewardResponse}} {{history.dateFormatter(query.date)}}
    - Page: {{history.sortData.pageIndex}} / {{history .sortData.totalPages}} +


    diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-counts-table.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-counts-table.tpl.html deleted file mode 100644 index 306e51f6c..000000000 --- a/apps/steward-app/src/main/js/app/client/statistics/query-counts-table.tpl.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - -
    - Query Counts By User -
    UserQuery Count
    {{user._1.userName}}{{user._2}}
    Total: {{queriesPerUser.total}}
    - - diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.factory.js b/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.factory.js index 46e2e1d5d..63ce2d750 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.factory.js +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.factory.js @@ -1,45 +1,45 @@ (function () { angular.module('shrine.steward.statistics') .factory('OntologyTerm', OntologyTermFactory); function OntologyTermFactory() { return OntologyTerm; } function OntologyTerm(key) { this.key = key; this.children = {}; this.queries = {}; this.queryCount = 0; } /* static */ OntologyTerm.prototype.maxTermUsedCount = 0; OntologyTerm.prototype.addQuery = function (queryId, queryData) { this.queryCount++; - if (this.queryCount > OntologyTerm.prototype.maxTermUsedCount) { - OntologyTerm.prototype.maxTermUsedCount = this.queryCount; + OntologyTerm.prototype.maxTermUsedCount = this.queryCount; } + if (!this.queries[queryId]) { this.queries[queryId] = queryData; } }; OntologyTerm.prototype.getQueries = function () { return Object.keys(this.queries); }; OntologyTerm.prototype.getQueryCount = function () { return this.queryCount; }; OntologyTerm.prototype.hasChildren = function () { return !!(Object.keys(this.children).length > 0); }; })(); \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.service.js b/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.service.js index 8e3b50c92..6432618c7 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.service.js +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/ontology-term.service.js @@ -1,162 +1,168 @@ (function () { 'use strict'; angular.module('shrine.steward.statistics') .service('OntologyTermService', OntologyTermService); OntologyTermService.$inject = ['OntologyTerm']; function OntologyTermService(OntologyTerm) { return { buildOntology: buildOntology, getMax: function () { var maxTermUsedCount = OntologyTerm.prototype.maxTermUsedCount; OntologyTerm.prototype.maxTermUsedCount = 0; return maxTermUsedCount; } }; /** * Build Ontology from array of Query Records. */ function buildOntology(queryRecords, topicId) { var ln = queryRecords.length; var ontology = new OntologyTerm('SHRINE'); var topics = {}; var parsingAllTopics = topicId === undefined; - var filteringByTopic; + var filteringByThisTopic; + var filteredCount = 0; for (var i = 0; i < ln; i++) { var record = queryRecords[i]; var topic = record.topic; - filteringByTopic = !parsingAllTopics && (topic && topicId === topic.id); + filteringByThisTopic = !parsingAllTopics && (topic && topicId === topic.id); appendTopicIfUnique(topic, topics); - if (parsingAllTopics || filteringByTopic) { + if (parsingAllTopics || filteringByThisTopic) { var str = record.queryContents; - ontology = traverse(str.queryDefinition.expr, record.externalId, ontology); + ontology = traverse(str.queryDefinition.expr || str.queryDefinition.subQuery, record.externalId, ontology); + filteredCount++; } } ontology.userName = (ln) ? queryRecords[0].user.userName : ''; if (parsingAllTopics) { ontology.topics = formatTopicsToArray(topics); } - ontology.queryCount = ln; + ontology.queryCount = parsingAllTopics ? ln : filteredCount; return ontology; } function formatTopicsToArray(topicObject) { var topicArray = []; var keys = Object.keys(topicObject); Object.keys(topicObject) .forEach(function (key) { topicArray.push(topicObject[key]); }); return topicArray; } function appendTopicIfUnique(topic, topics) { if (!topic) { return; } var id = topic.id; if (!topics[id]) { topics[id] = topic; } return topics; } /** * Recursive Traversal of ontological hierarchy. */ function traverse(obj, queryId, terms) { + if (typeof (obj) !== 'object') { + return terms; + } + var keys = Object.keys(obj); // -- nothing to search, we're done. -- // - if (typeof (obj) !== 'object' || !keys.length) { + if (!keys.length) { return terms; } // -- traverse each object key, parse all the terms. -- // for (var i = 0; i < keys.length; i++) { var key = keys[i]; if (key === 'term') { if (!Array.isArray(obj.term)) { processKey(terms, obj.term, queryId); } else { // -- iterate through the termlist -- // var termList = obj.term; for (var j = 0; j < termList.length; j++) { var term = termList[j]; processKey(terms, term, queryId); } } } else { traverse(obj[key], queryId, terms); } } return terms; } /** * Parse out a term hierarchy of a term key. */ function processKey(terms, key, queryId) { var prunedKey = key.split('\\\\SHRINE\\SHRINE')[1]; var ontologyList = prunedKey.split('\\').slice(1); ontologyList = ontologyList.slice(0, ontologyList.length - 1); var currentLevel = terms; for (var i = 0; i < ontologyList.length; i++) { var term = ontologyList[i]; processTerm(currentLevel, term, queryId); currentLevel = currentLevel.children[term]; } } /** * If a term is already a child of the parent, then add it's query to the list of queries for that term. * Otherwise, create a new term object and add it to the parent. */ function processTerm(terms, key, queryId) { var term; if (!terms.children[key]) { term = addTerm(terms, key, queryId); } else { term = terms.children[key]; term.addQuery(queryId, queryId); } } /** * Add an instance of a query term to it's parent in the ontolgy. */ function addTerm(terms, key, queryId) { var term = getNewTerm(key); term.addQuery(queryId, queryId); terms.children[key] = term; return term; } /** * Return Instance of Ontology Term. */ function getNewTerm(name) { return new OntologyTerm(name); } } })(); \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/query-digest.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-digest/query-digest.tpl.html index 8aae241bc..38afb9a01 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/query-digest.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/query-digest.tpl.html @@ -1,20 +1,22 @@
    -
    +
    + +
    - +
    \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html index 61cb655b4..661463d3b 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html @@ -1,15 +1,15 @@
    - + {{value.key}} - {{svc.getPct(value.queryCount, digest.totalQueries)}}%
    \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.js b/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.js index 94ec19e7f..214f5cb5e 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.js +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.js @@ -1,43 +1,57 @@ (function () { 'use strict'; angular.module('shrine.steward.statistics') .directive('topicDropdown', TopicDropdownFactory); function TopicDropdownFactory() { var templateUrl = './app/client/statistics/' + 'query-digest/topic-dropdown.tpl.html'; var topicDropdown = { scope: { topics: '=', topicSelected: '&' }, restrict: 'E', controller: TopicDropdownController, link: TopicDropdownLinker, controllerAs: 'dropdown', templateUrl: templateUrl }; return topicDropdown; } TopicDropdownController.$inject = ['$scope', 'OntologyTermService']; function TopicDropdownController($scope, OntologyTermService) { var dropdown = this; dropdown.ontologyTermService = OntologyTermService; dropdown.topics = $scope.topics; + dropdown.reset = reset; + + /** + * Reset handler. + */ + function reset(events, data) { + dropdown.selected = { + title: 'All' + }; + } } function TopicDropdownLinker(scope) { - var clearWatch = scope.$watch('topics', function (after, before) { + var clearTopicsWatch = scope.$watch('topics', function (after, before) { var dropdown = scope.dropdown; + + // -- listen for reset --// + scope.$on('reset-digest', dropdown.reset); + if (after && after.length) { dropdown.topics = after; } }); } })(); diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.tpl.html index a578603a8..8ead45444 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/topic-dropdown.tpl.html @@ -1,16 +1,16 @@ -
    +
    - Filter By Topic: -
    - -
    \ No newline at end of file + Filter By Topic: +
    + + +
    diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-topic-status-table.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-topic-status-table.tpl.html deleted file mode 100644 index 8382eefdc..000000000 --- a/apps/steward-app/src/main/js/app/client/statistics/query-topic-status-table.tpl.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - -
    - Query Topics By Status -
    - Status - - Query Topic Count -
    {{parseStateTitle(state)}}{{parseStateCount(state)}}
    Total: {{topicsPerState.total}}
    \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html index c6b06507c..e5ba47d4d 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html @@ -1,8 +1,8 @@
    {{username}}
    -
    +
    {{value}} queries
    \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js index 5dd69cb42..2179f2d55 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js @@ -1,50 +1,56 @@ (function () { angular.module('shrine.steward.statistics') .directive('statisticsGraph', StatisticsGraphDirective); function StatisticsGraphDirective() { var templateUrl = './app/client/statistics/statistics-graph/' + 'statistics-graph.tpl.html'; var statisticsGraph = { restrict: 'E', templateUrl: templateUrl, controller: StatisticsGraphController, controllerAs: 'graph', link: StatisticsGraphLink, scope: { graphData: '=', graphClick: '&' /** todo pass in click handler. **/ } }; return statisticsGraph; } StatisticsGraphController.$inject = ['$scope', 'StatisticsGraphService']; function StatisticsGraphController($scope, svc) { var graph = this; //graphService = svc; graph.graphData = $scope.graphData; graph.toPercentage = toPercentage; + graph.formatUsername = formatUsername; graph.graphClick = $scope.graphClick; graph.clearUsers = svc.clearUsers; + graph.formatUsername = formatUsername; function toPercentage(value) { var maxQueryCount = svc.getMaxUserQueryCount(graph.graphData.users) || 1; return svc.getCountAsPercentage(value, maxQueryCount); } + + function formatUsername(username) { + return svc.formatUsername(username); + } } StatisticsGraphLink.$inject = ['scope']; function StatisticsGraphLink(scope) { - scope.$watch('graphData', function(before, after) { + scope.$watch('graphData', function (before, after) { var graph = scope.graph; graph.graphData = scope.graphData; graph.clearUsers(); }); var test = arguments; } })(); \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js index 716db85da..21545768d 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js @@ -1,54 +1,59 @@ var Service; (function () { 'use strict'; angular.module('shrine.steward.statistics') .service('StatisticsGraphService', StatisticsGraphService); var sortedUsers = []; function StatisticsGraphService() { return { getSortedUsers: getSortedUsers, getMaxUser: getMaxUser, getMaxUserQueryCount: getMaxUserQueryCount, getCountAsPercentage: getCountAsPercentage, + formatUsername: formatUsername, clearUsers: clearUsers }; function getSortedUsers(users) { if (!sortedUsers.length) { sortedUsers = _.sortBy(users, [function (o) { console.log(o); return o._2; }]).reverse(); } return sortedUsers; } function getMaxUser(users) { var sortedUsers = getSortedUsers(users); return (!!sortedUsers.length) ? sortedUsers[0] : sortedUsers; } function getMaxUserQueryCount(users) { var maxUser = getMaxUser(users); return maxUser._2; } function getCountAsPercentage(userQueryCount, maxQueryCount) { var basePct = 20; return 100 * (userQueryCount / maxQueryCount) + basePct; } + function formatUsername(username) { + return (username.length > 10) ? username.substring(0, 10) + '...' : username; + } + function clearUsers() { sortedUsers = []; } } Service = StatisticsGraphService; })(); diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js index 935de3a7f..ff513f8d8 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js @@ -1,38 +1,52 @@ (function () { 'use strict'; describe('Statistics Graph Service Tests', StatisticsGraphServiceSpec); function StatisticsGraphServiceSpec() { var statsGraphService; function setup() { module('shrine.steward.statistics'); inject(function (_StatisticsGraphService_) { statsGraphService = _StatisticsGraphService_; }); } beforeEach(setup); it('Statistics Graph Service should exist', function () { expect(typeof (statsGraphService)).toBe('object'); }); it('getSortedUsers should exist', function () { expect(typeof (statsGraphService.getSortedUsers)).toBe('function'); }); it('getMaxUser should exist', function () { expect(typeof (statsGraphService.getMaxUser)).toBe('function'); }); it('getMaxUserQueryCount should exist', function () { expect(typeof (statsGraphService.getMaxUserQueryCount)).toBe('function'); }); it('getCountAsPercentage should exist', function () { expect(typeof (statsGraphService.getCountAsPercentage)).toBe('function'); }); + it('formatUserame should exist', function () { + expect(typeof (statsGraphService.formatUsername)).toBe('function'); + }); + + it('formatUsername should truncate benajamindanielcarmen to benjaminda...', function () { + var result = statsGraphService.formatUsername('benjamindanielcarmen'); + expect(result).toBe('benjaminda...'); + }); + + it('formatUsername should not truncate ben', function () { + var result = statsGraphService.formatUsername('ben'); + expect(result).toBe('ben'); + }); + } })(); \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html index 44a5c0156..992a960a4 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html @@ -1,11 +1,11 @@
    There is no Data for this time period.
    diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js b/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js index 03f15f503..ac8b90b9f 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js @@ -1,140 +1,145 @@ (function () { 'use strict'; angular .module('shrine.steward.statistics') .controller('StatisticsController', StatisticsController); StatisticsController.$inject = ['StatisticsModel', 'StewardService', '$scope']; function StatisticsController(model, service, $scope) { var showOntClass = 'ont-overlay'; var hideOntClass = 'ont-hidden'; - var stats = this; - stats.ontClass = hideOntClass; var startDate = new Date(); var endDate = new Date(); - startDate.setDate(endDate.getDate() - 7); + // -- public vars --// + stats.ontClass = hideOntClass; + startDate.setDate(endDate.getDate() - 7); stats.getDateString = service.commonService.dateService.utcToMMDDYYYY; stats.timestampToUtc = service.commonService.dateService.timestampToUtc; stats.viewDigest = viewDigest; - stats.startDate = startDate; + stats.startDate = startDate; stats.endDate = endDate; - stats.isValid = true; stats.startOpened = false; stats.endOpened = false; stats.topicsPerState = {}; stats.ontology = {}; - stats.graphData = { total: 0, users: [] }; - stats.format = 'MM/dd/yyyy'; + // -- public methods --// stats.openStart = openStart; stats.openEnd = openEnd; stats.validateRange = validateRange; stats.addDateRange = addDateRange; stats.parseStateTitle = parseStateTitle; stats.parseStateCount = parseStateCount; stats.getResults = getResults; stats.viewDigest = viewDigest; - + stats.broadcastReset = broadcastReset; // -- start -- // init(); // -- private -- // function init() { addDateRange(); } function openStart() { stats.startOpened = true; } function openEnd() { stats.endOpened = true; } + // -- @todo: code sloppy, refactor --// function validateRange() { var startUtc, endUtc; - var secondsPerDay = 86400000; if (stats.startDate === undefined || stats.endDate === undefined) { stats.isValid = false; return; } - //can validate date range here. + //@todo: abstract to date methods. i.e. dateService.floor(date) and dateService.ceil(date) + stats.startDate.setHours(0, 0, 0, 0); + stats.endDate.setHours(23, 59, 59, 999); + + //@todo: abstract to reusable Date validate range method i.e dateService.validate(startDate, endDate). startUtc = stats.timestampToUtc(stats.startDate); - endUtc = stats.timestampToUtc(stats.endDate) + secondsPerDay; + endUtc = stats.timestampToUtc(stats.endDate); - if (endUtc - startUtc <= 0) { + if (endUtc - startUtc <= 0) { stats.isValid = false; } else { stats.isValid = true; } return stats.isValid; } function addDateRange() { if (stats.validateRange()) { - var secondsPerDay = 86400000; stats.getResults(stats.timestampToUtc(stats.startDate), - stats.timestampToUtc(stats.endDate) + secondsPerDay); + stats.timestampToUtc(stats.endDate)); } } function parseStateTitle(state) { var title = ''; if (state.Approved !== undefined) { title = 'Approved'; } else { title = (state.Rejected !== undefined) ? 'Rejected' : 'Pending'; } return title; } function viewDigest(data) { - var secondsPerDay = 86400000; var startUtc = stats.timestampToUtc(stats.startDate); - var endUtc = stats.timestampToUtc(stats.endDate) + secondsPerDay; + var endUtc = stats.timestampToUtc(stats.endDate); + model.getUserQueryHistory(data.userName.toLowerCase(), startUtc, endUtc) - .then(function (result) { - stats.ontology = result.queryRecords; - stats.ontClass = showOntClass; - }); + .then(function (result) { + stats.ontology = result.queryRecords; + stats.ontClass = showOntClass; + }); } function parseStateCount(state) { var member = stats.parseStateTitle(state); return state[member]; } function getResults(startUtc, endUtc) { model.getQueriesPerUser(startUtc, endUtc) .then(function (result) { stats.graphData = result; }); model.getTopicsPerState(startUtc, endUtc) .then(function (result) { stats.topicsPerState = result; }); } + + function broadcastReset() { + $scope.$broadcast('reset-digest'); + } } -})(); \ No newline at end of file +})(); diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html index 4d1bc92bf..403c1fb0c 100644 --- a/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html +++ b/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html @@ -1,88 +1,83 @@


    Query Counts By User

    - + - - - - -
    Query Topics By Status
    Status Query Topic Count
    {{stats.parseStateTitle(state)}} {{stats.parseStateCount(state)}}
    Total: {{stats.topicsPerState.total}}
    \ No newline at end of file diff --git a/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss b/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss index 7f7421e68..ec213bc71 100644 --- a/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss +++ b/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss @@ -1,127 +1,130 @@ .topic-dropdown { div.title{ text-transform: uppercase; font-weight: 500; font-size: 1.3em; color: #63BCF8; } z-index: 20000; margin: 4rem; left: 0%; color: rgba(0, 0, 0, 0.87); font-family: "Roboto", sans-serif; font-size: 14px; font-weight: 300; line-height: 1.5em; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; position: fixed; .tdd-btn { &:hover{ cursor: pointer; } text-align: left; max-width: 1000px; outline: 0; /* takes care of blue outline */ display: -webkit-inline-box; display: -webkit-inline-flex; display: -ms-inline-flexbox; display: inline-flex; -webkit-box-align: center; -webkit-align-items: center; -ms-flex-align: center; align-items: center; -webkit-box-pack: justify; -webkit-justify-content: space-between; -ms-flex-pack: justify; justify-content: space-between; background: rgba(0,0,0, .75); border: 1px solid #63BCF8 !important; min-width: 400px; border: 0; border-radius: 4px; box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); box-sizing: border-box; padding: 16px 20px; color: #FFFFFF; font-size: 12px; font-weight: 300; letter-spacing: 1.2px; text-transform: uppercase; overflow: hidden; cursor: pointer; + }; .tdd-list { position: absolute; top: 100%; left: 0px; background: rgba(0,0,0, .75); border: 1px solid #63BCF8 !important; width: 100%; border-radius: 4px; box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); text-align: left; opacity: 0; visibility: hidden; -webkit-transition: 0.3s ease; transition: 0.3s ease; + max-height: 75vh; + overflow: auto; a { display: block; border-bottom: 1px solid rgba(0, 0, 0, 0.05); padding: 16px 0; color: inherit; text-decoration: none; background: none !important; }; &:before { content: ''; position: absolute; top: -6px; left: 20px; width: 0; height: 0; box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05); border-top: 6px solid #63BCF8;; border-right: 6px solid #63BCF8;; border-bottom: 6px solid transparent; border-left: 6px solid transparent; -webkit-transform: rotate(-45deg); transform: rotate(-45deg); mix-blend-mode: multiple; }; li { z-index: 100; position: relative; padding: 0 20px; + color: white; &.active { color: #5380F7; } &:first-child { border-radius: 4px 4px 0 0; } &:last-child { border-radius: 0 0 4px 4px; } &:last-child a { border-bottom: 0; } }; }; - &:focus .tdd-list, &:active .tdd-list { + .tdd-list-open { -webkit-transform: translate(0, 20px); transform: translate(0, 20px); - opacity: 1; - visibility: visible; + opacity: 1 !important; + visibility: visible !important; }; - }; } diff --git a/apps/steward-app/src/main/js/bower.json b/apps/steward-app/src/main/js/bower.json index eda35eff5..5e1051acd 100644 --- a/apps/steward-app/src/main/js/bower.json +++ b/apps/steward-app/src/main/js/bower.json @@ -1,62 +1,65 @@ { "name": "shrine-data-steward-2.0", "description": "HMS Data Steward", "main": "start.js", "authors": [ "Ben Carmen" ], "license": "MIT", "keywords": [ "hms", "steward", "app" ], "moduleType": [], "homepage": "https://open.med.harvard.edu/project/shrine/", "ignore": [ "**/.*", "node_modules", "bower_components", "src/vendor", "test", "tests" ], "dependencies": { "bootstrap": "v3.3.7", "metisMenu": "v2.5.2", "angular-loading-bar": "v0.9.0", "font-awesome": "v4.6.3", "jquery": "v2.2.4", "angular": "v1.5.7", "json3": "v3.3.2", "angular-ui-router": "v0.3.1", "angular-route": "v1.5.7", "angular-cookies": "v1.5.7", "oclazyload": "ocLazyLoad#v1.0.9", "lodash": "v4.13.1", "angular-bootstrap": "v1.3.3", "Chart.js": "v2.1.6", "x2js": "x2js#v1.2.0", "angular-animate": "v1.5.8" }, "devDependencies": { "angular-mocks": "v1.5.7", "font-awesome": "v4.6.3" }, "overrides": { "bootstrap": { "main": [ "dist/js/bootstrap.min.js", "dist/css/bootstrap.css", "less/bootstrap.less" ] }, "font-awesome": { "main": [ "css/font-awesome.css", "less/font-awesome.less", "scss/font-awesome.scss" ] } + }, + "resolutions": { + "angular": "1.5.8" } } diff --git a/apps/steward-app/src/main/js/bower_components/angular/.bower.json b/apps/steward-app/src/main/js/bower_components/angular/.bower.json index 00d0f0d8b..8eb867dcf 100644 --- a/apps/steward-app/src/main/js/bower_components/angular/.bower.json +++ b/apps/steward-app/src/main/js/bower_components/angular/.bower.json @@ -1,18 +1,18 @@ { "name": "angular", "version": "1.5.8", "license": "MIT", "main": "./angular.js", "ignore": [], "dependencies": {}, "homepage": "https://github.com/angular/bower-angular", "_release": "1.5.8", "_resolution": { "type": "version", "tag": "v1.5.8", "commit": "7e0e546eb6caedbb298c91a9f6bf7de7eeaa4ad2" }, "_source": "https://github.com/angular/bower-angular.git", - "_target": ">=1.4.0", + "_target": "1.5.8", "_originalSource": "angular" } \ No newline at end of file diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala index a94f3c75c..727306998 100644 --- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala +++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala @@ -1,740 +1,766 @@ package net.shrine.steward.db import java.sql.SQLException +import java.util.concurrent.TimeoutException import java.util.concurrent.atomic.AtomicInteger import javax.sql.DataSource import com.typesafe.config.Config import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearcherToAudit, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole} import net.shrine.i2b2.protocol.pm.User import net.shrine.log.Loggable -import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator} +import net.shrine.problem.{AbstractProblem, ProblemSources} +import net.shrine.slick.{CouldNotRunDbIoActionException, NeedsWarmUp, TestableDataSourceCreator} import net.shrine.source.ConfigSource import net.shrine.steward.CreateTopicsMode import slick.dbio.Effect.Read import slick.driver.JdbcProfile import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, Future, blocking} import scala.language.postfixOps import scala.util.Try +import scala.util.control.NonFatal /** * Database access code for the data steward service. * * I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace. * * @author dwalend * @since 1.19 */ case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable { import schemaDef._ import jdbcProfile.api._ val database = Database.forDataSource(dataSource) def createTables() = schemaDef.createTables(database) def dropTables() = schemaDef.dropTables(database) + //todo share code from DashboardProblemDatabase.scala . It's a lot richer. See SHRINE-1835 def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = { - val future: Future[R] = database.run(action) - blocking { - Await.result(future, 10 seconds) + try { + val future: Future[R] = database.run(action) + blocking { + Await.result(future, 10 seconds) + } + } catch { + case tax:TopicAcessException => throw tax + case tx:TimeoutException => + val x = CouldNotRunDbIoActionException(dataSource, tx) + StewardDatabaseProblem(x) + throw x + case NonFatal(nfx) => + val x = CouldNotRunDbIoActionException(dataSource, nfx) + StewardDatabaseProblem(x) + throw x } } def warmUp = { dbRun(allUserQuery.size.result) } def selectUsers:Seq[UserRecord] = { dbRun(allUserQuery.result) } // todo use whenever a shrine query is logged def upsertUser(user:User):Unit = { val userRecord = UserRecord(user) dbRun(allUserQuery.insertOrUpdate(userRecord)) } def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = { val createInState = CreateTopicsMode.createTopicsInState val now = System.currentTimeMillis() val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState) val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now) dbRun(for{ _ <- allTopicQuery += topicRecord _ <- allUserTopicQuery += userTopicRecord } yield topicRecord) } def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try { dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option => val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId)) if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy) if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId) val updatedTopic = oldTopicRecord.copy(name = topicRequest.name, description = topicRequest.description, changedBy = user.username, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).flatMap{_ => outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic) } } ) } def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = { require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id") val (count,topics,userNamesToOutboundUsers) = dbRun( for{ count <- topicCountQuery(parameters).length.result topics <- topicSelectQuery(parameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count, topics,userNamesToOutboundUsers)) ResearchersTopics(parameters.researcherIdOption.get, count, parameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } //treat as private (currently used in test) def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = { dbRun(topicSelectQuery(queryParameters).result) } def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = { val (count,topics,userNamesToOutboundUsers) = dbRun{ for{ count <- topicCountQuery(queryParameters).length.result topics <- topicSelectQuery(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count,topics,userNamesToOutboundUsers) } StewardsTopics(count, queryParameters.skipOption.getOrElse(0), topics.map(_.toOutboundTopic(userNamesToOutboundUsers))) } private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val countFilter = topicCountQuery(queryParameters) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(countFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countFilter)( columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "id" => x.id case "name" => x.name case "description" => x.description case "createdBy" => x.createdBy case "createDate" => x.createDate case "state" => x.state case "changedBy" => x.changedBy case "changeDate" => x.changeDate }))) val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = { val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId)) val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate)) maxDateFilter } def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = { val noTopicRecord:Option[TopicRecord] = None val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord) dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap( _.fold(noOpDBIO){ originalTopic => val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis()) (allTopicQuery += updatedTopic).map(_ => Option(updatedTopic)) } )) } def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = { dbRun(for{ totalTopics <- topicCountQuery(queryParameters).length.result topicsPerStateName <- topicCountsPerState(queryParameters).result } yield TopicsPerState(totalTopics,topicsPerStateName)) } private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = { val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state) groupedByState.map{case (state,result) => (state,result.length)} } def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = { //todo upsertUser(user) when the info is available from the PM val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful { if (CreateTopicsMode.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved else TopicState.createTopicsModeRequiresTopic } val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None} val (state,topicName) = dbRun(for{ state <- topicId.fold(noOpDBIOForState)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map( _.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state))) ) topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId => mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption ) _ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state) } yield (state,topicName)) val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match { case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name)) case (None,None) => None case (Some(id),None) => if(state == TopicState.unknownForUser) None else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery") case (None,Some(name)) => if(state == TopicState.unknownForUser) None else throw new IllegalStateException(s"How did you get here for $userId with no topic id but a topic name of $name and $state for $shrineQuery") } (state,topicIdAndName) } def selectQueryHistory(queryParameters: QueryParameters, topicParameter:Option[TopicId]): QueryHistory = { val topicQuery = for { count <- shrineQueryCountQuery(queryParameters, topicParameter).length.result shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set]) } yield (count, shrineQueries, topics, userNamesToOutboundUsers) val (count, shrineQueries, topics, userNamesToOutboundUsers) = dbRun(topicQuery) val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = { val topic = topicIdsToTopics.get(queryRecord.topicId) val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers)) val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId) //todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId)) queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser) } QueryHistory(count, queryParameters.skipOption.getOrElse(0), shrineQueries.map(toOutboundShrineQuery)) } private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = { allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap) } private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val countQuery = shrineQueryCountQuery(queryParameters,topicParameter) //todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one. // val orderByQuery = queryParameters.sortByOption.fold(limitFilter)( // columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName)))) val orderByQuery = queryParameters.sortByOption.fold(countQuery) { case "topicName" => val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id) joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1) case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match { case "stewardId" => x.stewardId case "externalId" => x.externalId case "researcherId" => x.researcherId case "name" => x.name case "topic" => x.topicId case "queryContents" => x.queryContents case "stewardResponse" => x.stewardResponse case "date" => x.date })) } val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip)) val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit)) limitFilter } private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = { val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId)) val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId)) //todo this is probably a binary Approved/Not approved val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name)) val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate)) val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate)) maxDateFilter } def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = { val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for { totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set]) } yield (totalQueries,queriesPerUser,userNamesToOutboundUsers)) val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2)) QueriesPerUser(totalQueries,queriesPerOutboundUser) } private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = { val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None) val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId) groupedByResearcher.map{case (researcher,result) => (researcher,result.length)} } lazy val nextTopicId:AtomicInteger = new AtomicInteger({ dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1 }) def selectAllAuditRequests: Seq[UserAuditRecord] = { dbRun(allUserAudits.result) } def selectMostRecentAuditRequests: Seq[UserAuditRecord] = { dbRun(mostRecentUserAudits.result) } def selectResearchersToAudit(maxQueryCountBetweenAudits:Int,minTimeBetweenAudits:Duration,now:Date):Seq[ResearcherToAudit] = { //todo one round with the db instead of O(researchers) //for each researcher //horizon = if the researcher has had an audit // date of last audit // else if no audit yet // date of first query val researchersToHorizons: Map[UserName, Date] = dbRun(for{ dateOfFirstQuery: Seq[(UserName, Date)] <- leastRecentUserQuery.map(record => record.researcherId -> record.date).result mostRecentAudit: Seq[(UserName, Date)] <- mostRecentUserAudits.map(record => record.researcher -> record.changeDate).result } yield { dateOfFirstQuery.toMap ++ mostRecentAudit.toMap }) val researchersToHorizonsAndCounts = researchersToHorizons.map{ researcherDate => val queryParameters = QueryParameters(researcherIdOption = Some(researcherDate._1), minDate = Some(researcherDate._2)) val count:Int = dbRun(shrineQueryCountQuery(queryParameters,None).length.result) (researcherDate._1,(researcherDate._2,count)) } //audit if oldest query within the horizon is >= minTimeBetweenAudits in the past and the researcher has run at least one query since. val oldestAllowed = System.currentTimeMillis() - minTimeBetweenAudits.toMillis val timeBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 > 0 && x._2._1 <= oldestAllowed) //audit if the researcher has run >= maxQueryCountBetweenAudits queries since horizon? val queryBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 >= maxQueryCountBetweenAudits) val toAudit = timeBasedAudit ++ queryBasedAudit val namesToOutboundUsers: Map[UserName, OutboundUser] = dbRun(outboundUsersForNamesAction(toAudit.keySet)) toAudit.map(x => ResearcherToAudit(namesToOutboundUsers(x._1),x._2._2,x._2._1,now)).to[Seq] } def logAuditRequests(auditRequests:Seq[ResearcherToAudit],now:Date) { dbRun{ allUserAudits ++= auditRequests.map(x => UserAuditRecord(researcher = x.researcher.userName, queryCount = x.count, changeDate = now )) } } } /** * Separate class to support schema generation without actually connecting to the database. * * @param jdbcProfile Database profile to use for the schema */ case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable { import jdbcProfile.api._ def ddlForAllTables = { allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema ++ allUserAudits.schema } //to get the schema, use the REPL //println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n")) def createTables(database:Database) = { try { val future = database.run(ddlForAllTables.create) Await.result(future,10 seconds) } catch { //I'd prefer to check and create schema only if absent. No way to do that with Oracle. case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x) } } def dropTables(database:Database) = { val future = database.run(ddlForAllTables.drop) //Really wait forever for the cleanup Await.result(future,Duration.Inf) } class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") { def userName = column[UserName]("userName",O.PrimaryKey) def fullName = column[String]("fullName") def isSteward = column[Boolean]("isSteward") def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply) } class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") { def id = column[TopicId]("id") def name = column[String]("name") def description = column[String]("description") def createdBy = column[UserName]("createdBy") def createDate = column[Date]("createDate") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def idIndex = index("idIndex",id,unique = false) def topicNameIndex = index("topicNameIndex",name,unique = false) def createdByIndex = index("createdByIndex",createdBy,unique = false) def createDateIndex = index("createDateIndex",createDate,unique = false) def stateIndex = index("stateIndex",state,unique = false) def changedByIndex = index("changedByIndex",changedBy,unique = false) def changeDateIndex = index("changeDateIndex",changeDate,unique = false) def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply) def fromRow = (fromParams _).tupled def fromParams(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, stateName:String, changedBy:UserName, changeDate:Date): TopicRecord = { TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(topicRecord: TopicRecord) = Some((topicRecord.id, topicRecord.name, topicRecord.description, topicRecord.createdBy, topicRecord.createDate, topicRecord.state.name, topicRecord.changedBy, topicRecord.changeDate )) } class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") { def researcher = column[UserName]("researcher") def topicId = column[TopicId]("topicId") def state = column[TopicStateName]("state") def changedBy = column[UserName]("changedBy") def changeDate = column[Date]("changeDate") def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true) def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow) def fromRow = (fromParams _).tupled def fromParams(researcher:UserName, topicId:TopicId, stateName:String, changedBy:UserName, changeDate:Date): UserTopicRecord = { UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate) } def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] = Some((userTopicRecord.researcher, userTopicRecord.topicId, userTopicRecord.state.name, userTopicRecord.changedBy, userTopicRecord.changeDate )) } class UserAuditTable(tag:Tag) extends Table[UserAuditRecord](tag,"userAudit") { def researcher = column[UserName]("researcher") def queryCount = column[Int]("queryCount") def changeDate = column[Date]("changeDate") def * = (researcher, queryCount, changeDate) <> (fromRow, toRow) def fromRow = (fromParams _).tupled def fromParams(researcher:UserName, queryCount:Int, changeDate:Date): UserAuditRecord = { UserAuditRecord(researcher,queryCount, changeDate) } def toRow(record: UserAuditRecord):Option[(UserName,Int,Date)] = Some((record.researcher, record.queryCount, record.changeDate )) } class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") { def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc) def externalId = column[ExternalQueryId]("id") def name = column[String]("name") def researcherId = column[UserName]("researcher") def topicId = column[Option[TopicId]]("topic") def queryContents = column[QueryContents]("queryContents") def stewardResponse = column[String]("stewardResponse") def date = column[Date]("date") def externalIdIndex = index("externalIdIndex",externalId,unique = false) def queryNameIndex = index("queryNameIndex",name,unique = false) def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false) def topicIdIndex = index("topicIdIndex",topicId,unique = false) def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false) def dateIndex = index("dateIndex",date,unique = false) def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow) def fromRow = (fromParams _).tupled def fromParams(stewardId:Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:String, date:Date): ShrineQueryRecord = { ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date) } def toRow(queryRecord: ShrineQueryRecord):Option[( Option[StewardQueryId], ExternalQueryId, String, UserName, Option[TopicId], QueryContents, String, Date )] = Some((queryRecord.stewardId, queryRecord.externalId, queryRecord.name, queryRecord.userId, queryRecord.topicId, queryRecord.queryContents, queryRecord.stewardResponse.name, queryRecord.date) ) } val allUserQuery = TableQuery[UserTable] val allTopicQuery = TableQuery[TopicTable] val allQueryTable = TableQuery[QueryTable] val allUserTopicQuery = TableQuery[UserTopicTable] val allUserAudits = TableQuery[UserAuditTable] val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for( topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists ) yield topic val mostRecentUserAudits: Query[UserAuditTable, UserAuditRecord, Seq] = for( record <- allUserAudits if !allUserAudits.filter(_.researcher === record.researcher).filter(_.changeDate > record.changeDate).exists ) yield record val leastRecentUserQuery: Query[QueryTable, ShrineQueryRecord, Seq] = for( record <- allQueryTable if !allQueryTable.filter(_.researcherId === record.researcherId).filter(_.date < record.date).exists ) yield record } object StewardSchema { val allConfig:Config = ConfigSource.config val config:Config = allConfig.getConfig("shrine.steward.database") val slickProfile:JdbcProfile = ConfigSource.getObject("slickProfileClassName", config) val schema = StewardSchema(slickProfile) } object StewardDatabase extends NeedsWarmUp { val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config) val db = StewardDatabase(StewardSchema.schema,dataSource) val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart") if(createTablesOnStart) StewardDatabase.db.createTables() override def warmUp() = StewardDatabase.db.warmUp } //API help sealed case class SortOrder(name:String){ import slick.lifted.ColumnOrdered def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = { if(this == SortOrder.ascending) column.asc else column.desc } } object SortOrder { val ascending = SortOrder("ascending") val descending = SortOrder("descending") val sortOrders = Seq(ascending,descending) val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_)) } case class QueryParameters(researcherIdOption:Option[UserName] = None, stateOption:Option[TopicState] = None, skipOption:Option[Int] = None, limitOption:Option[Int] = None, sortByOption:Option[String] = None, sortOrder:SortOrder = SortOrder.ascending, minDate:Option[Date] = None, maxDate:Option[Date] = None ) //DAO case classes, exposed for testing only case class ShrineQueryRecord(stewardId: Option[StewardQueryId], externalId:ExternalQueryId, name:String, userId:UserName, topicId:Option[TopicId], queryContents: QueryContents, stewardResponse:TopicState, date:Date) { def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = { OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,queryContents,stewardResponse.name,date) } } object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) { def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = { ShrineQueryRecord( None, shrineQuery.externalId, shrineQuery.name, userId, topicId, shrineQuery.queryContents, stewardResponse, System.currentTimeMillis()) } } case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) { lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole) else Set(researcherRole)) } object UserRecord extends ((UserName,String,Boolean) => UserRecord) { def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true"))) } case class TopicRecord(id:Option[TopicId] = None, name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState, changedBy:UserName, changeDate:Date) { def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = { OutboundTopic(id.get, name, description, userNamesToOutboundUsers(createdBy), createDate, state.name, userNamesToOutboundUsers(changedBy), changeDate) } } object TopicRecord { def apply(id:Option[TopicId], name:String, description:String, createdBy:UserName, createDate:Date, state:TopicState ):TopicRecord = TopicRecord(id, name, description, createdBy, createDate, state, createdBy, createDate) } case class UserTopicRecord(researcher:UserName, topicId:TopicId, state:TopicState, changedBy:UserName, changeDate:Date) case class UserAuditRecord(researcher:UserName, queryCount:Int, changeDate:Date) { def sameExceptForTimes(userAuditRecord: UserAuditRecord):Boolean = { (researcher == userAuditRecord.researcher) && (queryCount == userAuditRecord.queryCount) } } -case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId") +abstract class TopicAcessException(topicId: TopicId,message:String) extends IllegalArgumentException(message) + +case class TopicDoesNotExist(topicId:TopicId) extends TopicAcessException(topicId,s"No topic for id $topicId") + +case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends TopicAcessException(topicId,s"Topic $topicId has been ${TopicState.approved}") + +case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends TopicAcessException(topicId,s"$userId does not own $topicId; $ownerId owns it.") + +case class StewardDatabaseProblem(cnrdiax:CouldNotRunDbIoActionException) extends AbstractProblem(ProblemSources.Dsa) { + override def summary: String = "The DSA's database failed due to an exception." -case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}") + override def description: String = s"TThe DSAs database failed due to $cnrdiax" -case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.") \ No newline at end of file + override def throwable = Some(cnrdiax) +} \ No newline at end of file diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala b/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala index 508e4db4c..144626ce0 100644 --- a/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala +++ b/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala @@ -1,144 +1,144 @@ package net.shrine.steward.email import java.util.Date import javax.mail.internet.InternetAddress import akka.actor.Actor import com.typesafe.config.Config import courier.{Envelope, Mailer, Text} import net.shrine.authorization.steward.ResearcherToAudit import net.shrine.config.{ConfigExtensions, DurationConfigParser} import net.shrine.email.ConfiguredMailer import net.shrine.log.Log import net.shrine.problem.{AbstractProblem, ProblemSources} import net.shrine.source.ConfigSource import net.shrine.steward.db.StewardDatabase import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.blocking import scala.concurrent.duration.FiniteDuration import scala.concurrent.duration._ import scala.util.control.NonFatal import scala.xml.NodeSeq /** * @author david * @since 1.22 */ case class AuditEmailer(maxQueryCountBetweenAudits:Int, minTimeBetweenAudits:FiniteDuration, researcherLineTemplate:String, emailTemplate:String, emailSubject:String, from:InternetAddress, to:InternetAddress, stewardBaseUrl:String, //todo not an option mailer:Mailer ) { def audit() = { //gather a list of users to audit - Log.info("Auditing users") val now = System.currentTimeMillis() val researchersToAudit: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(maxQueryCountBetweenAudits, minTimeBetweenAudits, now) + Log.info(s"Auditing users ${researchersToAudit.map(_.researcher.userName).mkString(", ")}") if (researchersToAudit.nonEmpty){ val auditLines = researchersToAudit.sortBy(_.count).reverse.map { researcher => researcherLineTemplate.replaceAll("FULLNAME",researcher.researcher.fullName) .replaceAll("USERNAME",researcher.researcher.userName) .replaceAll("COUNT",researcher.count.toString) .replaceAll("LAST_AUDIT_DATE",new Date(researcher.leastRecentQueryDate).toString) }.mkString("\n") //build up the email body val emailBody = Text(emailTemplate.replaceAll("AUDIT_LINES",auditLines) .replaceAll("STEWARD_BASE_URL",stewardBaseUrl)) val envelope:Envelope = Envelope.from(from).to(to).subject(emailSubject).content(emailBody) Log.debug(s"About to send $envelope .") //send the email val future = mailer(envelope) try { blocking { Await.result(future, 60.seconds) } StewardDatabase.db.logAuditRequests(researchersToAudit, now) Log.info(s"Sent and logged $envelope .") } catch { case NonFatal(x) => CouldNotSendAuditEmail(envelope,x) } } } } object AuditEmailer { /** * * @param config All of shrine.conf */ def apply(config:Config):AuditEmailer = { val config = ConfigSource.config val emailConfig = config.getConfig("shrine.steward.emailDataSteward") AuditEmailer( maxQueryCountBetweenAudits = emailConfig.getInt("maxQueryCountBetweenAudits"), minTimeBetweenAudits = emailConfig.get("minTimeBetweenAudits", DurationConfigParser.parseDuration), researcherLineTemplate = emailConfig.getString("researcherLine"), emailTemplate = emailConfig.getString("emailBody"), emailSubject = emailConfig.getString("subject"), from = emailConfig.get("from", new InternetAddress(_)), to = emailConfig.get("to", new InternetAddress(_)), stewardBaseUrl = config.getString("shrine.queryEntryPoint.shrineSteward.stewardBaseUrl"), mailer = ConfiguredMailer.createMailerFromConfig(config.getConfig("shrine.email"))) } /** * Check the emailer's config, log any problems * * @param config All of shrine.conf */ def configCheck(config:Config):Boolean = try { val autoEmailer = apply(config) Log.info(s"DSA will request audits from ${autoEmailer.to}") true } catch { case NonFatal(x) => CannotConfigureAuditEmailer(x) false } } class AuditEmailerActor extends Actor { override def receive: Receive = {case _ => val config = ConfigSource.config AuditEmailer(config).audit() } } case class CannotConfigureAuditEmailer(ex:Throwable) extends AbstractProblem(ProblemSources.Dsa) { override def summary: String = "The DSA will not email audit requests due to a misconfiguration." override def description: String = s"The DSA will not email audit requests due to ${throwable.get}" override def throwable = Some(ex) } case class CouldNotSendAuditEmail(envelope:Envelope,ex:Throwable) extends AbstractProblem(ProblemSources.Dsa) { override def summary: String = "The DSA was not able to send an audit email." override def description: String = s"The DSA was not able to send an audit request to ${envelope.to} due to ${throwable.get}" override def throwable = Some(ex) override def detailsXml:NodeSeq =
    {s"Could not send $envelope"} {throwableDetail}
    } \ No newline at end of file diff --git a/apps/steward-app/src/main/sql/sqlserver.ddl b/apps/steward-app/src/main/sql/mssql.ddl similarity index 100% rename from apps/steward-app/src/main/sql/sqlserver.ddl rename to apps/steward-app/src/main/sql/mssql.ddl diff --git a/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala b/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala index fb363fdb4..603400622 100644 --- a/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala +++ b/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala @@ -1,60 +1,60 @@ package net.shrine.crypto import java.math.BigInteger import java.security.{KeyPairGenerator, PrivateKey, SecureRandom} import java.util.Date import net.shrine.util.NonEmptySeq import org.bouncycastle.asn1.ASN1Sequence import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x509._ import org.bouncycastle.cert.X509v3CertificateBuilder import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import org.scalatest.{FlatSpec, Matchers} /** * Created by ty on 11/1/16. */ @RunWith(classOf[JUnitRunner]) class DownStreamCertCollectionTest extends FlatSpec with Matchers { - val descriptor = NewTestKeyStore.descriptor - val heyo = "Heyo!".getBytes("UTF-8") + val descriptor: KeyStoreDescriptor = NewTestKeyStore.descriptor + val heyo : Array[Byte] = "Heyo!".getBytes("UTF-8") "A down stream cert collection" should "build and verify its own messages" in { val hubCertCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(descriptor) match { case hub: DownStreamCertCollection => hub case _ => fail("This should generate a DownstreamCertCollection!") } val testEntry = CertificateCreator.createSelfSignedCertEntry("notTrusted", "testing", "stillTesting") hubCertCollection.allEntries.size shouldBe 2 hubCertCollection.myEntry.privateKey.isDefined shouldBe true hubCertCollection.caEntry.privateKey.isDefined shouldBe false hubCertCollection.myEntry.aliases.first shouldBe "shrine-test" hubCertCollection.caEntry.aliases.first shouldBe "shrine-test-ca" hubCertCollection.caEntry.wasSignedBy(hubCertCollection.myEntry) shouldBe false hubCertCollection.myEntry.wasSignedBy(hubCertCollection.caEntry) shouldBe true val mySigned = hubCertCollection.myEntry.sign(heyo).get val testSigned = testEntry.sign(heyo).get testEntry.verify(mySigned, heyo) shouldBe false testEntry.verify(testSigned, heyo) shouldBe true testEntry.signed(testEntry.cert) shouldBe true hubCertCollection.myEntry.verify(testSigned, heyo) shouldBe false hubCertCollection.myEntry.verify(mySigned, heyo) shouldBe true hubCertCollection.caEntry.verify(testSigned, heyo) shouldBe false hubCertCollection.caEntry.verify(mySigned, heyo) shouldBe false hubCertCollection.verifyBytes(hubCertCollection.signBytes(heyo), heyo) shouldBe true } } diff --git a/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala b/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala index c0e3e8afb..5cd46ff73 100644 --- a/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala +++ b/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala @@ -1,26 +1,25 @@ package net.shrine.crypto import net.shrine.util.{PeerToPeerModel, SingleHubModel} /** - * @author clint - * @date Nov 27, 2013 - */ + * @author ty + */ object NewTestKeyStore { val fileName = "crypto2/shrine-test.jks" - + val password = "justatestpassword" - + val privateKeyAlias: Option[String] = Some("shrine-test") - + val keyStoreType: KeyStoreType = KeyStoreType.JKS - + val caCertAliases = Seq("shrine-test-ca") - + lazy val descriptor = KeyStoreDescriptor(fileName, password, privateKeyAlias, caCertAliases, SingleHubModel(false), Seq(RemoteSiteDescriptor("hub", Some("shrine-test-ca"), "localhost", "8080")), keyStoreType) - + lazy val certCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(descriptor) - + lazy val trustParam: TrustParam = TrustParam.BouncyKeyStore(certCollection) } \ No newline at end of file diff --git a/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf b/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf index ea9f4e305..05dbf19cb 100644 Binary files a/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf and b/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf differ