diff --git a/adapter/adapter-service/src/main/sql/mssql.ddl b/adapter/adapter-service/src/main/sql/mssql.ddl
index 1af337b5f..7a9875173 100644
--- a/adapter/adapter-service/src/main/sql/mssql.ddl
+++ b/adapter/adapter-service/src/main/sql/mssql.ddl
@@ -1,85 +1,88 @@
/* Audit db tables in adapterAuditDB */
create table "queriesReceived" ("shrineNodeId" VARCHAR(MAX) NOT NULL,"userName" VARCHAR(MAX) NOT NULL,"networkQueryId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"topicId" VARCHAR(MAX),"topicName" VARCHAR(MAX),"timeQuerySent" BIGINT NOT NULL,"timeReceived" BIGINT NOT NULL);
create table "executionsStarted" ("networkQueryId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeExecutionStarted" BIGINT NOT NULL);
create table "executionsCompleted" ("networkQueryId" BIGINT NOT NULL,"replyId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeExecutionCompleted" BIGINT NOT NULL);
create table "resultsSent" ("networkQueryId" BIGINT NOT NULL,"replyId" BIGINT NOT NULL,"queryName" VARCHAR(MAX) NOT NULL,"timeResultsSent" BIGINT NOT NULL);
/* Working tables in shrine_query_history */
+create database shrine_query_history;
+use shrine_query_history;
+
create table SHRINE_QUERY(
[id] [int] not null IDENTITY(1,1),
local_id [varchar](255) not null,
network_id bigint not null,
username [varchar](255) not null,
domain [varchar](255) not null,
query_name [varchar](255) not null,
query_expression text,
date_created datetime default current_timestamp,
has_been_run bit not null default 0,
flagged bit not null default 0,
flag_message [varchar](255) null,
constraint query_id_pk primary key clustered (id asc),
query_xml text
-)
-CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_network_id] ON [dbo].[SHRINE_QUERY] ([network_id] ASC)
-CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_local_id] ON [dbo].[SHRINE_QUERY] ([local_id] ASC)
-CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_username_domain] ON [dbo].[SHRINE_QUERY] (username, domain ASC)
+);
+CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_network_id] ON [dbo].[SHRINE_QUERY] ([network_id] ASC);
+CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_local_id] ON [dbo].[SHRINE_QUERY] ([local_id] ASC);
+CREATE NONCLUSTERED INDEX [ix_SHRINE_QUERY_username_domain] ON [dbo].[SHRINE_QUERY] (username, domain ASC);
-alter table SHRINE_QUERY alter column flag_message [varchar](MAX)
+alter table SHRINE_QUERY alter column flag_message [varchar](MAX);
create table QUERY_RESULT(
id int not null identity(1,1),
local_id varchar(255) not null,
query_id int not null,
[type] varchar(255) not null check ([type] in ('PATIENTSET','PATIENT_COUNT_XML','PATIENT_AGE_COUNT_XML','PATIENT_RACE_COUNT_XML','PATIENT_VITALSTATUS_COUNT_XML','PATIENT_GENDER_COUNT_XML','ERROR')),
[status] varchar(50) not null check ([status] in ('FINISHED', 'ERROR', 'PROCESSING', 'QUEUED')),
time_elapsed int null,
last_updated datetime default current_timestamp,
constraint QUERY_RESULT_id_pk primary key(id),
constraint fk_QUERY_RESULT_query_id foreign key (query_id) references SHRINE_QUERY (id) on delete cascade
-)
+);
create table ERROR_RESULT(
id int not null identity(1,1),
result_id int not null,
message varchar(255) not null,
constraint ERROR_RESULT_id_pk primary key(id),
constraint fk_ERROR_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade
-)
-alter table ERROR_RESULT add column 'CODEC' varchar not null default "Pre-1.20 Error"
-alter table ERROR_RESULT add column 'SUMMARY' text not null default "Pre-1.20 Error"
-alter table ERROR_RESULT add column 'DESCRIPTION' text not null default "Pre-1.20 Error"
-alter table ERROR_RESULT add column 'DETAILS' text not null default "Pre-1.20 Error"
+);
+alter table ERROR_RESULT add CODEC text not null default 'Pre-1.20 Error';
+alter table ERROR_RESULT add SUMMARY text not null default 'Pre-1.20 Error';
+alter table ERROR_RESULT add DESCRIPTION text not null default 'Pre-1.20 Error';
+alter table ERROR_RESULT add DETAILS text not null default 'Pre-1.20 Error';
create table COUNT_RESULT(
id int not null identity(1,1),
result_id int not null,
original_count int not null,
obfuscated_count int not null,
date_created datetime default current_timestamp,
constraint COUNT_RESULT_id_pk primary key(id),
constraint fk_COUNT_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade
-)
+);
create table BREAKDOWN_RESULT(
id int not null identity(1,1),
result_id int not null,
data_key varchar(255) not null,
original_value int not null,
obfuscated_value int not null,
constraint BREAKDOWN_RESULT_id_pk primary key(id),
constraint fk_BREAKDOWN_RESULT_QUERY_RESULT_id foreign key (result_id) references QUERY_RESULT (id) on delete cascade
-)
+);
create table PRIVILEGED_USER(
id int not null identity(1,1),
username varchar(255) not null,
domain varchar(255) not null,
threshold int not null,
override_date timestamp null,
constraint priviliged_user_pk primary key(id),
constraint ix_PRIVILEGED_USER_username_domain unique (username, domain)
-)
+);
create table "problems" ("id" INTEGER NOT NULL PRIMARY KEY IDENTITY, "codec" VARCHAR(254) NOT NULL,"stampText" VARCHAR(500) NOT NULL,"summary" VARCHAR(254) NOT NULL,"description" VARCHAR(MAX) NOT NULL,"detailsXml" VARCHAR(MAX) NOT NULL,"epoch" BIGINT NOT NULL);
create index "idx_epoch" on "problems" ("epoch");
diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js
index ef1f52aac..e027520b5 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/diagnostic.model.js
@@ -1,247 +1,247 @@
(function (){
'use strict';
// -- angular module -- //
angular.module('shrine-tools')
.factory('DiagnosticModel', DiagnosticModel);
DiagnosticModel.$inject = ['$http', '$q', 'UrlGetter', '$location'];
function DiagnosticModel (h, q, urlGetter, $location) {
var toDashboard = {url:''};
var cache = {};
// used solely for remote dashboard persistence
var m = {};
m.remoteSiteStatuses = [];
m.siteAlias = '';
// -- private const -- //
var Config = {
AdapterEndpoint: 'admin/status/adapter',
ConfigEndpoint: 'admin/status/config',
HubEndpoint: 'admin/status/hub',
I2B2Endpoint: 'admin/status/i2b2',
KeystoreEndpoint: 'admin/status/keystore',
OptionsEndpoint: 'admin/status/optionalParts',
ProblemEndpoint: 'admin/status/problems',
QepEndpoint: 'admin/status/qep',
SummaryEndpoint: 'admin/status/summary'
};
// -- public -- //
return {
getAdapter: getJsonMaker(Config.AdapterEndpoint, 'adapter'),
getConfig: getJsonMaker(Config.ConfigEndpoint, 'config', parseConfig),
getHub: getJsonMaker(Config.HubEndpoint, 'hub'),
getI2B2: getJsonMaker(Config.I2B2Endpoint, 'i2b2'),
getKeystore: getJsonMaker(Config.KeystoreEndpoint, 'keystore', storeRemoteSites),
getOptionalParts: getJsonMaker(Config.OptionsEndpoint, 'optionalParts'),
getProblems: getProblemsMaker(),
getQep: getJsonMaker(Config.QepEndpoint, 'qep'),
getSummary: getJsonMaker(Config.SummaryEndpoint, 'summary'),
safeLogout: safeLogout,
clearCache: clearCache,
map: map,
formatDate: formatDate,
cache: cache,
toDashboard: toDashboard,
m: m
};
function map(func, list) {
var result = [];
for(var i = 0; i < list.length; i++) {
result.push(func(list[i]))
}
return result;
}
function formatDate(dateObject) {
return [dateObject.getUTCFullYear(), "-",
pad2(dateObject.getUTCMonth() + 1), "-",
- pad2(dateObject.getUTCDay()), " ",
+ pad2(dateObject.getUTCDate()), " ",
pad2(dateObject.getUTCHours()), ":",
pad2(dateObject.getUTCMinutes()), ":",
pad2(dateObject.getUTCSeconds())].join("");
}
function pad2(stringLikeThing) {
// Does javascript provide a string format thing? Would love to write %02d here.
var stringed = "" + stringLikeThing;
if (stringed.length > 2) {
return stringed;
}
return ("00" + stringed).slice(-2);
}
/**
* Clears the current remote dashboard before logging out.
*/
function safeLogout() {
clearCache();
toDashboard.url = '';
m.siteAlias = '';
$location.path('/login');
}
function clearCache() {
for (var member in cache) {
if(cache.hasOwnProperty(member)) delete cache[member];
}
}
/**
* Method for Handling a failed rest call.
* @param failedResult
* @returns {*}
*/
function onFail(failedResult) {
return q.reject(failedResult);
}
/***
* Method for handling a successful rest call. Simply caches it and returns it.
* @param result
* @param cacheKey
* @returns {*}
*/
function parseJsonResult(result, cacheKey) {
cache[cacheKey] = result.data;
return result.data;
}
/**
* Still cache and return the result, however, save the RemoteSites outside of the cache,
* as we don't want these values to change between cache resets (which occur when switching sites)
* @param result
* @param cacheKey
*/
function storeRemoteSites(result, cacheKey) {
cache[cacheKey] = result.data;
if (m.remoteSiteStatuses.length == 0) {
m.remoteSiteStatuses = result.data.remoteSiteStatuses;
}
return result.data
}
/**
* Parses the json config map and turns it into a nested json object
* @param json the flat config map
* @param cacheKey a unique identifier for the function
*/
function parseConfig (json, cacheKey) {
var configMap = json.data.configMap;
var processed = preProcessJson(configMap);
cache[cacheKey] = processed;
return processed;
}
// IE11 doesn't support string includes
// This only searchers for characters, not arbitrary strings
function stringIncludes(haystack, needle) {
var arr = haystack.split("");
for (var i = 0; i < arr.length; i++) {
if (arr[i] == needle) {
return true;
}
}
return false;
}
// "explodes" and merges the flag config map.
// e.g., {"key.foo": 10, "key.baz": 5} -> {"key": {"foo": 10, "baz": 5}}
function preProcessJson (object) {
var result = {};
for (var key in object) {
if (object.hasOwnProperty(key)) {
if (!stringIncludes(key, ".")) {
result[key] = object[key]
} else {
var split = key.split(".");
var prev = result;
for (var i = 0; i < split.length; i++) {
var cur = split[i];
if (!(cur in prev)) {
prev[cur] = {}
}
if (i == split.length - 1) {
prev[cur] = object[key];
} else {
prev = prev[cur]
}
}
}
}
}
return result;
}
/**
* There's a lot going on here. Essentially, this is a function factory that allows one to
* define backend calls just through the path. It also implements a simple caching
* strategy.
* Essentially the get function only needs to be called once, and from then on it will spit
* back a cached promise. This lets you write the code and not care whether it's cached or
* not, but also get the caching performance anyways. For this function to work, the
* resolver function has to take in the http response and the cache key to set, and make
* sure that it caches what it returns (see parseJsonResult or parseConfig).
* @param endpoint
* @param cacheKey
* @param resolverDefault
* @returns {Function}
*/
function getJsonMaker(endpoint, cacheKey, resolverDefault) {
var resolver = (typeof resolverDefault !== 'undefined')?
function (response) { return resolverDefault(response, cacheKey) }:
function (response) { return parseJsonResult(response, cacheKey) };
return function() {
var cachedValue = cache[cacheKey];
if (cachedValue === undefined) {
var url = urlGetter(endpoint, undefined, toDashboard.url);
return h.get(url)
.then(resolver, onFail)
} else {
return q(function(resolver) { resolver(cachedValue)});
}
}
}
function getProblemsMaker() {
// Caches the last offset and page size to hold onto it between different views
var prevOffset = 0;
var prevN = 20;
/**
* ProblemEndpoint: 'admin/status/problems',
* @returns {*}
*/
return function(offset, n, epoch) {
if (offset != null) {
prevOffset = offset;
} else {
offset = prevOffset;
}
if (n != null) {
prevN = n;
} else {
n = prevN;
}
var epochString = epoch && isFinite(epoch) ? '&epoch=' + epoch : '';
var url = urlGetter(
Config.ProblemEndpoint + '?offset=' + offset + '&n=' + n + epochString,
undefined,
toDashboard.url
);
return h.get(url)
.then(parseJsonResult, onFail);
}
}
}
})();
diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js
index 7c97bb946..ccdbe9d71 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.controller.js
@@ -1,92 +1,92 @@
/**
* Controller for the Remote Dashboards panel.
* Parses the keystore get call, and has to handle
* some tricky caching logic with the model backend
* (namely we don't want to reset the dashboard links themselves
* between remoteDashboard visits)
*/
(function () {
'use strict';
// -- register controller with angular -- //
angular.module('shrine-tools')
.controller('DashboardController', DashboardController);
/**
*
* @type {string[]}
*/
DashboardController.$inject = ['$app', '$log', '$location'];
function DashboardController ($app, $log, $location) {
var vm = this;
var map = $app.model.map;
vm.keyStoreError = false;
init();
/**
*
*/
function init () {
$app.model.getKeystore()
.then(setDashboard, handleFailure);
}
function handleFailure (failure) {
vm.keyStoreError = failure;
}
/**
*
* @param keystore
*/
function setDashboard (keystore) {
var modelStatuses = $app.model.m.remoteSiteStatuses;
var tempList = [];
for (var i = 0; i < modelStatuses.length; i++) {
var abbreviatedEntry = modelStatuses[i];
if (abbreviatedEntry.url != "") // ignore self
tempList.push(abbreviatedEntry)
}
- vm.otherDashboards = [['Hub', '']].concat(map(entryToPair, tempList));
+ vm.otherDashboards = [['Hub', '', true]].concat(map(entryToPair, tempList));
vm.otherDashboards.sort(comparator);
vm.clearCache = clearCache;
vm.switchDashboard = switchDashboard;
}
/**
* Lexicographic sort where Hub is always first. I'm sure there's a more
* golf-way of writing this.
*/
function comparator(first, second) {
if (first[0] == 'Hub') {
return -2;
} else if (second[0] == 'Hub') {
return 2;
} else {
var less = first[0].toLowerCase() < second[0].toLowerCase();
var eq = first[0].toLowerCase() == second[0].toLowerCase();
return less? -1: eq? 0 : 1
}
}
//todo remove duplication with header.js
function switchDashboard(url, alias) {
$app.model.toDashboard.url = url;
$app.model.m.siteAlias = alias == 'Hub'? '': alias;
clearCache();
$location.url("/diagnostic/summary");
}
function clearCache() {
$app.model.clearCache();
}
function entryToPair(entry){
- return [entry.siteAlias, entry.url];
+ return [entry.siteAlias, entry.url, entry.theyHaveMine && entry.haveTheirs && !entry.timeOutError];
}
}
})();
diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html
index a30ca468b..5bd8c8af0 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/dashboard.tpl.html
@@ -1,17 +1,19 @@
+ flex-flow: column wrap;
+ float: left">
-
-
\ No newline at end of file
diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js
index 6faf3d130..6e86e4d77 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.controller.js
@@ -1,71 +1,81 @@
(function () {
'use strict';
// -- register controller with shrine-tools module
angular.module('shrine-tools')
.controller('SummaryController', SummaryController);
/**
* Summary Controller.
*
*/
- SummaryController.$inject = ['$app', '$sce', '$log']
- function SummaryController ($app, $sce, $log) {
+ SummaryController.$inject = ['$app', '$sce', '$log', '$timeout'];
+ function SummaryController ($app, $sce, $log, $timeout) {
var vm = this;
var unknown = 'UNKNOWN';
vm.summaryError = false;
vm.i2b2Error = false;
+ vm.loading = true;
$app.model.reloadSummary = init;
init();
/**
*
*/
function init() {
+ vm.loading = true;
+ var fifteenSeconds = 15*1000;
$app.model.getSummary()
.then(setSummary, handleSummaryFailure);
$app.model.getI2B2()
.then(setI2B2, handleI2B2Failure);
+
+ $timeout(setTimeoutError, fifteenSeconds);
+ }
+
+ function setTimeoutError() {
+ vm.loading = false;
}
function handleSummaryFailure(failure) {
vm.summaryError = failure;
}
function handleI2B2Failure(failure) {
vm.i2b2Error = failure;
}
function formatDate(maybeEpoch) {
if (!(maybeEpoch && isFinite(maybeEpoch))) {
return unknown;
} else {
var d = new Date(maybeEpoch);
return $app.model.formatDate(d);
}
}
/**
*
* @param summary
*/
function setSummary(summary) {
- vm.summary = summary;
+ vm.loading = false;
+ vm.summary = summary;
if (vm.summary.adapterMappingsFileName === undefined) {
vm.summary.adapterMappingsFileName = unknown;
} else if (vm.summary.adapterMappingsDate === undefined) {
vm.summary.adapterMappingsDate = unknown;
} else {
vm.summary.adapterMappingsDate = formatDate(vm.summary.adapterMappingsDate);
}
return this;
}
function setI2B2(i2b2) {
vm.ontProject = i2b2.ontProject;
}
}
})();
diff --git a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html
index 1985b7071..fe1eb4bb6 100644
--- a/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html
+++ b/apps/dashboard-app/src/main/js/src/app/diagnostic/views/summary.tpl.html
@@ -1,121 +1,134 @@
Version Info
This site is running SHRINE {{vm.summary.shrineVersion}} built on
This site is currently using ontology version
Based on concept term:
This site is currently using for mappings, last edited on
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html
index 61cb655b4..661463d3b 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html
+++ b/apps/steward-app/src/main/js/app/client/statistics/query-digest/term.tpl.html
@@ -1,15 +1,15 @@
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html
index c6b06507c..e5ba47d4d 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/bar.tpl.html
@@ -1,8 +1,8 @@
{{username}}
-
+
{{value}} queries
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js
index 5dd69cb42..2179f2d55 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.js
@@ -1,50 +1,56 @@
(function () {
angular.module('shrine.steward.statistics')
.directive('statisticsGraph', StatisticsGraphDirective);
function StatisticsGraphDirective() {
var templateUrl = './app/client/statistics/statistics-graph/' +
'statistics-graph.tpl.html';
var statisticsGraph = {
restrict: 'E',
templateUrl: templateUrl,
controller: StatisticsGraphController,
controllerAs: 'graph',
link: StatisticsGraphLink,
scope: {
graphData: '=',
graphClick: '&' /** todo pass in click handler. **/
}
};
return statisticsGraph;
}
StatisticsGraphController.$inject = ['$scope', 'StatisticsGraphService'];
function StatisticsGraphController($scope, svc) {
var graph = this;
//graphService = svc;
graph.graphData = $scope.graphData;
graph.toPercentage = toPercentage;
+ graph.formatUsername = formatUsername;
graph.graphClick = $scope.graphClick;
graph.clearUsers = svc.clearUsers;
+ graph.formatUsername = formatUsername;
function toPercentage(value) {
var maxQueryCount = svc.getMaxUserQueryCount(graph.graphData.users) || 1;
return svc.getCountAsPercentage(value, maxQueryCount);
}
+
+ function formatUsername(username) {
+ return svc.formatUsername(username);
+ }
}
StatisticsGraphLink.$inject = ['scope'];
function StatisticsGraphLink(scope) {
- scope.$watch('graphData', function(before, after) {
+ scope.$watch('graphData', function (before, after) {
var graph = scope.graph;
graph.graphData = scope.graphData;
graph.clearUsers();
});
var test = arguments;
}
})();
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js
index 716db85da..21545768d 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.js
@@ -1,54 +1,59 @@
var Service;
(function () {
'use strict';
angular.module('shrine.steward.statistics')
.service('StatisticsGraphService', StatisticsGraphService);
var sortedUsers = [];
function StatisticsGraphService() {
return {
getSortedUsers: getSortedUsers,
getMaxUser: getMaxUser,
getMaxUserQueryCount: getMaxUserQueryCount,
getCountAsPercentage: getCountAsPercentage,
+ formatUsername: formatUsername,
clearUsers: clearUsers
};
function getSortedUsers(users) {
if (!sortedUsers.length) {
sortedUsers = _.sortBy(users, [function (o) {
console.log(o);
return o._2;
}]).reverse();
}
return sortedUsers;
}
function getMaxUser(users) {
var sortedUsers = getSortedUsers(users);
return (!!sortedUsers.length) ? sortedUsers[0] : sortedUsers;
}
function getMaxUserQueryCount(users) {
var maxUser = getMaxUser(users);
return maxUser._2;
}
function getCountAsPercentage(userQueryCount, maxQueryCount) {
var basePct = 20;
return 100 * (userQueryCount / maxQueryCount) + basePct;
}
+ function formatUsername(username) {
+ return (username.length > 10) ? username.substring(0, 10) + '...' : username;
+ }
+
function clearUsers() {
sortedUsers = [];
}
}
Service = StatisticsGraphService;
})();
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js
index 935de3a7f..ff513f8d8 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.service.spec.js
@@ -1,38 +1,52 @@
(function () {
'use strict';
describe('Statistics Graph Service Tests', StatisticsGraphServiceSpec);
function StatisticsGraphServiceSpec() {
var statsGraphService;
function setup() {
module('shrine.steward.statistics');
inject(function (_StatisticsGraphService_) {
statsGraphService = _StatisticsGraphService_;
});
}
beforeEach(setup);
it('Statistics Graph Service should exist', function () {
expect(typeof (statsGraphService)).toBe('object');
});
it('getSortedUsers should exist', function () {
expect(typeof (statsGraphService.getSortedUsers)).toBe('function');
});
it('getMaxUser should exist', function () {
expect(typeof (statsGraphService.getMaxUser)).toBe('function');
});
it('getMaxUserQueryCount should exist', function () {
expect(typeof (statsGraphService.getMaxUserQueryCount)).toBe('function');
});
it('getCountAsPercentage should exist', function () {
expect(typeof (statsGraphService.getCountAsPercentage)).toBe('function');
});
+ it('formatUserame should exist', function () {
+ expect(typeof (statsGraphService.formatUsername)).toBe('function');
+ });
+
+ it('formatUsername should truncate benajamindanielcarmen to benjaminda...', function () {
+ var result = statsGraphService.formatUsername('benjamindanielcarmen');
+ expect(result).toBe('benjaminda...');
+ });
+
+ it('formatUsername should not truncate ben', function () {
+ var result = statsGraphService.formatUsername('ben');
+ expect(result).toBe('ben');
+ });
+
}
})();
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html
index 44a5c0156..992a960a4 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics-graph/statistics-graph.tpl.html
@@ -1,11 +1,11 @@
There is no Data for this time period.
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js b/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js
index 03f15f503..ac8b90b9f 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics.controller.js
@@ -1,140 +1,145 @@
(function () {
'use strict';
angular
.module('shrine.steward.statistics')
.controller('StatisticsController', StatisticsController);
StatisticsController.$inject = ['StatisticsModel', 'StewardService', '$scope'];
function StatisticsController(model, service, $scope) {
var showOntClass = 'ont-overlay';
var hideOntClass = 'ont-hidden';
-
var stats = this;
- stats.ontClass = hideOntClass;
var startDate = new Date();
var endDate = new Date();
- startDate.setDate(endDate.getDate() - 7);
+ // -- public vars --//
+ stats.ontClass = hideOntClass;
+ startDate.setDate(endDate.getDate() - 7);
stats.getDateString = service.commonService.dateService.utcToMMDDYYYY;
stats.timestampToUtc = service.commonService.dateService.timestampToUtc;
stats.viewDigest = viewDigest;
- stats.startDate = startDate;
+ stats.startDate = startDate;
stats.endDate = endDate;
-
stats.isValid = true;
stats.startOpened = false;
stats.endOpened = false;
stats.topicsPerState = {};
stats.ontology = {};
-
stats.graphData = {
total: 0,
users: []
};
-
stats.format = 'MM/dd/yyyy';
+ // -- public methods --//
stats.openStart = openStart;
stats.openEnd = openEnd;
stats.validateRange = validateRange;
stats.addDateRange = addDateRange;
stats.parseStateTitle = parseStateTitle;
stats.parseStateCount = parseStateCount;
stats.getResults = getResults;
stats.viewDigest = viewDigest;
-
+ stats.broadcastReset = broadcastReset;
// -- start -- //
init();
// -- private -- //
function init() {
addDateRange();
}
function openStart() {
stats.startOpened = true;
}
function openEnd() {
stats.endOpened = true;
}
+ // -- @todo: code sloppy, refactor --//
function validateRange() {
var startUtc, endUtc;
- var secondsPerDay = 86400000;
if (stats.startDate === undefined || stats.endDate === undefined) {
stats.isValid = false;
return;
}
- //can validate date range here.
+ //@todo: abstract to date methods. i.e. dateService.floor(date) and dateService.ceil(date)
+ stats.startDate.setHours(0, 0, 0, 0);
+ stats.endDate.setHours(23, 59, 59, 999);
+
+ //@todo: abstract to reusable Date validate range method i.e dateService.validate(startDate, endDate).
startUtc = stats.timestampToUtc(stats.startDate);
- endUtc = stats.timestampToUtc(stats.endDate) + secondsPerDay;
+ endUtc = stats.timestampToUtc(stats.endDate);
- if (endUtc - startUtc <= 0) {
+ if (endUtc - startUtc <= 0) {
stats.isValid = false;
} else {
stats.isValid = true;
}
return stats.isValid;
}
function addDateRange() {
if (stats.validateRange()) {
- var secondsPerDay = 86400000;
stats.getResults(stats.timestampToUtc(stats.startDate),
- stats.timestampToUtc(stats.endDate) + secondsPerDay);
+ stats.timestampToUtc(stats.endDate));
}
}
function parseStateTitle(state) {
var title = '';
if (state.Approved !== undefined) {
title = 'Approved';
}
else {
title = (state.Rejected !== undefined) ? 'Rejected' : 'Pending';
}
return title;
}
function viewDigest(data) {
- var secondsPerDay = 86400000;
var startUtc = stats.timestampToUtc(stats.startDate);
- var endUtc = stats.timestampToUtc(stats.endDate) + secondsPerDay;
+ var endUtc = stats.timestampToUtc(stats.endDate);
+
model.getUserQueryHistory(data.userName.toLowerCase(), startUtc, endUtc)
- .then(function (result) {
- stats.ontology = result.queryRecords;
- stats.ontClass = showOntClass;
- });
+ .then(function (result) {
+ stats.ontology = result.queryRecords;
+ stats.ontClass = showOntClass;
+ });
}
function parseStateCount(state) {
var member = stats.parseStateTitle(state);
return state[member];
}
function getResults(startUtc, endUtc) {
model.getQueriesPerUser(startUtc, endUtc)
.then(function (result) {
stats.graphData = result;
});
model.getTopicsPerState(startUtc, endUtc)
.then(function (result) {
stats.topicsPerState = result;
});
}
+
+ function broadcastReset() {
+ $scope.$broadcast('reset-digest');
+ }
}
-})();
\ No newline at end of file
+})();
diff --git a/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html b/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html
index 4d1bc92bf..403c1fb0c 100644
--- a/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html
+++ b/apps/steward-app/src/main/js/app/client/statistics/statistics.tpl.html
@@ -1,88 +1,83 @@
Query Counts By User
- x
+ x
-
-
-
-
-
Query Topics By Status
Status
Query Topic Count
{{stats.parseStateTitle(state)}}
{{stats.parseStateCount(state)}}
Total:
{{stats.topicsPerState.total}}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss b/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss
index 7f7421e68..ec213bc71 100644
--- a/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss
+++ b/apps/steward-app/src/main/js/app/sass/topic-dropdown.scss
@@ -1,127 +1,130 @@
.topic-dropdown {
div.title{
text-transform: uppercase;
font-weight: 500;
font-size: 1.3em;
color: #63BCF8;
}
z-index: 20000;
margin: 4rem;
left: 0%;
color: rgba(0, 0, 0, 0.87);
font-family: "Roboto", sans-serif;
font-size: 14px;
font-weight: 300;
line-height: 1.5em;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
position: fixed;
.tdd-btn {
&:hover{
cursor: pointer;
}
text-align: left;
max-width: 1000px;
outline: 0; /* takes care of blue outline */
display: -webkit-inline-box;
display: -webkit-inline-flex;
display: -ms-inline-flexbox;
display: inline-flex;
-webkit-box-align: center;
-webkit-align-items: center;
-ms-flex-align: center;
align-items: center;
-webkit-box-pack: justify;
-webkit-justify-content: space-between;
-ms-flex-pack: justify;
justify-content: space-between;
background: rgba(0,0,0, .75);
border: 1px solid #63BCF8 !important;
min-width: 400px;
border: 0;
border-radius: 4px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
box-sizing: border-box;
padding: 16px 20px;
color: #FFFFFF;
font-size: 12px;
font-weight: 300;
letter-spacing: 1.2px;
text-transform: uppercase;
overflow: hidden;
cursor: pointer;
+ };
.tdd-list {
position: absolute;
top: 100%;
left: 0px;
background: rgba(0,0,0, .75);
border: 1px solid #63BCF8 !important;
width: 100%;
border-radius: 4px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
text-align: left;
opacity: 0;
visibility: hidden;
-webkit-transition: 0.3s ease;
transition: 0.3s ease;
+ max-height: 75vh;
+ overflow: auto;
a {
display: block;
border-bottom: 1px solid rgba(0, 0, 0, 0.05);
padding: 16px 0;
color: inherit;
text-decoration: none;
background: none !important;
};
&:before {
content: '';
position: absolute;
top: -6px;
left: 20px;
width: 0;
height: 0;
box-shadow: 2px -2px 6px rgba(0, 0, 0, 0.05);
border-top: 6px solid #63BCF8;;
border-right: 6px solid #63BCF8;;
border-bottom: 6px solid transparent;
border-left: 6px solid transparent;
-webkit-transform: rotate(-45deg);
transform: rotate(-45deg);
mix-blend-mode: multiple;
};
li {
z-index: 100;
position: relative;
padding: 0 20px;
+ color: white;
&.active {
color: #5380F7;
}
&:first-child {
border-radius: 4px 4px 0 0;
}
&:last-child {
border-radius: 0 0 4px 4px;
}
&:last-child a {
border-bottom: 0;
}
};
};
- &:focus .tdd-list, &:active .tdd-list {
+ .tdd-list-open {
-webkit-transform: translate(0, 20px);
transform: translate(0, 20px);
- opacity: 1;
- visibility: visible;
+ opacity: 1 !important;
+ visibility: visible !important;
};
- };
}
diff --git a/apps/steward-app/src/main/js/bower.json b/apps/steward-app/src/main/js/bower.json
index eda35eff5..5e1051acd 100644
--- a/apps/steward-app/src/main/js/bower.json
+++ b/apps/steward-app/src/main/js/bower.json
@@ -1,62 +1,65 @@
{
"name": "shrine-data-steward-2.0",
"description": "HMS Data Steward",
"main": "start.js",
"authors": [
"Ben Carmen"
],
"license": "MIT",
"keywords": [
"hms",
"steward",
"app"
],
"moduleType": [],
"homepage": "https://open.med.harvard.edu/project/shrine/",
"ignore": [
"**/.*",
"node_modules",
"bower_components",
"src/vendor",
"test",
"tests"
],
"dependencies": {
"bootstrap": "v3.3.7",
"metisMenu": "v2.5.2",
"angular-loading-bar": "v0.9.0",
"font-awesome": "v4.6.3",
"jquery": "v2.2.4",
"angular": "v1.5.7",
"json3": "v3.3.2",
"angular-ui-router": "v0.3.1",
"angular-route": "v1.5.7",
"angular-cookies": "v1.5.7",
"oclazyload": "ocLazyLoad#v1.0.9",
"lodash": "v4.13.1",
"angular-bootstrap": "v1.3.3",
"Chart.js": "v2.1.6",
"x2js": "x2js#v1.2.0",
"angular-animate": "v1.5.8"
},
"devDependencies": {
"angular-mocks": "v1.5.7",
"font-awesome": "v4.6.3"
},
"overrides": {
"bootstrap": {
"main": [
"dist/js/bootstrap.min.js",
"dist/css/bootstrap.css",
"less/bootstrap.less"
]
},
"font-awesome": {
"main": [
"css/font-awesome.css",
"less/font-awesome.less",
"scss/font-awesome.scss"
]
}
+ },
+ "resolutions": {
+ "angular": "1.5.8"
}
}
diff --git a/apps/steward-app/src/main/js/bower_components/angular/.bower.json b/apps/steward-app/src/main/js/bower_components/angular/.bower.json
index 00d0f0d8b..8eb867dcf 100644
--- a/apps/steward-app/src/main/js/bower_components/angular/.bower.json
+++ b/apps/steward-app/src/main/js/bower_components/angular/.bower.json
@@ -1,18 +1,18 @@
{
"name": "angular",
"version": "1.5.8",
"license": "MIT",
"main": "./angular.js",
"ignore": [],
"dependencies": {},
"homepage": "https://github.com/angular/bower-angular",
"_release": "1.5.8",
"_resolution": {
"type": "version",
"tag": "v1.5.8",
"commit": "7e0e546eb6caedbb298c91a9f6bf7de7eeaa4ad2"
},
"_source": "https://github.com/angular/bower-angular.git",
- "_target": ">=1.4.0",
+ "_target": "1.5.8",
"_originalSource": "angular"
}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
index a94f3c75c..727306998 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/db/StewardDatabase.scala
@@ -1,740 +1,766 @@
package net.shrine.steward.db
import java.sql.SQLException
+import java.util.concurrent.TimeoutException
import java.util.concurrent.atomic.AtomicInteger
import javax.sql.DataSource
import com.typesafe.config.Config
import net.shrine.authorization.steward.{Date, ExternalQueryId, InboundShrineQuery, InboundTopicRequest, OutboundShrineQuery, OutboundTopic, OutboundUser, QueriesPerUser, QueryContents, QueryHistory, ResearcherToAudit, ResearchersTopics, StewardQueryId, StewardsTopics, TopicId, TopicIdAndName, TopicState, TopicStateName, TopicsPerState, UserName, researcherRole, stewardRole}
import net.shrine.i2b2.protocol.pm.User
import net.shrine.log.Loggable
-import net.shrine.slick.{NeedsWarmUp, TestableDataSourceCreator}
+import net.shrine.problem.{AbstractProblem, ProblemSources}
+import net.shrine.slick.{CouldNotRunDbIoActionException, NeedsWarmUp, TestableDataSourceCreator}
import net.shrine.source.ConfigSource
import net.shrine.steward.CreateTopicsMode
import slick.dbio.Effect.Read
import slick.driver.JdbcProfile
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.{Duration, DurationInt}
import scala.concurrent.{Await, Future, blocking}
import scala.language.postfixOps
import scala.util.Try
+import scala.util.control.NonFatal
/**
* Database access code for the data steward service.
*
* I'm not letting Slick handle foreign key resolution for now. I want to keep that logic separate to handle dirty data with some grace.
*
* @author dwalend
* @since 1.19
*/
case class StewardDatabase(schemaDef:StewardSchema,dataSource: DataSource) extends Loggable {
import schemaDef._
import jdbcProfile.api._
val database = Database.forDataSource(dataSource)
def createTables() = schemaDef.createTables(database)
def dropTables() = schemaDef.dropTables(database)
+ //todo share code from DashboardProblemDatabase.scala . It's a lot richer. See SHRINE-1835
def dbRun[R](action: DBIOAction[R, NoStream, Nothing]):R = {
- val future: Future[R] = database.run(action)
- blocking {
- Await.result(future, 10 seconds)
+ try {
+ val future: Future[R] = database.run(action)
+ blocking {
+ Await.result(future, 10 seconds)
+ }
+ } catch {
+ case tax:TopicAcessException => throw tax
+ case tx:TimeoutException =>
+ val x = CouldNotRunDbIoActionException(dataSource, tx)
+ StewardDatabaseProblem(x)
+ throw x
+ case NonFatal(nfx) =>
+ val x = CouldNotRunDbIoActionException(dataSource, nfx)
+ StewardDatabaseProblem(x)
+ throw x
}
}
def warmUp = {
dbRun(allUserQuery.size.result)
}
def selectUsers:Seq[UserRecord] = {
dbRun(allUserQuery.result)
}
// todo use whenever a shrine query is logged
def upsertUser(user:User):Unit = {
val userRecord = UserRecord(user)
dbRun(allUserQuery.insertOrUpdate(userRecord))
}
def createRequestForTopicAccess(user:User,topicRequest:InboundTopicRequest):TopicRecord = {
val createInState = CreateTopicsMode.createTopicsInState
val now = System.currentTimeMillis()
val topicRecord = TopicRecord(Some(nextTopicId.getAndIncrement),topicRequest.name,topicRequest.description,user.username,now,createInState.topicState)
val userTopicRecord = UserTopicRecord(user.username,topicRecord.id.get,TopicState.approved,user.username,now)
dbRun(for{
_ <- allTopicQuery += topicRecord
_ <- allUserTopicQuery += userTopicRecord
} yield topicRecord)
}
def updateRequestForTopicAccess(user:User,topicId:TopicId,topicRequest:InboundTopicRequest):Try[OutboundTopic] = Try {
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap{ option =>
val oldTopicRecord = option.getOrElse(throw TopicDoesNotExist(topicId = topicId))
if(user.username != oldTopicRecord.createdBy) throw DetectedAttemptByWrongUserToChangeTopic(topicId,user.username,oldTopicRecord.createdBy)
if(oldTopicRecord.state == TopicState.approved) throw ApprovedTopicCanNotBeChanged(topicId)
val updatedTopic = oldTopicRecord.copy(name = topicRequest.name,
description = topicRequest.description,
changedBy = user.username,
changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).flatMap{_ =>
outboundUsersForNamesAction(Set(updatedTopic.createdBy,updatedTopic.changedBy)).map(updatedTopic.toOutboundTopic)
}
}
)
}
def selectTopicsForResearcher(parameters:QueryParameters):ResearchersTopics = {
require(parameters.researcherIdOption.isDefined,"A researcher's parameters must supply a user id")
val (count,topics,userNamesToOutboundUsers) = dbRun(
for{
count <- topicCountQuery(parameters).length.result
topics <- topicSelectQuery(parameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count, topics,userNamesToOutboundUsers))
ResearchersTopics(parameters.researcherIdOption.get,
count,
parameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
//treat as private (currently used in test)
def selectTopics(queryParameters: QueryParameters):Seq[TopicRecord] = {
dbRun(topicSelectQuery(queryParameters).result)
}
def selectTopicsForSteward(queryParameters: QueryParameters):StewardsTopics = {
val (count,topics,userNamesToOutboundUsers) = dbRun{
for{
count <- topicCountQuery(queryParameters).length.result
topics <- topicSelectQuery(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction((topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count,topics,userNamesToOutboundUsers)
}
StewardsTopics(count,
queryParameters.skipOption.getOrElse(0),
topics.map(_.toOutboundTopic(userNamesToOutboundUsers)))
}
private def topicSelectQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val countFilter = topicCountQuery(queryParameters)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(countFilter.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countFilter)(
columnName => countFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "id" => x.id
case "name" => x.name
case "description" => x.description
case "createdBy" => x.createdBy
case "createDate" => x.createDate
case "state" => x.state
case "changedBy" => x.changedBy
case "changeDate" => x.changeDate
})))
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def topicCountQuery(queryParameters: QueryParameters):Query[TopicTable, TopicTable#TableElementType, Seq] = {
val allTopics:Query[TopicTable, TopicTable#TableElementType, Seq] = mostRecentTopicQuery
val researcherFilter = queryParameters.researcherIdOption.fold(allTopics)(userId => allTopics.filter(_.createdBy === userId))
val stateFilter = queryParameters.stateOption.fold(researcherFilter)(state => researcherFilter.filter(_.state === state.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.changeDate >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.changeDate <= maxDate))
maxDateFilter
}
def changeTopicState(topicId:TopicId,state:TopicState,userId:UserName):Option[TopicRecord] = {
val noTopicRecord:Option[TopicRecord] = None
val noOpDBIO:DBIOAction[Option[TopicRecord], NoStream, Effect.Write] = DBIO.successful(noTopicRecord)
dbRun(mostRecentTopicQuery.filter(_.id === topicId).result.headOption.flatMap(
_.fold(noOpDBIO){ originalTopic =>
val updatedTopic = originalTopic.copy(state = state, changedBy = userId, changeDate = System.currentTimeMillis())
(allTopicQuery += updatedTopic).map(_ => Option(updatedTopic))
}
))
}
def selectTopicCountsPerState(queryParameters: QueryParameters):TopicsPerState = {
dbRun(for{
totalTopics <- topicCountQuery(queryParameters).length.result
topicsPerStateName <- topicCountsPerState(queryParameters).result
} yield TopicsPerState(totalTopics,topicsPerStateName))
}
private def topicCountsPerState(queryParameters: QueryParameters): Query[(Rep[TopicStateName], Rep[Int]), (TopicStateName, Int), Seq] = {
val groupedByState = topicCountQuery(queryParameters).groupBy(topicRecord => topicRecord.state)
groupedByState.map{case (state,result) => (state,result.length)}
}
def logAndCheckQuery(userId:UserName,topicId:Option[TopicId],shrineQuery:InboundShrineQuery):(TopicState,Option[TopicIdAndName]) = {
//todo upsertUser(user) when the info is available from the PM
val noOpDBIOForState: DBIOAction[TopicState, NoStream, Effect.Read] = DBIO.successful {
if (CreateTopicsMode.createTopicsInState == CreateTopicsMode.TopicsIgnoredJustLog) TopicState.approved
else TopicState.createTopicsModeRequiresTopic
}
val noOpDBIOForTopicName: DBIOAction[Option[String], NoStream, Read] = DBIO.successful{None}
val (state,topicName) = dbRun(for{
state <- topicId.fold(noOpDBIOForState)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.state).result.headOption.map(
_.fold(TopicState.unknownForUser)(state => TopicState.namesToStates(state)))
)
topicName <- topicId.fold(noOpDBIOForTopicName)( someTopicId =>
mostRecentTopicQuery.filter(_.id === someTopicId).filter(_.createdBy === userId).map(_.name).result.headOption
)
_ <- allQueryTable += ShrineQueryRecord(userId,topicId,shrineQuery,state)
} yield (state,topicName))
val topicIdAndName:Option[TopicIdAndName] = (topicId,topicName) match {
case (Some(id),Some(name)) => Option(TopicIdAndName(id.toString,name))
case (None,None) => None
case (Some(id),None) =>
if(state == TopicState.unknownForUser) None
else throw new IllegalStateException(s"How did you get here for $userId with $id and $state for $shrineQuery")
case (None,Some(name)) =>
if(state == TopicState.unknownForUser) None
else throw new IllegalStateException(s"How did you get here for $userId with no topic id but a topic name of $name and $state for $shrineQuery")
}
(state,topicIdAndName)
}
def selectQueryHistory(queryParameters: QueryParameters, topicParameter:Option[TopicId]):
QueryHistory = {
val topicQuery = for {
count <- shrineQueryCountQuery(queryParameters, topicParameter).length.result
shrineQueries <- shrineQuerySelectQuery(queryParameters, topicParameter).result
topics <- mostRecentTopicQuery.filter(_.id.inSet(shrineQueries.map(_.topicId).to[Set].flatten)).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(shrineQueries.map(_.userId).to[Set] ++ (topics.map(_.createdBy) ++ topics.map(_.changedBy)).to[Set])
} yield (count, shrineQueries, topics, userNamesToOutboundUsers)
val (count, shrineQueries, topics, userNamesToOutboundUsers) = dbRun(topicQuery)
val topicIdsToTopics: Map[Option[TopicId], TopicRecord] = topics.map(x => (x.id, x)).toMap
def toOutboundShrineQuery(queryRecord: ShrineQueryRecord): OutboundShrineQuery = {
val topic = topicIdsToTopics.get(queryRecord.topicId)
val outboundTopic: Option[OutboundTopic] = topic.map(_.toOutboundTopic(userNamesToOutboundUsers))
val outboundUserOption = userNamesToOutboundUsers.get(queryRecord.userId)
//todo if a user is unknown and the system is in a mode that requires everyone to log into the data steward notify the data steward
val outboundUser: OutboundUser = outboundUserOption.getOrElse(OutboundUser.createUnknownUser(queryRecord.userId))
queryRecord.createOutboundShrineQuery(outboundTopic, outboundUser)
}
QueryHistory(count, queryParameters.skipOption.getOrElse(0), shrineQueries.map(toOutboundShrineQuery))
}
private def outboundUsersForNamesAction(userNames:Set[UserName]):DBIOAction[Map[UserName, OutboundUser], NoStream, Read] = {
allUserQuery.filter(_.userName.inSet(userNames)).result.map(_.map(x => (x.userName,x.asOutboundUser)).toMap)
}
private def shrineQuerySelectQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val countQuery = shrineQueryCountQuery(queryParameters,topicParameter)
//todo is there some way to do something with a map from column names to columns that I don't have to update? I couldn't find one.
// val orderByQuery = queryParameters.sortByOption.fold(limitFilter)(
// columnName => limitFilter.sortBy(x => queryParameters.sortOrder.orderForColumn(allQueryTable.columnForName(columnName))))
val orderByQuery = queryParameters.sortByOption.fold(countQuery) {
case "topicName" =>
val joined = countQuery.join(mostRecentTopicQuery).on(_.topicId === _.id)
joined.sortBy(x => queryParameters.sortOrder.orderForColumn(x._2.name)).map(x => x._1)
case columnName => countQuery.sortBy(x => queryParameters.sortOrder.orderForColumn(columnName match {
case "stewardId" => x.stewardId
case "externalId" => x.externalId
case "researcherId" => x.researcherId
case "name" => x.name
case "topic" => x.topicId
case "queryContents" => x.queryContents
case "stewardResponse" => x.stewardResponse
case "date" => x.date
}))
}
val skipFilter = queryParameters.skipOption.fold(orderByQuery)(skip => orderByQuery.drop(skip))
val limitFilter = queryParameters.limitOption.fold(skipFilter)(limit => skipFilter.take(limit))
limitFilter
}
private def shrineQueryCountQuery(queryParameters: QueryParameters,topicParameter:Option[TopicId]):Query[QueryTable, QueryTable#TableElementType, Seq] = {
val allShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = allQueryTable
val topicFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = topicParameter.fold(allShrineQueries)(topicId => allShrineQueries.filter(_.topicId === topicId))
val researcherFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.researcherIdOption.fold(topicFilter)(researcherId => topicFilter.filter(_.researcherId === researcherId))
//todo this is probably a binary Approved/Not approved
val stateFilter:Query[QueryTable, QueryTable#TableElementType, Seq] = queryParameters.stateOption.fold(researcherFilter)(stewardResponse => researcherFilter.filter(_.stewardResponse === stewardResponse.name))
val minDateFilter = queryParameters.minDate.fold(stateFilter)(minDate => stateFilter.filter(_.date >= minDate))
val maxDateFilter = queryParameters.maxDate.fold(minDateFilter)(maxDate => minDateFilter.filter(_.date <= maxDate))
maxDateFilter
}
def selectShrineQueryCountsPerUser(queryParameters: QueryParameters):QueriesPerUser = {
val (totalQueries,queriesPerUser,userNamesToOutboundUsers) = dbRun(for {
totalQueries <- shrineQueryCountQuery(queryParameters,None).length.result
queriesPerUser <- shrineQueryCountsPerResearcher(queryParameters).result
userNamesToOutboundUsers <- outboundUsersForNamesAction(queriesPerUser.map(x => x._1).to[Set])
} yield (totalQueries,queriesPerUser,userNamesToOutboundUsers))
val queriesPerOutboundUser:Seq[(OutboundUser,Int)] = queriesPerUser.map(x => (userNamesToOutboundUsers(x._1),x._2))
QueriesPerUser(totalQueries,queriesPerOutboundUser)
}
private def shrineQueryCountsPerResearcher(queryParameters: QueryParameters): Query[(Rep[UserName],Rep[Int]),(UserName,Int),Seq] = {
val filteredShrineQueries:Query[QueryTable, QueryTable#TableElementType, Seq] = shrineQueryCountQuery(queryParameters,None)
val groupedByResearcher = filteredShrineQueries.groupBy(shrineQuery => shrineQuery.researcherId)
groupedByResearcher.map{case (researcher,result) => (researcher,result.length)}
}
lazy val nextTopicId:AtomicInteger = new AtomicInteger({
dbRun(allTopicQuery.map(_.id).max.result).getOrElse(0) + 1
})
def selectAllAuditRequests: Seq[UserAuditRecord] = {
dbRun(allUserAudits.result)
}
def selectMostRecentAuditRequests: Seq[UserAuditRecord] = {
dbRun(mostRecentUserAudits.result)
}
def selectResearchersToAudit(maxQueryCountBetweenAudits:Int,minTimeBetweenAudits:Duration,now:Date):Seq[ResearcherToAudit] = {
//todo one round with the db instead of O(researchers)
//for each researcher
//horizon = if the researcher has had an audit
// date of last audit
// else if no audit yet
// date of first query
val researchersToHorizons: Map[UserName, Date] = dbRun(for{
dateOfFirstQuery: Seq[(UserName, Date)] <- leastRecentUserQuery.map(record => record.researcherId -> record.date).result
mostRecentAudit: Seq[(UserName, Date)] <- mostRecentUserAudits.map(record => record.researcher -> record.changeDate).result
} yield {
dateOfFirstQuery.toMap ++ mostRecentAudit.toMap
})
val researchersToHorizonsAndCounts = researchersToHorizons.map{ researcherDate =>
val queryParameters = QueryParameters(researcherIdOption = Some(researcherDate._1),
minDate = Some(researcherDate._2))
val count:Int = dbRun(shrineQueryCountQuery(queryParameters,None).length.result)
(researcherDate._1,(researcherDate._2,count))
}
//audit if oldest query within the horizon is >= minTimeBetweenAudits in the past and the researcher has run at least one query since.
val oldestAllowed = System.currentTimeMillis() - minTimeBetweenAudits.toMillis
val timeBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 > 0 && x._2._1 <= oldestAllowed)
//audit if the researcher has run >= maxQueryCountBetweenAudits queries since horizon?
val queryBasedAudit = researchersToHorizonsAndCounts.filter(x => x._2._2 >= maxQueryCountBetweenAudits)
val toAudit = timeBasedAudit ++ queryBasedAudit
val namesToOutboundUsers: Map[UserName, OutboundUser] = dbRun(outboundUsersForNamesAction(toAudit.keySet))
toAudit.map(x => ResearcherToAudit(namesToOutboundUsers(x._1),x._2._2,x._2._1,now)).to[Seq]
}
def logAuditRequests(auditRequests:Seq[ResearcherToAudit],now:Date) {
dbRun{
allUserAudits ++= auditRequests.map(x => UserAuditRecord(researcher = x.researcher.userName,
queryCount = x.count,
changeDate = now
))
}
}
}
/**
* Separate class to support schema generation without actually connecting to the database.
*
* @param jdbcProfile Database profile to use for the schema
*/
case class StewardSchema(jdbcProfile: JdbcProfile) extends Loggable {
import jdbcProfile.api._
def ddlForAllTables = {
allUserQuery.schema ++ allTopicQuery.schema ++ allQueryTable.schema ++ allUserTopicQuery.schema ++ allUserAudits.schema
}
//to get the schema, use the REPL
//println(StewardSchema.schema.ddlForAllTables.createStatements.mkString(";\n"))
def createTables(database:Database) = {
try {
val future = database.run(ddlForAllTables.create)
Await.result(future,10 seconds)
} catch {
//I'd prefer to check and create schema only if absent. No way to do that with Oracle.
case x:SQLException => info("Caught exception while creating tables. Recover by assuming the tables already exist.",x)
}
}
def dropTables(database:Database) = {
val future = database.run(ddlForAllTables.drop)
//Really wait forever for the cleanup
Await.result(future,Duration.Inf)
}
class UserTable(tag:Tag) extends Table[UserRecord](tag,"users") {
def userName = column[UserName]("userName",O.PrimaryKey)
def fullName = column[String]("fullName")
def isSteward = column[Boolean]("isSteward")
def * = (userName,fullName,isSteward) <> (UserRecord.tupled,UserRecord.unapply)
}
class TopicTable(tag:Tag) extends Table[TopicRecord](tag,"topics") {
def id = column[TopicId]("id")
def name = column[String]("name")
def description = column[String]("description")
def createdBy = column[UserName]("createdBy")
def createDate = column[Date]("createDate")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def idIndex = index("idIndex",id,unique = false)
def topicNameIndex = index("topicNameIndex",name,unique = false)
def createdByIndex = index("createdByIndex",createdBy,unique = false)
def createDateIndex = index("createDateIndex",createDate,unique = false)
def stateIndex = index("stateIndex",state,unique = false)
def changedByIndex = index("changedByIndex",changedBy,unique = false)
def changeDateIndex = index("changeDateIndex",changeDate,unique = false)
def * = (id.?, name, description, createdBy, createDate, state, changedBy, changeDate) <> (fromRow, toRow) //(TopicRecord.tupled,TopicRecord.unapply)
def fromRow = (fromParams _).tupled
def fromParams(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
stateName:String,
changedBy:UserName,
changeDate:Date): TopicRecord = {
TopicRecord(id, name, description, createdBy, createDate, TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(topicRecord: TopicRecord) =
Some((topicRecord.id,
topicRecord.name,
topicRecord.description,
topicRecord.createdBy,
topicRecord.createDate,
topicRecord.state.name,
topicRecord.changedBy,
topicRecord.changeDate
))
}
class UserTopicTable(tag:Tag) extends Table[UserTopicRecord](tag,"userTopic") {
def researcher = column[UserName]("researcher")
def topicId = column[TopicId]("topicId")
def state = column[TopicStateName]("state")
def changedBy = column[UserName]("changedBy")
def changeDate = column[Date]("changeDate")
def researcherTopicIdIndex = index("researcherTopicIdIndex",(researcher,topicId),unique = true)
def * = (researcher, topicId, state, changedBy, changeDate) <> (fromRow, toRow)
def fromRow = (fromParams _).tupled
def fromParams(researcher:UserName,
topicId:TopicId,
stateName:String,
changedBy:UserName,
changeDate:Date): UserTopicRecord = {
UserTopicRecord(researcher,topicId,TopicState.namesToStates(stateName), changedBy, changeDate)
}
def toRow(userTopicRecord: UserTopicRecord):Option[(UserName,TopicId,String,UserName,Date)] =
Some((userTopicRecord.researcher,
userTopicRecord.topicId,
userTopicRecord.state.name,
userTopicRecord.changedBy,
userTopicRecord.changeDate
))
}
class UserAuditTable(tag:Tag) extends Table[UserAuditRecord](tag,"userAudit") {
def researcher = column[UserName]("researcher")
def queryCount = column[Int]("queryCount")
def changeDate = column[Date]("changeDate")
def * = (researcher, queryCount, changeDate) <> (fromRow, toRow)
def fromRow = (fromParams _).tupled
def fromParams(researcher:UserName,
queryCount:Int,
changeDate:Date): UserAuditRecord = {
UserAuditRecord(researcher,queryCount, changeDate)
}
def toRow(record: UserAuditRecord):Option[(UserName,Int,Date)] =
Some((record.researcher,
record.queryCount,
record.changeDate
))
}
class QueryTable(tag:Tag) extends Table[ShrineQueryRecord](tag,"queries") {
def stewardId = column[StewardQueryId]("stewardId",O.PrimaryKey,O.AutoInc)
def externalId = column[ExternalQueryId]("id")
def name = column[String]("name")
def researcherId = column[UserName]("researcher")
def topicId = column[Option[TopicId]]("topic")
def queryContents = column[QueryContents]("queryContents")
def stewardResponse = column[String]("stewardResponse")
def date = column[Date]("date")
def externalIdIndex = index("externalIdIndex",externalId,unique = false)
def queryNameIndex = index("queryNameIndex",name,unique = false)
def researcherIdIndex = index("researcherIdIndex",stewardId,unique = false)
def topicIdIndex = index("topicIdIndex",topicId,unique = false)
def stewardResponseIndex = index("stewardResponseIndex",stewardResponse,unique = false)
def dateIndex = index("dateIndex",date,unique = false)
def * = (stewardId.?,externalId,name,researcherId,topicId,queryContents,stewardResponse,date) <> (fromRow,toRow)
def fromRow = (fromParams _).tupled
def fromParams(stewardId:Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:String,
date:Date): ShrineQueryRecord = {
ShrineQueryRecord(stewardId,externalId, name, userId, topicId, queryContents,TopicState.namesToStates(stewardResponse),date)
}
def toRow(queryRecord: ShrineQueryRecord):Option[(
Option[StewardQueryId],
ExternalQueryId,
String,
UserName,
Option[TopicId],
QueryContents,
String,
Date
)] =
Some((queryRecord.stewardId,
queryRecord.externalId,
queryRecord.name,
queryRecord.userId,
queryRecord.topicId,
queryRecord.queryContents,
queryRecord.stewardResponse.name,
queryRecord.date)
)
}
val allUserQuery = TableQuery[UserTable]
val allTopicQuery = TableQuery[TopicTable]
val allQueryTable = TableQuery[QueryTable]
val allUserTopicQuery = TableQuery[UserTopicTable]
val allUserAudits = TableQuery[UserAuditTable]
val mostRecentTopicQuery: Query[TopicTable, TopicRecord, Seq] = for(
topic <- allTopicQuery if !allTopicQuery.filter(_.id === topic.id).filter(_.changeDate > topic.changeDate).exists
) yield topic
val mostRecentUserAudits: Query[UserAuditTable, UserAuditRecord, Seq] = for(
record <- allUserAudits if !allUserAudits.filter(_.researcher === record.researcher).filter(_.changeDate > record.changeDate).exists
) yield record
val leastRecentUserQuery: Query[QueryTable, ShrineQueryRecord, Seq] = for(
record <- allQueryTable if !allQueryTable.filter(_.researcherId === record.researcherId).filter(_.date < record.date).exists
) yield record
}
object StewardSchema {
val allConfig:Config = ConfigSource.config
val config:Config = allConfig.getConfig("shrine.steward.database")
val slickProfile:JdbcProfile = ConfigSource.getObject("slickProfileClassName", config)
val schema = StewardSchema(slickProfile)
}
object StewardDatabase extends NeedsWarmUp {
val dataSource:DataSource = TestableDataSourceCreator.dataSource(StewardSchema.config)
val db = StewardDatabase(StewardSchema.schema,dataSource)
val createTablesOnStart = StewardSchema.config.getBoolean("createTablesOnStart")
if(createTablesOnStart) StewardDatabase.db.createTables()
override def warmUp() = StewardDatabase.db.warmUp
}
//API help
sealed case class SortOrder(name:String){
import slick.lifted.ColumnOrdered
def orderForColumn[T](column:ColumnOrdered[T]):ColumnOrdered[T] = {
if(this == SortOrder.ascending) column.asc
else column.desc
}
}
object SortOrder {
val ascending = SortOrder("ascending")
val descending = SortOrder("descending")
val sortOrders = Seq(ascending,descending)
val namesToSortOrders = sortOrders.map(x => (x.name,x)).toMap
def sortOrderForStringOption(option:Option[String]) = option.fold(ascending)(namesToSortOrders(_))
}
case class QueryParameters(researcherIdOption:Option[UserName] = None,
stateOption:Option[TopicState] = None,
skipOption:Option[Int] = None,
limitOption:Option[Int] = None,
sortByOption:Option[String] = None,
sortOrder:SortOrder = SortOrder.ascending,
minDate:Option[Date] = None,
maxDate:Option[Date] = None
)
//DAO case classes, exposed for testing only
case class ShrineQueryRecord(stewardId: Option[StewardQueryId],
externalId:ExternalQueryId,
name:String,
userId:UserName,
topicId:Option[TopicId],
queryContents: QueryContents,
stewardResponse:TopicState,
date:Date) {
def createOutboundShrineQuery(outboundTopic:Option[OutboundTopic],outboundUser:OutboundUser): OutboundShrineQuery = {
OutboundShrineQuery(stewardId.get,externalId,name,outboundUser,outboundTopic,queryContents,stewardResponse.name,date)
}
}
object ShrineQueryRecord extends ((Option[StewardQueryId],ExternalQueryId,String,UserName,Option[TopicId],QueryContents,TopicState,Date) => ShrineQueryRecord) {
def apply(userId:UserName,topicId:Option[TopicId],shrineQuery: InboundShrineQuery,stewardResponse:TopicState): ShrineQueryRecord = {
ShrineQueryRecord(
None,
shrineQuery.externalId,
shrineQuery.name,
userId,
topicId,
shrineQuery.queryContents,
stewardResponse,
System.currentTimeMillis())
}
}
case class UserRecord(userName:UserName,fullName:String,isSteward:Boolean) {
lazy val asOutboundUser:OutboundUser = OutboundUser(userName,fullName,if(isSteward) Set(stewardRole,researcherRole)
else Set(researcherRole))
}
object UserRecord extends ((UserName,String,Boolean) => UserRecord) {
def apply(user:User):UserRecord = UserRecord(user.username,user.fullName,user.params.toList.contains((stewardRole,"true")))
}
case class TopicRecord(id:Option[TopicId] = None,
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState,
changedBy:UserName,
changeDate:Date) {
def toOutboundTopic(userNamesToOutboundUsers: Map[UserName, OutboundUser]): OutboundTopic = {
OutboundTopic(id.get,
name,
description,
userNamesToOutboundUsers(createdBy),
createDate,
state.name,
userNamesToOutboundUsers(changedBy),
changeDate)
}
}
object TopicRecord {
def apply(id:Option[TopicId],
name:String,
description:String,
createdBy:UserName,
createDate:Date,
state:TopicState
):TopicRecord = TopicRecord(id,
name,
description,
createdBy,
createDate,
state,
createdBy,
createDate)
}
case class UserTopicRecord(researcher:UserName,
topicId:TopicId,
state:TopicState,
changedBy:UserName,
changeDate:Date)
case class UserAuditRecord(researcher:UserName,
queryCount:Int,
changeDate:Date) {
def sameExceptForTimes(userAuditRecord: UserAuditRecord):Boolean = {
(researcher == userAuditRecord.researcher) &&
(queryCount == userAuditRecord.queryCount)
}
}
-case class TopicDoesNotExist(topicId:TopicId) extends IllegalArgumentException(s"No topic for id $topicId")
+abstract class TopicAcessException(topicId: TopicId,message:String) extends IllegalArgumentException(message)
+
+case class TopicDoesNotExist(topicId:TopicId) extends TopicAcessException(topicId,s"No topic for id $topicId")
+
+case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends TopicAcessException(topicId,s"Topic $topicId has been ${TopicState.approved}")
+
+case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends TopicAcessException(topicId,s"$userId does not own $topicId; $ownerId owns it.")
+
+case class StewardDatabaseProblem(cnrdiax:CouldNotRunDbIoActionException) extends AbstractProblem(ProblemSources.Dsa) {
+ override def summary: String = "The DSA's database failed due to an exception."
-case class ApprovedTopicCanNotBeChanged(topicId:TopicId) extends IllegalStateException(s"Topic $topicId has been ${TopicState.approved}")
+ override def description: String = s"TThe DSAs database failed due to $cnrdiax"
-case class DetectedAttemptByWrongUserToChangeTopic(topicId:TopicId,userId:UserName,ownerId:UserName) extends IllegalArgumentException(s"$userId does not own $topicId; $ownerId owns it.")
\ No newline at end of file
+ override def throwable = Some(cnrdiax)
+}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala b/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala
index 508e4db4c..144626ce0 100644
--- a/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala
+++ b/apps/steward-app/src/main/scala/net/shrine/steward/email/AuditEmailer.scala
@@ -1,144 +1,144 @@
package net.shrine.steward.email
import java.util.Date
import javax.mail.internet.InternetAddress
import akka.actor.Actor
import com.typesafe.config.Config
import courier.{Envelope, Mailer, Text}
import net.shrine.authorization.steward.ResearcherToAudit
import net.shrine.config.{ConfigExtensions, DurationConfigParser}
import net.shrine.email.ConfiguredMailer
import net.shrine.log.Log
import net.shrine.problem.{AbstractProblem, ProblemSources}
import net.shrine.source.ConfigSource
import net.shrine.steward.db.StewardDatabase
import scala.concurrent.Await
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.blocking
import scala.concurrent.duration.FiniteDuration
import scala.concurrent.duration._
import scala.util.control.NonFatal
import scala.xml.NodeSeq
/**
* @author david
* @since 1.22
*/
case class AuditEmailer(maxQueryCountBetweenAudits:Int,
minTimeBetweenAudits:FiniteDuration,
researcherLineTemplate:String,
emailTemplate:String,
emailSubject:String,
from:InternetAddress,
to:InternetAddress,
stewardBaseUrl:String, //todo not an option
mailer:Mailer
) {
def audit() = {
//gather a list of users to audit
- Log.info("Auditing users")
val now = System.currentTimeMillis()
val researchersToAudit: Seq[ResearcherToAudit] = StewardDatabase.db.selectResearchersToAudit(maxQueryCountBetweenAudits,
minTimeBetweenAudits,
now)
+ Log.info(s"Auditing users ${researchersToAudit.map(_.researcher.userName).mkString(", ")}")
if (researchersToAudit.nonEmpty){
val auditLines = researchersToAudit.sortBy(_.count).reverse.map { researcher =>
researcherLineTemplate.replaceAll("FULLNAME",researcher.researcher.fullName)
.replaceAll("USERNAME",researcher.researcher.userName)
.replaceAll("COUNT",researcher.count.toString)
.replaceAll("LAST_AUDIT_DATE",new Date(researcher.leastRecentQueryDate).toString)
}.mkString("\n")
//build up the email body
val emailBody = Text(emailTemplate.replaceAll("AUDIT_LINES",auditLines)
.replaceAll("STEWARD_BASE_URL",stewardBaseUrl))
val envelope:Envelope = Envelope.from(from).to(to).subject(emailSubject).content(emailBody)
Log.debug(s"About to send $envelope .")
//send the email
val future = mailer(envelope)
try {
blocking {
Await.result(future, 60.seconds)
}
StewardDatabase.db.logAuditRequests(researchersToAudit, now)
Log.info(s"Sent and logged $envelope .")
} catch {
case NonFatal(x) => CouldNotSendAuditEmail(envelope,x)
}
}
}
}
object AuditEmailer {
/**
*
* @param config All of shrine.conf
*/
def apply(config:Config):AuditEmailer = {
val config = ConfigSource.config
val emailConfig = config.getConfig("shrine.steward.emailDataSteward")
AuditEmailer(
maxQueryCountBetweenAudits = emailConfig.getInt("maxQueryCountBetweenAudits"),
minTimeBetweenAudits = emailConfig.get("minTimeBetweenAudits", DurationConfigParser.parseDuration),
researcherLineTemplate = emailConfig.getString("researcherLine"),
emailTemplate = emailConfig.getString("emailBody"),
emailSubject = emailConfig.getString("subject"),
from = emailConfig.get("from", new InternetAddress(_)),
to = emailConfig.get("to", new InternetAddress(_)),
stewardBaseUrl = config.getString("shrine.queryEntryPoint.shrineSteward.stewardBaseUrl"),
mailer = ConfiguredMailer.createMailerFromConfig(config.getConfig("shrine.email")))
}
/**
* Check the emailer's config, log any problems
*
* @param config All of shrine.conf
*/
def configCheck(config:Config):Boolean = try {
val autoEmailer = apply(config)
Log.info(s"DSA will request audits from ${autoEmailer.to}")
true
} catch {
case NonFatal(x) =>
CannotConfigureAuditEmailer(x)
false
}
}
class AuditEmailerActor extends Actor {
override def receive: Receive = {case _ =>
val config = ConfigSource.config
AuditEmailer(config).audit()
}
}
case class CannotConfigureAuditEmailer(ex:Throwable) extends AbstractProblem(ProblemSources.Dsa) {
override def summary: String = "The DSA will not email audit requests due to a misconfiguration."
override def description: String = s"The DSA will not email audit requests due to ${throwable.get}"
override def throwable = Some(ex)
}
case class CouldNotSendAuditEmail(envelope:Envelope,ex:Throwable) extends AbstractProblem(ProblemSources.Dsa) {
override def summary: String = "The DSA was not able to send an audit email."
override def description: String = s"The DSA was not able to send an audit request to ${envelope.to} due to ${throwable.get}"
override def throwable = Some(ex)
override def detailsXml:NodeSeq =
{s"Could not send $envelope"}
{throwableDetail}
}
\ No newline at end of file
diff --git a/apps/steward-app/src/main/sql/sqlserver.ddl b/apps/steward-app/src/main/sql/mssql.ddl
similarity index 100%
rename from apps/steward-app/src/main/sql/sqlserver.ddl
rename to apps/steward-app/src/main/sql/mssql.ddl
diff --git a/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala b/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala
index fb363fdb4..603400622 100644
--- a/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala
+++ b/commons/crypto/src/test/scala/net/shrine/crypto/DownStreamCertCollectionTest.scala
@@ -1,60 +1,60 @@
package net.shrine.crypto
import java.math.BigInteger
import java.security.{KeyPairGenerator, PrivateKey, SecureRandom}
import java.util.Date
import net.shrine.util.NonEmptySeq
import org.bouncycastle.asn1.ASN1Sequence
import org.bouncycastle.asn1.x500.X500Name
import org.bouncycastle.asn1.x509._
import org.bouncycastle.cert.X509v3CertificateBuilder
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import org.scalatest.{FlatSpec, Matchers}
/**
* Created by ty on 11/1/16.
*/
@RunWith(classOf[JUnitRunner])
class DownStreamCertCollectionTest extends FlatSpec with Matchers {
- val descriptor = NewTestKeyStore.descriptor
- val heyo = "Heyo!".getBytes("UTF-8")
+ val descriptor: KeyStoreDescriptor = NewTestKeyStore.descriptor
+ val heyo : Array[Byte] = "Heyo!".getBytes("UTF-8")
"A down stream cert collection" should "build and verify its own messages" in {
val hubCertCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(descriptor) match {
case hub: DownStreamCertCollection => hub
case _ => fail("This should generate a DownstreamCertCollection!")
}
val testEntry = CertificateCreator.createSelfSignedCertEntry("notTrusted", "testing", "stillTesting")
hubCertCollection.allEntries.size shouldBe 2
hubCertCollection.myEntry.privateKey.isDefined shouldBe true
hubCertCollection.caEntry.privateKey.isDefined shouldBe false
hubCertCollection.myEntry.aliases.first shouldBe "shrine-test"
hubCertCollection.caEntry.aliases.first shouldBe "shrine-test-ca"
hubCertCollection.caEntry.wasSignedBy(hubCertCollection.myEntry) shouldBe false
hubCertCollection.myEntry.wasSignedBy(hubCertCollection.caEntry) shouldBe true
val mySigned = hubCertCollection.myEntry.sign(heyo).get
val testSigned = testEntry.sign(heyo).get
testEntry.verify(mySigned, heyo) shouldBe false
testEntry.verify(testSigned, heyo) shouldBe true
testEntry.signed(testEntry.cert) shouldBe true
hubCertCollection.myEntry.verify(testSigned, heyo) shouldBe false
hubCertCollection.myEntry.verify(mySigned, heyo) shouldBe true
hubCertCollection.caEntry.verify(testSigned, heyo) shouldBe false
hubCertCollection.caEntry.verify(mySigned, heyo) shouldBe false
hubCertCollection.verifyBytes(hubCertCollection.signBytes(heyo), heyo) shouldBe true
}
}
diff --git a/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala b/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala
index c0e3e8afb..5cd46ff73 100644
--- a/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala
+++ b/commons/crypto/src/test/scala/net/shrine/crypto/NewTestKeyStore.scala
@@ -1,26 +1,25 @@
package net.shrine.crypto
import net.shrine.util.{PeerToPeerModel, SingleHubModel}
/**
- * @author clint
- * @date Nov 27, 2013
- */
+ * @author ty
+ */
object NewTestKeyStore {
val fileName = "crypto2/shrine-test.jks"
-
+
val password = "justatestpassword"
-
+
val privateKeyAlias: Option[String] = Some("shrine-test")
-
+
val keyStoreType: KeyStoreType = KeyStoreType.JKS
-
+
val caCertAliases = Seq("shrine-test-ca")
-
+
lazy val descriptor = KeyStoreDescriptor(fileName, password, privateKeyAlias, caCertAliases, SingleHubModel(false),
Seq(RemoteSiteDescriptor("hub", Some("shrine-test-ca"), "localhost", "8080")), keyStoreType)
-
+
lazy val certCollection = BouncyKeyStoreCollection.fromFileRecoverWithClassPath(descriptor)
-
+
lazy val trustParam: TrustParam = TrustParam.BouncyKeyStore(certCollection)
}
\ No newline at end of file
diff --git a/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf b/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf
index ea9f4e305..05dbf19cb 100644
Binary files a/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf and b/shrine-webclient/src/main/html/help/pdf/shrine-client-guide.pdf differ