diff --git a/apps/dashboard-app/src/main/resources/reference.conf b/apps/dashboard-app/src/main/resources/reference.conf index b1dd0f233..f0740198c 100644 --- a/apps/dashboard-app/src/main/resources/reference.conf +++ b/apps/dashboard-app/src/main/resources/reference.conf @@ -1,82 +1,85 @@ shrine { + problem { + problemHandler = "net.shrine.problem.DatabaseProblemHandler$" + } dashboard { gruntWatch = false //false for production, true for mvn tomcat7:run . Allows the client javascript and html files to be loaded via gruntWatch . happyBaseUrl = "https://localhost:6443/shrine/rest/happy" statusBaseUrl = "https://localhost:6443/shrine/rest/internalstatus" remoteDashboard { protocol = "https://" port = ":6443" pathPrefix = "shrine-dashboard/fromDashboard" } database { dataSourceFrom = "JNDI" //Can be JNDI or testDataSource . Use testDataSource for tests, JNDI everywhere else jndiDataSourceName = "java:comp/env/jdbc/adapterAuditDB" //or leave out for tests slickProfileClassName = "slick.driver.MySQLDriver$" // Can be // slick.driver.H2Driver$ // slick.driver.MySQLDriver$ // slick.driver.PostgresDriver$ // slick.driver.SQLServerDriver$ // slick.driver.JdbcDriver$ // freeslick.OracleProfile$ // freeslick.MSSQLServerProfile$ // // (Yes, with the $ on the end) // For testing without JNDI // testDataSource { //typical test settings for unit tests //driverClassName = "org.h2.Driver" //url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests //url = "jdbc:h2:~/stewardTest.h2" //H2 embedded on disk at ~/test // } createTablesOnStart = false //for testing with H2 in memory, when not running unit tests. Set to false normally } } pmEndpoint { url = "http://changeme.com/i2b2/services/PMService/getServices" //"http://services.i2b2.org/i2b2/services/PMService/getServices" acceptAllCerts = true timeout { seconds = 10 } } authenticate { realm = "SHRINE Steward API" usersource { type = "PmUserSource" //Must be ConfigUserSource (for isolated testing) or PmUserSource (for everything else) domain = "set shrine.authenticate.usersource.domain to the PM authentication domain in dashboard.conf" //"i2b2demo" } } // If the pmEndpoint acceptAllCerts = false then you need to supply a keystore // Or if you would like dashboard-to-dashboard comms to work. // keystore { // file = "shrine.keystore" // password = "chiptesting" // privateKeyAlias = "test-cert" // keyStoreType = "JKS" // caCertAliases = [carra ca] // } } //todo typesafe config precedence seems to do the right thing, but I haven't found the rules that say this reference.conf should override others akka { loglevel = INFO // log-config-on-start = on loggers = ["akka.event.slf4j.Slf4jLogger"] // logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" // Toggles whether the threads created by this ActorSystem should be daemons or not daemonic = on } spray.servlet { boot-class = "net.shrine.dashboard.Boot" request-timeout = 30s } diff --git a/apps/dashboard-app/src/test/resources/dashboard.conf b/apps/dashboard-app/src/test/resources/dashboard.conf index 66d1e6e38..a3edd6875 100644 --- a/apps/dashboard-app/src/test/resources/dashboard.conf +++ b/apps/dashboard-app/src/test/resources/dashboard.conf @@ -1,51 +1,54 @@ shrine { + problem { + problemHandler = "net.shrine.problem.DatabaseProblemHandler$" + } authenticate { usersource { //Bogus security for testing type = "ConfigUserSource" //Must be ConfigUserSource (for isolated testing) or PmUserSource (for everything else) researcher { username = "ben" password = "kapow" } steward { username = "dave" password = "kablam" } qep{ username = "qep" password = "trustme" } admin{ username = "keith" password = "shh!" } } } dashboard { happyBaseUrl = "classpath://resources/testhappy" statusBaseUrl = "classpath://resources/teststatus" database { dataSourceFrom = "testDataSource" slickProfileClassName = "slick.driver.H2Driver$" createTestValuesOnStart = true createTablesOnStart = true // For testing without JNDI testDataSource { //typical test settings for unit tests driverClassName = "org.h2.Driver" url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" } } } keystore { file = "shrine.keystore" password = "chiptesting" privateKeyAlias = "test-cert" keyStoreType = "JKS" caCertAliases = [carra ca] } } \ No newline at end of file diff --git a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala index 540a02c66..34c53204c 100644 --- a/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala +++ b/commons/util/src/main/scala/net/shrine/problem/DashboardProblemDatabase.scala @@ -1,193 +1,191 @@ package net.shrine.problem import java.util.concurrent.TimeoutException import javax.sql.DataSource import com.typesafe.config.Config import net.shrine.slick.{CouldNotRunDbIoActionException, TestableDataSourceCreator} import slick.dbio.SuccessAction import slick.driver.JdbcProfile import slick.jdbc.meta.MTable import scala.collection.mutable import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.concurrent.{Await, Future} import scala.util.control.NonFatal import scala.xml.XML /** * Problems database object, defines the PROBLEMS table schema and related queries, * as well as all interactions with the database. * @author ty * @since 07/16 */ object Problems { val config:Config = ProblemConfigSource.config.getConfig("shrine.dashboard.database") - val slickProfileClassName = config.getString("slickProfileClassName") - // TODO: Can we not pay this 2 second cost here? - val slickProfile:JdbcProfile = ProblemConfigSource.objectForName(slickProfileClassName) + val slickProfile:JdbcProfile = ProblemConfigSource.get("slickProfileClassName", config) import slickProfile.api._ val dataSource: DataSource = TestableDataSourceCreator.dataSource(config) lazy val db = { val db = Database.forDataSource(dataSource) val createTables: String = "createTablesOnStart" if (config.hasPath(createTables) && config.getBoolean(createTables)) { val duration = FiniteDuration(3, SECONDS) Await.ready(db.run(IOActions.createIfNotExists), duration) val testValues: String = "createTestValuesOnStart" if (config.hasPath(testValues) && config.getBoolean(testValues)) { def dumb(id: Int) = ProblemDigest(s"codec($id)", s"stamp($id)", s"sum($id)", s"desc($id)",
{id}
, id) val dummyValues: Seq[ProblemDigest] = Seq(0, 1, 2, 3, 4, 5, 6).map(dumb) Await.ready(db.run(Queries ++= dummyValues), duration) } } db } /** * The Problems Table. This is the table schema. */ class ProblemsT(tag: Tag) extends Table[ProblemDigest](tag, Queries.tableName) { def id = column[Int]("id", O.PrimaryKey, O.AutoInc) def codec = column[String]("codec") def stampText = column[String]("stampText") def summary = column[String]("summary") def description = column[String]("description") def xml = column[String]("detailsXml", O.SqlType("Clob")) def epoch= column[Long]("epoch") // projection between table row and problem digest def * = (id, codec, stampText, summary, description, xml, epoch) <> (rowToProblem, problemToRow) def idx = index("idx_epoch", epoch, unique=false) /** * Converts a table row into a ProblemDigest. * @param args the table row, represented as a five-tuple string * @return the corresponding ProblemDigest */ def rowToProblem(args: (Int, String, String, String, String, String, Long)): ProblemDigest = args match { case (id, codec, stampText, summary, description, detailsXml, epoch) => ProblemDigest(codec, stampText, summary, description, XML.loadString(detailsXml), epoch) } /** * Converts a ProblemDigest into an Option of a table row. For now there is no failure * condition, ie a ProblemDigest can always be a table row, but this is a place for * possible future error handling * @param problem the ProblemDigest to convert * @return an Option of a table row. */ def problemToRow(problem: ProblemDigest): Option[(Int, String, String, String, String, String, Long)] = problem match { case ProblemDigest(codec, stampText, summary, description, detailsXml, epoch) => // 7 is ignored on insert and replaced with an auto incremented id Some((7, codec, stampText, summary, description, detailsXml.toString, epoch)) } } /** * Queries related to the Problems table. */ object Queries extends TableQuery(new ProblemsT(_)) { /** * The table name */ val tableName = "problems" /** * Equivalent to Select * from Problems; */ val selectAll = this /** * Selects all the details xml sorted by the problem's time stamp. */ val selectDetails = this.map(_.xml) /** * Sorts the problems in descending order */ val descending = this.sortBy(_.epoch.desc) /** * Selects the last N problems, after the offset */ def lastNProblems(n: Int, offset: Int = 0) = this.descending.drop(offset).take(n) } /** * DBIO Actions. These are pre-defined IO actions that may be useful. * Using it to centralize the location of DBIOs. */ object IOActions { val problems = Queries val tableExists = MTable.getTables(problems.tableName).map(_.nonEmpty) val createIfNotExists = tableExists.flatMap( if (_) SuccessAction(NoOperation) else problems.schema.create) val dropIfExists = tableExists.flatMap( if (_) problems.schema.drop else SuccessAction(NoOperation)) val resetTable = createIfNotExists >> problems.selectAll.delete val selectAll = problems.result def sizeAndProblemDigest(n: Int, offset: Int = 0) = problems.lastNProblems(n, offset).result.zip(problems.size.result) def findIndexOfDate(date: Long) = (problems.size - problems.filter(_.epoch <= date).size).result } /** * Entry point for interacting with the database. Runs IO actions. */ object DatabaseConnector { val IO = IOActions /** * Executes a series of IO actions as a single transactions */ def executeTransaction(actions: DBIOAction[_, NoStream, _]*): Future[Unit] = { db.run(DBIO.seq(actions:_*).transactionally) } /** * Executes a series of IO actions as a single transaction, synchronous */ def executeTransactionBlocking(actions: DBIOAction[_, NoStream, _]*)(implicit timeout: Duration): Unit = { try { Await.ready(this.executeTransaction(actions: _*), timeout) } catch { // TODO: Handle this better case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx) case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x) } } /** * Straight copy of db.run */ def run[R](dbio: DBIOAction[R, NoStream, _]): Future[R] = { db.run(dbio) } /** * Synchronized copy of db.run */ def runBlocking[R](dbio: DBIOAction[R, NoStream, _])(implicit timeout: Duration = new FiniteDuration(15, SECONDS)): R = { try { Await.result(this.run(dbio), timeout) } catch { case tx:TimeoutException => throw CouldNotRunDbIoActionException(Problems.dataSource, tx) case NonFatal(x) => throw CouldNotRunDbIoActionException(Problems.dataSource, x) } } def insertProblem(problem: ProblemDigest, timeout: Duration = new FiniteDuration(15, SECONDS)) = { runBlocking(Queries += problem)(timeout) } } } // For SuccessAction, just a no_op. case object NoOperation \ No newline at end of file diff --git a/commons/util/src/main/scala/net/shrine/problem/Problem.scala b/commons/util/src/main/scala/net/shrine/problem/Problem.scala index b9ac30111..6cf855934 100644 --- a/commons/util/src/main/scala/net/shrine/problem/Problem.scala +++ b/commons/util/src/main/scala/net/shrine/problem/Problem.scala @@ -1,225 +1,227 @@ package net.shrine.problem import java.net.InetAddress import java.util.Date import java.util.concurrent.Executors import net.shrine.log.Loggable import net.shrine.serialization.{XmlMarshaller, XmlUnmarshaller} import scala.concurrent.{ExecutionContext, Future, Promise} import scala.xml.{Elem, Node, NodeSeq} /** * Describes what information we have about a problem at the site in code where we discover it. * * @author david * @since 8/6/15 */ trait Problem { def summary:String def problemName = getClass.getName def throwable:Option[Throwable] = None def stamp:Stamp def description:String def exceptionXml(exception:Option[Throwable]): Option[Elem] = { exception.map{x => {x.getClass.getName} {x.getMessage} {x.getStackTrace.map(line => {line})}{exceptionXml(Option(x.getCause)).getOrElse("")} }} def throwableDetail: Option[Elem] = exceptionXml(throwable) def detailsXml: NodeSeq = NodeSeq.fromSeq(
{throwableDetail.getOrElse("")}
) def toDigest:ProblemDigest = ProblemDigest(problemName,stamp.pretty,summary,description,detailsXml,stamp.time) /** * Temporary replacement for onCreate, which will be released come Scala 2.13 * TODO: remove when Scala 2.13 releases */ def hackToHandleAfterInitialization(handler:ProblemHandler):Future[Unit] = { import ProblemExecutionContext.ioThreadPool Future { var continue = true while (continue) { try { - handler.handleProblem(this) + synchronized(handler.handleProblem(this)) continue = false } catch { case un:UninitializedFieldError => Thread.sleep(5) continue = true } } Unit } } } case class ProblemDigest(codec: String, stampText: String, summary: String, description: String, detailsXml: NodeSeq, epoch: Long) extends XmlMarshaller { override def toXml: Node = { {codec} {stampText} {summary} {description} {epoch} {detailsXml} } /** * Ignores detailXml. equals with scala.xml is impossible. See http://www.scala-lang.org/api/2.10.3/index.html#scala.xml.Equality$ */ override def equals(other: Any): Boolean = other match { case that: ProblemDigest => (that canEqual this) && codec == that.codec && stampText == that.stampText && summary == that.summary && description == that.description && epoch == that.epoch case _ => false } /** * Ignores detailXml */ override def hashCode: Int = { val prime = 67 codec.hashCode + prime * (stampText.hashCode + prime *(summary.hashCode + prime * (description.hashCode + prime * epoch.hashCode()))) } } object ProblemDigest extends XmlUnmarshaller[ProblemDigest] with Loggable { override def fromXml(xml: NodeSeq): ProblemDigest = { val problemNode = xml \ "problem" require(problemNode.nonEmpty,s"No problem tag in $xml") def extractText(tagName:String) = (problemNode \ tagName).text val codec = extractText("codec") val stampText = extractText("stamp") val summary = extractText("summary") val description = extractText("description") val detailsXml: NodeSeq = problemNode \ "details" val epoch = try { extractText("epoch").toLong } catch { case nx:NumberFormatException => error(s"While parsing xml representing a ProblemDigest, the epoch could not be parsed into a long", nx) 0 } ProblemDigest(codec,stampText,summary,description,detailsXml,epoch) } } case class Stamp(host:InetAddress,time:Long,source:ProblemSources.ProblemSource) { def pretty = s"${new Date(time)} on ${host.getHostName} ${source.pretty}" } object Stamp { //TODO: val dateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")? //TODO: Currently the stamp text is locale specific, which can change depending on the jre/computer running it... def apply(source:ProblemSources.ProblemSource, timer: => Long): Stamp = Stamp(InetAddress.getLocalHost, timer, source) } abstract class AbstractProblem(source:ProblemSources.ProblemSource) extends Problem { def timer = System.currentTimeMillis override val stamp = Stamp(source, timer) - hackToHandleAfterInitialization(DatabaseProblemHandler) + private val config = ProblemConfigSource.config.getConfig("shrine.problem") + hackToHandleAfterInitialization(ProblemConfigSource.get("problemHandler", config)) } trait ProblemHandler { def handleProblem(problem:Problem) } /** * An example problem handler */ object LoggingProblemHandler extends ProblemHandler with Loggable { override def handleProblem(problem: Problem): Unit = { problem.throwable.fold(error(problem.toString))(throwable => error(problem.toString,throwable) ) } } object DatabaseProblemHandler extends ProblemHandler { override def handleProblem(problem: Problem): Unit = { + Thread.sleep(5) if (!ProblemConfigSource.turnOffConnector) Problems.DatabaseConnector.insertProblem(problem.toDigest) } } object ProblemSources{ sealed trait ProblemSource { def pretty = getClass.getSimpleName.dropRight(1) } case object Adapter extends ProblemSource case object Hub extends ProblemSource case object Qep extends ProblemSource case object Dsa extends ProblemSource case object Unknown extends ProblemSource def problemSources = Set(Adapter,Hub,Qep,Dsa,Unknown) } case class ProblemNotYetEncoded(internalSummary:String,t:Option[Throwable] = None) extends AbstractProblem(ProblemSources.Unknown){ override val summary = "An unanticipated problem encountered." override val throwable = { val rx = t.fold(new IllegalStateException(s"$summary"))( new IllegalStateException(s"$summary",_) ) rx.fillInStackTrace() Some(rx) } val reportedAtStackTrace = new IllegalStateException("Capture reporting stack trace.") override val description = "This problem is not yet classified in Shrine source code. Please report the details to the Shrine dev team." override val detailsXml: NodeSeq = NodeSeq.fromSeq(
{internalSummary} {throwableDetail.getOrElse("")}
) } object ProblemNotYetEncoded { def apply(summary:String,x:Throwable):ProblemNotYetEncoded = ProblemNotYetEncoded(summary,Some(x)) } object ProblemExecutionContext { private val processes = Runtime.getRuntime.availableProcessors() private val factor = 3 private val threads = processes * factor implicit val ioThreadPool: ExecutionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(threads)) } \ No newline at end of file diff --git a/commons/util/src/main/scala/net/shrine/problem/ProblemConfigSource.scala b/commons/util/src/main/scala/net/shrine/problem/ProblemConfigSource.scala index 1c1b21a78..42eadba49 100644 --- a/commons/util/src/main/scala/net/shrine/problem/ProblemConfigSource.scala +++ b/commons/util/src/main/scala/net/shrine/problem/ProblemConfigSource.scala @@ -1,19 +1,22 @@ package net.shrine.problem -import com.typesafe.config.{ConfigValue, ConfigValueFactory} +import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory} import net.shrine.source.ConfigSource /** * Source of typesafe config for the problems database * * @author ty * @since 7/22/16 */ object ProblemConfigSource extends ConfigSource { override val configName: String = "dashboard" // Makes it so constructing a problem in this context won't log it to the connector // Does not stop you from constructing the connector and using it manually var turnOffConnector = false - // var turnOffConnector = config.getProblemHandler + + def get[T](path: String, config:Config):T = { + objectForName(config.getString(path)) + } } diff --git a/commons/util/src/test/resources/dashboard.conf b/commons/util/src/test/resources/dashboard.conf index a6c6913d3..6ab34572d 100644 --- a/commons/util/src/test/resources/dashboard.conf +++ b/commons/util/src/test/resources/dashboard.conf @@ -1,17 +1,20 @@ shrine{ + problem { + problemHandler = "net.shrine.problem.DatabaseProblemHandler$" + } dashboard { database { dataSourceFrom = "testDataSource" slickProfileClassName = "slick.driver.H2Driver$" createTablesOnStart = true // For testing without JNDI testDataSource { //typical test settings for unit tests driverClassName = "org.h2.Driver" url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" } } } } \ No newline at end of file diff --git a/integration/src/test/resources/dashboard.conf b/integration/src/test/resources/dashboard.conf index 6fea03987..af0eac07f 100644 --- a/integration/src/test/resources/dashboard.conf +++ b/integration/src/test/resources/dashboard.conf @@ -1,18 +1,21 @@ shrine{ + problem { + problemHandler = "net.shrine.problem.DatabaseProblemHandler$" + } dashboard { database { dataSourceFrom = "testDataSource" slickProfileClassName = "slick.driver.H2Driver$" createTablesOnStart = true createTestValuesOnStart = false // For testing without JNDI testDataSource { //typical test settings for unit tests driverClassName = "org.h2.Driver" numThreads = 30 url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" } } } } \ No newline at end of file