diff --git a/adapter/adapter-service/src/main/scala/net/shrine/adapter/AdapterComponents.scala b/adapter/adapter-service/src/main/scala/net/shrine/adapter/AdapterComponents.scala index a6bab4f69..70b758f0e 100644 --- a/adapter/adapter-service/src/main/scala/net/shrine/adapter/AdapterComponents.scala +++ b/adapter/adapter-service/src/main/scala/net/shrine/adapter/AdapterComponents.scala @@ -1,154 +1,157 @@ package net.shrine.adapter import scala.collection.JavaConverters._ import com.typesafe.config.Config import net.shrine.adapter.dao.{AdapterDao, I2b2AdminDao} import net.shrine.adapter.dao.squeryl.{SquerylAdapterDao, SquerylI2b2AdminDao} import net.shrine.adapter.dao.squeryl.tables.Tables import net.shrine.adapter.service.{AdapterService, I2b2AdminService} import net.shrine.adapter.translators.{ExpressionTranslator, QueryDefinitionTranslator} import net.shrine.client.{EndpointConfig, Poster} import net.shrine.config.mappings.{AdapterMappings, AdapterMappingsSource, ClasspathFormatDetectingAdapterMappingsSource} import net.shrine.crypto.{DefaultSignerVerifier, KeyStoreCertCollection} import net.shrine.dao.squeryl.SquerylInitializer import net.shrine.protocol.{HiveCredentials, NodeId, RequestType, ResultOutputType} import net.shrine.config.{ConfigExtensions, DurationConfigParser} import net.shrine.log.Log import scala.concurrent.duration.Duration /** * All the parts required for an adapter. * * @author david * @since 1.22 */ case class AdapterComponents( adapterService: AdapterService, i2b2AdminService: I2b2AdminService, adapterDao: AdapterDao, - adapterMappings: AdapterMappings) + adapterMappings: AdapterMappings, + lastModified: Long + ) object AdapterComponents { //todo try and trim this argument list back def apply( adapterConfig:Config, //config is "shrine.adapter" certCollection: KeyStoreCertCollection, squerylInitializer: SquerylInitializer, breakdownTypes: Set[ResultOutputType], crcHiveCredentials: HiveCredentials, signerVerifier: DefaultSignerVerifier, pmPoster: Poster, nodeId: NodeId ):AdapterComponents = { val crcEndpoint: EndpointConfig = adapterConfig.getConfigured("crcEndpoint",EndpointConfig(_)) val crcPoster: Poster = Poster(certCollection,crcEndpoint) val squerylAdapterTables: Tables = new Tables val adapterDao: AdapterDao = new SquerylAdapterDao(squerylInitializer, squerylAdapterTables)(breakdownTypes) //NB: Is i2b2HiveCredentials.projectId the right project id to use? val i2b2AdminDao: I2b2AdminDao = new SquerylI2b2AdminDao(crcHiveCredentials.projectId, squerylInitializer, squerylAdapterTables) val adapterMappingsFile = adapterConfig.getString("adapterMappingsFileName") val adapterMappingsSource: AdapterMappingsSource = ClasspathFormatDetectingAdapterMappingsSource(adapterMappingsFile) //NB: Fail fast val adapterMappings: AdapterMappings = adapterMappingsSource.load(adapterMappingsFile).get val expressionTranslator: ExpressionTranslator = ExpressionTranslator(adapterMappings) val queryDefinitionTranslator: QueryDefinitionTranslator = new QueryDefinitionTranslator(expressionTranslator) val doObfuscation = adapterConfig.getBoolean("setSizeObfuscation") val collectAdapterAudit = adapterConfig.getBoolean("audit.collectAdapterAudit") val botCountTimeThresholds: Seq[(Long, Duration)] = { import scala.concurrent.duration._ val countsAndMilliseconds: Seq[Config] = adapterConfig.getConfig("botDefense").getConfigList("countsAndMilliseconds").asScala countsAndMilliseconds.map(pairConfig => (pairConfig.getLong("count"),pairConfig.getLong("milliseconds").milliseconds)) } val obfuscator:Obfuscator = adapterConfig.getConfigured("obfuscation",Obfuscator(_)) Log.info(s"obfuscator is $obfuscator") val runQueryAdapter = RunQueryAdapter( poster = crcPoster, dao = adapterDao, hiveCredentials = crcHiveCredentials, conceptTranslator = queryDefinitionTranslator, adapterLockoutAttemptsThreshold = adapterConfig.getInt("adapterLockoutAttemptsThreshold"), doObfuscation = doObfuscation, runQueriesImmediately = adapterConfig.getOption("immediatelyRunIncomingQueries", _.getBoolean).getOrElse(true), //todo use reference.conf breakdownTypes = breakdownTypes, collectAdapterAudit = collectAdapterAudit, botCountTimeThresholds = botCountTimeThresholds, obfuscator = obfuscator ) val readInstanceResultsAdapter: Adapter = new ReadInstanceResultsAdapter( poster = crcPoster, hiveCredentials = crcHiveCredentials, dao = adapterDao, doObfuscation = doObfuscation, breakdownTypes = breakdownTypes, collectAdapterAudit = collectAdapterAudit, obfuscator = obfuscator ) val readQueryResultAdapter: Adapter = new ReadQueryResultAdapter( crcPoster, crcHiveCredentials, adapterDao, doObfuscation, breakdownTypes, collectAdapterAudit, obfuscator = obfuscator ) val readPreviousQueriesAdapter: Adapter = new ReadPreviousQueriesAdapter(adapterDao) val deleteQueryAdapter: Adapter = new DeleteQueryAdapter(adapterDao) val renameQueryAdapter: Adapter = new RenameQueryAdapter(adapterDao) val readQueryDefinitionAdapter: Adapter = new ReadQueryDefinitionAdapter(adapterDao) val readTranslatedQueryDefinitionAdapter: Adapter = new ReadTranslatedQueryDefinitionAdapter(nodeId, queryDefinitionTranslator) val flagQueryAdapter: Adapter = new FlagQueryAdapter(adapterDao) val unFlagQueryAdapter: Adapter = new UnFlagQueryAdapter(adapterDao) val adapterMap = AdapterMap(Map( RequestType.QueryDefinitionRequest -> runQueryAdapter, RequestType.GetRequestXml -> readQueryDefinitionAdapter, RequestType.UserRequest -> readPreviousQueriesAdapter, RequestType.InstanceRequest -> readInstanceResultsAdapter, RequestType.MasterDeleteRequest -> deleteQueryAdapter, RequestType.MasterRenameRequest -> renameQueryAdapter, RequestType.GetQueryResult -> readQueryResultAdapter, RequestType.ReadTranslatedQueryDefinitionRequest -> readTranslatedQueryDefinitionAdapter, RequestType.FlagQueryRequest -> flagQueryAdapter, RequestType.UnFlagQueryRequest -> unFlagQueryAdapter)) AdapterComponents( adapterService = new AdapterService( nodeId = nodeId, signatureVerifier = signerVerifier, maxSignatureAge = adapterConfig.getConfigured("maxSignatureAge", DurationConfigParser(_)), adapterMap = adapterMap ), i2b2AdminService = new I2b2AdminService( dao = adapterDao, i2b2AdminDao = i2b2AdminDao, pmPoster = pmPoster, runQueryAdapter = runQueryAdapter ), adapterDao = adapterDao, - adapterMappings = adapterMappings) + adapterMappings = adapterMappings, + lastModified = adapterMappingsSource.lastModified) } } \ No newline at end of file diff --git a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala index 4bdc74434..b4cc79d24 100644 --- a/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala +++ b/apps/shrine-app/src/main/scala/net/shrine/status/StatusJaxrs.scala @@ -1,452 +1,445 @@ package net.shrine.status import java.io.File import java.net.URL import java.security.MessageDigest import java.security.cert.X509Certificate import java.util.Date import javax.ws.rs.{GET, Path, Produces, WebApplicationException} import javax.ws.rs.core.{MediaType, Response} import com.sun.jersey.spi.container.{ContainerRequest, ContainerRequestFilter} import com.typesafe.config.{Config => TsConfig} +import net.shrine.adapter.AdapterComponents import net.shrine.authorization.{QueryAuthorizationService, StewardQueryAuthorizationService} import net.shrine.broadcaster._ import net.shrine.client.PosterOntClient import net.shrine.wiring.ShrineOrchestrator import org.json4s.{DefaultFormats, Formats} import org.json4s.native.Serialization import net.shrine.log.{Log, Loggable} import scala.collection.JavaConverters._ import scala.collection.immutable.{Map, Seq, Set} import net.shrine.config.ConfigExtensions import net.shrine.crypto.{KeyStoreCertCollection, KeyStoreDescriptor, SigningCertStrategy} import net.shrine.protocol.query.{OccuranceLimited, QueryDefinition, Term} import net.shrine.protocol._ import net.shrine.serialization.NodeSeqSerializer import net.shrine.util.Versions import scala.concurrent.Await import scala.util.{Success, Try} import scala.util.control.NonFatal /** * A subservice that shares internal state of the shrine servlet. * * @author david * @since 12/2/15 */ @Path("/internalstatus") @Produces(Array(MediaType.APPLICATION_JSON)) case class StatusJaxrs(shrineConfig:TsConfig) extends Loggable { implicit def json4sFormats: Formats = DefaultFormats + new NodeSeqSerializer @GET @Path("version") def version: String = { val version = Version("changeMe") val versionString = Serialization.write(version) versionString } @GET @Path("config") def config: String = { //todo probably better to reach out and grab the config from ManuallyWiredShrineJaxrsResources once it is a singleton Serialization.write(Json4sConfig(shrineConfig)) } @GET @Path("summary") def summary: String = { val summary = Summary() Serialization.write(summary) } @GET @Path("i2b2") def i2b2: String = { val i2b2 = I2b2() Serialization.write(i2b2) } @GET @Path("optionalParts") def optionalParts: String = { val optionalParts = OptionalParts() Serialization.write(optionalParts) } @GET @Path("hub") def hub: String = { val hub = Hub() Serialization.write(hub) } @GET @Path("adapter") def adapter: String = { val adapter = Adapter() Serialization.write(adapter) } @GET @Path("qep") def qep: String = { val qep = Qep() Serialization.write(qep) } @GET @Path("keystore") def keystore: String = { Serialization.write(KeyStoreReport()) } } /* todo fill in later when you take the time to get the right parts in place SHRINE-1529 case class KeyStoreEntryReport( alias:String, commonName:String, md5Signature:String ) */ case class KeyStoreReport( fileName:String, password:String = "REDACTED", privateKeyAlias:Option[String], owner:Option[String], issuer:Option[String], expires:Option[Date], signature:Option[String], caTrustedAlias:Option[String], caTrustedSignature:Option[String] // keyStoreContents:List[KeyStoreEntryReport] //todo SHRINE-1529 ) //todo build new API for the dashboard to use to check signatures object KeyStoreReport { def apply(): KeyStoreReport = { val keystoreDescriptor: KeyStoreDescriptor = ShrineOrchestrator.keyStoreDescriptor val certCollection: KeyStoreCertCollection = ShrineOrchestrator.certCollection def toMd5(cert:X509Certificate): String = { val md5 = MessageDigest.getInstance("MD5") def toHex(buf: Array[Byte]): String = buf.map("%02X".format(_)).mkString(":") toHex(md5.digest(cert.getEncoded)) } new KeyStoreReport( fileName = keystoreDescriptor.file, privateKeyAlias = keystoreDescriptor.privateKeyAlias, owner = certCollection.myCert.map(cert => cert.getSubjectDN.getName), issuer = certCollection.myCert.map(cert => cert.getIssuerDN.getName), expires = certCollection.myCert.map(cert => cert.getNotAfter), signature = certCollection.myCert.map(cert => toMd5(cert)), //todo sha1 signature if needed caTrustedAlias = certCollection.caCertAliases.headOption, caTrustedSignature = certCollection.headOption.map(cert => toMd5(cert)) // keyStoreContents = certCollection.caCerts.zipWithIndex.map((cert: ((Principal, X509Certificate), Int)) => KeyStoreEntryReport(keystoreDescriptor.caCertAliases(cert._2),cert._1._1.getName,toMd5(cert._1._2))).to[List] ) } } case class I2b2(pmUrl:String, crcUrl:Option[String], ontUrl:String, i2b2Domain:String, username:String, crcProject:String, ontProject:String) object I2b2 { def apply(): I2b2 = new I2b2( pmUrl = ShrineOrchestrator.pmPoster.url, crcUrl = ShrineOrchestrator.adapterComponents.map(_.i2b2AdminService.crcUrl), ontUrl = "", //todo i2b2Domain = ShrineOrchestrator.crcHiveCredentials.domain, username = ShrineOrchestrator.crcHiveCredentials.username, crcProject = ShrineOrchestrator.crcHiveCredentials.projectId, ontProject = ShrineOrchestrator.ontologyMetadata.client match { case client: PosterOntClient => client.hiveCredentials.projectId case _ => "" } ) } case class DownstreamNode(name:String, url:String) // Replaces StewardQueryAuthorizationService so that we never transmit a password case class Steward(stewardBaseUrl: String, qepUsername: String, password:String = "REDACTED") case class Qep( maxQueryWaitTimeMillis:Long, create:Boolean, attachSigningCert:Boolean, authorizationType:String, includeAggregateResults:Boolean, authenticationType:String, steward:Option[Steward], broadcasterUrl:Option[String] ) object Qep{ val key = "shrine.queryEntryPoint." import ShrineOrchestrator.queryEntryPointComponents def apply():Qep = new Qep( maxQueryWaitTimeMillis = queryEntryPointComponents.fold(0L)(_.i2b2Service.queryTimeout.toMicros), create = queryEntryPointComponents.isDefined, attachSigningCert = queryEntryPointComponents.fold(false)(_.i2b2Service.broadcastAndAggregationService.attachSigningCert), authorizationType = queryEntryPointComponents.fold("")(_.i2b2Service.authorizationService.getClass.getSimpleName), includeAggregateResults = queryEntryPointComponents.fold(false)(_.i2b2Service.includeAggregateResult), authenticationType = queryEntryPointComponents.fold("")(_.i2b2Service.authenticator.getClass.getSimpleName), steward = queryEntryPointComponents.flatMap(qec => checkStewardAuthorization(qec.shrineService.authorizationService)), broadcasterUrl = queryEntryPointComponents.flatMap(qec => checkBroadcasterUrl(qec.i2b2Service.broadcastAndAggregationService))) def checkStewardAuthorization(auth: QueryAuthorizationService): Option[Steward] = auth match { case sa:StewardQueryAuthorizationService => Some(Steward(sa.stewardBaseUrl.toString, sa.qepUserName)) case _ => None } //TODO: Double check with Dave that this is the right url def checkBroadcasterUrl(broadcaster: BroadcastAndAggregationService): Option[String] = broadcaster match { case a:HubBroadcastAndAggregationService => a.broadcasterClient match { case PosterBroadcasterClient(poster, _) => Some(poster.url) case _ => None } case _ => None } } object DownstreamNodes { def get():Seq[DownstreamNode] = { ShrineOrchestrator.hubComponents.fold(Seq.empty[DownstreamNode])(_.broadcastDestinations.map(DownstreamNode(_)).to[Seq]) } } object DownstreamNode { def apply(nodeHandle: NodeHandle): DownstreamNode = new DownstreamNode( nodeHandle.nodeId.name, nodeHandle.client.url.map(_.toString).getOrElse("not applicable")) } case class Adapter(crcEndpointUrl:String, setSizeObfuscation:Boolean, adapterLockoutAttemptsThreshold:Int, adapterMappingsFilename:Option[String], adapterMappingsDate:Option[Long] ) object Adapter{ def apply():Adapter = { val crcEndpointUrl = ShrineOrchestrator.adapterComponents.fold("")(_.i2b2AdminService.crcUrl) val setSizeObfuscation = ShrineOrchestrator.adapterComponents.fold(false)(_.i2b2AdminService.obfuscate) val adapterLockoutAttemptsThreshold = ShrineOrchestrator.adapterComponents.fold(0)(_.i2b2AdminService.adapterLockoutAttemptsThreshold) - val adapterMappingsFileInfo = mappingFileInfo + val adapterMappingsFileName = mappingFileInfo.map(_._1) + val adapterMappingsFileDate = mappingFileInfo.map(_._2) - Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileInfo._1, adapterMappingsFileInfo._2) + Adapter(crcEndpointUrl, setSizeObfuscation, adapterLockoutAttemptsThreshold, adapterMappingsFileName, adapterMappingsFileDate) } - def mappingFileInfo: (Option[String], Option[Long], Option[String]) = { - val adapterMappingsFileName = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.source) - val adapterMappingsVersion = ShrineOrchestrator.adapterComponents.map(_.adapterMappings.version) //todo use this? - val noDate:Option[Long] = None - val adapterMappingsDate:Option[Long] = adapterMappingsFileName.fold(noDate){ fileName => - val file:File = new File(fileName) - if(file.exists) Some(file.lastModified()) - else None - } - (adapterMappingsFileName,adapterMappingsDate,adapterMappingsVersion) - } + def mappingFileInfo: Option[(String, Long, String)] = + ShrineOrchestrator.adapterComponents.map(ac => (ac.adapterMappings.source, ac.lastModified, ac.adapterMappings.version)) } case class Hub(shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible create:Boolean, downstreamNodes:Seq[DownstreamNode]) object Hub{ def apply():Hub = { val shouldQuerySelf = false val create = ShrineOrchestrator.hubComponents.isDefined val downstreamNodes = DownstreamNodes.get() Hub(shouldQuerySelf, create, downstreamNodes) } } case class OptionalParts(isHub:Boolean, stewardEnabled:Boolean, shouldQuerySelf:Boolean, //todo don't use this field any more. Drop it when possible downstreamNodes:Seq[DownstreamNode]) object OptionalParts { def apply(): OptionalParts = { OptionalParts( ShrineOrchestrator.hubComponents.isDefined, ShrineOrchestrator.queryEntryPointComponents.fold(false)(_.shrineService.authorizationService.isInstanceOf[StewardQueryAuthorizationService]), shouldQuerySelf = false, DownstreamNodes.get() ) } } case class Summary( isHub:Boolean, shrineVersion:String, shrineBuildDate:String, ontologyVersion:String, ontologyTerm:String, queryResult: Option[SingleNodeResult], adapterMappingsFileName:Option[String], adapterMappingsDate:Option[Long], adapterOk:Boolean, keystoreOk:Boolean, hubOk:Boolean, qepOk:Boolean ) object Summary { val term = Term(ShrineOrchestrator.shrineConfig.getString("networkStatusQuery")) def runQueryRequest: BroadcastMessage = { val domain = "happy" val username = "happy" val networkAuthn = AuthenticationInfo(domain, username, Credential("", isToken = false)) val queryDefinition = QueryDefinition("TestQuery", OccuranceLimited(1, term)) import scala.concurrent.duration._ val req = RunQueryRequest( "happyProject", 3.minutes, networkAuthn, None, None, Set(ResultOutputType.PATIENT_COUNT_XML), queryDefinition) ShrineOrchestrator.signerVerifier.sign(BroadcastMessage(req.networkQueryId, networkAuthn, req), SigningCertStrategy.Attach) } def apply(): Summary = { val message = runQueryRequest val queryResult: Option[SingleNodeResult] = ShrineOrchestrator.adapterService.map{ adapterService => import scala.concurrent.duration._ val start = System.currentTimeMillis val resultAttempt: Try[Result] = Try(adapterService.handleRequest(message)) val end = System.currentTimeMillis val elapsed = (end - start).milliseconds resultAttempt match { case scala.util.Success(result) => result case scala.util.Failure(throwable) => FailureResult(NodeId("Local"), throwable) } } val adapterOk = queryResult.fold(true) { case r:Result => true case f:FailureResult => false } val hubOk = ShrineOrchestrator.hubComponents.fold(true){ hubComponents => val maxQueryWaitTime = hubComponents.broadcasterMultiplexerService.maxQueryWaitTime val broadcaster: Broadcaster = hubComponents.broadcasterMultiplexerService.broadcaster val message = runQueryRequest val triedMultiplexer = Try(broadcaster.broadcast(message)) //todo just use fold()() in scala 2.12 triedMultiplexer.toOption.fold(false) { multiplexer => val responses = Await.result(multiplexer.responses, maxQueryWaitTime).toSeq val failures = responses.collect { case f: FailureResult => f } val timeouts = responses.collect { case t: Timeout => t } val validResults = responses.collect { case r: Result => r } failures.isEmpty && timeouts.isEmpty && (validResults.size == broadcaster.destinations.size) } } val adapterMappingInfo = Adapter.mappingFileInfo val ontologyVersion = try { ShrineOrchestrator.ontologyMetadata.ontologyVersion } catch { case NonFatal(x) => Log.info("Problem while getting ontology version",x) s"Unavailable due to: ${x.getMessage}" } Summary( isHub = ShrineOrchestrator.hubComponents.isDefined, shrineVersion = Versions.version, shrineBuildDate = Versions.buildDate, //todo in scala 2.12, do better ontologyVersion = ontologyVersion, ontologyTerm = term.value, queryResult = queryResult, adapterMappingsFileName = adapterMappingInfo._1, adapterMappingsDate = adapterMappingInfo._2, adapterOk = adapterOk, keystoreOk = true, //todo something for this hubOk = hubOk, qepOk = true //todo something for this ) } } case class Version(version:String) //todo SortedMap when possible case class Json4sConfig(keyValues:Map[String,String]) object Json4sConfig{ def isPassword(key:String):Boolean = { if(key.toLowerCase.contains("password")) true else false } def apply(config:TsConfig):Json4sConfig = { val entries: Set[(String, String)] = config.entrySet.asScala.to[Set].map(x => (x.getKey,x.getValue.render())).filterNot(x => isPassword(x._1)) val sortedMap: Map[String, String] = entries.toMap Json4sConfig(sortedMap) } } class PermittedHostOnly extends ContainerRequestFilter { //todo generalize for happy, too //todo for tomcat 8 see https://jersey.java.net/documentation/latest/filters-and-interceptors.html for a cleaner version //shell code from http://stackoverflow.com/questions/17143514/how-to-add-custom-response-and-abort-request-in-jersey-1-11-filters //how to apply in http://stackoverflow.com/questions/4358213/how-does-one-intercept-a-request-during-the-jersey-lifecycle override def filter(requestContext: ContainerRequest): ContainerRequest = { val hostOfOrigin = requestContext.getBaseUri.getHost val shrineConfig:TsConfig = ShrineOrchestrator.config val permittedHostOfOrigin:String = shrineConfig.getOption("shrine.status.permittedHostOfOrigin",_.getString).getOrElse("localhost") val path = requestContext.getPath //happy and internalstatus API calls must come from the same host as tomcat is running on (hopefully the dashboard servlet). // todo access to the happy service permitted for SHRINE 1.21 per SHRINE-1366 // restrict access to happy service when database work resumes as part of SHRINE- // if ((path.contains("happy") || path.contains("internalstatus")) && (hostOfOrigin != permittedHostOfOrigin)) { if (path.contains("internalstatus") && (hostOfOrigin != permittedHostOfOrigin)) { val response = Response.status(Response.Status.UNAUTHORIZED).entity(s"Only available from $permittedHostOfOrigin, not $hostOfOrigin, controlled by shrine.status.permittedHostOfOrigin in shrine.conf").build() throw new WebApplicationException(response) } else requestContext } } \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/mappings/AdapterMappingsSource.scala b/commons/config/src/main/scala/net/shrine/config/mappings/AdapterMappingsSource.scala index 167fca552..2cffe2b7a 100644 --- a/commons/config/src/main/scala/net/shrine/config/mappings/AdapterMappingsSource.scala +++ b/commons/config/src/main/scala/net/shrine/config/mappings/AdapterMappingsSource.scala @@ -1,13 +1,15 @@ package net.shrine.config.mappings import scala.util.Try /** * @author clint * @since Mar 6, 2012 */ //todo this whole hierarchy of traits is really suspect. Maybe get rid of the whole works, or at least any unused levels. trait AdapterMappingsSource { def load(source:String): Try[AdapterMappings] + + def lastModified: Long } \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathAdapterMappingsSource.scala b/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathAdapterMappingsSource.scala index f73d418ee..5d65047d0 100644 --- a/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathAdapterMappingsSource.scala +++ b/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathAdapterMappingsSource.scala @@ -1,23 +1,33 @@ package net.shrine.config.mappings import java.io.InputStreamReader import java.io.Reader +import java.net.URL /** * @author clint * @date Mar 6, 2012 */ trait ClasspathAdapterMappingsSource extends ReaderAdapterMappingsSource { def mappingFileName: String //NB: Will blow up loudly if mapping file isn't found - final override protected def reader: Reader = { + final override protected def reader: Reader = new InputStreamReader(helpError.openStream()) + + final override def lastModified: Long = { + val conn = helpError.openConnection() + val lastModified = conn.getLastModified + + // release the resources + conn.getInputStream.close() + + lastModified + } + + private def helpError: URL = { require(mappingFileName != null) - - val mappingStream = getClass.getClassLoader.getResourceAsStream(mappingFileName) - - require(mappingStream != null, s"Couldn't find adapter mapping file '$mappingFileName' on the classpath") - - new InputStreamReader(mappingStream) + val url = getClass.getClassLoader.getResource(mappingFileName) + require(url != null, s"Couldn't find adapter mapping file '$mappingFileName' on the classpath") + url } } \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathFormatDetectingAdapterMappingsSource.scala b/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathFormatDetectingAdapterMappingsSource.scala index fcdabf91e..fcba470f2 100644 --- a/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathFormatDetectingAdapterMappingsSource.scala +++ b/commons/config/src/main/scala/net/shrine/config/mappings/ClasspathFormatDetectingAdapterMappingsSource.scala @@ -1,8 +1,9 @@ package net.shrine.config.mappings /** * @author clint * @date Sep 3, 2014 */ final case class ClasspathFormatDetectingAdapterMappingsSource(mappingFileName: String) - extends ClasspathAdapterMappingsSource with FormatDetectingAdapterMappingsSource \ No newline at end of file + extends ClasspathAdapterMappingsSource with FormatDetectingAdapterMappingsSource { +} \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/mappings/Csv.scala b/commons/config/src/main/scala/net/shrine/config/mappings/Csv.scala index af5bf9200..3bad05072 100644 --- a/commons/config/src/main/scala/net/shrine/config/mappings/Csv.scala +++ b/commons/config/src/main/scala/net/shrine/config/mappings/Csv.scala @@ -1,81 +1,82 @@ package net.shrine.config.mappings import au.com.bytecode.opencsv.CSVReader import java.io.Reader + import scala.collection.mutable.Buffer import scala.collection.mutable.ArrayBuffer -import scala.collection.AbstractIterator +import scala.collection.{AbstractIterator, mutable} import scala.util.control.NonFatal object Csv { private def makeCsvReader(reader: Reader): CSVReader = { //Post-reader params are: //seperator char //quote char //escape char //NB: escape char can't be the default, which is '\', since that char is all over i2b2 paths. #windowsfetish :( //Use '`' instead, as that isn't present in the Shrine ontology (that we know of) :\ new CSVReader(reader, ',', '"', '`') } def slurp(reader: Reader): Seq[(String, String)] = { val csvReader = makeCsvReader(reader) - val lines: Buffer[(String, String)] = new ArrayBuffer + val lines: mutable.Buffer[(String, String)] = new ArrayBuffer try { var rawLine = csvReader.readNext() while (rawLine != null) { val Array(shrine, i2b2) = rawLine lines += (shrine -> i2b2) rawLine = csvReader.readNext() } } finally { csvReader.close() } lines } def lazySlurp(reader: Reader): Iterator[(String, String)] = { val csvReader = makeCsvReader(reader) new AbstractIterator[(String, String)] { private[this] var lastLine = csvReader.readNext() private[this] var lineNumber = 1 override def hasNext: Boolean = lastLine != null override def next(): (String, String) = closeOnException { val result = parse(lastLine) advance() result } private[this] def advance(): Unit = { lastLine = csvReader.readNext() lineNumber += 1 if (!hasNext) { csvReader.close() } } private[this] def closeOnException[T](f: => T): T = { try { f } catch { case NonFatal(e) => csvReader.close(); throw e } } private[this] def parse(line: Array[String]): (String, String) = { require(line.size == 2, s"Line $lineNumber: Expected two 'columns' in csv line, but got $line") val Array(shrine, i2b2) = line shrine -> i2b2 } } } } \ No newline at end of file diff --git a/commons/config/src/main/scala/net/shrine/config/mappings/FileSystemAdapterMappingsSource.scala b/commons/config/src/main/scala/net/shrine/config/mappings/FileSystemAdapterMappingsSource.scala index fb892500a..b1f313474 100644 --- a/commons/config/src/main/scala/net/shrine/config/mappings/FileSystemAdapterMappingsSource.scala +++ b/commons/config/src/main/scala/net/shrine/config/mappings/FileSystemAdapterMappingsSource.scala @@ -1,22 +1,29 @@ package net.shrine.config.mappings import java.io.File import java.io.FileReader import java.io.Reader /** * @author clint * @date Mar 26, 2013 */ trait FileSystemAdapterMappingsSource extends ReaderAdapterMappingsSource { final def mappingFile: File = new File(mappingFileName) - + + final override def lastModified: Long = { + requireHelp + mappingFile.lastModified + } + //NB: Will blow up loudly if mapping file isn't found final override protected def reader: Reader = { + requireHelp + new FileReader(mappingFile) + } + + private final def requireHelp = { require(mappingFile != null) - require(mappingFile.exists, s"Couldn't find adapter mapping file '${ mappingFile.getCanonicalPath }'") - - new FileReader(mappingFile) } } \ No newline at end of file