diff --git a/tools/adapter-queries-to-qep/pom.xml b/tools/adapter-queries-to-qep/pom.xml index 94d9d842e..e3056c6ab 100644 --- a/tools/adapter-queries-to-qep/pom.xml +++ b/tools/adapter-queries-to-qep/pom.xml @@ -1,52 +1,58 @@ 4.0.0 SHRINE Copy Adapter Queries to QEP adapter-queries-to-qep jar net.shrine shrine-tools 1.21.0-SNAPSHOT net.shrine shrine-data-commons ${project.version} com.typesafe config net.shrine shrine-adapter-service ${project.version} net.shrine shrine-qep ${project.version} + + org.slf4j + slf4j-log4j12 + ${slf4j-version} + runtime + src/main/scala src/test/scala net.alchim31.maven scala-maven-plugin maven-assembly-plugin adapter-queries-to-qep-${project.version} false diff --git a/tools/adapter-queries-to-qep/src/main/examples/adapter-queries-to-qep.conf b/tools/adapter-queries-to-qep/src/main/examples/adapter-queries-to-qep.conf index 06c37972a..391983d2e 100644 --- a/tools/adapter-queries-to-qep/src/main/examples/adapter-queries-to-qep.conf +++ b/tools/adapter-queries-to-qep/src/main/examples/adapter-queries-to-qep.conf @@ -1,42 +1,34 @@ shrine { adapter { query { - database { - dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests and command-line tools, JNDI inside tomcat - testDataSource { driverClassName="com.mysql.jdbc.Driver" - url="jdbc:mysql://localhost:3306/shrine_query_history" // url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1" //H2 embedded in-memory for unit tests ;TRACE_LEVEL_SYSTEM_OUT=2 for H2's trace //optional credentials { username="shrine" - password="demouser" + password="CHANGEME" } } } } } queryEntryPoint { audit { database { - dataSourceFrom = "testDataSource" //Can be JNDI or testDataSource . Use testDataSource for tests and command-line tools, JNDI inside tomcat - testDataSource { driverClassName="com.mysql.jdbc.Driver" - url="jdbc:mysql://localhost:3306/qepAuditDB" credentials { username="shrine" - password="demouser" + password="CHANGEME" } } } } } - //breakdownResultOutputTypes {} should come from shrine.conf } \ No newline at end of file diff --git a/tools/adapter-queries-to-qep/src/main/scala/net/shrine/utilities/adapterqueriestoqep/AdapterQueriesToQep.scala b/tools/adapter-queries-to-qep/src/main/scala/net/shrine/utilities/adapterqueriestoqep/AdapterQueriesToQep.scala index 4a8ef1dee..72c39bf10 100644 --- a/tools/adapter-queries-to-qep/src/main/scala/net/shrine/utilities/adapterqueriestoqep/AdapterQueriesToQep.scala +++ b/tools/adapter-queries-to-qep/src/main/scala/net/shrine/utilities/adapterqueriestoqep/AdapterQueriesToQep.scala @@ -1,100 +1,133 @@ package net.shrine.utilities.adapterqueriestoqep import java.io.File import javax.sql.DataSource import javax.xml.datatype.XMLGregorianCalendar import com.typesafe.config.{Config, ConfigFactory} import net.shrine.adapter.dao.AdapterDao import net.shrine.adapter.dao.model.ShrineQuery import net.shrine.adapter.dao.squeryl.SquerylAdapterDao import net.shrine.audit.NetworkQueryId import net.shrine.dao.squeryl.{DataSourceSquerylInitializer, SquerylInitializer, SquerylDbAdapterSelecter} import net.shrine.adapter.dao.squeryl.tables.{Tables => AdapterTables} import net.shrine.protocol.ResultOutputTypes import net.shrine.qep.QepConfigSource import net.shrine.qep.queries.{QepQueryDb, QepQuery, QepQueryFlag} import net.shrine.slick.TestableDataSourceCreator import org.squeryl.internals.DatabaseAdapter import net.shrine.config.ConfigExtensions /** * @author dwalend * @since 1.21 */ object AdapterQueriesToQep { def main(args: Array[String]): Unit = { - if(args.length < 3) throw new IllegalArgumentException("Requires three arguments: the domain to transfer, the full path to the adapter-queries-to-qep.conf file, and the full path to the shrine.conf file.") + try { + if (args.length != 3) throw new WrongNumberOfArguments("Requires three arguments: the domain to transfer, the full path to the adapter-queries-to-qep.conf file, and the full path to the shrine.conf file.") - val domain = args(0) - val localConfig = args(1) - val shrineConfig = args(2) + val domain = args(0) + val localConfig = args(1) + val shrineConfig = args(2) - val config: Config = ConfigFactory.parseFile(new File(localConfig)).withFallback(ConfigFactory.parseFile(new File(shrineConfig))).withFallback(ConfigFactory.load()) + val config: Config = ConfigFactory.parseFile(new File(localConfig)).withFallback(ConfigFactory.parseFile(new File(shrineConfig))).withFallback(ConfigFactory.load()) - val adapterDataSource: DataSource = TestableDataSourceCreator.dataSource(config.getConfig("shrine.adapter.query.database")) - val squerylAdapter: DatabaseAdapter = SquerylDbAdapterSelecter.determineAdapter(config.getString("shrine.shrineDatabaseType")) - val squerylInitializer: SquerylInitializer = new DataSourceSquerylInitializer(adapterDataSource, squerylAdapter) + val adapterDataSource: DataSource = TestableDataSourceCreator.dataSource(config.getConfig("shrine.adapter.query.database")) + val squerylAdapter: DatabaseAdapter = SquerylDbAdapterSelecter.determineAdapter(config.getString("shrine.shrineDatabaseType")) + val squerylInitializer: SquerylInitializer = new DataSourceSquerylInitializer(adapterDataSource, squerylAdapter) - val squerylAdapterTables: AdapterTables = new AdapterTables - val breakdownTypes = config.getOptionConfigured("breakdownResultOutputTypes",ResultOutputTypes.fromConfig).getOrElse(Set.empty) + val squerylAdapterTables: AdapterTables = new AdapterTables + val breakdownTypes = config.getOptionConfigured("breakdownResultOutputTypes", ResultOutputTypes.fromConfig).getOrElse(Set.empty) - val adapterDao: AdapterDao = new SquerylAdapterDao(squerylInitializer, squerylAdapterTables)(breakdownTypes) + val adapterDao: AdapterDao = new SquerylAdapterDao(squerylInitializer, squerylAdapterTables)(breakdownTypes) - val adapterQueries: Seq[ShrineQuery] = adapterDao.findQueriesByDomain(domain) + val adapterQueries: Seq[ShrineQuery] = adapterDao.findQueriesByDomain(domain) - println(s"Found ${adapterQueries.size} queries for $domain in the adapter's table") + println(s"Found ${adapterQueries.size} queries for $domain in the adapter's table") - QepConfigSource.configForBlock(config,getClass.getSimpleName) { + QepConfigSource.configForBlock(config, getClass.getSimpleName) { - //filter out any queries that already exist - val queriesInQep: Set[NetworkQueryId] = QepQueryDb.db.selectAllQepQueries.map(_.networkId).to[Set] + //filter out any queries that already exist + val queriesInQep: Set[NetworkQueryId] = QepQueryDb.db.selectAllQepQueries.map(_.networkId).to[Set] - //turn each ShrineQuery into a QepQuery and store it - val queriesToInsert: Seq[ShrineQuery] = adapterQueries.filterNot(q => queriesInQep.contains(q.networkId)) - println(s"Will insert ${queriesToInsert.size} rows into the qep's previousQueries table. (${adapterQueries.size - queriesToInsert.size} have matching network ids.)") + //turn each ShrineQuery into a QepQuery and store it + val queriesToInsert: Seq[ShrineQuery] = adapterQueries.filterNot(q => queriesInQep.contains(q.networkId)) + println(s"Will insert ${queriesToInsert.size} rows into the qep's previousQueries table. (${adapterQueries.size - queriesToInsert.size} have matching network ids.)") - queriesToInsert.map(shrineQueryToQepQuery).foreach(QepQueryDb.db.insertQepQuery) + queriesToInsert.map(shrineQueryToQepQuery).foreach(QepQueryDb.db.insertQepQuery) - //only carry over flags for queries that don't already have flag entries - val flagsInQep: Set[NetworkQueryId] = QepQueryDb.db.selectMostRecentQepQueryFlagsFor(adapterQueries.map(q => q.networkId).to[Set]).keySet - val flagsToInsert: Seq[QepQueryFlag] = adapterQueries.filterNot(q => flagsInQep.contains(q.networkId)).flatMap(shrineQueryToQepQueryFlag) - println(s"Will insert ${flagsToInsert.size} rows into the qep's queryFlags table.") + //only carry over flags for queries that don't already have flag entries + val flagsInQep: Set[NetworkQueryId] = QepQueryDb.db.selectMostRecentQepQueryFlagsFor(adapterQueries.map(q => q.networkId).to[Set]).keySet + val flagsToInsert: Seq[QepQueryFlag] = adapterQueries.filterNot(q => flagsInQep.contains(q.networkId)).flatMap(shrineQueryToQepQueryFlag) + println(s"Will insert ${flagsToInsert.size} rows into the qep's queryFlags table.") - //make flags for each ShrineQuery that has them and store that - flagsToInsert.foreach(QepQueryDb.db.insertQepQueryFlag) + //make flags for each ShrineQuery that has them and store that + flagsToInsert.foreach(QepQueryDb.db.insertQepQueryFlag) + } + + println("previousQueries transferred from the local adapter to the QEP cache.") + } + catch { + case wnoa:WrongNumberOfArguments => + printUsage() + println(wnoa.message) + System.exit(1) + case x: Throwable => + printUsage() + x.printStackTrace() + System.exit(2) + } + + def printUsage(): Unit = { + println( + """ + |Usage: ./adapter-queries-to-qep DOMAIN ADAPTER_CONF SHRINE_CONF + |Copy previous queries from users in DOMAIN from the local shrine_query_history database to the local qepAudit database. + | + |DOMAIN is an I2B2 PM Cell domain name used by the local QEPs researchers. + |ADAPTER_CONF is a configuration file for this tool. This conf file takes precedence over the SHRINE_CONF file. + | See conf/adapter-queries-to-qep.conf for an example. + |SHRINE_CONF is the local configuration file, usually /opt/shrine/tomcat/lib/shrine.conf . + | + |Exit codes: 0 success + | 1 known error + | 2 unknown error (with an accompanying stack trace). + | + |Example: ./adapter-queries-to-qep i2b2demo conf/adapter-queries-to-qep.conf /opt/shrine/tomcat/lib/shrine.conf + """.stripMargin) } - println("previousQueries transferred from the local adapter.") + case class WrongNumberOfArguments(message:String) extends Exception(message) } def shrineQueryToQepQuery(shrineQuery: ShrineQuery):QepQuery = { val date:Long = toMillis(shrineQuery.dateCreated) new QepQuery( networkId = shrineQuery.networkId, userName = shrineQuery.username, userDomain = shrineQuery.domain, queryName = shrineQuery.name, expression = shrineQuery.queryDefinition.expr.fold("")(expression => expression.toXml.text), dateCreated = date, deleted = false, queryXml = shrineQuery.queryDefinition.toXml.text, changeDate = date ) } def shrineQueryToQepQueryFlag(shrineQuery: ShrineQuery):Option[QepQueryFlag] = { if(shrineQuery.isFlagged){ Some(new QepQueryFlag( networkQueryId = shrineQuery.networkId, flagged = shrineQuery.isFlagged, flagMessage = shrineQuery.flagMessage.getOrElse(""), changeDate = toMillis(shrineQuery.dateCreated) )) } else None } private def toMillis(xmlGc: XMLGregorianCalendar): Long = xmlGc.toGregorianCalendar.getTimeInMillis } \ No newline at end of file