diff --git a/src/client/ch/epfl/lca1/medco/I2B2MedCoCell.java b/src/client/ch/epfl/lca1/medco/I2B2MedCoCell.java index 3c087bd..5afa0a0 100755 --- a/src/client/ch/epfl/lca1/medco/I2B2MedCoCell.java +++ b/src/client/ch/epfl/lca1/medco/I2B2MedCoCell.java @@ -1,73 +1,60 @@ package ch.epfl.lca1.medco; import ch.epfl.lca1.medco.i2b2.I2B2Cell; -import ch.epfl.lca1.medco.i2b2.I2b2Status; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryRequest; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryResponse; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import ch.epfl.lca1.medco.util.Logger; import edu.harvard.i2b2.common.exception.I2B2Exception; -import edu.harvard.i2b2.common.util.axis2.ServiceClient; -import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; import edu.harvard.i2b2.crc.datavo.i2b2message.RequestMessageType; import edu.harvard.i2b2.crc.datavo.i2b2message.ResponseMessageType; -import edu.harvard.i2b2.crc.datavo.pdo.EidType; -import edu.harvard.i2b2.crc.datavo.pdo.PatientType; -import edu.harvard.i2b2.crc.datavo.pdo.PidType; -import edu.harvard.i2b2.crc.datavo.pdo.query.*; -import edu.harvard.i2b2.crc.loader.datavo.loader.query.*; -import edu.harvard.i2b2.crc.loader.datavo.loader.query.InputOptionListType; -import edu.harvard.i2b2.crc.loader.datavo.loader.query.OutputOptionListType; -import edu.harvard.i2b2.crc.loader.datavo.loader.query.OutputOptionType; import org.apache.axiom.om.OMElement; -import org.javatuples.Pair; import javax.xml.bind.JAXBElement; -import java.util.*; // todo: harmonize error handling (i2b2status + exceptions) public class I2B2MedCoCell extends I2B2Cell { private static final String URL_PATH_MEDCO_REQ = "/request"; - public I2B2MedCoCell(String medcoCellUrl, UserAuthentication auth) { + public I2B2MedCoCell(String medcoCellUrl, MedCoI2b2MessageHeader auth) { super(medcoCellUrl, auth); } /** * * @param medcoRequest * @return * @throws I2B2Exception */ public I2B2QueryResponse medcoQuery(I2B2QueryRequest medcoRequest) throws I2B2Exception { // make query request (from query definition) with a patient set result output I2B2QueryResponse parsedResp; try { // TODO: create custom service client that is used by all cells Logger.info("New request to cell " + cellURL); RequestMessageType newReqMessage = createRequestMessage(medcoRequest.getMessageBody()); OMElement reqMessageOM = msgUtil.buildOMElement(newReqMessage); String answerMessage = CustomServiceClient.sendRESTCustomTimeout(cellURL + URL_PATH_MEDCO_REQ, reqMessageOM); Logger.debug("Cell " + cellURL + " answered: " + answerMessage); // extract response JAXBElement responseJaxb = msgUtil.unMashallFromString(answerMessage); ResponseMessageType resp = (ResponseMessageType) responseJaxb.getValue(); parsedResp = new I2B2QueryResponse(resp); Logger.info("crc query request result: " + parsedResp.getI2b2Status()); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } return parsedResp; } } diff --git a/src/client/ch/epfl/lca1/medco/MedCoLoadingClient.java b/src/client/ch/epfl/lca1/medco/MedCoLoadingClient.java index 7c9bf94..899733b 100644 --- a/src/client/ch/epfl/lca1/medco/MedCoLoadingClient.java +++ b/src/client/ch/epfl/lca1/medco/MedCoLoadingClient.java @@ -1,465 +1,457 @@ package ch.epfl.lca1.medco; -import ch.epfl.lca1.medco.axis2.MedCoQueryRequestDelegate; import ch.epfl.lca1.medco.dao.MedCoDatabase; -import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryRequest; -import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryResponse; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; import ch.epfl.lca1.medco.loader.DataType; -import ch.epfl.lca1.medco.unlynx.UnlynxDecrypt; import ch.epfl.lca1.medco.util.MedCoUtil; import ch.epfl.lca1.medco.util.exceptions.UnlynxException; import com.opencsv.CSVReader; import edu.harvard.i2b2.common.exception.I2B2Exception; -import org.apache.commons.cli.*; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.javatuples.Triplet; import org.postgresql.ds.PGSimpleDataSource; -import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.util.*; /** * Created by misbach on 15.07.17. */ public class MedCoLoadingClient { public static void main(String[] args) throws InterruptedException, I2B2Exception, IOException, UnlynxException { Logger.getRootLogger().setLevel(Level.INFO); // rerun for each dataset setting: //loadStandardDataset("quarter"); //quadruple, double, normal, half, quarter //loadClearDataset("quadruple"); //System.out.println(StopWatch.misc.prettyPrint()); //loadFullGenomicClearOntology(); loadProfileManyNode(); } public static void loadFullGenomicClearOntology() throws I2B2Exception, IOException { loadSrv5Conf(); String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/"; MedCoDatabase dao = new MedCoDatabase(); CSVReader genomicReader = new CSVReader(new FileReader(dir + "data_mutations_extended_skcm_broad_clear_i2b2.txt"), '\t', '\u0000', 0); String[] genomicHeader = genomicReader.readNext(); String[] genomicEntry; int count = 0; while ((genomicEntry = genomicReader.readNext()) != null) { try { // ontology entry dao.batchSqlStatements.add( "insert into i2b2metadata.clinical_non_sensitive" + "(c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_basecode, c_facttablecolumn, c_tablename, c_columnname, c_columndatatype, c_operator, c_dimcode, update_date, valuetype_cd, m_applied_path) values(" + "'4', '\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + genomicEntry[2] + "\\', '" + genomicEntry[2] + "', 'N', 'LA', '" + genomicEntry[2] + "', 'concept_cd', 'concept_dimension', 'concept_path', 'T', 'LIKE', " + "'\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + genomicEntry[2] + "\\', now(), 'MEDCO_CLEAR', '@') ON CONFLICT DO NOTHING"); } catch (Throwable e) { System.out.println("ignoring entry"); } } dao.sqlBatchUpdate(); // concept dim entry: all at once dao.sqlUpdate("insert into i2b2demodata.concept_dimension(concept_path, concept_cd, name_char, import_date)" + "select c_fullname, c_basecode, c_name, update_date from i2b2metadata.clinical_non_sensitive where c_basecode is not null ON CONFLICT DO NOTHING;");//c_fullname like '%VARIANT_ID%' and } public static void loadClearDataset(String datasetSplit) throws UnlynxException, IOException, I2B2Exception { // change me node number loadSrv1Conf(); //StopWatch.misc.start("clear dataset loading: clinical ontology"); String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/", nodeId = "1", datasetId = "skcm_broad_clear_i2b2_part"+nodeId+"_" + datasetSplit; System.out.println("Starting loading node " + nodeId); MedCoDataLoader loader1 = new MedCoDataLoader("chuvclear" + nodeId, datasetId, "i2b2demotest", "Demo", "demo", "demouser"); loader1.loadClinicalFileOntology(dir + "data_clinical_skcm_broad.txt", '\t', '\u0000', 5, typesClinical); System.out.println("Node " + nodeId + ": ontology clinical OK"); //StopWatch.misc.stop(); //StopWatch.misc.start("clear dataset loading: clinical data"); loader1.loadClinicalFileData(dir + "data_clinical_skcm_broad.txt", '\t', '\u0000', 5, typesClinical); System.out.println("Node " + nodeId + ": clinical data OK"); // for each entry of the dataset: // generate ontology entry, generate concept dim entry generate obs fact //StopWatch.misc.stop(); //StopWatch.misc.start("clear dataset loading: variants data"); MedCoDatabase dao = new MedCoDatabase(); //CSVReader genomicReader = new CSVReader(new FileReader(dir + "data_mutations_extended_skcm_broad_clear_i2b2_part"+ nodeId + "_"+datasetSplit+".txt"), '\t', '\u0000', 0); CSVReader genomicReader = new CSVReader(new FileReader(dir + "data_mutations_extended_skcm_broad_clear_i2b2.txt"), '\t', '\u0000', 0); String[] genomicHeader = genomicReader.readNext(); // [sample_id, patient_id, variant_id] Map patientsMap = new HashMap<>(), samplesMap = new HashMap<>(); String[] genomicEntry; int count = 0; while ((genomicEntry = genomicReader.readNext()) != null) { try { // ontology entry dao.batchSqlStatements.add( "insert into i2b2metadata.clinical_non_sensitive" + "(c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_basecode, c_facttablecolumn, c_tablename, c_columnname, c_columndatatype, c_operator, c_dimcode, update_date, valuetype_cd, m_applied_path) values(" + "'4', '\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + genomicEntry[2] + "\\', '" + genomicEntry[2] + "', 'N', 'LA', '" + genomicEntry[2] + "', 'concept_cd', 'concept_dimension', 'concept_path', 'T', 'LIKE', " + "'\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + genomicEntry[2] + "\\', now(), 'MEDCO_CLEAR', '@') ON CONFLICT DO NOTHING"); // todo: remove me //dao.batchSqlStatements.add( // "insert into i2b2metadata.clinical_non_sensitive" + // "(c_hlevel, c_fullname, c_name, c_synonym_cd, c_visualattributes, c_basecode, c_facttablecolumn, c_tablename, c_columnname, c_columndatatype, c_operator, c_dimcode, update_date, valuetype_cd, m_applied_path) values(" + // "'4', '\\medco\\clinical\\nonsensitive\\VARIANT_ID\\S" + genomicEntry[2] + "\\', 'S" + genomicEntry[2] + "', 'N', 'LA', 'S" + genomicEntry[2] + "', 'concept_cd', 'concept_dimension', 'concept_path', 'T', 'LIKE', " + // "'\\medco\\clinical\\nonsensitive\\VARIANT_ID\\S" + genomicEntry[2] + "\\', now(), 'MEDCO_CLEAR', '@') ON CONFLICT DO NOTHING"); // obs fact entry int patient_id = -1, sample_id = -1; if (patientsMap.containsKey(genomicEntry[1])) { patient_id = patientsMap.get(genomicEntry[1]); } else { patient_id = dao.sqlSelectInt("select patient_num from i2b2demodata.patient_mapping where patient_ide=?", genomicEntry[1]); patientsMap.put(genomicEntry[1], patient_id); } if (samplesMap.containsKey(genomicEntry[0])) { sample_id = samplesMap.get(genomicEntry[0]); } else { sample_id = dao.sqlSelectInt("select encounter_num from i2b2demodata.encounter_mapping where encounter_ide=?", genomicEntry[0]); samplesMap.put(genomicEntry[0], sample_id); } int instanceNum=1;//count++%4;//change me for quadruple dao.batchSqlStatements.add("insert into i2b2demodata.observation_fact(encounter_num, patient_num, concept_cd, provider_id, start_date, instance_num) values(" + "'" + sample_id + "', '" + patient_id + "', '" + genomicEntry[2] + "', 'chuvclear"+nodeId+datasetSplit+"', NOW(), '"+instanceNum+"')"); //todo: remove me //instanceNum=2; //dao.batchSqlStatements.add("insert into i2b2demodata.observation_fact(encounter_num, patient_num, concept_cd, provider_id, start_date, instance_num) values(" + // "'" + sample_id + "', '" + patient_id + "', '" + genomicEntry[2] + "', 'chuvclear"+nodeId+datasetSplit+"', NOW(), '"+instanceNum+"')"); System.out.println(patient_id + " -- " + sample_id); } catch (Throwable e) { e.printStackTrace(); System.err.println("ignoring genomic entry ..."); } } //dao.exec the batchie dao.sqlBatchUpdate(); // concept dim entry: all at once dao.sqlUpdate("insert into i2b2demodata.concept_dimension(concept_path, concept_cd, name_char, import_date)" + "select c_fullname, c_basecode, c_name, update_date from i2b2metadata.clinical_non_sensitive where c_basecode is not null;");//c_fullname like '%VARIANT_ID%' and //StopWatch.misc.stop(); // loader1.loadClinicalFileOntology(dir + "data_mutations_extended_skcm_broad_clear_i2b2_part1_normal.txt", '\t', '\u0000', 0, typesGenomicClear); //System.out.println("Node " + nodeId + ": ontology clinical 2 OK"); //loader1.loadClinicalFileData(dir + "data_mutations_extended_skcm_broad_clear_i2b2_part1_normal.txt", '\t', '\u0000',0, typesGenomicClear); //System.out.println("Node " + nodeId + ": genomic data OK"); //loader1.translateIdsToNums(); //System.out.println("Node " + nodeId + ": translate OK"); } public static void loadStandardDataset(String datasetSplit) throws UnlynxException, IOException, I2B2Exception { { loadSrv1Conf(); String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/", datasetId = "skcm_broad_part1_" + datasetSplit; nodeLoading("1", datasetId, dir + "data_clinical_skcm_broad.txt", dir + "data_clinical_skcm_broad_part1.txt", dir + "data_mutations_extended_" + datasetId + ".txt"); } { loadSrv3Conf(); String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/", datasetId = "skcm_broad_part2_" + datasetSplit; nodeLoading("2", datasetId, dir + "data_clinical_skcm_broad.txt", dir + "data_clinical_skcm_broad_part2.txt", dir + "data_mutations_extended_" + datasetId + ".txt"); } { loadSrv5Conf(); String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/", datasetId = "skcm_broad_part3_" + datasetSplit; nodeLoading("3", datasetId, dir + "data_clinical_skcm_broad.txt", dir + "data_clinical_skcm_broad_part3.txt", dir + "data_mutations_extended_" + datasetId + ".txt"); } } public static void loadProfileManyNode() throws UnlynxException, IOException, I2B2Exception { loadSrv1Conf(); String datasetSplit = "normal"; String dir = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/", datasetId = "skcm_broad_part1_" + datasetSplit; nodeLoading("1", datasetId, dir + "data_clinical_skcm_broad.txt", dir + "data_clinical_skcm_broad_part1.txt", dir + "data_mutations_extended_" + datasetId + ".txt"); } public static void nodeLoading(String nodeId, String datasetId, String clinicalPath_full, String clinicalPath_part, String genomicPath_part) throws I2B2Exception, IOException, UnlynxException { System.out.println("Starting loading node " + nodeId); MedCoDataLoader loader1 = new MedCoDataLoader("chuv" + nodeId, datasetId, "medcodeployment", "Demo", "demo", "demouser"); loader1.loadClinicalFileOntology(clinicalPath_full, '\t', '\u0000', 5, typesClinical); System.out.println("Node " + nodeId + ": ontology OK"); loader1.loadClinicalFileData(clinicalPath_part, '\t', '\u0000', 0, typesClinical); System.out.println("Node " + nodeId + ": clinical data OK"); loader1.loadGenomicFile(genomicPath_part, '\t', '\u0000',0, typesGenomic); System.out.println("Node " + nodeId + ": genomic data OK"); loader1.translateIdsToNums(); System.out.println("Node " + nodeId + ": translate OK"); } protected static void loadMedCoConf(String hostname, int i2b2Port, int psqlPort, String unlynxEntryPoint) { MedCoUtil.getTestInstance().setProperty(MedCoUtil.ONTCELL_WS_URL_PROPERTIES, "http://" + hostname + ":" + i2b2Port + "/i2b2/services/OntologyService"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.FRCELL_WS_URL_PROPERTIES, "http://" + hostname + ":" + i2b2Port + "/i2b2/services/FRService"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.CRCCELL_WS_URL_PROPERTIES, "http://" + hostname + ":" + i2b2Port + "/i2b2/services/QueryToolService"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.I2B2CELLS_WS_WAITTIME_PROPERTIES, "180000"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_BINARY_PATH_PROPERTIES, "unlynxI2b2"); // assumed in bin path MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_GROUP_FILE_PATH_PROPERTIES, "/home/misbach/repositories/medco-deployment/configuration/keys/dev-3nodes-samehost/group.toml"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_DEBUG_LEVEL_PROPERTIES, "5"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_PROOFS_PROPERTIES, "0"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_ENTRY_POINT_IDX_PROPERTIES, unlynxEntryPoint); PGSimpleDataSource ds = new PGSimpleDataSource(); ds.setServerName(hostname); ds.setDatabaseName("medcodeployment"); ds.setPortNumber(psqlPort); ds.setUser("postgres"); ds.setPassword("prigen2017"); //ds.setCurrentSchema("medco_data"); MedCoUtil.getTestInstance().setDataSource(ds); } protected static void loadSrv1Conf() { loadMedCoConf("localhost", 8082, 5434, "0"); } protected static void loadSrv3Conf() { loadMedCoConf("iccluster062.iccluster.epfl.ch", 8080, 5432, "1"); } protected static void loadSrv5Conf() { loadMedCoConf("iccluster063.iccluster.epfl.ch", 8080, 5432, "2"); } private static DataType[] typesClinical = new DataType[]{ DataType.SAMPLE_ID, DataType.PATIENT_ID, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.ENC, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.ENC }, typesGenomicClear = new DataType[]{ DataType.SAMPLE_ID, DataType.PATIENT_ID, DataType.CLEAR },typesGenomic = new DataType[]{ // Hugo_Symbol Entrez_Gene_Id Center NCBI_Build Chromosome Start_Position End_Position Strand DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.CHROMOSOME, DataType.START_POS, DataType.ANNOTATION, DataType.ANNOTATION, // Variant_Classification Variant_Type Reference_Allele Tumor_Seq_Allele1 Tumor_Seq_Allele2 dbSNP_RS DataType.ANNOTATION, DataType.ANNOTATION, DataType.REF_ALLELES, DataType.ALT_ALLELES, DataType.ANNOTATION, DataType.ANNOTATION, // dbSNP_Val_Status Tumor_Sample_Barcode Matched_Norm_Sample_Barcode Match_Norm_Seq_Allele1 DataType.ANNOTATION, DataType.SAMPLE_ID, DataType.ANNOTATION, DataType.ANNOTATION, // Match_Norm_Seq_Allele2 Tumor_Validation_Allele1 Tumor_Validation_Allele2 Match_Norm_Validation_Allele1 DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Match_Norm_Validation_Allele2 Verification_Status Validation_Status Mutation_Status Sequencing_Phase DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Sequence_Source Validation_Method Score BAM_File Sequencer MA:FImpact MA:FIS MA:protein.change DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // MA:link.MSA MA:link.PDB MA:link.var Tumor_Sample_UUID Matched_Norm_Sample_UUID HGVSc HGVSp DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // HGVSp_Short Transcript_ID Exon_Number t_depth t_ref_count t_alt_count n_depth n_ref_count n_alt_count DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // all_effects Allele Gene Feature Feature_type Consequence cDNA_position CDS_position DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Protein_position Amino_acids Codons Existing_variation ALLELE_NUM DISTANCE SYMBOL SYMBOL_SOURCE DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // HGNC_ID BIOTYPE CANONICAL CCDS ENSP SWISSPROT TREMBL UNIPARC RefSeq SIFT PolyPhen EXON DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // INTRON DOMAINS GMAF AFR_MAF AMR_MAF ASN_MAF EAS_MAF EUR_MAF SAS_MAF AA_MAF EA_MAF CLIN_SIG DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // SOMATIC PUBMED MOTIF_NAME MOTIF_POS HIGH_INF_POS MOTIF_SCORE_CHANGE IMPACT PICK VARIANT_CLASS DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // TSL HGVS_OFFSET PHENO MINIMISED ExAC_AF ExAC_AF_AFR ExAC_AF_AMR ExAC_AF_EAS ExAC_AF_FIN ExAC_AF_NFE DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // ExAC_AF_OTH ExAC_AF_SAS GENE_PHENO FILTER DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION }; } diff --git a/src/client/ch/epfl/lca1/medco/MedCoQueryClient.java b/src/client/ch/epfl/lca1/medco/MedCoQueryClient.java index 3d136d7..81a0146 100644 --- a/src/client/ch/epfl/lca1/medco/MedCoQueryClient.java +++ b/src/client/ch/epfl/lca1/medco/MedCoQueryClient.java @@ -1,707 +1,718 @@ package ch.epfl.lca1.medco; import ch.epfl.lca1.medco.axis2.MedCoQueryRequestDelegate; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryRequest; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryResponse; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import ch.epfl.lca1.medco.unlynx.UnlynxDecrypt; import ch.epfl.lca1.medco.util.MedCoUtil; import com.eclipsesource.json.Json; import com.eclipsesource.json.JsonObject; import org.apache.commons.cli.*; +import org.apache.log4j.BasicConfigurator; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.javatuples.Triplet; import java.io.PrintWriter; import java.io.StringWriter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; /** * Created by misbach on 15.07.17. */ public class MedCoQueryClient { /** * Output on stderr information of the query, output on stdout the times * * @param args */ public static void main(String[] args) throws InterruptedException { // disable all logging (to control all output) - Logger.getRootLogger().setLevel(Level.INFO); + BasicConfigurator.configure(); + Logger.getRootLogger().setLevel(Level.DEBUG); // get parameters from command-line and set the client configuration CommandLine cmd = parseCli(args); String[] serversUrl = cmd.getOptionValues("server"); String queryName = cmd.getOptionValue("name"); String queryId = cmd.getOptionValue("query"); int numRepetitions; try{ numRepetitions = Integer.parseInt(cmd.getOptionValue("repetitions")); } catch (NumberFormatException e){ e.printStackTrace(); return; } String filename = cmd.getOptionValue("filename"); String username = cmd.getOptionValue("username"); String password = cmd.getOptionValue("password"); String domain = cmd.getOptionValue("domain"); String projectId = cmd.getOptionValue("project"); setClientConfig(cmd.getOptionValue("unlynxBinary"), cmd.getOptionValue("unlynxGroupFile")); // initialize timers ClientTimers ts = new ClientTimers(); final Map timesJsonOutput = new ConcurrentHashMap<>(); for (int rep = 0; rep < numRepetitions; rep++) { final int repFinal = rep; // generate request - UserAuthentication auth = new UserAuthentication(domain, projectId, username, false, 0, password); + MedCoI2b2MessageHeader auth = new MedCoI2b2MessageHeader(domain, projectId, username, false, 0, password); List> parsedQuery = parseQuery(Integer.parseInt(queryId)); I2B2QueryRequest request = new I2B2QueryRequest(auth); request.setQueryDefinition(queryName, parsedQuery); // make request to every specified servers in a thread final UnlynxDecrypt decrypt = new UnlynxDecrypt(); Thread[] queryThreads = new Thread[serversUrl.length]; for (int i = 0; i < serversUrl.length; i++) { final int i_cpy = i; queryThreads[i] = new Thread(() -> { try { I2B2MedCoCell medCoCell = new I2B2MedCoCell(serversUrl[i_cpy], auth); I2B2QueryResponse response = medCoCell.medcoQuery(request); Triplet results = response.getQueryResults(); timesJsonOutput.put(i_cpy, results.getValue2()); System.err.println("Query results for node " + serversUrl[i_cpy] + ", id " + i_cpy + ", iter " + repFinal + ":\n" + " - pub key used: " + results.getValue0() + "\n" + " - enc result: " + results.getValue1() + "\n" + " - dec result: " + decrypt.decryptInt(results.getValue1(), MedCoQueryRequestDelegate.clientSeckey) + "\n" + " - times:" + results.getValue2() + "\n"); } catch (Exception e) { e.printStackTrace(); } }); } for (Thread queryThread : queryThreads) { queryThread.start(); } for (Thread queryThread : queryThreads) { queryThread.join(); } // output in stdout the times double tmp1, tmp2, tmp3, tmp4; tmp1 = tmp2 = tmp3 = tmp4 = 0; for (Map.Entry result : timesJsonOutput.entrySet()) { System.out.println("{\"" + result.getKey() + "\":" + result.getValue() + "}"); JsonObject jsonResult = Json.parse(result.getValue()).asObject(); ts.tt += jsonResult.getInt("Overall (axis2 in/out)", 0); ts.qpi2b2 += jsonResult.getInt("Query parsing/splitting", 0); ts.clearqueryi2b2 += jsonResult.getInt("Clear query: i2b2 query", 0); ts.psretrieval += jsonResult.getInt("Clear query: patient set retrieval", 0); ts.eqp += jsonResult.getInt("Patient set encrypted data retrieval", 0); ts.utt += jsonResult.getInt("Unlynx query", 0); ts.uet += jsonResult.getInt("Unlynx execution time", 0); ts.uct += jsonResult.getInt("Unlynx communication time", 0); ts.qpunlynx += jsonResult.getInt("Parsing time", 0); ts.broadcast += jsonResult.getInt("Broadcasting time", 0); ts.ddtqet += jsonResult.getInt("DDT Query execution time", 0); if (jsonResult.getInt("DDT Query communication time", 0) > tmp1) tmp1 = jsonResult.getInt("DDT Query communication time", 0); ts.ddtdet += jsonResult.getInt("DDT Data execution time", 0); if (jsonResult.getInt("DDT Data communication time", 0) > tmp2) tmp2 = jsonResult.getInt("DDT Data communication time", 0); ts.aet += jsonResult.getInt("Aggregation time", 0); ts.set += jsonResult.getInt("Shuffling execution time", 0); if (jsonResult.getInt("Shuffling communication time", 0) > tmp3) tmp3 = jsonResult.getInt("Shuffling communication time", 0); ts.kset += jsonResult.getInt("Key Switching execution time", 0); if (jsonResult.getInt("Key Switching communication time", 0) > tmp4) tmp4 = jsonResult.getInt("Key Switching communication time", 0); } ts.ddtqct += tmp1; ts.ddtdct += tmp2; ts.sct += tmp3; ts.ksct += tmp4; System.out.flush(); } // 1000 is to convert ms to secs ts.Divide(timesJsonOutput.size(), 1000*numRepetitions); // todo: client pub / priv keys --> hardcoded in axis2 service // store average times (in seconds) in the output file storeTimers(filename, ts); } /** * store timers in an output file (to ease out simulations) */ private static void storeTimers(String filename, ClientTimers ts){ try{ PrintWriter writer = new PrintWriter(filename, "UTF-8"); writer.println("QUERY: "); writer.println("Overall time: " + Double.toString(ts.tt)); writer.println(""); writer.println("Query parsing i2b2: " + Double.toString(ts.qpi2b2)); writer.println("Clear query i2b2: " + Double.toString(ts.clearqueryi2b2)); writer.println("Patient set retrieval: " + Double.toString(ts.psretrieval)); writer.println(""); writer.println("Encrypted Query preparation: " + Double.toString(ts.eqp)); writer.println(""); writer.println("Unlynx overall time: " + Double.toString(ts.utt)); writer.println("Unlynx total execution time: " + Double.toString(ts.uet)); writer.println("Unlynx total communication time: " + Double.toString(ts.uct)); writer.println(""); writer.println("Query/Data parsing Unlynx: " + Double.toString(ts.qpunlynx)); writer.println("Total broadcast time (unlynx client -> unlynx server): " + Double.toString(ts.broadcast)); writer.println(""); writer.println("DDT Query (execution time): " + Double.toString(ts.ddtqet)); writer.println("DDT Query (communication time): " + Double.toString(ts.ddtqct)); writer.println(""); writer.println("DDT Data (execution time): " + Double.toString(ts.ddtdet)); writer.println("DDT Data (communication time): " + Double.toString(ts.ddtdct)); writer.println(""); writer.println("Aggregation (execution time): " + Double.toString(ts.aet)); writer.println(""); writer.println("Shuffling (execution time): " + Double.toString(ts.set)); writer.println("Shuffling (communication time): " + Double.toString(ts.sct)); writer.println(""); writer.println("Key Switching (execution time): " + Double.toString(ts.kset)); writer.println("Key Switching (communication time): " + Double.toString(ts.ksct)); writer.close(); } catch (Exception e) { e.printStackTrace(); } } /** * parse query, ex: X OR Y AND F AND Z OR H ==> (X OR Y) AND (F) AND (Z OR H) */ private static List> parseQuery(int queryId) { List> parsedQuery = new ArrayList<>(); String[] orTerms = getQueryUseCase(queryId).split(" AND "); for (String orTerm : orTerms) { String[] terms = orTerm.split(" OR "); List orTermsList = new ArrayList<>(); Collections.addAll(orTermsList, terms); parsedQuery.add(orTermsList); } return parsedQuery; } private static void setClientConfig(String unlynxBinPath, String groupFilePath) { MedCoUtil.getInstance().setProperty("medco.i2b2.waittimems", "600000"); // 10m timeout MedCoUtil.getInstance().setProperty("medco.unlynx.binarypath", unlynxBinPath); MedCoUtil.getInstance().setProperty("medco.unlynx.debuglevel", "0"); MedCoUtil.getInstance().setProperty("medco.unlynx.proofs", "0"); MedCoUtil.getInstance().setProperty("medco.unlynx.groupfilepath", groupFilePath); } // command line parsing private static CommandLine parseCli(String[] args) { Options options = new Options(); Option serverInput = new Option("s", "server", true, "MedCo cell server URL"); serverInput.setRequired(true); options.addOption(serverInput); Option nameInput = new Option("n", "name", true, "Query name"); nameInput.setRequired(true); options.addOption(nameInput); Option queryInput = new Option("q", "query", true, "The query in i2b2 format: OR groups separated by AND"); queryInput.setRequired(true); options.addOption(queryInput); Option repetitionsInput = new Option("r", "repetitions", true, "Number of repetitions for a given query"); repetitionsInput.setRequired(true); options.addOption(repetitionsInput); Option filenameInput = new Option("f", "filename", true, "Output filename"); filenameInput.setRequired(true); options.addOption(filenameInput); Option usernameInput = new Option("u", "username", true, "The login username"); usernameInput.setRequired(true); options.addOption(usernameInput); Option pwInput = new Option("w", "password", true, "The login password"); pwInput.setRequired(true); options.addOption(pwInput); Option domainInput = new Option("d", "domain", true, "The login domain"); domainInput.setRequired(true); options.addOption(domainInput); Option projectInput = new Option("p", "project", true, "The login project id"); projectInput.setRequired(true); options.addOption(projectInput); Option unlynxBinInput = new Option("b", "unlynxBinary", true, "Path to the unlynx binary"); unlynxBinInput.setRequired(true); options.addOption(unlynxBinInput); Option unlynxGroupFileInput = new Option("g", "unlynxGroupFile", true, "Path to the unlynx group file (servers public keys)"); unlynxGroupFileInput.setRequired(true); options.addOption(unlynxGroupFileInput); CommandLineParser parser = new GnuParser(); HelpFormatter formatter = new HelpFormatter(); CommandLine cmd; try { cmd = parser.parse(options, args); } catch (ParseException e) { System.out.println(e.getMessage()); formatter.printHelp("medcoClient", options); System.exit(1); return null; } return cmd; } private static String getQueryUseCase(int nb) { switch (nb) { + + case 100: // use-case medco-normal + return + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + + " AND " + + + "\\\\SENSITIVE_TAGGED\\medco\\encrypted\\/98y5inj97O+26HXW8fJnbHDH0CCohmlCYNgMfgJ2mzufKVl8PBffruVGm1C05tqWxrXKPNF9AMghe8ELmNmzA==\\"; + case 1: // use-case 2 encrypted encrypted return // clear query terms to retrieve all dataset "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + // skin -> MEDCO_ENC:4 ; cut melanoma: MEDCO_ENC:6 "MEDCO_ENC:jGKcRJrrJSnDmCprGdelcW/rmD8YuhK8C1E/GDTkkboLzZgA6FSuyL+LkZi1Is0wXmPrA5UAhjJ8DwPBKA6ZeA==" + " AND " + "MEDCO_ENC:vs+rMPIpUv3v998nHZPfuhNo7RfD6a5WJH2VIg8ETZLnap1k5B5oCrV7tZelOUK2RmtpBEk9GvJSKBKo4hSZ/A==" + " AND " + // variants ids for [Hugo_Symbol=BRAF AND Protein_Position=600/766] "MEDCO_GEN:ZnMBnoLz8TfFfSoZyV0pvz2rIkbaVMauxnGkZBsUMEUDz83Q4gWV5oxdU18ubelJfaqFlX7e6yqf7Eo7dfnQaw== OR " + "MEDCO_GEN:0TprAeIFodXk9jJN+BC4+MzAP+XRsv3BflcrgO5b3WBj81ajoLxyHmuPP05MTUmve2A0DS4/hKvOKb76rjTiMA== OR " + "MEDCO_GEN:3si3xWkTxIv7TnIdPVUG7GBDZXtGQSrsR8DC7ND/aTpCEaE/QPaW5qykPFreFDVF5hzGGBFbYKrosM3z0jQwew== OR " + "MEDCO_GEN:QvT5VI9Jk9u3kMIZDEp+Yzl5TzEEIgQ1mWjJn9kQMlP+d/tLI0nkjaq7fBSBnP3h45rsijtD1IzqTT5fPBc50A=="; case 2: // use-case 2 encrypted encrypted return // clear query terms to retrieve all dataset "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + // skin -> MEDCO_ENC:4 ; cut melanoma: MEDCO_ENC:6 "MEDCO_ENC:a23H2wyfoA30NJXIcSybAotf6b+ApUo2v0YJpqHz21ZjOpaswVvQgqfJL105BDExHWuQlkPFEuQ3lvgYALryeg==" + " AND " + "MEDCO_ENC:xe0zFWnw5x1VmMfltC+QOU09Ecnm35V8QS8/kDSISimdi4LQ8o1ZNh1Ic4W2rH2e8aIkCMZJtzTR7GxyreGIAg==" + " AND " + // variants ids for [Hugo_Symbol=BRAF] UPDATE ME "MEDCO_GEN:D44NedbMM08x5dc2r1DzyT1rAd4JkLW3d1S5RVs746pm28W/vemwlABDAtuCp4gevhIzMqnmh18vMYhHzr1zxw== OR " + "MEDCO_GEN:cdKFuPfwe5TgRc1UNesh2lrkhpkDlJH2/zRqIR0KvLnirK5k3gPANs+5coCAmyu1bz9v9V8K4akLbnIQ0kzwvg== OR " + "MEDCO_GEN:LwMtilcXN9GUVqV+QClI901NJZIXPe+nQov7CiN0yLBmU1Fy2zZ5zsN4g/0srGnMT1IZxE4OmNcMA6P4S3FLVg== OR " + "MEDCO_GEN:ZXxNMgjtIZT7nDAnGYyw9oZRKSROn/q+k3n2c32QJnr+/SymYMQJkfNpp05BpIsyigV55J16LWOarrwittCRQQ== OR " + "MEDCO_GEN:hnPltOs9qCGLSzDamuM49IU+Zu8JvOxQ8Z9m3bBH8yNb63z6TYbzL0C/Xm+2bNLHs6gR1tf9pJpYMbWJmfoQ0g== OR " + "MEDCO_GEN:uHoRSX2jTv+KkOymknNgCk+IHjyJNfGJSEjpSHYuLW5+fSTYZKznwnx4HgjmOzI+p5zcUg1sIkty3SJXGZB/HQ== OR " + "MEDCO_GEN:pxYjFNgUMXwZfpnnaxmZHQ3ImXYyJ5HmkD4ipNW2Psph6r2AOYZWZAQ90vSENaOKB67pXd8rmPizUaf32cy+Og== OR " + "MEDCO_GEN:f5coQKWJIWwNsb+/D/LF1PyNh4doqsbwyRgKbAX23K0VpDr7BQQlo+NDhm7TC7RSauz2m1r9DzGdMgGoREQyfg== OR " + "MEDCO_GEN:aOgmh0MjCaw4unBcB3y3bS6BdLxPiiY5S2CChGkWn49yxUm8jHZtOChWaYHmkgOYfK8sNlULVEL+7hq98g+u+g== OR " + "MEDCO_GEN:EjMWyWOEt6dcGpqBHNLPokJHTDwVGlmLLCNJrIvhnX86KEbmbV1hL2H4MxBRan+aKS1/CNY+z7+VSGrHj35dvA== OR " + "MEDCO_GEN:p/xNDZxug7nZ3DendKeaIPYfH64Y7OF25hM83l64Wz6v+4iNXg39ZjOIAZJCp21SJcZjFzQMdf5bxaWy9Z5zxw== OR " + "MEDCO_GEN:wdEJNZOFEM51BKYyTs6SOJCMCvCu6wV9WxYRkAfPq4tCGCklQdiT+qWMClFDlcn9gIMm1VchPQCWlIRDqob8/g== OR " + "MEDCO_GEN:JhS0oP28wbDiW/w9PQ1ZFC0eLiHJHx6n16fIWFHzdD8AzeYjs9xDvbRkR77sTakF4qUjb9H560fZWhAp42miyQ== OR " + "MEDCO_GEN:DCmwF6wL93kvgAwI1CH4YD9p25DPkiOFDgB+Wz1CdrFlqFLYLPtiHUjCgRNdGmRZ6TG8vIcqR87IcHtIxfoOTQ== OR " + "MEDCO_GEN:s37uFhzZDq9Z+9qKQu3SC17jIzMtLRhq3OaJlrvZwrfscdJNR+ddQWsLINWTnamBID7S4P4ziJsrn3ghx+TMnA== OR " + "MEDCO_GEN:e3YQ/riWgdK54UDKyWystehgF4ln59lT+wecIQinA0qS1dpnkfoqAYIVx3yIp9S4Wb7M24GcAwimLTuu+RxNaw== OR " + "MEDCO_GEN:LTUxRns5SeEqruXDhIVhk8T4U7AyDtEmMiLbJnJ81FvfkeWHBg5KUYvbBuaGTzf4ndZQ7W7OH8rRB3vibZDeMw== OR " + "MEDCO_GEN:hxhUEb80v7qeWjCGT5rCZ1DYCP8y89bX13lVhifDNOdi/d/VsB+qu93y0fQJEVFk5VCU6he5jXkEZLoef5L3uw== OR " + "MEDCO_GEN:dYrlZfdk8wlJqxsB32IoyPdjwXqkbsi3Co48GlBwS795J62+BqD1xXVLuiOylGnnjtxiMtfPP18TjIVZuFF3aA== OR " + "MEDCO_GEN:hF2A4LW4nL521qn10yNObIEqvFYinm3XyoFRDXSkKSWgi9eHykdVywm461FecDZZC/3f1Y+pFZ++1PHRB1Qftw== OR " + "MEDCO_GEN:2QXAN9BRmiOL0q2eRo1JbqMQSXMNGnOFqt8mgrCuPXBJn2C/aVx1UAMBF2DboRDIfdx7YjKC8mbjUKvexzGYow== OR " + "MEDCO_GEN:n0hJayzg6aQqSyeUf85WHjG5AlKbIqeu/VWQFVvHLNbK5Svq/Q6duaP3Wn/9iwBTWjA/nrPz0vXdOzIQToV0GQ== OR " + "MEDCO_GEN:rk7d9KnFyY3+y8t3C54yApN2T4MMwtEGEhydLV9kALwbw6XvGsSlM/B52GzVUuSkwjuYVg5wMxFXcKXk99NUSw== OR " + "MEDCO_GEN:rccYa/K6+AC6tJ8y+Py+aX4TJRBx0eKVWQhvGQEhLfdINqjRxkAFG6QbIBifCSqFWAKr1eI2QOs286SdFX/btA== OR " + "MEDCO_GEN:hUkpNGCHdMjE5/NLRl6QvJJLCQQ7PV4epnKPYbko4fgi/NyR27cmAt/oIy19oCOqp3hnxe1aHZXVn40Idq7+sA==" + " AND " + "MEDCO_GEN:hsI74szdwDidAOwHJiTy8HHBozqmuij52L+veJ+IhqyO2wwRBNRUQxxzH4VyEUPOxJ0Zg0enQwt+0/oX4X1cAA== OR " + "MEDCO_GEN:bYC1en7ce/yDbFK0NEXcINxscePn3tzOop0Ntzue1LZpR0RM7OKi46AUiZS8lT0pJBobyowuqsXg1EciXyS9Wg== OR " + "MEDCO_GEN:WD3hdnirdqqqMVzKI+XarDB5BgM5lzyYGoGogagghI1/LIHLTbw3uxNrGb+3d9Udj0lWXl3b8xlWuJesTcV9nA== OR " + "MEDCO_GEN:eRX3hSgsitWpwCa29/3u1nso+NyE20HwisNRmaFNAGp3AFfr4BjJMVJ1jZ9U6s8f22a8Vesub7XvXL76shmbIg== OR " + "MEDCO_GEN:CSeUe8thcSVS6SZt15C5D81VArJ7UNBIvOhsx9Ex2kJ4cwEOYzOnexTAKw4i8zwkz+xXAtSzIyH/KQfqzfwDvQ== OR " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR " + "MEDCO_GEN:HjY+NM9pr5ch6dUmmwJRMzEMFThtgT8nt91A98KDMm+lG10qILx7YcYFFMUwutQJQ9GVoFDRnd09GwynpIGB2A== OR " + "MEDCO_GEN:E3hzRYM/x12Ed89lI0TT5fQUo6xseTTp4Pu3++Nd+n3H2zsWlnqg6dpNSMZtud/YtzVgCQ7zRZWfUzZYU+u5jw== OR " + "MEDCO_GEN:uYMtnHnG+oc9hrGRuRwzDyX58ONnG1mCRfH7+suvw7DWQ4hSwxLSBk/2DjNMd57vYIFU224xw2cjbxx1kyOPbw== OR " + "MEDCO_GEN:+sNzExEMRWy7DI8c6aUzExs+H2qv5MXqCHAPcOf9DGrpTX1c5L8e5buyGJG2tIf0rQWnsDNMwRDpJc7Gpip/PA== OR " + "MEDCO_GEN:c9vzOssmq5msrUbWHU/zk2gtjDRPowUra2NNlrKbqbyV5N23eTR/gUoVzk674bgsiO6zL5fAWZiDJbD3NQ3CJA== OR " + "MEDCO_GEN:Qu+iZEppFtco3VtrOGC3dkxpNh57bTRdA8IQ/q/9YXScXXCVu5F9yqbz3xnk041T1OGFR4cPZBEN/pNAtMgzKw== OR " + "MEDCO_GEN:HFOBzYw3aPS1q804wKKioQNwCyRVl7MGu0dHGe6WdNhhsujWF5yYy9Db1Cmi3xfqeOGfEJcVbrq5+7eok5rxLw== OR " + "MEDCO_GEN:hntXZLRcTCX1daL9N73eINjIxrVJyNPSgVAPuZc2ts/w1G81PkpR09d6YRriX7pJdlXXx+gtIJb/DS7moyHcqQ== OR " + "MEDCO_GEN:KHveMaISTYiYTdNSPIOMafz7Ddk1kd7vtyICgvjcaFQwF16SUxUcxCsmE9ylSjaNiCmec0TOR5NOINnEHGm+3Q== OR " + "MEDCO_GEN:adxS2akG/2X93CbnPv0ifMnhjv/gAFKTD0sRhXe/wdmukPiJltlxZm9Gq5po/J6LopxNCPXb//tePx+WbCGq1w== OR " + "MEDCO_GEN:Y5Q9i2RWtx/xUAN8Mat4Gvv7vFxgxjJHuiPkPtYaM+iK7d209jFxNSQU2YpiQmK2fk23w/mymR+QhsMy1fIxFw== OR " + "MEDCO_GEN:/8lX/KIhRuXZ4mtZh+/4Cmgmat+PbEPP93fxs1OCOa1dVlKZZ7Yji/E2bAXzFhhXUYaQidAq0LTR9XP6RuAA/g== OR " + "MEDCO_GEN:QvJbptudB9Y6GVL72gv2LNVmP/AfxEzens6o+Kv4hPDnlaSx6rw2N/Zb0UNB5lIAqjVVnlvIIX9ScqMlLy7H6Q== OR " + "MEDCO_GEN:cMrETOs1qwdQ/ayz+jVWbryQLDJbcJMymDo/fHq/Kyw96XsBqdWuTDN98a9czLhxKnZqWQKaEvQsItlEk3vtsA== OR " + "MEDCO_GEN:nIBSBZq6+RMVOF1/azmILuiyP5yXCoiwB0hbnu0+RXbAtKMXBbufoSggtw8BUOSmEAkkicpScMjw04RUkBTbpA== OR " + "MEDCO_GEN:9Fb9EuGOgJuXjEMZtOreJdQW90U7wS6P07TC5h+LQis34WIqZVTqYB4B+p112GS3I58hIUqCa9k3G8PjeiRZTg== OR " + "MEDCO_GEN:xnOouQ/1cwcIYkkqdq80zHBmG/pOnk7dT9fYWWxCkVDsLDbjslQp+Dk//KNEBgC+7YPXEEY90LxpUdDXXzHG4g== OR " + "MEDCO_GEN:N0bGEVQAVb/AM8R/XKhz52Rlfe0U41DQBtqgTo9/VHp/jNwOjVI9G5Cw+31HgQv/Di+Tz38XXaBjirC3eRbWWA== OR " + "MEDCO_GEN:T9sTYaf1NmbXRKIc+Zi72ESQzlKSDy481B4hq4WtpQ5NYmNr0E0k/ePaLzye8BBJxHFy6bY+aSEWyukJgJxf5Q== OR " + "MEDCO_GEN:gAE6wiUw4cxTo88dVnfJuO6vhUsVBU7TyE5Vvnajz9uJAE2PwmY9GyalVuOvKtDb49VRLS6x205+PFOvDXJsag== OR " + "MEDCO_GEN:pVAuBBBJqNsHYetWBcjQa5x/ZISP+LTSu3lQConTCh3Jt/PohQnlkLe/Kr+xCDWCGVo/jfgz61ouXcj6ixgZsQ== OR " + "MEDCO_GEN:uESdfo/ULrcav5efi4Pq/WV0wM9/xKaAGHm2Oui4v3xcFT5ilUYvtwizmKCq+rdVwE5pvA4Fso3DJt8hZzBSuA== OR " + "MEDCO_GEN:tr8zvS4UC9uzSv6YmonwE6dUJifZAQzurPr/Bpwyc2+rkC8NeE8wR6SxG1WJ939vRkIAlRUdiUDwDFYbx3ZpIA== OR " + "MEDCO_GEN:hMzOJZ6Yvlu/II7IvZf9dcqJvBSdvHY+I+JEQvvTc5S7zAJ9BkD+lnQPbLUASOJ6kI2pGPhkC/JXyTLSg5K0Eg== OR " + "MEDCO_GEN:mOciI5Tc+eXsmNHFhxo/LXTyPN2gUeYHKcbYz3EHhuR6/Tb18WM3d/cUyJBzrBjUEyINyNX4TEP5lVxRr92+ZA== OR " + "MEDCO_GEN:mN/ss8CHskOgizEjTl8RdUEhdP1w+SRHUMP9RPCMOyCO3TIPyBSUNLGNDCDGx599dNqvjVRyP/QD4JxjL/7F9Q== OR " + "MEDCO_GEN:ZFjzyMBPvyS+89bZW4BMmmlb/3yVYN0OtbHKAx74J5O45dnQ2DHNEJfUIrX/1bWUfhDZ3uH1ZKaq3xt6tPDmfg== OR " + "MEDCO_GEN:qyPFS8DdLNU0AvSI5VjH2O0FHrCaUAe8TPSbWiu9+aeGHuhqMHLQwiGCx7b7yII7FGsCE5ToWO7n5r2uUhOk5w== OR " + "MEDCO_GEN:GssT+XY8vNmpWew/FpyAvtP9JHCUt7KyYAQxqsf2scc5XgxZ9rDePLnn05AUbdURex1wyP/03AmUIY2IAB8cZg== OR " + "MEDCO_GEN:8DzmmlUEgYe69zYNrHSpRf7QBfAUjJHwv9x6H1jvifQeimZmQqIbQy9u6zTNQ84meMBr/dKQdI1UxbBtDEgLxA== OR " + "MEDCO_GEN:Vj1F94YSanP6TgY7WJr/OFAA3gxLfyARagxkw0Szr4DmqCRR0a3gfczdFSDnR1BkdVkiGSkMcR+lL7YZi9yOQg== OR " + "MEDCO_GEN:FNGWJv3hlyD+D/bHuT8jlrKanPYH5kAFCDvvJVSiYM2PiRzZ5YpndbNQ9nbwiCuCQEGXMqugiH06cJgkyiPoYw== OR " + "MEDCO_GEN:E9WpRiE/woD+jAhPN1EfiTgyFV1edXm2Pr858JO/4WIhT7BiErYW+lfy5TEFNq0lypFISUOaZPw/jHaiqQwPqw== OR " + "MEDCO_GEN:e+ToN0Z665neUv5RmPK2wQW14bD9oXqoucqtgIkHOg7GR4ULrOsQh8qnzd/IN5mmQY/YdGz1hoc5wrai4ebNaQ== OR " + "MEDCO_GEN:buxxNxhfLZuIZh15RVAvCMBtXCNqPIxozOkyiOM/4Bu0FvhpnQeE8gR+uwg0qtN1OiMXHliRhKNoqcXK90eOGQ== OR " + "MEDCO_GEN:sWIJpNie8wFBxvSmbelIQmCH5Ky66h92xprHFi46N83K2mgdb5TZD4lmVojj3jWTyzoUMJoq5jRzdr4NrRIwxw== OR " + "MEDCO_GEN:rHumWgTO53thX0syMAPtd98clhxV2nqUgHoqOOkzqOHZg0GB6JK3EhaGD/PWyQhFyRfhXohD8ISMFUPfNe2Teg== OR " + "MEDCO_GEN:TexkrOStADoEc//+tazhkXAy3WXvKmzX1mwKtsUK3YR8UHLA8IsO6XTRSZcceD2l7CeCq09q915x/TkjA7T5XQ== OR " + "MEDCO_GEN:TGwWbEnnl5SKR0g53EMcSJW+Ae0jCAuHd4rH5+4ZKGO2Wt4YNCPhzVuB8TT7340eQKuPxmtW/j0C+w1dC2HJxQ== OR " + "MEDCO_GEN:LHCYuvig8PYpfrzCrS8nnruWPDNLUGHHAqjC/U5L7L2mOh+b8UVXegG0tVadmddccvv0tdfHLPUys2qfvTqMSQ== OR " + "MEDCO_GEN:zzKtD2/ywSQ5f28ikVIOh2Ir7yZR2UfBUY0jW5VLyXcj0VAa1WpgihNrDmSWqeaRvOk4Csvj61CfgJ6AQmrqEQ== OR " + "MEDCO_GEN:MEBPXmSKjcnwkNYgywNQVMMmWnrysh2v5Ig1h/KCPZgAOQfpjVsaAqOHTrGZ6OL/Fxf95lAGubfhtdVB51qclA== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:qM8KzlNN+oA+TfqShe5oqaDuHphZfw/n6p+xr2bmsMFGA7J+Gwax0ZZcFgyEIiL3FrfMPNTGsz0WxJe8iLhTCQ=="; case 3: return // one genomic term test // clear query terms to retrieve all dataset "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_GEN:k2EKXqzZ9c672FIoD/2YJZt9x3NvC2I0/A056DMJYrdkF8MJURvsGhIf8KxaccuJj+ApPlLYpn2tV/gIODGDjg=="; case 4: // use-case 1 in clear return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997223410688\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997064022784\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997064020736\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948998062267136\\"; case 5: //use-case 2 in clear return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054904932905773056\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054898625779855360\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054898626853597184\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054968999892742144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054922546600210432\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054904954414166016\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054923379857424384\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054923381098933248\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054918645662608384\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054905185268658176\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054923607457132544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054861517262417920\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948987408734208\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997223410688\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997064022784\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948997064020736\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948998062267136\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948999337337856\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054861823278837760\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054953138544961536\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054904773018905600\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054917142457610240\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054949048695910400\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054861692282335232\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054948050082458624\\" + " AND " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244731450050998272\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761148709661696\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828257969198066688\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742591331743035392\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244734719527742464\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761165822422016\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828266574613577728\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244734599302213632\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244734615609652992\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828258079793473536\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742581279372079104\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828214681296432128\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742593391179853824\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244731379150482432\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707619996790784\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707607111888896\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707450345581568\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707451419324416\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742593229011282944\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707438500867072\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742581136564416512\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828266483970731008\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742593253707344896\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828266501150601216\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828266502224342016\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707399913269248\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707503026037760\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707487054123520\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707493295259648\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244803225329855488\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244835122005142528\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-3742595392634613760\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828266596713621504\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707304132149248\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761290443582464\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-4828218249340513280\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761261385445376\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707269956960256\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761257090477056\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244761331212216320\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703514008055808\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707363372498944\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560164864\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560165888\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519225765120\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519376764928\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244803244478955520\\"; case 6: // i2b2 clear: 5 attributes return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054904932905773056\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054898625779855360\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054898626853597184\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054968999892742144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-7054922546600210432\\"; case 7: // i2b2 clear:10 attributes return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560164864\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560165888\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519225765120\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519376764928\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6244803244478955520\\"; case 8: // i2b2 clear: 100 attributes StringWriter sw = new StringWriter(); for (int i = 0 ; i < 10 ; i++) { sw.append( "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560164864\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560165888\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519225765120\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519376764928\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\ OR "); } sw.append( "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560164864\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560165888\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519225765120\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519376764928\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\"); return sw.toString(); case 9: // i2b2 clear: 500 attributes StringWriter sw2 = new StringWriter(); for (int i = 0 ; i < 55 ; i++) { sw2.append( "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560164864\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707366560165888\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519225765120\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605703519376764928\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\ OR "); } sw2.append( "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707347467692032\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707351594886144\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340823916544\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707339750174720\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\-6605707340672917504\\"); return sw2.toString(); case 10: // medco: 5 attributes return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR " + "MEDCO_GEN:HjY+NM9pr5ch6dUmmwJRMzEMFThtgT8nt91A98KDMm+lG10qILx7YcYFFMUwutQJQ9GVoFDRnd09GwynpIGB2A=="; case 11: // medco:10 attributes return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_GEN:hsI74szdwDidAOwHJiTy8HHBozqmuij52L+veJ+IhqyO2wwRBNRUQxxzH4VyEUPOxJ0Zg0enQwt+0/oX4X1cAA== OR " + "MEDCO_GEN:bYC1en7ce/yDbFK0NEXcINxscePn3tzOop0Ntzue1LZpR0RM7OKi46AUiZS8lT0pJBobyowuqsXg1EciXyS9Wg== OR " + "MEDCO_GEN:WD3hdnirdqqqMVzKI+XarDB5BgM5lzyYGoGogagghI1/LIHLTbw3uxNrGb+3d9Udj0lWXl3b8xlWuJesTcV9nA== OR " + "MEDCO_GEN:eRX3hSgsitWpwCa29/3u1nso+NyE20HwisNRmaFNAGp3AFfr4BjJMVJ1jZ9U6s8f22a8Vesub7XvXL76shmbIg== OR " + "MEDCO_GEN:CSeUe8thcSVS6SZt15C5D81VArJ7UNBIvOhsx9Ex2kJ4cwEOYzOnexTAKw4i8zwkz+xXAtSzIyH/KQfqzfwDvQ== OR " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR " + "MEDCO_GEN:HjY+NM9pr5ch6dUmmwJRMzEMFThtgT8nt91A98KDMm+lG10qILx7YcYFFMUwutQJQ9GVoFDRnd09GwynpIGB2A=="; case 12: // medco: 100 attributes StringWriter sw3 = new StringWriter(); sw3.append("\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND "); for (int i = 0 ; i < 10 ; i++) { sw3.append( "MEDCO_GEN:hsI74szdwDidAOwHJiTy8HHBozqmuij52L+veJ+IhqyO2wwRBNRUQxxzH4VyEUPOxJ0Zg0enQwt+0/oX4X1cAA== OR " + "MEDCO_GEN:bYC1en7ce/yDbFK0NEXcINxscePn3tzOop0Ntzue1LZpR0RM7OKi46AUiZS8lT0pJBobyowuqsXg1EciXyS9Wg== OR " + "MEDCO_GEN:WD3hdnirdqqqMVzKI+XarDB5BgM5lzyYGoGogagghI1/LIHLTbw3uxNrGb+3d9Udj0lWXl3b8xlWuJesTcV9nA== OR " + "MEDCO_GEN:eRX3hSgsitWpwCa29/3u1nso+NyE20HwisNRmaFNAGp3AFfr4BjJMVJ1jZ9U6s8f22a8Vesub7XvXL76shmbIg== OR " + "MEDCO_GEN:CSeUe8thcSVS6SZt15C5D81VArJ7UNBIvOhsx9Ex2kJ4cwEOYzOnexTAKw4i8zwkz+xXAtSzIyH/KQfqzfwDvQ== OR " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR "); } sw3.append( "MEDCO_GEN:hsI74szdwDidAOwHJiTy8HHBozqmuij52L+veJ+IhqyO2wwRBNRUQxxzH4VyEUPOxJ0Zg0enQwt+0/oX4X1cAA== OR " + "MEDCO_GEN:bYC1en7ce/yDbFK0NEXcINxscePn3tzOop0Ntzue1LZpR0RM7OKi46AUiZS8lT0pJBobyowuqsXg1EciXyS9Wg== OR " + "MEDCO_GEN:WD3hdnirdqqqMVzKI+XarDB5BgM5lzyYGoGogagghI1/LIHLTbw3uxNrGb+3d9Udj0lWXl3b8xlWuJesTcV9nA== OR " + "MEDCO_GEN:eRX3hSgsitWpwCa29/3u1nso+NyE20HwisNRmaFNAGp3AFfr4BjJMVJ1jZ9U6s8f22a8Vesub7XvXL76shmbIg== OR " + "MEDCO_GEN:CSeUe8thcSVS6SZt15C5D81VArJ7UNBIvOhsx9Ex2kJ4cwEOYzOnexTAKw4i8zwkz+xXAtSzIyH/KQfqzfwDvQ== OR " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR " + "MEDCO_GEN:HjY+NM9pr5ch6dUmmwJRMzEMFThtgT8nt91A98KDMm+lG10qILx7YcYFFMUwutQJQ9GVoFDRnd09GwynpIGB2A=="); return sw3.toString(); case 13: // medco: 500 attributes StringWriter sw4 = new StringWriter(); sw4.append("\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND "); for (int i = 0 ; i < 55 ; i++) { sw4.append( "MEDCO_GEN:hsI74szdwDidAOwHJiTy8HHBozqmuij52L+veJ+IhqyO2wwRBNRUQxxzH4VyEUPOxJ0Zg0enQwt+0/oX4X1cAA== OR " + "MEDCO_GEN:bYC1en7ce/yDbFK0NEXcINxscePn3tzOop0Ntzue1LZpR0RM7OKi46AUiZS8lT0pJBobyowuqsXg1EciXyS9Wg== OR " + "MEDCO_GEN:WD3hdnirdqqqMVzKI+XarDB5BgM5lzyYGoGogagghI1/LIHLTbw3uxNrGb+3d9Udj0lWXl3b8xlWuJesTcV9nA== OR " + "MEDCO_GEN:eRX3hSgsitWpwCa29/3u1nso+NyE20HwisNRmaFNAGp3AFfr4BjJMVJ1jZ9U6s8f22a8Vesub7XvXL76shmbIg== OR " + "MEDCO_GEN:CSeUe8thcSVS6SZt15C5D81VArJ7UNBIvOhsx9Ex2kJ4cwEOYzOnexTAKw4i8zwkz+xXAtSzIyH/KQfqzfwDvQ== OR " + "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR "); } sw4.append( "MEDCO_GEN:NETvU3/IYHvBL5NOzW2h7lpi8AGlPMSsMSKUNvwnhdM9bmOmRVsd4Hpywg/tR3tLHygKBTEW5XeaM2G6l/0SfQ== OR " + "MEDCO_GEN:MkcVUYjRD71jgWnu8/kfPiac+Ezmh5hL5wzP8IANdDZUs9gy9tyCVbnKVFVLOTQnaBTWZYtUnQwTfQwmOsHdgw== OR " + "MEDCO_GEN:MviI3uTGsPlKw2aZsNkmHHVv9XY/ycD6p9bArjlSMvjdcbwCA620/PMtI9MrpQEXVOe6pw2auGMahdNarn/WuQ== OR " + "MEDCO_GEN:NkGR6ZYbrxArjdG6REfxmo9qzN4JKew7lykKLI7VGb7wWky7dptR3eY2HeUPlK5l8JLOZ2iuX+nRHF7/X7bNEw== OR " + "MEDCO_GEN:HjY+NM9pr5ch6dUmmwJRMzEMFThtgT8nt91A98KDMm+lG10qILx7YcYFFMUwutQJQ9GVoFDRnd09GwynpIGB2A=="); return sw4.toString(); case 14: // medco 10 servers return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_ENC:LDPgWORjARoU1afmLsEU59o970xQspOpSb1nB49F3FuFMzZGI0siGWbYyROKDYjORurNmAQteEbYpDggANQWFg==" + " AND " + "MEDCO_ENC:zaIfZW3VMrSURkMmYQ3g7VZu58Of9K/XDHT3ln3UEhLrbVrM+tHpLDs4Hdpv3hQpOzThrlomyh6Hg68HnduxQg==" + " AND " + "MEDCO_GEN:36qhWlgIKzLgR+bW+bQadf6YtG0geyqkkDO/dgtAjYUTBSWEwfEJzt54fyqfiSdzmE9wD4L4X7SUBFefNIkIgw== OR " + "MEDCO_GEN:BuOMhbWOZ9ISr+FMosyPj3IW1u0yzSbzwMjgv52qr9RAF3ti/CRF5UN/ELJq5FoUgnfTsjvuTEdsT46cMYC1pA== OR " + "MEDCO_GEN:LCd6f8ykhyIrPh2R9fpetTmxlkGsAeKxpKP8n7szIHPkCS4VfcHl5OG7mYW6RH+O1UbnEsKO5fV0sBuor8PpYg== OR " + "MEDCO_GEN:qtvSnylwP7PlT4KTB1MBKXDLhUF9CdNkzfEKAjvnLstoqU29HfYBARZQDRxrgQvaLaOZSfSZB/KamqvmNniO0w=="; case 15: // medco 6 servers return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_ENC:FzXxSbBn86gMmF7WT6a4kHDcHrOg3SEkaojcPm7U3qsQp0bhzaLZLYenL/+yNS5j39TFcLU1uSUE5I8tD3Qryw==" + " AND " + "MEDCO_ENC:66xaTIbPcE8V/4u9cE7UWFjgBPpu7yBMSfLSsNrDeTssfy57z5DfTAI+ynrVMzosOapo2SqQxRrrKFSWIljEbw==" + " AND " + "MEDCO_GEN:ZwneoQQyvDUckDcxlOvS+1IvDckXgw7n13IpznyAcHaU6r3uHuSZOXFHUHxhqINh0q6PFj9htw4Ogrt0TR7b5w== OR " + "MEDCO_GEN:auWQ7K+BBDlyRZdSlyIHUDh6d+z60Gsop7aanz4kyYRMrCNNm9jUOqHmbpDawm05fygUZ5jZLEcRJoQuP2Ao9g== OR " + "MEDCO_GEN:LnTrW21XUj7DTSzYdhilO39xBqFNFmkL72q1m8xoSwyI8IGHgQGmw9rDno7iGYKxoVCBarwLHrqykVHji4+tQw== OR " + "MEDCO_GEN:2zO57byps/7sa0cXQbIE3fQ7niglT0td5ew21ZH214lUGdLl3Bq94bEHnEhiSHn5FrV9SesmrnI0NuWQQQHe2w=="; case 16: // medco 9 servers return "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Male\\ OR " + "\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\GENDER\\Female\\" + " AND " + "MEDCO_ENC:ucrIw7mP2vVfFivx0ENbVqiVNJA/rUpfYPpPijeavMl2qa0etxn7uhSJXVtPSfM6Hg5fHyX+v2mjvU+zHdTRpA==" + " AND " + "MEDCO_ENC:Un8OlW0bfCM/z3UFY2kP3CiDDTMZn0lemyFNyda68uHGjpPY2lEB8wOmn+lq9lxBxiZnD0BuUNxqzousBh6AYA==" + " AND " + "MEDCO_GEN:QoYNiucM/dRnN3L569tTNK4ETW2zhMYM2LqtWf3hoIzvXchr7VpjSrMVnssAr4SGv7n2PcGcdoB5hNJ47fyvYQ== OR " + "MEDCO_GEN:/gQIdPCUVw3SXMBs4WozPbco6V9s/glvH+E/oqeBVWO/+dCMT2SYlDaNnkFzYoIUciBFsLhB4ffmGihurYGscA== OR " + "MEDCO_GEN:sPWZvOLnVNltU8vqaum9gm4aMUhcyNb5kGi/iXATVGyyWapHYJ2auq6xEb80LdrDh8Sc3zJxPI5ncLCkMYTILw== OR " + "MEDCO_GEN:IsRF1+L3xp7j0t+MK694nDCman52R7TifUgXk739jIlAq4Ri70ovhXjNKoK2y6MW9g2LlWwKkP7g+s7Rd6M3Zg=="; default: return null; } } } diff --git a/src/server/ch/epfl/lca1/medco/MedCoDataLoader.java b/src/server/ch/epfl/lca1/medco/MedCoDataLoader.java index 064d539..b5a44c0 100644 --- a/src/server/ch/epfl/lca1/medco/MedCoDataLoader.java +++ b/src/server/ch/epfl/lca1/medco/MedCoDataLoader.java @@ -1,123 +1,123 @@ package ch.epfl.lca1.medco; import ch.epfl.lca1.medco.dao.MedCoDatabase; import ch.epfl.lca1.medco.i2b2.crc.I2B2CRCCell; import ch.epfl.lca1.medco.i2b2.fr.I2B2FRCell; import ch.epfl.lca1.medco.i2b2.ont.I2B2ONTCell; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import ch.epfl.lca1.medco.loader.clinical.DataLoader; import ch.epfl.lca1.medco.loader.clinical.OntologyLoader; import ch.epfl.lca1.medco.loader.DataType; import ch.epfl.lca1.medco.loader.genomic.GenomicLoader; import ch.epfl.lca1.medco.util.exceptions.UnlynxException; import com.opencsv.CSVReader; import edu.harvard.i2b2.common.exception.I2B2Exception; import org.javatuples.Pair; import java.io.FileReader; import java.io.IOException; import java.util.Map; /** * Created by misbach on 06.06.17. */ // todo: think about how to treat mismatch when exist in genomic but not clinical! (right now the ids to num are not updated) public class MedCoDataLoader { public static final String EMPTY_VALUE_FLAG = ""; private String providerId, datasetId; private MedCoDatabase medcoDao; - private UserAuthentication auth; + private MedCoI2b2MessageHeader auth; private I2B2ONTCell ontCell; private I2B2CRCCell crcCell; private I2B2FRCell frCell; public MedCoDataLoader(String providerId, String datasetId, String domainId, String projectId, String username, String password) throws I2B2Exception { this.providerId = providerId; this.datasetId = datasetId; // init connection to i2b2 cells - auth = new UserAuthentication(domainId, projectId, username, false, 0, password); + auth = new MedCoI2b2MessageHeader(domainId, projectId, username, false, 0, password); ontCell = new I2B2ONTCell(auth); //crcCell = new I2B2CRCCell(auth); frCell = new I2B2FRCell(auth); medcoDao = new MedCoDatabase(); } /** * First line = header *'\t', * @param nbLinesToSkip * @param dataTypes * @throws IOException */ // split up in 2 fonctins: ontology and data ; pdoconcepts/concept dim -> during ontology public void loadClinicalFileOntology(String clinicalFilePath, char csvSeparator, char csvQuote, int nbLinesToSkip, DataType[] dataTypes) throws IOException, I2B2Exception, UnlynxException { CSVReader reader = new CSVReader(new FileReader(clinicalFilePath), csvSeparator, csvQuote, nbLinesToSkip); String[] csvHeader = reader.readNext(); String[] csvEntry; // first pass: load ontology OntologyLoader clinicalOntLoader = new OntologyLoader(csvHeader, dataTypes, providerId); while ((csvEntry = reader.readNext()) != null) { clinicalOntLoader.parseEntry(csvEntry); } clinicalOntLoader.loadOntology(ontCell); // clinicalOntLoader.loadConceptDimension(frCell, crcCell); //todo: commented in order to load big things, use sql query to update: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //insert into i2b2demodata.concept_dimension(concept_path, concept_cd, name_char, import_date) //select c_fullname, c_basecode, c_name, update_date from i2b2metadata.clinical_non_sensitive where c_fullname like '%VARIANT_ID%' and c_basecode is not null; reader.close(); } public void loadClinicalFileData(String clinicalFilePath, char csvSeparator, char csvQuote, int nbLinesToSkip, DataType[] dataTypes) throws IOException, I2B2Exception, UnlynxException { CSVReader reader = new CSVReader(new FileReader(clinicalFilePath), csvSeparator, csvQuote, nbLinesToSkip); String[] csvHeader = reader.readNext(); String[] csvEntry; // second pass: load data DataLoader clinicalData = new DataLoader(csvHeader, dataTypes, providerId); reader = new CSVReader(new FileReader(clinicalFilePath), csvSeparator, csvQuote, nbLinesToSkip + 1); while ((csvEntry = reader.readNext()) != null) { clinicalData.loadEntry(ontCell, csvEntry); } clinicalData.loadData(frCell, crcCell); } public void loadGenomicFile(String genomicFilePath, char csvSeparator, char csvQuote, int nbLinesToSkip, DataType[] dataTypes) throws IOException, I2B2Exception, UnlynxException { CSVReader reader = new CSVReader(new FileReader(genomicFilePath), csvSeparator, csvQuote, nbLinesToSkip); String[] csvHeader = reader.readNext(); String[] csvEntry; GenomicLoader genOntLoader = new GenomicLoader(csvHeader, dataTypes, providerId); while ((csvEntry = reader.readNext()) != null) { genOntLoader.parseEntry(csvEntry); } genOntLoader.loadEntries(); //genOntLoader.loadOntology(datasetId, ontCell); todo: tmp disabled!! reader.close(); } /** * Should be called after loading * @throws I2B2Exception */ public void translateIdsToNums() throws I2B2Exception { Map> maps = crcCell.getSampleAndPatientNums(medcoDao.getUniqueSampleIds()); medcoDao.updateSampleAndPatientIds(maps); } } diff --git a/src/server/ch/epfl/lca1/medco/StandardQuery.java b/src/server/ch/epfl/lca1/medco/StandardQuery.java index ca261b7..4664f93 100644 --- a/src/server/ch/epfl/lca1/medco/StandardQuery.java +++ b/src/server/ch/epfl/lca1/medco/StandardQuery.java @@ -1,260 +1,259 @@ /* * Copyright (c) 2006-2007 Massachusetts General Hospital * All rights reserved. This program and the accompanying materials * are made available under the terms of the i2b2 Software License v1.0 * which accompanies this distribution. * * Contributors: * Rajesh Kuttan */ package ch.epfl.lca1.medco; import ch.epfl.lca1.medco.i2b2.crc.I2B2CRCCell; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryRequest; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryResponse; import ch.epfl.lca1.medco.i2b2.pm.I2B2PMCell; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; import ch.epfl.lca1.medco.i2b2.pm.UserInformation; import ch.epfl.lca1.medco.unlynx.UnlynxClient; import ch.epfl.lca1.medco.util.Constants; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.Timers; import ch.epfl.lca1.medco.util.exceptions.MedCoError; import ch.epfl.lca1.medco.util.exceptions.MedCoException; import edu.harvard.i2b2.common.exception.I2B2Exception; import edu.harvard.i2b2.common.util.jaxb.JAXBUtilException; import edu.harvard.i2b2.crc.datavo.setfinder.query.PanelType; import edu.harvard.i2b2.crc.datavo.setfinder.query.QueryDefinitionType; import org.javatuples.Pair; import java.io.StringWriter; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; // //todo doc: https://github.com/chb/shrine/tree/master/doc /** * Represents a query to MedCo. * From the XML query (in CRC format), parse to extract the sensitive attributes, * make query to CRC for non-sensitive attributes, get the patient set from CRC, * query the cothority with the patient sets and sensitive attributes and answer. * * * everything under that sohuld not use the config!! */ public class StandardQuery { private I2B2QueryRequest queryRequest; private I2B2CRCCell crcCell; private I2B2PMCell pmCell; private UnlynxClient unlynxClient; //int resultMode, String clientPubKey, long timoutSeconds public StandardQuery(I2B2QueryRequest request, String unlynxBinPath, String unlynxGroupFilePath, int unlynxDebugLevel, int unlynxEntryPointIdx, int unlynxProofsFlag, long unlynxTimeoutSeconds, - String crcCellUrl, String pmCellUrl, UserAuthentication medcoI2b2Auth) throws I2B2Exception { + String crcCellUrl, String pmCellUrl) throws I2B2Exception { this.queryRequest = request; unlynxClient = new UnlynxClient(unlynxBinPath, unlynxGroupFilePath, unlynxDebugLevel, unlynxEntryPointIdx, unlynxProofsFlag, unlynxTimeoutSeconds); - crcCell = new I2B2CRCCell(crcCellUrl, medcoI2b2Auth); - pmCell = new I2B2PMCell(pmCellUrl, medcoI2b2Auth); + crcCell = new I2B2CRCCell(crcCellUrl, queryRequest.getMessageHeader()); + pmCell = new I2B2PMCell(pmCellUrl, queryRequest.getMessageHeader()); } /** * * @return the query answer in CRC XML format. * @throws JAXBUtilException */ public I2B2QueryResponse executeQuery() throws MedCoException, I2B2Exception { Timers.resetTimers(); Timers.get("overall").start(); // get user information (auth., privacy budget, authorizations, public key) // todo: get and check budget query / user // todo: get user permissions Timers.get("steps").start("User information retrieval"); UserInformation user = pmCell.getUserInformation(queryRequest.getMessageHeader()); if (!user.isAuthenticated()) { Logger.warn("Authentication failed for user " + user.getUsername()); // todo: proper auth failed response return null; } QueryType queryType = QueryType.resolveUserPermission(user.getRoles()); Timers.get("steps").stop(); // retrieve the encrypted query terms Timers.get("steps").start("Query parsing/splitting"); List encryptedQueryItems = extractEncryptedQueryTerms(false, false); Timers.get("steps").stop(); // query unlynx to tag the query terms Timers.get("steps").start("Query tagging"); List taggedItems = unlynxClient.computeDistributedDetTags(queryRequest.getQueryName(), encryptedQueryItems); Timers.addAdditionalTimes(unlynxClient.getLastTimingMeasurements()); Timers.get("steps").stop(); // replace the query terms, query i2b2 with the original clear query terms + the tagged ones Timers.get("steps").start("i2b2 query"); replaceEncryptedQueryTerms(taggedItems); I2B2QueryResponse i2b2Response = crcCell.queryRequest(queryRequest); Timers.get("steps").stop(); // retrieve the patient set, including the encrypted dummy flags Timers.get("steps").start("i2b2 patient set retrieval"); Pair, List> patientSet = crcCell.queryForPatientSet(i2b2Response.getPatientSetId(), true); Timers.get("steps").stop(); String aggResult; switch (queryType) { case AGGREGATED_PER_SITE: aggResult = unlynxClient.aggregateData(queryRequest.getQueryName(), user.getUserPublicKey(), patientSet.getValue1()); Timers.addAdditionalTimes(unlynxClient.getLastTimingMeasurements()); break; case OBFUSCATED_PER_SITE: case AGGREGATED_TOTAL: default: throw new MedCoError("Query type not supported yet."); } i2b2Response.resetResultInstanceListToEncryptedCountOnly(); i2b2Response.setQueryResults(user.getUserPublicKey(), aggResult, Timers.generateFullReport()); Logger.info("MedCo query successful (" + queryRequest.getQueryName() + ")."); Timers.get("overall").stop(); return i2b2Response; } /** * TODO * No checks on panels are done (i.e. if they contain mixed query types or not) * * @param taggedItems * @throws MedCoException */ private void replaceEncryptedQueryTerms(List taggedItems) throws MedCoException { QueryDefinitionType qd = queryRequest.getQueryDefinition(); int encTermCount = 0; // iter on the panels for (int p = 0; p < qd.getPanel().size(); p++) { PanelType panel = qd.getPanel().get(p); // iter on the items int nbItems = panel.getItem().size(); for (int i = 0; i < nbItems; i++) { // replace encryptem item with its tagged version Matcher medcoKeyMatcher = Constants.REGEX_QUERY_KEY_ENC.matcher(panel.getItem().get(i).getItemKey()); if (medcoKeyMatcher.matches()) { panel.getItem().get(i).setItemKey(medcoKeyMatcher.replaceFirst( "$1" + Constants.CONCEPT_PATH_NODE_TAGGED + "$3" + taggedItems.get(encTermCount++) + "$5")); } } } // check the provided taggedItems match the number of encrypted terms if (encTermCount != taggedItems.size()) { Logger.warn("Mismatch in provided number of tagged items (" + taggedItems.size() + ") and number of encrypted items in query (" + encTermCount + ")"); } } /** * Extract from the i2b2 query the sensitive / encrypted items recognized by the prefix defined in {@link Constants}. * Accepts only panels fully clear or encrypted, i.e. no mix is allowed. *

* The predicate, if returned, has the following format: * (exists(v0, r) || exists(v1, r)) && (exists(v2, r) || exists(v3, r)) && exists(v4, r) * * @param removePanels removes the encrypted panels when encountered if true * @param getPredicate adds at the end of the returned list the corresponding predicate if true * @return the list of encrypted query terms and optionally the corresponding predicate * @throws MedCoException if a panel contains mixed clear and encrypted query terms */ private List extractEncryptedQueryTerms(boolean removePanels, boolean getPredicate) throws MedCoException { // todo: handle cases: only clear no encrypt / only encrypt no clear // todo: must be modified if invertion implementation QueryDefinitionType qd = queryRequest.getQueryDefinition(); StringWriter predicateSw = new StringWriter(); List extractedItems = new ArrayList<>(); int encTermCount = 0; // iter on the panels for (int p = 0; p < qd.getPanel().size(); p++) { boolean panelIsEnc = false, panelIsClear = false; PanelType panel = qd.getPanel().get(p); // iter on the items int nbItems = panel.getItem().size(); for (int i = 0; i < nbItems; i++) { // check if item is clear or encrypted, extract and generate predicate if yes Matcher medcoKeyMatcher = Constants.REGEX_QUERY_KEY_ENC.matcher(panel.getItem().get(i).getItemKey()); if (medcoKeyMatcher.matches()) { if (i == 0) { predicateSw.append("("); } extractedItems.add(medcoKeyMatcher.group(4)); predicateSw.append("exists(v" + encTermCount++ + ", r)"); if (i < nbItems - 1) { predicateSw.append(" || "); } else if (i == nbItems - 1) { predicateSw.append(")"); if (p < qd.getPanel().size() - 1) { predicateSw.append(" && "); } } Logger.debug("Extracted item " + extractedItems.get(extractedItems.size() - 1)); panelIsEnc = true; } else { panelIsClear = true; } // enforce that a panel can only be one type if (panelIsClear && panelIsEnc) { throw Logger.error(new MedCoException("Encountered panel with mixed clear and encrypted query terms: not allowed.")); } } // remove panel and log if (panelIsEnc) { if (removePanels) { qd.getPanel().remove(panel); p--; Logger.debug("Removed encrypted panel"); } Logger.debug("Encountered encrypted panel"); } else if (panelIsClear) { Logger.debug("Encountered clear panel"); } else { Logger.warn("Encountered empty panel in query " + qd.getQueryName()); } } String predicate = predicateSw.toString(); Logger.info("Extracted " + extractedItems.size() + " encrypted query terms and generated unlynx predicate with " + encTermCount + " terms: " + predicate + " for query " + queryRequest.getQueryName()); if (getPredicate) { extractedItems.add(predicate); } return extractedItems; } } diff --git a/src/server/ch/epfl/lca1/medco/axis2/MedCoQueryRequestDelegate.java b/src/server/ch/epfl/lca1/medco/axis2/MedCoQueryRequestDelegate.java index bad5f85..758bb36 100644 --- a/src/server/ch/epfl/lca1/medco/axis2/MedCoQueryRequestDelegate.java +++ b/src/server/ch/epfl/lca1/medco/axis2/MedCoQueryRequestDelegate.java @@ -1,103 +1,102 @@ /* * Copyright (c) 2006-2007 Massachusetts General Hospital * All rights reserved. This program and the accompanying materials * are made available under the terms of the i2b2 Software License v1.0 * which accompanies this distribution. * * Contributors: * Rajesh Kuttan */ package ch.epfl.lca1.medco.axis2; import ch.epfl.lca1.medco.StandardQuery; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryResponse; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.MedCoUtil; import edu.harvard.i2b2.common.util.jaxb.JAXBUtilException; import edu.harvard.i2b2.crc.datavo.i2b2message.*; import ch.epfl.lca1.medco.util.exceptions.MedCoException; import ch.epfl.lca1.medco.i2b2.crc.I2B2QueryRequest; import edu.harvard.i2b2.common.exception.I2B2Exception; import java.io.PrintWriter; import java.io.StringWriter; /** * Setfinder query request delegate class $Id: QueryRequestDelegate.java,v 1.17 * 2008/05/08 15:13:45 rk903 Exp $ * * @author rkuttan */ // todo: no authentication done for now, implement! public class MedCoQueryRequestDelegate extends RequestHandlerDelegate { protected static edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory i2b2OF = new edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory(); protected static MedCoUtil medCoUtil = MedCoUtil.getInstance(); // todo: hardcoded, should be taken from the PM public static final String clientPubKey = "eQviK90cvJ2lRx8ox6GgQKFmOtbgoG9RXa7UnmemtRA="; public static final String clientSeckey = "iqLQz3zMlRjCyBrg4+303hsxL7F5vDtIaBxO0oc7gQA="; /** */ public String handleRequest(String requestString) throws I2B2Exception { Logger.info("Handling new MedCo query request"); try { I2B2QueryRequest request = new I2B2QueryRequest(requestString); //I2B2Cell.authenticate(xx); // returns objects with infos about request StandardQuery medcoQuery = new StandardQuery(request, medCoUtil.getUnlynxBinPath(), medCoUtil.getUnlynxGroupFilePath(), medCoUtil.getUnlynxDebugLevel(), medCoUtil.getUnlynxEntryPointIdx(), medCoUtil.getUnlynxProofsFlag(), - medCoUtil.getI2b2Waittimems(), medCoUtil.getDataRepositoryCellUrl(), medCoUtil.getProjectManagementCellUrl(), - null); + medCoUtil.getI2b2Waittimems(), medCoUtil.getDataRepositoryCellUrl(), medCoUtil.getProjectManagementCellUrl()); int resultMode = 0; int timeoutSeconds = MedCoUtil.getInstance().getI2b2Waittimems();//todo: from configuration add entry specific to unlynx I2B2QueryResponse queryAnswer = medcoQuery.executeQuery(); StringWriter strWriter = new StringWriter(); MedCoUtil.getMsgUtil().marshallerWithCDATA(i2b2OF.createResponse(queryAnswer), strWriter, new String[]{"observation_blob","patient_blob","observer_blob","concept_blob","event_blob"}); return strWriter.toString(); } catch (MedCoException | I2B2Exception | JAXBUtilException e) { Logger.error("Error during query", e); ResponseMessageType errorResponse = i2b2OF.createResponseMessageType(); ResponseHeaderType respHeader = i2b2OF.createResponseHeaderType(); errorResponse.setResponseHeader(respHeader); InfoType info = i2b2OF.createInfoType(); respHeader.setInfo(info); // detailed error message StringWriter infoMessage = new StringWriter(); PrintWriter exceptionPrint = new PrintWriter(infoMessage); infoMessage.append("Exception:" + e.getMessage() + "\n"); e.printStackTrace(exceptionPrint); infoMessage.append("Caused by: " + (e.getCause() == null ? "no cause" : e.getCause().getMessage()) + "\n"); e.getCause().printStackTrace(exceptionPrint); info.setValue(infoMessage.toString()); try { StringWriter strWriter = new StringWriter(); MedCoUtil.getMsgUtil().marshallerWithCDATA(i2b2OF.createResponse(errorResponse), strWriter, new String[]{"observation_blob", "patient_blob", "observer_blob", "concept_blob", "event_blob"}); return strWriter.toString(); } catch (Throwable e1) { Logger.error("Error while generating error message", e); } } // all hope lost at this point return null; } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/I2B2Cell.java b/src/server/ch/epfl/lca1/medco/i2b2/I2B2Cell.java index febfce6..281daf5 100755 --- a/src/server/ch/epfl/lca1/medco/i2b2/I2B2Cell.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/I2B2Cell.java @@ -1,123 +1,123 @@ package ch.epfl.lca1.medco.i2b2; import javax.xml.bind.JAXBElement; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; import edu.harvard.i2b2.crc.datavo.i2b2message.RequestHeaderType; import edu.harvard.i2b2.crc.datavo.i2b2message.RequestMessageType; import edu.harvard.i2b2.crc.datavo.i2b2message.ResponseMessageType; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; import org.apache.axiom.om.OMElement; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.MedCoUtil; import edu.harvard.i2b2.common.util.axis2.ServiceClient; import org.javatuples.Pair; /** * keep internal the translations string <> OMelement <> JAXB * TODO: check if it makes to refactor and have abstratec object to manipulate those // maybe a medco jaxbutil ? * @author misbach * */ public abstract class I2B2Cell { // TODO: send message method + construction class here /** * URL of the i2b2 cell this object connects to. */ protected final String cellURL; /** * JAXB utility class to manipulate XML. */ protected static MessagesUtil msgUtil = MedCoUtil.getMsgUtil(); /** * General usage utility class. */ protected static MedCoUtil medCoUtil = MedCoUtil.getInstance(); - protected UserAuthentication auth; + protected MedCoI2b2MessageHeader auth; // todo: centralize object factories in 1 point (msgutil) // JAXB object factories protected static edu.harvard.i2b2.crc.loader.datavo.loader.query.ObjectFactory loaderCrcOF = new edu.harvard.i2b2.crc.loader.datavo.loader.query.ObjectFactory(); protected static edu.harvard.i2b2.crc.loader.datavo.fr.ObjectFactory loaderFrOF = new edu.harvard.i2b2.crc.loader.datavo.fr.ObjectFactory(); protected static edu.harvard.i2b2.crc.datavo.ontology.ObjectFactory ontOF = new edu.harvard.i2b2.crc.datavo.ontology.ObjectFactory(); protected static edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory i2b2OF = new edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory(); protected static edu.harvard.i2b2.crc.datavo.pdo.ObjectFactory pdoOF = new edu.harvard.i2b2.crc.datavo.pdo.ObjectFactory(); protected static edu.harvard.i2b2.crc.datavo.pdo.query.ObjectFactory queryPdoOF = new edu.harvard.i2b2.crc.datavo.pdo.query.ObjectFactory(); protected static edu.harvard.i2b2.crc.datavo.setfinder.query.ObjectFactory querySetFinderOF = new edu.harvard.i2b2.crc.datavo.setfinder.query.ObjectFactory(); - protected I2B2Cell(String cellURL, UserAuthentication auth) { + protected I2B2Cell(String cellURL, MedCoI2b2MessageHeader auth) { this.cellURL = cellURL; this.auth = auth; } /** * Sends a request to the cell, and get back an answer. * extract the body to make new request! * * @param reqMessage * @return */ protected ResponseMessageType requestToCell(String urlPath, RequestMessageType reqMessage) throws Exception { Logger.info("New request to cell " + cellURL); RequestMessageType newReqMessage = createRequestMessage(reqMessage.getMessageBody()); OMElement reqMessageOM = msgUtil.buildOMElement(newReqMessage); String answerMessage = ServiceClient.sendREST(cellURL + urlPath, reqMessageOM); Logger.debug("Cell " + cellURL + " answered: " + answerMessage); // extract response JAXBElement responseJaxb = msgUtil.unMashallFromString(answerMessage); return (ResponseMessageType) responseJaxb.getValue(); } protected Pair requestToCell(String urlPath, BodyType reqBody) throws Exception { // encapsulate and send request Logger.info("New request to cell " + cellURL); RequestMessageType reqMessage = createRequestMessage(reqBody); OMElement reqMessageOM = msgUtil.buildOMElement(reqMessage); String answerMessage = ServiceClient.sendREST(cellURL + urlPath, reqMessageOM); Logger.debug("Cell " + cellURL + " answered: " + answerMessage); return msgUtil.parseI2b2Response(answerMessage); } protected RequestMessageType createRequestMessage(BodyType body) { // setup request header RequestHeaderType requestHeader = i2b2OF.createRequestHeaderType(); requestHeader.setResultWaittimeMs(MedCoUtil.getInstance().getI2b2Waittimems()); // pack in request message RequestMessageType requestMessage = i2b2OF.createRequestMessageType(); requestMessage.setMessageBody(body); requestMessage.setMessageHeader(auth); requestMessage.setRequestHeader(requestHeader); Logger.debug("Correctly created new i2b2 request message" + requestMessage); return requestMessage; } /** * * @return the cell URL */ protected String getCellUrl() { return cellURL; } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2CRCCell.java b/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2CRCCell.java index 9f70ef0..47fed7a 100755 --- a/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2CRCCell.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2CRCCell.java @@ -1,375 +1,372 @@ package ch.epfl.lca1.medco.i2b2.crc; import ch.epfl.lca1.medco.util.Constants; import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; import edu.harvard.i2b2.crc.datavo.i2b2message.ResponseMessageType; import edu.harvard.i2b2.crc.datavo.pdo.EidType; import edu.harvard.i2b2.crc.datavo.pdo.ParamType; import edu.harvard.i2b2.crc.datavo.pdo.PatientType; import edu.harvard.i2b2.crc.datavo.pdo.PidType; import edu.harvard.i2b2.crc.datavo.pdo.query.*; -import edu.harvard.i2b2.crc.datavo.setfinder.query.MasterInstanceResultResponseType; -import edu.harvard.i2b2.crc.datavo.setfinder.query.QueryResultInstanceType; -import edu.harvard.i2b2.crc.datavo.setfinder.query.QueryStatusTypeType; import ch.epfl.lca1.medco.i2b2.I2B2Cell; import ch.epfl.lca1.medco.i2b2.I2b2Status; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import edu.harvard.i2b2.crc.loader.datavo.loader.query.*; import edu.harvard.i2b2.crc.loader.datavo.loader.query.InputOptionListType; import edu.harvard.i2b2.crc.loader.datavo.loader.query.OutputOptionListType; import edu.harvard.i2b2.crc.loader.datavo.loader.query.OutputOptionType; import ch.epfl.lca1.medco.util.Logger; import edu.harvard.i2b2.common.exception.I2B2Exception; import org.javatuples.Pair; import java.util.*; // todo: harmonize error handling (i2b2status + exceptions) public class I2B2CRCCell extends I2B2Cell { private static final String URL_PATH_I2B2_PUBLISHDATAREQ = "/publishDataRequest", URL_PATH_I2B2_PDOREQ = "/pdorequest", URL_PATH_I2B2_REQ = "/request"; private static final String[] RESULT_OUTPUT_TYPES = {"PATIENTSET", "PATIENT_COUNT_XML"};//, "PATIENT_ENCOUNTER_SET", "PATIENT_COUNT_XML"}; - public I2B2CRCCell(String crcCellUrl , UserAuthentication auth) { + public I2B2CRCCell(String crcCellUrl , MedCoI2b2MessageHeader auth) { super(crcCellUrl, auth); // get url } public I2B2QueryResponse queryRequest(I2B2QueryRequest origRequest) throws I2B2Exception { // make query request (from query definition) with a patient set result output origRequest.setOutputTypes(RESULT_OUTPUT_TYPES); I2B2QueryResponse parsedResp; try { ResponseMessageType resp = requestToCell(URL_PATH_I2B2_REQ, origRequest); parsedResp = new I2B2QueryResponse(resp); Logger.info("crc query request result: " + parsedResp.getI2b2Status()); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } return parsedResp; } /** * Given the original i2b2 query that has been stripped from the encrypted query terms, query the i2b2 crc for * the corresponding patient set, includes the enc dummies * * @return list of ids + list of dummy flags * @throws I2B2Exception */ public Pair, List> queryForPatientSet(String patientSetId, boolean getDummyFlags) throws I2B2Exception { // make query to get the actual patient set with the id PatientListType pdoQueryPatientList = queryPdoOF.createPatientListType(); pdoQueryPatientList.setPatientSetCollId(patientSetId); Pair patientSetResponse = pdoQueryFromInputList(null, null, pdoQueryPatientList, null, false, false,false, true, false, false, false, false); if (patientSetResponse.getValue0() != I2b2Status.DONE) { throw Logger.error(new I2B2Exception("Query for patient set id " + patientSetId + " to i2b2 CRC failed.")); } // extract the list of patient ids and enc dummy flags List patientIds = new ArrayList<>(); List patientEncDummyFlags = new ArrayList<>(); for (PatientType patientType : patientSetResponse.getValue1().getPatientData().getPatientSet().getPatient()) { patientIds.add(patientType.getPatientId().getValue()); Logger.debug("Patient id extracted: " + patientIds.get(patientIds.size() - 1)); if (getDummyFlags) { for (ParamType paramType : patientType.getParam()) { if (paramType.getColumn().trim().equals(Constants.PATIENT_DUMMY_FLAG_COL_NAME)) { patientEncDummyFlags.add(paramType.getValue()); Logger.debug("Patient dummy flag extracted: " + patientEncDummyFlags.get(patientEncDummyFlags.size() - 1)); } } } } if (getDummyFlags && patientEncDummyFlags.size() != patientIds.size()) { Logger.warn("Dummy flags requested but mismatch between nb patient ids (" + patientIds.size() + ") and nb dummy flags (" + patientEncDummyFlags.size() + ")"); } Logger.info("Query for patient set id " + patientSetId + " returning a patient set of size " + patientIds.size()); return new Pair<>(patientIds, patientEncDummyFlags); } /** * Error checking left for the caller * @param fileRepoLocationURI * @param providerId * @return * @throws I2B2Exception */ public Pair publishData(String fileRepoLocationURI, String providerId) throws I2B2Exception { PublishDataRequestType publishReq = loaderCrcOF.createPublishDataRequestType(); { InputOptionListType input = loaderCrcOF.createInputOptionListType(); DataListType dataList = loaderCrcOF.createDataListType(); DataListType.LocationUri locationUri = loaderCrcOF.createDataListTypeLocationUri(); locationUri.setProtocolName("FR"); locationUri.setValue(fileRepoLocationURI); dataList.setLocationUri(locationUri); dataList.setDataFormatType(DataFormatType.PDO); dataList.setLoadLabel("MedCo dataset loading"); dataList.setSourceSystemCd(providerId); input.setDataFile(dataList); publishReq.setInputList(input); } { LoadType loadList = loaderCrcOF.createLoadType(); loadList.setCommitFlag(true); loadList.setClearTempLoadTables(true); FactLoadOptionType factLoad = loaderCrcOF.createFactLoadOptionType(); factLoad.setAppendFlag(true); factLoad.setIgnoreBadData(true); loadList.setLoadObservationSet(factLoad); LoadOptionType loadOption = loaderCrcOF.createLoadOptionType(); loadOption.setIgnoreBadData(true); loadList.setLoadEventSet(loadOption); loadList.setLoadPatientSet(loadOption); loadList.setLoadObserverSet(loadOption); loadList.setLoadPidSet(loadOption); loadList.setLoadEidSet(loadOption); loadList.setLoadConceptSet(loadOption); //loadList.setLoadModifierSet(loadOption); // XXX: modifiers not used in current implementation publishReq.setLoadList(loadList); } { OutputOptionListType outputList = loaderCrcOF.createOutputOptionListType(); OutputOptionType outputOption = loaderCrcOF.createOutputOptionType(); outputOption.setBlob(false); outputOption.setOnlykeys(true); outputOption.setTechdata(false); outputList.setObservationSet(outputOption); outputList.setPatientSet(outputOption); outputList.setEventSet(outputOption); outputList.setObserverSet(outputOption); outputList.setConceptSet(outputOption); //outputList.setModifierSet(outputOption); // XXX: modifiers not used in current implementation outputList.setPidSet(outputOption); outputList.setEidSet(outputOption); publishReq.setOutputList(outputList); } BodyType body = i2b2OF.createBodyType(); body.getAny().add(loaderCrcOF.createPublishDataRequest(publishReq)); try { Pair resp = requestToCell(URL_PATH_I2B2_PUBLISHDATAREQ, body); Logger.info("publish data req result: " + resp.getValue0()); // error checking and handling completely left to the caller return new Pair<>( resp.getValue0(), (LoadDataResponseType) msgUtil.getUnwrapHelper().getObjectByClass( resp.getValue1().getAny(), LoadDataResponseType.class)); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } /** * Returns the HIVE id of sample and patients (the internal i2b2 number). * * @param ids pairs of {sample_id, source/provider_id} * @return pair of maps: [key=sample_id ; value=sample_num] , [key=patient_id ; value=patient_num] * nope: sample_id->sample_num,patient_num */ public Map> getSampleAndPatientNums(Set> ids) throws I2B2Exception { // construct xml request for samples EidListType eids = queryPdoOF.createEidListType(); for (Pair id: ids) { EidListType.Eid eid = queryPdoOF.createEidListTypeEid(); eid.setValue(id.getValue0()); eid.setSource(id.getValue1()); //XXX: index parameter ??? what does it do? eids.getEid().add(eid); } // make request for samples Pair pdoResultSamples = pdoQueryFromInputList(eids, null, null, null, false, true, false, false, false, false, false, true); if (pdoResultSamples.getValue0() != I2b2Status.DONE) { Logger.warn("PDO request status is not done, expect problems: " + pdoResultSamples.getValue0()); } // extract the results / samples and construct xml for patients Map> resultMap = new HashMap<>(); List resultEids = pdoResultSamples.getValue1().getPatientData().getEidSet().getEid(); PidListType pids = queryPdoOF.createPidListType(); for (EidType resultEid : resultEids) { for (EidType.EventMapId resultMapEid: resultEid.getEventMapId()) { // add to result map for sample resultMap.put( resultMapEid.getValue(), // the ID in source system (not hive) new Pair<>( resultEid.getEventId().getValue(), // the sample_num (HIVE number) resultEid.getEventId().getPatientId() // the patient_id (not num for now) )); } // construct query for patient PidListType.Pid pid = queryPdoOF.createPidListTypePid(); pid.setValue(resultEid.getEventId().getPatientId()); // patient_id pid.setSource(resultEid.getEventId().getPatientIdSource()); // source / provider pids.getPid().add(pid); } // make request for patients Pair pdoResultPatients = pdoQueryFromInputList(null, pids, null, null, false, true, false, false, false, false, true, false); if (pdoResultPatients.getValue0() != I2b2Status.DONE) { Logger.warn("PDO request status is not done, expect problems: " + pdoResultPatients.getValue0()); } // extract the results / patients Map patientsMap = new HashMap<>(); List resultPids = pdoResultPatients.getValue1().getPatientData().getPidSet().getPid(); for (PidType resultPid : resultPids) { for (PidType.PatientMapId resultMapPid: resultPid.getPatientMapId()) { patientsMap.put( resultMapPid.getValue(), // the ID in source system (not hive) resultPid.getPatientId().getValue() // the patient_num (HIVE number) ); } } // merge the patients map into the result map for (Map.Entry> entry : resultMap.entrySet()) { resultMap.replace( entry.getKey(), entry.getValue().setAt1( patientsMap.get( entry.getValue().getValue1())) ); } return resultMap; } /** * Note: only 1 of the 4 is taken into account (put the rest to null) * * @param eidList * @param pidList * @param patientList * @param eventList * * @param getBlob true to get blob column * @param getOnlykeys true to get only the primary keys * @param getTechdata true to get the tech data * @return * @throws I2B2Exception */ private Pair pdoQueryFromInputList( EidListType eidList, PidListType pidList, PatientListType patientList, EventListType eventList, boolean getBlob, boolean getOnlykeys, boolean getTechdata, boolean queryPatientSet, boolean queryObservationSet, boolean queryEventSet, boolean queryPidSet, boolean queryEidSet) throws I2B2Exception { PdoQryHeaderType queryHeader = queryPdoOF.createPdoQryHeaderType(); queryHeader.setRequestType(PdoRequestTypeType.GET_PDO_FROM_INPUT_LIST); GetPDOFromInputListRequestType queryRequest = queryPdoOF.createGetPDOFromInputListRequestType(); { edu.harvard.i2b2.crc.datavo.pdo.query.InputOptionListType inputOptionList = queryPdoOF.createInputOptionListType(); if (eidList != null) { inputOptionList.setEidList(eidList); } else if (pidList != null) { inputOptionList.setPidList(pidList); } else if (patientList != null) { inputOptionList.setPatientList(patientList); } else if (eventList != null) { inputOptionList.setEventList(eventList); } else { throw Logger.error(new IllegalArgumentException("All input lists are null.")); } queryRequest.setInputList(inputOptionList); } { FilterListType filterList = queryPdoOF.createFilterListType(); // todo: left empty for now queryRequest.setFilterList(filterList); } { edu.harvard.i2b2.crc.datavo.pdo.query.OutputOptionListType outputOptionList = queryPdoOF.createOutputOptionListType(); outputOptionList.setNames(OutputOptionNameType.NONE); // do not use code_lookup to resolve code to their names // common attributes for the following returned elements edu.harvard.i2b2.crc.datavo.pdo.query.OutputOptionType outputOption = queryPdoOF.createOutputOptionType(); outputOption.setBlob(getBlob); outputOption.setOnlykeys(getOnlykeys); outputOption.setTechdata(getTechdata); outputOption.setSelect(OutputOptionSelectType.USING_INPUT_LIST); if (queryPatientSet) { outputOptionList.setPatientSet(outputOption); } if (queryObservationSet) { edu.harvard.i2b2.crc.datavo.pdo.query.FactOutputOptionType factOutputOption = queryPdoOF.createFactOutputOptionType(); factOutputOption.setBlob(outputOption.isBlob()); factOutputOption.setOnlykeys(outputOption.isOnlykeys()); factOutputOption.setTechdata(outputOption.isTechdata()); factOutputOption.setSelect(outputOption.getSelect()); outputOptionList.setObservationSet(factOutputOption); } if (queryEventSet) { outputOptionList.setEventSet(outputOption); } if (queryPidSet) { outputOptionList.setPidSet(outputOption); } if (queryEidSet) { outputOptionList.setEidSet(outputOption); } queryRequest.setOutputOption(outputOptionList); } BodyType body = i2b2OF.createBodyType(); body.getAny().add(queryPdoOF.createPdoheader(queryHeader)); body.getAny().add(queryPdoOF.createRequest(queryRequest)); try { Pair resp = requestToCell(URL_PATH_I2B2_PDOREQ, body); Logger.info("pdo query result: " + resp.getValue0()); // error checking and handling completely left to the caller return new Pair<>( resp.getValue0(), (PatientDataResponseType) msgUtil.getUnwrapHelper().getObjectByClass( resp.getValue1().getAny(), PatientDataResponseType.class)); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2QueryRequest.java b/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2QueryRequest.java index d13450e..251e4ec 100644 --- a/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2QueryRequest.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/crc/I2B2QueryRequest.java @@ -1,185 +1,183 @@ package ch.epfl.lca1.medco.i2b2.crc; import java.util.List; -import ch.epfl.lca1.medco.i2b2.I2b2Status; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import edu.harvard.i2b2.crc.datavo.i2b2message.*; import edu.harvard.i2b2.crc.datavo.setfinder.query.*; import ch.epfl.lca1.medco.i2b2.MessagesUtil; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.MedCoUtil; import edu.harvard.i2b2.common.exception.I2B2Exception; import edu.harvard.i2b2.common.util.jaxb.JAXBUtilException; -import edu.harvard.i2b2.crc.datavo.setfinder.query.ObjectFactory; -import org.javatuples.Triplet; /** * Represents an I2B2 query request. * Use for server side: * Use for client side * * @author mickael * */ public class I2B2QueryRequest extends RequestMessageType { // live references to the 2 elements of the request body private PsmQryHeaderType queryHeader; private QueryDefinitionRequestType queryBody; - private UserAuthentication authentication; + private MedCoI2b2MessageHeader authentication; private static final String I2B2_RESULT_OPTION_TYPE = "PATIENT_COUNT_XML"; private static MedCoUtil util = MedCoUtil.getInstance(); private static MessagesUtil msgUtil = util.getMsgUtil(); private static edu.harvard.i2b2.crc.datavo.setfinder.query.ObjectFactory querySetFinderOF = new edu.harvard.i2b2.crc.datavo.setfinder.query.ObjectFactory(); private static edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory i2b2OF = new edu.harvard.i2b2.crc.datavo.i2b2message.ObjectFactory(); /** * Used on server-side to parse an incoming request. * Unmashall the query from string. * Checks type is with query definition. (both in header annnnnnnd for the type of xml for qrequestbody) * * @param requestString * @throws JAXBUtilException if the string could not be parsed / unmashalled * @throws I2B2Exception if the request is of the wrong type */ public I2B2QueryRequest(String requestString) throws I2B2Exception { try { // JAXB parsing RequestMessageType parsedRequest = msgUtil.parseI2b2Request(requestString); - authentication = new UserAuthentication(parsedRequest.getMessageHeader()); + authentication = new MedCoI2b2MessageHeader(parsedRequest.getMessageHeader()); setMessageHeader(authentication); setMessageBody(parsedRequest.getMessageBody()); setRequestHeader(parsedRequest.getRequestHeader()); queryHeader = (PsmQryHeaderType) msgUtil.getUnwrapHelper().getObjectByClass( getMessageBody().getAny(), PsmQryHeaderType.class); queryBody = (QueryDefinitionRequestType) msgUtil.getUnwrapHelper().getObjectByClass( getMessageBody().getAny(), QueryDefinitionRequestType.class); // check type is query request with query definition if (!queryHeader.getRequestType().equals(PsmRequestTypeType.CRC_QRY_RUN_QUERY_INSTANCE_FROM_QUERY_DEFINITION)) { throw new I2B2Exception("Only query from query definition supported, got:" + queryHeader.getRequestType()); } } catch (ClassCastException | JAXBUtilException e) { throw Logger.error(new I2B2Exception("JAXB unwrap failed.", e)); } } /** * Used on client-side to create a request. */ - public I2B2QueryRequest(UserAuthentication auth) { + public I2B2QueryRequest(MedCoI2b2MessageHeader auth) { // authentication authentication = auth; setMessageHeader(authentication); // standard information generated by i2b2 webclient RequestHeaderType reqHeader = i2b2OF.createRequestHeaderType(); reqHeader.setResultWaittimeMs(util.getI2b2Waittimems()); setRequestHeader(reqHeader); queryHeader = querySetFinderOF.createPsmQryHeaderType(); queryHeader.setRequestType(PsmRequestTypeType.CRC_QRY_RUN_QUERY_INSTANCE_FROM_QUERY_DEFINITION); UserType user = querySetFinderOF.createUserType(); user.setGroup(authentication.getSecurity().getDomain()); user.setLogin(authentication.getSecurity().getUsername()); user.setValue(authentication.getSecurity().getUsername()); queryHeader.setUser(user); queryHeader.setEstimatedTime(0); // query request = query definition (created later) + result output list queryBody = querySetFinderOF.createQueryDefinitionRequestType(); // result output ResultOutputOptionListType outputList = querySetFinderOF.createResultOutputOptionListType(); ResultOutputOptionType resultOption = querySetFinderOF.createResultOutputOptionType(); resultOption.setPriorityIndex(1); resultOption.setName(I2B2_RESULT_OPTION_TYPE); outputList.getResultOutput().add(resultOption); queryBody.setResultOutputList(outputList); // create the body BodyType body = i2b2OF.createBodyType(); body.getAny().add(querySetFinderOF.createPsmheader(queryHeader)); body.getAny().add(querySetFinderOF.createRequest(queryBody)); setMessageBody(body); } public void setQueryDefinition(String queryName, List> itemKeys) { QueryDefinitionType queryDef = querySetFinderOF.createQueryDefinitionType(); queryDef.setQueryName(queryName); queryDef.setQueryId(queryName); queryDef.setQueryDescription("Query generated by the Java client: " + queryName); queryDef.setSpecificityScale(0); int panelCount = 0; for (List panel : itemKeys) { PanelType panelXml = querySetFinderOF.createPanelType(); panelXml.setPanelNumber(++panelCount); panelXml.setPanelAccuracyScale(0); for (String itemKey : panel) { ItemType itemXml = querySetFinderOF.createItemType(); itemXml.setItemKey(itemKey); itemXml.setItemIsSynonym(false); panelXml.getItem().add(itemXml); } queryDef.getPanel().add(panelXml); } queryBody.setQueryDefinition(queryDef); } /** * Extracts the query definition from the request. * * @return the query definition * @throws JAXBUtilException */ public QueryDefinitionType getQueryDefinition() { return queryBody.getQueryDefinition(); } /** * Resets and sets to the provided output types the list of result output type. * * @param outputTypeNames */ public void setOutputTypes(String[] outputTypeNames) { List outputsList = queryBody.getResultOutputList().getResultOutput(); outputsList.clear(); for (String outputTypeName : outputTypeNames) { ResultOutputOptionType outputType = querySetFinderOF.createResultOutputOptionType(); outputType.setName(outputTypeName); outputsList.add(outputType); } } public String getQueryName() { return queryBody.getQueryDefinition().getQueryName(); } - public UserAuthentication getUserAuthentication() { + @Override + public MedCoI2b2MessageHeader getMessageHeader() { return authentication; } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/fr/I2B2FRCell.java b/src/server/ch/epfl/lca1/medco/i2b2/fr/I2B2FRCell.java index b129741..0608c08 100755 --- a/src/server/ch/epfl/lca1/medco/i2b2/fr/I2B2FRCell.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/fr/I2B2FRCell.java @@ -1,168 +1,168 @@ package ch.epfl.lca1.medco.i2b2.fr; import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; import edu.harvard.i2b2.crc.datavo.i2b2message.RequestMessageType; import edu.harvard.i2b2.crc.datavo.pdo.PatientDataType; import ch.epfl.lca1.medco.i2b2.I2B2Cell; import ch.epfl.lca1.medco.i2b2.I2b2Status; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import edu.harvard.i2b2.crc.loader.datavo.fr.File; import edu.harvard.i2b2.crc.loader.datavo.fr.SendfileRequestType; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.MedCoUtil; import edu.harvard.i2b2.common.exception.I2B2Exception; import edu.harvard.i2b2.common.util.axis2.ServiceClient; import org.apache.axiom.om.OMAbstractFactory; import org.apache.axiom.om.OMElement; import org.apache.axiom.soap.SOAP11Constants; import org.apache.axiom.soap.SOAPEnvelope; import org.apache.axis2.AxisFault; import org.apache.axis2.Constants; import org.apache.axis2.addressing.EndpointReference; import org.apache.axis2.client.OperationClient; import org.apache.axis2.client.Options; import org.apache.axis2.context.MessageContext; import org.apache.axis2.wsdl.WSDLConstants; import org.w3c.dom.Document; import javax.activation.DataHandler; import javax.activation.FileDataSource; import javax.xml.bind.JAXBElement; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import java.io.FileWriter; import java.io.IOException; import java.nio.file.*; // todo: harmonize error handling (i2b2status + exceptions) public class I2B2FRCell extends I2B2Cell { private static final long UPLOAD_TIMEOUT_MS = 10000L; private static final String AXIS2_ACTION_SENDFILE = "urn:sendfileRequest", AXIS2_TMP_DIR = "temp", AXIS2_FILE_SIZE_THRESHOLD = "4000"; - public I2B2FRCell(UserAuthentication auth) { + public I2B2FRCell(MedCoI2b2MessageHeader auth) { super(medCoUtil.getFileRepositoryCellUrl(), auth); // get url } /** * Wrapper around uploadFile to easily upload a Patient Data Object. * * @param pdo the patient data object * @param uploadName the name the file should take on the File Repository * @throws I2B2Exception in case of failure or error */ public void uploadPDO(PatientDataType pdo, String uploadName) throws I2B2Exception { JAXBElement wrappedPdo = pdoOF.createPatientData(pdo); Document doc = MedCoUtil.getMsgUtil().documentFromJAXBElement(wrappedPdo); try { // create the temp file Path tmpFilePath = Files.createTempFile(null, null); // write XML to temp file DOMSource source = new DOMSource(doc); FileWriter writer = new FileWriter(tmpFilePath.toFile()); StreamResult result = new StreamResult(writer); TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); transformer.transform(source, result); // make the request uploadFile(tmpFilePath.toAbsolutePath().toString(), uploadName); // delete temp file //Files.delete(tmpFilePath); //todo: temporarly disabled System.out.println(tmpFilePath.toAbsolutePath()); Files.copy(tmpFilePath, Paths.get("/home/misbach/pdo-temp.xml"), StandardCopyOption.REPLACE_EXISTING); } catch (IOException | TransformerException e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } /** * Upload a file into the file repository. * Succeed if no exception is thrown. * * @param filePath local path of the file to upload * @param uploadName name used on the file repository * @throws I2B2Exception in case of failure or error */ public void uploadFile(String filePath, String uploadName) throws I2B2Exception { // generate axis2 options Options options = new Options(); options.setTo(new EndpointReference(this.getCellUrl())); options.setAction(AXIS2_ACTION_SENDFILE); options.setSoapVersionURI(SOAP11Constants.SOAP_ENVELOPE_NAMESPACE_URI); options.setTimeOutInMilliSeconds(UPLOAD_TIMEOUT_MS); options.setProperty(Constants.Configuration.ENABLE_SWA, Constants.VALUE_TRUE); options.setProperty(Constants.Configuration.CACHE_ATTACHMENTS, Constants.VALUE_TRUE); options.setProperty(Constants.Configuration.ATTACHMENT_TEMP_DIR, AXIS2_TMP_DIR); options.setProperty(Constants.Configuration.FILE_SIZE_THRESHOLD, AXIS2_FILE_SIZE_THRESHOLD); // generate i2b2 request message SendfileRequestType sendfileRequest = loaderFrOF.createSendfileRequestType(); File file = loaderFrOF.createFile(); file.setName(uploadName); file.setOverwrite("true"); sendfileRequest.setUploadFile(file); BodyType bodyType = i2b2OF.createBodyType(); bodyType.getAny().add(loaderFrOF.createSendfileRequest(sendfileRequest)); RequestMessageType requestMessage = createRequestMessage(bodyType); OMElement requestElement = msgUtil.buildOMElement(requestMessage); // request org.apache.axis2.client.ServiceClient sender = null; try { sender = ServiceClient.getServiceClient(); sender.setOptions(options); OperationClient mepClient = sender.createClient(org.apache.axis2.client.ServiceClient.ANON_OUT_IN_OP); // message context and envelope MessageContext mc = new MessageContext(); mc.addAttachment(uploadName, new DataHandler(new FileDataSource(filePath))); mc.setDoingSwA(true); SOAPEnvelope env = OMAbstractFactory.getSOAP11Factory().getDefaultEnvelope(); env.getBody().addChild(requestElement); mc.setEnvelope(env); mepClient.addMessageContext(mc); Logger.debug("Ready to send query: " + requestElement); // send request and interpret answer mepClient.execute(true); MessageContext response = mepClient.getMessageContext(WSDLConstants.MESSAGE_LABEL_IN_VALUE); OMElement frResponse = (OMElement) response.getEnvelope().getBody().getFirstOMChild(); Logger.debug("Received answer to query: " + frResponse); if (msgUtil.parseStatus(frResponse) != I2b2Status.DONE) { throw Logger.error(new I2B2Exception("Request failed.")); } Logger.info("Successfully uploaded file."); } catch (AxisFault e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } finally { if (sender != null) { try{ sender.cleanupTransport(); sender.cleanup(); } catch (AxisFault e) { Logger.warn("Error cleaning up", e); } } } } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/ont/I2B2ONTCell.java b/src/server/ch/epfl/lca1/medco/i2b2/ont/I2B2ONTCell.java index edf0169..de2e14d 100755 --- a/src/server/ch/epfl/lca1/medco/i2b2/ont/I2B2ONTCell.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/ont/I2B2ONTCell.java @@ -1,496 +1,496 @@ package ch.epfl.lca1.medco.i2b2.ont; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import edu.harvard.i2b2.crc.datavo.ontology.*; import edu.harvard.i2b2.crc.datavo.ontology.ConceptType; import ch.epfl.lca1.medco.i2b2.I2B2Cell; import ch.epfl.lca1.medco.i2b2.I2b2Status; import ch.epfl.lca1.medco.i2b2.MessagesUtil; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.util.exceptions.ConceptNotFoundException; import ch.epfl.lca1.medco.util.exceptions.I2B2XMLException; import ch.epfl.lca1.medco.util.exceptions.UnlynxException; import edu.harvard.i2b2.common.exception.I2B2Exception; import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; import org.javatuples.Pair; import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; import java.util.*; // todo: harmonize error handling (i2b2status + exceptions) public class I2B2ONTCell extends I2B2Cell { private static final String URL_PATH_I2B2_LOADMETADATA = "/loadMetadata", URL_PATH_I2B2_GETTERMINFO = "/getTermInfo", URL_PATH_I2B2_GETCHILDREN = "/getChildren", URL_PATH_I2B2_DELETECHILD = "/deleteChild", URL_PATH_I2B2_ADDCHILD = "/addChild", URL_PATH_I2B2_GETCODEINFO = "/getCodeInfo"; public static final String ONT_PATH_ROOT = "\\medco\\", ONT_PATH_CLINICAL = ONT_PATH_ROOT + "clinical\\", ONT_PATH_GENOMIC = ONT_PATH_ROOT + "genomic\\", ONT_PATH_CLINICAL_SENSITIVE = ONT_PATH_CLINICAL + "sensitive\\", ONT_PATH_CLINICAL_NONSENSITIVE = ONT_PATH_CLINICAL + "nonsensitive\\"; public static final String ONT_PATH_NEXT_ID_SENSITIVE = ONT_PATH_CLINICAL_SENSITIVE + "next_usable_id\\", ONT_PATH_NEXT_ID_NONSENSITIVE = ONT_PATH_CLINICAL_NONSENSITIVE + "next_usable_id\\"; public static final String BASECODE_PREFIX_ENC = "MEDCO_ENC", BASECODE_PREFIX_CLEAR = "MEDCO_CLEAR", BASECODE_PREFIX_ADMIN = "MEDCO_ADMIN", BASECODE_PREFIX_GEN = "MEDCO_GEN"; public static final String TABLE_CD_CLINICAL_SENSITIVE = "CLINICAL_SENSITIVE", TABLE_CD_CLINICAL_NON_SENSITIVE = "CLINICAL_NON_SENSITIVE", TABLE_CD_GENOMIC = "GENOMIC"; /** * Pairs of tables in ontology + concepts to be loaded. */ private Map> conceptsListByTable; - public I2B2ONTCell(UserAuthentication auth) { + public I2B2ONTCell(MedCoI2b2MessageHeader auth) { super(medCoUtil.getOntologyCellUrl(), auth); // get url conceptsListByTable = new HashMap<>();//ontOF.createMetadataLoadType(); } // for node: a contianer not queryable, lvl=1, + the values of container, lvl=2 // //public void addConcept(String path, String uniqueName, // long ontologyId, String valueType) throws I2B2Exception { //have enum for categories?? // key: \\C_TABLE_CD\C_FULLNAME //C_TABLE_NAME: medco_sensitive, medco_non_sensitive // // C_TABLE_CD: medco_clinical_sensitive, medco_clinical_non_sensitive, medco_genomic_annotations // fullname: \medco\clinical\sensitive // TODO // basecode === valuetype:id # // value type: UNLYNX_ENC, etc // prefill table access, schemes (categeries in basecode) // String fullName = path + uniqueName + "\\"; // addConcept( // 1, uniqueName, "N", "LA", 0, valueType + ":" + ontologyId, fullName, // "concept_cd", "concept_path", "T", "LIKE", "", // fullName, valueType, fullName, "concept_dimension", "@", new GregorianCalendar(), // null); //} public void accumulateConcept(String name, String path, String visualAttr, String code, String basecodeSuffix, String comment, String ontologyTable, String metadataxml, List pdoConcepts) throws I2B2Exception { String fullName = path + name + "\\"; int lvl = path.split("\\\\").length - 1; String basecode = basecodeSuffix == null ? null : code + ":" + basecodeSuffix; accumulateConcept( lvl, name, "N", visualAttr, 0, basecode, fullName, "concept_cd", "concept_path", "T", "LIKE", comment, fullName, code, fullName, "concept_dimension", "@", new GregorianCalendar(), metadataxml, ontologyTable); Logger.debug("Concept accumulated: " + fullName + ", " + basecode); // concept_dimension table in CRC, only for queriable concepts if (pdoConcepts != null && basecode != null && !basecodeSuffix.equals(BASECODE_PREFIX_ADMIN)) { edu.harvard.i2b2.crc.datavo.pdo.ConceptType pdoConcept = pdoOF.createConceptType(); pdoConcept.setConceptPath(fullName); pdoConcept.setConceptCd(basecode); pdoConcept.setNameChar(name); pdoConcepts.add(pdoConcept); Logger.debug("Concept added to pdo concept dimension: " + fullName + ", " + basecode); } } /* medco ontology: lvl 0 clinical / genomical: lvl 1 etc. doesn't need the table cd here */ private void accumulateConcept(int level, String name, String synonymCd, String visualAttributes, int totalNum, String basecode, String dimcode, String factTableColumn, String columnName, String columnDataType, String operator, String comment, String tooltip, String valueTypeCd, String fullName, String dimtablename, String appliedPath, GregorianCalendar updateDate, String metadataXml, String ontologyTable) throws I2B2Exception { OntologyDataType concept = ontOF.createOntologyDataType(); concept.setLevel(level); concept.setName(name); concept.setSynonymCd(synonymCd); concept.setVisualattributes(visualAttributes); concept.setTotalnum(totalNum); concept.setBasecode(basecode); concept.setFacttablecolumn(factTableColumn); concept.setColumnname(columnName); concept.setColumndatatype(columnDataType); concept.setOperator(operator); concept.setDimcode(dimcode); concept.setComment(comment); concept.setTooltip(tooltip); concept.setValuetypeCd(valueTypeCd); concept.setFullname(fullName); concept.setDimtablename(dimtablename); concept.setAppliedPath(appliedPath); try { concept.setUpdateDate(DatatypeFactory.newInstance().newXMLGregorianCalendar(updateDate)); } catch (DatatypeConfigurationException e) { throw Logger.error(new I2B2XMLException("Could not create calendar", e)); } if (metadataXml != null) { XmlValueType xmlMetadataXml = ontOF.createXmlValueType(); xmlMetadataXml.getAny().add(edu.harvard.i2b2.common.util.xml.XMLUtil.convertStringToDOM(metadataXml).getDocumentElement()); concept.setMetadataxml(xmlMetadataXml); } // ontologyTable if (!conceptsListByTable.containsKey(ontologyTable)) { conceptsListByTable.put(ontologyTable, new ArrayList<>()); } conceptsListByTable.get(ontologyTable).add(concept); } /* public void addAdmin(String path, String uniqueName, int ontologyId, String valueType) throws I2B2Exception { //have enum for categories?? // key: \\C_TABLE_CD\C_FULLNAME //C_TABLE_NAME: medco_sensitive, medco_non_sensitive // // C_TABLE_CD: medco_clinical_sensitive, medco_clinical_non_sensitive, medco_genomic_annotations // fullname: \medco\clinical\sensitive // TODO // basecode === valuetype:id # // value type: UNLYNX_ENC, etc // prefill table access, schemes (categeries in basecode) String fullName = path + uniqueName + "\\"; addConcept( 1, uniqueName, "N", "LA", 0, valueType + ontologyId, fullName, "concept_cd", "concept_path", "T", "LIKE", "", fullName, valueType, fullName, "concept_dimension", "@", new GregorianCalendar(), null); }*/ public String getClearConceptCd(String fieldName, String fieldValue) throws I2B2Exception, UnlynxException { String searchPath = "\\\\" + TABLE_CD_CLINICAL_NON_SENSITIVE + ONT_PATH_CLINICAL_NONSENSITIVE + fieldName + "\\" + fieldValue + "\\"; ConceptsType concepts = getTermInfo(searchPath, "default", false, false, false, 1); if (concepts.getConcept().size() > 1) { throw Logger.error(new UnlynxException("Database inconsistent, size 1 != " + concepts.getConcept().size())); } else if (concepts.getConcept().size() == 0) { throw Logger.warn(new ConceptNotFoundException("Database inconsistent, size 1 != " + concepts.getConcept().size())); } return concepts.getConcept().get(0).getBasecode(); } public long getEncryptId(String fieldName, String fieldValue) throws I2B2Exception, UnlynxException { String searchPath = "\\\\" + TABLE_CD_CLINICAL_SENSITIVE + ONT_PATH_CLINICAL_SENSITIVE + fieldName + "\\" + fieldValue + "\\"; ConceptsType concepts = getTermInfo(searchPath, "default", false, false, false, 1); if (concepts.getConcept().size() > 1) { throw Logger.error(new UnlynxException("Database inconsistent, size 1 != " + concepts.getConcept().size())); } else if (concepts.getConcept().size() == 0) { throw Logger.warn(new ConceptNotFoundException("Database inconsistent, size 1 != " + concepts.getConcept().size())); } try { return Long.parseLong(concepts.getConcept().get(0).getBasecode().split(":")[1]); } catch (NumberFormatException e) { throw Logger.error(new UnlynxException("Database inconsistent", e)); } } public long getNextEncUsableId() throws I2B2Exception, UnlynxException { return getNextUsableId("\\\\" + TABLE_CD_CLINICAL_SENSITIVE + ONT_PATH_NEXT_ID_SENSITIVE); } public long getNextClearUsableId() throws I2B2Exception, UnlynxException { return getNextUsableId("\\\\" + TABLE_CD_CLINICAL_NON_SENSITIVE + ONT_PATH_NEXT_ID_NONSENSITIVE); } private long getNextUsableId(String searchkey) throws I2B2Exception, UnlynxException { ConceptsType concepts = getChildren(searchkey, "default", 1, true, false, false); if (concepts.getConcept().size() != 1) { throw Logger.error(new UnlynxException("Database inconsistent")); } try { return Long.parseLong(concepts.getConcept().get(0).getName()); } catch (NumberFormatException e) { throw Logger.error(new UnlynxException("Database inconsistent", e)); } } public void updateNextEncUsableId(long id) throws I2B2Exception { updateNextUsableId(id, "\\\\" + TABLE_CD_CLINICAL_SENSITIVE + ONT_PATH_NEXT_ID_SENSITIVE, "nextEncUsableId", ONT_PATH_NEXT_ID_SENSITIVE, "The next usable id for Unlynx encryption."); } public void updateNextClearUsableId(long id) throws I2B2Exception { updateNextUsableId(id, "\\\\" + TABLE_CD_CLINICAL_NON_SENSITIVE + ONT_PATH_NEXT_ID_NONSENSITIVE, "nextClearUsableId", ONT_PATH_NEXT_ID_NONSENSITIVE, "The next usable id for non sensitive clinical data."); } private void updateNextUsableId(long id, String searchKey, String basecodeSuffix, String dimcode, String comment) throws I2B2Exception { ConceptsType concepts = getChildren(searchKey, "default", 1, true, false, false); // delete previous one (normally only 1 entry) for (ConceptType concept : concepts.getConcept()) { I2b2Status status = deleteChild(concept.getKey(), concept.getBasecode(), true, concept.getLevel(), concept.getName(), "Y", null); if (status != I2b2Status.DONE) { Logger.warn("deleteChild status: " + status + ", " + status.getStatusMessage()); } else { Logger.info("Deleted old next usable ID"); } } // add the new one String basecode = BASECODE_PREFIX_ADMIN + ":" + basecodeSuffix; I2b2Status status = addChild( 4, searchKey + id + "\\", id + "", basecode, "T", "concept_path", dimcode + id + "\\", "concept_cd", comment, "LH", BASECODE_PREFIX_ADMIN, "N", "LIKE", "concept_dimension", null, 0, new GregorianCalendar(), null); if (status != I2b2Status.DONE) { Logger.warn("addChild status: " + status + ", " + status.getStatusMessage()); } else { Logger.info("Updated next usable ID"); } } public boolean codeExists(String code) throws I2B2Exception { ConceptsType concepts = getCodeInfo(code, "default", "contains", false, true, false); return (concepts.getConcept().size() != 0); } // ------------------------------------------------------------ // raw methods to contact I2b2 cell // ------------------------------------------------------------ private I2b2Status deleteChild(String key, String basecode, boolean includeChildren, int lvl, String name, String synonyms, String visualAttr) throws I2B2Exception { DeleteChildType deleteChild = new DeleteChildType(); deleteChild.setKey(key); deleteChild.setBasecode(basecode); deleteChild.setIncludeChildren(includeChildren); deleteChild.setLevel(lvl); deleteChild.setName(name); deleteChild.setSynonymCd(synonyms); deleteChild.setVisualattribute(visualAttr); BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createDeleteChild(deleteChild)); try { Pair resp = requestToCell(URL_PATH_I2B2_DELETECHILD, body); Logger.info("delete child result: " + resp.getValue0()); return resp.getValue0(); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } private I2b2Status addChild(int lvl, String key, String name, String basecode, String columnDataType, String columnName, String dimcode, String factTableCol, String comment, String visualAttr, String valueType, String synonyms, String operator, String tableName, String tooltip, Integer totalNum, GregorianCalendar updateDate, String metadataXml) throws I2B2Exception { ConceptType add = new ConceptType(); add.setLevel(lvl); add.setKey(key); add.setName(name); add.setBasecode(basecode); add.setColumndatatype(columnDataType); add.setColumnname(columnName); add.setDimcode(dimcode); add.setFacttablecolumn(factTableCol); add.setComment(comment); add.setVisualattributes(visualAttr); add.setValuetypeCd(valueType); add.setSynonymCd(synonyms); add.setOperator(operator); add.setTablename(tableName); add.setTooltip(tooltip); add.setTotalnum(totalNum); try { add.setUpdateDate(DatatypeFactory.newInstance().newXMLGregorianCalendar(updateDate)); } catch (DatatypeConfigurationException e) { throw Logger.error(new I2B2XMLException("Could not create calendar", e)); } if (metadataXml != null) { XmlValueType xmlMetadataXml = ontOF.createXmlValueType(); xmlMetadataXml.getAny().add(edu.harvard.i2b2.common.util.xml.XMLUtil.convertStringToDOM(metadataXml).getDocumentElement()); add.setMetadataxml(xmlMetadataXml); } BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createAddChild(add)); try { Pair resp = requestToCell(URL_PATH_I2B2_ADDCHILD, body); Logger.info("add child result: " + resp.getValue0()); return resp.getValue0(); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } private ConceptsType getChildren(String parent, String type, int max, boolean hiddens, boolean synonyms, boolean blob) throws I2B2Exception { GetChildrenType getChildren = new GetChildrenType(); getChildren.setMax(max);//1 getChildren.setHiddens(hiddens); getChildren.setSynonyms(synonyms); getChildren.setType(type);// default, all, core getChildren.setParent(parent); getChildren.setBlob(blob); BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createGetChildren(getChildren)); try { Pair resp = requestToCell(URL_PATH_I2B2_GETCHILDREN, body); Logger.info("get children result: " + resp.getValue0() + ", " + resp.getValue0().getStatusMessage()); return (ConceptsType) msgUtil.getUnwrapHelper().getObjectByClass(resp.getValue1().getAny(), ConceptsType.class); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } /** * TODO doc * Sends request to i2b2 ontology cell of loading the concepts accumulated with addConcept in the field conceptsList. * conceptsList is emptied if the process succeed.XXX * * @param path the ontology table in which the concept should be stored * @param dataType: default, code, all * @return the {@link I2b2Status} * @throws I2B2Exception if the i2b2 request fails not gracefully */ private ConceptsType getTermInfo(String path, String dataType, boolean getBlob, boolean getHiddens, boolean getSynonyms, int max) throws I2B2Exception { GetTermInfoType termInfo = ontOF.createGetTermInfoType(); termInfo.setBlob(getBlob); termInfo.setHiddens(getHiddens); termInfo.setSynonyms(getSynonyms); termInfo.setType(dataType); termInfo.setMax(max); termInfo.setSelf(path); BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createGetTermInfo(termInfo)); try { Pair resp = requestToCell(URL_PATH_I2B2_GETTERMINFO, body); Logger.info("get term info result: " + resp.getValue0()); return (ConceptsType) MessagesUtil.getUnwrapHelper().getObjectByClass( resp.getValue1().getAny(), ConceptsType.class); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } private ConceptsType getCodeInfo(String code, String dataType, String strategy, boolean getBlob, boolean getHiddens, boolean getSynonyms) throws I2B2Exception { VocabRequestType vocab = ontOF.createVocabRequestType(); vocab.setBlob(getBlob); vocab.setHiddens(getHiddens); vocab.setSynonyms(getSynonyms); vocab.setType(dataType); MatchStrType matchStr = ontOF.createMatchStrType(); matchStr.setStrategy(strategy); matchStr.setValue(code); vocab.setMatchStr(matchStr); BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createGetCodeInfo(vocab)); try { Pair resp = requestToCell(URL_PATH_I2B2_GETCODEINFO, body); Logger.info("get code info result: " + resp.getValue0()); return (ConceptsType) resp.getValue1().getAny().get(0); } catch (Exception e) { throw Logger.error(new I2B2Exception("Request failed.", e)); } } /** * Sends request to i2b2 ontology cell of loading the concepts accumulated with addConcept in the field conceptsList. * conceptsList is emptied if the process succeed. * Send one message per annotation! * * @return the {@link I2b2Status} * @throws I2B2Exception if the i2b2 request fails not gracefully */ public I2b2Status loadConcepts() throws I2B2Exception { I2b2Status reqStatus = null; int conceptsLoadedCount = 0; for (Map.Entry> entries : conceptsListByTable.entrySet()) { for (OntologyDataType concept : entries.getValue()) { MetadataLoadType loadType = ontOF.createMetadataLoadType(); loadType.setTableName(entries.getKey()); loadType.getMetadata().add(concept); BodyType body = i2b2OF.createBodyType(); body.getAny().add(ontOF.createLoadMetadata(loadType)); try { Pair resp = requestToCell(URL_PATH_I2B2_LOADMETADATA, body); reqStatus = resp.getValue0(); Logger.info("load metadata result: " + reqStatus + ", msg: " + reqStatus.getStatusMessage()); if (reqStatus != I2b2Status.DONE) { Logger.error("Request failed for " + concept.getName() + ", continuing"); //return reqStatus; continue; } } catch (Exception e) { Logger.error("Request failed for " + concept.getName() + ", continuing", e); } conceptsLoadedCount++; } } // everything loaded, clear the concepts Logger.info("Loaded " + conceptsLoadedCount + " concepts in i2b2 ontology"); conceptsListByTable.clear(); return reqStatus; } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/pm/I2B2PMCell.java b/src/server/ch/epfl/lca1/medco/i2b2/pm/I2B2PMCell.java index 00c5469..c78058c 100755 --- a/src/server/ch/epfl/lca1/medco/i2b2/pm/I2B2PMCell.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/pm/I2B2PMCell.java @@ -1,178 +1,178 @@ package ch.epfl.lca1.medco.i2b2.pm; import edu.harvard.i2b2.crc.datavo.i2b2message.MessageHeaderType; import edu.harvard.i2b2.crc.datavo.i2b2message.RequestMessageType; import edu.harvard.i2b2.crc.datavo.pm.ProjectType; import org.apache.axiom.om.OMElement; import ch.epfl.lca1.medco.util.Logger; import ch.epfl.lca1.medco.i2b2.I2B2Cell; import edu.harvard.i2b2.common.exception.I2B2Exception; //import edu.harvard.i2b2.crc.datavo.CRCJAXBUtil; //import edu.harvard.i2b2.crc.datavo.i2b2message.BodyType; //import edu.harvard.i2b2.crc.datavo.i2b2message.FacilityType; //import edu.harvard.i2b2.crc.datavo.i2b2message.MessageHeaderType; //import edu.harvard.i2b2.crc.datavo.i2b2message.RequestHeaderType; //import edu.harvard.i2b2.crc.datavo.i2b2message.RequestMessageType; //import edu.harvard.i2b2.crc.datavo.i2b2message.ResponseHeaderType; //import edu.harvard.i2b2.crc.datavo.i2b2message.ResponseMessageType; //import edu.harvard.i2b2.crc.datavo.i2b2message.SecurityType; //import edu.harvard.i2b2.crc.datavo.i2b2message.StatusType; //import edu.harvard.i2b2.crc.datavo.pm.ConfigureType; //import edu.harvard.i2b2.crc.datavo.pm.GetUserConfigurationType; //import edu.harvard.i2b2.crc.datavo.pm.ObjectFactory; //import edu.harvard.i2b2.crc.datavo.pm.ProjectType; //import edu.harvard.i2b2.crc.datavo.pm.UserType; //import edu.harvard.i2b2.crc.util.QueryProcessorUtil; public class I2B2PMCell extends I2B2Cell { // take from crc query request delegate // fmethod check validuser does nothing escept call the callpmutil // public static String callUserResponse(SecurityType securityType, String projectId ) throws AxisFault, I2B2Exception { // RequestMessageType requestMessageType = getI2B2RequestMessage(securityType, projectId); // OMElement requestElement = null; // String response = null; // try { // requestElement = buildOMElement(requestMessageType); // log.debug("CRC PM call's request xml " + requestElement); // response = ServiceClient.sendREST(QueryProcessorUtil.getInstance() // .getProjectManagementCellUrl(), requestElement); // log.debug("Got Response"); // } catch (XMLStreamException e) { // e.printStackTrace(); // throw new I2B2Exception("" + StackTraceUtil.getStackTrace(e)); // } catch (Exception e) { // e.printStackTrace(); // throw new I2B2Exception("" + StackTraceUtil.getStackTrace(e)); // } // // log.debug("Returning ProjectType"); // return response; // } - public I2B2PMCell(String pmCellUrl, UserAuthentication medcoI2b2Auth) { + public I2B2PMCell(String pmCellUrl, MedCoI2b2MessageHeader medcoI2b2Auth) { super(medCoUtil.getProjectManagementCellUrl(), null); // get url } /** * Returns the Project part of the authentication with the PM. * Contains TODO... * * @return * * @throws I2B2Exception if the original request message is missing required parts */ - public UserAuthentication doAuthentication(RequestMessageType origReqMessage) { + public MedCoI2b2MessageHeader doAuthentication(RequestMessageType origReqMessage) { if (origReqMessage == null || origReqMessage.getMessageHeader() == null || origReqMessage.getMessageHeader().getSecurity() == null) { Logger.warn("Authentication failed because some parts of the original request message is missing."); - return null;// new UserAuthentication(origReqMessage, false); + return null;// new MedCoI2b2MessageHeader(origReqMessage, false); } // construct authentication request //RequestMessageType authReqMessage = newReqMessageFromReqMessage(origReqMessage); OMElement requestElement = null; ProjectType projectType = null; //TODO change: have a object connecting to PM return null; } public UserInformation getUserInformation(MessageHeaderType messageHeader) { //todo: implement me return new UserInformation(); } /* public static ProjectType callUserProject(SecurityType securityType, String projectId ) throws AxisFault, I2B2Exception { RequestMessageType requestMessageType = getI2B2RequestMessage(securityType, projectId); OMElement requestElement = null; ProjectType projectType = null; try { requestElement = buildOMElement(requestMessageType); log.debug("CRC PM call's request xml " + requestElement); String response = ServiceClient.sendREST(QueryProcessorUtil.getInstance() .getProjectManagementCellUrl(), requestElement); log.debug("Got Response"); projectType = getUserProjectFromResponse(response, securityType, projectId); log.debug("Parsed Projcet Type: " + projectType.getName()); } catch (XMLStreamException e) { e.printStackTrace(); throw new I2B2Exception("" + StackTraceUtil.getStackTrace(e)); } catch (Exception e) { e.printStackTrace(); throw new I2B2Exception("" + StackTraceUtil.getStackTrace(e)); } log.debug("Returning ProjectType"); return projectType; } public static ProjectType getUserProjectFromResponse(String responseXml, SecurityType securityType, String projectId) throws JAXBUtilException, I2B2Exception { JAXBElement responseJaxb = msgUtil.unMashallFromString(responseXml); //CRCJAXBUtil.getJAXBUtil().unMashallFromString(responseXml); ResponseMessageType pmRespMessageType = (ResponseMessageType) responseJaxb .getValue(); log.debug("CRC's PM call response xml" + responseXml); ResponseHeaderType responseHeader = pmRespMessageType .getResponseHeader(); StatusType status = responseHeader.getResultStatus().getStatus(); String procStatus = status.getType(); String procMessage = status.getValue(); if (procStatus.equals("ERROR")) { log.info("PM Error reported by CRC web Service " + procMessage); throw new I2B2Exception("PM Error reported by CRC web Service " + procMessage); } else if (procStatus.equals("WARNING")) { log.info("PM Warning reported by CRC web Service" + procMessage); throw new I2B2Exception("PM Warning reported by CRC web Service" + procMessage); } JAXBUnWrapHelper helper = new JAXBUnWrapHelper(); ConfigureType configureType = (ConfigureType) helper.getObjectByClass( pmRespMessageType.getMessageBody().getAny(), ConfigureType.class); UserType userType = configureType.getUser(); List projectTypeList = userType.getProject(); ProjectType projectType = null; if (projectTypeList != null && projectTypeList.size() > 0) { for (ProjectType pType : projectTypeList) { if (pType.getId().equalsIgnoreCase(projectId)) { projectType = pType; break; } } if (projectType == null) { throw new I2B2Exception("User not registered to the project[" + projectId + "]"); } } return projectType; }*/ } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/pm/UserAuthentication.java b/src/server/ch/epfl/lca1/medco/i2b2/pm/MedCoI2b2MessageHeader.java similarity index 88% rename from src/server/ch/epfl/lca1/medco/i2b2/pm/UserAuthentication.java rename to src/server/ch/epfl/lca1/medco/i2b2/pm/MedCoI2b2MessageHeader.java index 701c45e..745f31b 100644 --- a/src/server/ch/epfl/lca1/medco/i2b2/pm/UserAuthentication.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/pm/MedCoI2b2MessageHeader.java @@ -1,63 +1,63 @@ package ch.epfl.lca1.medco.i2b2.pm; import edu.harvard.i2b2.crc.datavo.i2b2message.*; import ch.epfl.lca1.medco.util.MedCoUtil; /** * Simple wrapper around the {@link MessageHeaderType} class that contains the I2B2 credentials. */ -public class UserAuthentication extends MessageHeaderType { +public class MedCoI2b2MessageHeader extends MessageHeaderType { private static MedCoUtil util = MedCoUtil.getInstance(); private static final String SENDING_FACILITY_NAME = "i2b2_medco", RECEIVING_FACILITY_NAME = "i2b2", RECEIVING_APPLICATION = "i2b2 cell", RECEIVING_APPLICATION_VERSION = "1.7"; - public UserAuthentication(String domainId, String projectId, String username, - boolean passwordIsToken, int tokenTimeoutMs, String passwordValue) { + public MedCoI2b2MessageHeader(String domainId, String projectId, String username, + boolean passwordIsToken, int tokenTimeoutMs, String passwordValue) { // XXX message control id [messagenum + sessionid] not implemented SecurityType security = new SecurityType(); security.setDomain(domainId); security.setUsername(username); PasswordType password = new PasswordType(); password.setIsToken(passwordIsToken); password.setTokenMsTimeout(tokenTimeoutMs); password.setValue(passwordValue); security.setPassword(password); setSecurity(security); ApplicationType sendingApp = new ApplicationType(); sendingApp.setApplicationName(util.getApplicationName()); //"MedCo cell" sendingApp.setApplicationVersion(util.getApplicationVersion()); // "0.01" setSendingApplication(sendingApp); ApplicationType receivingApp = new ApplicationType(); receivingApp.setApplicationName(RECEIVING_APPLICATION); receivingApp.setApplicationVersion(RECEIVING_APPLICATION_VERSION); setReceivingApplication(receivingApp); FacilityType sendingFacility = new FacilityType(); sendingFacility.setFacilityName(SENDING_FACILITY_NAME); setSendingFacility(sendingFacility); FacilityType recvFacility = new FacilityType(); recvFacility.setFacilityName(RECEIVING_FACILITY_NAME); setReceivingFacility(recvFacility); setProjectId(projectId); } - public UserAuthentication(MessageHeaderType messageHeader) { + public MedCoI2b2MessageHeader(MessageHeaderType messageHeader) { this(messageHeader.getSecurity().getDomain(), messageHeader.getProjectId(), messageHeader.getSecurity().getUsername(), messageHeader.getSecurity().getPassword().isIsToken(), messageHeader.getSecurity().getPassword().getTokenMsTimeout() == null ? 0 : messageHeader.getSecurity().getPassword().getTokenMsTimeout(), messageHeader.getSecurity().getPassword().getValue()); } } diff --git a/src/server/ch/epfl/lca1/medco/i2b2/pm/UserInformation.java b/src/server/ch/epfl/lca1/medco/i2b2/pm/UserInformation.java index 529e53c..2690d2a 100644 --- a/src/server/ch/epfl/lca1/medco/i2b2/pm/UserInformation.java +++ b/src/server/ch/epfl/lca1/medco/i2b2/pm/UserInformation.java @@ -1,45 +1,45 @@ package ch.epfl.lca1.medco.i2b2.pm; import edu.harvard.i2b2.crc.datavo.pm.RoleType; import edu.harvard.i2b2.crc.datavo.pm.RolesType; // todo: implement me public class UserInformation { - private UserAuthentication auth; + private MedCoI2b2MessageHeader auth; - public UserAuthentication getAuth() { + public MedCoI2b2MessageHeader getAuth() { return auth; } public double getPrivacyBudget() { return Double.MAX_VALUE; } public boolean isAuthenticated() { // todo: implement me return true; } public RolesType getRoles() { // todo: implement me RolesType tmpRoles = new RolesType(); RoleType role = new RoleType(); role.setProjectId("Demo"); role.setRole("DATA_AGG"); role.setUserName("demo"); tmpRoles.getRole().add(role); return tmpRoles; } public String getUsername() { // todo: implement me return "Jean Ca"; } public String getUserPublicKey() { return "eQviK90cvJ2lRx8ox6GgQKFmOtbgoG9RXa7UnmemtRA="; } } diff --git a/src/test/ch/epfl/lca1/medco/i2b2/I2b2CellsTests.java b/src/test/ch/epfl/lca1/medco/i2b2/I2b2CellsTests.java index ba13e0c..cb75301 100644 --- a/src/test/ch/epfl/lca1/medco/i2b2/I2b2CellsTests.java +++ b/src/test/ch/epfl/lca1/medco/i2b2/I2b2CellsTests.java @@ -1,73 +1,73 @@ package ch.epfl.lca1.medco.i2b2; import ch.epfl.lca1.medco.i2b2.fr.I2B2FRCell; import ch.epfl.lca1.medco.i2b2.ont.I2B2ONTCell; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import ch.epfl.lca1.medco.util.MedCoUtil; import ch.epfl.lca1.medco.util.exceptions.UnlynxException; import edu.harvard.i2b2.common.exception.I2B2Exception; import edu.harvard.i2b2.common.util.jaxb.JAXBUtilException; import org.junit.Test; import javax.xml.datatype.DatatypeConfigurationException; import static org.junit.Assert.*; /** * Created by misbach on 12.06.17. */ public class I2b2CellsTests { static { MedCoUtil.getTestInstance().setProperty(MedCoUtil.ONTCELL_WS_URL_PROPERTIES, "http://localhost:8080/i2b2/services/OntologyService"); MedCoUtil.getTestInstance().setProperty(MedCoUtil.FRCELL_WS_URL_PROPERTIES, "http://localhost:8080/i2b2/services/FRService"); } - private UserAuthentication auth = new UserAuthentication("i2b2demotest", "Demo", + private MedCoI2b2MessageHeader auth = new MedCoI2b2MessageHeader("i2b2demotest", "Demo", "demo", false, 0, "demouser"); private I2B2ONTCell ontCell = new I2B2ONTCell(auth); private I2B2FRCell frCell = new I2B2FRCell(auth); @Test public void ontAddConceptTest() throws I2B2Exception, JAXBUtilException, DatatypeConfigurationException { ontCell.accumulateConcept("test1", "\\test1\\", "FA", "codetest", "suffix", "comment", "CUSTOM_META", null, null); ontCell.accumulateConcept("test2", "\\test2\\", "FA", "codetest", "suffix2", "comment", "CUSTOM_META", null, null); I2b2Status res = ontCell.loadConcepts(); System.out.println(res); System.out.println(res.getStatusMessage()); assertEquals(res, I2b2Status.DONE); } @Test public void ontIdTest() throws I2B2Exception, UnlynxException { ontCell.updateNextEncUsableId(3); ontCell.updateNextClearUsableId(40); long idE = ontCell.getNextEncUsableId(); long idC = ontCell.getNextClearUsableId(); assertEquals(idE, 3); assertEquals(idC, 40); System.out.println(idE + " , " + idC); ontCell.updateNextEncUsableId(idE + 1); ontCell.updateNextClearUsableId(idC + 1); } @Test public void frUploadTest() throws I2B2Exception { try { frCell.uploadFile("testfiles/medco_request_query_def_reference.xml", "testFile.xml"); } catch (Throwable t) { t.printStackTrace(); fail(t.getMessage()); } assertTrue(true); } } diff --git a/src/test/ch/epfl/lca1/medco/loader/MedCoDataLoaderTests.java b/src/test/ch/epfl/lca1/medco/loader/MedCoDataLoaderTests.java index 074bd8b..20de6fc 100644 --- a/src/test/ch/epfl/lca1/medco/loader/MedCoDataLoaderTests.java +++ b/src/test/ch/epfl/lca1/medco/loader/MedCoDataLoaderTests.java @@ -1,482 +1,482 @@ package ch.epfl.lca1.medco.loader; import ch.epfl.lca1.medco.ConfiguredTests; import ch.epfl.lca1.medco.MedCoDataLoader; import ch.epfl.lca1.medco.i2b2.ont.I2B2ONTCell; -import ch.epfl.lca1.medco.i2b2.pm.UserAuthentication; +import ch.epfl.lca1.medco.i2b2.pm.MedCoI2b2MessageHeader; import ch.epfl.lca1.medco.loader.clinical.OntologyLoader; import ch.epfl.lca1.medco.util.exceptions.UnlynxException; import edu.harvard.i2b2.common.exception.I2B2Exception; import org.junit.Test; import java.io.IOException; /** * Created by misbach on 16.06.17. */ public class MedCoDataLoaderTests extends ConfiguredTests { @Test public void testClinicalOntLoader() throws I2B2Exception, UnlynxException { String[] headers = new String[] { "sample_id", "patient_id", "test_enc_1", "test enc 2", "some clear1", "clear2", "ency3" }; DataType[] types = new DataType[] { DataType.SAMPLE_ID, DataType.PATIENT_ID, DataType.ENC, DataType.ENC, DataType.CLEAR, DataType.CLEAR, DataType.ENC }; - UserAuthentication auth = new UserAuthentication("i2b2demotest", "Demo", "demo", false, 0, "demouser"); + MedCoI2b2MessageHeader auth = new MedCoI2b2MessageHeader("i2b2demotest", "Demo", "demo", false, 0, "demouser"); OntologyLoader loader = new OntologyLoader(headers, types, "chuv"); loader.parseEntry(new String[]{"22", "333F", "A", "B", "A", "33", "5630"}); loader.parseEntry(new String[]{"22", "33F", "B", "B", "B", "33", "511630"}); loader.parseEntry(new String[]{"223", "33F", "A", "C", "A", "353", "5630"}); loader.parseEntry(new String[]{"22F", "33f3F", "B", "B", "C", "333", "5630"}); loader.parseEntry(new String[]{"22F", "3f33F", "C", "L", "A", "33", "563220"}); loader.loadOntology(new I2B2ONTCell(auth)); } @Test public void testClinicalLoader() throws UnlynxException, IOException, I2B2Exception { String clinicalPath_full = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad.txt", clinicalPath_part1 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_part1.txt", clinicalPath_part2 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_part2.txt", clinicalPath_part3 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_part3.txt"; // --- mapping for the test dataset //SAMPLE_ID PATIENT_ID AGE DISTANT_METASTASIS_PATHOLOGIC_SPREAD GENDER // HISTOLOGICAL_SUBTYPE ICD_O_3_HISTOLOGY ICD_O_3_SITE PRETREATMENT_HISTORY // PRIMARY_TUMOR_PATHOLOGIC_SPREAD PRIOR_DIAGNOSIS RESIDUAL_TUMOR // TOBACCO_SMOKING_HISTORY_INDICATOR TUMOR_STAGE_2009 OS_STATUS OS_MONTHS // DFS_STATUS DFS_MONTHS CANCER_TYPE CANCER_TYPE_DETAILED /* DataType[] typesClinical = new DataType[]{ DataType.SAMPLE_ID, DataType.PATIENT_ID, DataType.CLEAR, DataType.ENC, DataType.CLEAR, DataType.ENC, DataType.ENC, DataType.ENC, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.ENC, DataType.IGNORE, DataType.ENC, DataType.CLEAR, DataType.CLEAR, DataType.ENC, DataType.ENC, DataType.CLEAR, DataType.IGNORE };*/ // --- mapping for the skcm_broad dataset // SAMPLE_ID PATIENT_ID GENDER(C) AGE(C) AGE_AT_PROCUREMENT(C) PRIMARY_DIAGNOSIS(C) // [PRIMARY_TUMOR_LOCALIZATION_TYPE](S) // PRIMARY_SITE(C) BRESLOW_DEPTH(C) CANCER_TYPE(C) [CANCER_TYPE_DETAILED](S) DataType[] typesClinical = new DataType[]{ DataType.SAMPLE_ID, DataType.PATIENT_ID, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.ENC, DataType.CLEAR, DataType.CLEAR, DataType.CLEAR, DataType.ENC }; { loadSrv1Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv1", "skcm_broad_part1", "i2b2demotest", "Demo", "demo", "demouser"); loader.loadClinicalFileOntology(clinicalPath_full, '\t', '\u0000', 5, typesClinical); loader.loadClinicalFileData(clinicalPath_part1, '\t', '\u0000', 0, typesClinical); } { loadSrv3Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv2", "skcm_broad_part2", "i2b2demotest", "Demo", "demo", "demouser"); loader.loadClinicalFileOntology(clinicalPath_full, '\t', '\u0000', 5, typesClinical); loader.loadClinicalFileData(clinicalPath_part2, '\t', '\u0000', 0, typesClinical); } { loadSrv5Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv3", "skcm_broad_part3", "i2b2demotest", "Demo", "demo", "demouser"); loader.loadClinicalFileOntology(clinicalPath_full, '\t', '\u0000', 5, typesClinical); loader.loadClinicalFileData(clinicalPath_part3, '\t', '\u0000', 0, typesClinical); } /* 1 demo /opt/FRC_files/Demo/pdoUploadchuv.xml COMPLETED 2017-07-20T16:54:25.121Z 2017-07-20T16:54:26.761Z */ } @Test public void testGenomicLoader() throws UnlynxException, IOException, I2B2Exception { String genomicPath_part1 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_part1.txt", genomicPath_part2 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_part2.txt", genomicPath_part3 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_part3.txt"; DataType[] typesGenomic = new DataType[]{ // Hugo_Symbol Entrez_Gene_Id Center NCBI_Build Chromosome Start_Position End_Position Strand DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.CHROMOSOME, DataType.START_POS, DataType.ANNOTATION, DataType.ANNOTATION, // Variant_Classification Variant_Type Reference_Allele Tumor_Seq_Allele1 Tumor_Seq_Allele2 dbSNP_RS DataType.ANNOTATION, DataType.ANNOTATION, DataType.REF_ALLELES, DataType.ALT_ALLELES, DataType.ANNOTATION, DataType.ANNOTATION, // dbSNP_Val_Status Tumor_Sample_Barcode Matched_Norm_Sample_Barcode Match_Norm_Seq_Allele1 DataType.ANNOTATION, DataType.SAMPLE_ID, DataType.ANNOTATION, DataType.ANNOTATION, // Match_Norm_Seq_Allele2 Tumor_Validation_Allele1 Tumor_Validation_Allele2 Match_Norm_Validation_Allele1 DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Match_Norm_Validation_Allele2 Verification_Status Validation_Status Mutation_Status Sequencing_Phase DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Sequence_Source Validation_Method Score BAM_File Sequencer MA:FImpact MA:FIS MA:protein.change DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // MA:link.MSA MA:link.PDB MA:link.var Tumor_Sample_UUID Matched_Norm_Sample_UUID HGVSc HGVSp DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // HGVSp_Short Transcript_ID Exon_Number t_depth t_ref_count t_alt_count n_depth n_ref_count n_alt_count DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // all_effects Allele Gene Feature Feature_type Consequence cDNA_position CDS_position DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Protein_position Amino_acids Codons Existing_variation ALLELE_NUM DISTANCE SYMBOL SYMBOL_SOURCE DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // HGNC_ID BIOTYPE CANONICAL CCDS ENSP SWISSPROT TREMBL UNIPARC RefSeq SIFT PolyPhen EXON DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // INTRON DOMAINS GMAF AFR_MAF AMR_MAF ASN_MAF EAS_MAF EUR_MAF SAS_MAF AA_MAF EA_MAF CLIN_SIG DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // SOMATIC PUBMED MOTIF_NAME MOTIF_POS HIGH_INF_POS MOTIF_SCORE_CHANGE IMPACT PICK VARIANT_CLASS DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // TSL HGVS_OFFSET PHENO MINIMISED ExAC_AF ExAC_AF_AFR ExAC_AF_AMR ExAC_AF_EAS ExAC_AF_FIN ExAC_AF_NFE DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // ExAC_AF_OTH ExAC_AF_SAS GENE_PHENO FILTER DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION }; // dataset test /* DataType[] typesGenomic = new DataType[] { // Hugo_Symbol Entrez_Gene_Id Center NCBI_Build Chromosome Start_Position End_Position Strand DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.CHROMOSOME, DataType.START_POS, DataType.ANNOTATION, DataType.ANNOTATION, // Variant_Classification Variant_Type Reference_Allele Tumor_Seq_Allele1 Tumor_Seq_Allele2 dbSNP_RS DataType.ANNOTATION, DataType.ANNOTATION, DataType.REF_ALLELES, DataType.ALT_ALLELES, DataType.ANNOTATION, DataType.ANNOTATION, // dbSNP_Val_Status Tumor_Sample_Barcode Matched_Norm_Sample_Barcode Match_Norm_Seq_Allele1 Match_Norm_Seq_Allele2 DataType.ANNOTATION, DataType.IGNORE, DataType.IGNORE, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, // Tumor_Validation_Allele1 Tumor_Validation_Allele2 Match_Norm_Validation_Allele1 Match_Norm_Validation_Allele2 DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, // Verification_Status Validation_Status Mutation_Status Sequencing_Phase Sequence_Source Validation_Method DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Score BAM_File Sequencer MA:FImpact MA:FIS MA:protein.change MA:link.MSA MA:link.PDB MA:link.var DataType.ANNOTATION, DataType.ANNOTATION, DataType.SAMPLE_DATA, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Tumor_Sample_UUID Matched_Norm_Sample_UUID HGVSc HGVSp HGVSp_Short Transcript_ID Exon_Number DataType.IGNORE, DataType.IGNORE, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // t_depth t_ref_count t_alt_count n_depth n_ref_count n_alt_count all_effects Allele Gene Feature Feature_type DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.SAMPLE_DATA, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // Consequence cDNA_position CDS_position Protein_position Amino_acids Codons Existing_variation ALLELE_NUM DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // DISTANCE SYMBOL SYMBOL_SOURCE HGNC_ID BIOTYPE CANONICAL CCDS ENSP SWISSPROT TREMBL UNIPARC RefSeq SIFT DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // PolyPhen EXON INTRON DOMAINS GMAF AFR_MAF AMR_MAF ASN_MAF EAS_MAF EUR_MAF SAS_MAF AA_MAF EA_MAF CLIN_SIG DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // SOMATIC PUBMED MOTIF_NAME MOTIF_POS HIGH_INF_POS MOTIF_SCORE_CHANGE IMPACT PICK VARIANT_CLASS DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // TSL HGVS_OFFSET PHENO MINIMISED ExAC_AF ExAC_AF_AFR ExAC_AF_AMR ExAC_AF_EAS ExAC_AF_FIN ExAC_AF_NFE ExAC_AF_OTH DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, // ExAC_AF_SAS GENE_PHENO FILTER SAMPLE_ID DataType.ANNOTATION, DataType.ANNOTATION, DataType.ANNOTATION, DataType.SAMPLE_ID }; */ { loadSrv1Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv1", "skcm_broad_part1","i2b2demotest", "Demo", "demo", "demouser"); loader.loadGenomicFile(genomicPath_part1, '\t', '\u0000',0, typesGenomic); } { loadSrv3Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv2", "skcm_broad_part2","i2b2demotest", "Demo", "demo", "demouser"); loader.loadGenomicFile(genomicPath_part2, '\t', '\u0000',0, typesGenomic); } { loadSrv5Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv3", "skcm_broad_part3","i2b2demotest", "Demo", "demo", "demouser"); loader.loadGenomicFile(genomicPath_part3, '\t', '\u0000',0, typesGenomic); } } @Test public void testTranslateIdsToNums() throws UnlynxException, IOException, I2B2Exception { { loadSrv1Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv1", "skcm_broad_part1","i2b2demotest", "Demo", "demo", "demouser"); loader.translateIdsToNums(); } { loadSrv3Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv2", "skcm_broad_part2","i2b2demotest", "Demo", "demo", "demouser"); loader.translateIdsToNums(); } { loadSrv5Conf(); MedCoDataLoader loader = new MedCoDataLoader("chuv3", "skcm_broad_part3","i2b2demotest", "Demo", "demo", "demouser"); loader.translateIdsToNums(); } } }