Diffusion MedCo I2b2 Cell (master) Browse

DatasetsManipulations.java
27ae07e5b3damaster

DatasetsManipulations.java
View Options

			package ch.epfl.lca1.medco;

			import ch.epfl.lca1.medco.unlynx.UnlynxEncrypt;
			import ch.epfl.lca1.medco.util.MedCoUtil;
			import com.opencsv.CSVReader;
			import com.opencsv.CSVWriter;
			import org.apache.log4j.Level;
			import org.apache.log4j.Logger;
			import org.postgresql.ds.PGSimpleDataSource;

			import java.io.*;
			import java.util.HashMap;
			import java.util.HashSet;
			import java.util.Map;
			import java.util.Set;

			/**
			* Created by misbach on 20.07.17.
			*/
			public class DatasetsManipulations {

			public static void main(String[] args) throws IOException {
			//splitting();
			//generation();
			getInfoUseCases();
			//generateClearGenomicDataset();
			//timeEncrypt();
			//generateHistogramsForLeakage();
			}

			public static void generateHistogramsForLeakage() throws IOException {
			// generate histograms

			//todo: MedCoLoadingClient.loadSrv1Conf();
			String genomicFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad.txt",
			clinicalFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad.txt",
			clearGenomicOutputPath1 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/histogram_nb_patients.txt",
			clearGenomicOutputPath2 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/histogram_nb_facts.txt";

			// maps sample id -> patient id
			Map<String, String> samplePatientMap = new HashMap<>();

			// read clinical: store patient id - sample ids mapping
			{
			CSVReader clinicalReader = new CSVReader(new FileReader(clinicalFilePath), '\t', '\u0000', 5);
			String[] clinicalHeader = clinicalReader.readNext();

			// headers idx clinical
			int sampleIdId = -1, patientIdId = -1;
			for (int i = 0; i < clinicalHeader.length; i++) {
			if (clinicalHeader[i].trim().equals("SAMPLE_ID")) {
			sampleIdId = i;
			} else if (clinicalHeader[i].trim().equals("PATIENT_ID")) {
			patientIdId = i;
			}
			}

			// dataset store mapping
			String[] clinicalEntry;
			while ((clinicalEntry = clinicalReader.readNext()) != null) {
			try {
			samplePatientMap.put(clinicalEntry[sampleIdId], clinicalEntry[patientIdId]);
			} catch (Throwable e) {
			System.err.println("ignoring clinical entry ...");
			}
			}
			}

			// map variant id -> patients set
			Map<Long, Set<String>> nbPatientsMap = new HashMap<>();
			// map patient -> nb facts
			Map<String, Integer> nbFacts = new HashMap<>();
			{

			CSVReader genomicReader = new CSVReader(new FileReader(genomicFilePath), '\t', '\u0000', 1);
			String[] genomicHeader = genomicReader.readNext();


			// headers idx genomic
			int sampleIdId = -1, chromId = -1, startPosId = -1, altAlleleId = -1, refAlleleId = -1;
			for (int i = 0; i < genomicHeader.length; i++) {
			if (genomicHeader[i].trim().equals("Tumor_Sample_Barcode")) {
			sampleIdId = i;
			} else if (genomicHeader[i].trim().equals("Chromosome")) {
			chromId = i;
			} else if (genomicHeader[i].trim().equals("Start_Position")) {
			startPosId = i;
			} else if (genomicHeader[i].trim().equals("Reference_Allele")) {
			refAlleleId = i;
			} else if (genomicHeader[i].trim().equals("Tumor_Seq_Allele1")) {
			altAlleleId = i;
			}
			}

			// dataset genomic reading + generating [sample_id, patient_id, variant_id]
			String[] genomicEntry;
			while ((genomicEntry = genomicReader.readNext()) != null) {
			try {
			// extract from entry
			long variantId = EncryptedIdentifiersManager.getVariantId(genomicEntry[chromId].trim(),
			Long.parseLong(genomicEntry[startPosId].trim()), genomicEntry[refAlleleId].trim(), genomicEntry[altAlleleId].trim());

			String patientId = samplePatientMap.get(genomicEntry[sampleIdId]);

			// add patients to the set corresponding to variant
			if (!nbPatientsMap.containsKey(variantId)) {
			nbPatientsMap.put(variantId, new HashSet<>());
			}
			nbPatientsMap.get(variantId).add(patientId);

			// update nb facts / patient
			if (!nbFacts.containsKey(patientId)) {
			nbFacts.put(patientId, 0);
			}
			nbFacts.replace(patientId, nbFacts.get(patientId) + 1);

			} catch (Throwable e) {
			System.err.println("ignoring genomic entry ...");
			}
			}
			}

			CSVWriter genomicWriter1 = new CSVWriter(new FileWriter(clearGenomicOutputPath1), '\t', '\u0000');
			genomicWriter1.writeNext(new String[]{"VARIANT_ID", "NB_PATIENTS"});
			for (Map.Entry<Long, Set<String>> entry : nbPatientsMap.entrySet()) {
			genomicWriter1.writeNext(new String[]{String.valueOf(entry.getKey()), String.valueOf(entry.getValue().size())});
			}
			CSVWriter genomicWriter2 = new CSVWriter(new FileWriter(clearGenomicOutputPath2), '\t', '\u0000');
			genomicWriter2.writeNext(new String[]{"PATIENT_ID", "NB_GENOMIC_FACTS"});
			for (Map.Entry<String, Integer> entry : nbFacts.entrySet()) {
			genomicWriter2.writeNext(new String[]{String.valueOf(entry.getKey()), String.valueOf(entry.getValue())});
			}
			genomicWriter1.close();
			genomicWriter2.close();
			}

			public static void timeEncrypt() throws IOException {
			Logger.getRootLogger().setLevel(Level.OFF);
			loadSrv1Conf();
			UnlynxEncrypt encrypt = new UnlynxEncrypt();

			//StopWatch.overall.start();
			for (int i = 0 ; i < 1000000 ; i++) {
			String hey = encrypt.encryptInt(i);

			if (i % 250000 == 0) {
			//long time = StopWatch.overall.getTotalTimeMillis();
			//System.out.println(time);
			}

			if (i % 10000 == 0) {
			System.out.println(i);
			}
			}
			}

			public static void generateClearGenomicDataset() throws IOException {
			// generate full dataset for clear i2b2 loading, then split using the existing things

			loadSrv1Conf();
			String genomicFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad.txt",
			clinicalFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad.txt",
			clearGenomicOutputPath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2.txt";

			// maps sample id -> patient id
			Map<String, String> samplePatientMap = new HashMap<>();

			// read clinical: store patient id - sample ids mapping
			{
			CSVReader clinicalReader = new CSVReader(new FileReader(clinicalFilePath), '\t', '\u0000', 5);
			String[] clinicalHeader = clinicalReader.readNext();

			// headers idx clinical
			int sampleIdId = -1, patientIdId = -1;
			for (int i = 0; i < clinicalHeader.length; i++) {
			if (clinicalHeader[i].trim().equals("SAMPLE_ID")) {
			sampleIdId = i;
			} else if (clinicalHeader[i].trim().equals("PATIENT_ID")) {
			patientIdId = i;
			}
			}

			// dataset store mapping
			String[] clinicalEntry;
			while ((clinicalEntry = clinicalReader.readNext()) != null) {
			try {
			samplePatientMap.put(clinicalEntry[sampleIdId], clinicalEntry[patientIdId]);
			} catch (Throwable e) {
			System.err.println("ignoring clinical entry ...");
			}
			}
			}

			// read genomic and output dataset we want
			{

			CSVReader genomicReader = new CSVReader(new FileReader(genomicFilePath), '\t', '\u0000', 1);
			String[] genomicHeader = genomicReader.readNext();

			CSVWriter genomicWriter = new CSVWriter(new FileWriter(clearGenomicOutputPath), '\t', '\u0000');
			genomicWriter.writeNext(new String[]{"SAMPLE_ID", "PATIENT_ID", "VARIANT_ID"});

			// headers idx genomic
			int sampleIdId = -1, chromId = -1, startPosId = -1, altAlleleId = -1, refAlleleId = -1;
			for (int i = 0; i < genomicHeader.length; i++) {
			if (genomicHeader[i].trim().equals("Tumor_Sample_Barcode")) {
			sampleIdId = i;
			} else if (genomicHeader[i].trim().equals("Chromosome")) {
			chromId = i;
			} else if (genomicHeader[i].trim().equals("Start_Position")) {
			startPosId = i;
			} else if (genomicHeader[i].trim().equals("Reference_Allele")) {
			refAlleleId = i;
			} else if (genomicHeader[i].trim().equals("Tumor_Seq_Allele1")) {
			altAlleleId = i;
			}
			}

			// dataset genomic reading + generating [sample_id, patient_id, variant_id]
			String[] genomicEntry;
			while ((genomicEntry = genomicReader.readNext()) != null) {
			try {
			long variantId = EncryptedIdentifiersManager.getVariantId(genomicEntry[chromId].trim(),
			Long.parseLong(genomicEntry[startPosId].trim()), genomicEntry[refAlleleId].trim(), genomicEntry[altAlleleId].trim());

			genomicWriter.writeNext(new String[]{
			genomicEntry[sampleIdId],
			samplePatientMap.get(genomicEntry[sampleIdId]),
			variantId + ""});
			} catch (Throwable e) {
			System.err.println("ignoring genomic entry ...");
			}
			}
			}
			}

			public static void getInfoUseCases() throws IOException {
			// cancer type detailed = cutaneous melanoma
			// primary tumor localization = skin
			// annotation protein position = 600/766
			// hugo symbol = braf

			loadSrv1Conf();
			Logger.getRootLogger().setLevel(Level.DEBUG);

			String genomicFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad.txt";

			Set<Long>
			clearQueryVariantsUseCase1 = new HashSet<>(),
			clearQueryVariantsUseCase2Braf = new HashSet<>(),
			clearQueryVariantsUseCase2Others = new HashSet<>();


			{
			CSVReader genomicReader = new CSVReader(new FileReader(genomicFilePath), '\t', '\u0000', 1);
			String[] genomicHeader = genomicReader.readNext();

			// headers idx genomic
			int proteinPosId = -1, hugoSymbolId = -1, sampleIdId = -1,
			chromId = -1, startPosId = -1, altAlleleId = -1, refAlleleId = -1;
			for (int i = 0; i < genomicHeader.length; i++) {
			if (genomicHeader[i].trim().equals("Hugo_Symbol")) {
			hugoSymbolId = i;
			} else if (genomicHeader[i].trim().equals("Protein_position")) {
			proteinPosId = i;
			} else if (genomicHeader[i].trim().equals("Tumor_Sample_Barcode")) {
			sampleIdId = i;
			} else if (genomicHeader[i].trim().equals("Chromosome")) {
			chromId = i;
			} else if (genomicHeader[i].trim().equals("Start_Position")) {
			startPosId = i;
			} else if (genomicHeader[i].trim().equals("Reference_Allele")) {
			refAlleleId = i;
			} else if (genomicHeader[i].trim().equals("Tumor_Seq_Allele1")) {
			altAlleleId = i;
			}
			}

			// read dataset
			String[] genomicEntry;

			while ((genomicEntry = genomicReader.readNext()) != null) {
			try {

			// use case 1
			if (genomicEntry[proteinPosId].trim().equals("600/766") &&
			genomicEntry[hugoSymbolId].trim().equals("BRAF")) {

			long id = EncryptedIdentifiersManager.getVariantId(genomicEntry[chromId].trim(),
			Long.parseLong(genomicEntry[startPosId].trim()), genomicEntry[refAlleleId].trim(), genomicEntry[altAlleleId].trim());

			clearQueryVariantsUseCase1.add(id);

			}

			// use case 2 - braf
			if (genomicEntry[hugoSymbolId].trim().equals("BRAF")) {
			long id = EncryptedIdentifiersManager.getVariantId(genomicEntry[chromId].trim(),
			Long.parseLong(genomicEntry[startPosId].trim()), genomicEntry[refAlleleId].trim(), genomicEntry[altAlleleId].trim());

			clearQueryVariantsUseCase2Braf.add(id);
			}

			// use case 2 - other mutations
			if (genomicEntry[hugoSymbolId].trim().equals("PTEN") \|\|
			genomicEntry[hugoSymbolId].trim().equals("CDKN2A") \|\|
			genomicEntry[hugoSymbolId].trim().equals("MAP2K2") \|\|
			genomicEntry[hugoSymbolId].trim().equals("MAP2K1")) {

			long id = EncryptedIdentifiersManager.getVariantId(genomicEntry[chromId].trim(),
			Long.parseLong(genomicEntry[startPosId].trim()), genomicEntry[refAlleleId].trim(), genomicEntry[altAlleleId].trim());

			clearQueryVariantsUseCase2Others.add(id);
			}

			} catch (Throwable e) {
			System.err.println("ignoring genomic variant ...");
			}
			}

			System.err.println("nbs: " + clearQueryVariantsUseCase1.size() + " / " + clearQueryVariantsUseCase2Braf.size() + " / " + clearQueryVariantsUseCase2Others.size());

			}

			UnlynxEncrypt encrypt = new UnlynxEncrypt();

			// gen queries encrypted
			System.out.println("--- use case 1 ---\n");
			System.out.print("MEDCO_ENC:" + encrypt.encryptInt(1) + " AND "); // val
			System.out.println("MEDCO_ENC:" + encrypt.encryptInt(2) + " AND "); // val
			for (Long encId : clearQueryVariantsUseCase1) {
			System.out.println("MEDCO_GEN:" + encrypt.encryptInt(encId) + " OR ");
			}

			System.out.println("\n\n\n--- use case 2 ---\n");
			System.out.print(" -- clinical reminder ");
			for (Long encId : clearQueryVariantsUseCase2Braf) {
			System.out.println("\\\\SENSITIVE_TAGGED\\medco\\encrypted\\" + encrypt.encryptInt(encId) + "\\ OR ");
			}
			System.out.println(" AND ");
			for (Long encId : clearQueryVariantsUseCase2Others) {
			System.out.println("\\\\SENSITIVE_TAGGED\\medco\\encrypted\\" + encrypt.encryptInt(encId) + "\\ OR ");
			}

			// gen queries clear
			System.out.println("\n\n\n--- use case 1 clear ---\n");
			for (Long id : clearQueryVariantsUseCase1) {
			System.out.println("\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + id + "\\ OR ");
			}

			System.out.println("\n\n\n--- use case 2 clear ---\n");
			for (Long id : clearQueryVariantsUseCase2Braf) {
			System.out.println("\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + id + "\\ OR ");
			}
			System.out.println(" AND ");
			for (Long id : clearQueryVariantsUseCase2Others) {
			System.out.println("\\\\CLINICAL_NON_SENSITIVE\\medco\\clinical\\nonsensitive\\VARIANT_ID\\" + id + "\\ OR ");
			}
			}

			private static void generateLessData(int factor, String genomicInput, String genomicOuput) throws IOException {

			CSVWriter genomicWriter = new CSVWriter(new FileWriter(genomicOuput), '\t', '\u0000');

			CSVReader genomicReader = new CSVReader(new FileReader(genomicInput), '\t', '\u0000', 0);
			String[] genomicHeader = genomicReader.readNext();
			genomicWriter.writeNext(genomicHeader);

			String[] genomicEntry;
			int entriesCount = 0, writtenEntriesCount = 0;
			while ( (genomicEntry = genomicReader.readNext()) != null) {

			if (entriesCount % factor == 0) {
			genomicWriter.writeNext(genomicEntry);
			writtenEntriesCount++;
			}

			entriesCount++;
			}

			genomicWriter.close();
			System.out.println("Less data factor: " + factor + ", parsed " + entriesCount + " entries, written " + writtenEntriesCount);
			}

			private static void generateMoreData(int factor, String genomicInput, String genomicOuput) throws IOException {

			CSVWriter genomicWriter = new CSVWriter(new FileWriter(genomicOuput), '\t', '\u0000');

			CSVReader genomicReader = new CSVReader(new FileReader(genomicInput), '\t', '\u0000', 0);
			String[] genomicHeader = genomicReader.readNext();
			genomicWriter.writeNext(genomicHeader);

			String[] genomicEntry;
			int entriesCount = 0, writtenEntriesCount = 0;
			while ( (genomicEntry = genomicReader.readNext()) != null) {

			for (int i = 0 ; i < factor ; i++) {
			genomicWriter.writeNext(genomicEntry);
			writtenEntriesCount++;
			}

			entriesCount++;
			}

			genomicWriter.close();
			System.out.println("More data factor: " + factor + ", parsed " + entriesCount + " entries, written " + writtenEntriesCount);
			}

			/*
			from each part file: generate
			- clinical_half_patients: remove half of the patients, assign rest of of data to others remaining
			- clinical_double_patients: duplicate all patients
			- genomic_half_patients,
			- genomic_double_patients

			data generation:
			when less --> retain only 1 out of 2 or 1 out of 4
			when more --> all records duplicated / quadrupled

			--> clinical remains the same, only genomic touched (because nb clinical << nb genomic)
			*/
			public static void generation() throws IOException {
			String genomicPart1 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part1",
			genomicPart2 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part2",
			genomicPart3 = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part3";

			// half generation
			generateLessData(2, genomicPart1 + ".txt", genomicPart1 + "_half.txt");
			generateLessData(2, genomicPart2 + ".txt", genomicPart2 + "_half.txt");
			generateLessData(2, genomicPart3 + ".txt", genomicPart3 + "_half.txt");

			// quarter generation
			generateLessData(4, genomicPart1 + ".txt", genomicPart1 + "_quarter.txt");
			generateLessData(4, genomicPart2 + ".txt", genomicPart2 + "_quarter.txt");
			generateLessData(4, genomicPart3 + ".txt", genomicPart3 + "_quarter.txt");

			// double generation
			generateMoreData(2, genomicPart1 + ".txt", genomicPart1 + "_double.txt");
			generateMoreData(2, genomicPart2 + ".txt", genomicPart2 + "_double.txt");
			generateMoreData(2, genomicPart3 + ".txt", genomicPart3 + "_double.txt");

			// quadruple generation
			generateMoreData(4, genomicPart1 + ".txt", genomicPart1 + "_quadruple.txt");
			generateMoreData(4, genomicPart2 + ".txt", genomicPart2 + "_quadruple.txt");
			generateMoreData(4, genomicPart3 + ".txt", genomicPart3 + "_quadruple.txt");

			}

			public static void splitting() throws IOException {
			String clinicalFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad.txt",
			genomicFilePath = "/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad.txt";

			FileOutputStream
			os11 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_clear_i2b2_part1_encodingOK.txt"),
			os12 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_clear_i2b2_part2_encodingOK.txt"),
			os13 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_clinical_skcm_broad_clear_i2b2_part3_encodingOK.txt"),
			os21 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part1_encodingOK.txt"),
			os22 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part2_encodingOK.txt"),
			os23 = new FileOutputStream("/home/misbach/repositories/i2b2-core-server-medco/ch.epfl.lca1.medco/testfiles/datasets/full/skcm_broad/data_mutations_extended_skcm_broad_clear_i2b2_part3_encodingOK.txt");

			CSVWriter[] clinicalWriters = new CSVWriter[]{
			new CSVWriter(new OutputStreamWriter(os11, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			new CSVWriter(new OutputStreamWriter(os12, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			new CSVWriter(new OutputStreamWriter(os13, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			};
			CSVWriter[] genomicWriters = new CSVWriter[]{
			new CSVWriter(new OutputStreamWriter(os21, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			new CSVWriter(new OutputStreamWriter(os22, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			new CSVWriter(new OutputStreamWriter(os23, "UTF-8"),
			'\t', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.RFC4180_LINE_END),
			};



			CSVReader clinicalReader = new CSVReader(new InputStreamReader(new FileInputStream(clinicalFilePath), "ISO-8859-1"), '\t', '\u0000', 5);
			String[] clinicalHeader = clinicalReader.readNext();
			clinicalWriters[0].writeNext(clinicalHeader);
			clinicalWriters[1].writeNext(clinicalHeader);
			clinicalWriters[2].writeNext(clinicalHeader);

			CSVReader genomicReader = new CSVReader(new InputStreamReader(new FileInputStream(genomicFilePath), "ISO-8859-1"), '\t', '\u0000', 1);
			String[] genomicHeader = genomicReader.readNext();
			genomicWriters[0].writeNext(genomicHeader);
			genomicWriters[1].writeNext(genomicHeader);
			genomicWriters[2].writeNext(genomicHeader);

			Set<String>[] sampleIdsSets = new HashSet[] {
			new HashSet<>(),
			new HashSet<>(),
			new HashSet<>()
			};

			// read clinical file
			String[] clinicalEntry;
			int clinicalEntriesCount = 0;
			while ( (clinicalEntry = clinicalReader.readNext()) != null) {
			clinicalEntriesCount++;

			if (clinicalEntriesCount <= 40) {
			clinicalWriters[0].writeNext(clinicalEntry);
			sampleIdsSets[0].add(clinicalEntry[0].trim());
			} else if (clinicalEntriesCount > 40 && clinicalEntriesCount <= 80) {
			clinicalWriters[1].writeNext(clinicalEntry);
			sampleIdsSets[1].add(clinicalEntry[0].trim());
			} else {
			clinicalWriters[2].writeNext(clinicalEntry);
			sampleIdsSets[2].add(clinicalEntry[0].trim());
			}
			}

			System.out.println("Parsed " + clinicalEntriesCount + " entries, sizes: " + sampleIdsSets[0].size() + " - " +
			sampleIdsSets[1].size() + " - " + sampleIdsSets[2].size());

			// read genomic file
			int sampleIdIdx = 15; //15 for skcm broad originial, 2 for generated clear i2b2 dataset
			String[] genomicEntry;
			int genomicCount = 0;
			while ( (genomicEntry = genomicReader.readNext()) != null) {
			genomicCount++;

			if (sampleIdsSets[0].contains(genomicEntry[sampleIdIdx].trim())) {
			genomicWriters[0].writeNext(genomicEntry);
			} else if (sampleIdsSets[1].contains(genomicEntry[sampleIdIdx].trim())) {
			genomicWriters[1].writeNext(genomicEntry);
			} else if (sampleIdsSets[2].contains(genomicEntry[sampleIdIdx].trim())){
			genomicWriters[2].writeNext(genomicEntry);
			} else {
			System.out.println("Non existing sample in clinical: " + genomicEntry[sampleIdIdx]);
			}
			}

			System.out.println("Parsed " + genomicCount + " entries");

			clinicalWriters[0].close();
			clinicalWriters[1].close();
			clinicalWriters[2].close();
			genomicWriters[0].close();
			genomicWriters[1].close();
			genomicWriters[2].close();

			}

			protected static void loadMedCoConf(String hostname, int i2b2Port, int psqlPort, String unlynxEntryPoint) {
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.ONTCELL_WS_URL_PROPERTIES,
			"http://" + hostname + ":" + i2b2Port + "/i2b2/services/OntologyService");
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.FRCELL_WS_URL_PROPERTIES,
			"http://" + hostname + ":" + i2b2Port + "/i2b2/services/FRService");
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.CRCCELL_WS_URL_PROPERTIES,
			"http://" + hostname + ":" + i2b2Port + "/i2b2/services/QueryToolService");
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.I2B2CELLS_WS_WAITTIME_PROPERTIES,
			"180000");

			MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_BINARY_PATH_PROPERTIES, "i2b2"); // assumed in bin path
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_GROUP_FILE_PATH_PROPERTIES,
			"/home/misbach/repositories/medco-deployment/configuration-profiles/exp-3nodes-icclusters/group.toml");

			MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_DEBUG_LEVEL_PROPERTIES, "5");
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_PROOFS_PROPERTIES, "0");
			MedCoUtil.getTestInstance().setProperty(MedCoUtil.UNLYNX_ENTRY_POINT_IDX_PROPERTIES, unlynxEntryPoint);

			PGSimpleDataSource ds = new PGSimpleDataSource();
			ds.setServerName(hostname);
			ds.setDatabaseName("medcodeployment");
			ds.setPortNumber(psqlPort);
			ds.setUser("postgres");
			ds.setPassword("prigen2017");
			//ds.setCurrentSchema("medco_data");

			MedCoUtil.getTestInstance().setDataSource(ds);
			}

			protected static void loadSrv1Conf() {
			loadMedCoConf("localhost", 8082,
			5434, "0");
			}

			protected static void loadSrv3Conf() {
			loadMedCoConf("iccluster062.iccluster.epfl.ch", 8080,
			5432, "1");
			}

			protected static void loadSrv5Conf() {
			loadMedCoConf("iccluster063.iccluster.epfl.ch", 8080,
			5432, "2");
			}

			}

medco-i2b2-cell/src/client/ch/epfl/lca1/medco/DatasetsManipulations.java27ae07e5b3damaster

DatasetsManipulations.javaView Options

medco-i2b2-cell/src/client/ch/epfl/lca1/medco/DatasetsManipulations.java
27ae07e5b3damaster

DatasetsManipulations.java
View Options