Homec4science
Diffusion i2b2-webclient (newGui) Browse

R4897/tools/shrine_add_schema/createMetadata.go
8ffca5381ce3newGui

Last Change
Raw File

createMetadata.go
View Options

package main
import (
"log"
"os"
"bufio"
"strings"
)
// path to the folder in with to read/write files
const path = "/home/niccolo/SHRINE/"
const fromFile = "SHRINE_ONT_GENOMIC_ANNOTATIONS.csv"
const toAnnotationMeta = "SHRINE_ONT_ANNOTATIONS_METADATA.csv"
const toOntGenomicNew = "SHRINE_ONT_GENOMIC_ANNOTATIONS_NEW.csv"
const toVariantsName = "SHRINE_VARIANTS_NAMES.csv"
const toGenes = "SHRINE_GENES.csv"
func main(){
// create the three ontology genomic, variants ID and genes tables
loadAndCreate()
// create the annotation-meta file
//annotationToValues := loadAnnotationToValuesMap()
//for annotation, values := range annotationToValues{
// println(annotation + ": " + strings.Join(values,"||"))
//}
//createAnnotationMetadataFile(annotationToValues)
}
func loadAndCreate() error{
// open all the files, I'll write in them as I read to avoid to load all the csv in go (just one line per time)
from, err := os.Open(path + fromFile)
defer from.Close()
if err != nil {
log.Fatal("Error while opening", path + fromFile)
return nil
}
ontGenomicFile, err := os.Create(path + toOntGenomicNew)
defer ontGenomicFile.Close()
if err != nil {
log.Fatal("Error while opening", path + toOntGenomicNew)
return err
}
variantsNameFile, err := os.Create(path + toVariantsName)
defer variantsNameFile.Close()
if err != nil {
log.Fatal("Error while opening", path + toVariantsName)
return err
}
genesFile, err := os.Create(path + toGenes)
defer genesFile.Close()
if err != nil {
log.Fatal("Error while opening", path + toGenes)
return err
}
// replacers for strange annotation values. repl2(repl1(str))
repl1 := strings.NewReplacer(
",;", " ",
";,", " ",
";;", " ",
",,", " ",
)
repl2 := strings.NewReplacer(
",", " ",
";", " ",
)
// just keep the list of Hugo_symbol values already written in the file (I don't want to repeat values)
var hugoValues = []string{"?"} // ? so that if some hugo symbol is missing in a row I don't write "?" in genesFile
var annotationToValue = make(map[string]string)
// scanner.Text() returns "-8772697417129649152","{""Hugo_Symbol"":""PI4KB"", ""Entrez_Gene_Id"":""0"", ""Center"":""broad.mit.edu"", ""NCBI_Build"":""GRCh37"", ""Chromosome"":""1"", ""Start_Position"":""151287991"", ""End_Position"":""151287991"", ""Strand"":""-1"", ""Variant_Classification"":""Intron"", ""Variant_Type"":""SNP"", ""Reference_Allele"":""T"", ""Tumor_Seq_Allele1"":""C"", ""Tumor_Seq_Allele2"":""C"", ""dbSNP_RS"":"""", ""dbSNP_Val_Status"":"""", ""Tumor_Sample_Barcode"":""ME002"", ""Matched_Norm_Sample_Barcode"":""ME002-Normal"", ""Match_Norm_Seq_Allele1"":""T"", ""Match_Norm_Seq_Allele2"":""T"", ""Tumor_Validation_Allele1"":"""", ""Tumor_Validation_Allele2"":"""", ""Match_Norm_Validation_Allele1"":"""", ""Match_Norm_Validation_Allele2"":"""", ""Verification_Status"":"""", ""Validation_Status"":"""", ""Mutation_Status"":"""", ""Sequencing_Phase"":"""", ""Sequence_Source"":"""", ""Validation_Method"":"""", ""Score"":"""", ""BAM_File"":"""", ""Sequencer"":""Illumina GAIIx"", ""MA:FImpact"":"""", ""MA:FIS"":"""", ""MA:protein.change"":"""", ""MA:link.MSA"":"""", ""MA:link.PDB"":"""", ""MA:link.var"":"""", ""Tumor_Sample_UUID"":"""", ""Matched_Norm_Sample_UUID"":"""", ""HGVSc"":""c.909+58A>G"", ""HGVSp"":"""", ""HGVSp_Short"":"""", ""Transcript_ID"":""ENST00000368873"", ""Exon_Number"":"""", ""t_depth"":""0"", ""t_ref_count"":""."", ""t_alt_count"":""."", ""n_depth"":""0"", ""n_ref_count"":""."", ""n_alt_count"":""."", ""all_effects"":""PI4KB,intron_variant,,ENST00000368875,NM_002651.2;PI4KB,intron_variant,,ENST00000368874,NM_001198774.1;PI4KB,intron_variant,,ENST00000271657,;PI4KB,intron_variant,,ENST00000368873,;PI4KB,intron_variant,,ENST00000368872,NM_001198773.1;PI4KB,intron_variant,,ENST00000529142,NM_001198775.1;PI4KB,intron_variant,,ENST00000430800,;PI4KB,intron_variant,,ENST00000489223,;PI4KB,downstream_gene_variant,,ENST00000438243,;"", ""Allele"":""C"", ""Gene"":""ENSG00000143393"", ""Feature"":""ENST00000368873"", ""Feature_type"":""Transcript"", ""Consequence"":""intron_variant"", ""cDNA_position"":""-/3340"", ""CDS_position"":""-/2451"", ""Protein_position"":""-/816"", ""Amino_acids"":"""", ""Codons"":"""", ""Existing_variation"":"""", ""ALLELE_NUM"":""1"", ""DISTANCE"":"""", ""SYMBOL"":""PI4KB"", ""SYMBOL_SOURCE"":""HGNC"", ""HGNC_ID"":""8984"", ""BIOTYPE"":""protein_coding"", ""CANONICAL"":"""", ""CCDS"":"""", ""ENSP"":""ENSP00000357867"", ""SWISSPROT"":""PI4KB_HUMAN"", ""TREMBL"":""F8W860_HUMAN,E9PL47_HUMAN,E9PIH4_HUMAN"", ""UNIPARC"":""UPI0000040634"", ""RefSeq"":"""", ""SIFT"":"""", ""PolyPhen"":"""", ""EXON"":"""", ""INTRON"":""02.nov"", ""DOMAINS"":"""", ""GMAF"":"""", ""AFR_MAF"":"""", ""AMR_MAF"":"""", ""ASN_MAF"":"""", ""EAS_MAF"":"""", ""EUR_MAF"":"""", ""SAS_MAF"":"""", ""AA_MAF"":"""", ""EA_MAF"":"""", ""CLIN_SIG"":"""", ""SOMATIC"":"""", ""PUBMED"":"""", ""MOTIF_NAME"":"""", ""MOTIF_POS"":"""", ""HIGH_INF_POS"":"""", ""MOTIF_SCORE_CHANGE"":"""", ""IMPACT"":""MODIFIER"", ""PICK"":"""", ""VARIANT_CLASS"":""SNV"", ""TSL"":"""", ""HGVS_OFFSET"":"""", ""PHENO"":"""", ""MINIMISED"":""1"", ""ExAC_AF"":"""", ""ExAC_AF_AFR"":"""", ""ExAC_AF_AMR"":"""", ""ExAC_AF_EAS"":"""", ""ExAC_AF_FIN"":"""", ""ExAC_AF_NFE"":"""", ""ExAC_AF_OTH"":"""", ""ExAC_AF_SAS"":"""", ""GENE_PHENO"":"""", ""FILTER"":"".""}"
scanner := bufio.NewScanner(from)
// todo remove i (get all rows)
i := 0 // there are 232 211 lines to read
for scanner.Scan() {
i = i+1
if i % 10000 == 0{
println(i)
//return nil
}
var variantNumber string
var variantName string
// split the row in variant identifier and list of genomic annotations
row := strings.SplitN(scanner.Text(), ",", 2)
variantNumber = row[0][1:len(row[0])-1]
// divide the annotations (list of "annotationName"":""annotationValue")
annotationsList := strings.Split(row[1][4:len(row[1])-4], "\"\", \"\"")
// store the current row in map
annotationToValue = map[string]string{}
for _, annot := range annotationsList {
temp := strings.Split(annot, "\"\":\"\"")
annotationName := temp[0]
annotationValue := temp[1]
if annotationValue == "" { continue }
if annotationValue == "-" { continue }
if annotationValue == "Unknown" { continue }
// just check this line don't a duplicated annotation (shouldn't happen)
_, ok := annotationToValue[annotationName]
if ok {println("Error! Variant", variantNumber, "contains duplicated", annotationName)}
annotationToValue[annotationName] = repl2.Replace(repl1.Replace(annotationValue))
}
// once the whole line is loaded let's write it into the files
// first get the t_depth and build the variant name chr:pos:r>A
t_depth := "0" // todo if it not specified what value should I put?
d, ok := annotationToValue["t_depth"]
if ok{t_depth = d}
delete(annotationToValue, "t_depth")
chromosome := "Chromosome"
start_pos := "Start_Position"
ref_allele := "Reference_Allele"
tum_allele1 := "Tumor_Seq_Allele1"
tum_allele2 := "Tumor_Seq_Allele2"
// if there is no value then put "?"
chrom := "?"
if a, ok := annotationToValue[chromosome]; ok {chrom = a}
startPos := "?"
if a, ok := annotationToValue[start_pos]; ok {startPos = a}
refAllele := "?"
if a, ok := annotationToValue[ref_allele]; ok {refAllele = a}
// just get one of the two
allele := "?"
if a, ok := annotationToValue[tum_allele1]; ok {
allele = a
}else if a, ok := annotationToValue[tum_allele2]; ok {
allele = a
}
variantName =
chrom + ":" +
startPos + ":" +
refAllele + ">"+
allele
// we don't need these entries anymore
delete(annotationToValue, chromosome)
delete(annotationToValue, start_pos)
delete(annotationToValue, "End_Position")
delete(annotationToValue, ref_allele)
// GENOMIC ONTOLOGY
// first column: variant number
ontGenomicLine := "\"" + variantNumber + "\""
// second column: variant name
ontGenomicLine += ",\"" + variantName + "\""
// third column: Hugo_SymbolValue;(homo/hetero)zygous/both/missing;annotationName1=annotationValue1;annotationName2=annotationValue2;...
hugo_s := "Hugo_Symbol"
hugoValue := "?"
if a, ok := annotationToValue[hugo_s]; ok {
hugoValue = a
}else{
// todo lot don't have the Hugo_Symbol field
// println("Error! Variant", variantNumber, "contains no HugoSymbol")
}
// tum_allele1 tum_allele2
// nil nil Unknown
// A nil Unknown
// nil B Unknown
// A B Heterozygous
// A A Homozygous
alleleType := ""
a1, ok1 := annotationToValue[tum_allele1]
a2, ok2 := annotationToValue[tum_allele2]
if !ok1 || !ok2{
alleleType = "Unknown"
}else if a1==a2{
alleleType = "Homozygous"
}else{
alleleType = "Heterozygous"
}
ontGenomicLine += ",\"" + hugoValue + ";" + alleleType
delete(annotationToValue, hugo_s)
delete(annotationToValue, tum_allele1)
delete(annotationToValue, tum_allele2)
for anName, anValue := range annotationToValue {
ontGenomicLine += ";" + anName + "=" + anValue
}
ontGenomicLine += "\""
// fourth column: t_depth
ontGenomicLine += ",\"" + t_depth + "\""
newline := "\n" // write the row in a new line
if i == 1 {
newline = ""
}
_, err = ontGenomicFile.WriteString(newline + ontGenomicLine)
if err != nil {
log.Fatal("Error in writing in the genomic file", err)
return err
}
// VARIANT NAMES
_, err = variantsNameFile.WriteString(newline + "\"" + variantName + "\"")
if err != nil {
log.Fatal("Error in writing in the genomic file", err)
return err
}
// GENES
if !containsOrAdd(&hugoValues, hugoValue) {
_, err = genesFile.WriteString(newline + "\"" + hugoValue + "\"")
if err != nil {
log.Fatal("Error in writing in the genomic file", err)
return err
}
}
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
return err
}
return nil
}
func loadAnnotationToValuesMap() map[string][]string{
fp, err := os.Open(path + fromFile)
defer fp.Close()
if err != nil {
log.Fatal("Error while opening", path + fromFile)
return nil
}
var annotationToValues = make(map[string][]string)
// scanner.Text() returns "-8772697417129649152","{""Hugo_Symbol"":""PI4KB"", ""Entrez_Gene_Id"":""0"", ""Center"":""broad.mit.edu"", ""NCBI_Build"":""GRCh37"", ""Chromosome"":""1"", ""Start_Position"":""151287991"", ""End_Position"":""151287991"", ""Strand"":""-1"", ""Variant_Classification"":""Intron"", ""Variant_Type"":""SNP"", ""Reference_Allele"":""T"", ""Tumor_Seq_Allele1"":""C"", ""Tumor_Seq_Allele2"":""C"", ""dbSNP_RS"":"""", ""dbSNP_Val_Status"":"""", ""Tumor_Sample_Barcode"":""ME002"", ""Matched_Norm_Sample_Barcode"":""ME002-Normal"", ""Match_Norm_Seq_Allele1"":""T"", ""Match_Norm_Seq_Allele2"":""T"", ""Tumor_Validation_Allele1"":"""", ""Tumor_Validation_Allele2"":"""", ""Match_Norm_Validation_Allele1"":"""", ""Match_Norm_Validation_Allele2"":"""", ""Verification_Status"":"""", ""Validation_Status"":"""", ""Mutation_Status"":"""", ""Sequencing_Phase"":"""", ""Sequence_Source"":"""", ""Validation_Method"":"""", ""Score"":"""", ""BAM_File"":"""", ""Sequencer"":""Illumina GAIIx"", ""MA:FImpact"":"""", ""MA:FIS"":"""", ""MA:protein.change"":"""", ""MA:link.MSA"":"""", ""MA:link.PDB"":"""", ""MA:link.var"":"""", ""Tumor_Sample_UUID"":"""", ""Matched_Norm_Sample_UUID"":"""", ""HGVSc"":""c.909+58A>G"", ""HGVSp"":"""", ""HGVSp_Short"":"""", ""Transcript_ID"":""ENST00000368873"", ""Exon_Number"":"""", ""t_depth"":""0"", ""t_ref_count"":""."", ""t_alt_count"":""."", ""n_depth"":""0"", ""n_ref_count"":""."", ""n_alt_count"":""."", ""all_effects"":""PI4KB,intron_variant,,ENST00000368875,NM_002651.2;PI4KB,intron_variant,,ENST00000368874,NM_001198774.1;PI4KB,intron_variant,,ENST00000271657,;PI4KB,intron_variant,,ENST00000368873,;PI4KB,intron_variant,,ENST00000368872,NM_001198773.1;PI4KB,intron_variant,,ENST00000529142,NM_001198775.1;PI4KB,intron_variant,,ENST00000430800,;PI4KB,intron_variant,,ENST00000489223,;PI4KB,downstream_gene_variant,,ENST00000438243,;"", ""Allele"":""C"", ""Gene"":""ENSG00000143393"", ""Feature"":""ENST00000368873"", ""Feature_type"":""Transcript"", ""Consequence"":""intron_variant"", ""cDNA_position"":""-/3340"", ""CDS_position"":""-/2451"", ""Protein_position"":""-/816"", ""Amino_acids"":"""", ""Codons"":"""", ""Existing_variation"":"""", ""ALLELE_NUM"":""1"", ""DISTANCE"":"""", ""SYMBOL"":""PI4KB"", ""SYMBOL_SOURCE"":""HGNC"", ""HGNC_ID"":""8984"", ""BIOTYPE"":""protein_coding"", ""CANONICAL"":"""", ""CCDS"":"""", ""ENSP"":""ENSP00000357867"", ""SWISSPROT"":""PI4KB_HUMAN"", ""TREMBL"":""F8W860_HUMAN,E9PL47_HUMAN,E9PIH4_HUMAN"", ""UNIPARC"":""UPI0000040634"", ""RefSeq"":"""", ""SIFT"":"""", ""PolyPhen"":"""", ""EXON"":"""", ""INTRON"":""02.nov"", ""DOMAINS"":"""", ""GMAF"":"""", ""AFR_MAF"":"""", ""AMR_MAF"":"""", ""ASN_MAF"":"""", ""EAS_MAF"":"""", ""EUR_MAF"":"""", ""SAS_MAF"":"""", ""AA_MAF"":"""", ""EA_MAF"":"""", ""CLIN_SIG"":"""", ""SOMATIC"":"""", ""PUBMED"":"""", ""MOTIF_NAME"":"""", ""MOTIF_POS"":"""", ""HIGH_INF_POS"":"""", ""MOTIF_SCORE_CHANGE"":"""", ""IMPACT"":""MODIFIER"", ""PICK"":"""", ""VARIANT_CLASS"":""SNV"", ""TSL"":"""", ""HGVS_OFFSET"":"""", ""PHENO"":"""", ""MINIMISED"":""1"", ""ExAC_AF"":"""", ""ExAC_AF_AFR"":"""", ""ExAC_AF_AMR"":"""", ""ExAC_AF_EAS"":"""", ""ExAC_AF_FIN"":"""", ""ExAC_AF_NFE"":"""", ""ExAC_AF_OTH"":"""", ""ExAC_AF_SAS"":"""", ""GENE_PHENO"":"""", ""FILTER"":"".""}"
scanner := bufio.NewScanner(fp)
// todo remove i (get all rows)
i := 0 // there are 232 211 lines to read
for scanner.Scan() {
i = i+1
if i % 10000 == 0{
println(i)
//println(scanner.Text())
//return nil
//return annotationToValues
}
// drop the mutation number
annotationsString := strings.SplitN(scanner.Text(), ",", 2)[1]
// divide the annotations (list of "annotationName"":""annotationValue")
annotationsList := strings.Split(annotationsString[4:len(annotationsString)-4], "\"\", \"\"")
for _, annot := range annotationsList {
temp := strings.Split(annot, "\"\":\"\"")
annotationName := temp[0]
annotationValue := temp[1]
if annotationValue == "" { continue }
_, ok := annotationToValues[annotationName]
// it is the first time I see the annotations, then add to the map
if !ok {
annotationToValues[annotationName] = []string{}
}
// check if the value is already in the map
alreadypresent := false
for _, val := range annotationToValues[annotationName]{
if val == annotationValue {
alreadypresent = true
break
}
}
if !alreadypresent{
annotationToValues[annotationName] = append(annotationToValues[annotationName], annotationValue)
}
}
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
return annotationToValues
}
func createAnnotationMetadataFile(annotationToValues map[string][]string) error{
fp, err := os.Create(path + toAnnotationMeta)
defer fp.Close()
if err != nil {
log.Fatal("Error while opening", path + toAnnotationMeta)
return err
}
// for every annotation write a row in the file
// the csv has two columns, one with the annotation name and one with the list of corresponding values (in json format)
for annotation, values := range annotationToValues {
if len(values) == 0{
continue
}
valuesStr := "[\"\""+ strings.Join(values, "\"\",\"\"")+"\"\"]"
// correct format: [""1247579"",""183775705"",""70136913"",""96256790"",""60696559"",""39603381""]
annotation := `"` + annotation + `","{` + valuesStr + `}"` + "\n"
_, err = fp.WriteString(annotation)
if err != nil {
log.Fatal("Error in the createAnnotationMetadataFile", err)
return err
}
}
//// for every couple (annotation, value) write a row in the file
//// the csv has two columns, one with the annotation name and one with a value
//for annotation, values := range annotationToValues {
// for _, val := range values {
// annotation := `"` + annotation + `","` + val + `"` + "\n"
// _, err := fp.WriteString(annotation)
// if err != nil {
// log.Fatal("Error in the createAnnotationMetadataFile", err)
// return err
// }
// }
//}
return nil
}
func containsOrAdd(array *[]string, str string) bool {
for _, s := range *array {
if s == str {
return true
}
}
*array = append(*array, str)
return false
}
c4science · Help