diff --git a/python/BlackDynamite/build_tables.sql b/python/BlackDynamite/build_tables.sql index 83d5211..d8d89d9 100644 --- a/python/BlackDynamite/build_tables.sql +++ b/python/BlackDynamite/build_tables.sql @@ -1,124 +1,128 @@ -- quantities CREATE TABLE SCHEMAS_IDENTIFIER.quantities ( id SERIAL PRIMARY KEY, --id, standard name TEXT NOT NULL, -- human readable name idendifying the quantity is_integer BOOLEAN NOT NULL, --whether the quantity is an integer quantity is_vector BOOLEAN NOT NULL, --whether the quantity is vectorial or scalar description TEXT , --general description the quantity (like units) UNIQUE (name)); -- integer scalar values CREATE TABLE SCHEMAS_IDENTIFIER.scalar_integer ( id SERIAL PRIMARY KEY, --id, standard run_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.runs quantity_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.quantities measurement INTEGER NOT NULL, -- measured value step INTEGER NOT NULL, --step at which this measurement has been taken computed_at TIMESTAMP NOT NULL, --time when the measurement has been added to the database. this column might be dropped later on FOREIGN KEY (run_id) REFERENCES SCHEMAS_IDENTIFIER.runs ON DELETE CASCADE, - FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities) ON DELETE CASCADE; + FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities ON DELETE CASCADE); -- real scalar values CREATE TABLE SCHEMAS_IDENTIFIER.scalar_real ( id SERIAL PRIMARY KEY, --id, standard run_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.runs quantity_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.quantities measurement DOUBLE PRECISION NOT NULL, -- measured value step INTEGER NOT NULL, --step at which this measurement has been taken computed_at TIMESTAMP NOT NULL, --time when the measurement has been added to the database. this column might be dropped later on FOREIGN KEY (run_id) REFERENCES SCHEMAS_IDENTIFIER.runs ON DELETE CASCADE, - FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities) ON DELETE CASCADE; + FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities ON DELETE CASCADE); -- real vector values CREATE TABLE SCHEMAS_IDENTIFIER.vector_real ( id SERIAL PRIMARY KEY, --id, standard run_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.runs quantity_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.quantities measurement DOUBLE PRECISION[] NOT NULL, -- measured value step INTEGER NOT NULL, --step at which this measurement has been taken computed_at TIMESTAMP NOT NULL, --time when the measurement has been added to the database. this column might be dropped later on FOREIGN KEY (run_id) REFERENCES SCHEMAS_IDENTIFIER.runs ON DELETE CASCADE, - FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities) ON DELETE CASCADE; + FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities ON DELETE CASCADE); -- int vector values CREATE TABLE SCHEMAS_IDENTIFIER.vector_integer ( id SERIAL PRIMARY KEY, --id, standard run_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.runs quantity_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.quantities measurement INTEGER [] NOT NULL, -- measured value step INTEGER NOT NULL, --step at which this measurement has been taken computed_at TIMESTAMP NOT NULL, --time when the measurement has been added to the database. this column might be dropped later on FOREIGN KEY (run_id) REFERENCES SCHEMAS_IDENTIFIER.runs ON DELETE CASCADE, - FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities) ON DELETE CASCADE; + FOREIGN KEY (quantity_id) REFERENCES SCHEMAS_IDENTIFIER.quantities ON DELETE CASCADE); -- configuration files CREATE TABLE SCHEMAS_IDENTIFIER.configfiles ( id SERIAL PRIMARY KEY, --id, standard filename TEXT NOT NULL, -- name of the config file - file TEXT NOT NULL, -- content of the config file - UNIQUE (file)); + file TEXT NOT NULL -- content of the config file + ); + +CREATE UNIQUE INDEX configfiles_md5file_key ON SCHEMAS_IDENTIFIER.configfiles ((md5(file))); +--ALTER TABLE SCHEMAS_IDENTIFIER.configfiles ADD UNIQUE (SCHEMAS_IDENTIFIER.configfiles_md5file_key); -- configuration files CREATE TABLE SCHEMAS_IDENTIFIER.runconfig ( id SERIAL PRIMARY KEY, --id, standard run_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.runs configfile_id INTEGER NOT NULL, --referencing SCHEMAS_IDENTIFIER.configfiles FOREIGN KEY (run_id) REFERENCES SCHEMAS_IDENTIFIER.runs ON DELETE CASCADE, FOREIGN KEY (configfile_id) REFERENCES SCHEMAS_IDENTIFIER.configfiles); -- create functions for the different triggers --create run_inserter CREATE FUNCTION SCHEMAS_IDENTIFIER.run_inserter() RETURNS TRIGGER AS $run_inserter$ BEGIN NEW.state := 'CREATED'; NEW.run_name := 'run' || NEW.id; RETURN NEW; END; $run_inserter$ LANGUAGE 'PLPGSQL' VOLATILE; CREATE TRIGGER run_inserter BEFORE INSERT ON SCHEMAS_IDENTIFIER.runs FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.run_inserter(); --create measurement inserter CREATE FUNCTION SCHEMAS_IDENTIFIER.measurement_inserter() RETURNS TRIGGER AS $measurement_inserter$ BEGIN NEW.computed_at := CURRENT_TIMESTAMP; RETURN NEW; END; $measurement_inserter$ LANGUAGE 'PLPGSQL' VOLATILE; --create set start time trigger CREATE FUNCTION SCHEMAS_IDENTIFIER.set_start_time() RETURNS TRIGGER AS $set_start_time$ BEGIN IF NEW.state = 'START' THEN NEW.start_time := CURRENT_TIMESTAMP; END IF; RETURN NEW; END; $set_start_time$ LANGUAGE 'PLPGSQL' VOLATILE; --autmagic clean of config files CREATE FUNCTION SCHEMAS_IDENTIFIER.delete_file() RETURNS TRIGGER AS $delete_file$ BEGIN - DELETE FROM configfiles; - EXCEPTION WHEN foreign_key_violation THEN NULL; + DELETE FROM configfiles; + RETURN NEW; + EXCEPTION WHEN FOREIGN_KEY_VIOLATION THEN NULL; RETURN NEW; END; - $delete_file$ LANGUAGE 'PLPGSQL' VOLATILE; + $delete_file$ LANGUAGE 'PLPGSQL'; CREATE TRIGGER int_scal_insert BEFORE INSERT ON SCHEMAS_IDENTIFIER.scalar_integer FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.measurement_inserter(); CREATE TRIGGER int_vect_insert BEFORE INSERT ON SCHEMAS_IDENTIFIER.vector_integer FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.measurement_inserter(); CREATE TRIGGER real_scal_insert BEFORE INSERT ON SCHEMAS_IDENTIFIER.scalar_real FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.measurement_inserter(); CREATE TRIGGER real_vect_insert BEFORE INSERT ON SCHEMAS_IDENTIFIER.vector_real FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.measurement_inserter(); CREATE TRIGGER update_start_time BEFORE UPDATE ON SCHEMAS_IDENTIFIER.runs FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.set_start_time(); CREATE TRIGGER delete_conffile AFTER DELETE ON SCHEMAS_IDENTIFIER.runs FOR EACH ROW EXECUTE PROCEDURE SCHEMAS_IDENTIFIER.delete_file(); diff --git a/python/BlackDynamite/conffile.py b/python/BlackDynamite/conffile.py index 72bb58b..77fe9f8 100644 --- a/python/BlackDynamite/conffile.py +++ b/python/BlackDynamite/conffile.py @@ -1,42 +1,55 @@ #!/usr/bin/env python from __future__ import print_function import sqlobject +import re class ConfFile(sqlobject.SQLObject): """ """ - def addFile(self,filename,base,content=None): + def addFile(self,filename,base,params=None,regex_params=None,content=None): self.prepare(base) self.entries["filename"] = filename if (content): self.entries["file"] = content else: self.entries["file"] = open(filename, 'r').read() - + + if (regex_params): + for p in params: + lowerp = p.lower() + rp = regex_params.replace("%p","(" + p + ")") + rp = rp.replace("%v","(.*)") + rr = regex_params.replace("%p","\\1") + rr = rr.replace("%v","__BLACKDYNAMITE__" + p + "__") +# print (rp) +# print (rr) + self.entries["file"] = re.sub(rp,rr,self.entries["file"],flags=re.IGNORECASE) +# print (self.entries["file"]) + filelist = base.getMatchedObjectList(self) if (len(filelist) == 0): base.insert(self) elif (len(filelist) == 1): self.entries = filelist[0].entries self.id = filelist[0].id - + def __init__ (self): sqlobject.SQLObject.__init__(self) self.table_name = "configfiles" self.types["filename"] = str self.types["file"] = str -def addFile(filename,base,content=None): +def addFile(filename,base,**kwargs): cnffile = ConfFile() - cnffile.addFile(filename,base,content) + cnffile.addFile(filename,base,**kwargs) return cnffile diff --git a/python/BlackDynamite/getRunInfo.py b/python/BlackDynamite/getRunInfo.py index cd9cf65..f689445 100755 --- a/python/BlackDynamite/getRunInfo.py +++ b/python/BlackDynamite/getRunInfo.py @@ -1,128 +1,128 @@ #!/usr/bin/env python import base import job import run import os , sys, stat import runconfig import subprocess import conffile import getopt import socket def getRunInfo(runid): myrun = run.Run() myrun["id"] = runid myrun.id = runid myrun.prepare(base) run_list = base.getMatchedObjectList(myrun) if (len(run_list) == 0): print ("no run found with id " + runid) sys.exit(1) myrun = run_list[0] myjob = job.Job() myjob.id = myrun["job_id"] myjob["id"] = myrun["job_id"] myjob.prepare(base) job_list = base.getMatchedObjectList(myjob) if (len(job_list) == 0): print ("no job found with id " + myjob.id) sys.exit(1) myjob = job_list[0] list_entries = myjob.entries.keys() print ("************************ job info *********************************") for entry in list_entries: if (myjob[entry]): print (entry + ": " + str(myjob[entry])) print ("************************ run info *********************************") list_entries = myrun.entries.keys() regular_run_entries = ("run_name", \ "job_id" , \ "state", \ "start_time", \ "machine_name" , \ "exec", \ "nproc", \ "wait_id" \ ) for entry in regular_run_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) list_entries.remove(entry) for entry in list_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) conffiles = myrun.getConfigFiles(base) for conf in conffiles: print ("****************************************************************") - print ("file: " + conf["filename"]) + print ("file #" + str(conf.id) + ": " + conf["filename"]) print ("****************************************************************") print (conf["file"]) ################################################################ options, remainder = getopt.getopt(sys.argv[1:],"", ["study=","host=","runid=","user=", "password=" , "order="]) study = None connection_args = {} connection_args["host"] = "lsmssrv1" order = "id" runid = None for opt, arg in options: if opt == '--study' : study = arg if opt == '--host' : connection_args["host"] = arg if opt == '--user' : connection_args["user"] = arg if opt == '--password' : connection_args["password"] = arg if opt == '--runid' : runid = arg if opt == '--order' : order = arg if (not study): print ("A study should be provided. use --study ") sys.exit(-1) base = base.Base(study,**connection_args) if (runid): getRunInfo(runid) else: infos = [] infos.append("run_name") infos.append("id") infos.append("job_id") infos.append("state") infos.append("nproc") infos.append("machine_name") format_string = " {:<20} | {:^4} | {:^6} | {:<10} | {:^5} | {:<20}" line = format_string.format(*infos) separator = "-" * len(line) print (separator) print (line) print (separator) myrun = run.Run() run_list = base.getObjectList(myrun,order) for r in run_list: run_infos = [r[col] for col in infos] line = format_string.format(*run_infos) print (line) diff --git a/python/BlackDynamite/launchRuns.py b/python/BlackDynamite/launchRuns.py index ab6020c..cd27040 100644 --- a/python/BlackDynamite/launchRuns.py +++ b/python/BlackDynamite/launchRuns.py @@ -1,109 +1,118 @@ #!/usr/bin/env python import base import job import run import os , sys, stat import runconfig import subprocess import conffile import getopt import socket truerun = False dbhost = None study = None outpath = None generator = None -options, remainder = getopt.getopt(sys.argv[1:],"", ["truerun","study=","dbhost=","outpath=","generator=","nruns="]) +options, remainder = getopt.getopt(sys.argv[1:],"", ["truerun","study=","dbhost=","outpath=","generator=","nruns=","runid="]) nruns = 0 +runid = None for opt, arg in options: if opt == '--study' : study = arg if opt == '--truerun' : truerun = True if opt == '--dbhost' : dbhost = arg if opt == '--outpath' : outpath = arg if opt == '--generator' : generator = arg if opt == '--nruns' : nruns = int(arg) + if opt == '--runid' : + runid = int(arg) if (not study): print ("A study should be provided. use --study ") sys.exit(-1) if (not outpath): print ("A directory where to create temp files should be provided. use --outpath ") sys.exit(-1) if (truerun == False): print ("this is a dryrun use --truerun to make effective") if (not dbhost): print ("The default database lsmssrv1 has been used. use --dbhost to change that") dbhost = "lsmssrv1.epfl.ch" base = base.Base(study,host=dbhost) mydir = outpath + "BD-" + study + "-runs" if not os.path.exists(mydir): os.makedirs(mydir) os.chdir(mydir) myrun = run.Run() -myrun["state"] = "CREATED" -myrun["machine_name"] = socket.gethostname() + +if (runid): + myrun["id"] = runid + myrun.id = runid + myrun.prepare(base) +else: + myrun["state"] = "CREATED" + myrun["machine_name"] = socket.gethostname() run_list = base.getMatchedObjectList(myrun) if (nruns > 0): run_list = [run_list[i] for i in range(0,nruns)] if (len(run_list) == 0): print "No runs to be launched" for j in run_list: j["run_path"] = mydir + "/" + "run-" + str(j.id) j.prepare(base) base.update(j) if not os.path.exists("run-" + str(j.id)): os.makedirs("run-" + str(j.id)) os.chdir("run-" + str(j.id)) conffiles = j.getConfigFiles(base) for conf in conffiles: print ("create file " + conf["filename"]) f = open(conf["filename"], 'w') f.write(conf["file"]) f.close() if (generator): mymod = __import__(generator) print (mymod) mymod.launch(j,base,truerun) else: _exec = j.getExecFile(base) _exec["file"] = "#!/bin/bash\n" + _exec["file"] f = open(_exec["filename"], 'w') f.write(_exec["file"]) f.close() os.chmod(_exec["filename"], stat.S_IRWXU) print ("execute ./" + _exec["filename"]) if (truerun == True): ret = subprocess.call("./" + _exec["filename"],shell=True) os.chdir("../") if (truerun == True): base.commit()