diff --git a/python/BlackDynamite/BDstat.py b/python/BlackDynamite/BDstat.py index 8ff78b2..2251a06 100644 --- a/python/BlackDynamite/BDstat.py +++ b/python/BlackDynamite/BDstat.py @@ -1,140 +1,140 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- from __future__ import print_function __all__ = [ "BDStat" ] import numpy as np import job import copy class BDStat(object): """ """ def average(self,quantity,run_list,entries_to_average): result = dict() run_info = dict() cpt = dict() run_counter = 0 nb_runs = len(run_list) entries_to_average.append("id") entries_to_consider = [] if (run_list is not []): for e in run_list[0][1].entries: if e not in entries_to_average: entries_to_consider.append(e) for r,j in run_list: q = r.getScalarQuantity(quantity) run_counter += 1 - if (q == None): + if (q is None): continue print ("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format(r.id,q.shape,run_counter,nb_runs,1.*run_counter/nb_runs*100)) ent = [j[i] for i in entries_to_consider] ent = tuple(ent) if (ent not in result.keys()): # print (ent) result[ent] = np.zeros([q.shape[0],3]) cpt[ent] = 0 # print (result[ent].shape) # print (q.shape) sz1 = q.shape[0] sz2 = result[ent].shape[0] if (sz1 > sz2): q = q[:sz2] elif (sz2 > sz1): result[ent] = result[ent][:sz1] result[ent][:,0] += q[:,0] result[ent][:,1] += q[:,1] result[ent][:,2] += q[:,1]**2 cpt[ent] += 1 for ent in result.keys(): # print (ent) myjob = job.Job(self.base) for i in range(0,len(entries_to_consider)): myjob.entries[entries_to_consider[i]] = ent[i] result[ent] /= cpt[ent] result[ent][:,2] -= result[ent][:,1]**2 result[ent][:,2] = np.maximum(np.zeros(result[ent][:,2].shape[0]),result[ent][:,2]) result[ent][:,2] = np.sqrt(result[ent][:,2]) result[ent] = {"ref_job":myjob , "averaged_number":cpt[ent], "data":result[ent]} return result def averageVector(self,quantity,run_list,entries_to_average): result = dict() all_steps = dict() run_info = dict() cpt = dict() run_counter = 0 nb_runs = len(run_list) entries_to_average.append("id") entries_to_consider = [] if (run_list is not []): for e in run_list[0][1].entries: if e not in entries_to_average: entries_to_consider.append(e) for r,j in run_list: steps,data = r.getAllVectorQuantity(quantity) run_counter += 1 if (data is None): continue print ("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format(r.id,data.shape,run_counter,nb_runs,1.*run_counter/nb_runs*100)) ent = [j[i] for i in entries_to_consider] ent = tuple(ent) if (ent not in result.keys()): # print (ent) result[ent] = np.zeros([data.shape[0],data.shape[1],2]) all_steps[ent] = steps cpt[ent] = 0 # print (result[ent].shape) # print (q.shape) sz1 = data.shape[0] sz2 = result[ent].shape[0] if (sz1 > sz2): data = data[:sz2] steps = steps[:sz2] elif (sz2 > sz1): result[ent] = result[ent][:sz1] all_steps[ent] = all_steps[ent][:sz1] result[ent][:,:,0] += data[:,:] result[ent][:,:,1] += data[:,:]**2 cpt[ent] += 1 for ent in result.keys(): # print (ent) myjob = job.Job(self.base) for i in range(0,len(entries_to_consider)): myjob.entries[entries_to_consider[i]] = ent[i] result[ent] /= cpt[ent] result[ent][:,:,1] -= result[ent][:,:,0]**2 result[ent][:,:,1] = np.sqrt(result[ent][:,:,1]) result[ent] = {"ref_job":myjob , "averaged_number":cpt[ent], "steps":all_steps[ent], "data":result[ent]} return result def __init__ (self,base,**params): self.base = base diff --git a/python/BlackDynamite/bdlogging.py b/python/BlackDynamite/bdlogging.py index 67d27d3..1d2d3fe 100644 --- a/python/BlackDynamite/bdlogging.py +++ b/python/BlackDynamite/bdlogging.py @@ -1,98 +1,98 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDLogging" ] import logging,traceback,os ################################################################ def invalidPrint(x): raise Exception('print should not be used in that class: use the logging system instead: "{0}"'.format(x)) ################################################################ file_handler = None def setFileHandler(logger,streamformatter): - if globals()['file_handler'] == None: + if globals()['file_handler'] is None: # print ("resetting the file for {0}".format(logger.name)) f = open('bd.log','w') f.close() globals()['file_handler'] = logging.FileHandler('bd.log') file_handler = globals()['file_handler'] file_handler.setFormatter(streamformatter) if not file_handler in logger.handlers: logger.addHandler(file_handler) ################################################################ Parent = logging.getLoggerClass() class BDlogger(Parent): def __init__(self,name): self.name = name self.logger_screen = Parent(name + 'screen') self.logger_file = Parent(name + 'file') self.streamformatter = logging.Formatter(fmt='%(levelname)s:%(foo)50s:%(f)15s:%(l)s:' + ' '*10 + '%(message)s') self.screen_handler = logging.StreamHandler() self.screen_handler.setFormatter(self.streamformatter) self.logger_screen.addHandler(self.screen_handler) self.enable_file = False self.setScreenLevel(logging.INFO) def setLogFileLevel(self,level): self.logger_file.setLevel(level) def setScreenLevel(self,level): self.logger_screen.setLevel(level) def activateFileLog(self,level = logging.DEBUG): setFileHandler(self.logger_file,self.streamformatter) self.enable_file = True self.setLogFileLevel(level) def getExtra(self): extra = {} tr = traceback.extract_stack(limit=3) tr = tr[0] fname = os.path.basename(tr[0]) extra['foo'] = tr[2] extra['f'] = fname extra['l'] = tr[1] return extra def debug(self,x): self.logger_screen.debug(x,extra=self.getExtra()) if self.enable_file is True: self.logger_file.debug(x,extra=self.getExtra()) def warning(self,x): self.logger_screen.warning(x,extra=self.getExtra()) if self.enable_file is True: self.logger_file.warning(x,extra=self.getExtra()) def info(self,x): self.logger_screen.info(x,extra=self.getExtra()) if self.enable_file is True: self.logger_file.info(x,extra=self.getExtra()) def error(self,x): self.logger_screen.error(x,extra=self.getExtra()) if self.enable_file is True: self.logger_file.error(x,extra=self.getExtra()) ################################################################ logging.setLoggerClass(BDlogger) ################################################################ import logging logger = logging.getLogger(__name__) ################################################################ def activateFileLogging(): for name,log in logging.Logger.manager.loggerDict.iteritems(): if isinstance(log,BDlogger): log.activateFileLog() diff --git a/python/BlackDynamite/bdparser.py b/python/BlackDynamite/bdparser.py index ddb97a4..9001c1f 100755 --- a/python/BlackDynamite/bdparser.py +++ b/python/BlackDynamite/bdparser.py @@ -1,590 +1,590 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDParser", "RunParser" ] import BlackDynamite as BD import sys import re import os import stat import pwd import base import run import argcomplete, argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp ################################################################ import bdlogging,logging print = bdlogging.invalidPrint logger = logging.getLogger(__name__) ################################################################ class BDParser(object): """ """ def listPossibleHosts(self): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = [] try: hosts += [h.strip() for h in open(bd_hosts)] except: pass return hosts def listPossibleModules(self,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] module_list = [] paths = [p.strip() for p in paths if not p.strip() == ''] for p in paths: files = os.listdir(p) files = [f for f in files if os.path.splitext(f)[1] == '.py'] files = [f for f in files if not f[0] == '_'] matching_string = ".*blackdynamite.*" files = [os.path.splitext(f)[0] for f in files if re.match(matching_string,open(os.path.join(p,f)).read().replace('\n',' '),flags=re.IGNORECASE)] module_list += files logger.debug("found these files " + str(module_list)) return module_list def updatePossibleHosts(self,new_host): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = set(self.listPossibleHosts()) hosts.add(new_host) f = open(bd_hosts,'w') for h in hosts: f.write(h+'\n') def completer(self,prefix,**kwargs): #bdlogging.activateFileLogging() try: params = vars(kwargs["parsed_args"]) if params['logging'] is True: bdlogging.activateFileLogging() logger.debug("in") logger.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.iteritems(): logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] logger.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() if 'BDconf' in params: self.readConfFiles(params,params['BDconf']) if 'host' in params: self.readConfFile(params,params['host']+'.bd') logger.debug("key " + str(key) + "\n") for k in params.keys(): logger.debug("params = " + str(k)) if (key == "host"): return self.listPossibleHosts() if (key == "study"): params["should_not_check_study"] = True mybase = base.Base(**params) return mybase.getSchemaList() if (key == 'quantity'): mybase = base.Base(**params) myrun = run.Run(mybase) return myrun.listQuantities() if (key in self.admissible_params and self.admissible_params[key] == ModuleType): logger.debug("trying to complete module list for '{}'".format(key) ) return self.listPossibleModules(params) except Exception as e: logger.debug(traceback.format_exc()) logger.debug(str(e)) return [] def listPossibleConf(self): logger.debug("in") files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) return files return files def readConfFiles(self,read_params,fnames): logger.debug("in") for f in fnames: self.readConfFile(read_params,f) def readConfFile(self,read_params,fname): logger.debug("in") if fname in self.readConfFileList: return self.readConfFileList.append(fname) if type(fname) == list: raise Exception('cannot use list in that function: ' + str(type(fname))) pre_args = {} for dir in ["./",os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir,fname) if (os.path.isfile(fullpath)): fname = fullpath break try: with open(fname) as fh: #print("loading file '{0}'".format(fname)) os.chmod(fname,stat.S_IREAD|stat.S_IWRITE) lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex,line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() pre_args[param] = val self.argv.append("--"+param) self.argv.append(val) read_params.update(self.createParamsMap(pre_args)) except Exception as e: logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc())) def checkParam(self,p,dico): logger.debug("in") print("****************") print("Obsolete: should use the mandatory argument for the declare_params function") print("It was used by object " + str(self) + " for keyword " + p) print("FATAL => ABORT") print("****************") sys.exit(-1) def loadModule(self,read_params,myscript,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] mymod = None for p in paths: try: modfile = os.path.join(p,myscript+".py") #print("loading file " + modfile) mymod = imp.load_source(myscript,modfile) break except IOError as io_err: logger.debug("loadModule " + str(io_err)) logger.debug("loadModule " + str(mymod)) if (mymod is None): logger.debug("cannot find module '" + myscript + "' from paths " + str(paths)) logger.debug("trace :" + traceback.format_exc()+ '\n') raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) return mymod def createParamsMap(self,pre_args): logger.debug("in") read_params = {} if ('logging' in pre_args) and (pre_args['logging'] is True): bdlogging.activateFileLogging() for opt,args in pre_args.iteritems(): logger.debug("createParamsMap1 " + str(opt) + " : " + str(args)) if (args is None): continue if (not type(args) == list): args = [args] if (type(args) == str) : args = [args] logger.debug("createParamsMap2 " + str(opt) + " : " + str(args)) for arg in args: if (arg is None): continue if (opt == 'BDconf'): if (not type(arg) == list): arg = [arg] self.readConfFiles(read_params,arg) continue if (opt == 'host'): self.updatePossibleHosts(arg) self.readConfFile(read_params,arg+'.bd') for param, typ in self.admissible_params.iteritems(): if opt == param: logger.debug("createParamsMap3 " + str(param) + " : " + str(typ)) if (typ == ModuleType): read_params[param] = self.loadModule(read_params,arg,pre_args) logger.debug("createParamsMap4 " + str(param) + " : " + str(typ)) elif (type(typ) == list): args = arg.split(",") if (not param in read_params): read_params[param] = [] if (not len(typ) == 1) : subtype = str else : subtype = typ[0] for i in range(0,len(args)): read_params[param].append(subtype(args[i])) else: read_params[param] = typ(arg) break return read_params def addModulesAdmissibleParameters(self,read_params): logger.debug("in") for k,v in read_params.iteritems(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update(mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys() if "default_params" in mymod.__dict__: self.default_params.update(mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) if "mandatory" in mymod.__dict__: self.mandatory.update(mymod.__dict__["mandatory"]) logger.debug(str(self.admissible_params)) def addModulesAdmissibleParametersForComplete(self,read_params): logger.debug("in") if not "_ARGCOMPLETE" in os.environ: return logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]) # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" logger.debug("break line " + os.environ["COMP_LINE"]) all_args = re.split(breaks,os.environ["COMP_LINE"]) logger.debug("break line " + str(all_args)) for i in range(0,len(all_args)): a = all_args[i] res = re.match("--(.*)",a) if res is None: continue a = res.group(1) logger.debug("treating a " + str(a) ) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: logger.debug("here treating a " + str(a) ) if i+1 >= len(all_args): continue b = all_args[i+1] logger.debug("treating b " + str(b) ) res = re.match("--(.*)",b) if res is not None: continue if not b.strip() == '': tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] logger.debug("tmp_read_params " + str(tmp_read_params) ) try: tmp_read_params = self.createParamsMap(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("AAAAAAAAA " + str(tmp_read_params)) try: self.addModulesAdmissibleParameters(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("CCCCCCCCCC" + str(self.admissible_params)) def constructArgParser(self,add_help=True,add_mandatory=True): logger.debug("in") parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g,param_list in self.group_params.iteritems(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.iteritems(): #print ("param {0}: {1}".format(param,typ) ) p_help = "help TODO" is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory) if (param in self.help) : p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else : grp = self.params_group["General"] if (typ is None): raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'") if (typ is bool): if (param in self.default_params and self.default_params[param] == True): grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_false', required = is_mandatory) else: grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_true', required = is_mandatory) elif (typ is list or typ == [str]): grp.add_argument("--" + param, action = 'append', dest = param, help = p_help, required = is_mandatory).completer = self.completer else: grp.add_argument("--" + param, dest = param, help = p_help, required = is_mandatory).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None): logger.debug("in") if (params is not None): self.admissible_params.update(params) if group not in self.group_params: self.group_params[group] = [] self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) for key in defaults.keys(): self.mandatory[key] = False if (help is not None): self.help.update(help) if (mandatory is not None): self.mandatory.update(mandatory) for param,typ in self.admissible_params.iteritems(): if typ == bool and param not in self.default_params: self.default_params[param] = False def addEnvBDArguments(self,parser): logger.debug("in") parser = self.constructArgParser(add_help=False,add_mandatory=False) pre_args = vars(parser.parse_known_args(args=self.argv)[0]) for name, value in os.environ.iteritems(): m = re.match("BLACKDYNAMITE_(.*)",name) if (m): var = m.group(1).lower() if (var not in pre_args or pre_args[var] is None): if (var in self.admissible_params): self.argv.append("--" + var) self.argv.append(value) def parseBDParameters(self, argv = None): logger.debug("in") - if argv == None: + if argv is None: self.argv = list(sys.argv[1:]) else: self.argv = list(argv) logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n") logger.debug("env is\n\n") for k,v in os.environ.iteritems(): logger.debug("export " + k + "='" + v + "'\n") logger.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False,add_mandatory=False) self.addEnvBDArguments(parser) logger.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=self.argv)[0] logger.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) logger.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) logger.debug("addModulesAdmissibleParametersForComplete\n") try: self.addModulesAdmissibleParametersForComplete(read_params) except KeyError as e: logger.debug("trace :" + traceback.format_exc()) logger.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=self.argv) read_params = self.createParamsMap(vars(pre_args)) if not "user" in read_params: read_params["user"] = pwd.getpwuid(os.getuid())[0] return read_params def __init__ (self): logger.debug("in") self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.mandatory = {} self.admissible_params["study"] = str self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language" self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" self.admissible_params["password"] = str self.help["password"] = "Provides the password" self.admissible_params["BDconf"] = list self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons" self.admissible_params["truerun"] = bool self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun" self.default_params["truerun"] = False self.admissible_params["job_constraints"] = [str] self.help["job_constraints"] = "This allows to constraint run selections by job properties" self.default_params["job_constraints"] = None self.admissible_params["run_constraints"] = [str] self.help["run_constraints"] = "This allows to constraint run selections by run properties" self.default_params["run_constraints"] = None self.admissible_params["binary_operator"] = str self.default_params["binary_operator"] = 'and' self.help["binary_operator"] = 'Set the default binary operator to make requests to database' self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["yes"] = bool self.default_params["yes"] = False self.help["yes"] = "Answer all questions to yes." self.admissible_params["logging"] = bool self.help["logging"] = "Activate the file logging system" self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "job_constraints", "run_constraints", "list_parameters"] self.readConfFileList = [] ################################################################ def validate_question(question, params, default_validated = True): logger.debug("in") if (default_validated): default_str = "(Y/n)" else: default_str = "(y/N)" if (params["yes"] == False): validated = raw_input("{0}? {1} ".format(question, default_str)) #print (validated) if (validated == "\n" or validated == ""): validated = default_validated elif(validated == "Y" or validated == "y"): validated = True else: validated = False else: logger.info("{0}? {1} Forced Y".format(question, default_str)) validated = True return validated ################################################################ def filterParams(sub_list,total_list): logger.debug("in") new_list = {} for p in sub_list: if (p in total_list and total_list[p] is not False): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): logger.debug("in") params = BDParser.parseBDParameters(self) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__ (self): logger.debug("in") BDParser.__init__(self) self.mandatory["machine_name"] = True self.mandatory["nproc"] = True self.mandatory["run_name"] = True self.admissible_params["machine_name"] = str self.help["machine_name"] = "Specify the name of the machine where the job is to be launched" self.admissible_params["nproc"] = int self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched" self.admissible_params["run_name"] = str self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind" self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"] diff --git a/python/BlackDynamite/graphhelper.py b/python/BlackDynamite/graphhelper.py index 1148d6a..396884f 100755 --- a/python/BlackDynamite/graphhelper.py +++ b/python/BlackDynamite/graphhelper.py @@ -1,394 +1,394 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- __all__ = [ "GraphHelper", "GraphParser" ] import base import run import job import runselector import jobselector import bdparser ############################################## ##Temp workaround for broken debian package ## ############################################## #import matplotlib############################# #matplotlib.use('Agg')######################### ############################################## #import matplotlib.pyplot as plt import re import sys import numpy as np class GraphHelper(object): """ """ def getMeasures(self,run_list): myresults = [] add_req = [] if (self.frequency): add_req += ["step %% {0} = 0".format(self.frequency)] if (self.start): add_req += ["step > {0}".format(self.start) ] if (self.end): add_req += ["step < {0}".format(self.end) ] for r,j in run_list: #print ("retrieve data from run " + r["run_name"]) res = r.getScalarQuantities(self.quantities,add_req) for key , value in res: - if (value == None): + if (value is None): del res[key] myresults.append([r,j,res]) return myresults def selectGraphs(self): # print 'AAAAAAAAA ' + self.binary_operator run_list = self.runSelector.selectRuns(self.run_constraints,self.job_constraints, self.sort_by,binary_operator=self.binary_operator) results = self.getMeasures(run_list) return results def show(self): import matplotlib.pyplot as plt plt.show() def makeGraphs(self,fig=None,**kwargs): import matplotlib.pyplot as plt results = self.selectGraphs() if fig is None: fig = plt.figure(figsize=self.figsize) for r,j,data in results: if data: self.makeCurve(data,fig=fig,myrun=r,myjob=j,**kwargs) return fig def replaceRunAndJobsParameters(self,name,myrun,myjob): res = name # print (res) codes = [["%r."+key,myrun[key]] for key in myrun.entries.keys()] codes += [["%j."+key,myjob[key]] for key in myjob.entries.keys()] for code,val in codes: res = res.replace(code,str(val)) return res def generateLabels(self,results,myrun,myjob): labels = [] names = [r[0] for r in results] for i in range(0,len(results)): name = results[i][0] if (not self.legend or i >= len(self.legend) or not self.legend[i]): labels.append(self.replaceRunAndJobsParameters(name,myrun,myjob)) continue # print (self.legend[i]) head_legend = self.legend[i].replace("{","{{") head_legend = head_legend.replace("}","}}") head_legend = re.sub(r"(%)([0-9]+)",r'{\2}',head_legend).format(*names) # print (head_legend) head_legend = self.replaceRunAndJobsParameters(head_legend,myrun,myjob) # print (head_legend) # if (not head_legend.find("%") == -1): # print("unknown variable name. Possible variables are:") # print "\n".join([c[0] for c in codes]) # sys.exit(-1) # print (head_legend) labels.append(head_legend) return labels def makeComposedQuantity(self,results,myrun,myjob): vecs = [r[1] for r in results] names = [r[0] for r in results] # print (vecs[0].shape) new_results = [] for comp in self.using: exprs = comp.split(":") tmp_res = [] for i in [0,1]: e = re.sub(r"(%)([0-9]+)\.(x)",r"vecs[\2][:,0]",exprs[i]) e = re.sub(r"(%)([0-9]+)\.(y)",r"vecs[\2][:,1]",e) e = self.replaceRunAndJobsParameters(e,myrun,myjob) try: tmp_res.append(eval(e)) except Exception as ex: print (names) print ("invalid expression: '" + exprs[i] + "'") print ("invalid expression: '" + e + "'") print (ex) i = 1 for v in vecs: print ('quantity {0}/{1} shape: {2}'.format(i,len(vecs),v.shape)) i += 1 sys.exit(-1) name = re.sub(r"(%)([0-9]+)\.([x|y])",r'(" + str(names[\2]) + ")',exprs[1]) res = np.zeros((tmp_res[0].shape[0],2)) res[:,0] = tmp_res[0] res[:,1] = tmp_res[1] # print (res.shape) expr = re.sub(r"(%)([0-9]+)",r"vecs[\2]",comp) # res[0] = eval(expr) # print (name) name = "\"" + name + "\"" # print (name) name = eval(name) # print (name) new_results.append([name,res]) return new_results def decorateGraph(self, fig, myrun, myjob, results): if not results: return if fig is None: fig = plt.figure() axe=fig.add_subplot(1,1,1) if self.xrange: axe.set_xlim(self.xrange) if self.yrange: axe.set_ylim(self.yrange) if (self.xlabel): axe.set_xlabel(self.xlabel) if (self.ylabel): axe.set_ylabel(self.ylabel) if (self.title): t = self.replaceRunAndJobsParameters(self.title,myrun,myjob) axe.set_title(t) axe.grid(True,linewidth=0.1) if (self.using): results = self.makeComposedQuantity(results,myrun,myjob) labels = self.generateLabels(results,myrun,myjob) # print (labels) return fig, axe, results, labels def makeCurve(self,results,myrun=None,myjob=None,fig=None,**kwargs): fig, axe, results, labels = self.decorateGraph(fig, myrun, myjob, results) for count, result in enumerate(results): name = result[0] vec = result[1] label = labels[count] # print (self.quantities) # print (name) style=dict() if (self.marker is not None): style["marker"] = self.marker if self.blackwhite: width_index = self.cycle_index/len(self.linestyle_cycle) style_index = self.cycle_index%len(self.linestyle_cycle) self.cycle_index += 1 style["linewidth"] = self.linewidth_cycle[width_index] style["linestyle"] = self.linestyle_cycle[style_index] style["color"] = 'k' axe.plot(vec[:,0]/self.xscale, vec[:,1]/self.yscale, label=label, **style) axe.legend(loc='best') if (self.fileout): fig.savefig(self.fileout) return fig def setJobConstraint(self,**params): self.job_constraints = [] if ("job_constraints" in params): self.job_constraints = params["job_constraints"] def setRunConstraint(self,**params): self.run_constraints = [] if ("run_constraints" in params): self.run_constraints = params["run_constraints"] def setBinaryOperator(self,**params): self.binary_operator = 'and' if ("binary_operator" in params): self.binary_operator = params["binary_operator"] def setQuantity(self,**params): if ("quantity" in params): self.quantities = params["quantity"] else: print ("quantity should be provided using option --quantity") self.quantities = "__BLACKDYNAMITE_ERROR__" def __init__ (self,base,**params): self.setJobConstraint(**params) self.setRunConstraint(**params) self.setBinaryOperator(**params) self.setQuantity(**params) self.base = base self.runSelector = runselector.RunSelector(self.base) self.fig = None self.xrange = None self.yrange = None self.xlabel = None self.ylabel = None self.xscale = None self.yscale = None self.fileout = None self.title = None self.using = None self.frequency = None self.start = None self.end = None self.figsize = None self.blackwhite = None self.legend = None self.sort_by = None self.marker = None ### set the members if keys are present in params members = set(self.__dict__.keys()) p = set(params.keys()) for key in members & p: setattr(self,key,params[key]) if(params["list_quantities"] == True): myrun = run.Run(base) print ("list of possible quantities:\n") print("\n".join(myrun.listQuantities())) sys.exit(0) if(params["list_parameters"] == True): myjob = job.Job(base) myjob.prepare() print ("****************************************************************") print ("Job parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myjob.types.iteritems() ] print("\n".join(params)) myrun = run.Run(base) myrun.prepare() print ("****************************************************************") print ("Run parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myrun.types.iteritems() ] print("\n".join(params)) sys.exit(0) self.linewidth_cycle = [1, 2, 4] self.linestyle_cycle = ['-', '--', '-.'] self.cycle_index = 0 ################################################################ class GraphParser(bdparser.BDParser): """ """ def __init__ (self): bdparser.BDParser.__init__(self) self.admissible_params["quantity"] = [str] self.help["quantity"] = "Specify the quantity to be outputed" self.admissible_params["xrange"] = [float] self.help["xrange"] = "Specify range of values in the X direction" self.admissible_params["yrange"] = [float] self.help["yrange"] = "Specify range of values in the Y direction" self.admissible_params["sort_by"] = [str] self.help["sort_by"] = "Specify a study parameter to be used in sorting the curves" self.admissible_params["xlabel"] = str self.help["xlabel"] = "Specify the label for the X axis" self.admissible_params["ylabel"] = str self.help["ylabel"] = "Specify the label for the Y axis" self.admissible_params["xscale"] = float self.default_params["xscale"] = 1. self.help["xscale"] = "Specify a scale factor for the X axis" self.admissible_params["yscale"] = float self.default_params["yscale"] = 1. self.help["yscale"] = "Specify a scale factor for the Y axis" self.admissible_params["title"] = str self.help["title"] = "Specify title for the graph" self.admissible_params["legend"] = [str] self.help["legend"] = "Specify a legend for the curves. The syntax can use %%j.param or %%r.param to use get job and run values" self.default_params["legend"] = None self.admissible_params["using"] = [str] self.help["using"] = "Allow to combine several quantities. The syntax uses python syntax where %%quantity1.column1:%%quantity2.column2 is the python numpy vector provided by quantity number (provided using the --quantities option) and column number (x or y). The sytax is comparable to the GNUPlot one in using the ':' to separate X from Y axis" self.admissible_params["list_quantities"] = bool self.help["list_quantities"] = "Request to list the possible quantities to be plotted" self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["frequency"] = int self.default_params["frequency"] = 1 self.help["frequency"] = "Set a frequency at which the quantity values should be retreived (helpful when the amount of data is very large)" self.admissible_params["start"] = float self.help["start"] = "Set the start X value for the graph" self.admissible_params["end"] = int self.help["end"] = "Set the end X value for the graph" self.admissible_params["figsize"] = [float] self.admissible_params["blackwhite"] = bool self.default_params["blackwhite"] = False self.help["blackwhite"] = "Request a black and white graph generation" self.default_params["blackwhite"] = False self.help["blackwhite"] = "Request to plot a black and white graph" self.admissible_params["marker"] = str self.help["marker"] = "Request a specific marker (matplotlib option)" self.admissible_params["fileout"] = str self.help["fileout"] = "Request to write a PDF file (given its name) containing the graph" self.group_params["GraphHelper"] = [ "quantity", "xrange", "yrange", "sort_by", "xlabel", "ylabel", "xscale", "yscale", "title", "legend", "using", "list_quantities", "list_parameters", "frequency", "start", "end", "figsize", "blackwhite", "marker", "fileout"] ################################################################ diff --git a/python/BlackDynamite/sqlobject.py b/python/BlackDynamite/sqlobject.py index 9aaa190..39329fd 100644 --- a/python/BlackDynamite/sqlobject.py +++ b/python/BlackDynamite/sqlobject.py @@ -1,321 +1,321 @@ from __future__ import print_function import copy import re import sys import datetime class SQLObject(object): """ """ def __str__(self): output = "" keys = set(self.entries.keys()) keys.remove('id') keys = list(keys) keys = ['id'] + keys outputs = [] for k in keys: v = self.entries[k] outputs += [k + ": " + str(v)] return "\n".join(outputs) def __getitem__(self, index) : return self.entries[index.lower()] def __setitem__(self, index, value) : self.entries[index.lower()] = value def commit(self): self.base.connection.commit() def setConstraint(self, constraints) : # print (constraints) for cons in constraints: _regex = "\s*(\w*)\s*((?:\=~)|(?:!=)|<|>|=)\s*(.*)" match = re.match(_regex,cons) # print (match.groups()) if (not match or (not len(match.groups()) == 3)): print ("malformed constraint: " + cons) sys.exit(-1) key = match.group(1).lower().strip() op = match.group(2) if (op == "=~"): op = "~" # print (op) val = match.group(3) # print (key) # print (op) # print (val) if (not key in self.types): print ("unknown key '{0}'".format(key)) print (constraints) print ("possible keys are:") for k in self.types.keys(): print ("\t" + k) sys.exit(-1) # print (val) # print (key) # print (self.types[key]) val = self.types[key](val) if (not key in self.entries): self.entries[key] = [] self.entries[key].append(val) if (not key in self.operators): self.operators[key] = [] self.operators[key].append(op) def setEntries(self,params): self.prepare() for p, val in params.iteritems(): if (p in self.types): self.entries[p] = val def setFields(self, constraints) : for cons in constraints: _regex = "(\w*)\s*=\s*(.*)" match = re.match(_regex,cons) if (not match or (not len(match.groups()) == 2)): print ("malformed assignment: " + cons) sys.exit(-1) key = match.group(1).lower().strip() val = match.group(2) if (not key in self.types): print ("unknown key '{0}'".format(key)) print ("possible keys are:") for k in self.types.keys(): print ("\t" + k) sys.exit(-1) val = self.types[key](val) self.entries[key] = val def __init__(self,base,my_id=0): self._is_input = True self.foreign_keys = {} self.allowNull = {} self.types = {} self.entries = {} self.id = my_id self.base = base self.operators = {} def __copy__(self): _cp.types = self.types.copy() _cp.entries = self.entries.copy() _cp.id = self.id _cp.foreign_keys = self.foreign_keys.copy() _cp.allowNull = self.foreign_keys.copy() _cp.connection = self.connection.copy() return _cp def __deepcopy__(self, memo): _cp = type(self)(self.base) _cp.types = copy.deepcopy(self.types.copy(),memo) _cp.entries = copy.deepcopy(self.entries.copy(),memo) _cp.id = self.id _cp.foreign_keys = copy.deepcopy(self.foreign_keys,memo) _cp.allowNull = copy.deepcopy(self.foreign_keys,memo) _cp.connection = self.base return _cp def prepare(self): self.base.setObjectItemTypes(self) @property def is_input(self): return _is_input @is_input.setter def is_input(self, value): self._is_input = value @property def id(self): if not hasattr(self, "_id"): raise Exception("You either forgot to save the id at insert for " "class '{0}' or id this object doesn't know its id " "yet because it has never been inserted into the" "data base".format(self.__class__.__name__)) return self._id @id.setter def id(self, value): self._id = value def __conform__(self, foo): if foo is ISQLQuote: return self @classmethod def from_select(cls, fetched_row): raise Exception("Your forgot to implement the select method for class " "'{0}'!".format(self.__class__.__name__)) def _sql_members(self): raise Exception("Till, you forgot to implement the property " "__sql_members for the class '{0}'".format( self.__class__.__name__)) def insert(self): params = list() # print (self.types) ex_msg = "" for key,value in self.types.items(): if key == "id": continue if ((key not in self.entries) and (key not in self.allowNull)): ex_msg += "key '" + key + "' must be given a value before proceeding insertion\n" if (not ex_msg == ""): raise Exception("\n****************\n"+ex_msg+"****************\n") for key,value in self.entries.items(): # print (key) # print (self.types[key]) # print (value) params.append(self.types[key.lower()](value)) request = "INSERT INTO {0}.{1} ({2}) VALUES ({3}) RETURNING id".format(self.base.schema,self.table_name,','.join(self.entries.keys()),','.join(["%s" for item in params])), params return request def delete(self): request,params = "DELETE FROM {0}.{1} WHERE id={2}".format(self.base.schema,self.table_name,self.id),[] self.base.performRequest(request,params) def update(self): params = list() keys = list() for key,value in self.entries.items(): - if (value == None): + if (value is None): continue _type = self.types[key] # print (_type) # print (key) # print (type(value)) if (_type == datetime.datetime): continue # _type = str keys.append(key) params.append(_type(value)) request = "UPDATE {0}.{1} SET ({2}) = ({3}) WHERE id = {4}".format(self.base.schema,self.table_name,','.join(keys),','.join(["%s" for item in params]),self.id) self.base.performRequest(request,params) def getquoted(self): objs = [sql_adapt(member) for member in self._sql_members()] for obj in objs: if hasattr(obj, 'prepare'): obj.prepare(self._conn) quoted_objs = [obj.getquoted() for obj in objs] return '(' + ', '.join(quoted_objs) + ')' def createTableRequest(self): query_string = "CREATE TABLE {0}.{1} ( id SERIAL PRIMARY KEY,".format(self.base.schema,self.table_name) for key,value in self.types.items(): if (value == float): type_string = "DOUBLE PRECISION" elif (value == int): type_string = "INTEGER" elif (value == str): type_string = "TEXT" elif (value == bool): type_string = "BOOLEAN" elif (value == datetime.datetime): type_string = "TIMESTAMP" else: print (value) raise Exception("type '{0}' not handled".format(value)) query_string += "{0} {1} ".format(key, type_string) if (key not in self.allowNull): query_string += " NOT NULL" query_string += "," for key, value in self.foreign_keys.items(): query_string += "FOREIGN KEY ({0}) REFERENCES {1}.{2},".format(key,self.base.schema,value) return query_string[:-1] +");" def makeMatchingCondition(self, binary_operator = "and"): self.prepare() condition = "" params = [] binary_operator = " " + binary_operator + " " for key,values in self.entries.items(): if (values is None): continue if (not type(values) == list): values = [values] for i in range(0,len(values)): try: if (not isinstance(self.types[key],type)): raise Exception("Key " + key + " needs to be given a type and not " + self.types[key]) except KeyError as err: raise KeyError("{0}, keys are {1}".format(err, self.types.keys())) # print (str(key) + "type:" + str(self.types[key]) + " " + str(values[i])) params.append(self.types[key](values[i])) op = "=" if (key in self.operators): op = self.operators[key][i] if (condition): condition += binary_operator if isinstance(values[i], float) and op == "=" and values[i] != 0: condition += "@({0}.{1} / %s -1) < 1e-10".format( self.table_name, key) else: condition += self.table_name + "." + key condition += " " + op + " " condition += " %s " return condition,params def makeSelectRequest(self): request = "SELECT * FROM {0}.{1} ".format(self.base.schema,self.table_name) return request def getObjectList(self, order="id"): self.prepare() request = "SELECT * FROM {0}.{1} ORDER BY ".format(self.base.schema,self.table_name) + order curs = self.base.performRequest(request) return self.buildList(curs) return self.buildList(sqlobject,curs) def getMatchedObjectList(self, order_condition = ""): condition, params = self.makeMatchingCondition() request = self.makeSelectRequest() # print (condition) # print (order_condition) if (condition): request += "WHERE " + condition if (order_condition): request += order_condition #print (request) #print (params) curs = self.base.performRequest(request, params) return self.buildList(curs) def buildList(self, curs): col_info = self.base.getColumnProperties(self) list_objects = [] for entries in curs: # print (col_info) obj = copy.deepcopy(self) for i in range(0,len(col_info)): col_name = col_info[i][0] obj[col_name] = entries[i] obj.id = obj["id"] # print (col_name + " " + str(entries[i])) list_objects.append(obj) # print (list_objects[0].entries) return list_objects