diff --git a/python/BlackDynamite/bdlogging.py b/python/BlackDynamite/bdlogging.py index 978a767..3d1064f 100644 --- a/python/BlackDynamite/bdlogging.py +++ b/python/BlackDynamite/bdlogging.py @@ -1,72 +1,82 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDLogging" ] -import logging +import logging,traceback,os ################################################################ def invalidPrint(x): raise Exception('print should not be used in that class: use the logging system instead: "{0}"'.format(x)) ################################################################ class BDlogger: def __init__(self,name,fileHandler): self.logger_screen = logging.getLogger(name) self.logger_file = logging.getLogger(name + 'file') self.logger_file.addHandler(fileHandler) self.logger_screen.addHandler(logging.StreamHandler()) def setLogFileLevel(self,level): self.logger_file.setLevel(level) def setScreenLevel(self,level): self.logger_screen.setLevel(level) - + def getExtra(self): + extra = {} + tr = traceback.extract_stack(limit=3) + tr = tr[0] + fname = os.path.basename(tr[0]) + + extra['foo'] = tr[2] + extra['f'] = fname + extra['l'] = tr[1] + return extra + def debug(self,x): - self.logger_screen.debug(x) - self.logger_file.debug(x) + self.logger_screen.debug(x,extra=self.getExtra()) + self.logger_file.debug(x,extra=self.getExtra()) def warning(self,x): - self.logger_screen.warning(x) - self.logger_file.warning(x) + self.logger_screen.warning(x,extra=self.getExtra()) + self.logger_file.warning(x,extra=self.getExtra()) def info(self,x): - self.logger_screen.info(x) - self.logger_file.info(x) + self.logger_screen.info(x,extra=self.getExtra()) + self.logger_file.info(x,extra=self.getExtra()) def error(self,x): - self.logger_screen.error(x) - self.logger_file.error(x) + self.logger_screen.error(x,extra=self.getExtra()) + self.logger_file.error(x,extra=self.getExtra()) class BDLogging: """ """ filehandler = None @classmethod def getLogger(cls,name,level=logging.WARNING): if cls.filehandler is None: cls.filehandler = logging.FileHandler('bd.log', mode='w') - streamformatter = logging.Formatter(fmt='%(levelname)s:%(funcName)50s:%(filename)15s:%(lineno)s:' + ' '*10 + '%(message)s') + streamformatter = logging.Formatter(fmt='%(levelname)s:%(foo)50s:%(f)15s:%(l)s:' + ' '*10 + '%(message)s') cls.filehandler.setFormatter(streamformatter) logger = BDlogger(name,cls.filehandler) logger.setScreenLevel(logging.WARNING) logger.setLogFileLevel(level) logger.debug('Starting to log') return logger diff --git a/python/BlackDynamite/bdparser.py b/python/BlackDynamite/bdparser.py index 6ca9e0b..ec0ccc9 100755 --- a/python/BlackDynamite/bdparser.py +++ b/python/BlackDynamite/bdparser.py @@ -1,581 +1,582 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDParser", "RunParser" ] import BlackDynamite as BD import sys import re import os import stat import pwd import base import run import argcomplete, argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp ################################################################ -import bdlogging +import bdlogging,logging print = bdlogging.invalidPrint +#logger = bdlogging.BDLogging.getLogger(__name__,level=logging.DEBUG) logger = bdlogging.BDLogging.getLogger(__name__) ################################################################ class BDParser(object): """ """ def listPossibleHosts(self): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = [] try: hosts += [h.strip() for h in open(bd_hosts)] except: pass return hosts def listPossibleModules(self,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] module_list = [] paths = [p.strip() for p in paths if not p.strip() == ''] for p in paths: files = os.listdir(p) files = [f for f in files if os.path.splitext(f)[1] == '.py'] files = [f for f in files if not f[0] == '_'] matching_string = ".*blackdynamite.*" files = [os.path.splitext(f)[0] for f in files if re.match(matching_string,open(os.path.join(p,f)).read().replace('\n',' '),flags=re.IGNORECASE)] module_list += files logger.debug("found these files " + str(module_list)) return module_list def updatePossibleHosts(self,new_host): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = set(self.listPossibleHosts()) hosts.add(new_host) f = open(bd_hosts,'w') for h in hosts: f.write(h+'\n') def completer(self,prefix,**kwargs): logger.debug("in") try: params = vars(kwargs["parsed_args"]) logger.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.iteritems(): logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] logger.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() if 'BDconf' in params: self.readConfFiles(params,params['BDconf']) if 'host' in params: self.readConfFile(params,params['host']+'.bd') logger.debug("key " + str(key) + "\n") for k in params.keys(): logger.debug("params = " + str(k)) if (key == "host"): return self.listPossibleHosts() if (key == "study"): params["should_not_check_study"] = True mybase = base.Base(**params) return mybase.getSchemaList() if (key == 'quantity'): mybase = base.Base(**params) myrun = run.Run(mybase) return myrun.listQuantities() if (key in self.admissible_params and self.admissible_params[key] == ModuleType): logger.debug("trying to complete module list for '{}'".format(key) ) return self.listPossibleModules(params) except Exception as e: logger.debug(traceback.format_exc()) logger.debug(str(e)) return [] def listPossibleConf(self): logger.debug("in") files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) return files return files def readConfFiles(self,read_params,fnames): logger.debug("in") for f in fnames: self.readConfFile(read_params,f) def readConfFile(self,read_params,fname): logger.debug("in") if fname in self.readConfFileList: return self.readConfFileList.append(fname) if type(fname) == list: raise Exception('cannot use list in that function: ' + str(type(fname))) pre_args = {} for dir in ["./",os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir,fname) if (os.path.isfile(fullpath)): fname = fullpath break try: with open(fname) as fh: #print("loading file '{0}'".format(fname)) os.chmod(fname,stat.S_IREAD|stat.S_IWRITE) lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex,line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() pre_args[param] = val self.argv.append("--"+param) self.argv.append(val) read_params.update(self.createParamsMap(pre_args)) except Exception as e: logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc())) def checkParam(self,p,dico): logger.debug("in") print("****************") print("Obsolete: should use the mandatory argument for the declare_params function") print("It was used by object " + str(self) + " for keyword " + p) print("FATAL => ABORT") print("****************") sys.exit(-1) def loadModule(self,read_params,myscript,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] mymod = None for p in paths: try: modfile = os.path.join(p,myscript+".py") #print("loading file " + modfile) mymod = imp.load_source(myscript,modfile) break except IOError as io_err: logger.debug("loadModule " + str(io_err)) logger.debug("loadModule " + str(mymod)) if (mymod is None): logger.debug("cannot find module '" + myscript + "' from paths " + str(paths)) logger.debug("trace :" + traceback.format_exc()+ '\n') raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) return mymod def createParamsMap(self,pre_args): logger.debug("in") read_params = {} for opt,args in pre_args.iteritems(): logger.debug("createParamsMap1 " + str(opt) + " : " + str(args)) if (args is None): continue if (not type(args) == list): args = [args] logger.debug("createParamsMap2 " + str(opt) + " : " + str(args)) for arg in args: if (arg is None): continue if (opt == 'BDconf'): self.readConfFiles(read_params,arg) continue if (opt == 'host'): self.updatePossibleHosts(arg) self.readConfFile(read_params,arg+'.bd') for param, typ in self.admissible_params.iteritems(): if opt == param: logger.debug("createParamsMap3 " + str(param) + " : " + str(typ)) if (typ == ModuleType): read_params[param] = self.loadModule(read_params,arg,pre_args) logger.debug("createParamsMap4 " + str(param) + " : " + str(typ)) elif (type(typ) == list): args = arg.split(",") if (not param in read_params): read_params[param] = [] if (not len(typ) == 1) : subtype = str else : subtype = typ[0] for i in range(0,len(args)): read_params[param].append(subtype(args[i])) else: read_params[param] = typ(arg) break return read_params def addModulesAdmissibleParameters(self,read_params): logger.debug("in") for k,v in read_params.iteritems(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update(mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys() if "default_params" in mymod.__dict__: self.default_params.update(mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) if "mandatory" in mymod.__dict__: self.mandatory.update(mymod.__dict__["mandatory"]) logger.debug(str(self.admissible_params)) def addModulesAdmissibleParametersForComplete(self,read_params): logger.debug("in") if not "_ARGCOMPLETE" in os.environ: return logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]) # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" logger.debug("break line " + os.environ["COMP_LINE"]) all_args = re.split(breaks,os.environ["COMP_LINE"]) logger.debug("break line " + str(all_args)) for i in range(0,len(all_args)): a = all_args[i] res = re.match("--(.*)",a) if res is None: continue a = res.group(1) logger.debug("treating a " + str(a) ) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: logger.debug("here treating a " + str(a) ) if i+1 >= len(all_args): continue b = all_args[i+1] logger.debug("treating b " + str(b) ) res = re.match("--(.*)",b) if res is not None: continue if not b.strip() == '': tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] logger.debug("tmp_read_params " + str(tmp_read_params) ) try: tmp_read_params = self.createParamsMap(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("AAAAAAAAA " + str(tmp_read_params)) try: self.addModulesAdmissibleParameters(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("CCCCCCCCCC" + str(self.admissible_params)) def constructArgParser(self,add_help=True,add_mandatory=True): logger.debug("in") parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g,param_list in self.group_params.iteritems(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.iteritems(): #print ("param {0}: {1}".format(param,typ) ) p_help = "help TODO" is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory) if (param in self.help) : p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else : grp = self.params_group["General"] if (typ is None): raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'") if (typ is bool): if (param in self.default_params and self.default_params[param] == True): grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_false', required = is_mandatory) else: grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_true', required = is_mandatory) elif (typ is list or typ == [str]): grp.add_argument("--" + param, action = 'append', dest = param, help = p_help, required = is_mandatory).completer = self.completer else: grp.add_argument("--" + param, dest = param, help = p_help, required = is_mandatory).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None): logger.debug("in") if (params is not None): self.admissible_params.update(params) if group not in self.group_params: self.group_params[group] = [] self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) for key in defaults.keys(): self.mandatory[key] = False if (help is not None): self.help.update(help) if (mandatory is not None): self.mandatory.update(mandatory) for param,typ in self.admissible_params.iteritems(): if typ == bool and param not in self.default_params: self.default_params[param] = False def addEnvBDArguments(self,parser): logger.debug("in") parser = self.constructArgParser(add_help=False,add_mandatory=False) pre_args = vars(parser.parse_known_args(args=self.argv)[0]) for name, value in os.environ.iteritems(): m = re.match("BLACKDYNAMITE_(.*)",name) if (m): var = m.group(1).lower() if (var not in pre_args or pre_args[var] is None): if (var in self.admissible_params): self.argv.append("--" + var) self.argv.append(value) def parseBDParameters(self, argv = None): logger.debug("in") if argv == None: self.argv = list(sys.argv[1:]) else: self.argv = list(argv) logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n") logger.debug("env is\n\n") for k,v in os.environ.iteritems(): logger.debug("export " + k + "='" + v + "'\n") logger.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False,add_mandatory=False) self.addEnvBDArguments(parser) logger.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=self.argv)[0] logger.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) logger.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) logger.debug("addModulesAdmissibleParametersForComplete\n") try: self.addModulesAdmissibleParametersForComplete(read_params) except KeyError as e: logger.debug("trace :" + traceback.format_exc()) logger.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=self.argv) read_params = self.createParamsMap(vars(pre_args)) if not "user" in read_params: read_params["user"] = pwd.getpwuid(os.getuid())[0] return read_params def __init__ (self): logger.debug("in") self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.mandatory = {} self.admissible_params["study"] = str self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language" self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" self.admissible_params["password"] = str self.help["password"] = "Provides the password" self.admissible_params["BDconf"] = list self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons" self.admissible_params["truerun"] = bool self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun" self.default_params["truerun"] = False self.admissible_params["job_constraints"] = [str] self.help["job_constraints"] = "This allows to constraint run selections by job properties" self.default_params["job_constraints"] = None self.admissible_params["run_constraints"] = [str] self.help["run_constraints"] = "This allows to constraint run selections by run properties" self.default_params["run_constraints"] = None self.admissible_params["binary_operator"] = str self.default_params["binary_operator"] = 'and' self.help["binary_operator"] = 'Set the default binary operator to make requests to database' self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["yes"] = bool self.default_params["yes"] = False self.help["yes"] = "Answer all questions to yes." self.admissible_params["logging"] = bool self.help["logging"] = "Activate the logging system" self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "job_constraints", "run_constraints", "list_parameters"] self.readConfFileList = [] ################################################################ def validate_question(question, params, default_validated = True): logger.debug("in") if (default_validated): default_str = "(Y/n)" else: default_str = "(y/N)" if (params["yes"] == False): validated = raw_input("{0}? {1} ".format(question, default_str)) #print (validated) if (validated == "\n" or validated == ""): validated = default_validated elif(validated == "Y" or validated == "y"): validated = True else: validated = False else: print("{0}? {1} Forced Y".format(question, default_str)) validated = True return validated ################################################################ def filterParams(sub_list,total_list): logger.debug("in") new_list = {} for p in sub_list: if (p in total_list and total_list[p] is not False): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): logger.debug("in") params = BDParser.parseBDParameters(self) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__ (self): logger.debug("in") BDParser.__init__(self) self.mandatory["machine_name"] = True self.mandatory["nproc"] = True self.mandatory["run_name"] = True self.admissible_params["machine_name"] = str self.help["machine_name"] = "Specify the name of the machine where the job is to be launched" self.admissible_params["nproc"] = int self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched" self.admissible_params["run_name"] = str self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind" self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"]