diff --git a/python/BlackDynamite/bdparser.py b/python/BlackDynamite/bdparser.py index 10a61bd..ddb97a4 100755 --- a/python/BlackDynamite/bdparser.py +++ b/python/BlackDynamite/bdparser.py @@ -1,590 +1,590 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDParser", "RunParser" ] import BlackDynamite as BD import sys import re import os import stat import pwd import base import run import argcomplete, argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp ################################################################ import bdlogging,logging print = bdlogging.invalidPrint logger = logging.getLogger(__name__) ################################################################ class BDParser(object): """ """ def listPossibleHosts(self): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = [] try: hosts += [h.strip() for h in open(bd_hosts)] except: pass return hosts def listPossibleModules(self,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] module_list = [] paths = [p.strip() for p in paths if not p.strip() == ''] for p in paths: files = os.listdir(p) files = [f for f in files if os.path.splitext(f)[1] == '.py'] files = [f for f in files if not f[0] == '_'] matching_string = ".*blackdynamite.*" files = [os.path.splitext(f)[0] for f in files if re.match(matching_string,open(os.path.join(p,f)).read().replace('\n',' '),flags=re.IGNORECASE)] module_list += files logger.debug("found these files " + str(module_list)) return module_list def updatePossibleHosts(self,new_host): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir,'hosts') hosts = set(self.listPossibleHosts()) hosts.add(new_host) f = open(bd_hosts,'w') for h in hosts: f.write(h+'\n') def completer(self,prefix,**kwargs): #bdlogging.activateFileLogging() try: params = vars(kwargs["parsed_args"]) if params['logging'] is True: bdlogging.activateFileLogging() logger.debug("in") logger.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.iteritems(): logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] logger.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() if 'BDconf' in params: self.readConfFiles(params,params['BDconf']) if 'host' in params: self.readConfFile(params,params['host']+'.bd') logger.debug("key " + str(key) + "\n") for k in params.keys(): logger.debug("params = " + str(k)) if (key == "host"): return self.listPossibleHosts() if (key == "study"): params["should_not_check_study"] = True mybase = base.Base(**params) return mybase.getSchemaList() if (key == 'quantity'): mybase = base.Base(**params) myrun = run.Run(mybase) return myrun.listQuantities() if (key in self.admissible_params and self.admissible_params[key] == ModuleType): logger.debug("trying to complete module list for '{}'".format(key) ) return self.listPossibleModules(params) except Exception as e: logger.debug(traceback.format_exc()) logger.debug(str(e)) return [] def listPossibleConf(self): logger.debug("in") files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) return files return files def readConfFiles(self,read_params,fnames): logger.debug("in") for f in fnames: self.readConfFile(read_params,f) def readConfFile(self,read_params,fname): logger.debug("in") if fname in self.readConfFileList: return self.readConfFileList.append(fname) if type(fname) == list: raise Exception('cannot use list in that function: ' + str(type(fname))) pre_args = {} for dir in ["./",os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir,fname) if (os.path.isfile(fullpath)): fname = fullpath break try: with open(fname) as fh: #print("loading file '{0}'".format(fname)) os.chmod(fname,stat.S_IREAD|stat.S_IWRITE) lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex,line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() pre_args[param] = val self.argv.append("--"+param) self.argv.append(val) read_params.update(self.createParamsMap(pre_args)) except Exception as e: logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc())) def checkParam(self,p,dico): logger.debug("in") print("****************") print("Obsolete: should use the mandatory argument for the declare_params function") print("It was used by object " + str(self) + " for keyword " + p) print("FATAL => ABORT") print("****************") sys.exit(-1) def loadModule(self,read_params,myscript,pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] mymod = None for p in paths: try: modfile = os.path.join(p,myscript+".py") #print("loading file " + modfile) mymod = imp.load_source(myscript,modfile) break except IOError as io_err: logger.debug("loadModule " + str(io_err)) logger.debug("loadModule " + str(mymod)) if (mymod is None): logger.debug("cannot find module '" + myscript + "' from paths " + str(paths)) logger.debug("trace :" + traceback.format_exc()+ '\n') raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) return mymod def createParamsMap(self,pre_args): logger.debug("in") read_params = {} if ('logging' in pre_args) and (pre_args['logging'] is True): bdlogging.activateFileLogging() for opt,args in pre_args.iteritems(): logger.debug("createParamsMap1 " + str(opt) + " : " + str(args)) if (args is None): continue if (not type(args) == list): args = [args] if (type(args) == str) : args = [args] logger.debug("createParamsMap2 " + str(opt) + " : " + str(args)) for arg in args: if (arg is None): continue if (opt == 'BDconf'): if (not type(arg) == list): arg = [arg] self.readConfFiles(read_params,arg) continue if (opt == 'host'): self.updatePossibleHosts(arg) self.readConfFile(read_params,arg+'.bd') for param, typ in self.admissible_params.iteritems(): if opt == param: logger.debug("createParamsMap3 " + str(param) + " : " + str(typ)) if (typ == ModuleType): read_params[param] = self.loadModule(read_params,arg,pre_args) logger.debug("createParamsMap4 " + str(param) + " : " + str(typ)) elif (type(typ) == list): args = arg.split(",") if (not param in read_params): read_params[param] = [] if (not len(typ) == 1) : subtype = str else : subtype = typ[0] for i in range(0,len(args)): read_params[param].append(subtype(args[i])) else: read_params[param] = typ(arg) break return read_params def addModulesAdmissibleParameters(self,read_params): logger.debug("in") for k,v in read_params.iteritems(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update(mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys() if "default_params" in mymod.__dict__: self.default_params.update(mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) if "mandatory" in mymod.__dict__: self.mandatory.update(mymod.__dict__["mandatory"]) logger.debug(str(self.admissible_params)) def addModulesAdmissibleParametersForComplete(self,read_params): logger.debug("in") if not "_ARGCOMPLETE" in os.environ: return logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]) # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" logger.debug("break line " + os.environ["COMP_LINE"]) all_args = re.split(breaks,os.environ["COMP_LINE"]) logger.debug("break line " + str(all_args)) for i in range(0,len(all_args)): a = all_args[i] res = re.match("--(.*)",a) if res is None: continue a = res.group(1) logger.debug("treating a " + str(a) ) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: logger.debug("here treating a " + str(a) ) if i+1 >= len(all_args): continue b = all_args[i+1] logger.debug("treating b " + str(b) ) res = re.match("--(.*)",b) if res is not None: continue if not b.strip() == '': tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] logger.debug("tmp_read_params " + str(tmp_read_params) ) try: tmp_read_params = self.createParamsMap(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("AAAAAAAAA " + str(tmp_read_params)) try: self.addModulesAdmissibleParameters(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e)) logger.debug("CCCCCCCCCC" + str(self.admissible_params)) def constructArgParser(self,add_help=True,add_mandatory=True): logger.debug("in") parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g,param_list in self.group_params.iteritems(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.iteritems(): #print ("param {0}: {1}".format(param,typ) ) p_help = "help TODO" is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory) if (param in self.help) : p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else : grp = self.params_group["General"] if (typ is None): raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'") if (typ is bool): if (param in self.default_params and self.default_params[param] == True): grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_false', required = is_mandatory) else: grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_true', required = is_mandatory) elif (typ is list or typ == [str]): grp.add_argument("--" + param, action = 'append', dest = param, help = p_help, required = is_mandatory).completer = self.completer else: grp.add_argument("--" + param, dest = param, help = p_help, required = is_mandatory).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None): logger.debug("in") if (params is not None): self.admissible_params.update(params) if group not in self.group_params: self.group_params[group] = [] self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) for key in defaults.keys(): self.mandatory[key] = False if (help is not None): self.help.update(help) if (mandatory is not None): self.mandatory.update(mandatory) for param,typ in self.admissible_params.iteritems(): if typ == bool and param not in self.default_params: self.default_params[param] = False def addEnvBDArguments(self,parser): logger.debug("in") parser = self.constructArgParser(add_help=False,add_mandatory=False) pre_args = vars(parser.parse_known_args(args=self.argv)[0]) for name, value in os.environ.iteritems(): m = re.match("BLACKDYNAMITE_(.*)",name) if (m): var = m.group(1).lower() if (var not in pre_args or pre_args[var] is None): if (var in self.admissible_params): self.argv.append("--" + var) self.argv.append(value) def parseBDParameters(self, argv = None): logger.debug("in") if argv == None: self.argv = list(sys.argv[1:]) else: self.argv = list(argv) logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n") logger.debug("env is\n\n") for k,v in os.environ.iteritems(): logger.debug("export " + k + "='" + v + "'\n") logger.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False,add_mandatory=False) self.addEnvBDArguments(parser) logger.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=self.argv)[0] logger.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) logger.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) logger.debug("addModulesAdmissibleParametersForComplete\n") try: self.addModulesAdmissibleParametersForComplete(read_params) except KeyError as e: logger.debug("trace :" + traceback.format_exc()) logger.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=self.argv) read_params = self.createParamsMap(vars(pre_args)) if not "user" in read_params: read_params["user"] = pwd.getpwuid(os.getuid())[0] return read_params def __init__ (self): logger.debug("in") self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.mandatory = {} self.admissible_params["study"] = str self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language" self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" self.admissible_params["password"] = str self.help["password"] = "Provides the password" self.admissible_params["BDconf"] = list self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons" self.admissible_params["truerun"] = bool self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun" self.default_params["truerun"] = False self.admissible_params["job_constraints"] = [str] self.help["job_constraints"] = "This allows to constraint run selections by job properties" self.default_params["job_constraints"] = None self.admissible_params["run_constraints"] = [str] self.help["run_constraints"] = "This allows to constraint run selections by run properties" self.default_params["run_constraints"] = None self.admissible_params["binary_operator"] = str self.default_params["binary_operator"] = 'and' self.help["binary_operator"] = 'Set the default binary operator to make requests to database' self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["yes"] = bool self.default_params["yes"] = False self.help["yes"] = "Answer all questions to yes." self.admissible_params["logging"] = bool self.help["logging"] = "Activate the file logging system" self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "job_constraints", "run_constraints", "list_parameters"] self.readConfFileList = [] ################################################################ def validate_question(question, params, default_validated = True): logger.debug("in") if (default_validated): default_str = "(Y/n)" else: default_str = "(y/N)" if (params["yes"] == False): validated = raw_input("{0}? {1} ".format(question, default_str)) #print (validated) if (validated == "\n" or validated == ""): validated = default_validated elif(validated == "Y" or validated == "y"): validated = True else: validated = False else: - print("{0}? {1} Forced Y".format(question, default_str)) + logger.info("{0}? {1} Forced Y".format(question, default_str)) validated = True return validated ################################################################ def filterParams(sub_list,total_list): logger.debug("in") new_list = {} for p in sub_list: if (p in total_list and total_list[p] is not False): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): logger.debug("in") params = BDParser.parseBDParameters(self) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__ (self): logger.debug("in") BDParser.__init__(self) self.mandatory["machine_name"] = True self.mandatory["nproc"] = True self.mandatory["run_name"] = True self.admissible_params["machine_name"] = str self.help["machine_name"] = "Specify the name of the machine where the job is to be launched" self.admissible_params["nproc"] = int self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched" self.admissible_params["run_name"] = str self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind" self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"] diff --git a/src/sql_query.hh b/src/sql_query.hh index 3f4ef12..6efb0f7 100644 --- a/src/sql_query.hh +++ b/src/sql_query.hh @@ -1,290 +1,288 @@ /* author : Nicolas RICHART author : Till JUNGE */ #ifndef __BLACKDYNAMITE_SQL_QUERY__ #define __BLACKDYNAMITE_SQL_QUERY__ #include #include #include #include #include namespace BlackDynamite { /* -------------------------------------------------------------------------- */ template inline std::string nan_avoider(const num& val) { return pqxx::to_string(val); } /* -------------------------------------------------------------------------- */ template <> inline std::string nan_avoider(const double& val) { if (val != val) { #if PQXX_VERSION_MAJOR < 4 return "('nan')::double precision"; #else return "NaN"; #endif } else { /* Postgres has its own range definitions */ double fval = std::abs(val); if (fval <= 1e-307) { #if PQXX_VERSION_MAJOR < 4 return "(0.0)::double precision"; #else return "0."; #endif } else if (fval >=1e308) { if (val > 0.) { #if PQXX_VERSION_MAJOR < 4 return "('infinity')::double precision"; #else return "Infinity"; #endif } else { #if PQXX_VERSION_MAJOR < 4 return "('-infinity')::double precision"; #else return "-Infinity"; #endif } } std::stringstream val_rep; val_rep.precision(16); val_rep << std::scientific << val; return val_rep.str(); } } /** * Simple query handler as a secured transaction */ template< class transaction = pqxx::transaction > class SQLQuery : public pqxx::transactor { public: SQLQuery(const std::string & sql_query) : sql_query(sql_query) { } protected: SQLQuery() {} public: - virtual void operator() (transaction & trans) throw { + virtual void operator() (transaction & trans) { UInt retry = 10000; bool success = false; std::string message; - std::runtime_error * error; while (retry != 0){ try { this->result = this->execute(trans); retry = 0; success = true; } catch (std::runtime_error & e) { --retry; message = e.what(); - error = &e; } } if (!success){ std::cerr << "Failed to execute query after " << retry << " retries " << sql_query << std::endl << " with message: " << message; - throw *error; + throw std::runtime_error(message); } } protected: virtual pqxx::result execute(pqxx::transaction_base & trans) { return trans.exec(sql_query); } private: std::string sql_query; pqxx::result result; }; /* ------------------------------------------------------------------------ */ /** * Simple prepared query this query rely on sql queries previously prepared, * the SQLQueryPrepareHelper is done to help in this process */ template, typename... Arguments> class SQLPreparedQuery : public SQLQuery { public: SQLPreparedQuery(const std::string & query_name, Arguments... parameters) : query_name(query_name), parameters(std::make_tuple(parameters...)) { } /* ---------------------------------------------------------------------- */ /// execute a prepared function protected: virtual pqxx::result execute(pqxx::transaction_base & trans) { pqxx::prepare::invocation invoc = trans.prepared(query_name); return this->prepared<0, Arguments...>(invoc).exec(); } private: template inline pqxx::prepare::invocation & prepared(pqxx::prepare::invocation & invoc) { return invoc; } template inline pqxx::prepare::invocation & prepared(pqxx::prepare::invocation & invoc) { return this->prepared(this->addParameter(invoc, std::get(this->parameters))); } template inline pqxx::prepare::invocation & addParameter(pqxx::prepare::invocation & invoc, const T & param) { return invoc(param); } template struct NanAvoiderIT { const std::string operator()(typename std::vector::iterator & it) { return nan_avoider(*it); } }; template inline pqxx::prepare::invocation & addParameter(pqxx::prepare::invocation & invoc, const std::vector & param) { std::string join = nan_avoider(param[0]); for(auto it = param.begin() + 1; it != param.end(); ++it) { join += ", "; join += nan_avoider(*it); } return invoc("{" + join + "}"); // return invoc("ARRAY [" + pqxx::separated_list(",", param.begin(), param.end(), NanAvoiderIT*() + "]"); } private: std::string query_name; std::tuple parameters; }; /* ------------------------------------------------------------------------ */ #if PQXX_VERSION_MAJOR < 4 /* ---------------------------------------------------------------------- */ // Convertion functions /// Function to convert a c++ type in a SQL type for the preparation template struct InternalPrepareHelper { }; /// convert std::string to "varchar" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("varchar", pqxx::prepare::treat_string); } }; /// convert double to "real" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("double precision"); } }; /// convert std::vector to "double precision[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("double precision[]"); } }; /// convert std::vector to "interger[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer[]"); } }; /// convert std::vector to "interger[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer[]"); } }; /// convert int to "interger" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer"); } }; /// convert unsigned int to "interger" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer"); } }; /// convert bool to "boolean" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("boolean"); } }; #endif /** * SQLQueryPrepareHelper */ class SQLQueryPrepareHelper { /* ---------------------------------------------------------------------- */ // preparation function public: /// prepare the request in the connection static void prepare(pqxx::connection_base & connection, const std::string & request_name, const std::string & sql_request) { connection.prepare(request_name, sql_request); } /// prepare the request in the connection template static void prepare(pqxx::connection_base & connection, const std::string & request_name, const std::string & sql_request) { #if PQXX_VERSION_MAJOR < 4 pqxx::prepare::declaration decl = connection.prepare(request_name, sql_request); prepare(decl); #else connection.prepare(request_name, sql_request); #endif } static void unprepare(pqxx::connection_base & connection, const std::string & request_name) { connection.unprepare(request_name); } #if PQXX_VERSION_MAJOR < 4 private: template static const pqxx::prepare::declaration & prepare(const pqxx::prepare::declaration & decl) { return decl; } template static const pqxx::prepare::declaration & prepare(const pqxx::prepare::declaration & decl) { return prepare(InternalPrepareHelper::addArgumentType(decl)); } #endif }; } #endif //__BLACKDYNAMITE_SQL_QUERY__