diff --git a/python/BlackDynamite/base.py b/python/BlackDynamite/base.py index b955e02..a1c7ab8 100755 --- a/python/BlackDynamite/base.py +++ b/python/BlackDynamite/base.py @@ -1,271 +1,270 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BaseError", "Base" ] import job import os import psycopg2 import re import copy import numpy as np import psycopg2 import bdparser import sys import getpass import datetime import run class BaseError (Exception): pass class Base(object): """ """ def createBase(self,job_desc,run_desc,quantities={},**kwargs): print (quantities) self.createSchema(kwargs) self.createTable(job_desc) self.createTable(run_desc) self.createGenericTables() for qname,type in quantities.iteritems(): self.pushQuantity(qname,type) if (self.truerun): self.commit() def getObject(self,sqlobject): curs = self.connection.cursor() curs.execute("SELECT * FROM {0}.{1} WHERE id = {2}".format(self.schema,sqlobject.table_name,sqlobject.id)) col_info = self.getColumnProperties(sqlobject) line = curs.fetchone() for i in range(0,len(col_info)): col_name = col_info[i][0] sqlobject[col_name] = line[i] def createSchema(self, params = {"yes": False}): # create the schema of the simulation curs = self.connection.cursor() curs.execute("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{0}'".format(self.schema).lower()) if (curs.rowcount): validated = bdparser.validate_question("Are you sure you want to drop the schema named '" + self.schema + "'", params, False) if (validated == True): curs.execute("DROP SCHEMA {0} cascade".format(self.schema)) else: print ("creation canceled: exit program") sys.exit(-1) curs.execute("CREATE SCHEMA {0}".format(self.schema)) def createTypeCodes(self): curs = self.connection.cursor() curs.execute("SELECT typname,oid from pg_type;") self.type_code = {} for i in curs: # print (i[0]) if (i[0] == 'float8'): self.type_code[i[1]] = float if (i[0] == 'text'): self.type_code[i[1]] = str if (i[0] == 'int8'): self.type_code[i[1]] = int if (i[0] == 'int4'): self.type_code[i[1]] = int if (i[0] == 'bool'): self.type_code[i[1]] = bool if (i[0] == 'timestamp'): self.type_code[i[1]] = datetime.datetime def createTable(self, object): request = object.createTableRequest() curs = self.connection.cursor() # print (request) curs.execute(request) def createGenericTables(self,): sql_script_name = os.path.join(os.path.dirname(__file__),"build_tables.sql") curs = self.connection.cursor() # create generic tables query_list = list() with open(sql_script_name,"r") as fh: for line in fh: query_list.append(re.sub("SCHEMAS_IDENTIFIER",self.schema,line)) curs.execute("\n".join(query_list)) def getColumnProperties(self, sqlobject): curs = self.connection.cursor() curs.execute("SELECT * FROM {0}.{1} LIMIT 0".format(self.schema,sqlobject.table_name)) column_names = [ desc[0] for desc in curs.description] column_type = [ desc[1] for desc in curs.description] return zip(column_names,column_type) def setObjectItemTypes(self, sqlobject): col_info = self.getColumnProperties(sqlobject) for i,j in col_info: sqlobject.types[i] = self.type_code[j] # print (str(i) + " " + str(self.type_code[j])) def insert(self, sqlobject): sqlobject.prepare() curs = self.performRequest(*(sqlobject.insert())) sqlobject.id = curs.fetchone()[0] def performRequest(self, request, params=[]): curs = self.connection.cursor() # print (request) # print (params) try: curs.execute(request,params) except psycopg2.ProgrammingError as err: raise psycopg2.ProgrammingError( ("While trying to execute the query '{0}' with parameters " + "'{1}', I caught this: '{2}'").format(request, params, err)) return curs def createParameterSpace(self,myjob,entry_nb=0,tmp_job=None): keys = myjob.entries.keys() nparam = len(keys) # print (nparam) # print (entry_nb) if (entry_nb == nparam): if (not tmp_job): print ("internal error") sys.exit(-1) print (tmp_job.entries) if (len(tmp_job.getMatchedObjectList()) > 0): return print ("insert job " + str(tmp_job.entries)) self.insert(tmp_job) return if (not tmp_job): tmp_job = job.Job(self) key = keys[entry_nb] e = myjob[key] if (type(e) == list): for typ in e: tmp_job[key.lower()] = typ self.createParameterSpace(myjob,entry_nb+1,tmp_job) else: tmp_job[key.lower()] = e self.createParameterSpace(myjob,entry_nb+1,tmp_job) if (self.truerun): self.commit() def pushQuantity(self,name, type_code, description=None): """ implemented type_codes: "int" "float" "int.vector" "float.vector" """ if ((type_code == "int") or (type_code == int)): is_integer = True is_vector = False elif (type_code == "int.vector"): is_integer = True is_vector = True elif ((type_code == "float") or (type_code == float)): is_integer = False is_vector = False elif (type_code == "float.vector"): is_integer = False is_vector = True else: raise Exception("invalid type '{0}' for a quantity".format(type_code)) curs = self.connection.cursor() curs.execute("INSERT INTO {0}.quantities (name,is_integer,is_vector,description) VALUES (%s , %s , %s, %s) RETURNING id".format(self.schema),(name,is_integer,is_vector,description) ) item = curs.fetchone() if (item is None): raise Exception("Counld not create quantity \"" + name + "\"") return item[0] def commit(self): self.connection.commit() def getSchemaList(self): curs = self.connection.cursor() curs.execute("SELECT distinct(table_schema) from information_schema.tables where table_name='runs'") schemas = [ desc[0] for desc in curs] return schemas def checkStudy(self,dico): if not "study" in dico: print (dico) print ("****************") print ("Parameter 'study' must be provided at command line") print ("possibilities are:") schemas = self.getSchemaList() for s in schemas: print ("\t" + s) print ("") print ("FATAL => ABORT") print ("****************") sys.exit(-1) def __init__ (self, truerun=False,**kwargs): psycopg2_params = ["host","user","port","password"] connection_params = bdparser.filterParams(psycopg2_params,kwargs) - if ("password" in connection_params and not - isinstance(connection_params["password"], str)): + if "password" in connection_params and connection_params["password"] == 'ask': connection_params["password"] = getpass.getpass() try: self.connection = psycopg2.connect(**connection_params) except Exception as e: print ("Connection failed: check you connection settings:\n" + str(e)) sys.exit(-1) assert(isinstance(self.connection,psycopg2._psycopg.connection)) self.dbhost = kwargs["host"] if "host" in kwargs.keys() else "localhost" if ("should_not_check_study" not in kwargs): self.checkStudy(kwargs) self.schema = kwargs["study"] self.createTypeCodes() self.truerun = truerun if("list_parameters" in kwargs and kwargs["list_parameters"] == True): myjob = job.Job(self) myjob.prepare() print ("****************************************************************") print ("Job parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myjob.types.iteritems() ] print("\n".join(params)) myrun = run.Run(self) myrun.prepare() print ("****************************************************************") print ("Run parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myrun.types.iteritems() ] print("\n".join(params)) sys.exit(0) ################################################################ if __name__ == "__main__": connection = psycopg2.connect(host = "localhost") job_description = job.Job(dict(hono=int,lulu=float,toto=str)) base = Base("honoluluSchema",connection,job_description) base.create() connection.commit() base.pushJob(dict(hono=12,lulu=24.2,toto="toto")) base.pushQuantity("ekin", "float") connection.commit() diff --git a/python/BlackDynamite/bdparser.py b/python/BlackDynamite/bdparser.py index e40261f..b43e00c 100755 --- a/python/BlackDynamite/bdparser.py +++ b/python/BlackDynamite/bdparser.py @@ -1,487 +1,492 @@ #!/usr/bin/env python from __future__ import print_function __all__ = [ "BDParser", "RunParser" ] import BlackDynamite as BD import sys import re import os import pwd import base import argcomplete, argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp class BDParser(object): """ """ def debug(self,mesg,mode = 'a'): if self.debugfname: with open(self.debugfname, mode) as fh: print(mesg, file=fh) def completer(self,prefix,**kwargs): self.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.iteritems(): self.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") self.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") params = vars(kwargs["parsed_args"]) for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] self.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() + if 'BDconf' in params: + self.readConfFile(params,params['BDconf']) + + for k in params.keys(): + self.debug("params = " + str(k)) + + if (key == "host"): return ["localhost","lsmssrv1.epfl.ch"] if (key == "study"): - - if ("password" in params): - del params["password"] - params["should_not_check_study"] = True - if ("host" not in params): + if ("host" not in params): params["host"] = "lsmssrv1.epfl.ch" try: mybase = base.Base(**params) except Exception as e: self.debug("params " + str(params) + "\n") self.debug("Exception :" + str(e) + "\n") self.debug("trace :" + traceback.format_exc()) return ["ConnectionError"] - + return mybase.getSchemaList() return [] def listPossibleConf(self): files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) -# print (files) + return files + + return files def readConfFile(self,read_params,fname): pre_args = {} for dir in ["./",os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir,fname) # print ("searching BDconf " + fname + " in dir " + dir) if (os.path.isfile(fullpath)): fname = fullpath break # print (fname) with open(fname) as fh: lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex,line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() pre_args[param] = val self.argv.append("--"+param) self.argv.append(val) read_params.update(self.createParamsMap(pre_args)) def checkParam(self,p,dico): print ("****************") print ("Obsolete: should use the mandatory argument for the declare_params function") print ("It was used by object " + str(self) + " for keyword " + p) print ("FATAL => ABORT") print ("****************") sys.exit(-1) def createParamsMap(self,pre_args): read_params = {} # print (pre_args) for opt,args in pre_args.iteritems(): if (args is None): continue if (not type(args) == list): args = [args] for arg in args: if (arg is None): continue if (opt == "BDconf"): self.readConfFile(read_params,arg) continue for param, typ in self.admissible_params.iteritems(): if opt == param: if (typ == ModuleType): myscript = arg paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') # print (paths) if ("module_path" in pre_args): paths += pre_args["module_path"].split(':') paths += BD.__path__ paths += [ path + "/coating" for path in BD.__path__ ] mymod = None for p in paths: try: modfile = os.path.join(p,myscript+".py") print ("loading file " + modfile) mymod = imp.load_source(myscript,modfile) break except IOError as io_err: pass if (mymod is None): raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) read_params[param] = mymod elif (type(typ) == list): args = arg.split(",") if (not param in read_params): read_params[param] = [] if (not len(typ) == 1): subtype = str else: subtype = typ[0] for i in range(0,len(args)): read_params[param].append(subtype(args[i])) else: read_params[param] = typ(arg) break return read_params def addModulesAdmissibleParameters(self,read_params): for k,v in read_params.iteritems(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update(mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys() if "default_params" in mymod.__dict__: self.default_params.update(mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) if "mandatory" in mymod.__dict__: self.mandatory.update(mymod.__dict__["mandatory"]) self.debug("BBBBBB " + str(self.admissible_params)+"\n") def addModulesAdmissibleParametersForComplete(self,read_params): if not "_ARGCOMPLETE" in os.environ: return self.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]+"\n") # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" self.debug("break line " + os.environ["COMP_LINE"]+"\n") all_args = re.split(breaks,os.environ["COMP_LINE"]) self.debug("break line " + str(all_args)+"\n") for i in range(0,len(all_args)): a = all_args[i] res = re.match("--(.*)",a) if res is None: continue a = res.group(1) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: if i+1 >= len(all_args): continue b = all_args[i+1] # print (b) res = re.match("--(.*)",b) if res is not None: continue tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] tmp_read_params = self.createParamsMap(tmp_read_params) self.debug("AAAAAAAAA " + str(tmp_read_params)+"\n") self.addModulesAdmissibleParameters(tmp_read_params) self.debug("CCCCCCCCCC" + str(self.admissible_params)+"\n") def constructArgParser(self,add_help=True,add_mandatory=True): parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g,param_list in self.group_params.iteritems(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.iteritems(): p_help = "help TODO" is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory) # print (param + ": " + str(is_mandatory) ) if (param in self.help): p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else: grp = self.params_group["General"] if (typ is None): raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'") if (typ is bool): if (param in self.default_params and self.default_params[param] == True): grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_false', required = is_mandatory) else: grp.add_argument("--" + param, help = p_help, dest = param, action = 'store_true', required = is_mandatory) elif (typ is list): grp.add_argument("--" + param, action = 'append', dest = param, help = p_help, required = is_mandatory).completer = self.completer else: grp.add_argument("--" + param, dest = param, help = p_help, required = is_mandatory).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None): if (params is not None): self.admissible_params.update(params) if group not in self.group_params: self.group_params[group] = [] self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) for key in defaults.keys(): self.mandatory[key] = False if (help is not None): self.help.update(help) if (mandatory is not None): self.mandatory.update(mandatory) for param,typ in self.admissible_params.iteritems(): if typ == bool and param not in self.default_params: self.default_params[param] = False def addEnvBDArguments(self,parser): parser = self.constructArgParser(add_help=False,add_mandatory=False) pre_args = vars(parser.parse_known_args(args=self.argv)[0]) for name, value in os.environ.iteritems(): m = re.match("BLACKDYNAMITE_(.*)",name) if (m): var = m.group(1).lower() if (var not in pre_args or pre_args[var] is None): if (var in self.admissible_params): self.argv.append("--" + var) self.argv.append(value) def parseBDParameters(self, argv = None): if argv == None: self.argv = list(sys.argv[1:]) else: self.argv = list(argv) self.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n") self.debug("env is\n\n") for k,v in os.environ.iteritems(): self.debug("export " + k + "='" + v + "'\n") self.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False,add_mandatory=False) self.addEnvBDArguments(parser) self.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=self.argv)[0] self.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) self.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) self.debug("addModulesAdmissibleParametersForComplete\n") try: - self.addModulesAdmissibleParametersForComplete(read_params) + self.addModulesAdmissibleParametersForComplete(read_params) except KeyError as e: - self.debug("trace :" + traceback.format_exc()) + self.debug("trace :" + traceback.format_exc()) self.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=self.argv) # print (self.argv) # print (pre_args) read_params = self.createParamsMap(vars(pre_args)) if not "user" in read_params: # read_params["user"] = os.getlogin() read_params["user"] = pwd.getpwuid(os.getuid())[0] return read_params def __init__ (self): - #self.debugfname = "debug-completion" - self.debugfname = None + self.debugfname = "debug-completion" + #self.debugfname = None self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.mandatory = {} self.admissible_params["study"] = str self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language" self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" - self.admissible_params["password"] = bool - self.help["password"] = "Flag to request prompt for typing password" + self.admissible_params["password"] = str + self.help["password"] = "Provides the password" self.admissible_params["BDconf"] = str self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons" self.admissible_params["truerun"] = bool self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun" self.default_params["truerun"] = False self.admissible_params["job_constraints"] = [str] self.help["job_constraints"] = "This allows to constraint run selections by job properties" self.default_params["job_constraints"] = None self.admissible_params["run_constraints"] = [str] self.help["run_constraints"] = "This allows to constraint run selections by run properties" self.default_params["run_constraints"] = None self.admissible_params["binary_operator"] = str self.default_params["binary_operator"] = 'and' self.help["binary_operator"] = 'Set the default binary operator to make requests to database' self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["yes"] = bool self.default_params["yes"] = False self.help["yes"] = "Answer all questions to yes." self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "job_constraints", "run_constraints", "list_parameters"] ################################################################ def validate_question(question, params, default_validated = True): if (default_validated): default_str = "(Y/n)" else: default_str = "(y/N)" if (params["yes"] == False): validated = raw_input("{0}? {1} ".format(question, default_str)) #print (validated) if (validated == "\n" or validated == ""): validated = default_validated elif(validated == "Y" or validated == "y"): validated = True else: validated = False else: print("{0}? {1} Forced Y".format(question, default_str)) validated = True return validated ################################################################ def filterParams(sub_list,total_list): new_list = {} for p in sub_list: if (p in total_list and total_list[p] is not False): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): params = BDParser.parseBDParameters(self) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__ (self): BDParser.__init__(self) self.mandatory["machine_name"] = True self.mandatory["nproc"] = True self.mandatory["run_name"] = True self.admissible_params["machine_name"] = str self.help["machine_name"] = "Specify the name of the machine where the job is to be launched" self.admissible_params["nproc"] = int self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched" self.admissible_params["run_name"] = str self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind" self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"]