Page MenuHomec4science
Files F62813331

bdparser.py
No OneTemporary
Actions

File Metadata

Created
Wed, May 15, 20:24

bdparser.py
View Options

#!/usr/bin/env python
from __future__ import print_function
__all__ = [ "BDParser", "RunParser" ]
import BlackDynamite as BD
import sys
import re
import os
import stat
import pwd
import base
import run
import argcomplete, argparse
from argcomplete.completers import EnvironCompleter
import traceback
from types import ModuleType
import imp
################################################################
import bdlogging,logging
print = bdlogging.invalidPrint
logger = logging.getLogger(__name__)
################################################################
class BDParser(object):
"""
"""
def listPossibleHosts(self):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir,'hosts')
hosts = []
try: hosts += [h.strip() for h in open(bd_hosts)]
except: pass
return hosts
def listPossibleModules(self,pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':')
paths += BD.__path__
paths += [ path + "/coating" for path in BD.__path__ ]
module_list = []
paths = [p.strip() for p in paths if not p.strip() == '']
for p in paths:
files = os.listdir(p)
files = [f for f in files if os.path.splitext(f)[1] == '.py']
files = [f for f in files if not f[0] == '_']
matching_string = ".*blackdynamite.*"
files = [os.path.splitext(f)[0] for f in files if re.match(matching_string,open(os.path.join(p,f)).read().replace('\n',' '),flags=re.IGNORECASE)]
module_list += files
logger.debug("found these files " + str(module_list))
return module_list
def updatePossibleHosts(self,new_host):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir,'hosts')
hosts = set(self.listPossibleHosts())
hosts.add(new_host)
f = open(bd_hosts,'w')
for h in hosts: f.write(h+'\n')
def completer(self,prefix,**kwargs):
#bdlogging.activateFileLogging()
try:
params = vars(kwargs["parsed_args"])
if params['logging'] is True: bdlogging.activateFileLogging()
logger.debug("in")
logger.debug("BDparser prefix " + str(prefix) + "\n")
for k, v in kwargs.iteritems():
logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")
for k in params.keys():
if params[k] is None: del params[k]
key = vars(kwargs["action"])["dest"]
logger.debug("key " + str(key) + "\n")
if (key == "BDconf"): return self.listPossibleConf()
if 'BDconf' in params:
self.readConfFiles(params,params['BDconf'])
if 'host' in params:
self.readConfFile(params,params['host']+'.bd')
logger.debug("key " + str(key) + "\n")
for k in params.keys(): logger.debug("params = " + str(k))
if (key == "host"): return self.listPossibleHosts()
if (key == "study"):
params["should_not_check_study"] = True
mybase = base.Base(**params)
return mybase.getSchemaList()
if (key == 'quantity'):
mybase = base.Base(**params)
myrun = run.Run(mybase)
return myrun.listQuantities()
if (key in self.admissible_params and self.admissible_params[key] == ModuleType):
logger.debug("trying to complete module list for '{}'".format(key) )
return self.listPossibleModules(params)
except Exception as e:
logger.debug(traceback.format_exc())
logger.debug(str(e))
return []
def listPossibleConf(self):
logger.debug("in")
files = []
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
for filename in os.listdir(dir):
fileName, fileExtension = os.path.splitext(filename)
if (fileExtension == ".bd"):
files.append(filename)
return files
return files
def readConfFiles(self,read_params,fnames):
logger.debug("in")
for f in fnames: self.readConfFile(read_params,f)
def readConfFile(self,read_params,fname):
logger.debug("in")
logger.debug("readConfFileList {0}".format(self.readConfFileList))
if fname in self.readConfFileList:
return
self.readConfFileList.append(fname)
if type(fname) == list: raise Exception('cannot use list in that function: ' + str(type(fname)))
pre_args = {}
for dir in ["./",os.path.expanduser("~/.blackdynamite")]:
fullpath = os.path.join(dir,fname)
if (os.path.isfile(fullpath)):
fname = fullpath
break
try:
with open(fname) as fh:
logger.debug("loading file '{0}'".format(fname))
os.chmod(fname,stat.S_IREAD|stat.S_IWRITE)
lines = [line.strip() for line in fh]
regex = "(.*)=(.*)"
for line in lines:
match = re.match(regex,line)
if (not match):
print("malformed line:" + line)
sys.exit(-1)
param = match.group(1).strip()
val = match.group(2).strip()
pre_args[param] = val
self.argv.append("--"+param)
self.argv.append(val)
logger.debug("read parameters: '{0}'".format(self.argv))
logger.debug("pre args : '{0}'".format(pre_args))
read_params.update(self.createParamsMap(pre_args))
except Exception as e:
logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc()))
logger.debug("out")
def checkParam(self,p,dico):
logger.debug("in")
print("****************")
print("Obsolete: should use the mandatory argument for the declare_params function")
print("It was used by object " + str(self) + " for keyword " + p)
print("FATAL => ABORT")
print("****************")
sys.exit(-1)
def loadModule(self,read_params,myscript,pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':')
paths += BD.__path__
paths += [ path + "/coating" for path in BD.__path__ ]
mymod = None
for p in paths:
try:
modfile = os.path.join(p,myscript+".py")
#print("loading file " + modfile)
mymod = imp.load_source(myscript,modfile)
break
except IOError as io_err:
logger.debug("loadModule " + str(io_err))
logger.debug("loadModule " + str(mymod))
if (mymod is None):
logger.debug("cannot find module '" + myscript + "' from paths " + str(paths))
logger.debug("trace :" + traceback.format_exc()+ '\n')
raise Exception("cannot find module '" + myscript + "' from paths " + str(paths))
return mymod
def createParamsMap(self,pre_args):
logger.debug("in")
read_params = {}
if ('logging' in pre_args) and (pre_args['logging'] is True):
bdlogging.activateFileLogging()
for opt,args in pre_args.iteritems():
logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
if (args is None): continue
if (not type(args) == list): args = [args]
if (type(args) == str) : args = [args]
logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))
for arg in args:
if (arg is None): continue
if (opt == 'BDconf'):
if (not type(arg) == list): arg = [arg]
self.readConfFiles(read_params,arg)
continue
if (opt == 'host'):
self.updatePossibleHosts(arg)
self.readConfFile(read_params,arg+'.bd')
for param, typ in self.admissible_params.iteritems():
if opt == param:
logger.debug("createParamsMap3 " + str(param) + " : " + str(typ))
if (typ == ModuleType):
read_params[param] = self.loadModule(read_params,arg,pre_args)
logger.debug("createParamsMap4 " + str(param) + " : " + str(typ))
elif (type(typ) == list):
args = arg.split(",")
if (not param in read_params): read_params[param] = []
if (not len(typ) == 1) : subtype = str
else : subtype = typ[0]
for i in range(0,len(args)):
read_params[param].append(subtype(args[i]))
else: read_params[param] = typ(arg)
break
return read_params
def addModulesAdmissibleParameters(self,read_params):
logger.debug("in")
for k,v in read_params.iteritems():
if self.admissible_params[k] == ModuleType:
mymod = read_params[k]
modname = mymod.__name__
if "admissible_params" in mymod.__dict__:
self.admissible_params.update(mymod.__dict__["admissible_params"])
self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys()
if "default_params" in mymod.__dict__:
self.default_params.update(mymod.__dict__["default_params"])
if "help" in mymod.__dict__:
self.help.update(mymod.__dict__["help"])
if "mandatory" in mymod.__dict__:
self.mandatory.update(mymod.__dict__["mandatory"])
logger.debug(str(self.admissible_params))
def addModulesAdmissibleParametersForComplete(self,read_params):
logger.debug("in")
if not "_ARGCOMPLETE" in os.environ:
return
logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])
# breaks = os.environ["COMP_WORDBREAKS"]
tmp_read_params = {}
breaks = " |=|&|<|>|;"
logger.debug("break line " + os.environ["COMP_LINE"])
all_args = re.split(breaks,os.environ["COMP_LINE"])
logger.debug("break line " + str(all_args))
for i in range(0,len(all_args)):
a = all_args[i]
res = re.match("--(.*)",a)
if res is None: continue
a = res.group(1)
logger.debug("treating a " + str(a) )
if a in self.admissible_params:
if self.admissible_params[a] == ModuleType:
logger.debug("here treating a " + str(a) )
if i+1 >= len(all_args): continue
b = all_args[i+1]
logger.debug("treating b " + str(b) )
res = re.match("--(.*)",b)
if res is not None: continue
if not b.strip() == '':
tmp_read_params[a] = b
if ("module_path" in read_params):
tmp_read_params["module_path"] = read_params["module_path"]
logger.debug("tmp_read_params " + str(tmp_read_params) )
try: tmp_read_params = self.createParamsMap(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e))
logger.debug("AAAAAAAAA " + str(tmp_read_params))
try: self.addModulesAdmissibleParameters(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e))
logger.debug("CCCCCCCCCC" + str(self.admissible_params))
def constructArgParser(self,add_help=True,add_mandatory=True):
logger.debug("in")
parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help)
self.params_group = {}
group = parser.add_argument_group("General")
self.params_group["General"] = group
for g,param_list in self.group_params.iteritems():
group = parser.add_argument_group(g)
for p in param_list:
self.params_group[p] = group
for param, typ in self.admissible_params.iteritems():
#print ("param {0}: {1}".format(param,typ) )
p_help = "help TODO"
is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory)
if (param in self.help) : p_help = self.help[param]
if (param in self.params_group): grp = self.params_group[param]
else : grp = self.params_group["General"]
if (typ is None):
raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'")
if (typ is bool):
if (param in self.default_params and self.default_params[param] == True):
grp.add_argument("--" + param,
help = p_help,
dest = param,
action = 'store_false',
required = is_mandatory)
else:
grp.add_argument("--" + param,
help = p_help,
dest = param,
action = 'store_true',
required = is_mandatory)
elif (typ is list or typ == [str]):
grp.add_argument("--" + param,
action = 'append',
dest = param,
help = p_help,
required = is_mandatory).completer = self.completer
else:
grp.add_argument("--" + param,
dest = param,
help = p_help,
required = is_mandatory).completer = self.completer
parser.set_defaults(**self.default_params)
return parser
def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None):
logger.debug("in")
if (params is not None):
self.admissible_params.update(params)
if group not in self.group_params:
self.group_params[group] = []
self.group_params[group] += params.keys()
if (defaults is not None):
self.default_params.update(defaults)
for key in defaults.keys():
self.mandatory[key] = False
if (help is not None):
self.help.update(help)
if (mandatory is not None):
self.mandatory.update(mandatory)
for param,typ in self.admissible_params.iteritems():
if typ == bool and param not in self.default_params:
self.default_params[param] = False
def addEnvBDArguments(self,parser):
logger.debug("in")
parser = self.constructArgParser(add_help=False,add_mandatory=False)
pre_args = vars(parser.parse_known_args(args=self.argv)[0])
for name, value in os.environ.iteritems():
m = re.match("BLACKDYNAMITE_(.*)",name)
if (m):
var = m.group(1).lower()
if (var not in pre_args or pre_args[var] is None):
if (var in self.admissible_params):
self.argv.append("--" + var)
self.argv.append(value)
def parseBDParameters(self, argv = None):
logger.debug("in")
if argv is None:
self.argv = list(sys.argv[1:])
else:
self.argv = list(argv)
logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n")
logger.debug("env is\n\n")
for k,v in os.environ.iteritems():
logger.debug("export " + k + "='" + v + "'\n")
logger.debug("constructArgParser\n")
parser = self.constructArgParser(add_help=False,add_mandatory=False)
self.addEnvBDArguments(parser)
logger.debug("parse_known_args\n")
pre_args = parser.parse_known_args(args=self.argv)[0]
logger.debug("createParamsMap\n")
read_params = self.createParamsMap(vars(pre_args))
logger.debug("addModuleAdmissibleParameters\n")
self.addModulesAdmissibleParameters(read_params)
logger.debug("addModulesAdmissibleParametersForComplete\n")
try:
self.addModulesAdmissibleParametersForComplete(read_params)
except KeyError as e:
logger.debug("trace :" + traceback.format_exc())
logger.debug("constructArgParser\n")
parser = self.constructArgParser()
argcomplete.autocomplete(parser)
pre_args = parser.parse_args(args=self.argv)
read_params = self.createParamsMap(vars(pre_args))
if not "user" in read_params:
read_params["user"] = pwd.getpwuid(os.getuid())[0]
return read_params
def __init__ (self):
logger.debug("in")
self.admissible_params = {}
self.help = {}
self.default_params = {}
self.group_params = {}
self.mandatory = {}
self.admissible_params["study"] = str
self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language"
self.admissible_params["host"] = str
self.help["host"] = "Specify data base server address"
self.admissible_params["port"] = int
self.help["port"] = "Specify data base server port"
self.admissible_params["user"] = str
self.help["user"] = "Specify user name to connect to data base server"
self.admissible_params["password"] = str
self.help["password"] = "Provides the password"
self.admissible_params["BDconf"] = list
self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons"
self.admissible_params["truerun"] = bool
self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun"
self.default_params["truerun"] = False
self.admissible_params["constraints"] = [str]
self.help["constraints"] = "This allows to constraint run/job selections by properties"
self.default_params["constraints"] = None
self.admissible_params["job_constraints"] = [str]
self.help["job_constraints"] = "This allows to constraint run selections by job properties"
self.default_params["job_constraints"] = None
self.admissible_params["run_constraints"] = [str]
self.help["run_constraints"] = "This allows to constraint run selections by run properties"
self.default_params["run_constraints"] = None
self.admissible_params["binary_operator"] = str
self.default_params["binary_operator"] = 'and'
self.help["binary_operator"] = 'Set the default binary operator to make requests to database'
self.admissible_params["list_parameters"] = bool
self.help["list_parameters"] = "Request to list the possible job/run parameters"
self.admissible_params["yes"] = bool
self.default_params["yes"] = False
self.help["yes"] = "Answer all questions to yes."
self.admissible_params["logging"] = bool
self.help["logging"] = "Activate the file logging system"
self.group_params["BDParser"] = ["study",
"host",
"port",
"user",
"password",
"BDconf",
"truerun",
"constraints",
"job_constraints",
"run_constraints",
"list_parameters"]
self.readConfFileList = []
################################################################
def validate_question(question, params, default_validated = True):
logger.debug("in")
if (default_validated):
default_str = "(Y/n)"
else:
default_str = "(y/N)"
if (params["yes"] == False):
validated = raw_input("{0}? {1} ".format(question, default_str))
#print (validated)
if (validated == "\n" or validated == ""):
validated = default_validated
elif(validated == "Y" or validated == "y"):
validated = True
else:
validated = False
else:
logger.info("{0}? {1} Forced Y".format(question, default_str))
validated = True
return validated
################################################################
def filterParams(sub_list,total_list):
logger.debug("in")
new_list = {}
for p in sub_list:
if (p in total_list and total_list[p] is not False):
new_list[p] = total_list[p]
return new_list
################################################################
class RunParser(BDParser):
"""
"""
def parseBDParameters(self):
logger.debug("in")
params = BDParser.parseBDParameters(self)
params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
return params
def __init__ (self):
logger.debug("in")
BDParser.__init__(self)
self.mandatory["machine_name"] = True
self.mandatory["nproc"] = True
self.mandatory["run_name"] = True
self.admissible_params["machine_name"] = str
self.help["machine_name"] = "Specify the name of the machine where the job is to be launched"
self.admissible_params["nproc"] = int
self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched"
self.admissible_params["run_name"] = str
self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind"
self.default_params = {}
self.default_params["job_constraints"] = None
self.group_params["RunParser"] = [
"machine_name",
"nproc",
"run_name"]

Event Timeline

c4science · Help