Page MenuHomec4science
Files F62420631

bdparser.py
No OneTemporary
Actions

File Metadata

Created
Mon, May 13, 02:08

bdparser.py
View Options

#!/usr/bin/env python3
from __future__ import print_function
from . import __path__ as BD_path
from . import base
from . import run
from . import bdlogging
import sys
import re
import os
import stat
import pwd
import argcomplete
import argparse
from argcomplete.completers import EnvironCompleter
import traceback
from types import ModuleType
import imp
################################################################
import logging
print = bdlogging.invalidPrint
logger = logging.getLogger(__name__)
################################################################
class BDParser(object):
def listPossibleHosts(self):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = []
try:
hosts += [h.strip() for h in open(bd_hosts)]
except Exception:
pass
return hosts
def listPossibleModules(self, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
module_list = []
paths = [p.strip() for p in paths if not p.strip() == '']
for p in paths:
files = os.listdir(p)
files = [f for f in files if os.path.splitext(f)[1] == '.py']
files = [f for f in files if not f[0] == '_']
matching_string = ".*blackdynamite.*"
files = [os.path.splitext(f)[0] for
f in files if re.match(
matching_string,
open(os.path.join(p, f)).read().replace('\n', ' '),
flags=re.IGNORECASE)]
module_list += files
logger.debug("found these files " + str(module_list))
return module_list
def updatePossibleHosts(self, new_host):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = set(self.listPossibleHosts())
hosts.add(new_host)
f = open(bd_hosts, 'w')
for h in hosts:
f.write(h+'\n')
def completer(self, prefix, **kwargs):
# bdlogging.activateFileLogging()
try:
params = vars(kwargs["parsed_args"])
if params['logging'] is True:
bdlogging.activateFileLogging()
logger.debug("in")
logger.debug("BDparser prefix " + str(prefix) + "\n")
for k, v in kwargs.items():
logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")
for k in params.keys():
if params[k] is None:
del params[k]
key = vars(kwargs["action"])["dest"]
logger.debug("key " + str(key) + "\n")
if (key == "BDconf"):
return self.listPossibleConf()
if 'BDconf' in params:
self.readConfFiles(params, params['BDconf'])
if 'host' in params:
self.readConfFile(params, params['host']+'.bd')
logger.debug("key " + str(key) + "\n")
for k in params.keys():
logger.debug("params = " + str(k))
if (key == "host"):
return self.listPossibleHosts()
if (key == "study"):
params["should_not_check_study"] = True
mybase = base.Base(**params)
return mybase.getSchemaList()
if (key == 'quantity'):
mybase = base.Base(**params)
myrun = run.Run(mybase)
return myrun.listQuantities()
if (key in self.admissible_params and
self.admissible_params[key] == ModuleType):
logger.debug(
"trying to complete module list for '{}'".format(key))
return self.listPossibleModules(params)
except Exception as e:
logger.debug(traceback.format_exc())
logger.debug(str(e))
return []
def listPossibleConf(self):
logger.debug("in")
files = []
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
for filename in os.listdir(dir):
fileName, fileExtension = os.path.splitext(filename)
if (fileExtension == ".bd"):
files.append(filename)
return files
return files
def readConfFiles(self, read_params, fnames):
logger.debug("in")
for f in fnames:
self.readConfFile(read_params, f)
def readConfFile(self, read_params, fname):
logger.debug("in")
logger.debug("readConfFileList {0}".format(self.readConfFileList))
if fname in self.readConfFileList:
return
self.readConfFileList.append(fname)
if type(fname) == list:
raise Exception(
'cannot use list in that function: ' + str(type(fname)))
pre_args = {}
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
fullpath = os.path.join(dir, fname)
if (os.path.isfile(fullpath)):
fname = fullpath
break
try:
with open(fname) as fh:
logger.debug("loading file '{0}'".format(fname))
os.chmod(fname, stat.S_IREAD | stat.S_IWRITE)
lines = [line.strip() for line in fh]
regex = "(.*)=(.*)"
for line in lines:
match = re.match(regex, line)
if (not match):
print("malformed line:" + line)
sys.exit(-1)
param = match.group(1).strip()
val = match.group(2).strip()
pre_args[param] = val
self.argv.append("--"+param)
self.argv.append(val)
logger.debug("read parameters: '{0}'".format(self.argv))
logger.debug("pre args : '{0}'".format(pre_args))
read_params.update(self.createParamsMap(pre_args))
except Exception as e:
logger.debug("cannot open file " + fname + '\n' +
str(e) + '\n' + str(traceback.format_exc()))
logger.debug("out")
def checkParam(self, p, dico):
logger.debug("in")
print("****************")
print("Obsolete: should use the mandatory argument"
" for the declare_params function")
print("It was used by object " + str(self) + " for keyword " + p)
print("FATAL => ABORT")
print("****************")
sys.exit(-1)
def loadModule(self, read_params, myscript, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
mymod = None
for p in paths:
try:
modfile = os.path.join(p, myscript+".py")
# print("loading file " + modfile)
mymod = imp.load_source(myscript, modfile)
break
except IOError as io_err:
logger.debug("loadModule " + str(io_err))
logger.debug("loadModule " + str(mymod))
if (mymod is None):
logger.debug("cannot find module '" + myscript +
"' from paths " + str(paths))
logger.debug("trace :" + traceback.format_exc() + '\n')
raise Exception("cannot find module '" +
myscript + "' from paths " + str(paths))
return mymod
def createParamsMap(self, pre_args):
logger.debug("in")
read_params = {}
if ('logging' in pre_args) and (pre_args['logging'] is True):
bdlogging.activateFileLogging()
for opt, args in pre_args.items():
logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
if (args is None):
continue
if (not type(args) == list):
args = [args]
if (type(args) == str):
args = [args]
logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))
for arg in args:
if (arg is None):
continue
if (opt == 'BDconf'):
if (not type(arg) == list):
arg = [arg]
self.readConfFiles(read_params, arg)
continue
if (opt == 'host'):
self.updatePossibleHosts(arg)
self.readConfFile(read_params, arg+'.bd')
for param, typ in self.admissible_params.items():
if opt == param:
logger.debug("createParamsMap3 " +
str(param) + " : " + str(typ))
if (typ == ModuleType):
read_params[param] = self.loadModule(
read_params, arg, pre_args)
logger.debug("createParamsMap4 " +
str(param) + " : " + str(typ))
elif (type(typ) == list):
args = arg.split(",")
if (param not in read_params):
read_params[param] = []
if (not len(typ) == 1):
subtype = str
else:
subtype = typ[0]
for i in range(0, len(args)):
read_params[param].append(subtype(args[i]))
else:
read_params[param] = typ(arg)
break
return read_params
def addModulesAdmissibleParameters(self, read_params):
logger.debug("in")
for k, v in read_params.items():
if self.admissible_params[k] == ModuleType:
mymod = read_params[k]
modname = mymod.__name__
if "admissible_params" in mymod.__dict__:
self.admissible_params.update(
mymod.__dict__["admissible_params"])
self.group_params["'" + modname + "' module options"] = (
mymod.__dict__["admissible_params"].keys())
if "default_params" in mymod.__dict__:
self.default_params.update(
mymod.__dict__["default_params"])
if "help" in mymod.__dict__:
self.help.update(mymod.__dict__["help"])
if "mandatory" in mymod.__dict__:
self.mandatory.update(mymod.__dict__["mandatory"])
logger.debug(str(self.admissible_params))
def addModulesAdmissibleParametersForComplete(self, read_params):
logger.debug("in")
if "_ARGCOMPLETE" not in os.environ:
return
logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])
# breaks = os.environ["COMP_WORDBREAKS"]
tmp_read_params = {}
breaks = " |=|&|<|>|;"
logger.debug("break line " + os.environ["COMP_LINE"])
all_args = re.split(breaks, os.environ["COMP_LINE"])
logger.debug("break line " + str(all_args))
for i in range(0, len(all_args)):
a = all_args[i]
res = re.match("--(.*)", a)
if res is None:
continue
a = res.group(1)
logger.debug("treating a " + str(a))
if a in self.admissible_params:
if self.admissible_params[a] == ModuleType:
logger.debug("here treating a " + str(a))
if i+1 >= len(all_args):
continue
b = all_args[i+1]
logger.debug("treating b " + str(b))
res = re.match("--(.*)", b)
if res is not None:
continue
if not b.strip() == '':
tmp_read_params[a] = b
if ("module_path" in read_params):
tmp_read_params["module_path"] = read_params["module_path"]
logger.debug("tmp_read_params " + str(tmp_read_params))
try:
tmp_read_params = self.createParamsMap(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("AAAAAAAAA " + str(tmp_read_params))
try:
self.addModulesAdmissibleParameters(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("CCCCCCCCCC" + str(self.admissible_params))
def constructArgParser(self, add_help=True, add_mandatory=True):
logger.debug("in")
parser = argparse.ArgumentParser(
description="BlackDynamite option parser",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
add_help=add_help)
self.params_group = {}
group = parser.add_argument_group("General")
self.params_group["General"] = group
for g, param_list in self.group_params.items():
group = parser.add_argument_group(g)
for p in param_list:
self.params_group[p] = group
for param, typ in self.admissible_params.items():
# print("param {0}: {1}".format(param,typ) )
p_help = "help TODO"
is_mandatory = (param in self.mandatory.keys() and
self.mandatory[param] is True and
add_mandatory)
if (param in self.help):
p_help = self.help[param]
if (param in self.params_group):
grp = self.params_group[param]
else:
grp = self.params_group["General"]
if (typ is None):
raise Exception("Deprectated option type for " +
param + " : should be changed to 'bool'")
if (typ is bool):
if (param in self.default_params and
self.default_params[param] is True):
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_false',
required=is_mandatory)
else:
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_true',
required=is_mandatory)
elif (typ is list or typ == [str]):
grp.add_argument(
"--" + param,
action='append',
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
else:
grp.add_argument(
"--" + param,
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
parser.set_defaults(**self.default_params)
return parser
def register_params(self, group="General", params=None, defaults=None,
help=None, mandatory=None):
logger.debug("in")
if (params is not None):
self.admissible_params.update(params)
if group not in self.group_params:
self.group_params[group] = []
self.group_params[group] += params.keys()
if (defaults is not None):
self.default_params.update(defaults)
for key in defaults.keys():
self.mandatory[key] = False
if (help is not None):
self.help.update(help)
if (mandatory is not None):
self.mandatory.update(mandatory)
for param, typ in self.admissible_params.items():
if typ == bool and param not in self.default_params:
self.default_params[param] = False
def addEnvBDArguments(self, parser):
logger.debug("in")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
pre_args = vars(parser.parse_known_args(args=self.argv)[0])
for name, value in os.environ.items():
m = re.match("BLACKDYNAMITE_(.*)", name)
if (m):
var = m.group(1).lower()
if (var not in pre_args or pre_args[var] is None):
if (var in self.admissible_params):
self.argv.append("--" + var)
self.argv.append(value)
def parseBDParameters(self, argv=None):
logger.debug("in")
if argv is None:
self.argv = list(sys.argv[1:])
else:
self.argv = list(argv)
logger.debug("program called with " +
str(len(self.argv)) +
" args " + str(self.argv) + "\n")
logger.debug("env is\n\n")
for k, v in os.environ.items():
logger.debug("export " + k + "='" + v + "'\n")
logger.debug("constructArgParser\n")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
self.addEnvBDArguments(parser)
logger.debug("parse_known_args\n")
pre_args = parser.parse_known_args(args=self.argv)[0]
logger.debug("createParamsMap\n")
read_params = self.createParamsMap(vars(pre_args))
logger.debug("addModuleAdmissibleParameters\n")
self.addModulesAdmissibleParameters(read_params)
logger.debug("addModulesAdmissibleParametersForComplete\n")
try:
self.addModulesAdmissibleParametersForComplete(read_params)
except KeyError as e:
logger.debug("trace :" + traceback.format_exc())
logger.debug("constructArgParser\n")
parser = self.constructArgParser()
argcomplete.autocomplete(parser)
pre_args = parser.parse_args(args=self.argv)
read_params = self.createParamsMap(vars(pre_args))
if "user" not in read_params:
read_params["user"] = pwd.getpwuid(os.getuid())[0]
return read_params
def __init__(self):
logger.debug("in")
self.admissible_params = {}
self.help = {}
self.default_params = {}
self.group_params = {}
self.mandatory = {}
self.admissible_params["study"] = str
self.help["study"] = (
"Specify the study from the BlackDynamite database"
". This refers to the schemas in PostgreSQL language")
self.admissible_params["host"] = str
self.help["host"] = "Specify data base server address"
self.admissible_params["port"] = int
self.help["port"] = "Specify data base server port"
self.admissible_params["user"] = str
self.help["user"] = "Specify user name to connect to data base server"
self.admissible_params["password"] = str
self.help["password"] = "Provides the password"
self.admissible_params["BDconf"] = list
self.help["BDconf"] = (
"Path to a BlackDynamite file (*.bd) configuring current optons")
self.admissible_params["truerun"] = bool
self.help["truerun"] = (
"Set this flag if you want to truly perform the"
" action on base. If not set all action are mainly dryrun")
self.default_params["truerun"] = False
self.admissible_params["constraints"] = [str]
self.help["constraints"] = (
"This allows to constraint run/job selections by properties")
self.default_params["constraints"] = None
self.admissible_params["binary_operator"] = str
self.default_params["binary_operator"] = 'and'
self.help["binary_operator"] = (
'Set the default binary operator to make requests to database')
self.admissible_params["list_parameters"] = bool
self.help["list_parameters"] = (
"Request to list the possible job/run parameters")
self.admissible_params["yes"] = bool
self.default_params["yes"] = False
self.help["yes"] = "Answer all questions to yes."
self.admissible_params["logging"] = bool
self.help["logging"] = "Activate the file logging system"
self.group_params["BDParser"] = ["study",
"host",
"port",
"user",
"password",
"BDconf",
"truerun",
"constraints",
"job_constraints",
"run_constraints",
"list_parameters"]
self.readConfFileList = []
################################################################
def validate_question(question, params, default_validated=True):
logger.debug("in")
if (default_validated):
default_str = "(Y/n)"
else:
default_str = "(y/N)"
if params["yes"] is False:
validated = input("{0}? {1} ".format(question, default_str))
# print (validated)
if (validated == "\n" or validated == ""):
validated = default_validated
elif(validated == "Y" or validated == "y"):
validated = True
else:
validated = False
else:
logger.info("{0}? {1} Forced Y".format(question, default_str))
validated = True
return validated
################################################################
def filterParams(sub_list, total_list):
logger.debug("in")
new_list = {}
for p in sub_list:
if (p in total_list and total_list[p] is not False):
new_list[p] = total_list[p]
return new_list
################################################################
class RunParser(BDParser):
"""
"""
def parseBDParameters(self):
logger.debug("in")
params = BDParser.parseBDParameters(self)
params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
return params
def __init__(self):
logger.debug("in")
BDParser.__init__(self)
self.mandatory["machine_name"] = True
self.mandatory["nproc"] = True
self.mandatory["run_name"] = True
self.admissible_params["machine_name"] = str
self.help["machine_name"] = ("Specify the name of the machine where"
" the job is to be launched")
self.admissible_params["nproc"] = int
self.help["nproc"] = ("Specify the number of processors onto which"
" this run is supposed to be launched")
self.admissible_params["run_name"] = str
self.help["run_name"] = (
'User friendly name given to this run.'
' This is usually helpful to recall a run kind')
self.default_params = {}
self.default_params["job_constraints"] = None
self.group_params["RunParser"] = [
"machine_name",
"nproc",
"run_name"]
################################################################
__all__ = ["BDParser", "RunParser"]

Event Timeline

c4science · Help