bdparser.py
12fd59de4991master

bdparser.py
View Options

			#!/usr/bin/env python

			from __future__ import print_function

			__all__ = [ "BDParser", "RunParser" ]

			import BlackDynamite as BD
			from . import base
			from . import run
			import sys
			import re
			import os
			import stat
			import pwd
			import argcomplete, argparse
			from argcomplete.completers import EnvironCompleter
			import traceback
			from types import ModuleType
			import imp
			from copy import copy
			################################################################
			import logging
			from . import bdlogging
			print = bdlogging.invalidPrint
			logger = logging.getLogger(__name__)
			################################################################

			class BDParser(object):
			"""
			"""

			def listPossibleHosts(self):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir,'hosts')
			hosts = []
			try: hosts += [h.strip() for h in open(bd_hosts)]
			except: pass
			return hosts

			def listPossibleModules(self,pre_args):
			logger.debug("in")
			paths = copy(sys.path)
			if 'module_path' in pre_args:
			paths.extend(pre_args['module_path'].split(':'))
			paths += BD.__path__
			paths += [ path + "/coating" for path in BD.__path__ ]
			module_list = []
			paths = [p.strip() for p in paths if not p.strip() == '']
			for p in paths:
			files = os.listdir(p)
			files = [f for f in files if os.path.splitext(f)[1] == '.py']
			files = [f for f in files if not f[0] == '_']
			matching_string = ".blackdynamite."
			files = [os.path.splitext(f)[0] for f in files if re.match(matching_string,open(os.path.join(p,f)).read().replace('\n',' '),flags=re.IGNORECASE)]
			module_list += files
			logger.debug("found these files " + str(module_list))

			return module_list

			def updatePossibleHosts(self,new_host):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir,'hosts')
			hosts = set(self.listPossibleHosts())
			hosts.add(new_host)
			f = open(bd_hosts,'w')
			for h in hosts: f.write(h+'\n')


			def completer(self,prefix,**kwargs):
			#bdlogging.activateFileLogging()

			try:
			params = vars(kwargs["parsed_args"])
			if params['logging'] is True: bdlogging.activateFileLogging()
			logger.debug("in")
			logger.debug("BDparser prefix " + str(prefix) + "\n")
			for k, v in kwargs.iteritems():
			logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
			logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")


			for k in params.keys():
			if params[k] is None: del params[k]

			key = vars(kwargs["action"])["dest"]

			logger.debug("key " + str(key) + "\n")

			if (key == "BDconf"): return self.listPossibleConf()

			if 'BDconf' in params:
			self.readConfFiles(params,params['BDconf'])

			if 'host' in params:
			self.readConfFile(params,params['host']+'.bd')


			logger.debug("key " + str(key) + "\n")

			for k in params.keys(): logger.debug("params = " + str(k))

			if (key == "host"): return self.listPossibleHosts()
			if (key == "study"):
			params["should_not_check_study"] = True
			mybase = base.Base(**params)
			return mybase.getSchemaList()
			if (key == 'quantity'):
			mybase = base.Base(**params)
			myrun = run.Run(mybase)
			return myrun.listQuantities()

			if (key in self.admissible_params and self.admissible_params[key] == ModuleType):
			logger.debug("trying to complete module list for '{}'".format(key) )
			return self.listPossibleModules(params)

			except Exception as e:
			logger.debug(traceback.format_exc())
			logger.debug(str(e))

			return []

			def listPossibleConf(self):
			logger.debug("in")
			files = []
			for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
			for filename in os.listdir(dir):
			fileName, fileExtension = os.path.splitext(filename)
			if (fileExtension == ".bd"):
			files.append(filename)
			return files


			return files

			def readConfFiles(self,read_params,fnames):
			logger.debug("in")
			for f in fnames: self.readConfFile(read_params,f)

			def readConfFile(self,read_params,fname):
			logger.debug("in")
			logger.debug("readConfFileList {0}".format(self.readConfFileList))

			if fname in self.readConfFileList:
			return
			self.readConfFileList.append(fname)

			if type(fname) == list: raise Exception('cannot use list in that function: ' + str(type(fname)))

			pre_args = {}
			for dir in ["./",os.path.expanduser("~/.blackdynamite")]:
			fullpath = os.path.join(dir,fname)
			if (os.path.isfile(fullpath)):
			fname = fullpath
			break


			try:
			with open(fname) as fh:
			logger.debug("loading file '{0}'".format(fname))
			os.chmod(fname,stat.S_IREAD\|stat.S_IWRITE)
			lines = [line.strip() for line in fh]
			regex = "(.)=(.)"
			for line in lines:
			match = re.match(regex,line)
			if (not match):
			print("malformed line:" + line)
			sys.exit(-1)

			param = match.group(1).strip()
			val = match.group(2).strip()
			pre_args[param] = val
			self.argv.append("--"+param)
			self.argv.append(val)
			logger.debug("read parameters: '{0}'".format(self.argv))
			logger.debug("pre args : '{0}'".format(pre_args))
			read_params.update(self.createParamsMap(pre_args))
			except Exception as e:
			logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc()))
			logger.debug("out")


			def checkParam(self,p,dico):
			logger.debug("in")
			print("****************")
			print("Obsolete: should use the mandatory argument for the declare_params function")
			print("It was used by object " + str(self) + " for keyword " + p)
			print("FATAL => ABORT")
			print("****************")
			sys.exit(-1)

			def loadModule(self,read_params,myscript,pre_args):
			logger.debug("in")
			paths = []
			if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':')
			if ("module_path" in pre_args) : paths += pre_args["module_path"].split(':')
			paths += BD.__path__
			paths += [ path + "/coating" for path in BD.__path__ ]

			mymod = None
			for p in paths:
			try:
			modfile = os.path.join(p,myscript+".py")
			#print("loading file " + modfile)
			mymod = imp.load_source(myscript,modfile)
			break
			except IOError as io_err:

			logger.debug("loadModule " + str(io_err))

			logger.debug("loadModule " + str(mymod))
			if (mymod is None):
			logger.debug("cannot find module '" + myscript + "' from paths " + str(paths))
			logger.debug("trace :" + traceback.format_exc()+ '\n')
			raise Exception("cannot find module '" + myscript + "' from paths " + str(paths))

			return mymod


			def createParamsMap(self,pre_args):
			logger.debug("in")
			read_params = {}

			if ('logging' in pre_args) and (pre_args['logging'] is True):
			bdlogging.activateFileLogging()


			try:
			iterator = pre_args.iteritems()
			except:
			iterator = pre_args.items()

			for opt,args in iterator:
			logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
			if (args is None): continue
			if (not type(args) == list): args = [args]
			if (type(args) == str) : args = [args]
			logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))

			for arg in args:
			if (arg is None): continue
			if (opt == 'BDconf'):
			if (not type(arg) == list): arg = [arg]
			self.readConfFiles(read_params,arg)
			continue
			if (opt == 'host'):
			self.updatePossibleHosts(arg)
			self.readConfFile(read_params,arg+'.bd')


			try:
			iterator = self.admissible_params.iteritems()
			except:
			iterator = self.admissible_params.items()

			for param, typ in iterator:
			if opt == param:
			logger.debug("createParamsMap3 " + str(param) + " : " + str(typ))

			if (typ == ModuleType):
			read_params[param] = self.loadModule(read_params,arg,pre_args)
			logger.debug("createParamsMap4 " + str(param) + " : " + str(typ))
			elif (type(typ) == list):
			args = arg.split(",")
			if (not param in read_params): read_params[param] = []
			if (not len(typ) == 1) : subtype = str
			else : subtype = typ[0]

			for i in range(0,len(args)):
			read_params[param].append(subtype(args[i]))


			else: read_params[param] = typ(arg)

			break

			return read_params

			def addModulesAdmissibleParameters(self,read_params):
			logger.debug("in")

			try:
			iterator = read_params.iteritems()
			except:
			iterator = read_params.items()

			for k, v in iterator:
			if self.admissible_params[k] == ModuleType:
			mymod = read_params[k]
			modname = mymod.__name__
			if "admissible_params" in mymod.__dict__:
			self.admissible_params.update(mymod.__dict__["admissible_params"])
			self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys()
			if "default_params" in mymod.__dict__:
			self.default_params.update(mymod.__dict__["default_params"])
			if "help" in mymod.__dict__:
			self.help.update(mymod.__dict__["help"])
			if "mandatory" in mymod.__dict__:
			self.mandatory.update(mymod.__dict__["mandatory"])

			logger.debug(str(self.admissible_params))

			def addModulesAdmissibleParametersForComplete(self,read_params):
			logger.debug("in")
			if not "_ARGCOMPLETE" in os.environ:
			return

			logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])

			# breaks = os.environ["COMP_WORDBREAKS"]
			tmp_read_params = {}
			breaks = " \|=\|&\|<\|>\|;"

			logger.debug("break line " + os.environ["COMP_LINE"])

			all_args = re.split(breaks,os.environ["COMP_LINE"])
			logger.debug("break line " + str(all_args))

			for i in range(0,len(all_args)):
			a = all_args[i]
			res = re.match("--(.*)",a)
			if res is None: continue

			a = res.group(1)
			logger.debug("treating a " + str(a) )
			if a in self.admissible_params:
			if self.admissible_params[a] == ModuleType:
			logger.debug("here treating a " + str(a) )
			if i+1 >= len(all_args): continue
			b = all_args[i+1]
			logger.debug("treating b " + str(b) )
			res = re.match("--(.*)",b)
			if res is not None: continue
			if not b.strip() == '':
			tmp_read_params[a] = b

			if ("module_path" in read_params):
			tmp_read_params["module_path"] = read_params["module_path"]

			logger.debug("tmp_read_params " + str(tmp_read_params) )
			try: tmp_read_params = self.createParamsMap(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e))
			logger.debug("AAAAAAAAA " + str(tmp_read_params))

			try: self.addModulesAdmissibleParameters(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc()+ '\n' + str(e))

			logger.debug("CCCCCCCCCC" + str(self.admissible_params))

			def constructArgParser(self,add_help=True,add_mandatory=True):
			logger.debug("in")
			parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help)

			self.params_group = {}
			group = parser.add_argument_group("General")
			self.params_group["General"] = group
			try:
			for g, param_list in self.group_params.iteritems():
			group = parser.add_argument_group(g)
			for p in param_list:
			self.params_group[p] = group
			except:
			for g, param_list in self.group_params.items():
			group = parser.add_argument_group(g)
			for p in param_list:
			self.params_group[p] = group

			try:
			iterator = self.admissible_params.iteritems()
			except:
			iterator = self.admissible_params.items()

			for param, typ in iterator:
			#print ("param {0}: {1}".format(param,typ) )

			p_help = "help TODO"
			is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] == True and add_mandatory)

			if (param in self.help) : p_help = self.help[param]
			if (param in self.params_group): grp = self.params_group[param]
			else : grp = self.params_group["General"]

			if (typ is None):
			raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'")
			if (typ is bool):
			if (param in self.default_params and self.default_params[param] == True):
			grp.add_argument("--" + param,
			help = p_help,
			dest = param,
			action = 'store_false',
			required = is_mandatory)
			else:
			grp.add_argument("--" + param,
			help = p_help,
			dest = param,
			action = 'store_true',
			required = is_mandatory)
			elif (typ is list or typ == [str]):
			grp.add_argument("--" + param,
			action = 'append',
			dest = param,
			help = p_help,
			required = is_mandatory).completer = self.completer
			else:
			grp.add_argument("--" + param,
			dest = param,
			help = p_help,
			required = is_mandatory).completer = self.completer

			parser.set_defaults(**self.default_params)
			return parser

			def register_params(self,group="General",params=None,defaults=None,help=None, mandatory=None):
			logger.debug("in")
			if (params is not None):
			self.admissible_params.update(params)
			if group not in self.group_params:
			self.group_params[group] = []
			self.group_params[group] += params.keys()

			if (defaults is not None):
			self.default_params.update(defaults)
			for key in defaults.keys():
			self.mandatory[key] = False
			if (help is not None):
			self.help.update(help)
			if (mandatory is not None):
			self.mandatory.update(mandatory)

			try:
			for param, typ in self.admissible_params.iteritems():
			if typ == bool and param not in self.default_params:
			self.default_params[param] = False
			except:
			for param, typ in self.admissible_params.items():
			if typ == bool and param not in self.default_params:
			self.default_params[param] = False


			def addEnvBDArguments(self, parser):
			logger.debug("in")
			parser = self.constructArgParser(add_help=False,add_mandatory=False)
			pre_args = vars(parser.parse_known_args(args=self.argv)[0])

			try:
			iterator = os.environ.iteritems()
			except:
			iterator = os.environ.items()

			for name, value in iterator:
			m = re.match("BLACKDYNAMITE_(.*)",name)
			if (m):
			var = m.group(1).lower()
			if (var not in pre_args or pre_args[var] is None):
			if (var in self.admissible_params):
			self.argv.append("--" + var)
			self.argv.append(value)

			def parseBDParameters(self, argv = None):
			logger.debug("in")

			if argv is None:
			self.argv = list(sys.argv[1:])
			else:
			self.argv = list(argv)

			logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n")
			logger.debug("env is\n\n")

			try:
			for k, v in os.environ.iteritems():
			logger.debug("export " + k + "='" + v + "'\n")
			except:
			for k, v in os.environ.items():
			logger.debug("export " + k + "='" + v + "'\n")


			logger.debug("constructArgParser\n")
			parser = self.constructArgParser(add_help=False,add_mandatory=False)
			self.addEnvBDArguments(parser)
			logger.debug("parse_known_args\n")
			pre_args = parser.parse_known_args(args=self.argv)[0]
			logger.debug("createParamsMap\n")
			read_params = self.createParamsMap(vars(pre_args))
			logger.debug("addModuleAdmissibleParameters\n")
			self.addModulesAdmissibleParameters(read_params)
			logger.debug("addModulesAdmissibleParametersForComplete\n")
			try:
			self.addModulesAdmissibleParametersForComplete(read_params)
			except KeyError as e:
			logger.debug("trace :" + traceback.format_exc())

			logger.debug("constructArgParser\n")
			parser = self.constructArgParser()

			argcomplete.autocomplete(parser)
			pre_args = parser.parse_args(args=self.argv)
			read_params = self.createParamsMap(vars(pre_args))

			if not "user" in read_params:
			read_params["user"] = pwd.getpwuid(os.getuid())[0]

			return read_params

			def __init__ (self):
			logger.debug("in")
			self.admissible_params = {}
			self.help = {}
			self.default_params = {}
			self.group_params = {}
			self.mandatory = {}

			self.admissible_params["study"] = str
			self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language"
			self.admissible_params["host"] = str
			self.help["host"] = "Specify data base server address"
			self.admissible_params["port"] = int
			self.help["port"] = "Specify data base server port"
			self.admissible_params["user"] = str
			self.help["user"] = "Specify user name to connect to data base server"
			self.admissible_params["password"] = str
			self.help["password"] = "Provides the password"

			self.admissible_params["BDconf"] = list
			self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons"

			self.admissible_params["truerun"] = bool
			self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun"
			self.default_params["truerun"] = False

			self.admissible_params["job_constraints"] = [str]
			self.help["job_constraints"] = "This allows to constraint run selections by job properties"
			self.default_params["job_constraints"] = None

			self.admissible_params["run_constraints"] = [str]
			self.help["run_constraints"] = "This allows to constraint run selections by run properties"
			self.default_params["run_constraints"] = None

			self.admissible_params["binary_operator"] = str
			self.default_params["binary_operator"] = 'and'
			self.help["binary_operator"] = 'Set the default binary operator to make requests to database'

			self.admissible_params["list_parameters"] = bool
			self.help["list_parameters"] = "Request to list the possible job/run parameters"

			self.admissible_params["yes"] = bool
			self.default_params["yes"] = False
			self.help["yes"] = "Answer all questions to yes."

			self.admissible_params["logging"] = bool
			self.help["logging"] = "Activate the file logging system"

			self.group_params["BDParser"] = ["study",
			"host",
			"port",
			"user",
			"password",
			"BDconf",
			"truerun",
			"job_constraints",
			"run_constraints",
			"list_parameters"]

			self.readConfFileList = []

			################################################################

			def validate_question(question, params, default_validated = True):
			logger.debug("in")
			if (default_validated):
			default_str = "(Y/n)"
			else:
			default_str = "(y/N)"

			if (params["yes"] == False):
			validated = input("{0}? {1} ".format(question, default_str))
			#print (validated)
			if (validated == "\n" or validated == ""):
			validated = default_validated
			elif(validated == "Y" or validated == "y"):
			validated = True
			else:
			validated = False
			else:
			logger.info("{0}? {1} Forced Y".format(question, default_str))
			validated = True

			return validated

			################################################################

			def filterParams(sub_list,total_list):
			logger.debug("in")
			new_list = {}
			for p in sub_list:
			if (p in total_list and total_list[p] is not False):
			new_list[p] = total_list[p]
			return new_list


			################################################################


			class RunParser(BDParser):
			"""
			"""


			def parseBDParameters(self):
			logger.debug("in")
			params = BDParser.parseBDParameters(self)
			params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
			return params


			def __init__ (self):
			logger.debug("in")
			BDParser.__init__(self)

			self.mandatory["machine_name"] = True
			self.mandatory["nproc"] = True
			self.mandatory["run_name"] = True


			self.admissible_params["machine_name"] = str
			self.help["machine_name"] = "Specify the name of the machine where the job is to be launched"
			self.admissible_params["nproc"] = int
			self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched"
			self.admissible_params["run_name"] = str
			self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind"
			self.default_params = {}
			self.default_params["job_constraints"] = None

			self.group_params["RunParser"] = [
			"machine_name",
			"nproc",
			"run_name"]

R3127/BlackDynamite/bdparser.py12fd59de4991master

bdparser.pyView Options

R3127/BlackDynamite/bdparser.py
12fd59de4991master

bdparser.py
View Options