Diffusion blackdynamite (multiuser) Browse

bdparser.py
181b6efb8049multiuser

bdparser.py
View Options

			#!/usr/bin/env python3

			from __future__ import print_function

			import BlackDynamite as BD
			import sys
			import re
			import os
			import stat
			import pwd
			import base
			import run
			import argcomplete
			import argparse
			from argcomplete.completers import EnvironCompleter
			import traceback
			from types import ModuleType
			import imp
			################################################################
			import bdlogging
			import logging
			print = bdlogging.invalidPrint
			logger = logging.getLogger(__name__)
			################################################################


			class BDParser(object):

			def listPossibleHosts(self):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir, 'hosts')
			hosts = []
			try:
			hosts += [h.strip() for h in open(bd_hosts)]
			except:
			pass
			return hosts

			def listPossibleModules(self, pre_args):
			logger.debug("in")
			paths = []
			if ("PYTHONPATH" in os.environ):
			paths = os.environ["PYTHONPATH"].split(':')
			if ("module_path" in pre_args):
			paths += pre_args["module_path"].split(':')
			paths += BD.__path__
			paths += [path + "/coating" for path in BD.__path__]
			module_list = []
			paths = [p.strip() for p in paths if not p.strip() == '']
			for p in paths:
			files = os.listdir(p)
			files = [f for f in files if os.path.splitext(f)[1] == '.py']
			files = [f for f in files if not f[0] == '_']
			matching_string = ".blackdynamite."
			files = [os.path.splitext(f)[0] for
			f in files if re.match(
			matching_string,
			open(os.path.join(p, f)).read().replace('\n', ' '),
			flags=re.IGNORECASE)]

			module_list += files
			logger.debug("found these files " + str(module_list))

			return module_list

			def updatePossibleHosts(self, new_host):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir, 'hosts')
			hosts = set(self.listPossibleHosts())
			hosts.add(new_host)
			f = open(bd_hosts, 'w')
			for h in hosts:
			f.write(h+'\n')

			def completer(self, prefix, **kwargs):
			# bdlogging.activateFileLogging()

			try:
			params = vars(kwargs["parsed_args"])
			if params['logging'] is True:
			bdlogging.activateFileLogging()
			logger.debug("in")
			logger.debug("BDparser prefix " + str(prefix) + "\n")
			for k, v in kwargs.iteritems():
			logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
			logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")

			for k in params.keys():
			if params[k] is None:
			del params[k]

			key = vars(kwargs["action"])["dest"]

			logger.debug("key " + str(key) + "\n")

			if (key == "BDconf"):
			return self.listPossibleConf()

			if 'BDconf' in params:
			self.readConfFiles(params, params['BDconf'])

			if 'host' in params:
			self.readConfFile(params, params['host']+'.bd')

			logger.debug("key " + str(key) + "\n")

			for k in params.keys():
			logger.debug("params = " + str(k))

			if (key == "host"):
			return self.listPossibleHosts()
			if (key == "study"):
			params["should_not_check_study"] = True
			mybase = base.Base(**params)
			return mybase.getSchemaList()
			if (key == 'quantity'):
			mybase = base.Base(**params)
			myrun = run.Run(mybase)
			return myrun.listQuantities()

			if (key in self.admissible_params and
			self.admissible_params[key] == ModuleType):
			logger.debug(
			"trying to complete module list for '{}'".format(key))
			return self.listPossibleModules(params)

			except Exception as e:
			logger.debug(traceback.format_exc())
			logger.debug(str(e))

			return []

			def listPossibleConf(self):
			logger.debug("in")
			files = []
			for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
			for filename in os.listdir(dir):
			fileName, fileExtension = os.path.splitext(filename)
			if (fileExtension == ".bd"):
			files.append(filename)
			return files

			return files

			def readConfFiles(self, read_params, fnames):
			logger.debug("in")
			for f in fnames:
			self.readConfFile(read_params, f)

			def readConfFile(self, read_params, fname):
			logger.debug("in")
			logger.debug("readConfFileList {0}".format(self.readConfFileList))

			if fname in self.readConfFileList:
			return
			self.readConfFileList.append(fname)

			if type(fname) == list:
			raise Exception(
			'cannot use list in that function: ' + str(type(fname)))

			pre_args = {}
			for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
			fullpath = os.path.join(dir, fname)
			if (os.path.isfile(fullpath)):
			fname = fullpath
			break

			try:
			with open(fname) as fh:
			logger.debug("loading file '{0}'".format(fname))
			os.chmod(fname, stat.S_IREAD \| stat.S_IWRITE)
			lines = [line.strip() for line in fh]
			regex = "(.)=(.)"
			for line in lines:
			match = re.match(regex, line)
			if (not match):
			print("malformed line:" + line)
			sys.exit(-1)

			param = match.group(1).strip()
			val = match.group(2).strip()
			pre_args[param] = val
			self.argv.append("--"+param)
			self.argv.append(val)
			logger.debug("read parameters: '{0}'".format(self.argv))
			logger.debug("pre args : '{0}'".format(pre_args))
			read_params.update(self.createParamsMap(pre_args))
			except Exception as e:
			logger.debug("cannot open file " + fname + '\n' +
			str(e) + '\n' + str(traceback.format_exc()))
			logger.debug("out")

			def checkParam(self, p, dico):
			logger.debug("in")
			print("****************")
			print("Obsolete: should use the mandatory argument"
			" for the declare_params function")
			print("It was used by object " + str(self) + " for keyword " + p)
			print("FATAL => ABORT")
			print("****************")
			sys.exit(-1)

			def loadModule(self, read_params, myscript, pre_args):
			logger.debug("in")
			paths = []
			if ("PYTHONPATH" in os.environ):
			paths = os.environ["PYTHONPATH"].split(':')
			if ("module_path" in pre_args):
			paths += pre_args["module_path"].split(':')
			paths += BD.__path__
			paths += [path + "/coating" for path in BD.__path__]

			mymod = None
			for p in paths:
			try:
			modfile = os.path.join(p, myscript+".py")
			# print("loading file " + modfile)
			mymod = imp.load_source(myscript, modfile)
			break
			except IOError as io_err:

			logger.debug("loadModule " + str(io_err))

			logger.debug("loadModule " + str(mymod))
			if (mymod is None):
			logger.debug("cannot find module '" + myscript +
			"' from paths " + str(paths))
			logger.debug("trace :" + traceback.format_exc() + '\n')
			raise Exception("cannot find module '" +
			myscript + "' from paths " + str(paths))

			return mymod

			def createParamsMap(self, pre_args):
			logger.debug("in")
			read_params = {}

			if ('logging' in pre_args) and (pre_args['logging'] is True):
			bdlogging.activateFileLogging()

			for opt, args in pre_args.iteritems():
			logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
			if (args is None):
			continue
			if (not type(args) == list):
			args = [args]
			if (type(args) == str):
			args = [args]
			logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))

			for arg in args:
			if (arg is None):
			continue
			if (opt == 'BDconf'):
			if (not type(arg) == list):
			arg = [arg]
			self.readConfFiles(read_params, arg)
			continue
			if (opt == 'host'):
			self.updatePossibleHosts(arg)
			self.readConfFile(read_params, arg+'.bd')

			for param, typ in self.admissible_params.iteritems():
			if opt == param:
			logger.debug("createParamsMap3 " +
			str(param) + " : " + str(typ))

			if (typ == ModuleType):
			read_params[param] = self.loadModule(
			read_params, arg, pre_args)
			logger.debug("createParamsMap4 " +
			str(param) + " : " + str(typ))
			elif (type(typ) == list):
			args = arg.split(",")
			if (param not in read_params):
			read_params[param] = []
			if (not len(typ) == 1):
			subtype = str
			else:
			subtype = typ[0]

			for i in range(0, len(args)):
			read_params[param].append(subtype(args[i]))

			else:
			read_params[param] = typ(arg)

			break

			return read_params

			def addModulesAdmissibleParameters(self, read_params):
			logger.debug("in")
			for k, v in read_params.iteritems():
			if self.admissible_params[k] == ModuleType:
			mymod = read_params[k]
			modname = mymod.__name__
			if "admissible_params" in mymod.__dict__:
			self.admissible_params.update(
			mymod.__dict__["admissible_params"])
			self.group_params["'" + modname + "' module options"] = (
			mymod.__dict__["admissible_params"].keys())
			if "default_params" in mymod.__dict__:
			self.default_params.update(
			mymod.__dict__["default_params"])
			if "help" in mymod.__dict__:
			self.help.update(mymod.__dict__["help"])
			if "mandatory" in mymod.__dict__:
			self.mandatory.update(mymod.__dict__["mandatory"])

			logger.debug(str(self.admissible_params))

			def addModulesAdmissibleParametersForComplete(self, read_params):
			logger.debug("in")
			if "_ARGCOMPLETE" not in os.environ:
			return

			logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])

			# breaks = os.environ["COMP_WORDBREAKS"]
			tmp_read_params = {}
			breaks = " \|=\|&\|<\|>\|;"

			logger.debug("break line " + os.environ["COMP_LINE"])

			all_args = re.split(breaks, os.environ["COMP_LINE"])
			logger.debug("break line " + str(all_args))

			for i in range(0, len(all_args)):
			a = all_args[i]
			res = re.match("--(.*)", a)
			if res is None:
			continue

			a = res.group(1)
			logger.debug("treating a " + str(a))
			if a in self.admissible_params:
			if self.admissible_params[a] == ModuleType:
			logger.debug("here treating a " + str(a))
			if i+1 >= len(all_args):
			continue
			b = all_args[i+1]
			logger.debug("treating b " + str(b))
			res = re.match("--(.*)", b)
			if res is not None:
			continue
			if not b.strip() == '':
			tmp_read_params[a] = b

			if ("module_path" in read_params):
			tmp_read_params["module_path"] = read_params["module_path"]

			logger.debug("tmp_read_params " + str(tmp_read_params))
			try:
			tmp_read_params = self.createParamsMap(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
			logger.debug("AAAAAAAAA " + str(tmp_read_params))

			try:
			self.addModulesAdmissibleParameters(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))

			logger.debug("CCCCCCCCCC" + str(self.admissible_params))

			def constructArgParser(self, add_help=True, add_mandatory=True):
			logger.debug("in")
			parser = argparse.ArgumentParser(
			description="BlackDynamite option parser",
			formatter_class=argparse.ArgumentDefaultsHelpFormatter,
			add_help=add_help)

			self.params_group = {}
			group = parser.add_argument_group("General")
			self.params_group["General"] = group
			for g, param_list in self.group_params.iteritems():
			group = parser.add_argument_group(g)
			for p in param_list:
			self.params_group[p] = group

			for param, typ in self.admissible_params.iteritems():
			# print("param {0}: {1}".format(param,typ) )

			p_help = "help TODO"
			is_mandatory = (param in self.mandatory.keys() and
			self.mandatory[param] is True and
			add_mandatory)

			if (param in self.help):
			p_help = self.help[param]
			if (param in self.params_group):
			grp = self.params_group[param]
			else:
			grp = self.params_group["General"]

			if (typ is None):
			raise Exception("Deprectated option type for " +
			param + " : should be changed to 'bool'")
			if (typ is bool):
			if (param in self.default_params and
			self.default_params[param] is True):
			grp.add_argument("--" + param,
			help=p_help,
			dest=param,
			action='store_false',
			required=is_mandatory)
			else:
			grp.add_argument("--" + param,
			help=p_help,
			dest=param,
			action='store_true',
			required=is_mandatory)
			elif (typ is list or typ == [str]):
			grp.add_argument(
			"--" + param,
			action='append',
			dest=param,
			help=p_help,
			required=is_mandatory).completer = self.completer
			else:
			grp.add_argument(
			"--" + param,
			dest=param,
			help=p_help,
			required=is_mandatory).completer = self.completer

			parser.set_defaults(**self.default_params)
			return parser

			def register_params(self, group="General", params=None, defaults=None,
			help=None, mandatory=None):
			logger.debug("in")
			if (params is not None):
			self.admissible_params.update(params)
			if group not in self.group_params:
			self.group_params[group] = []
			self.group_params[group] += params.keys()

			if (defaults is not None):
			self.default_params.update(defaults)
			for key in defaults.keys():
			self.mandatory[key] = False
			if (help is not None):
			self.help.update(help)
			if (mandatory is not None):
			self.mandatory.update(mandatory)

			for param, typ in self.admissible_params.iteritems():
			if typ == bool and param not in self.default_params:
			self.default_params[param] = False

			def addEnvBDArguments(self, parser):
			logger.debug("in")
			parser = self.constructArgParser(add_help=False, add_mandatory=False)
			pre_args = vars(parser.parse_known_args(args=self.argv)[0])

			for name, value in os.environ.iteritems():
			m = re.match("BLACKDYNAMITE_(.*)", name)
			if (m):
			var = m.group(1).lower()
			if (var not in pre_args or pre_args[var] is None):
			if (var in self.admissible_params):
			self.argv.append("--" + var)
			self.argv.append(value)

			def parseBDParameters(self, argv=None):
			logger.debug("in")

			if argv is None:
			self.argv = list(sys.argv[1:])
			else:
			self.argv = list(argv)

			logger.debug("program called with " +
			str(len(self.argv)) +
			" args " + str(self.argv) + "\n")
			logger.debug("env is\n\n")
			for k, v in os.environ.iteritems():
			logger.debug("export " + k + "='" + v + "'\n")

			logger.debug("constructArgParser\n")
			parser = self.constructArgParser(add_help=False, add_mandatory=False)
			self.addEnvBDArguments(parser)
			logger.debug("parse_known_args\n")
			pre_args = parser.parse_known_args(args=self.argv)[0]
			logger.debug("createParamsMap\n")
			read_params = self.createParamsMap(vars(pre_args))
			logger.debug("addModuleAdmissibleParameters\n")
			self.addModulesAdmissibleParameters(read_params)
			logger.debug("addModulesAdmissibleParametersForComplete\n")
			try:
			self.addModulesAdmissibleParametersForComplete(read_params)
			except KeyError as e:
			logger.debug("trace :" + traceback.format_exc())

			logger.debug("constructArgParser\n")
			parser = self.constructArgParser()

			argcomplete.autocomplete(parser)
			pre_args = parser.parse_args(args=self.argv)
			read_params = self.createParamsMap(vars(pre_args))

			if "user" not in read_params:
			read_params["user"] = pwd.getpwuid(os.getuid())[0]

			return read_params

			def __init__(self):
			logger.debug("in")
			self.admissible_params = {}
			self.help = {}
			self.default_params = {}
			self.group_params = {}
			self.mandatory = {}

			self.admissible_params["study"] = str
			self.help["study"] = (
			"Specify the study from the BlackDynamite database"
			". This refers to the schemas in PostgreSQL language")
			self.admissible_params["host"] = str
			self.help["host"] = "Specify data base server address"
			self.admissible_params["port"] = int
			self.help["port"] = "Specify data base server port"
			self.admissible_params["user"] = str
			self.help["user"] = "Specify user name to connect to data base server"
			self.admissible_params["password"] = str
			self.help["password"] = "Provides the password"

			self.admissible_params["BDconf"] = list
			self.help["BDconf"] = (
			"Path to a BlackDynamite file (*.bd) configuring current optons")

			self.admissible_params["truerun"] = bool
			self.help["truerun"] = (
			"Set this flag if you want to truly perform the"
			" action on base. If not set all action are mainly dryrun")
			self.default_params["truerun"] = False

			self.admissible_params["constraints"] = [str]
			self.help["constraints"] = (
			"This allows to constraint run/job selections by properties")
			self.default_params["constraints"] = None

			self.admissible_params["binary_operator"] = str
			self.default_params["binary_operator"] = 'and'
			self.help["binary_operator"] = (
			'Set the default binary operator to make requests to database')

			self.admissible_params["list_parameters"] = bool
			self.help["list_parameters"] = (
			"Request to list the possible job/run parameters")

			self.admissible_params["yes"] = bool
			self.default_params["yes"] = False
			self.help["yes"] = "Answer all questions to yes."

			self.admissible_params["logging"] = bool
			self.help["logging"] = "Activate the file logging system"

			self.group_params["BDParser"] = ["study",
			"host",
			"port",
			"user",
			"password",
			"BDconf",
			"truerun",
			"constraints",
			"job_constraints",
			"run_constraints",
			"list_parameters"]

			self.readConfFileList = []

			################################################################


			def validate_question(question, params, default_validated=True):
			logger.debug("in")
			if (default_validated):
			default_str = "(Y/n)"
			else:
			default_str = "(y/N)"

			if params["yes"] is False:
			validated = raw_input("{0}? {1} ".format(question, default_str))
			# print (validated)
			if (validated == "\n" or validated == ""):
			validated = default_validated
			elif(validated == "Y" or validated == "y"):
			validated = True
			else:
			validated = False
			else:
			logger.info("{0}? {1} Forced Y".format(question, default_str))
			validated = True

			return validated

			################################################################


			def filterParams(sub_list, total_list):
			logger.debug("in")
			new_list = {}
			for p in sub_list:
			if (p in total_list and total_list[p] is not False):
			new_list[p] = total_list[p]
			return new_list


			################################################################


			class RunParser(BDParser):
			"""
			"""

			def parseBDParameters(self):
			logger.debug("in")
			params = BDParser.parseBDParameters(self)
			params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
			return params

			def __init__(self):
			logger.debug("in")
			BDParser.__init__(self)

			self.mandatory["machine_name"] = True
			self.mandatory["nproc"] = True
			self.mandatory["run_name"] = True

			self.admissible_params["machine_name"] = str
			self.help["machine_name"] = ("Specify the name of the machine where"
			" the job is to be launched")
			self.admissible_params["nproc"] = int
			self.help["nproc"] = ("Specify the number of processors onto which"
			" this run is supposed to be launched")
			self.admissible_params["run_name"] = str
			self.help["run_name"] = (
			'User friendly name given to this run.'
			' This is usually helpful to recall a run kind')
			self.default_params = {}
			self.default_params["job_constraints"] = None

			self.group_params["RunParser"] = [
			"machine_name",
			"nproc",
			"run_name"]


			################################################################


			__all__ = ["BDParser", "RunParser"]

R3127/BlackDynamite/bdparser.py181b6efb8049multiuser

bdparser.pyView Options

R3127/BlackDynamite/bdparser.py
181b6efb8049multiuser

bdparser.py
View Options