Diffusion blackdynamite (multiuser) Browse

bdparser.py
20d1d78382b5multiuser

bdparser.py
View Options

			#!/usr/bin/env python3

			from __future__ import print_function

			from . import __path__ as BD_path
			from . import base
			from . import run
			from . import bdlogging

			import sys
			import re
			import os
			import stat
			import pwd
			import argcomplete
			import argparse
			from argcomplete.completers import EnvironCompleter
			import traceback
			from types import ModuleType
			import imp
			################################################################
			import logging
			print = bdlogging.invalidPrint
			logger = logging.getLogger(__name__)
			################################################################


			class BDParser(object):

			def listPossibleHosts(self):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir, 'hosts')
			hosts = []
			try:
			hosts += [h.strip() for h in open(bd_hosts)]
			except Exception:
			pass
			return hosts

			def listPossibleModules(self, pre_args):
			logger.debug("in")
			paths = []
			if ("PYTHONPATH" in os.environ):
			paths = os.environ["PYTHONPATH"].split(':')
			if ("module_path" in pre_args):
			paths += pre_args["module_path"].split(':')
			paths += BD_path
			paths += [path + "/coating" for path in BD_path]
			module_list = []
			paths = [p.strip() for p in paths if not p.strip() == '']
			for p in paths:
			files = os.listdir(p)
			files = [f for f in files if os.path.splitext(f)[1] == '.py']
			files = [f for f in files if not f[0] == '_']
			matching_string = ".blackdynamite."
			files = [os.path.splitext(f)[0] for
			f in files if re.match(
			matching_string,
			open(os.path.join(p, f)).read().replace('\n', ' '),
			flags=re.IGNORECASE)]

			module_list += files
			logger.debug("found these files " + str(module_list))

			return module_list

			def updatePossibleHosts(self, new_host):
			logger.debug("in")
			bd_dir = os.path.expanduser("~/.blackdynamite")
			bd_hosts = os.path.join(bd_dir, 'hosts')
			hosts = set(self.listPossibleHosts())
			hosts.add(new_host)
			f = open(bd_hosts, 'w')
			for h in hosts:
			f.write(h+'\n')

			def completer(self, prefix, **kwargs):
			# bdlogging.activateFileLogging()

			try:
			params = vars(kwargs["parsed_args"])
			if params['logging'] is True:
			bdlogging.activateFileLogging()
			logger.debug("in")
			logger.debug("BDparser prefix " + str(prefix) + "\n")
			for k, v in kwargs.items():
			logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
			logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")

			for k in params.keys():
			if params[k] is None:
			del params[k]

			key = vars(kwargs["action"])["dest"]

			logger.debug("key " + str(key) + "\n")

			if (key == "BDconf"):
			return self.listPossibleConf()

			if 'BDconf' in params:
			self.readConfFiles(params, params['BDconf'])

			if 'host' in params:
			self.readConfFile(params, params['host']+'.bd')

			logger.debug("key " + str(key) + "\n")

			for k in params.keys():
			logger.debug("params = " + str(k))

			if (key == "host"):
			return self.listPossibleHosts()
			if (key == "study"):
			params["should_not_check_study"] = True
			mybase = base.Base(**params)
			return mybase.getSchemaList()
			if (key == 'quantity'):
			mybase = base.Base(**params)
			myrun = run.Run(mybase)
			return myrun.listQuantities()

			if (key in self.admissible_params and
			self.admissible_params[key] == ModuleType):
			logger.debug(
			"trying to complete module list for '{}'".format(key))
			return self.listPossibleModules(params)

			except Exception as e:
			logger.debug(traceback.format_exc())
			logger.debug(str(e))

			return []

			def listPossibleConf(self):
			logger.debug("in")
			files = []
			for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
			for filename in os.listdir(dir):
			fileName, fileExtension = os.path.splitext(filename)
			if (fileExtension == ".bd"):
			files.append(filename)
			return files

			return files

			def readConfFiles(self, read_params, fnames):
			logger.debug("in")
			for f in fnames:
			self.readConfFile(read_params, f)

			def readConfFile(self, read_params, fname):
			logger.debug("in")
			logger.debug("readConfFileList {0}".format(self.readConfFileList))

			if fname in self.readConfFileList:
			return
			self.readConfFileList.append(fname)

			if type(fname) == list:
			raise Exception(
			'cannot use list in that function: ' + str(type(fname)))

			pre_args = {}
			for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
			fullpath = os.path.join(dir, fname)
			if (os.path.isfile(fullpath)):
			fname = fullpath
			break

			try:
			with open(fname) as fh:
			logger.debug("loading file '{0}'".format(fname))
			os.chmod(fname, stat.S_IREAD \| stat.S_IWRITE)
			lines = [line.strip() for line in fh]
			regex = "(.)=(.)"
			for line in lines:
			match = re.match(regex, line)
			if (not match):
			print("malformed line:" + line)
			sys.exit(-1)

			param = match.group(1).strip()
			val = match.group(2).strip()
			pre_args[param] = val
			self.argv.append("--"+param)
			self.argv.append(val)
			logger.debug("read parameters: '{0}'".format(self.argv))
			logger.debug("pre args : '{0}'".format(pre_args))
			read_params.update(self.createParamsMap(pre_args))
			except Exception as e:
			logger.debug("cannot open file " + fname + '\n' +
			str(e) + '\n' + str(traceback.format_exc()))
			logger.debug("out")

			def checkParam(self, p, dico):
			logger.debug("in")
			print("****************")
			print("Obsolete: should use the mandatory argument"
			" for the declare_params function")
			print("It was used by object " + str(self) + " for keyword " + p)
			print("FATAL => ABORT")
			print("****************")
			sys.exit(-1)

			def loadModule(self, read_params, myscript, pre_args):
			logger.debug("in")
			paths = []
			if ("PYTHONPATH" in os.environ):
			paths = os.environ["PYTHONPATH"].split(':')
			if ("module_path" in pre_args):
			paths += pre_args["module_path"].split(':')
			paths += BD_path
			paths += [path + "/coating" for path in BD_path]

			mymod = None
			for p in paths:
			try:
			modfile = os.path.join(p, myscript+".py")
			# print("loading file " + modfile)
			mymod = imp.load_source(myscript, modfile)
			break
			except IOError as io_err:

			logger.debug("loadModule " + str(io_err))

			logger.debug("loadModule " + str(mymod))
			if (mymod is None):
			logger.debug("cannot find module '" + myscript +
			"' from paths " + str(paths))
			logger.debug("trace :" + traceback.format_exc() + '\n')
			raise Exception("cannot find module '" +
			myscript + "' from paths " + str(paths))

			return mymod

			def createParamsMap(self, pre_args):
			logger.debug("in")
			read_params = {}

			if ('logging' in pre_args) and (pre_args['logging'] is True):
			bdlogging.activateFileLogging()

			for opt, args in pre_args.items():
			logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
			if (args is None):
			continue
			if (not type(args) == list):
			args = [args]
			if (type(args) == str):
			args = [args]
			logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))

			for arg in args:
			if (arg is None):
			continue
			if (opt == 'BDconf'):
			if (not type(arg) == list):
			arg = [arg]
			self.readConfFiles(read_params, arg)
			continue
			if (opt == 'host'):
			self.updatePossibleHosts(arg)
			self.readConfFile(read_params, arg+'.bd')

			for param, typ in self.admissible_params.items():
			if opt == param:
			logger.debug("createParamsMap3 " +
			str(param) + " : " + str(typ))

			if (typ == ModuleType):
			read_params[param] = self.loadModule(
			read_params, arg, pre_args)
			logger.debug("createParamsMap4 " +
			str(param) + " : " + str(typ))
			elif (type(typ) == list):
			args = arg.split(",")
			if (param not in read_params):
			read_params[param] = []
			if (not len(typ) == 1):
			subtype = str
			else:
			subtype = typ[0]

			for i in range(0, len(args)):
			read_params[param].append(subtype(args[i]))

			else:
			read_params[param] = typ(arg)

			break

			return read_params

			def addModulesAdmissibleParameters(self, read_params):
			logger.debug("in")
			for k, v in read_params.items():
			if self.admissible_params[k] == ModuleType:
			mymod = read_params[k]
			modname = mymod.__name__
			if "admissible_params" in mymod.__dict__:
			self.admissible_params.update(
			mymod.__dict__["admissible_params"])
			self.group_params["'" + modname + "' module options"] = (
			mymod.__dict__["admissible_params"].keys())
			if "default_params" in mymod.__dict__:
			self.default_params.update(
			mymod.__dict__["default_params"])
			if "help" in mymod.__dict__:
			self.help.update(mymod.__dict__["help"])
			if "mandatory" in mymod.__dict__:
			self.mandatory.update(mymod.__dict__["mandatory"])

			logger.debug(str(self.admissible_params))

			def addModulesAdmissibleParametersForComplete(self, read_params):
			logger.debug("in")
			if "_ARGCOMPLETE" not in os.environ:
			return

			logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])

			# breaks = os.environ["COMP_WORDBREAKS"]
			tmp_read_params = {}
			breaks = " \|=\|&\|<\|>\|;"

			logger.debug("break line " + os.environ["COMP_LINE"])

			all_args = re.split(breaks, os.environ["COMP_LINE"])
			logger.debug("break line " + str(all_args))

			for i in range(0, len(all_args)):
			a = all_args[i]
			res = re.match("--(.*)", a)
			if res is None:
			continue

			a = res.group(1)
			logger.debug("treating a " + str(a))
			if a in self.admissible_params:
			if self.admissible_params[a] == ModuleType:
			logger.debug("here treating a " + str(a))
			if i+1 >= len(all_args):
			continue
			b = all_args[i+1]
			logger.debug("treating b " + str(b))
			res = re.match("--(.*)", b)
			if res is not None:
			continue
			if not b.strip() == '':
			tmp_read_params[a] = b

			if ("module_path" in read_params):
			tmp_read_params["module_path"] = read_params["module_path"]

			logger.debug("tmp_read_params " + str(tmp_read_params))
			try:
			tmp_read_params = self.createParamsMap(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
			logger.debug("AAAAAAAAA " + str(tmp_read_params))

			try:
			self.addModulesAdmissibleParameters(tmp_read_params)
			except Exception as e:
			logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))

			logger.debug("CCCCCCCCCC" + str(self.admissible_params))

			def constructArgParser(self, add_help=True, add_mandatory=True):
			logger.debug("in")
			parser = argparse.ArgumentParser(
			description="BlackDynamite option parser",
			formatter_class=argparse.ArgumentDefaultsHelpFormatter,
			add_help=add_help)

			self.params_group = {}
			group = parser.add_argument_group("General")
			self.params_group["General"] = group
			for g, param_list in self.group_params.items():
			group = parser.add_argument_group(g)
			for p in param_list:
			self.params_group[p] = group

			for param, typ in self.admissible_params.items():
			# print("param {0}: {1}".format(param,typ) )

			p_help = "help TODO"
			is_mandatory = (param in self.mandatory.keys() and
			self.mandatory[param] is True and
			add_mandatory)

			if (param in self.help):
			p_help = self.help[param]
			if (param in self.params_group):
			grp = self.params_group[param]
			else:
			grp = self.params_group["General"]

			if (typ is None):
			raise Exception("Deprectated option type for " +
			param + " : should be changed to 'bool'")
			if (typ is bool):
			if (param in self.default_params and
			self.default_params[param] is True):
			grp.add_argument("--" + param,
			help=p_help,
			dest=param,
			action='store_false',
			required=is_mandatory)
			else:
			grp.add_argument("--" + param,
			help=p_help,
			dest=param,
			action='store_true',
			required=is_mandatory)
			elif (typ is list or typ == [str]):
			grp.add_argument(
			"--" + param,
			action='append',
			dest=param,
			help=p_help,
			required=is_mandatory).completer = self.completer
			else:
			grp.add_argument(
			"--" + param,
			dest=param,
			help=p_help,
			required=is_mandatory).completer = self.completer

			parser.set_defaults(**self.default_params)
			return parser

			def register_params(self, group="General", params=None, defaults=None,
			help=None, mandatory=None):
			logger.debug("in")
			if (params is not None):
			self.admissible_params.update(params)
			if group not in self.group_params:
			self.group_params[group] = []
			self.group_params[group] += params.keys()

			if (defaults is not None):
			self.default_params.update(defaults)
			for key in defaults.keys():
			self.mandatory[key] = False
			if (help is not None):
			self.help.update(help)
			if (mandatory is not None):
			self.mandatory.update(mandatory)

			for param, typ in self.admissible_params.items():
			if typ == bool and param not in self.default_params:
			self.default_params[param] = False

			def addEnvBDArguments(self, parser):
			logger.debug("in")
			parser = self.constructArgParser(add_help=False, add_mandatory=False)
			pre_args = vars(parser.parse_known_args(args=self.argv)[0])

			for name, value in os.environ.items():
			m = re.match("BLACKDYNAMITE_(.*)", name)
			if (m):
			var = m.group(1).lower()
			if (var not in pre_args or pre_args[var] is None):
			if (var in self.admissible_params):
			self.argv.append("--" + var)
			self.argv.append(value)

			def parseBDParameters(self, argv=None):
			logger.debug("in")

			if argv is None:
			self.argv = list(sys.argv[1:])
			else:
			self.argv = list(argv)

			logger.debug("program called with " +
			str(len(self.argv)) +
			" args " + str(self.argv) + "\n")
			logger.debug("env is\n\n")
			for k, v in os.environ.items():
			logger.debug("export " + k + "='" + v + "'\n")

			logger.debug("constructArgParser\n")
			parser = self.constructArgParser(add_help=False, add_mandatory=False)
			self.addEnvBDArguments(parser)
			logger.debug("parse_known_args\n")
			pre_args = parser.parse_known_args(args=self.argv)[0]
			logger.debug("createParamsMap\n")
			read_params = self.createParamsMap(vars(pre_args))
			logger.debug("addModuleAdmissibleParameters\n")
			self.addModulesAdmissibleParameters(read_params)
			logger.debug("addModulesAdmissibleParametersForComplete\n")
			try:
			self.addModulesAdmissibleParametersForComplete(read_params)
			except KeyError as e:
			logger.debug("trace :" + traceback.format_exc())

			logger.debug("constructArgParser\n")
			parser = self.constructArgParser()

			argcomplete.autocomplete(parser)
			pre_args = parser.parse_args(args=self.argv)
			read_params = self.createParamsMap(vars(pre_args))

			if "user" not in read_params:
			read_params["user"] = pwd.getpwuid(os.getuid())[0]

			return read_params

			def __init__(self):
			logger.debug("in")
			self.admissible_params = {}
			self.help = {}
			self.default_params = {}
			self.group_params = {}
			self.mandatory = {}

			self.admissible_params["study"] = str
			self.help["study"] = (
			"Specify the study from the BlackDynamite database"
			". This refers to the schemas in PostgreSQL language")
			self.admissible_params["host"] = str
			self.help["host"] = "Specify data base server address"
			self.admissible_params["port"] = int
			self.help["port"] = "Specify data base server port"
			self.admissible_params["user"] = str
			self.help["user"] = "Specify user name to connect to data base server"
			self.admissible_params["password"] = str
			self.help["password"] = "Provides the password"

			self.admissible_params["BDconf"] = list
			self.help["BDconf"] = (
			"Path to a BlackDynamite file (*.bd) configuring current optons")

			self.admissible_params["truerun"] = bool
			self.help["truerun"] = (
			"Set this flag if you want to truly perform the"
			" action on base. If not set all action are mainly dryrun")
			self.default_params["truerun"] = False

			self.admissible_params["constraints"] = [str]
			self.help["constraints"] = (
			"This allows to constraint run/job selections by properties")
			self.default_params["constraints"] = None

			self.admissible_params["binary_operator"] = str
			self.default_params["binary_operator"] = 'and'
			self.help["binary_operator"] = (
			'Set the default binary operator to make requests to database')

			self.admissible_params["list_parameters"] = bool
			self.help["list_parameters"] = (
			"Request to list the possible job/run parameters")

			self.admissible_params["yes"] = bool
			self.default_params["yes"] = False
			self.help["yes"] = "Answer all questions to yes."

			self.admissible_params["logging"] = bool
			self.help["logging"] = "Activate the file logging system"

			self.group_params["BDParser"] = ["study",
			"host",
			"port",
			"user",
			"password",
			"BDconf",
			"truerun",
			"constraints",
			"job_constraints",
			"run_constraints",
			"list_parameters"]

			self.readConfFileList = []

			################################################################


			def validate_question(question, params, default_validated=True):
			logger.debug("in")
			if (default_validated):
			default_str = "(Y/n)"
			else:
			default_str = "(y/N)"

			if params["yes"] is False:
			validated = input("{0}? {1} ".format(question, default_str))
			# print (validated)
			if (validated == "\n" or validated == ""):
			validated = default_validated
			elif(validated == "Y" or validated == "y"):
			validated = True
			else:
			validated = False
			else:
			logger.info("{0}? {1} Forced Y".format(question, default_str))
			validated = True

			return validated

			################################################################


			def filterParams(sub_list, total_list):
			logger.debug("in")
			new_list = {}
			for p in sub_list:
			if (p in total_list and total_list[p] is not False):
			new_list[p] = total_list[p]
			return new_list


			################################################################


			class RunParser(BDParser):
			"""
			"""

			def parseBDParameters(self):
			logger.debug("in")
			params = BDParser.parseBDParameters(self)
			params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
			return params

			def __init__(self):
			logger.debug("in")
			BDParser.__init__(self)

			self.mandatory["machine_name"] = True
			self.mandatory["nproc"] = True
			self.mandatory["run_name"] = True

			self.admissible_params["machine_name"] = str
			self.help["machine_name"] = ("Specify the name of the machine where"
			" the job is to be launched")
			self.admissible_params["nproc"] = int
			self.help["nproc"] = ("Specify the number of processors onto which"
			" this run is supposed to be launched")
			self.admissible_params["run_name"] = str
			self.help["run_name"] = (
			'User friendly name given to this run.'
			' This is usually helpful to recall a run kind')
			self.default_params = {}
			self.default_params["job_constraints"] = None

			self.group_params["RunParser"] = [
			"machine_name",
			"nproc",
			"run_name"]


			################################################################


			__all__ = ["BDParser", "RunParser"]

R3127/BlackDynamite/bdparser.py20d1d78382b5multiuser

bdparser.pyView Options

R3127/BlackDynamite/bdparser.py
20d1d78382b5multiuser

bdparser.py
View Options