diff --git a/python/BlackDynamite/base.py b/python/BlackDynamite/base.py index 6e2474b..1c24aac 100644 --- a/python/BlackDynamite/base.py +++ b/python/BlackDynamite/base.py @@ -1,258 +1,259 @@ #!/usr/bin/env python from __future__ import print_function import os import job import psycopg2 import re import copy import numpy as np import psycopg2 import bdparser import sys import getpass import datetime import run class BaseError (Exception): pass class Base(object): """ """ def createBase(self,job_desc,run_desc,quantities,**kwargs): self.createSchema() self.createTable(job_desc) self.createTable(run_desc) self.createGenericTables() for qname,type in quantities.iteritems(): self.pushQuantity(qname,type) if (self.truerun): self.commit() def getObject(self,sqlobject): curs = self.connection.cursor() curs.execute("SELECT * FROM {0}.{1} WHERE id = {2}".format(self.schema,sqlobject.table_name,sqlobject.id)) col_info = self.getColumnProperties(sqlobject) line = curs.fetchone() for i in range(0,len(col_info)): col_name = col_info[i][0] sqlobject[col_name] = line[i] def createSchema(self,): # create the schema of the simulation curs = self.connection.cursor() curs.execute("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{0}'".format(self.schema).lower()) if (curs.rowcount): curs.execute("DROP SCHEMA {0} cascade".format(self.schema)) curs.execute("CREATE SCHEMA {0}".format(self.schema)) def createTypeCodes(self): curs = self.connection.cursor() curs.execute("SELECT typname,oid from pg_type;") self.type_code = {} for i in curs: # print (i[0]) if (i[0] == 'float8'): self.type_code[i[1]] = float if (i[0] == 'text'): self.type_code[i[1]] = str if (i[0] == 'int8'): self.type_code[i[1]] = int if (i[0] == 'int4'): self.type_code[i[1]] = int if (i[0] == 'bool'): self.type_code[i[1]] = bool if (i[0] == 'timestamp'): self.type_code[i[1]] = datetime.datetime def createTable(self, object): request = object.createTableRequest() curs = self.connection.cursor() # print (request) curs.execute(request) def createGenericTables(self,): sql_script_name = os.path.join(os.path.dirname(__file__),"build_tables.sql") curs = self.connection.cursor() # create generic tables query_list = list() with open(sql_script_name,"r") as fh: for line in fh: query_list.append(re.sub("SCHEMAS_IDENTIFIER",self.schema,line)) curs.execute("\n".join(query_list)) def getColumnProperties(self, sqlobject): curs = self.connection.cursor() curs.execute("SELECT * FROM {0}.{1} LIMIT 0".format(self.schema,sqlobject.table_name)) column_names = [ desc[0] for desc in curs.description] column_type = [ desc[1] for desc in curs.description] return zip(column_names,column_type) def setObjectItemTypes(self, sqlobject): col_info = self.getColumnProperties(sqlobject) for i,j in col_info: sqlobject.types[i] = self.type_code[j] # print (str(i) + " " + str(self.type_code[j])) def insert(self, sqlobject): sqlobject.prepare() curs = self.performRequest(*(sqlobject.insert())) sqlobject.id = curs.fetchone()[0] def performRequest(self, request, params=[]): curs = self.connection.cursor() # print (request) # print (params) curs.execute(request,params) return curs def createParameterSpace(self,myjob,entry_nb=0,tmp_job=None): keys = myjob.entries.keys() nparam = len(keys) # print (nparam) # print (entry_nb) if (entry_nb == nparam): if (not tmp_job): print ("internal error") sys.exit(-1) + print (tmp_job.entries) if (len(tmp_job.getMatchedObjectList()) > 0): return print ("insert job " + str(tmp_job.entries)) self.insert(tmp_job) return if (not tmp_job): tmp_job = job.Job(self) key = keys[entry_nb] e = myjob[key] if (type(e) == list): for typ in e: tmp_job[key.lower()] = typ self.createParameterSpace(myjob,entry_nb+1,tmp_job) else: tmp_job[key.lower()] = e self.createParameterSpace(myjob,entry_nb+1,tmp_job) if (self.truerun): self.commit() def pushQuantity(self,name, type_code, description=None): """ implemented type_codes: "int" "float" "int.vector" "float.vector" """ if (type_code == "int"): is_integer = True is_vector = False if (type_code == "int.vector"): is_integer = True is_vector = True if (type_code == "float"): is_integer = False is_vector = False if (type_code == "float.vector"): is_integer = False is_vector = True curs = self.connection.cursor() curs.execute("INSERT INTO {0}.quantities (name,is_integer,is_vector,description) VALUES (%s , %s , %s, %s) RETURNING id".format(self.schema),(name,is_integer,is_vector,description) ) item = curs.fetchone() if (item is None): raise Exception("Counld not create quantity \"" + name + "\"") return item[0] def commit(self): self.connection.commit() def getSchemaList(self): curs = self.connection.cursor() curs.execute("SELECT distinct(table_schema) from information_schema.tables where table_name='runs'") schemas = [ desc[0] for desc in curs] return schemas def checkStudy(self,dico): if not "study" in dico: print (dico) print ("****************") print ("Parameter 'study' must be provided at command line") print ("possibilities are:") schemas = self.getSchemaList() for s in schemas: print ("\t" + s) print ("") print ("FATAL => ABORT") print ("****************") sys.exit(-1) def __init__ (self, truerun=False,**kwargs): psycopg2_params = ["host","user","port","password"] connection_params = bdparser.filterParams(psycopg2_params,kwargs) if ("password" in connection_params and not isinstance(connection_params["password"], str)): connection_params["password"] = getpass.getpass() try: self.connection = psycopg2.connect(**connection_params) except Exception as e: print ("Connection failed: check you connection settings:\n" + str(e)) sys.exit(-1) assert(isinstance(self.connection,psycopg2._psycopg.connection)) self.dbhost = kwargs["host"] if ("should_not_check_study" not in kwargs): self.checkStudy(kwargs) self.schema = kwargs["study"] self.createTypeCodes() self.truerun = truerun if("list_parameters" in kwargs and kwargs["list_parameters"] == True): myjob = job.Job(self) myjob.prepare() print ("****************************************************************") print ("Job parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myjob.types.iteritems() ] print("\n".join(params)) myrun = run.Run(self) myrun.prepare() print ("****************************************************************") print ("Run parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myrun.types.iteritems() ] print("\n".join(params)) sys.exit(0) ################################################################ if __name__ == "__main__": connection = psycopg2.connect(host = "localhost") job_description = job.Job(dict(hono=int,lulu=float,toto=str)) base = Base("honoluluSchema",connection,job_description) base.create() connection.commit() base.pushJob(dict(hono=12,lulu=24.2,toto="toto")) base.pushQuantity("ekin", "float") connection.commit() diff --git a/python/BlackDynamite/bdparser.py b/python/BlackDynamite/bdparser.py index 33528f5..0922362 100644 --- a/python/BlackDynamite/bdparser.py +++ b/python/BlackDynamite/bdparser.py @@ -1,417 +1,415 @@ #!/usr/bin/env python from __future__ import print_function import sys import re import os import base import argcomplete , argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp class BDParser(object): """ """ def debug(self,mesg,mode = 'a'): if self.debugfname: fh = open(self.debugfname, mode) fh.write(mesg) fh.flush() fh.close() def completer(self,prefix,**kwargs): self.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.iteritems(): self.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") self.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") params = vars(kwargs["parsed_args"]) for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] self.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() if (key == "host"): return ["localhost","lsmssrv1.epfl.ch"] if (key == "study"): if ("password" in params): del params["password"] params["should_not_check_study"] = True if ("host" not in params): params["host"] = "lsmssrv1.epfl.ch" try: mybase = base.Base(**params) except Exception as e: self.debug("params " + str(params) + "\n") self.debug("Exception :" + str(e) + "\n") self.debug("trace :" + traceback.format_exc()) return ["ConnectionError"] return mybase.getSchemaList() return [] def listPossibleConf(self): files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) # print (files) return files def readConfFile(self,read_params,fname): for dir in ["./",os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir,fname) # print ("searching BDconf " + fname + " in dir " + dir) if (os.path.isfile(fullpath)): fname = fullpath break # print (fname) with open(fname) as fh: lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex,line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() if (param in self.admissible_params): read_params[param] = val def checkParam(self,p,dico): if not p in dico: print ("****************") print ("Parameter '" + p + "' must be provided at command line") print ("FATAL => ABORT") print ("****************") sys.exit(-1) def createParamsMap(self,pre_args): read_params = {} # print (pre_args) for opt,args in pre_args.iteritems(): if (args is None): continue if (not type(args) == list): args = [args] for arg in args: if (arg is None): continue if (opt == "BDconf"): self.readConfFile(read_params,arg) continue for param, typ in self.admissible_params.iteritems(): # print ("AAAAAAA " + str(typ) + str(opt) + " " + str(param)) if opt == param: if (typ == bool): # print ("AAAAAAAAAAAAAAA " + str(arg)) if (arg == "true"): read_params[param] = True else: read_params[param] = False elif (typ == ModuleType): myscript = arg paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') # print (paths) if ("module_path" in pre_args): paths += pre_args["module_path"].split(':') mymod = None for p in paths: try: modfile = os.path.join(p,myscript+".py") # print ("loading file " + modfile) mymod = imp.load_source(myscript,modfile) except Exception as e: pass if (mymod is None): raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) read_params[param] = mymod elif (type(typ) == list): args = arg.split(",") if (not param in read_params): read_params[param] = [] if (not len(typ) == 1): subtype = str else: subtype = typ[0] for i in range(0,len(args)): read_params[param].append(subtype(args[i])) else: # print (param) # print (arg) # print (typ) if (typ is not None): read_params[param] = typ(arg) elif (arg is True): read_params[param] = arg elif (arg is False and opt in self.default_params): read_params[param] = arg break return read_params def addModulesAdmissibleParameters(self,read_params): for k,v in read_params.iteritems(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update(mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options" ] = mymod.__dict__["admissible_params"].keys() if "default_params" in mymod.__dict__: self.default_params.update(mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) self.debug("BBBBBB " + str(self.admissible_params)+"\n") def addModulesAdmissibleParametersForComplete(self,read_params): self.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]+"\n") if not "_ARGCOMPLETE" in os.environ: return # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" self.debug("break line " + os.environ["COMP_LINE"]+"\n") all_args = re.split(breaks,os.environ["COMP_LINE"]) self.debug("break line " + str(all_args)+"\n") for i in range(0,len(all_args)): a = all_args[i] res = re.match("--(.*)",a) if res is None: continue a = res.group(1) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: if i+1 >= len(all_args): continue b = all_args[i+1] # print (b) res = re.match("--(.*)",b) if res is not None: continue tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] tmp_read_params = self.createParamsMap(tmp_read_params) self.debug("AAAAAAAAA " + str(tmp_read_params)+"\n") self.addModulesAdmissibleParameters(tmp_read_params) self.debug("CCCCCCCCCC" + str(self.admissible_params)+"\n") def constructArgParser(self,add_help=True): parser = argparse.ArgumentParser(description = "BlackDynamite option parser",formatter_class=argparse.ArgumentDefaultsHelpFormatter,add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g,param_list in self.group_params.iteritems(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.iteritems(): p_help = "help TODO" if (param in self.help): p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else: grp = self.params_group["General"] if (typ is None): grp.add_argument("--" + param,help=p_help,action='store_true') elif (typ is list): grp.add_argument("--" + param,action='append',help=p_help).completer = self.completer else: grp.add_argument("--" + param,help=p_help).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self,group="General",params=None,defaults=None,help=None): - if (params is None): - return - - self.admissible_params.update(params) - if group not in self.group_params: - self.group_params[group] = [] - self.group_params[group] += params.keys() + if (params is not None): + self.admissible_params.update(params) + if group not in self.group_params: + self.group_params[group] = [] + self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) if (help is not None): self.help.update(help) def parseBDParameters(self, argv = None): self.debug("program called with " + str(len(sys.argv)) + " args " + str(sys.argv) + "\n") self.debug("env is\n\n") for k,v in os.environ.iteritems(): self.debug("export " + k + "='" + v + "'\n") self.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False) self.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=argv)[0] self.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) self.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) self.debug("addModulesAdmissibleParametersForComplete\n") try: self.addModulesAdmissibleParametersForComplete(read_params) except Exception as e: self.debug("trace :" + traceback.format_exc()) self.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=argv) read_params = self.createParamsMap(vars(pre_args)) return read_params def __init__ (self): #self.debugfname = "debug-completion" self.debugfname = None self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.admissible_params["study"] = str self.help["study"] = "Specify the study from the BlackDynamite database. This refers to the schemas in PostgreSQL language" self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" self.admissible_params["password"] = None self.help["password"] = "Flag to request prompt for typing password" self.admissible_params["BDconf"] = str self.help["BDconf"] = "Path to a BlackDynamite file (*.bd) configuring current optons" self.admissible_params["truerun"] = None self.help["truerun"] = "Set this flag if you want to truly perform the action on base. If not set all action are mainly dryrun" self.default_params["truerun"] = False self.admissible_params["job_constraints"] = [str] self.help["job_constraints"] = "This allows to constraint run selections by job properties" self.default_params["job_constraints"] = None self.admissible_params["run_constraints"] = [str] self.help["run_constraints"] = "This allows to constraint run selections by run properties" self.default_params["run_constraints"] = None self.admissible_params["list_parameters"] = None self.help["list_parameters"] = "Request to list the possible job/run parameters" self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "job_constraints", "run_constraints", "list_parameters"] ################################################################ def filterParams(sub_list,total_list): new_list = {} for p in sub_list: if (p in total_list): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): params = BDParser.parseBDParameters(self) self.checkParam('machine_name',params) self.checkParam('nproc',params) self.checkParam('run_name',params) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__ (self): BDParser.__init__(self) self.admissible_params["machine_name"] = str self.help["machine_name"] = "Specify the name of the machine where the job is to be launched" self.admissible_params["nproc"] = int self.help["nproc"] = "Specify the number of processors onto which this run is supposed to be launched" self.admissible_params["run_name"] = str self.help["run_name"] = "User friendly name given to this run. This is usually helpful to recall a run kind" self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"] diff --git a/python/BlackDynamite/getRunInfo.py b/python/BlackDynamite/getRunInfo.py index 4f25799..26dc31d 100755 --- a/python/BlackDynamite/getRunInfo.py +++ b/python/BlackDynamite/getRunInfo.py @@ -1,184 +1,190 @@ #!/usr/bin/env python import base import job import run import os , sys, stat import runconfig import subprocess import conffile import getopt import socket import bdparser import runselector import run import datetime ################################################################ def printSummary(mybase,params): runSelector = runselector.RunSelector(mybase) run_list = runSelector.selectRuns(params,params,quiet=True) print ("************************ run summary => study {0} *********************************".format(mybase.schema)) if (len(run_list) == 0): print ("no runs found") sys.exit(0) request = "SELECT run_name,state,count(state) from {0}.runs ".format(mybase.schema) if (len(run_list) > 0): request += "where id in (" + ",".join([str(r.id) for r,j in run_list ]) + ")" request += " group by state,run_name order by run_name,state" # print (request) curs = mybase.performRequest(request,[]) stats = {} for i in curs: if i[0] not in stats: stats[i[0]] = [] stats[i[0]].append([i[1] , int(i[2]) ]) for run_name,st in stats.iteritems(): tot = 0 for n,count in st: tot += count for n,count in st: print ("{:20} {:>20} => {:5} ({:>5.1f}%)".format(run_name,n,count,100.*count/tot)) print ("") sys.exit(0) ################################################################ def getRunInfo(runid,mybase): myrun = run.Run(mybase) myrun["id"] = runid myrun.id = runid run_list = myrun.getMatchedObjectList() if (len(run_list) == 0): print ("no run found with id " + runid) sys.exit(1) myrun = run_list[0] myjob = job.Job(mybase) myjob.id = myrun["job_id"] myjob["id"] = myrun["job_id"] job_list = myjob.getMatchedObjectList() if (len(job_list) == 0): print ("no job found with id " + myjob.id) sys.exit(1) myjob = job_list[0] list_entries = myjob.entries.keys() print ("************************ job info *********************************") for entry in list_entries: if (myjob[entry]): print (entry + ": " + str(myjob[entry])) print ("************************ run info *********************************") list_entries = myrun.entries.keys() regular_run_entries = ("run_name", \ "job_id" , \ "state", \ "start_time", \ "machine_name" , \ "exec", \ "nproc", \ "wait_id" \ ) for entry in regular_run_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) list_entries.remove(entry) for entry in list_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) conffiles = myrun.getConfigFiles() for conf in conffiles: print ("****************************************************************") print ("file #" + str(conf.id) + ": " + conf["filename"]) print ("****************************************************************") print (conf["file"]) ################################################################ parser = bdparser.BDParser() parser.register_params( group="getRunInfo", params={"runid":int, "order":str, "summary":None}, defaults={"order":"id"}) params = parser.parseBDParameters() mybase = base.Base(**params) if ("summary" in params): printSummary(mybase,params) if ("runid" in params): getRunInfo(params["runid"],mybase) else: infos = [] infos.append("run_name") infos.append("id") infos.append("job_id") infos.append("state") infos.append("nproc") infos.append("machine_name") infos.append("start_time") format_string = " {:<20} | {:^6} | {:^6} | {:<15} | {:^5} | {:<20} | {:14}" line = format_string.format(*infos) line += " | {:9} |".format("last step") line += " {:14} |".format("last update") - line += " {:9} |".format("Time/step") + line += " {:10} |".format("Time/step") line += " {:14} |".format("Total Time") separator = "-" * len(line) print (separator) print (line) print (separator) run_constraints = [] job_constraints = [] if ("run_constraints" in params): run_constraints = params["run_constraints"] if ("job_constraints" in params): job_constraints = params["job_constraints"] runSelector = runselector.RunSelector(mybase) run_list = runSelector.selectRuns(run_constraints,job_constraints,sort_by="runs." + params["order"],quiet=True) for r,j in run_list: run_infos = [r[col] for col in infos] step,steptime = r.getLastStep() run_infos.append(step) run_infos.append(steptime) if (step): run_infos.append((steptime-run_infos[6])/step) run_infos.append((steptime-run_infos[6])) else: run_infos.append("0") run_infos.append("0") for i in range(0,len(run_infos)): if type(run_infos[i]) == datetime.datetime: run_infos[i] = str(run_infos[i].strftime("%H:%M %d/%m/%y")) if type(run_infos[i]) == datetime.timedelta: - ms = run_infos[i].microseconds - run_infos[i] -= datetime.timedelta(microseconds=ms) -# print (str(run_infos[i])) + if (run_infos[i] < datetime.timedelta(seconds=1)): + if (run_infos[i] < datetime.timedelta(microseconds=1000)): + run_infos[i] = str(run_infos[i].microseconds) + u'\u00B5'.encode('UTF-8') + "s" + else: + run_infos[i] = str(1./1000.*run_infos[i].microseconds) + 'ms' + else: + ms = run_infos[i].microseconds + run_infos[i] -= datetime.timedelta(microseconds=ms) + ## # print (run_infos) # print ([type(i) for i in run_infos]) - line = (format_string + " | {:9} |" + " {:14} |" + " {:9} |" + " {:14} |").format(*run_infos) + line = (format_string + " | {:9} |" + " {:14} |" + " {:10} |" + " {:14} |").format(*run_infos) print (line) diff --git a/python/BlackDynamite/graphhelper.py b/python/BlackDynamite/graphhelper.py index 38ab5a9..bfa2eca 100644 --- a/python/BlackDynamite/graphhelper.py +++ b/python/BlackDynamite/graphhelper.py @@ -1,360 +1,360 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- import base import run import job import runselector import jobselector import matplotlib.pyplot as plt import re import bdparser import sys import run import numpy as np class GraphHelper(object): """ """ def getMeasures(self,run_list): myresults = [] add_req = [] if (self.frequency): add_req += ["step %% {0} = 0".format(self.frequency)] if (self.start): add_req += ["step > {0}".format(self.start) ] if (self.end): add_req += ["step < {0}".format(self.end) ] for r,j in run_list: print ("retreive data from run " + r["run_name"]) res = r.getScalarQuantities(self.quantities,add_req) for key , value in res: if (value == None): del res[key] myresults.append([r,j,res]) return myresults def selectGraphs(self): run_list = self.runSelector.selectRuns(self.run_constraints,self.job_constraints, self.sort_by) results = self.getMeasures(run_list) return results def makeGraphs(self,fig=None,**kwargs): results = self.selectGraphs() if fig is None: fig = plt.figure(figsize=self.figsize) for r,j,data in results: self.makeCurve(data,fig=fig,myrun=r,myjob=j,**kwargs) return fig def replaceRunAndJobsParameters(self,name,myrun,myjob): res = name # print (res) codes = [["%r."+key,myrun[key]] for key in myrun.entries.keys()] codes += [["%j."+key,myjob[key]] for key in myjob.entries.keys()] for code,val in codes: res = res.replace(code,str(val)) return res def generateLabels(self,results,myrun,myjob): labels = [] names = [r[0] for r in results] for i in range(0,len(results)): name = results[i][0] if (not self.legend or i >= len(self.legend) or not self.legend[i]): labels.append(self.replaceRunAndJobsParameters(name,myrun,myjob)) continue # print (self.legend[i]) head_legend = self.legend[i].replace("{","{{") head_legend = head_legend.replace("}","}}") head_legend = re.sub(r"(%)([0-9]+)",r'{\2}',head_legend).format(*names) # print (head_legend) head_legend = self.replaceRunAndJobsParameters(head_legend,myrun,myjob) # print (head_legend) # if (not head_legend.find("%") == -1): # print("unknown variable name. Possible variables are:") # print "\n".join([c[0] for c in codes]) # sys.exit(-1) # print (head_legend) labels.append(head_legend) return labels def makeComposedQuantity(self,results,myrun,myjob): vecs = [r[1] for r in results] names = [r[0] for r in results] # print (vecs[0].shape) new_results = [] for comp in self.using: exprs = comp.split(":") tmp_res = [] for i in [0,1]: e = re.sub(r"(%)([0-9]+)\.(x)",r"vecs[\2][:,0]",exprs[i]) e = re.sub(r"(%)([0-9]+)\.(y)",r"vecs[\2][:,1]",e) e = self.replaceRunAndJobsParameters(e,myrun,myjob) try: tmp_res.append(eval(e)) except: print (names) print ("invalid expression: '" + exprs[i] + "'") print ("invalid expression: '" + e + "'") sys.exit(-1) name = re.sub(r"(%)([0-9]+)\.([x|y])",r'(" + str(names[\2]) + ")',exprs[1]) res = np.zeros((tmp_res[0].shape[0],2)) res[:,0] = tmp_res[0] res[:,1] = tmp_res[1] # print (res.shape) expr = re.sub(r"(%)([0-9]+)",r"vecs[\2]",comp) # res[0] = eval(expr) # print (name) name = "\"" + name + "\"" # print (name) name = eval(name) # print (name) new_results.append([name,res]) return new_results def makeCurve(self,results,myrun=None,myjob=None,fig=None,**kwargs): if not results: return if fig is None: fig = plt.figure() axe=fig.add_subplot(1,1,1) if self.xrange: axe.set_xlim(self.xrange) if self.yrange: axe.set_ylim(self.yrange) if (self.xlabel): axe.set_xlabel(self.xlabel) if (self.ylabel): axe.set_ylabel(self.ylabel) if (self.title): t = self.replaceRunAndJobsParameters(self.title,myrun,myjob) axe.set_title(t) axe.grid(True,linewidth=0.1) if (self.using): results = self.makeComposedQuantity(results,myrun,myjob) labels = self.generateLabels(results,myrun,myjob) # print (labels) for i in range(0,len(results)): name = results[i][0] vec = results[i][1] label = labels[i] # print (self.quantities) # print (name) style=dict() if (self.marker is not None): style["marker"] = self.marker if self.blackwhite: width_index = self.cycle_index/len(self.linestyle_cycle) style_index = self.cycle_index%len(self.linestyle_cycle) self.cycle_index += 1 style["linewidth"] = self.linewidth_cycle[width_index] style["linestyle"] = self.linestyle_cycle[style_index] style["color"] = 'k' axe.plot(vec[:,0]/self.xscale, vec[:,1]/self.yscale, label=label, **style) axe.legend(loc='best') if (self.fileout): fig.savefig(self.fileout) return fig def setJobConstraint(self,**params): self.job_constraints = [] if ("job_constraints" in params): self.job_constraints = params["job_constraints"] def setRunConstraint(self,**params): self.run_constraints = [] if ("run_constraints" in params): self.run_constraints = params["run_constraints"] def setQuantity(self,**params): if ("quantity" in params): self.quantities = params["quantity"] else: print ("quantity should be provided using option --quantity") self.quantities = "__BLACKDYNAMITE_ERROR__" def __init__ (self,base,**params): self.setJobConstraint(**params) self.setRunConstraint(**params) self.setQuantity(**params) self.base = base self.runSelector = runselector.RunSelector(self.base) self.fig = None self.xrange = None self.yrange = None self.xlabel = None self.ylabel = None self.xscale = None self.yscale = None self.fileout = None self.title = None self.using = None self.frequency = None self.start = None self.end = None self.figsize = None self.blackwhite = None self.legend = None self.sort_by = None self.marker = None ### set the members if keys are present in params members = set(self.__dict__.keys()) p = set(params.keys()) for key in members & p: setattr(self,key,params[key]) if("list_quantities" in params): myrun = run.Run(base) print ("list of possible quantities:\n") print("\n".join(myrun.listQuantities())) sys.exit(0) if("list_parameters" in params): myjob = job.Job(base) myjob.prepare() print ("****************************************************************") print ("Job parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myjob.types.iteritems() ] print("\n".join(params)) myrun = run.Run(base) myrun.prepare() print ("****************************************************************") print ("Run parameters:") print ("****************************************************************") params = [str(j[0]) + ": " + str(j[1]) for j in myrun.types.iteritems() ] print("\n".join(params)) sys.exit(0) self.linewidth_cycle = [1, 2, 4] self.linestyle_cycle = ['-', '--', '-.'] self.cycle_index = 0 ################################################################ class GraphParser(bdparser.BDParser): """ """ def __init__ (self): bdparser.BDParser.__init__(self) self.admissible_params["quantity"] = [str] self.help["quantity"] = "Specify the quantity to be outputed" self.admissible_params["xrange"] = [float] self.help["xrange"] = "Specify range of values in the X direction" self.admissible_params["yrange"] = [float] self.help["yrange"] = "Specify range of values in the Y direction" self.admissible_params["sort_by"] = [str] self.help["sort_by"] = "Specify a study parameter to be used in sorting the curves" self.admissible_params["xlabel"] = str self.help["xlabel"] = "Specify the label for the X axis" self.admissible_params["ylabel"] = str self.help["ylabel"] = "Specify the label for the Y axis" self.admissible_params["xscale"] = float self.default_params["xscale"] = 1. self.help["xscale"] = "Specify a scale factor for the X axis" self.admissible_params["yscale"] = float self.default_params["yscale"] = 1. self.help["yscale"] = "Specify a scale factor for the Y axis" self.admissible_params["title"] = str self.help["title"] = "Specify title for the graph" self.admissible_params["legend"] = [str] self.help["legend"] = "Specify a legend for the curves. The syntax can use %%j.param or %%r.param to use get job and run values" self.default_params["legend"] = None self.admissible_params["using"] = [str] self.help["using"] = "Allow to combine several quantities. The syntax uses python syntax where %%quantity1.column1:%%quantity2.column2 is the python numpy vector provided by quantity number (provided using the --quantities option) and column number (x or y). The sytax is comparable to the GNUPlot one in using the ':' to separate X from Y axis" self.admissible_params["list_quantities"] = None self.help["list_quantities"] = "Request to list the possible quantities to be plotted" self.admissible_params["list_parameters"] = None self.help["list_parameters"] = "Request to list the possible job/run parameters" self.admissible_params["frequency"] = None self.help["frequency"] = "Set a frequency at which the quantity values should be retreived (helpful when the amount of data is very large)" - self.admissible_params["start"] = None + self.admissible_params["start"] = float self.help["start"] = "Set the start X value for the graph" self.admissible_params["end"] = None self.help["end"] = "Set the end X value for the graph" self.admissible_params["figsize"] = None self.default_params["figsize"] = None self.admissible_params["blackwhite"] = None self.help["blackwhite"] = "Request a black and white graph generation" self.default_params["blackwhite"] = False self.help["blackwhite"] = "Request to plot a black and white graph" self.admissible_params["marker"] = str self.help["marker"] = "Request a specific marker (matplotlib option)" self.admissible_params["fileout"] = str self.help["fileout"] = "Request to write a PDF file (given its name) containing the graph" self.group_params["GraphHelper"] = [ "quantity", "xrange", "yrange", "sort_by", "xlabel", "ylabel", "xscale", "yscale", "title", "legend", "using", "list_quantities", "list_parameters", "frequency", "start", "end", "figsize", "blackwhite", "marker", "fileout"] ################################################################ diff --git a/python/BlackDynamite/launchRuns.py b/python/BlackDynamite/launchRuns.py index ac9c8cc..5b7172d 100755 --- a/python/BlackDynamite/launchRuns.py +++ b/python/BlackDynamite/launchRuns.py @@ -1,106 +1,106 @@ #!/usr/bin/env python import base import job import run import os , sys, stat import runconfig import subprocess import conffile import getopt import socket import bdparser import runselector from types import ModuleType def main(argv = None): if (type(argv) == str): argv = argv.split() parser = bdparser.BDParser() parser.register_params( group = "launchRuns.py", params={ "outpath":str, "generator":ModuleType, "nruns":int, "state":str, "machine_name":str }, defaults = { "machine_name":socket.gethostname(), "nruns":-1, "generator":"bashCoat" }) parser.help.update({ "nruns":"Specify the number of runs to launch. This is useful when we want to launch only the first run from the stack.", "generator":"Specify the launcher generator" }) params = parser.parseBDParameters(argv = argv) mybase = base.Base(**params) if (not "outpath" in params): print ("A directory where to create temp files should be provided. use --outpath ") sys.exit(-1) mydir = os.path.join(params["outpath"], "BD-" + params["study"] + "-runs") if not os.path.exists(mydir): os.makedirs(mydir) os.chdir(mydir) run_constraints = [] job_constraints = [] runSelector = runselector.RunSelector(mybase) if ("run_constraints" in params): run_constraints = params["run_constraints"] def item_matcher(name, item): return item.lower().lstrip().startswith(name) if not any([item_matcher("state", item) for item in run_constraints]): run_constraints.append("state = CREATED") if not any([item_matcher("machine_name", item) for item in run_constraints]): run_constraints.append("machine_name = {0}".format(params["machine_name"])) if ("job_constraints" in params): job_constraints = params["job_constraints"] run_list = runSelector.selectRuns(run_constraints,job_constraints) if (params["nruns"] > 0): run_list = [run_list[i] for i in range(0,min(params["nruns"],len(run_list)))] if (len(run_list) == 0): print "No runs to be launched" for r,j in run_list: - print("Dealing with job {0.id}, run {1.id}".format(r, j)) + print("Dealing with job {0.id}, run {1.id}".format(j,r)) r["run_path"] = os.path.join(mydir, "run-" + str(r.id)) j.update() if not os.path.exists("run-" + str(r.id)): os.makedirs("run-" + str(r.id)) os.chdir("run-" + str(r.id)) conffiles = r.getConfigFiles() for conf in conffiles: print ("create file " + conf["filename"]) f = open(conf["filename"], 'w') f.write(conf["file"]) f.close() print ("launch in '" + mydir + "/" + "run-" + str(r.id) + "/'") mymod = params["generator"] print (mymod) mymod.launch(r,params) os.chdir("../") if (params["truerun"] == True): mybase.commit() if __name__ == '__main__': main() diff --git a/python/BlackDynamite/run.py b/python/BlackDynamite/run.py index 7e68a60..83416be 100644 --- a/python/BlackDynamite/run.py +++ b/python/BlackDynamite/run.py @@ -1,288 +1,293 @@ #!/usr/bin/env python from __future__ import print_function import job import runconfig import conffile import sqlobject import base import sys import bdparser as bdp import re import numpy as np import datetime class Run(sqlobject.SQLObject): """ """ def attachToJob(self,job): self["job_id"] = job.id self.base.insert(self) self.addConfigFile(self.execfile) for cnffile in self.configfiles: self.addConfigFile(cnffile) def getExecFile(self): return self.getUpdatedConfigFile(self.entries["exec"]) def addConfigFiles(self,file_list,regex_params=None): self.prepare() params_list = self.types.keys() myjob = job.Job(self.base) myjob.prepare() params_list += myjob.types.keys() #print (regex_params) - self.configfiles += [ conffile.addFile(fname,self.base,\ - regex_params=regex_params,\ - params=params_list) for fname in file_list] + file_ids = [f.id for f in self.configfiles] + files_to_add = [conffile.addFile(fname,self.base, + regex_params=regex_params, + params=params_list) for fname in file_list] + + for f in files_to_add: + if (f.id not in file_ids): + self.configfiles.append(f) return self.configfiles def addConfigFile(self,configfile): myrun_config = runconfig.RunConfig(self.base) myrun_config.prepare() myrun_config.attachToRun(self) myrun_config.addConfigFile(configfile) self.base.insert(myrun_config) def getConfigFiles(self): # myjob = job.Job(self.base) # myjob["id"] = self.entries["job_id"] # myjob = self.getMatchedObjectList()[0] runconf = runconfig.RunConfig(self.base) runconf["run_id"] = self.id runconf_list = runconf.getMatchedObjectList() conffiles = [self.getUpdatedConfigFile(f["configfile_id"]) for f in runconf_list] return conffiles def getConfigFile(self,file_id): runconf = runconfig.RunConfig(self.base) conf = conffile.ConfFile(self.base) conf.prepare() conf["id"] = file_id conf = conf.getMatchedObjectList()[0] return conf def getUpdatedConfigFile(self,file_id): conf = self.getConfigFile(file_id) myjob = job.Job(self.base) myjob.prepare() myjob["id"] = self.entries["job_id"] myjob = myjob.getMatchedObjectList()[0] for key,val in myjob.entries.iteritems(): tmp = conf["file"].replace("__BLACKDYNAMITE__" + key + "__",str(val)) if ((not tmp == conf["file"]) and val is None): raise Exception( "unset job parameter " + key) conf["file"] = tmp for key,val in self.entries.iteritems(): tmp = conf["file"].replace("__BLACKDYNAMITE__" + key + "__",str(val)) if ((not tmp == conf["file"]) and not val): raise Exception( "unset run parameter " + key) conf["file"] = tmp conf["file"] = conf["file"].replace("__BLACKDYNAMITE__dbhost__",self.base.dbhost) conf["file"] = conf["file"].replace("__BLACKDYNAMITE__study__",self.base.schema) conf["file"] = conf["file"].replace("__BLACKDYNAMITE__run_id__",str(self.id)) return conf def listQuantities(self): request = "SELECT id,name FROM {0}.quantities WHERE (is_vector) = (false)".format(self.base.schema) curs = self.base.performRequest(request) all_quantities = [res[1] for res in curs] return all_quantities def getScalarQuantities(self,names,additional_request=None): request = "SELECT id,name FROM {0}.quantities WHERE (is_vector) = (false)".format(self.base.schema) params = [] if (names): if (not type(names) == list): names = [names] request += " and (" for name in names: similar_op_match = re.match(r"\s*(~)\s*(.*)", name) op = " = " if (similar_op_match): op = " ~ " name = similar_op_match.group(2) request += " name " + op + "%s or" params.append(str(name)) request = request[:len(request)-3] + ")" # print (request) # print (params) curs = self.base.performRequest(request,params) quantities = [res[1] for res in curs] if (len(quantities) == 0): print ("No quantity matches " + str(names)) print ("Quantities declared in the database are \n" + "\n".join([res for res in self.listQuantities()])) sys.exit(-1) return None try: quant_indexes = [quantities.index(n) for n in names] quantities = names except: quant_indexes = None # print (quant) results = [] for key in quantities: q = self.getScalarQuantity(key,additional_request) if (q is not None): results.append([key,q]) print ("got Quantity " + str(key) + " : " + str(q.shape[0]) + " values") return results def getLastStep(self): request = """SELECT max(b.max),max(b.time) from ( SELECT max(step) as max,max(computed_at) as time from {0}.scalar_integer where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.scalar_real where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.vector_integer where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.vector_real where run_id = {1}) as b""".format(self.base.schema,self.id) # print (request) curs = self.base.performRequest(request,[]) item = curs.fetchone() if (item is not None): return item[0],item[1] def getQuantityID(self,name, is_integer = None, is_vector = None): request = "SELECT id,is_integer,is_vector FROM {0}.quantities WHERE (name) = (%s)".format(self.base.schema) curs = self.base.performRequest(request,[name]) item = curs.fetchone() if (item is None): raise Exception("unknown quantity \"" + name + "\"") if ((is_integer is not None) and (not is_integer == item[1])): raise Exception("quantity \"" + name + "\" has is_integer = " + str(item[1])) if ((is_vector is not None) and (not is_vector == item[2])): raise Exception("quantity \"" + name + "\" has is_vector = " + str(item[2])) return item[0],item[1],item[2] def getScalarQuantity(self,name,additional_request=None): quantity_id,is_integer,is_vector = self.getQuantityID(name) if (is_vector == True): raise Exception("Quantity " + name + " is not scalar") request = "SELECT step,measurement from {0}.{1} WHERE (run_id,quantity_id) = ({2},{3})".format(self.base.schema,"scalar_real" if (is_integer == False) else "scalar_integer",self.id,quantity_id) if (additional_request): request += " and " + " and ".join(additional_request) request += " ORDER BY step" curs = self.base.performRequest(request,[name]) fetch = curs.fetchall() if (not fetch): return None return (np.array([(val[0],val[1]) for val in fetch])) def getVectorQuantity(self,name, step): quantity_id,is_integer,is_vector = getQuantityID(self,name) if (is_vector == False): raise Exception("Quantity " + name + " is not vectorial") request = "SELECT measurement from {0}.{1} WHERE (run_id,quantity_id,step) = ({2},{3},{4})".format(self.base.schema,"vector_real" if (is_integer == False) else "vector_integer",self.id,quantity_id,step) curs = self.base.performRequest(request,[name]) fetch = curs.fetchone() if (fetch): return np.array(fetch[0]) return None def pushVectorQuantity(self,vec,step,name, is_integer, description = None): try: quantity_id,is_integer,is_vector = self.getQuantityID(name,is_integer = is_integer, is_vector = True) except Exception as e: print (e) typecode = "int" if is_integer == False: typecode = "float" typecode += ".vector" quantity_id = self.base.pushQuantity(name,typecode,description) array = [i for i in vec] if (is_integer == True): array_format = ",".join(["{:d}".format(i) for i in vec]) request = "INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s)".format(self.base.schema,"vector_real" if (is_integer == False) else "vector_integer") curs = self.base.performRequest(request,[self.id,quantity_id,array,step]) def pushScalarQuantity(self,val,step,name, is_integer, description = None): try: quantity_id,is_integer,is_vector = self.getQuantityID(name,is_integer = is_integer, is_vector = False) except Exception as e: print (e) typecode = "int" if is_integer == False: typecode = "float" quantity_id = self.base.pushQuantity(name,typecode,description) if (is_integer == True): array_format = ",".join(["{:d}".format(i) for i in vec]) request = "INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s)".format(self.base.schema,"vector_real" if (is_integer == False) else "vector_integer") curs = self.base.performRequest(request,[self.id,quantity_id,val,step]) def getAllVectorQuantity(self,name): quantity_id,is_integer,is_vector = self.getQuantityID(name,is_vector = True) request = "SELECT step,measurement from {0}.{1} WHERE (run_id,quantity_id) = ({2},{3}) order by step".format(self.base.schema,"vector_real" if (is_integer == False) else "vector_integer",self.id,quantity_id) curs = self.base.performRequest(request,[name]) fetch = curs.fetchall() if (not fetch): return None matres = np.array([val[1] for val in fetch]) stepres = np.array([val[0] for val in fetch]) return (stepres,matres) def deleteData(self): request,params = "DELETE FROM {0}.scalar_real WHERE run_id={1}".format(self.base.schema,self.id),[] self.base.performRequest(request,params) request,params = "DELETE FROM {0}.scalar_integer WHERE run_id={1}".format(self.base.schema,self.id),[] self.base.performRequest(request,params) request,params = "DELETE FROM {0}.vector_real WHERE run_id={1}".format(self.base.schema,self.id),[] self.base.performRequest(request,params) request,params = "DELETE FROM {0}.vector_integer WHERE run_id={1}".format(self.base.schema,self.id),[] self.base.performRequest(request,params) def __init__ (self,base): sqlobject.SQLObject.__init__(self,base) self.table_name = "runs" self.foreign_keys["job_id"] = "jobs" self.types["machine_name"] = str self.types["run_path"] = str self.allowNull["run_path"] = True self.types["job_id"] = int self.types["nproc"] = int self.types["run_name"] = str self.types["wait_id"] = int self.allowNull["wait_id"] = True self.types["start_time"] = datetime.datetime self.allowNull["start_time"] = True self.types["state"] = str self.allowNull["state"] = True self.execfile = None self.configfiles = [] self.types["exec"] = str ################################################################ diff --git a/python/BlackDynamite/runLM.py b/python/BlackDynamite/runLM.py index f96ce5d..92acac2 100644 --- a/python/BlackDynamite/runLM.py +++ b/python/BlackDynamite/runLM.py @@ -1,106 +1,109 @@ #!/usr/bin/env python from __future__ import print_function import job import runconfig import job import conffile import run import sys import bdparser class RunLM(run.Run): """ """ def attachToJob(self,job): if (not "exec" in self.entries): self.createLaunchFile() + + if ("lm_conf" in self.entries): + self.addConfigFiles([self.entries["lm_conf"]]) run.Run.attachToJob(self,job) + + def addConfigFiles(self,file_list,regex_params='LET\s+%p\s*=\s*%v'): - self.configfiles += run.Run.addConfigFiles(self,file_list,regex_params=regex_params) + run.Run.addConfigFiles(self,file_list,regex_params=regex_params) return self.configfiles def createLaunchFile(self): self.execfile = conffile.addFile("launch.sh", self.base, content=""" export HOST=__BLACKDYNAMITE__dbhost__ export SCHEMA=__BLACKDYNAMITE__study__ export RUN_ID=__BLACKDYNAMITE__run_id__ -export LMPATH=__BLACKDYNAMITE__lm_path__ export DEBUG_LEVEL=0 -mpirun __BLACKDYNAMITE__mpi_option__ -np __BLACKDYNAMITE__nproc__ __BLACKDYNAMITE__lm_path__ __BLACKDYNAMITE__lm_conf__ __BLACKDYNAMITE__nsteps__ +mpirun __BLACKDYNAMITE__mpi_option__ -np __BLACKDYNAMITE__nproc__ __BLACKDYNAMITE__amel_path__ __BLACKDYNAMITE__lm_conf__ __BLACKDYNAMITE__nsteps__ if [ $? != 0 ]; then updateRuns.py --study=$SCHEMA --host=$HOST --run_constraints="id = $RUN_ID" --updates="state = LM FAILED" --truerun fi """ ) self["exec"] = self.execfile.id return self.execfile def __init__ (self,base): run.Run.__init__(self,base) self.types["amel_path"] = str - self.types["lm_path"] = str self.types["nsteps"] = int self.types["lm_conf"] = str self.types["mpi_option"] = str self.types["lm_release_info"] = str - - self.group_params["RunLM"] = [ - "amel_path", - "lm_path", - "nsteps", - "lm_conf", - "mpi_option", - "lm_release_info"] - - + self.allowNull["lm_release_info"] = True ################################################################ class RunLMParser(bdparser.RunParser): """ """ def parseBDParameters(self): params = bdparser.RunParser.parseBDParameters(self) self.checkParam('nsteps',params) - self.checkParam('lm_path',params) + self.checkParam('amel_path',params) self.checkParam('mpi_option',params) self.checkParam('lm_conf',params) return params def declareBDParameters(self,additional_params=None,default_additional_params=None): tmp_params = self.runLM_params if (additional_params): tmp_params.update(additional_params) tmp_default_params = self.default_runLM_params if (default_additional_params): tmp_default_params.update(default_additional_params) bdparser.RunParser.declareBDParameters(self,additional_params=tmp_params,default_additional_params=tmp_default_params) def __init__ (self): bdparser.RunParser.__init__(self) - self.runLM_params = {} - self.runLM_params["nsteps"] = int - self.runLM_params["lm_path"] = str - self.runLM_params["lm_conf"] = str - self.runLM_params["mpi_option"] = str - - self.default_runLM_params = {} - self.default_runLM_params["mpi_option"] = " " + self.admissible_params["nsteps"] = int + self.help["nsteps"] = "Set the number of steps to run" + self.admissible_params["amel_path"] = str + self.help["amel_path"] = "The path to the libmultiscale client executable AMEL" + self.admissible_params["lm_conf"] = str + self.help["lm_conf"] = "The name of the configuration file for libmultiscale" + self.admissible_params["mpi_option"] = str + self.help["mpi_option"] = "optional MPI option" + + self.default_params["mpi_option"] = " " + + self.group_params["RunLM"] = [ + "amel_path", + "nsteps", + "lm_conf", + "mpi_option", + "lm_release_info"] ################################################################