diff --git a/BlackDynamite/base.py b/BlackDynamite/base.py
index 87c386e..ed8d62e 100755
--- a/BlackDynamite/base.py
+++ b/BlackDynamite/base.py
@@ -1,463 +1,463 @@
#!/usr/bin/env python3
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
################################################################
from __future__ import print_function
################################################################
from . import job
from . import bdparser
from . import run
from . import bdlogging
from . import jobselector
import os
import psycopg2
import re
import sys
import getpass
import datetime
import atexit
################################################################
__all__ = ["Base"]
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class Base(object):
"""
"""
def getRunFromID(self, run_id):
myrun = run.Run(self)
myrun["id"] = run_id
myrun.id = run_id
run_list = myrun.getMatchedObjectList()
if len(run_list) != 1:
raise Exception('Unknown run {0}'.format(run_id))
return run_list[0]
def getJobFromID(self, job_id):
myjob = job.Job(self)
myjob["id"] = job_id
myjob.id = job_id
job_list = myjob.getMatchedObjectList()
if len(job_list) != 1:
raise Exception('Unknown run {0}'.format(job_id))
return job_list[0]
def createBase(self, job_desc, run_desc, quantities={}, **kwargs):
# logger.debug (quantities)
self.createSchema(kwargs)
self.createTable(job_desc)
self.createTable(run_desc)
self.createGenericTables()
for qname, type in quantities.items():
self.pushQuantity(qname, type)
if self.truerun:
self.commit()
def getObject(self, sqlobject):
curs = self.connection.cursor()
curs.execute("SELECT * FROM {0}.{1} WHERE id = {2}".format(
self.schema, sqlobject.table_name, sqlobject.id))
col_info = self.getColumnProperties(sqlobject)
line = curs.fetchone()
for i in range(0, len(col_info)):
col_name = col_info[i][0]
sqlobject[col_name] = line[i]
def createSchema(self, params={"yes": False}):
# create the schema of the simulation
curs = self.connection.cursor()
curs.execute(("SELECT schema_name FROM information_schema.schemata"
" WHERE schema_name = '{0}'").format(
self.schema).lower())
if curs.rowcount:
validated = bdparser.validate_question(
"Are you sure you want to drop the schema named '" +
self.schema + "'", params, False)
if validated is True:
curs.execute("DROP SCHEMA {0} cascade".format(self.schema))
else:
logger.debug("creation canceled: exit program")
sys.exit(-1)
curs.execute("CREATE SCHEMA {0}".format(self.schema))
def createTypeCodes(self):
curs = self.connection.cursor()
curs.execute("SELECT typname,oid from pg_type;")
self.type_code = {}
for i in curs:
if i[0] == 'float8':
self.type_code[i[1]] = float
if i[0] == 'text':
self.type_code[i[1]] = str
if i[0] == 'int8':
self.type_code[i[1]] = int
if i[0] == 'int4':
self.type_code[i[1]] = int
if i[0] == 'bool':
self.type_code[i[1]] = bool
if i[0] == 'timestamp':
self.type_code[i[1]] = datetime.datetime
def createTable(self, obj):
request = obj.createTableRequest()
curs = self.connection.cursor()
logger.debug(request)
curs.execute(request)
def createGenericTables(self,):
sql_script_name = os.path.join(os.path.dirname(__file__),
"build_tables.sql")
curs = self.connection.cursor()
# create generic tables
query_list = list()
with open(sql_script_name, "r") as fh:
for line in fh:
query_list.append(re.sub("SCHEMAS_IDENTIFIER",
self.schema, line))
curs.execute("\n".join(query_list))
def getColumnProperties(self, sqlobject):
curs = self.connection.cursor()
curs.execute("SELECT * FROM {0}.{1} LIMIT 0".format(
self.schema, sqlobject.table_name))
column_names = [desc[0] for desc in curs.description]
column_type = [desc[1] for desc in curs.description]
return list(zip(column_names, column_type))
def setObjectItemTypes(self, sqlobject):
col_info = self.getColumnProperties(sqlobject)
for i, j in col_info:
sqlobject.types[i] = self.type_code[j]
# logger.debug (str(i) + " " + str(self.type_code[j]))
def insert(self, sqlobject):
curs = self.performRequest(*(sqlobject.insert()))
sqlobject.id = curs.fetchone()[0]
def performRequest(self, request, params=[]):
curs = self.connection.cursor()
# logger.debug (request)
# logger.debug (params)
try:
curs.execute(request, params)
except psycopg2.ProgrammingError as err:
raise psycopg2.ProgrammingError(
("While trying to execute the query '{0}' with parameters " +
"'{1}', I caught this: '{2}'").format(request, params, err))
return curs
def createParameterSpace(self, myjob, entry_nb=0,
tmp_job=None, nb_inserted=0):
"""
This function is a recursive call to generate the points
in the parametric space
The entries of the jobs are treated one by one
in a recursive manner
"""
# keys() gives a non-indexable view
keys = list(myjob.entries.keys())
nparam = len(keys)
# if this is the case I have done all the
# entries of the job
# it is time to insert it (after some checks)
if entry_nb == nparam:
if tmp_job is None:
raise RuntimeError("internal error")
# check if already inserted
jselect = jobselector.JobSelector(self)
jobs = jselect.selectJobs(tmp_job, quiet=True)
if len(jobs) > 0:
return nb_inserted
# insert it
nb_inserted += 1
logger.info("insert job #{0}".format(nb_inserted) +
': ' + str(tmp_job.entries))
self.insert(tmp_job)
return nb_inserted
if tmp_job is None:
tmp_job = job.Job(self)
# the key that I am currently treating
key = keys[entry_nb]
e = myjob[key]
# if this is a list I have to create several parametric points
if not isinstance(e, list):
e = [e]
for value in e:
tmp_job[key.lower()] = value
nb_inserted = self.createParameterSpace(
myjob, entry_nb+1, tmp_job, nb_inserted)
if self.truerun:
self.commit()
return nb_inserted
def pushQuantity(self, name, type_code, description=None):
""" implemented type_codes: "int" "float" "int.vector" "float.vector"
"""
if ((type_code == "int") or (type_code == int)):
is_integer = True
is_vector = False
elif (type_code == "int.vector"):
is_integer = True
is_vector = True
elif ((type_code == "float") or (type_code == float)):
is_integer = False
is_vector = False
elif (type_code == "float.vector"):
is_integer = False
is_vector = True
else:
raise Exception(
"invalid type '{0}' for a quantity".format(type_code))
curs = self.connection.cursor()
curs.execute("""
INSERT INTO {0}.quantities (name, is_integer, is_vector, description)
VALUES (%s , %s , %s, %s) RETURNING id
""".format(self.schema), (name, is_integer, is_vector, description))
item = curs.fetchone()
if (item is None):
raise Exception("Counld not create quantity \"" + name + "\"")
return item[0]
def commit(self):
logger.debug("commiting changes to base")
self.connection.commit()
def getUserList(self):
curs = self.connection.cursor()
curs.execute("""
select tableowner from pg_tables where tablename = 'runs';
""")
users = [desc[0] for desc in curs]
return users
def getStudySize(self, study):
curs = self.connection.cursor()
try:
logger.info(study)
curs.execute("""
select sz from (SELECT SUM(pg_total_relation_size(quote_ident(schemaname)
|| '.' || quote_ident(tablename)))::BIGINT
FROM pg_tables WHERE schemaname = '{0}') as sz
""".format(study))
size = curs.fetchone()[0]
curs.execute("""
select pg_size_pretty(cast({0} as bigint))
""".format(size))
size = curs.fetchone()[0]
curs.execute("""
select count({0}.runs.id) from {0}.runs
""".format(study))
nruns = curs.fetchone()[0]
curs.execute("""
select count({0}.jobs.id) from {0}.jobs
""".format(study))
njobs = curs.fetchone()[0]
except psycopg2.ProgrammingError:
self.connection.rollback()
size = '????'
return {'size': size, 'nruns': nruns, 'njobs': njobs}
def grantAccess(self, study, user):
curs = self.connection.cursor()
curs.execute("""
grant SELECT on ALL tables in schema {0} to {1};
grant USAGE on SCHEMA {0} to {1};
""".format(study, user))
self.commit()
def revokeAccess(self, study, user):
curs = self.connection.cursor()
curs.execute("""
revoke SELECT on ALL tables in schema {0} from {1};
revoke USAGE on SCHEMA {0} from {1};
""".format(study, user))
self.commit()
def getStudyOwner(self, schema):
curs = self.connection.cursor()
curs.execute("""
select grantor from information_schema.table_privileges
where (table_name,table_schema,privilege_type)
= ('runs','{0}','SELECT');
""".format(schema))
owners = [desc[0] for desc in curs]
return owners[0]
def getGrantedUsers(self, schema):
curs = self.connection.cursor()
curs.execute("""
select grantee from information_schema.table_privileges
where (table_name,table_schema,privilege_type)
= ('runs','{0}','SELECT');
""".format(schema))
granted_users = [desc[0] for desc in curs]
return granted_users
def getSchemaList(self, filter_names=True):
curs = self.connection.cursor()
curs.execute("""
SELECT distinct(table_schema) from information_schema.tables
where table_name='runs'
""")
schemas = [desc[0] for desc in curs]
filtered_schemas = []
if filter_names is True:
for s in schemas:
m = re.match('{0}_(.+)'.format(self.user), s)
if m:
s = m.group(1)
filtered_schemas.append(s)
else:
filtered_schemas = schemas
return filtered_schemas
def checkStudy(self, dico):
if "study" not in dico:
message = "\n" + "*"*30 + "\n"
message += "Parameter 'study' must be provided at command line\n"
message += "possibilities are:\n"
schemas = self.getSchemaList()
for s in schemas:
message += "\t" + s + "\n"
message += "\n"
message += "FATAL => ABORT\n"
message += "*"*30 + "\n"
logger.error(message)
sys.exit(-1)
def close(self):
if 'connection' in self.__dict__:
logger.debug('closing database session')
self.connection.close()
del (self.__dict__['connection'])
def __init__(self, truerun=False, creation=False, **kwargs):
psycopg2_params = ["host", "user", "port", "password"]
connection_params = bdparser.filterParams(psycopg2_params, kwargs)
connection_params['dbname'] = 'blackdynamite'
if ("password" in connection_params and
- connection_params["password"] == 'ask'):
- connection_params["password"] = getpass.getpass()
+ connection_params["password"] == 'ask'):
+ connection_params["password"] = getpass.getpass()
logger.debug('connection arguments: {0}'.format(connection_params))
try:
self.connection = psycopg2.connect(**connection_params)
logger.debug('connected to base')
except Exception as e:
logger.error(
"Connection failed: check your connection settings:\n" +
str(e))
sys.exit(-1)
assert(isinstance(self.connection, psycopg2._psycopg.connection))
self.dbhost = (kwargs["host"]
if "host" in kwargs.keys()
else "localhost")
if 'user' in kwargs:
self.user = kwargs["user"]
else:
self.user = os.getlogin()
if ("should_not_check_study" not in kwargs):
self.checkStudy(kwargs)
if 'study' in kwargs:
# Need this because getSchemaList strips prefix
match = re.match('(.+)_(.+)', kwargs["study"])
if match:
self.schema = kwargs["study"]
study_name = match.group(2)
else:
self.schema = kwargs["user"] + '_' + kwargs["study"]
study_name = kwargs["study"]
if (
(creation is not True) and
(study_name not in self.getSchemaList())):
raise RuntimeError("Study name '{}' invalid".format(
- study_name))
+ study_name))
self.createTypeCodes()
self.truerun = truerun
if("list_parameters" in kwargs and kwargs["list_parameters"] is True):
message = self.getPossibleParameters()
logger.info("\n{0}".format(message))
sys.exit(0)
# We should avoid using __del__ to close DB
def close_db():
self.close()
atexit.register(close_db)
def getPossibleParameters(self):
myjob = job.Job(self)
message = ""
message += ("*"*65 + "\n")
message += ("Job parameters:\n")
message += ("*"*65 + "\n")
params = [str(j[0]) + ": " + str(j[1])
for j in myjob.types.items()]
message += ("\n".join(params)+"\n")
myrun = run.Run(self)
message += ("*"*65 + "\n")
message += ("Run parameters:\n")
message += ("*"*65 + "\n")
params = [str(j[0]) + ": " + str(j[1])
for j in myrun.types.items()]
message += ("\n".join(params))
return message
################################################################
if __name__ == "__main__":
connection = psycopg2.connect(host="localhost")
job_description = job.Job(dict(hono=int, lulu=float, toto=str))
base = Base("honoluluSchema", connection, job_description)
base.create()
connection.commit()
base.pushJob(dict(hono=12, lulu=24.2, toto="toto"))
base.pushQuantity("ekin", "float")
connection.commit()
diff --git a/BlackDynamite/bdparser.py b/BlackDynamite/bdparser.py
index bff5c1d..50635c9 100755
--- a/BlackDynamite/bdparser.py
+++ b/BlackDynamite/bdparser.py
@@ -1,673 +1,676 @@
#!/usr/bin/env python3
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
from __future__ import print_function
from . import __path__ as BD_path
from . import base
from . import run
from . import bdlogging
import sys
import re
import os
import stat
import pwd
import argcomplete
import argparse
from argcomplete.completers import EnvironCompleter
import traceback
from types import ModuleType
import imp
################################################################
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class BDParser(object):
def listPossibleHosts(self):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = []
try:
hosts += [h.strip() for h in open(bd_hosts)]
except Exception:
pass
return hosts
def listPossibleModules(self, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
module_list = []
paths = [p.strip() for p in paths if not p.strip() == '']
for p in paths:
files = os.listdir(p)
files = [f for f in files if os.path.splitext(f)[1] == '.py']
files = [f for f in files if not f[0] == '_']
matching_string = ".*blackdynamite.*"
files = [os.path.splitext(f)[0] for
f in files if re.match(
matching_string,
open(os.path.join(p, f)).read().replace('\n', ' '),
flags=re.IGNORECASE)]
module_list += files
logger.debug("found these files " + str(module_list))
return module_list
def updatePossibleHosts(self, new_host):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = set(self.listPossibleHosts())
hosts.add(new_host)
f = open(bd_hosts, 'w')
for h in hosts:
f.write(h+'\n')
def completer(self, prefix, **kwargs):
# bdlogging.activateFileLogging()
try:
params = vars(kwargs["parsed_args"])
if params['logging'] is True:
bdlogging.activateFileLogging()
logger.debug("in")
logger.debug("BDparser prefix " + str(prefix) + "\n")
for k, v in kwargs.items():
logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")
to_delete = []
for k in params.keys():
if params[k] is None:
to_delete.append(k)
for k in to_delete:
del params[k]
key = vars(kwargs["action"])["dest"]
logger.debug("key " + str(key) + "\n")
- if (key == "BDconf"):
+ if (key == "bdconf"):
return self.listPossibleConf()
- if 'BDconf' in params:
- self.readConfFiles(params, params['BDconf'])
+ if 'bdconf' in params:
+ self.readConfFiles(params, params['bdconf'])
if 'host' in params:
self.readConfFile(params, params['host']+'.bd')
logger.debug("key " + str(key) + "\n")
for k in params.keys():
logger.debug("params = " + str(k))
if (key == "host"):
return self.listPossibleHosts()
if (key == "study"):
params["should_not_check_study"] = True
mybase = base.Base(**params)
return mybase.getSchemaList()
if (key == 'quantity'):
mybase = base.Base(**params)
myrun = run.Run(mybase)
return myrun.listQuantities()
if (key in self.admissible_params and
self.admissible_params[key] == ModuleType):
logger.debug(
"trying to complete module list for '{}'".format(key))
return self.listPossibleModules(params)
except Exception as e:
logger.debug(traceback.format_exc())
logger.debug(str(e))
return []
def listPossibleConf(self):
logger.debug("in")
files = []
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
for filename in os.listdir(dir):
fileName, fileExtension = os.path.splitext(filename)
if (fileExtension == ".bd"):
files.append(filename)
return files
return files
def readConfFiles(self, read_params, fnames):
logger.debug("in")
for f in fnames:
self.readConfFile(read_params, f)
def readConfFile(self, read_params, fname):
logger.debug("in")
logger.debug("readConfFileList {0}".format(self.readConfFileList))
if fname in self.readConfFileList:
return
self.readConfFileList.append(fname)
if type(fname) == list:
raise Exception(
'cannot use list in that function: ' + str(type(fname)))
pre_args = {}
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
fullpath = os.path.join(dir, fname)
if (os.path.isfile(fullpath)):
fname = fullpath
break
try:
with open(fname) as fh:
logger.debug("loading file '{0}'".format(fname))
os.chmod(fname, stat.S_IREAD | stat.S_IWRITE)
lines = [line.strip() for line in fh]
regex = "(.*)=(.*)"
for line in lines:
match = re.match(regex, line)
if (not match):
print("malformed line:" + line)
sys.exit(-1)
param = match.group(1).strip()
val = match.group(2).strip()
pre_args[param] = val
self.argv.append("--"+param)
self.argv.append(val)
logger.debug("read parameters: '{0}'".format(self.argv))
logger.debug("pre args : '{0}'".format(pre_args))
read_params.update(self.createParamsMap(pre_args))
except Exception as e:
logger.debug("cannot open file " + fname + '\n' +
str(e) + '\n' + str(traceback.format_exc()))
logger.debug("out")
def checkParam(self, p, dico):
logger.debug("in")
print("****************")
print("Obsolete: should use the mandatory argument"
" for the declare_params function")
print("It was used by object " + str(self) + " for keyword " + p)
print("FATAL => ABORT")
print("****************")
sys.exit(-1)
def loadModule(self, read_params, myscript, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
mymod = None
for p in paths:
try:
modfile = os.path.join(p, myscript+".py")
# print("loading file " + modfile)
mymod = imp.load_source(myscript, modfile)
break
except IOError as io_err:
logger.debug("loadModule " + str(io_err))
logger.debug("loadModule " + str(mymod))
if (mymod is None):
logger.debug("cannot find module '" + myscript +
"' from paths " + str(paths))
logger.debug("trace :" + traceback.format_exc() + '\n')
raise Exception("cannot find module '" +
myscript + "' from paths " + str(paths))
return mymod
def createParamsMap(self, pre_args):
logger.debug("in")
read_params = {}
if ('logging' in pre_args) and (pre_args['logging'] is True):
bdlogging.activateFileLogging()
for opt, args in pre_args.items():
logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
if (args is None):
continue
if (not type(args) == list):
args = [args]
if (type(args) == str):
args = [args]
logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))
for arg in args:
if (arg is None):
continue
- if (opt == 'BDconf'):
+ if (opt == 'bdconf'):
if (not type(arg) == list):
arg = [arg]
self.readConfFiles(read_params, arg)
continue
if (opt == 'host'):
self.updatePossibleHosts(arg)
self.readConfFile(read_params, arg+'.bd')
+ if (opt == 'user'):
+ self.readConfFile(read_params, arg+'.bd')
for param, typ in self.admissible_params.items():
if opt == param:
logger.debug("createParamsMap3 " +
str(param) + " : " + str(typ))
if (typ == ModuleType):
read_params[param] = self.loadModule(
read_params, arg, pre_args)
logger.debug("createParamsMap4 " +
str(param) + " : " + str(typ))
elif (type(typ) == list):
args = arg.split(",")
if (param not in read_params):
read_params[param] = []
if (not len(typ) == 1):
subtype = str
else:
subtype = typ[0]
for i in range(0, len(args)):
read_params[param].append(subtype(args[i]))
else:
read_params[param] = typ(arg)
break
+ logger.debug(read_params)
return read_params
def addModulesAdmissibleParameters(self, read_params):
logger.debug("in")
for k, v in read_params.items():
if self.admissible_params[k] == ModuleType:
mymod = read_params[k]
modname = mymod.__name__
if "admissible_params" in mymod.__dict__:
self.admissible_params.update(
mymod.__dict__["admissible_params"])
self.group_params["'" + modname + "' module options"] = (
mymod.__dict__["admissible_params"].keys())
if "default_params" in mymod.__dict__:
self.default_params.update(
mymod.__dict__["default_params"])
if "help" in mymod.__dict__:
self.help.update(mymod.__dict__["help"])
if "mandatory" in mymod.__dict__:
self.mandatory.update(mymod.__dict__["mandatory"])
logger.debug(str(self.admissible_params))
def addModulesAdmissibleParametersForComplete(self, read_params):
logger.debug("in")
if "_ARGCOMPLETE" not in os.environ:
return
logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])
# breaks = os.environ["COMP_WORDBREAKS"]
tmp_read_params = {}
breaks = " |=|&|<|>|;"
logger.debug("break line " + os.environ["COMP_LINE"])
all_args = re.split(breaks, os.environ["COMP_LINE"])
logger.debug("break line " + str(all_args))
for i in range(0, len(all_args)):
a = all_args[i]
res = re.match("--(.*)", a)
if res is None:
continue
a = res.group(1)
logger.debug("treating a " + str(a))
if a in self.admissible_params:
if self.admissible_params[a] == ModuleType:
logger.debug("here treating a " + str(a))
if i+1 >= len(all_args):
continue
b = all_args[i+1]
logger.debug("treating b " + str(b))
res = re.match("--(.*)", b)
if res is not None:
continue
if not b.strip() == '':
tmp_read_params[a] = b
if ("module_path" in read_params):
tmp_read_params["module_path"] = read_params["module_path"]
logger.debug("tmp_read_params " + str(tmp_read_params))
try:
tmp_read_params = self.createParamsMap(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("AAAAAAAAA " + str(tmp_read_params))
try:
self.addModulesAdmissibleParameters(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("CCCCCCCCCC" + str(self.admissible_params))
def constructArgParser(self, add_help=True, add_mandatory=True):
logger.debug("in")
parser = argparse.ArgumentParser(
description="BlackDynamite option parser",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
add_help=add_help)
self.params_group = {}
group = parser.add_argument_group("General")
self.params_group["General"] = group
for g, param_list in self.group_params.items():
group = parser.add_argument_group(g)
for p in param_list:
self.params_group[p] = group
for param, typ in self.admissible_params.items():
# print("param {0}: {1}".format(param,typ) )
p_help = "help TODO"
is_mandatory = (param in self.mandatory.keys() and
self.mandatory[param] is True and
add_mandatory)
if (param in self.help):
p_help = self.help[param]
if (param in self.params_group):
grp = self.params_group[param]
else:
grp = self.params_group["General"]
if (typ is None):
raise Exception("Deprectated option type for " +
param + " : should be changed to 'bool'")
if (typ is bool):
if (param in self.default_params and
self.default_params[param] is True):
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_false',
required=is_mandatory)
else:
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_true',
required=is_mandatory)
elif (typ is list or typ == [str]):
grp.add_argument(
"--" + param,
action='append',
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
else:
grp.add_argument(
"--" + param,
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
parser.set_defaults(**self.default_params)
return parser
def register_params(self, group="General", params=None, defaults=None,
help=None, mandatory=None):
logger.debug("in")
if (params is not None):
self.admissible_params.update(params)
if group not in self.group_params:
self.group_params[group] = []
self.group_params[group] += params.keys()
if (defaults is not None):
self.default_params.update(defaults)
for key in defaults.keys():
self.mandatory[key] = False
if (help is not None):
self.help.update(help)
if (mandatory is not None):
self.mandatory.update(mandatory)
for param, typ in self.admissible_params.items():
if typ == bool and param not in self.default_params:
self.default_params[param] = False
def addEnvBDArguments(self, parser):
logger.debug("in")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
pre_args = vars(parser.parse_known_args(args=self.argv)[0])
for name, value in os.environ.items():
m = re.match("BLACKDYNAMITE_(.*)", name)
if (m):
var = m.group(1).lower()
if (var not in pre_args or pre_args[var] is None):
if (var in self.admissible_params):
self.argv.append("--" + var)
self.argv.append(value)
def parseBDParameters(self, argv=None):
logger.debug("in")
if argv is None:
self.argv = list(sys.argv[1:])
else:
self.argv = list(argv)
logger.debug("program called with " +
str(len(self.argv)) +
" args " + str(self.argv) + "\n")
logger.debug("env is\n\n")
for k, v in os.environ.items():
logger.debug("export " + k + "='" + v + "'\n")
logger.debug("constructArgParser\n")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
self.addEnvBDArguments(parser)
logger.debug("parse_known_args\n")
pre_args = parser.parse_known_args(args=self.argv)[0]
logger.debug("createParamsMap\n")
read_params = self.createParamsMap(vars(pre_args))
logger.debug("addModuleAdmissibleParameters\n")
self.addModulesAdmissibleParameters(read_params)
logger.debug("addModulesAdmissibleParametersForComplete\n")
try:
self.addModulesAdmissibleParametersForComplete(read_params)
except KeyError as e:
logger.debug("trace :" + traceback.format_exc())
logger.debug("constructArgParser\n")
parser = self.constructArgParser()
argcomplete.autocomplete(parser)
pre_args = parser.parse_args(args=self.argv)
read_params = self.createParamsMap(vars(pre_args))
if "user" not in read_params:
read_params["user"] = pwd.getpwuid(os.getuid())[0]
return read_params
def __init__(self):
logger.debug("in")
self.admissible_params = {}
self.help = {}
self.default_params = {}
self.group_params = {}
self.mandatory = {}
self.admissible_params["study"] = str
self.help["study"] = (
"Specify the study from the BlackDynamite database"
". This refers to the schemas in PostgreSQL language")
self.admissible_params["host"] = str
self.help["host"] = "Specify data base server address"
self.admissible_params["port"] = int
self.help["port"] = "Specify data base server port"
self.admissible_params["user"] = str
self.help["user"] = "Specify user name to connect to data base server"
self.admissible_params["password"] = str
self.help["password"] = "Provides the password"
- self.admissible_params["BDconf"] = list
- self.help["BDconf"] = (
+ self.admissible_params["bdconf"] = list
+ self.help["bdconf"] = (
"Path to a BlackDynamite file (*.bd) configuring current optons")
self.admissible_params["truerun"] = bool
self.help["truerun"] = (
"Set this flag if you want to truly perform the"
" action on base. If not set all action are mainly dryrun")
self.default_params["truerun"] = False
self.admissible_params["constraints"] = [str]
self.help["constraints"] = (
"This allows to constraint run/job selections by properties")
self.default_params["constraints"] = None
self.admissible_params["binary_operator"] = str
self.default_params["binary_operator"] = 'and'
self.help["binary_operator"] = (
'Set the default binary operator to make requests to database')
self.admissible_params["list_parameters"] = bool
self.help["list_parameters"] = (
"Request to list the possible job/run parameters")
self.admissible_params["yes"] = bool
self.default_params["yes"] = False
self.help["yes"] = "Answer all questions to yes."
self.admissible_params["logging"] = bool
self.help["logging"] = "Activate the file logging system"
self.group_params["BDParser"] = ["study",
"host",
"port",
"user",
"password",
- "BDconf",
+ "bdconf",
"truerun",
"constraints",
"job_constraints",
"run_constraints",
"list_parameters"]
self.readConfFileList = []
################################################################
def validate_question(question, params, default_validated=True):
logger.debug("in")
if (default_validated):
default_str = "(Y/n)"
else:
default_str = "(y/N)"
if params["yes"] is False:
validated = input("{0}? {1} ".format(question, default_str))
# print (validated)
if (validated == "\n" or validated == ""):
validated = default_validated
elif(validated == "Y" or validated == "y"):
validated = True
else:
validated = False
else:
logger.info("{0}? {1} Forced Y".format(question, default_str))
validated = True
return validated
################################################################
def filterParams(sub_list, total_list):
logger.debug("in")
new_list = {}
for p in sub_list:
if (p in total_list and total_list[p] is not False):
new_list[p] = total_list[p]
return new_list
################################################################
class RunParser(BDParser):
"""
"""
def parseBDParameters(self):
logger.debug("in")
params = BDParser.parseBDParameters(self)
- params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
+ params['run_name'], nb_subs = re.subn(r'\s', '_', params['run_name'])
return params
def __init__(self):
logger.debug("in")
BDParser.__init__(self)
self.mandatory["machine_name"] = True
self.mandatory["nproc"] = True
self.mandatory["run_name"] = True
self.admissible_params["machine_name"] = str
self.help["machine_name"] = ("Specify the name of the machine where"
" the job is to be launched")
self.admissible_params["nproc"] = int
self.help["nproc"] = ("Specify the number of processors onto which"
" this run is supposed to be launched")
self.admissible_params["run_name"] = str
self.help["run_name"] = (
'User friendly name given to this run.'
' This is usually helpful to recall a run kind')
self.default_params = {}
self.default_params["job_constraints"] = None
self.group_params["RunParser"] = [
"machine_name",
"nproc",
"run_name"]
################################################################
__all__ = ["BDParser", "RunParser"]
diff --git a/BlackDynamite/coating/bashCoat.py b/BlackDynamite/coating/bashCoat.py
index 82f654d..e5874f1 100755
--- a/BlackDynamite/coating/bashCoat.py
+++ b/BlackDynamite/coating/bashCoat.py
@@ -1,75 +1,74 @@
#!/usr/bin/env python
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
# from BlackDynamite import *
import os
import stat
import subprocess
admissible_params = {"stdout": bool,
"stop_on_error": bool}
default_params = {"stop_on_error": False}
help = {"stdout": "Specify if you want the standard output instead of a file",
"stop_on_error":
"Specify if should raise an error in case "
"of an error in the bash script"}
def launch(run, params):
_exec = run.getExecFile()
head = \
"""#!/bin/bash
export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__
export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__
export BLACKDYNAMITE_USER={0}
-
""".format(params["user"])
_exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"]
f = open(_exec["filename"], 'w')
f.write(_exec["file"])
f.close()
os.chmod(_exec["filename"], stat.S_IRWXU)
print("execute ./" + _exec["filename"])
if params["truerun"] is True:
run["state"] = "launched"
run.update()
run.commit()
filename = run["run_name"] + ".o" + str(run.id)
filename_err = run["run_name"] + ".e" + str(run.id)
if params["stdout"] is True:
ret = subprocess.call("./" + _exec["filename"])
else:
with open(filename, "w") as outfile:
with open(filename_err, "w") as errfile:
ret = subprocess.call(
"./" + _exec["filename"],
stdout=outfile,
stderr=errfile)
if ret == 0:
run["state"] = "FINISHED"
else:
run["state"] = "BASH error"
run.update()
run.commit()
if params["stop_on_error"] is True and not ret == 0:
raise Exception(
"The underlying bash script returned "
"with the error code {0}.".format(ret))