diff --git a/BlackDynamite/BDstat.py b/BlackDynamite/BDstat.py
index 9ed56bf..e005a35 100644
--- a/BlackDynamite/BDstat.py
+++ b/BlackDynamite/BDstat.py
@@ -1,151 +1,163 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
from . import job
import numpy as np
class BDStat(object):
"""
"""
def average(self, quantity, run_list, entries_to_average):
result = dict()
run_info = dict()
cpt = dict()
run_counter = 0
nb_runs = len(run_list)
entries_to_average.append("id")
entries_to_consider = []
if (run_list is not []):
for e in run_list[0][1].entries:
if e not in entries_to_average:
entries_to_consider.append(e)
entries_to_consider = sorted(entries_to_consider)
# print (entries_to_consider)
for r, j in run_list:
q = r.getScalarQuantity(quantity)
run_counter += 1
if (q is None):
continue
print("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format(
r.id, q.shape, run_counter, nb_runs,
1.*run_counter/nb_runs*100))
ent = [j[i] for i in entries_to_consider]
ent = tuple(ent)
if (ent not in result.keys()):
# print (ent)
result[ent] = np.zeros([q.shape[0], 3])
cpt[ent] = 0
# print (result[ent].shape)
# print (q.shape)
sz1 = q.shape[0]
sz2 = result[ent].shape[0]
if (sz1 > sz2):
q = q[:sz2]
elif (sz2 > sz1):
result[ent] = result[ent][:sz1]
result[ent][:, 0] += q[:, 0]
result[ent][:, 1] += q[:, 1]
result[ent][:, 2] += q[:, 1]**2
cpt[ent] += 1
for ent in result.keys():
# print (ent)
myjob = job.Job(self.base)
for i in range(0, len(entries_to_consider)):
myjob.entries[entries_to_consider[i]] = ent[i]
result[ent] /= cpt[ent]
result[ent][:, 2] -= result[ent][:, 1]**2
result[ent][:, 2] = np.maximum(
np.zeros(result[ent][:, 2].shape[0]), result[ent][:, 2])
result[ent][:, 2] = np.sqrt(result[ent][:, 2])
result[ent] = {"ref_job": myjob,
"averaged_number": cpt[ent],
"data": result[ent]
}
return result
def averageVector(self, quantity, run_list, entries_to_average):
result = dict()
all_steps = dict()
run_info = dict()
cpt = dict()
run_counter = 0
nb_runs = len(run_list)
entries_to_average.append("id")
entries_to_consider = []
if (run_list is not []):
for e in run_list[0][1].entries:
if e not in entries_to_average:
entries_to_consider.append(e)
for r, j in run_list:
steps, data = r.getAllVectorQuantity(quantity)
run_counter += 1
if (data is None):
continue
print("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format(
r.id, data.shape, run_counter, nb_runs,
1.*run_counter/nb_runs*100))
ent = [j[i] for i in entries_to_consider]
ent = tuple(ent)
if (ent not in result.keys()):
# print (ent)
result[ent] = np.zeros([data.shape[0], data.shape[1], 2])
all_steps[ent] = steps
cpt[ent] = 0
# print (result[ent].shape)
# print (q.shape)
sz1 = data.shape[0]
sz2 = result[ent].shape[0]
if (sz1 > sz2):
data = data[:sz2]
steps = steps[:sz2]
elif (sz2 > sz1):
result[ent] = result[ent][:sz1]
all_steps[ent] = all_steps[ent][:sz1]
result[ent][:, :, 0] += data[:, :]
result[ent][:, :, 1] += data[:, :]**2
cpt[ent] += 1
for ent in result.keys():
# print (ent)
myjob = job.Job(self.base)
for i in range(0, len(entries_to_consider)):
myjob.entries[entries_to_consider[i]] = ent[i]
result[ent] /= cpt[ent]
result[ent][:, :, 1] -= result[ent][:, :, 0]**2
result[ent][:, :, 1] = np.sqrt(result[ent][:, :, 1])
result[ent] = {"ref_job": myjob,
"averaged_number": cpt[ent],
"steps": all_steps[ent],
"data": result[ent]}
return result
def __init__(self, base, **params):
self.base = base
################################################################
__all__ = ["BDStat"]
diff --git a/BlackDynamite/__init__.py b/BlackDynamite/__init__.py
index a665e6d..a46bde6 100644
--- a/BlackDynamite/__init__.py
+++ b/BlackDynamite/__init__.py
@@ -1,38 +1,50 @@
#!/usr/bin/env python3
# flake8: noqa
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from . import bdparser
from . import run
from . import job
from . import base
from . import runselector
from . import jobselector
from . import conffile
from . import BDstat
from .bdparser import *
from .run import *
from .job import *
from .base import *
from .runselector import *
from .jobselector import *
from .conffile import *
from .BDstat import *
__all__ = ["run", "job", "base"]
__all__.extend(bdparser.__all__)
__all__.extend(run.__all__)
__all__.extend(job.__all__)
__all__.extend(base.__all__)
__all__.extend(runselector.__all__)
__all__.extend(jobselector.__all__)
__all__.extend(conffile.__all__)
__all__.extend(BDstat.__all__)
# try:
# import graphhelper
# __all__.append("graphhelper")
# from graphhelper import *
# __all__.extend(graphhelper.__all__)
# except:
# print "graphhelper not loaded"
diff --git a/BlackDynamite/bdlogging.py b/BlackDynamite/bdlogging.py
index d00616e..09c5d33 100644
--- a/BlackDynamite/bdlogging.py
+++ b/BlackDynamite/bdlogging.py
@@ -1,84 +1,96 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
import logging
import traceback
import os
import sys
from . import __name__ as global_name
# Base level logger
root_logger = logging.getLogger(global_name)
root_logger.setLevel(logging.DEBUG) # Avoid hard-filtering
# Logging format
BD_FORMAT = "%(levelname)s:%(funcName)s [%(filename)s:%(lineno)d]: %(message)s"
sh = logging.StreamHandler(sys.stderr)
sh.setLevel(logging.WARNING) # Only show warnings to screen
sh.setFormatter(logging.Formatter(BD_FORMAT))
root_logger.addHandler(sh)
class ExtraContext:
"""Adds some context to logging"""
_wire = {
'foo': lambda x: x.foo(),
'l': lambda x: x.lololo(),
'f': lambda x: x.fname()
}
@staticmethod
def getTr():
return traceback.extract_stack(limit=20)[11]
def fname(self):
return os.path.basename(self.getTr()[0])
def lololo(self):
return self.getTr()[1]
def foo(self):
return self.getTr()[2]
def __getitem__(self, name):
return self._wire[name](self)
def __iter__(self):
d = {k: self._wire[k](self) for k in self._wire}
return iter(d)
def invalidPrint(x):
raise Exception('print should not be used in that class: '
'use the logging system instead: "{0}"'.format(x))
def activateFileLogging():
"""Activate logging to file (if not already enabled)"""
# formatter = logging.Formatter(fmt='%(levelname)s:%(foo)50s:%(f)15s:%(l)s:'
# + ' '*10 + '%(message)s')
formatter = logging.Formatter(BD_FORMAT)
# Handler for file
bd_file_handler = logging.FileHandler('bd.log', mode='a+')
bd_file_handler.setFormatter(formatter)
bd_file_handler.setLevel(logging.DEBUG) # Log everything to file
if '_has_file_handler' not in globals() \
or not globals()['_has_file_handler']:
logger = logging.getLogger(global_name)
logger.debug("Activating logging to file")
logger.addHandler(bd_file_handler)
# This should be the first line logged in file
logger.debug("Activated logging to file")
globals()['_has_file_handler'] = True
def getLogger(name):
logger = logging.getLogger(name)
logger.propagate = True
return logger
diff --git a/BlackDynamite/bdparser.py b/BlackDynamite/bdparser.py
index f7d8753..e673e1b 100755
--- a/BlackDynamite/bdparser.py
+++ b/BlackDynamite/bdparser.py
@@ -1,658 +1,670 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
from . import __path__ as BD_path
from . import base
from . import run
from . import bdlogging
import sys
import re
import os
import stat
import pwd
import argcomplete
import argparse
from argcomplete.completers import EnvironCompleter
import traceback
from types import ModuleType
import imp
################################################################
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class BDParser(object):
def listPossibleHosts(self):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = []
try:
hosts += [h.strip() for h in open(bd_hosts)]
except Exception:
pass
return hosts
def listPossibleModules(self, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
module_list = []
paths = [p.strip() for p in paths if not p.strip() == '']
for p in paths:
files = os.listdir(p)
files = [f for f in files if os.path.splitext(f)[1] == '.py']
files = [f for f in files if not f[0] == '_']
matching_string = ".*blackdynamite.*"
files = [os.path.splitext(f)[0] for
f in files if re.match(
matching_string,
open(os.path.join(p, f)).read().replace('\n', ' '),
flags=re.IGNORECASE)]
module_list += files
logger.debug("found these files " + str(module_list))
return module_list
def updatePossibleHosts(self, new_host):
logger.debug("in")
bd_dir = os.path.expanduser("~/.blackdynamite")
bd_hosts = os.path.join(bd_dir, 'hosts')
hosts = set(self.listPossibleHosts())
hosts.add(new_host)
f = open(bd_hosts, 'w')
for h in hosts:
f.write(h+'\n')
def completer(self, prefix, **kwargs):
# bdlogging.activateFileLogging()
try:
params = vars(kwargs["parsed_args"])
if params['logging'] is True:
bdlogging.activateFileLogging()
logger.debug("in")
logger.debug("BDparser prefix " + str(prefix) + "\n")
for k, v in kwargs.items():
logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n")
logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n")
for k in params.keys():
if params[k] is None:
del params[k]
key = vars(kwargs["action"])["dest"]
logger.debug("key " + str(key) + "\n")
if (key == "BDconf"):
return self.listPossibleConf()
if 'BDconf' in params:
self.readConfFiles(params, params['BDconf'])
if 'host' in params:
self.readConfFile(params, params['host']+'.bd')
logger.debug("key " + str(key) + "\n")
for k in params.keys():
logger.debug("params = " + str(k))
if (key == "host"):
return self.listPossibleHosts()
if (key == "study"):
params["should_not_check_study"] = True
mybase = base.Base(**params)
return mybase.getSchemaList()
if (key == 'quantity'):
mybase = base.Base(**params)
myrun = run.Run(mybase)
return myrun.listQuantities()
if (key in self.admissible_params and
self.admissible_params[key] == ModuleType):
logger.debug(
"trying to complete module list for '{}'".format(key))
return self.listPossibleModules(params)
except Exception as e:
logger.debug(traceback.format_exc())
logger.debug(str(e))
return []
def listPossibleConf(self):
logger.debug("in")
files = []
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
for filename in os.listdir(dir):
fileName, fileExtension = os.path.splitext(filename)
if (fileExtension == ".bd"):
files.append(filename)
return files
return files
def readConfFiles(self, read_params, fnames):
logger.debug("in")
for f in fnames:
self.readConfFile(read_params, f)
def readConfFile(self, read_params, fname):
logger.debug("in")
logger.debug("readConfFileList {0}".format(self.readConfFileList))
if fname in self.readConfFileList:
return
self.readConfFileList.append(fname)
if type(fname) == list:
raise Exception(
'cannot use list in that function: ' + str(type(fname)))
pre_args = {}
for dir in ["./", os.path.expanduser("~/.blackdynamite")]:
fullpath = os.path.join(dir, fname)
if (os.path.isfile(fullpath)):
fname = fullpath
break
try:
with open(fname) as fh:
logger.debug("loading file '{0}'".format(fname))
os.chmod(fname, stat.S_IREAD | stat.S_IWRITE)
lines = [line.strip() for line in fh]
regex = "(.*)=(.*)"
for line in lines:
match = re.match(regex, line)
if (not match):
print("malformed line:" + line)
sys.exit(-1)
param = match.group(1).strip()
val = match.group(2).strip()
pre_args[param] = val
self.argv.append("--"+param)
self.argv.append(val)
logger.debug("read parameters: '{0}'".format(self.argv))
logger.debug("pre args : '{0}'".format(pre_args))
read_params.update(self.createParamsMap(pre_args))
except Exception as e:
logger.debug("cannot open file " + fname + '\n' +
str(e) + '\n' + str(traceback.format_exc()))
logger.debug("out")
def checkParam(self, p, dico):
logger.debug("in")
print("****************")
print("Obsolete: should use the mandatory argument"
" for the declare_params function")
print("It was used by object " + str(self) + " for keyword " + p)
print("FATAL => ABORT")
print("****************")
sys.exit(-1)
def loadModule(self, read_params, myscript, pre_args):
logger.debug("in")
paths = []
if ("PYTHONPATH" in os.environ):
paths = os.environ["PYTHONPATH"].split(':')
if ("module_path" in pre_args):
paths += pre_args["module_path"].split(':')
paths += BD_path
paths += [path + "/coating" for path in BD_path]
mymod = None
for p in paths:
try:
modfile = os.path.join(p, myscript+".py")
# print("loading file " + modfile)
mymod = imp.load_source(myscript, modfile)
break
except IOError as io_err:
logger.debug("loadModule " + str(io_err))
logger.debug("loadModule " + str(mymod))
if (mymod is None):
logger.debug("cannot find module '" + myscript +
"' from paths " + str(paths))
logger.debug("trace :" + traceback.format_exc() + '\n')
raise Exception("cannot find module '" +
myscript + "' from paths " + str(paths))
return mymod
def createParamsMap(self, pre_args):
logger.debug("in")
read_params = {}
if ('logging' in pre_args) and (pre_args['logging'] is True):
bdlogging.activateFileLogging()
for opt, args in pre_args.items():
logger.debug("createParamsMap1 " + str(opt) + " : " + str(args))
if (args is None):
continue
if (not type(args) == list):
args = [args]
if (type(args) == str):
args = [args]
logger.debug("createParamsMap2 " + str(opt) + " : " + str(args))
for arg in args:
if (arg is None):
continue
if (opt == 'BDconf'):
if (not type(arg) == list):
arg = [arg]
self.readConfFiles(read_params, arg)
continue
if (opt == 'host'):
self.updatePossibleHosts(arg)
self.readConfFile(read_params, arg+'.bd')
for param, typ in self.admissible_params.items():
if opt == param:
logger.debug("createParamsMap3 " +
str(param) + " : " + str(typ))
if (typ == ModuleType):
read_params[param] = self.loadModule(
read_params, arg, pre_args)
logger.debug("createParamsMap4 " +
str(param) + " : " + str(typ))
elif (type(typ) == list):
args = arg.split(",")
if (param not in read_params):
read_params[param] = []
if (not len(typ) == 1):
subtype = str
else:
subtype = typ[0]
for i in range(0, len(args)):
read_params[param].append(subtype(args[i]))
else:
read_params[param] = typ(arg)
break
return read_params
def addModulesAdmissibleParameters(self, read_params):
logger.debug("in")
for k, v in read_params.items():
if self.admissible_params[k] == ModuleType:
mymod = read_params[k]
modname = mymod.__name__
if "admissible_params" in mymod.__dict__:
self.admissible_params.update(
mymod.__dict__["admissible_params"])
self.group_params["'" + modname + "' module options"] = (
mymod.__dict__["admissible_params"].keys())
if "default_params" in mymod.__dict__:
self.default_params.update(
mymod.__dict__["default_params"])
if "help" in mymod.__dict__:
self.help.update(mymod.__dict__["help"])
if "mandatory" in mymod.__dict__:
self.mandatory.update(mymod.__dict__["mandatory"])
logger.debug(str(self.admissible_params))
def addModulesAdmissibleParametersForComplete(self, read_params):
logger.debug("in")
if "_ARGCOMPLETE" not in os.environ:
return
logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"])
# breaks = os.environ["COMP_WORDBREAKS"]
tmp_read_params = {}
breaks = " |=|&|<|>|;"
logger.debug("break line " + os.environ["COMP_LINE"])
all_args = re.split(breaks, os.environ["COMP_LINE"])
logger.debug("break line " + str(all_args))
for i in range(0, len(all_args)):
a = all_args[i]
res = re.match("--(.*)", a)
if res is None:
continue
a = res.group(1)
logger.debug("treating a " + str(a))
if a in self.admissible_params:
if self.admissible_params[a] == ModuleType:
logger.debug("here treating a " + str(a))
if i+1 >= len(all_args):
continue
b = all_args[i+1]
logger.debug("treating b " + str(b))
res = re.match("--(.*)", b)
if res is not None:
continue
if not b.strip() == '':
tmp_read_params[a] = b
if ("module_path" in read_params):
tmp_read_params["module_path"] = read_params["module_path"]
logger.debug("tmp_read_params " + str(tmp_read_params))
try:
tmp_read_params = self.createParamsMap(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("AAAAAAAAA " + str(tmp_read_params))
try:
self.addModulesAdmissibleParameters(tmp_read_params)
except Exception as e:
logger.debug("trace :" + traceback.format_exc() + '\n' + str(e))
logger.debug("CCCCCCCCCC" + str(self.admissible_params))
def constructArgParser(self, add_help=True, add_mandatory=True):
logger.debug("in")
parser = argparse.ArgumentParser(
description="BlackDynamite option parser",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
add_help=add_help)
self.params_group = {}
group = parser.add_argument_group("General")
self.params_group["General"] = group
for g, param_list in self.group_params.items():
group = parser.add_argument_group(g)
for p in param_list:
self.params_group[p] = group
for param, typ in self.admissible_params.items():
# print("param {0}: {1}".format(param,typ) )
p_help = "help TODO"
is_mandatory = (param in self.mandatory.keys() and
self.mandatory[param] is True and
add_mandatory)
if (param in self.help):
p_help = self.help[param]
if (param in self.params_group):
grp = self.params_group[param]
else:
grp = self.params_group["General"]
if (typ is None):
raise Exception("Deprectated option type for " +
param + " : should be changed to 'bool'")
if (typ is bool):
if (param in self.default_params and
self.default_params[param] is True):
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_false',
required=is_mandatory)
else:
grp.add_argument("--" + param,
help=p_help,
dest=param,
action='store_true',
required=is_mandatory)
elif (typ is list or typ == [str]):
grp.add_argument(
"--" + param,
action='append',
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
else:
grp.add_argument(
"--" + param,
dest=param,
help=p_help,
required=is_mandatory).completer = self.completer
parser.set_defaults(**self.default_params)
return parser
def register_params(self, group="General", params=None, defaults=None,
help=None, mandatory=None):
logger.debug("in")
if (params is not None):
self.admissible_params.update(params)
if group not in self.group_params:
self.group_params[group] = []
self.group_params[group] += params.keys()
if (defaults is not None):
self.default_params.update(defaults)
for key in defaults.keys():
self.mandatory[key] = False
if (help is not None):
self.help.update(help)
if (mandatory is not None):
self.mandatory.update(mandatory)
for param, typ in self.admissible_params.items():
if typ == bool and param not in self.default_params:
self.default_params[param] = False
def addEnvBDArguments(self, parser):
logger.debug("in")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
pre_args = vars(parser.parse_known_args(args=self.argv)[0])
for name, value in os.environ.items():
m = re.match("BLACKDYNAMITE_(.*)", name)
if (m):
var = m.group(1).lower()
if (var not in pre_args or pre_args[var] is None):
if (var in self.admissible_params):
self.argv.append("--" + var)
self.argv.append(value)
def parseBDParameters(self, argv=None):
logger.debug("in")
if argv is None:
self.argv = list(sys.argv[1:])
else:
self.argv = list(argv)
logger.debug("program called with " +
str(len(self.argv)) +
" args " + str(self.argv) + "\n")
logger.debug("env is\n\n")
for k, v in os.environ.items():
logger.debug("export " + k + "='" + v + "'\n")
logger.debug("constructArgParser\n")
parser = self.constructArgParser(add_help=False, add_mandatory=False)
self.addEnvBDArguments(parser)
logger.debug("parse_known_args\n")
pre_args = parser.parse_known_args(args=self.argv)[0]
logger.debug("createParamsMap\n")
read_params = self.createParamsMap(vars(pre_args))
logger.debug("addModuleAdmissibleParameters\n")
self.addModulesAdmissibleParameters(read_params)
logger.debug("addModulesAdmissibleParametersForComplete\n")
try:
self.addModulesAdmissibleParametersForComplete(read_params)
except KeyError as e:
logger.debug("trace :" + traceback.format_exc())
logger.debug("constructArgParser\n")
parser = self.constructArgParser()
argcomplete.autocomplete(parser)
pre_args = parser.parse_args(args=self.argv)
read_params = self.createParamsMap(vars(pre_args))
if "user" not in read_params:
read_params["user"] = pwd.getpwuid(os.getuid())[0]
return read_params
def __init__(self):
logger.debug("in")
self.admissible_params = {}
self.help = {}
self.default_params = {}
self.group_params = {}
self.mandatory = {}
self.admissible_params["study"] = str
self.help["study"] = (
"Specify the study from the BlackDynamite database"
". This refers to the schemas in PostgreSQL language")
self.admissible_params["host"] = str
self.help["host"] = "Specify data base server address"
self.admissible_params["port"] = int
self.help["port"] = "Specify data base server port"
self.admissible_params["user"] = str
self.help["user"] = "Specify user name to connect to data base server"
self.admissible_params["password"] = str
self.help["password"] = "Provides the password"
self.admissible_params["BDconf"] = list
self.help["BDconf"] = (
"Path to a BlackDynamite file (*.bd) configuring current optons")
self.admissible_params["truerun"] = bool
self.help["truerun"] = (
"Set this flag if you want to truly perform the"
" action on base. If not set all action are mainly dryrun")
self.default_params["truerun"] = False
self.admissible_params["constraints"] = [str]
self.help["constraints"] = (
"This allows to constraint run/job selections by properties")
self.default_params["constraints"] = None
self.admissible_params["binary_operator"] = str
self.default_params["binary_operator"] = 'and'
self.help["binary_operator"] = (
'Set the default binary operator to make requests to database')
self.admissible_params["list_parameters"] = bool
self.help["list_parameters"] = (
"Request to list the possible job/run parameters")
self.admissible_params["yes"] = bool
self.default_params["yes"] = False
self.help["yes"] = "Answer all questions to yes."
self.admissible_params["logging"] = bool
self.help["logging"] = "Activate the file logging system"
self.group_params["BDParser"] = ["study",
"host",
"port",
"user",
"password",
"BDconf",
"truerun",
"constraints",
"job_constraints",
"run_constraints",
"list_parameters"]
self.readConfFileList = []
################################################################
def validate_question(question, params, default_validated=True):
logger.debug("in")
if (default_validated):
default_str = "(Y/n)"
else:
default_str = "(y/N)"
if params["yes"] is False:
validated = input("{0}? {1} ".format(question, default_str))
# print (validated)
if (validated == "\n" or validated == ""):
validated = default_validated
elif(validated == "Y" or validated == "y"):
validated = True
else:
validated = False
else:
logger.info("{0}? {1} Forced Y".format(question, default_str))
validated = True
return validated
################################################################
def filterParams(sub_list, total_list):
logger.debug("in")
new_list = {}
for p in sub_list:
if (p in total_list and total_list[p] is not False):
new_list[p] = total_list[p]
return new_list
################################################################
class RunParser(BDParser):
"""
"""
def parseBDParameters(self):
logger.debug("in")
params = BDParser.parseBDParameters(self)
params['run_name'], nb_subs = re.subn('\s', '_', params['run_name'])
return params
def __init__(self):
logger.debug("in")
BDParser.__init__(self)
self.mandatory["machine_name"] = True
self.mandatory["nproc"] = True
self.mandatory["run_name"] = True
self.admissible_params["machine_name"] = str
self.help["machine_name"] = ("Specify the name of the machine where"
" the job is to be launched")
self.admissible_params["nproc"] = int
self.help["nproc"] = ("Specify the number of processors onto which"
" this run is supposed to be launched")
self.admissible_params["run_name"] = str
self.help["run_name"] = (
'User friendly name given to this run.'
' This is usually helpful to recall a run kind')
self.default_params = {}
self.default_params["job_constraints"] = None
self.group_params["RunParser"] = [
"machine_name",
"nproc",
"run_name"]
################################################################
__all__ = ["BDParser", "RunParser"]
diff --git a/BlackDynamite/coating/__init__.py b/BlackDynamite/coating/__init__.py
index a9694fb..7913597 100644
--- a/BlackDynamite/coating/__init__.py
+++ b/BlackDynamite/coating/__init__.py
@@ -1,6 +1,18 @@
#!/bin/env python
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import bashCoat
import pbsCoat
import sgeCoat
import slurmCoat
diff --git a/BlackDynamite/coating/bashCoat.py b/BlackDynamite/coating/bashCoat.py
index 5ca88cb..82f654d 100755
--- a/BlackDynamite/coating/bashCoat.py
+++ b/BlackDynamite/coating/bashCoat.py
@@ -1,63 +1,75 @@
#!/usr/bin/env python
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
# from BlackDynamite import *
import os
import stat
import subprocess
admissible_params = {"stdout": bool,
"stop_on_error": bool}
default_params = {"stop_on_error": False}
help = {"stdout": "Specify if you want the standard output instead of a file",
"stop_on_error":
"Specify if should raise an error in case "
"of an error in the bash script"}
def launch(run, params):
_exec = run.getExecFile()
head = \
"""#!/bin/bash
export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__
export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__
export BLACKDYNAMITE_USER={0}
""".format(params["user"])
_exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"]
f = open(_exec["filename"], 'w')
f.write(_exec["file"])
f.close()
os.chmod(_exec["filename"], stat.S_IRWXU)
print("execute ./" + _exec["filename"])
if params["truerun"] is True:
run["state"] = "launched"
run.update()
run.commit()
filename = run["run_name"] + ".o" + str(run.id)
filename_err = run["run_name"] + ".e" + str(run.id)
if params["stdout"] is True:
ret = subprocess.call("./" + _exec["filename"])
else:
with open(filename, "w") as outfile:
with open(filename_err, "w") as errfile:
ret = subprocess.call(
"./" + _exec["filename"],
stdout=outfile,
stderr=errfile)
if ret == 0:
run["state"] = "FINISHED"
else:
run["state"] = "BASH error"
run.update()
run.commit()
if params["stop_on_error"] is True and not ret == 0:
raise Exception(
"The underlying bash script returned "
"with the error code {0}.".format(ret))
diff --git a/BlackDynamite/coating/pbsCoat.py b/BlackDynamite/coating/pbsCoat.py
index b0d95be..3e154f0 100755
--- a/BlackDynamite/coating/pbsCoat.py
+++ b/BlackDynamite/coating/pbsCoat.py
@@ -1,123 +1,135 @@
#!/usr/bin/env python
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from BlackDynamite import *
import os
import stat
import subprocess
import re
import socket
admissible_params = {"walltime": str,
"email": str,
"nproc": int,
"pbs_option": [str],
"module": [str],
"mpiprocs": int,
"ncpus": int,
"cwd": bool}
default_params = {"walltime": "48:00:00",
"cwd": True}
help = {"walltime": "Specify the wall time for the runs",
"email": "Specify the email to notify",
"nproc": "Force the number of processors and update the run",
"pbs_option": "Allow to provide additional PBS options",
"module": "List of module to load",
"ncpus": "Number of cpus per nodes",
"mpiprocs": "Number of mpi processus per nodes",
"cwd": "Run by default in the run folder"}
def launch(run, params):
_exec = run.getExecFile()
head = \
"""#!/bin/bash
#PBS -l walltime={0}
""".format(params["walltime"])
if ("email" in params):
head += "#PBS -m abe\n"
head += "#PBS -M {0}\n".format(params["email"])
pbs_head_name = "#PBS -N {0}_{1}\n".format(run["run_name"], run.id)
head += pbs_head_name
run["state"] = "PBS submit"
if ("nproc" in params):
run["nproc"] = params["nproc"]
nproc = run["nproc"]
if ("mpiprocs" in params or "npcus" in params):
args = []
if "ncpus" in params:
npernode = params["ncpus"]
args.append("ncpus={0}".format(npernode))
if "mpiprocs" in params:
npernode = min(params["mpiprocs"], nproc)
args.append("mpiprocs={0}".format(npernode))
select = max(1, nproc/npernode)
args.insert(0, "#PBS -l select={0}".format(select))
select_str = ":".join(args)
print select_str
head += select_str + "\n"
else:
head += "#PBS -l nodes=" + str(nproc) + "\n"
if ("pbs_option" in params):
for i in params["pbs_option"]:
head += "#PBS {0}\n".format(i)
if ("module" in params):
for i in params["module"]:
head += "module load {0}\n".format(i)
run.update()
head += """
export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__
export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__
export BLACKDYNAMITE_USER=""" + params["user"] + """
on_kill()
{
updateRuns.py --updates \"state = PBS killed\" --truerun
exit 0
}
on_stop()
{
updateRuns.py --updates \"state = PBS stopped\" --truerun
exit 0
}
# Execute function on_die() receiving TERM signal
#
trap on_stop SIGUSR1
trap on_stop SIGTERM
trap on_kill SIGUSR2
trap on_kill SIGKILL
"""
if (params["cwd"]):
head += """
cd __BLACKDYNAMITE__run_path__
"""
_exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"]
f = open(_exec["filename"], 'w')
f.write(_exec["file"])
f.close()
# os.chmod(_exec["filename"], stat.S_IRWXU)
print("execute qsub ./" + _exec["filename"])
print("in dir ")
subprocess.call("pwd")
if params["truerun"] is True:
ret = subprocess.call("qsub " + _exec["filename"], shell=True)
diff --git a/BlackDynamite/coating/sgeCoat.py b/BlackDynamite/coating/sgeCoat.py
index 0cab3b0..56f84bf 100755
--- a/BlackDynamite/coating/sgeCoat.py
+++ b/BlackDynamite/coating/sgeCoat.py
@@ -1,99 +1,111 @@
#!/usr/bin/env python
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from BlackDynamite import *
import subprocess
admissible_params = {"walltime": str, "email": str,
"nproc": int, "sge_option": [str], "module": [str]}
# default_params = {"walltime":"00:05:00"}
help = {"walltime": "Specify the wall time for the runs",
"email": "Specify the email to notify",
"nproc": "Force the number of processors and update the run",
"sge_option": "Allow to provide additional SGE options",
"module": "List of module to load"}
def launch(run, params):
_exec = run.getExecFile()
if ("walltime" not in params):
raise Exception("walltime not set for this job {0}".format(run.id))
head = \
"""#!/bin/bash
#$ -S /bin/sh
#$ -cwd
#$ -j y
#$ -notify
#$ -l walltime={0}
""".format(params["walltime"])
if ("email" in params):
head += "#$ -m eas -M {0}\n".format(params["email"])
sge_head_name = "#$ -N " + "run" + \
str(run.id) + "-" + run["run_name"] + "\n"
sge_head_name = sge_head_name.replace(":", "_")
head += sge_head_name
run["state"] = "SGE submit"
if ("nproc" in params):
run["nproc"] = params["nproc"]
nproc = run["nproc"]
if (nproc % 12 == 0 and nproc % 8 == 0):
head += "#$ -pe orte* " + str(nproc) + "\n"
elif (nproc % 12 == 0):
head += "#$ -pe orte12 " + str(nproc) + "\n"
elif (nproc % 8 == 0):
head += "#$ -pe orte8 " + str(nproc) + "\n"
if ("sge_option" in params):
for i in params["sge_option"]:
head += "#$ {0}\n".format(i)
if ("module" in params):
for i in params["module"]:
head += "module load {0}\n".format(i)
run.update()
head += """
export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__
export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__
export BLACKDYNAMITE_USER=""" + params["user"] + """
on_kill()
{
updateRuns.py --updates \"state = SGE killed\" --truerun
exit 0
}
on_stop()
{
updateRuns.py --updates \"state = SGE stopped\" --truerun
exit 0
}
# Execute function on_die() receiving TERM signal
#
trap on_stop SIGUSR1
trap on_kill SIGUSR2
"""
_exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"]
f = open(_exec["filename"], 'w')
f.write(_exec["file"])
f.close()
# os.chmod(_exec["filename"], stat.S_IRWXU)
print("execute qsub ./" + _exec["filename"])
print("in dir ")
subprocess.call("pwd")
if params["truerun"] is True:
ret = subprocess.call("qsub " + _exec["filename"], shell=True)
diff --git a/BlackDynamite/coating/slurmCoat.py b/BlackDynamite/coating/slurmCoat.py
index 7abfe84..4fd740a 100755
--- a/BlackDynamite/coating/slurmCoat.py
+++ b/BlackDynamite/coating/slurmCoat.py
@@ -1,127 +1,139 @@
#!/usr/bin/env python
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
from BlackDynamite import *
import os
import stat
import subprocess
import re
import socket
admissible_params = {"walltime": str,
"email": str,
"nodes": int,
"module": [str],
"cpus-per-task": int,
"cpus-per-node": int,
"cwd": bool,
"slurm_option": [str]}
default_params = {"walltime": "48:00:00",
"cwd": True,
"nodes": 1,
"cpus-per-task": 1}
help = {"walltime": "Specify the wall time for the runs",
"email": "Specify the email to notify",
"nodes": "Number of nodes for the job",
"slurm_option": "Allow to provide additional SLURM options",
"module": "List of modules to load",
"cpus-per-node": "Number of CPU per nodes",
"cpus-per-task": "Number of thread per MPI process",
"cwd": "Run by default in the run folder"}
mandatory = {"cpus-per-node": True}
def launch(run, params):
_exec = run.getExecFile()
head = "#!/bin/bash\n\n"
head += "#SBATCH --time={0}\n".format(params["walltime"])
if ("email" in params):
head += "#SBATCH --mail-type=ALL\n"
head += "#SBATCH --mail-user={0}\n".format(params["email"])
slurm_head_name = "#SBATCH --job-name={0}_{1}\n".format(
run["run_name"], run.id)
head += slurm_head_name
run["state"] = "SLURM submit"
if ("nproc" in params):
run["nproc"] = params["nproc"]
nproc = run["nproc"]
try:
nodes = max(nproc * params["cpus-per-task"] //
params["cpus-per-node"], 1)
except Exception as e:
print(params.keys())
print(e)
raise e
head += "#SBATCH --nodes={0}\n".format(nodes)
head += "#SBATCH --ntasks={0}\n".format(nproc)
head += "#SBATCH --cpus-per-task={0}\n".format(params["cpus-per-task"])
if "slurm_option" in params:
for option in params["slurm_option"]:
# To get consistent behavior between --slurm_option=""
# and --slurm_option ""
m = re.match(r'^--(\S+)$', option)
if m:
option = m.group(1)
head += "#SBATCH --{}\n".format(option)
if (params["cwd"]):
head += "#SBATCH --workdir=__BLACKDYNAMITE__run_path__\n"
if ("module" in params):
head += "\nmodule purge\n"
for i in params["module"]:
head += "module load {0}\n".format(i)
run.update()
head += """
export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__
export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__
export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__
export BLACKDYNAMITE_USER=""" + params["user"] + """
on_kill()
{
updateRuns.py --updates \"state = SLURM killed\" --truerun
exit 0
}
on_stop()
{
updateRuns.py --updates \"state = SLURM stopped\" --truerun
exit 0
}
# Execute function on_die() receiving TERM signal
#
trap on_stop SIGUSR1
trap on_stop SIGTERM
trap on_kill SIGUSR2
trap on_kill SIGKILL
"""
_exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"]
f = open(_exec["filename"], 'w')
f.write(_exec["file"])
f.close()
# os.chmod(_exec["filename"], stat.S_IRWXU)
print("execute sbatch ./" + _exec["filename"])
print("in dir ")
subprocess.call("pwd")
if params["truerun"] is True:
ret = subprocess.call("sbatch " + _exec["filename"], shell=True)
diff --git a/BlackDynamite/conffile.py b/BlackDynamite/conffile.py
index d2099a5..6f63675 100755
--- a/BlackDynamite/conffile.py
+++ b/BlackDynamite/conffile.py
@@ -1,80 +1,92 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
################################################################
from . import sqlobject
from . import selector
import re
import os
################################################################
__all__ = ["ConfFile", "addFile"]
################################################################
class ConfFile(sqlobject.SQLObject):
"""
"""
table_name = "configfiles"
def addFile(self, filename,
params=None,
regex_params=None,
content=None):
print("adding file " + filename)
self.entries["filename"] = os.path.basename(filename)
if content:
self.entries["file"] = content
else:
self.entries["file"] = open(filename, 'r').read()
if regex_params:
for p in params:
# lowerp = p.lower()
rp = "(" + regex_params
rp = rp.replace("%p", ")(" + p)
rp = rp.replace("%v", ")(.*)")
rr = "\\1\\2__BLACKDYNAMITE__" + p + "__"
# print (rp)
# print (rr)
self.entries["file"] = re.sub(rp, rr,
self.entries["file"],
flags=re.IGNORECASE)
# print (self.entries["file"])
file_select = selector.Selector(self.base)
filelist = file_select.select(ConfFile, self)
if (len(filelist) == 0):
tmp_conffile = ConfFile(self.base)
tmp_conffile.entries = dict(self.entries)
del tmp_conffile.entries['filename']
md5filelist = tmp_conffile.getMatchedObjectList()
if len(md5filelist) != 0:
import md5
for f in md5filelist:
raise Exception("""
There is already another file with same content but different name:
this is an impossible situation for BlackDynamite.
The file concerned is '{0}' md5:{1}
** If you want keep going, please rename the file before insertion **
""".format(f['filename'], md5.new(f['file']).hexdigest()))
self.base.insert(self)
elif (len(filelist) == 1):
self.entries = filelist[0].entries
self.id = filelist[0].id
def __init__(self, connection):
sqlobject.SQLObject.__init__(self, connection)
self.types["filename"] = str
self.types["file"] = str
def addFile(filename, base, **kwargs):
cnffile = ConfFile(base)
cnffile.addFile(filename, **kwargs)
return cnffile
diff --git a/BlackDynamite/constraints.py b/BlackDynamite/constraints.py
index f56e1d8..2005148 100644
--- a/BlackDynamite/constraints.py
+++ b/BlackDynamite/constraints.py
@@ -1,220 +1,233 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from __future__ import print_function
################################################################
from . import sqlobject
from . import job
from . import run
from . import bdlogging
import pyparsing as pp
################################################################
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class BDconstraints(object):
""
def __iadd__(self, constraints):
if not isinstance(constraints, BDconstraints):
# print('cons2', constraints)
constraints = BDconstraints(self.base, constraints)
self.constraints += constraints.constraints
return self
def __init__(self, base, constraints):
# print('cons', constraints)
self.constraints = constraints
self.base = base
self.conditions = None
if isinstance(constraints, BDconstraints):
self.constraints = constraints.constraints
elif isinstance(constraints, dict):
if "constraints" in constraints:
self.constraints = constraints["constraints"]
elif "run_id" in constraints:
self.constraints = [
'runs.id = {0}'.format(constraints['run_id'])]
elif "job_id" in constraints:
self.constraints = [
'jobs.id = {0}'.format(constraints['job_id'])]
else:
self.constraints = []
if not isinstance(self.constraints, list):
self.constraints = [self.constraints]
self.constraint_parser = BDconstraintsParser(self.base)
# print('cons(end) ', self.constraints)
def _pushCondition(self, _cond, _params):
if self.conditions != '':
self.conditions += ' and '
self.conditions += _cond
self.params += _params
def pushConditionFromSQLObject(self, _cstr):
_cond = []
_params = []
sql_obj = _cstr
for k, v in sql_obj.entries.items():
_cond.append('({0}.{1} = %s)'.format(
sql_obj.table_name, k))
_params.append(v)
_cond = ' and '.join(_cond)
self._pushCondition(_cond, _params)
def pushConditionFromString(self, _cstr):
_cond, _params = self.constraint_parser.parse(_cstr)
self._pushCondition(_cond, _params)
def getMatchingCondition(self):
self.conditions = ""
self.params = []
for _cstr in self.constraints:
if isinstance(_cstr, str):
self.pushConditionFromString(_cstr)
if isinstance(_cstr, sqlobject.SQLObject):
self.pushConditionFromSQLObject(_cstr)
logger.debug(self.conditions)
logger.debug(self.params)
return self.conditions, self.params
################################################################
class BDconstraintsParser(object):
def __init__(self, base):
self.base = base
self._params = []
self.ref_run = run.Run(self.base)
self.ref_job = job.Job(self.base)
# rule for entry in the sqlobject
var = pp.Word(pp.alphanums+'_')
prefix = (pp.Literal('runs') | pp.Literal('jobs')) + pp.Literal('.')
entry = pp.Optional(prefix) + var
def check_varname(tokens):
# print(tokens)
res = pp.ParseResults(''.join(tokens))
logger.debug(res)
if len(tokens) == 3:
obj_type = tokens[0]
var_name = tokens[2].lower()
else:
obj_type = None
var_name = tokens[0].lower()
if obj_type is None:
job_var = var_name in self.ref_job.types
run_var = var_name in self.ref_run.types
if job_var and run_var:
raise RuntimeError(
'ambiguous variable: {0}\n{1}'.format(
res, self.base.getPossibleParameters()))
if job_var:
res.type = self.ref_job.types[var_name]
elif run_var:
res.type = self.ref_run.types[var_name]
else:
raise RuntimeError(
'unknown variable: {0}\n{1}'.format(
res[0], self.base.getPossibleParameters()))
else:
if obj_type == 'runs':
ref_obj = self.ref_run
elif obj_type == 'jobs':
ref_obj = self.ref_job
if var_name not in ref_obj.types:
raise RuntimeError(
'unknown variable: "{0}"\n{1}'.format(
var_name, ref_obj.types))
res.type = ref_obj.types[var_name]
return res
entry = entry.setParseAction(check_varname)
# rule to parse the operators
operators = [
# '+', # addition 2 + 3 5
# '-', # subtraction 2 - 3 -1
# '*', # multiplication 2 * 3 6
# '/', # division (integer division truncates the result)
# '%', # modulo (remainder) 5 % 4 1
# '^', # exponentiation 2.0 ^ 3.0 8
'<', # less than
'>', # greater than
'<=', # less than or equal to
'>=', # greater than or equal to
'=', # equal
'!=', # not equal
'~', # Matches regular expression, case sensitive
'~*', # Matches regular expression, case insensitive
'!~', # Does not match regular expression, case sensitive
'!~*' # Does not match regular expression, case insensitive
]
ops = pp.Literal(operators[0])
for o in operators[1:]:
ops |= pp.Literal(o)
# parse a constraint of the form 'var operator value' and flatten it
constraint = pp.Group(entry + ops + pp.Word(pp.alphanums+'._'))
def regroup_constraints(tokens):
expected_type = tokens[0].type
key = tokens[0][0]
op = tokens[0][1]
val = tokens[0][2]
try:
parse_res = entry.parseString(val)
if parse_res.type != expected_type:
raise RuntimeError('no the correct type')
val = parse_res[0]
except Exception:
self._params.append(val)
val = '%s'
res = ('(' +
' '.join([str(key), str(op), str(val)]) + ')')
return res
constraint = constraint.setParseAction(regroup_constraints)
separator = (pp.Literal(',').setParseAction(
lambda tokens: 'and') | pp.Literal('and'))
self.constraints = (constraint + pp.Optional(
pp.OneOrMore(separator + constraint))).setParseAction(
lambda tokens: ' '.join(tokens))
def parse(self, _str):
self._params = []
logger.debug(_str)
res = self.constraints.parseString(_str)
res = ' '.join(res)
return res, self._params
diff --git a/BlackDynamite/graphhelper.py b/BlackDynamite/graphhelper.py
index ed15ab1..1caa867 100755
--- a/BlackDynamite/graphhelper.py
+++ b/BlackDynamite/graphhelper.py
@@ -1,374 +1,387 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from . import run
from . import runselector
from . import bdparser
import re
import sys
import numpy as np
################################################################
class GraphHelper(object):
"""
"""
def getMeasures(self, run_list):
myresults = []
add_req = []
if (self.frequency):
add_req += ["step %% {0} = 0".format(self.frequency)]
if (self.start):
add_req += ["step > {0}".format(self.start)]
if (self.end):
add_req += ["step < {0}".format(self.end)]
for r, j in run_list:
# print ("retrieve data from run " + r["run_name"])
res = r.getScalarQuantities(self.quantities, add_req)
for key, value in res:
if value is None:
del res[key]
myresults.append([r, j, res])
return myresults
def selectGraphs(self):
run_list = self.runSelector.selectRuns(self.constraints,
self.sort_by)
results = self.getMeasures(run_list)
return results
def show(self):
import matplotlib.pyplot as plt
plt.show()
def makeGraphs(self, fig=None, **kwargs):
import matplotlib.pyplot as plt
results = self.selectGraphs()
if fig is None:
fig = plt.figure(figsize=self.figsize)
for r, j, data in results:
if data:
self.makeCurve(data, fig=fig, myrun=r, myjob=j, **kwargs)
return fig
def replaceRunAndJobsParameters(self, name, myrun, myjob):
res = name
# print (res)
codes = [["%r." + key, myrun[key]] for key in myrun.entries.keys()]
codes += [["%j." + key, myjob[key]] for key in myjob.entries.keys()]
for code, val in codes:
res = res.replace(code, str(val))
return res
def generateLabels(self, results, myrun, myjob):
labels = []
names = [r[0] for r in results]
for i in range(0, len(results)):
name = results[i][0]
if (not self.legend or i >= len(self.legend) or
not self.legend[i]):
labels.append(
self.replaceRunAndJobsParameters(name, myrun, myjob)
)
continue
# print (self.legend[i])
head_legend = self.legend[i].replace("{", "{{")
head_legend = head_legend.replace("}", "}}")
head_legend = re.sub(r"(%)([0-9]+)", r'{\2}', head_legend).format(
*names)
# print (head_legend)
head_legend = self.replaceRunAndJobsParameters(
head_legend, myrun, myjob)
# print (head_legend)
# if (not head_legend.find("%") == -1):
# print("unknown variable name. Possible variables are:")
# print "\n".join([c[0] for c in codes])
# sys.exit(-1)
# print (head_legend)
labels.append(head_legend)
return labels
def makeComposedQuantity(self, results, myrun, myjob):
vecs = [r[1] for r in results]
names = [r[0] for r in results]
# print (vecs[0].shape)
new_results = []
for comp in self.using:
exprs = comp.split(":")
tmp_res = []
for i in [0, 1]:
e = re.sub(r"(%)([0-9]+)\.(x)", r"vecs[\2][:,0]", exprs[i])
e = re.sub(r"(%)([0-9]+)\.(y)", r"vecs[\2][:,1]", e)
e = self.replaceRunAndJobsParameters(e, myrun, myjob)
try:
tmp_res.append(eval(e))
except Exception as ex:
print(names)
print("invalid expression: '" + exprs[i] + "'")
print("invalid expression: '" + e + "'")
print(ex)
i = 1
for v in vecs:
print('quantity {0}/{1} shape: {2}'.format(
i, len(vecs), v.shape))
i += 1
sys.exit(-1)
name = re.sub(r"(%)([0-9]+)\.([x|y])", r'(" + str(names[\2]) + ")',
exprs[1])
res = np.zeros((tmp_res[0].shape[0], 2))
res[:, 0] = tmp_res[0]
res[:, 1] = tmp_res[1]
# print (res.shape)
# expr = re.sub(r"(%)([0-9]+)", r"vecs[\2]", comp)
# res[0] = eval(expr)
# print (name)
name = "\"" + name + "\""
# print (name)
name = eval(name)
# print (name)
new_results.append([name, res])
return new_results
def decorateGraph(self, fig, myrun, myjob, results):
if not results:
return
if fig is None:
import matplotlib.pyplot as plt
fig = plt.figure()
axe = fig.add_subplot(1, 1, 1)
if self.xrange:
axe.set_xlim(self.xrange)
if self.yrange:
axe.set_ylim(self.yrange)
if (self.xlabel):
axe.set_xlabel(self.xlabel)
if (self.ylabel):
axe.set_ylabel(self.ylabel)
if (self.title):
t = self.replaceRunAndJobsParameters(self.title, myrun, myjob)
axe.set_title(t)
axe.grid(True, linewidth=0.1)
if self.using:
results = self.makeComposedQuantity(results, myrun, myjob)
labels = self.generateLabels(results, myrun, myjob)
# print (labels)
return fig, axe, results, labels
def makeCurve(self, results, myrun=None, myjob=None, fig=None, **kwargs):
fig, axe, results, labels = self.decorateGraph(
fig, myrun, myjob, results)
for count, result in enumerate(results):
# name = result[0]
vec = result[1]
label = labels[count]
# print (self.quantities)
# print (name)
style = dict()
if (self.marker is not None):
style["marker"] = self.marker
if self.blackwhite:
width_index = self.cycle_index/len(self.linestyle_cycle)
style_index = self.cycle_index % len(self.linestyle_cycle)
self.cycle_index += 1
style["linewidth"] = self.linewidth_cycle[width_index]
style["linestyle"] = self.linestyle_cycle[style_index]
style["color"] = 'k'
axe.plot(vec[:, 0]/self.xscale, vec[:, 1]/self.yscale,
label=label, **style)
axe.legend(loc='best')
if (self.fileout):
fig.savefig(self.fileout)
return fig
def setConstraints(self, **params):
self.constraints = []
if "constraints" in params:
self.constraints = params["constraints"]
def setBinaryOperator(self, **params):
self.binary_operator = 'and'
if ("binary_operator" in params):
self.binary_operator = params["binary_operator"]
def setQuantity(self, **params):
if ("quantity" in params):
self.quantities = params["quantity"]
else:
print("quantity should be provided using option --quantity")
self.quantities = "__BLACKDYNAMITE_ERROR__"
def __init__(self, base, **params):
self.setConstraints(**params)
self.setQuantity(**params)
self.base = base
self.runSelector = runselector.RunSelector(self.base)
self.fig = None
self.xrange = None
self.yrange = None
self.xlabel = None
self.ylabel = None
self.xscale = None
self.yscale = None
self.fileout = None
self.title = None
self.using = None
self.frequency = None
self.start = None
self.end = None
self.figsize = None
self.blackwhite = None
self.legend = None
self.sort_by = None
self.marker = None
# set the members if keys are present in params
members = set(self.__dict__.keys())
p = set(params.keys())
for key in members & p:
setattr(self, key, params[key])
if params["list_quantities"] is True:
myrun = run.Run(base)
print("list of possible quantities:\n")
print("\n".join(myrun.listQuantities()))
sys.exit(0)
if params["list_parameters"] is True:
self.base.getPossibleParameters()
sys.exit(0)
self.linewidth_cycle = [1, 2, 4]
self.linestyle_cycle = ['-', '--', '-.']
self.cycle_index = 0
################################################################
class GraphParser(bdparser.BDParser):
"""
"""
def __init__(self):
bdparser.BDParser.__init__(self)
self.admissible_params["quantity"] = [str]
self.help["quantity"] = "Specify the quantity to be outputed"
self.admissible_params["xrange"] = [float]
self.help["xrange"] = "Specify range of values in the X direction"
self.admissible_params["yrange"] = [float]
self.help["yrange"] = "Specify range of values in the Y direction"
self.admissible_params["sort_by"] = [str]
self.help["sort_by"] = (
"Specify a study parameter to be used in sorting the curves")
self.admissible_params["xlabel"] = str
self.help["xlabel"] = "Specify the label for the X axis"
self.admissible_params["ylabel"] = str
self.help["ylabel"] = "Specify the label for the Y axis"
self.admissible_params["xscale"] = float
self.default_params["xscale"] = 1.
self.help["xscale"] = "Specify a scale factor for the X axis"
self.admissible_params["yscale"] = float
self.default_params["yscale"] = 1.
self.help["yscale"] = "Specify a scale factor for the Y axis"
self.admissible_params["title"] = str
self.help["title"] = "Specify title for the graph"
self.admissible_params["legend"] = [str]
self.help["legend"] = (
"Specify a legend for the curves."
" The syntax can use %%j.param or %%r.param to use"
" get job and run values")
self.default_params["legend"] = None
self.admissible_params["using"] = [str]
self.help["using"] = (
"Allow to combine several quantities. "
"The syntax uses python syntax where "
"%%quantity1.column1:%%quantity2.column2 is the python "
"numpy vector provided by quantity number (provided using the "
"--quantities option) and column number (x or y). "
"The sytax is comparable to the GNUPlot one in using the ':' "
"to separate X from Y axis")
self.admissible_params["list_quantities"] = bool
self.help["list_quantities"] = (
"Request to list the possible quantities to be plotted")
self.admissible_params["list_parameters"] = bool
self.help["list_parameters"] = (
"Request to list the possible job/run parameters")
self.admissible_params["frequency"] = int
self.default_params["frequency"] = 1
self.help["frequency"] = (
"Set a frequency at which the quantity values "
"should be retreived "
"(helpful when the amount of data is very large)")
self.admissible_params["start"] = float
self.help["start"] = "Set the start X value for the graph"
self.admissible_params["end"] = int
self.help["end"] = "Set the end X value for the graph"
self.admissible_params["figsize"] = [float]
self.admissible_params["blackwhite"] = bool
self.default_params["blackwhite"] = False
self.help["blackwhite"] = "Request a black and white graph generation"
self.default_params["blackwhite"] = False
self.help["blackwhite"] = "Request to plot a black and white graph"
self.admissible_params["marker"] = str
self.help["marker"] = "Request a specific marker (matplotlib option)"
self.admissible_params["fileout"] = str
self.help["fileout"] = (
'Request to write a PDF file'
' (given its name) containing the graph')
self.group_params["GraphHelper"] = [
"quantity",
"xrange",
"yrange",
"sort_by",
"xlabel",
"ylabel",
"xscale",
"yscale",
"title",
"legend",
"using",
"list_quantities",
"list_parameters",
"frequency",
"start",
"end",
"figsize",
"blackwhite",
"marker",
"fileout"]
################################################################
__all__ = ["GraphHelper", "GraphParser"]
################################################################
diff --git a/BlackDynamite/job.py b/BlackDynamite/job.py
index 43e7e23..1c5d301 100755
--- a/BlackDynamite/job.py
+++ b/BlackDynamite/job.py
@@ -1,18 +1,30 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
################################################################
from . import sqlobject
################################################################
__all__ = ["Job"]
################################################################
class Job(sqlobject.SQLObject):
"""
"""
table_name = 'jobs'
def __init__(self, base):
sqlobject.SQLObject.__init__(self, base)
self.table_name = "jobs"
diff --git a/BlackDynamite/jobselector.py b/BlackDynamite/jobselector.py
index 38f8185..528eb4a 100755
--- a/BlackDynamite/jobselector.py
+++ b/BlackDynamite/jobselector.py
@@ -1,31 +1,44 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from . import selector
from . import job
################################################################
class JobSelector(selector.Selector):
"""
"""
def selectJobs(self, constraints=None, sort_by=None, quiet=False):
job_list = self.select(job.Job, constraints=constraints,
sort_by=sort_by)
if quiet is False:
if not job_list:
print("no jobs found")
print("Selected jobs are: " + str([j.id for j in job_list]))
return job_list
def __init__(self, base):
selector.Selector.__init__(self, base)
################################################################
__all__ = ["JobSelector"]
diff --git a/BlackDynamite/run.py b/BlackDynamite/run.py
index 3df54d2..b8c7538 100755
--- a/BlackDynamite/run.py
+++ b/BlackDynamite/run.py
@@ -1,464 +1,476 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
################################################################
from . import job
from . import runconfig
from . import conffile
from . import sqlobject
from . import bdparser
from . import base
from . import runselector
from . import bdlogging
import sys
import re
import numpy as np
import datetime
import subprocess
import socket
import os
################################################################
__all__ = ['Run', 'getRunFromScript']
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class Run(sqlobject.SQLObject):
"""
"""
table_name = 'runs'
def getJob(self):
return self.base.getJobFromID(self.entries["job_id"])
def start(self):
self.entries['state'] = 'START'
logger.debug('starting run')
self.update()
logger.debug('update done')
self.base.commit()
logger.debug('commited')
def finish(self):
self.entries['state'] = 'FINISHED'
logger.debug('finish run')
self.update()
logger.debug('update done')
self.base.commit()
logger.debug('commited')
def attachToJob(self, job):
self["job_id"] = job.id
self.base.insert(self)
self.addConfigFile(self.execfile)
for cnffile in self.configfiles:
self.addConfigFile(cnffile)
def getExecFile(self):
return self.getUpdatedConfigFile(self.entries["exec"])
def setExecFile(self, file_name, **kwargs):
# check if the file is already in the config files
for f in self.configfiles:
if f.entries["filename"] == file_name:
self.execfile = f
self.entries["exec"] = f.id
return f.id
# the file is not in the current config files
# so it has to be added
conf = conffile.addFile(file_name, self.base, **kwargs)
self.configfiles.append(conf)
self.execfile = conf
self.entries["exec"] = conf.id
return conf.id
def listFiles(self, subdir=""):
"""List files in run directory / specified sub-directory"""
command = 'ls {0}'.format(os.path.join(self['run_path'], subdir))
if not self['machine_name'] == socket.gethostname():
command = 'ssh {0} "{1}"'.format(self['machine_name'], command)
logger.info(command)
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
out = p.stdout.readlines()
out = [o.strip() for o in out]
return out
def getFile(self, filename, outpath='/tmp'):
dest_path = os.path.join(
outpath, "BD-" + self.base.schema + "-cache",
"run-{0}".format(self.id))
dest_file = os.path.join(dest_path, filename)
if self['machine_name'] == socket.gethostname():
return self.getFullFileName(filename)
# In case filename contains sub-directories
dest_path = os.path.dirname(dest_file)
logger.info(dest_path)
logger.info(dest_file)
# Making directories
try:
os.makedirs(dest_path, exist_ok=True)
except Exception as e:
logger.error(e)
pass
if os.path.isfile(dest_file):
return dest_file
cmd = 'scp {0}:{1} {2}'.format(self['machine_name'],
self.getFullFileName(filename),
dest_file)
logger.info(cmd)
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
logger.info(p.stdout.read())
return dest_file
def getFullFileName(self, filename):
return os.path.join(self['run_path'], filename)
def addConfigFiles(self, file_list, regex_params=None):
if not type(file_list) == list:
file_list = [file_list]
params_list = list(self.types.keys())
myjob = job.Job(self.base)
params_list += list(myjob.types.keys())
# logger.debug (regex_params)
file_ids = [f.id for f in self.configfiles]
files_to_add = [
conffile.addFile(
fname, self.base,
regex_params=regex_params,
params=params_list)
for fname in file_list]
for f in files_to_add:
if (f.id not in file_ids):
self.configfiles.append(f)
return self.configfiles
def addConfigFile(self, configfile):
myrun_config = runconfig.RunConfig(self.base)
myrun_config.attachToRun(self)
myrun_config.addConfigFile(configfile)
self.base.insert(myrun_config)
def getConfigFiles(self):
# myjob = job.Job(self.base)
# myjob["id"] = self.entries["job_id"]
# myjob = self.getMatchedObjectList()[0]
runconf = runconfig.RunConfig(self.base)
runconf["run_id"] = self.id
runconf_list = runconf.getMatchedObjectList()
logger.info(runconf_list)
conffiles = [
self.getUpdatedConfigFile(f["configfile_id"])
for f in runconf_list]
return conffiles
def getConfigFile(self, file_id):
# runconf = runconfig.RunConfig(self.base)
conf = conffile.ConfFile(self.base)
conf["id"] = file_id
conf = conf.getMatchedObjectList()[0]
return conf
def replaceBlackDynamiteVariables(self, text):
myjob = job.Job(self.base)
myjob["id"] = self.entries["job_id"]
myjob = myjob.getMatchedObjectList()[0]
for key, val in myjob.entries.items():
tmp = text.replace("__BLACKDYNAMITE__" + key + "__",
str(val))
if ((not tmp == text) and val is None):
raise Exception("unset job parameter " + key)
text = tmp
for key, val in self.entries.items():
tmp = text.replace("__BLACKDYNAMITE__" + key + "__",
str(val))
if ((not tmp == text) and val is None):
logger.debug(self.entries)
raise Exception("unset run parameter " + key)
text = tmp
text = text.replace("__BLACKDYNAMITE__dbhost__",
self.base.dbhost)
text = text.replace("__BLACKDYNAMITE__study__",
self.base.schema)
text = text.replace("__BLACKDYNAMITE__run_id__",
str(self.id))
return text
def getUpdatedConfigFile(self, file_id):
conf = self.getConfigFile(file_id)
conf["file"] = self.replaceBlackDynamiteVariables(conf["file"])
return conf
def listQuantities(self):
request = "SELECT id,name FROM {0}.quantities".format(self.base.schema)
curs = self.base.performRequest(request)
all_quantities = [res[1] for res in curs]
return all_quantities
def getScalarQuantities(self, names, additional_request=None):
request = ("SELECT id,name FROM {0}.quantities "
"WHERE (is_vector) = (false)").format(
self.base.schema)
params = []
if (names):
if (not type(names) == list):
names = [names]
request += " and ("
for name in names:
similar_op_match = re.match(r"\s*(~)\s*(.*)", name)
op = " = "
if (similar_op_match):
op = " ~ "
name = similar_op_match.group(2)
request += " name " + op + "%s or"
params.append(str(name))
request = request[:len(request)-3] + ")"
# logger.debug (request)
# logger.debug (params)
curs = self.base.performRequest(request, params)
quantities = [res[1] for res in curs]
if (len(quantities) == 0):
logger.debug("No quantity matches " + str(names))
logger.debug("Quantities declared in the database are \n" +
"\n".join(
[res for res in self.listQuantities()]))
sys.exit(-1)
return None
try:
quant_indexes = [quantities.index(n) for n in names]
logger.debug(quant_indexes)
quantities = names
except:
# quant_indexes = None
pass
# logger.debug (quant)
results = []
for key in quantities:
q = self.getScalarQuantity(key, additional_request)
if (q is not None):
results.append([key, q])
logger.debug("got Quantity " + str(key) + " : " +
str(q.shape[0]) + " values")
return results
def getLastStep(self):
request = """SELECT max(b.max),max(b.time) from (
SELECT max(step) as max,max(computed_at) as time
from {0}.scalar_integer where run_id = {1}
union
SELECT max(step) as max,max(computed_at)
as time from {0}.scalar_real where run_id = {1}
union
SELECT max(step) as max,max(computed_at)
as time from {0}.vector_integer where run_id = {1}
union
SELECT max(step) as max,max(computed_at) as time
from {0}.vector_real where run_id = {1}) as b
""".format(self.base.schema, self.id)
# logger.debug (request)
curs = self.base.performRequest(request, [])
item = curs.fetchone()
if (item is not None):
return item[0], item[1]
def getQuantityID(self, name, is_integer=None, is_vector=None):
request = """
SELECT id,is_integer,is_vector FROM {0}.quantities WHERE (name) = (%s)
""".format(self.base.schema)
curs = self.base.performRequest(request, [name])
item = curs.fetchone()
if (item is None):
raise Exception("unknown quantity \"" + name + "\"")
if ((is_integer is not None) and (not is_integer == item[1])):
raise Exception("quantity \"" + name + "\" has is_integer = " +
str(item[1]))
if ((is_vector is not None) and (not is_vector == item[2])):
raise Exception("quantity \"" + name + "\" has is_vector = " +
str(item[2]))
return item[0], item[1], item[2]
def getScalarQuantity(self, name, additional_request=None):
quantity_id, is_integer, is_vector = self.getQuantityID(name)
if is_vector is True:
raise Exception("Quantity " + name + " is not scalar")
request = """
SELECT step,measurement from {0}.{1} WHERE (run_id,quantity_id) = ({2},{3})
""".format(self.base.schema,
"scalar_real" if (is_integer is False)
else "scalar_integer",
self.id,
quantity_id)
if (additional_request):
request += " and " + " and ".join(additional_request)
request += " ORDER BY step"
curs = self.base.performRequest(request, [name])
fetch = curs.fetchall()
if (not fetch):
return None
return (np.array([(val[0], val[1]) for val in fetch]))
def getVectorQuantity(self, name, step):
quantity_id, is_integer, is_vector = self.getQuantityID(name)
if (is_vector is False):
raise Exception("Quantity " + name + " is not vectorial")
request = """
SELECT measurement from {0}.{1} WHERE (run_id,quantity_id,step) = ({2},{3},{4})
""".format(self.base.schema, "vector_real"
if (is_integer is False)
else "vector_integer",
self.id, quantity_id, step)
curs = self.base.performRequest(request, [name])
fetch = curs.fetchone()
if (fetch):
return np.array(fetch[0])
return None
def pushVectorQuantity(self, vec, step, name,
is_integer, description=None):
logger.debug('pushing {0}'.format(name))
try:
quantity_id, is_integer, is_vector = self.getQuantityID(
name, is_integer=is_integer, is_vector=True)
except Exception as e:
typecode = "int"
if is_integer is False:
typecode = "float"
typecode += ".vector"
quantity_id = self.base.pushQuantity(name, typecode, description)
array = [i for i in vec]
# if is_integer is True:
# array_format = ",".join(["{:d}".format(i) for i in vec])
request = """
INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s)
""".format(self.base.schema, "vector_real"
if is_integer is False
else "vector_integer")
curs = self.base.performRequest(request, [self.id, quantity_id,
array, step])
logger.debug(curs)
logger.debug('ready to commit')
self.base.commit()
logger.debug('commited')
def pushScalarQuantity(self, val, step, name,
is_integer, description=None):
logger.debug('pushing {0}'.format(name))
try:
quantity_id, is_integer, is_vector = self.getQuantityID(
name, is_integer=is_integer, is_vector=False)
except Exception as e:
typecode = "int"
if is_integer is False:
typecode = "float"
quantity_id = self.base.pushQuantity(name, typecode, description)
request = """
INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s)
""".format(self.base.schema, "scalar_real"
if is_integer is False
else "scalar_integer")
curs = self.base.performRequest(request,
[self.id, quantity_id, val, step])
logger.debug(curs)
logger.debug('ready to commit')
self.base.commit()
logger.debug('commited')
def getAllVectorQuantity(self, name):
quantity_id, is_integer, is_vector = self.getQuantityID(
name, is_vector=True)
request = """
SELECT step,measurement from {0}.{1}
WHERE (run_id,quantity_id) = ({2},{3}) order by step
""".format(self.base.schema, "vector_real"
if is_integer is False
else "vector_integer", self.id, quantity_id)
curs = self.base.performRequest(request, [name])
fetch = curs.fetchall()
if (not fetch):
return [None, None]
matres = np.array([val[1] for val in fetch])
stepres = np.array([val[0] for val in fetch])
return (stepres, matres)
def deleteData(self):
request, params = (
"DELETE FROM {0}.scalar_real WHERE run_id={1}".format(
self.base.schema, self.id), [])
self.base.performRequest(request, params)
request, params = (
"DELETE FROM {0}.scalar_integer WHERE run_id={1}".format(
self.base.schema, self.id), [])
self.base.performRequest(request, params)
request, params = (
"DELETE FROM {0}.vector_real WHERE run_id={1}".format(
self.base.schema, self.id), [])
self.base.performRequest(request, params)
request, params = (
"DELETE FROM {0}.vector_integer WHERE run_id={1}".format(
self.base.schema, self.id), [])
self.base.performRequest(request, params)
def __init__(self, base):
sqlobject.SQLObject.__init__(self, base)
self.foreign_keys["job_id"] = "jobs"
self.types["machine_name"] = str
self.types["run_path"] = str
self.allowNull["run_path"] = True
self.types["job_id"] = int
self.types["nproc"] = int
self.types["run_name"] = str
self.types["wait_id"] = int
self.allowNull["wait_id"] = True
self.types["start_time"] = datetime.datetime
self.allowNull["start_time"] = True
self.types["state"] = str
self.allowNull["state"] = True
self.execfile = None
self.configfiles = []
self.types["exec"] = str
################################################################
def getRunFromScript():
parser = bdparser.BDParser()
parser.register_params(params={"run_id": int})
params = parser.parseBDParameters(argv=[])
mybase = base.Base(**params)
runSelector = runselector.RunSelector(mybase)
run_list = runSelector.selectRuns(params)
if len(run_list) > 1:
raise Exception('internal error')
if len(run_list) == 0:
raise Exception('internal error')
myrun, myjob = run_list[0]
# myrun.setEntries(params)
return myrun, myjob
diff --git a/BlackDynamite/runLM.py b/BlackDynamite/runLM.py
index 0f441ec..bc1f1ee 100644
--- a/BlackDynamite/runLM.py
+++ b/BlackDynamite/runLM.py
@@ -1,91 +1,103 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
from . import conffile
from . import run
from . import bdparser
class RunLM(run.Run):
"""
"""
def attachToJob(self, job):
if "exec" not in self.entries:
self.createLaunchFile()
if "lm_conf" in self.entries:
self.addConfigFiles([self.entries["lm_conf"]])
run.Run.attachToJob(self, job)
def addConfigFiles(self, file_list, regex_params='LET\s+%p\s*=\s*%v'):
run.Run.addConfigFiles(self, file_list, regex_params=regex_params)
return self.configfiles
def createLaunchFile(self):
self.execfile = conffile.addFile("launch.sh", self.base,
content="""
export HOST=__BLACKDYNAMITE__dbhost__
export SCHEMA=__BLACKDYNAMITE__study__
export RUN_ID=__BLACKDYNAMITE__run_id__
export DEBUG_LEVEL=0
mpirun __BLACKDYNAMITE__mpi_option__ -np __BLACKDYNAMITE__nproc__ \
__BLACKDYNAMITE__amel_path__ __BLACKDYNAMITE__lm_conf__ \
__BLACKDYNAMITE__nsteps__
if [ $? != 0 ]; then
updateRuns.py --study=$SCHEMA --host=$HOST --run_constraints="id = $RUN_ID" \
--updates="state = LM FAILED" --truerun
fi
""")
self["exec"] = self.execfile.id
return self.execfile
def __init__(self, base):
run.Run.__init__(self, base)
self.types["amel_path"] = str
self.types["nsteps"] = int
self.types["lm_conf"] = str
self.types["mpi_option"] = str
self.types["lm_release_info"] = str
self.allowNull["lm_release_info"] = True
################################################################
class RunLMParser(bdparser.RunParser):
def parseBDParameters(self):
params = bdparser.RunParser.parseBDParameters(self)
return params
def __init__(self):
bdparser.RunParser.__init__(self)
self.mandatory["nsteps"] = True
self.mandatory["amel_path"] = True
self.mandatory["lm_conf"] = True
self.admissible_params["nsteps"] = int
self.help["nsteps"] = "Set the number of steps to run"
self.admissible_params["amel_path"] = str
self.help["amel_path"] = (
"The path to the libmultiscale client executable AMEL")
self.admissible_params["lm_conf"] = str
self.help["lm_conf"] = (
"The name of the configuration file for libmultiscale")
self.admissible_params["mpi_option"] = str
self.help["mpi_option"] = "optional MPI option"
self.default_params["mpi_option"] = " "
self.group_params["RunLM"] = [
"amel_path",
"nsteps",
"lm_conf",
"mpi_option",
"lm_release_info"]
################################################################
diff --git a/BlackDynamite/runconfig.py b/BlackDynamite/runconfig.py
index 88a1b21..fd68871 100755
--- a/BlackDynamite/runconfig.py
+++ b/BlackDynamite/runconfig.py
@@ -1,26 +1,38 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
from . import sqlobject
class RunConfig(sqlobject.SQLObject):
"""
"""
table_name = 'runconfig'
def attachToRun(self, run):
self["run_id"] = run.id
def addConfigFile(self, configfile):
self["configfile_id"] = configfile.id
def __init__(self, base):
sqlobject.SQLObject.__init__(self, base)
self.table_name = "runconfig"
self.foreign_keys["run_id"] = "runs"
self.foreign_keys["configfile_id"] = "configfiles"
self.types["run_id"] = int
self.types["configfile_id"] = int
diff --git a/BlackDynamite/runselector.py b/BlackDynamite/runselector.py
index c32cf11..6cd536a 100755
--- a/BlackDynamite/runselector.py
+++ b/BlackDynamite/runselector.py
@@ -1,43 +1,56 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from . import run
from . import job
from . import selector
from . import constraints as BDconstraints
################################################################
class RunSelector(selector.Selector):
"""
"""
def selectRuns(self, constraints,
sort_by=None,
quiet=False):
run_constraints = BDconstraints.BDconstraints(self.base,
'runs.job_id = jobs.id')
run_constraints += constraints
run_list = self.select([run.Run, job.Job],
constraints=run_constraints,
sort_by=sort_by)
if quiet is False:
if not run_list:
print("no runs found")
print("Selected runs are: " + str([r[0].id for r in run_list]))
print("Selected jobs are: " + str([r[0]["job_id"]
for r in run_list]))
return run_list
def __init__(self, base):
selector.Selector.__init__(self, base)
################################################################
__all__ = ["RunSelector"]
diff --git a/BlackDynamite/selector.py b/BlackDynamite/selector.py
index 9132d09..449199b 100644
--- a/BlackDynamite/selector.py
+++ b/BlackDynamite/selector.py
@@ -1,90 +1,103 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from __future__ import print_function
################################################################
import copy
from . import constraints as BDcons
################################################################
from . import bdlogging
################################################################
print = bdlogging.invalidPrint
logger = bdlogging.getLogger(__name__)
################################################################
class Selector(object):
def __init__(self, base):
self.base = base
def buildList(self, curs, sqlobjs):
logger.debug(sqlobjs)
if not isinstance(sqlobjs, list):
sqlobjs = [sqlobjs]
col_infos = []
sqlobjs2 = []
for sqlobj in sqlobjs:
if isinstance(sqlobj, type):
sqlobj = sqlobj(self.base)
sqlobjs2.append(sqlobj)
col_infos.append(self.base.getColumnProperties(sqlobj))
sqlobjs = sqlobjs2
list_objects = []
for entries in curs:
# print(entries)
objs = []
offset = 0
logger.debug(sqlobjs)
for index, sqlobj in enumerate(sqlobjs):
obj = copy.deepcopy(sqlobj)
for col_name, size in col_infos[index]:
logger.debug((col_name, entries[offset]))
obj[col_name] = entries[offset]
offset += 1
objs.append(obj)
if len(objs) == 1:
list_objects.append(objs[0])
else:
list_objects.append(tuple(objs))
return list_objects
def select(self, _types, constraints=None, sort_by=None):
if (sort_by is not None) and (not isinstance(sort_by, str)):
raise RuntimeError(
'sort_by argument is not correct: {0}'.format(sort_by))
const = BDcons.BDconstraints(self.base, constraints)
condition, params = const.getMatchingCondition()
if not isinstance(_types, list):
_types = [_types]
selected_tables = ['{0}.{1}'.format(self.base.schema, t.table_name)
for t in _types]
selected_tables = ','.join(selected_tables)
request = "SELECT * FROM {0}".format(selected_tables)
if condition:
request += " WHERE " + condition
# print (sort_by)
if sort_by:
request += " ORDER BY " + sort_by
logger.debug(request)
logger.debug(params)
curs = self.base.performRequest(request, params)
obj_list = self.buildList(curs, _types)
return obj_list
diff --git a/BlackDynamite/sqlobject.py b/BlackDynamite/sqlobject.py
index 6644879..5c2eb4c 100644
--- a/BlackDynamite/sqlobject.py
+++ b/BlackDynamite/sqlobject.py
@@ -1,232 +1,245 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from __future__ import print_function
################################################################
import copy
import datetime
import psycopg2
import re
import sys
################################################################
class LowerCaseDict(object):
def __init__(self):
self.entries = {}
def __getattr__(self, attr):
if 'entries' not in self.__dict__:
raise AttributeError(attr)
key = attr.lower()
if key in self.entries:
return self.entries[key]
else:
raise AttributeError(attr)
def __setattr__(self, attr, value):
key = attr.lower()
if key == 'entries':
object.__setattr__(self, key, value)
entries = self.entries
if key in entries:
self.__setitem__(attr, value)
else:
object.__setattr__(self, attr, value)
def __getitem__(self, index):
return self.entries[index.lower()]
def keys(self):
return self.entries.keys()
def __iter__(self):
return self.entries.__iter__()
def __setitem__(self, index, value):
self.entries[index.lower()] = value
def items(self):
return self.entries.items()
def copy(self):
cp = LowerCaseDict()
cp.entries = self.entries
return cp
def setEntries(self, params):
for p, val in params.items():
if p in self.types:
self.entries[p] = val
################################################################
class SQLObject(LowerCaseDict):
" The generic object related to entries in the database "
def __str__(self):
keys = set(self.entries.keys())
if 'id' in self.entries:
keys.remove('id')
keys = list(keys)
if 'id' in self.entries:
keys = ['id'] + keys
outputs = []
for k in keys:
v = self.entries[k]
outputs += [k + ": " + str(v)]
return "\n".join(outputs)
def commit(self):
self.base.connection.commit()
def setFields(self, constraints):
for cons in constraints:
_regex = "(\w*)\s*=\s*(.*)"
match = re.match(_regex, cons)
if (not match or (not len(match.groups()) == 2)):
print("malformed assignment: " + cons)
sys.exit(-1)
key = match.group(1).lower().strip()
val = match.group(2)
if key not in self.types:
print("unknown key '{0}'".format(key))
print("possible keys are:")
for k in self.types.keys():
print("\t" + k)
sys.exit(-1)
val = self.types[key](val)
self.entries[key] = val
def __init__(self, base):
LowerCaseDict.__init__(self)
self.foreign_keys = {}
self.allowNull = {}
self.types = LowerCaseDict()
self.base = base
self.operators = {}
self._prepare()
def __deepcopy__(self, memo):
_cp = type(self)(self.base)
_cp.types = copy.deepcopy(self.types.copy(), memo)
_cp.entries = copy.deepcopy(self.entries.copy(), memo)
# _cp.id = self.id
_cp.foreign_keys = copy.deepcopy(self.foreign_keys, memo)
_cp.allowNull = copy.deepcopy(self.foreign_keys, memo)
_cp.connection = self.base
return _cp
def _prepare(self):
try:
self.base.setObjectItemTypes(self)
except psycopg2.ProgrammingError:
self.base.connection.rollback()
def insert(self):
params = list()
# print (self.types)
ex_msg = ""
for key, value in self.types.items():
if key == "id":
continue
if ((key not in self.entries) and (key not in self.allowNull)):
ex_msg += (
"key '" + key +
"' must be given a value before proceeding insertion\n")
if (not ex_msg == ""):
raise Exception("\n****************\n"+ex_msg+"****************\n")
for key, value in self.entries.items():
# print (key)
# print (self.types[key])
# print (value)
params.append(self.types[key.lower()](value))
request = """
INSERT INTO {0}.{1} ({2}) VALUES ({3}) RETURNING id
""".format(self.base.schema, self.table_name,
','.join(self.entries.keys()),
','.join(["%s" for item in params])), params
return request
def delete(self):
request, params = "DELETE FROM {0}.{1} WHERE id={2}".format(
self.base.schema,
self.table_name,
self.id), []
self.base.performRequest(request, params)
def update(self):
params = list()
keys = list()
for key, value in self.entries.items():
if (value is None):
continue
_type = self.types[key]
# print (_type)
# print (key)
# print (type(value))
if (_type == datetime.datetime):
continue
# _type = str
keys.append(key)
params.append(_type(value))
request = "UPDATE {0}.{1} SET ({2}) = ({3}) WHERE id = {4}".format(
self.base.schema,
self.table_name,
','.join(keys),
','.join(["%s" for item in params]), self.id)
self.base.performRequest(request, params)
def getquoted(self):
raise RuntimeError('code needs review')
# objs = [sql_adapt(member) for member in self._sql_members()]
# for obj in objs:
# if hasattr(obj, 'prepare'):
# obj.prepare(self._conn)
# quoted_objs = [obj.getquoted() for obj in objs]
# return '(' + ', '.join(quoted_objs) + ')'
def createTableRequest(self):
query_string = "CREATE TABLE {0}.{1} ( id SERIAL PRIMARY KEY,".format(
self.base.schema, self.table_name)
for key, value in self.types.items():
if key == 'id':
continue
if (value == float):
type_string = "DOUBLE PRECISION"
elif (value == int):
type_string = "INTEGER"
elif (value == str):
type_string = "TEXT"
elif (value == bool):
type_string = "BOOLEAN"
elif (value == datetime.datetime):
type_string = "TIMESTAMP"
else:
print(value)
raise Exception("type '{0}' not handled".format(value))
query_string += "{0} {1} ".format(key, type_string)
if (key not in self.allowNull):
query_string += " NOT NULL"
query_string += ","
for key, value in self.foreign_keys.items():
query_string += "FOREIGN KEY ({0}) REFERENCES {1}.{2},".format(
key, self.base.schema, value)
return query_string[:-1] + ");"
def getMatchedObjectList(self):
from . import selector
sel = selector.Selector(self.base)
return sel.select(self, self)
diff --git a/example/createDB.py b/example/createDB.py
index 12edbe6..b9bd76d 100755
--- a/example/createDB.py
+++ b/example/createDB.py
@@ -1,26 +1,38 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
# First we need to set the python headers and
# to import the blackdynamite modules
import BlackDynamite as BD
# Then you have to create a generic black dynamite parser
# and parse the system (including the connection parameters and credentials)
parser = BD.bdparser.BDParser()
params = parser.parseBDParameters()
# Then we can connect to the black dynamite database
base = BD.base.Base(**params, creation=True)
# Then you have to define the parametric space (the job pattern)
myjob_desc = BD.job.Job(base)
myjob_desc.types["param1"] = int
myjob_desc.types["param2"] = float
myjob_desc.types["param3"] = str
# Then you have to define the run pattern
myruns_desc = BD.run.Run(base)
myruns_desc.types["compiler"] = str
# Then we request for the creation of the database
base.createBase(myjob_desc, myruns_desc, **params)
diff --git a/example/createJobs.py b/example/createJobs.py
index 7cb84a7..e4bfdec 100755
--- a/example/createJobs.py
+++ b/example/createJobs.py
@@ -1,24 +1,36 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
# First we need to set the python headers
# and to import the \blackdynamite modules
import BlackDynamite as BD
# Then you have to create a generic black dynamite parser
# and parse the system (including the connection parameters and credentials)
parser = BD.bdparser.BDParser()
params = parser.parseBDParameters()
# Then we can connect to the black dynamite database
base = BD.base.Base(**params)
# create of job object
job = BD.job.Job(base)
# specify a range of jobs
job["param1"] = 10
job["param2"] = [3.14, 1., 2.]
job["param3"] = 'toto'
# creation of the jobs on the database
base.createParameterSpace(job)
diff --git a/example/createRuns.py b/example/createRuns.py
index 9e89a73..92a7ce3 100755
--- a/example/createRuns.py
+++ b/example/createRuns.py
@@ -1,39 +1,51 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
# First we need to set the python headers
# and to import the blackdynamite modules
import BlackDynamite as BD
# import a runparser (instead of a generic BD parser)
parser = BD.RunParser()
params = parser.parseBDParameters()
# Then we can connect to the black dynamite database
base = BD.Base(**params)
# create a run object
myrun = BD.Run(base)
# set the run parameters from the parsed entries
myrun.setEntries(params)
# add a configuration file
myrun.addConfigFiles("doIt.py")
# set the entry point (executable) file
myrun.setExecFile("launch.sh")
# create a job selector
jobSelector = BD.JobSelector(base)
# select the jobs that should be associated with the runs about to be created
job_list = jobSelector.selectJobs(params)
# create the runs
for j in job_list:
myrun['compiler'] = 'gcc'
myrun.attachToJob(j)
# if truerun, commit the changes to the base
if (params["truerun"] is True):
base.commit()
diff --git a/example/doIt.py b/example/doIt.py
index 6939cf1..5c5f998 100644
--- a/example/doIt.py
+++ b/example/doIt.py
@@ -1,16 +1,28 @@
#!/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
myrun, myjob = BD.getRunFromScript()
print(myjob)
myrun.start()
for step in range(0, 10):
_quantity = myrun.id*step
myrun.pushScalarQuantity(_quantity, step, "ekin", is_integer=False)
myrun.pushScalarQuantity(_quantity*2, step, "epot", is_integer=False)
myrun.finish()
diff --git a/example/post_treatment.py b/example/post_treatment.py
index 99c882b..c639278 100644
--- a/example/post_treatment.py
+++ b/example/post_treatment.py
@@ -1,44 +1,57 @@
#!/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
import BlackDynamite as BD
import matplotlib.pyplot as plt
################################################################
# basic connection
parser = BD.BDParser()
params = parser.parseBDParameters(
'--host lsmssrv1.epfl.ch --study mystudy'.split())
mybase = BD.Base(**params)
################################################################
# function to plot things (user's job)
def plot(run_list):
for r, j in run_list:
ekin = r.getScalarQuantity('ekin')
if ekin is None:
continue
print(j)
list_files = r.listFiles()
print(list_files)
fname = r.getFile(list_files[3])
print(fname + ':')
_file = open(fname)
print(_file.read())
plt.plot(ekin[:, 0], ekin[:, 1], 'o-',
label='$p_2 = {0}$'.format(j['param2']))
plt.legend(loc='best')
plt.show()
################################################################
# selecting some runs
runSelector = BD.RunSelector(mybase)
run_list = runSelector.selectRuns(params, params)
plot(run_list)
# selecting some other runs
params['run_constraints'] = ['run_name =~ test', 'state = FINISHED']
params['job_constraints'] = ['param2 > 1']
run_list = runSelector.selectRuns(params, params)
plot(run_list)
diff --git a/pythontests/__init__.py b/pythontests/__init__.py
index 5bfd17e..11f0c97 100644
--- a/pythontests/__init__.py
+++ b/pythontests/__init__.py
@@ -1,2 +1,14 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
diff --git a/pythontests/coding_convention.py b/pythontests/coding_convention.py
index 321e3ec..03115b8 100644
--- a/pythontests/coding_convention.py
+++ b/pythontests/coding_convention.py
@@ -1,63 +1,54 @@
#!/usr/bin/env python3
-"""
-@file coding_convention.py
-
-@author Till Junge
-
-@date 11 Aug 2016
-
-@brief Test whether the source files of game_engine follow our coding
- conventions
-
-@section LICENCE
-
- Copyright (C) 2016 Till Junge
-
-coding_convention.py is part of zegame and proprietary
-software; you can neither redistribute nor modify coding_convention.py.
-
-Only members of the zegame association are allowed to view, run,
-modify and copy coding_convention.py under the terms of the zegame
-association statutes.
-"""
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
try:
import unittest
import os
import pep8
import BlackDynamite
except ImportError as err:
import sys
print(err)
sys.exit(str(err))
class Pep8Test(unittest.TestCase):
"""
Test for pep8 conformity
"""
def setUp(self):
"""
builds a list of source files. If you find a smarter way, please let me
know
"""
print() # for emacs to evaluate the first line of errors
self.mod_files = list()
bd_path = os.path.join(BlackDynamite.__path__[0], "..")
for dirpath, _, filenames in os.walk(bd_path):
self.mod_files += [os.path.join(dirpath, filename)
for filename in filenames
if filename.endswith((".py", ".pyx"))]
for dirpath, _, filenames in os.walk(os.path.join(bd_path, 'scripts')):
self.mod_files += [os.path.join(dirpath, filename)
for filename in filenames
if not filename.endswith(".sh")]
def test_pep8_conformity(self):
"""
check all files for pep8 conformity
"""
pep8style = pep8.StyleGuide()
pep8style.check_files((mod_file for mod_file in self.mod_files))
diff --git a/scripts/canYouDigIt.py b/scripts/canYouDigIt.py
index e85d95a..19484ed 100755
--- a/scripts/canYouDigIt.py
+++ b/scripts/canYouDigIt.py
@@ -1,19 +1,31 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# -*- py-which-shell: "python"; -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
################################################################
import BlackDynamite as BD
from BlackDynamite.graphhelper import GraphParser
from BlackDynamite.graphhelper import GraphHelper
################################################################
parser = GraphParser()
params = parser.parseBDParameters()
base = BD.Base(**params)
gH = GraphHelper(base, **params)
gH.makeGraphs(**params)
gH.show()
diff --git a/scripts/cleanRuns.py b/scripts/cleanRuns.py
index a878848..1918632 100755
--- a/scripts/cleanRuns.py
+++ b/scripts/cleanRuns.py
@@ -1,120 +1,132 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
import os
import sys
import socket
import re
import shutil
################################################################
def validate(question):
if params["truerun"] is True:
validated = BD.bdparser.validate_question(question, params)
else:
print("{0}? Forced N".format(question))
validated = False
return validated
parser = BD.BDParser()
parser.register_params("clearRun",
params={
"runid": int,
"clean_orphans": str,
"machine_name": str,
"constraints": [str],
"delete": bool},
defaults={
"machine_name": socket.gethostname(),
"delete": False,
})
params = parser.parseBDParameters()
if "machine_name" in params:
if "constraints" in params:
params["constraints"].append(
"machine_name = " + params["machine_name"])
else:
params["constraints"] = ["machine_name = " + params["machine_name"]]
base = BD.Base(**params)
runSelector = BD.RunSelector(base)
if "clean_orphans" in params:
run_list = runSelector.selectRuns([])
run_ids = [r.id for r, j in run_list]
resdir = params["clean_orphans"] + "/BD-" + params["study"] + "-runs"
print("clean orphans from " + resdir)
if not os.path.exists(resdir):
print("Directory '" + resdir + "' do not exists")
sys.exit(-1)
to_delete = {}
for filename in os.listdir(resdir):
fullname = os.path.join(resdir, filename)
# print(fullname)
if (os.path.isdir(fullname)):
match = re.match("run-([0-9]+)", filename)
if (match):
# print(filename)
id = int(match.group(1))
if (id not in run_ids):
to_delete[id] = fullname
if (len(to_delete.keys()) == 0):
print("No orphans found")
sys.exit(0)
validated = validate("Delete output from runs " + str(to_delete.keys()))
if (validated):
for id, fullname in to_delete.items():
print("Delete output from run " + str(id))
shutil.rmtree(fullname)
sys.exit(0)
runSelector = BD.RunSelector(base)
run_list = runSelector.selectRuns(params)
if (len(run_list) == 0):
print("No runs to be cleared")
validated = validate("Delete runs " + str([r[0].id for r in run_list]))
for r, j in run_list:
delete_flag = params["delete"]
run_path = r["run_path"]
if run_path:
if os.path.exists(run_path):
if (validated):
print("Deleting directory: " + run_path)
shutil.rmtree(run_path)
else:
print("Simulate deletion of directory: " + run_path)
else:
print("output directory: '" + run_path +
"' not found: are we on the right machine ?")
if (delete_flag):
if validated:
print("Deleting run " + str(r.id) + " from base")
r.delete()
base.commit()
else:
print("Simulate deletion of run " + str(r.id) + " from base")
else:
if validated:
print("Deleting data associated with run " + str(r.id))
r.deleteData()
r["STATE"] = "CREATED"
r["start_time"] = None
r.update()
base.commit()
else:
print("Simulate deletion of data associated with run " + str(r.id))
diff --git a/scripts/createUser.py b/scripts/createUser.py
index 295b151..da375af 100755
--- a/scripts/createUser.py
+++ b/scripts/createUser.py
@@ -1,90 +1,103 @@
#!/usr/bin/python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
# import getpass
import os
import stat
import psycopg2
import string
import argparse
import getpass
import os
################################################################
from random import randint, choice
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
################################################################
def generatePassword():
characters = string.ascii_letters + string.digits
password = "".join(choice(characters) for x in range(randint(8, 16)))
return password
################################################################
def createUser(user, host):
connection_params = dict()
connection_params["user"] = user
if host is not None:
connection_params["host"] = host
if host is None:
host = 'localhost'
connection_params["password"] = getpass.getpass(
'{0}@{1} password: '.format(connection_params['user'], host))
try:
connection = psycopg2.connect(**connection_params)
except Exception as e:
raise Exception(str(e)+'\n'+'*'*30 +
'\ncannot connect to database\n' + '*'*30)
new_user = input('new login: ')
connection.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
curs = connection.cursor()
try:
curs.execute('create user {0}'.format(new_user))
except Exception as e:
print(e)
print('Setting new password')
curs.execute('grant create on database blackdynamite to {0}'.format(
new_user))
password = generatePassword()
curs.execute('alter role {0} with password \'{1}\' '.format(
new_user, password))
fname = '{0}.bd'.format(new_user)
print('Saving information to {0}'.format(fname))
try:
os.remove(fname)
except Exception:
pass
bdconf = open(fname, 'w')
bdconf.write('password = {0}\n'.format(password))
bdconf.write('host = {0}'.format(host))
bdconf.close()
os.chmod(fname, stat.S_IREAD)
################################################################
parser = argparse.ArgumentParser(
description='User creation tool for blackdynamite')
parser.add_argument("--user", type=str,
help="name of the user to create",
required=True)
parser.add_argument("--host", type=str,
help="host to connect where to create the user",
default=None)
args = parser.parse_args()
args = vars(args)
new_user = args['user']
host = args['host']
createUser(new_user, host)
diff --git a/scripts/enterRun.py b/scripts/enterRun.py
index 2cc58a6..a64839c 100755
--- a/scripts/enterRun.py
+++ b/scripts/enterRun.py
@@ -1,74 +1,87 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
import BlackDynamite as BD
import subprocess
import os
import sys
import socket
################################################################
parser = BD.BDParser()
parser.register_params(group="getRunInfo",
params={"run_id": int, "order": str},
help={"run_id": "Select a run_id for switching to it"})
params = parser.parseBDParameters()
mybase = BD.Base(**params)
if 'run_id' in params:
params['run_constraints'] = ['id = {0}'.format(params['run_id'])]
try:
del params['job_constraints']
except:
pass
runSelector = BD.RunSelector(mybase)
run_list = runSelector.selectRuns(params, quiet=True)
mybase.close()
if (len(run_list) == 0):
print("no run found")
sys.exit(1)
run, job = run_list[0]
run_id = run['id']
separator = '-'*30
print(separator)
print("JOB INFO")
print(separator)
print(job)
print(separator)
print("RUN INFO")
print(separator)
print(run)
print(separator)
print("LOGGING TO '{0}'".format(run['machine_name']))
print(separator)
if run['state'] == 'CREATED':
print("Cannot enter run: not yet started")
sys.exit(-1)
bashrc_filename = os.path.join(
'/tmp', 'bashrc.user{0}.study{1}.run{2}'.format(params['user'],
params['study'],
run_id))
bashrc = open(bashrc_filename, 'w')
bashrc.write('export PS1="\\u@\\h:<{0}|RUN-{1}> $ "\n'.format(
params['study'], run_id))
bashrc.write('cd {0}\n'.format(run['run_path']))
bashrc.write('echo ' + separator)
bashrc.close()
command_login = 'bash --rcfile {0} -i'.format(bashrc_filename)
if not run['machine_name'] == socket.gethostname():
command1 = 'scp -q {0} {1}:{0}'.format(bashrc_filename,
run['machine_name'])
subprocess.call(command1, shell=True)
command_login = 'ssh -X -A -t {0} "{1}"'.format(
run['machine_name'], command_login)
# print command_login
subprocess.call(command_login, shell=True)
diff --git a/scripts/getRunInfo.py b/scripts/getRunInfo.py
index e319aa4..e43b755 100755
--- a/scripts/getRunInfo.py
+++ b/scripts/getRunInfo.py
@@ -1,242 +1,254 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
import sys
import datetime
################################################################
def printSummary(mybase, params):
runSelector = BD.RunSelector(mybase)
run_list = runSelector.selectRuns(params, quiet=True)
print("*"*6 + " run summary => study {0} ".format(mybase.schema) + "*"*6)
if len(run_list) == 0:
print("no runs found")
sys.exit(0)
request = "SELECT run_name,state,count(state) from {0}.runs ".format(
mybase.schema)
if (len(run_list) > 0):
request += "where id in (" + ",".join(
[str(r.id) for r, j in run_list]) + ")"
request += " group by state,run_name order by run_name,state"
# print (request)
curs = mybase.performRequest(request, [])
stats = {}
for i in curs:
if i[0] not in stats:
stats[i[0]] = []
stats[i[0]].append([i[1], int(i[2])])
for run_name, st in stats.items():
tot = 0
for n, count in st:
tot += count
for n, count in st:
print("{:20} {:>20} => {:5} ({:>5.1f}%)".format(
run_name, n, count, 100.*count/tot))
print("")
sys.exit(0)
################################################################
def getRunInfo(run_id, mybase):
myrun = BD.Run(mybase)
myrun["id"] = run_id
myrun.id = run_id
run_list = myrun.getMatchedObjectList()
if (len(run_list) == 0):
print("no run found with id " + str(run_id))
sys.exit(1)
myrun = run_list[0]
myjob = BD.Job(mybase)
myjob.id = myrun["job_id"]
myjob["id"] = myrun["job_id"]
job_list = myjob.getMatchedObjectList()
if (len(job_list) == 0):
print("no job found with id " + myjob.id)
sys.exit(1)
myjob = job_list[0]
list_entries = myjob.entries.keys()
print("*"*6 + " job info " + "*"*6)
for entry in list_entries:
if (myjob[entry]):
print(entry + ": " + str(myjob[entry]))
print("*"*6 + " run info " + "*"*6)
list_entries = myrun.entries.keys()
regular_run_entries = ("run_name",
"job_id",
"state",
"start_time",
"machine_name",
"exec",
"nproc",
"wait_id")
for entry in regular_run_entries:
if (myrun[entry]):
print(entry + ": " + str(myrun[entry]))
list_entries.remove(entry)
for entry in list_entries:
if (myrun[entry]):
print(entry + ": " + str(myrun[entry]))
conffiles = myrun.getConfigFiles()
for conf in conffiles:
print("*"*6)
print("file #" + str(conf.id) + ": " + conf["filename"])
print("*"*6)
print(conf["file"])
################################################################
def getInfoNames():
infos = []
infos.append("run_name")
infos.append("id")
infos.append("job_id")
if "infos" in params:
infos += params['infos']
else:
infos += ["state", "nproc", "machine_name"]
infos.append("start_time")
infos.append("last step")
infos.append("last update")
infos.append("Time/step")
infos.append("Total Time")
return infos
################################################################
def getFormatString(infos):
format_string = " {:<20} | {:^6} | {:^6} |"
if "infos" in params:
format_string += " {:^10} |" * len(params['infos'])
else:
format_string += " {:<15} | {:^5} | {:<20} |"
format_string += " {:14} | {:>9} | {:>16} | {:>10} | {:>16} |"
return format_string
################################################################
def formatTimeDelta(t):
if (t < datetime.timedelta(seconds=1)):
if (t < datetime.timedelta(microseconds=1000)):
t = str(t.microseconds) + u'\u00B5'.encode('UTF-8') + "s"
else:
t = str(1./1000.*t.microseconds) + 'ms'
else:
ms = t.microseconds
t -= datetime.timedelta(microseconds=ms)
t = str(t)
return t
################################################################
def getTimeInfos(r):
step, steptime = r.getLastStep()
start_time = r['start_time']
time_perstep = None
total_time = None
if (step is not None and steptime and start_time):
time_perstep = (steptime-start_time)/(step+1)
total_time = steptime-start_time
time_perstep = formatTimeDelta(time_perstep)
total_time = formatTimeDelta(total_time)
if start_time:
start_time = start_time.strftime("%H:%M %d/%m/%y")
if steptime:
steptime = steptime.strftime("%H:%M %d/%m/%y")
run_infos = [start_time, step, steptime, time_perstep, total_time]
return run_infos
################################################################
def getRunInfos(r, j):
run_infos = []
for col in info_names[:-5]:
key_run = col.replace('%r.', '').strip()
if not key_run == 'start_time':
if key_run in r.entries:
run_infos.append(r[key_run])
else:
key_job = col.replace('%j.', '').strip()
if key_job in j.entries:
run_infos.append(j[key_job])
else:
raise Exception('Key {0} is not a valid parameter'.format(
key_run))
run_infos += getTimeInfos(r)
return run_infos
################################################################
parser = BD.BDParser()
parser.register_params(
group="getRunInfo",
params={"run_id": int, "order": str,
"summary": bool,
"infos": [str]},
defaults={"order": "id"},
help={"run_id": "Select a run_id for complete output",
"summary": "Output a summary of the completeness of the study",
"order": "specify the column which serves to order the lines"})
params = parser.parseBDParameters()
mybase = BD.Base(**params)
if params["summary"] is True:
printSummary(mybase, params)
if ("run_id" in params):
getRunInfo(params["run_id"], mybase)
else:
info_names = getInfoNames()
format_string = getFormatString(info_names)
header = format_string.format(*info_names)
separator = "-" * len(header)
print(separator)
print(header)
print(separator)
runSelector = BD.RunSelector(mybase)
run_list = runSelector.selectRuns(params,
sort_by="runs." + params["order"],
quiet=True)
for r, j in run_list:
try:
infos = getRunInfos(r, j)
def transform_None(x):
if x is None:
return 'None'
else:
return x
infos = [transform_None(x) for x in infos]
line = format_string.format(*infos)
print(line)
except Exception as e:
print(getRunInfos(r, j))
print(e)
diff --git a/scripts/launchRuns.py b/scripts/launchRuns.py
index 015f70e..f0ce038 100755
--- a/scripts/launchRuns.py
+++ b/scripts/launchRuns.py
@@ -1,105 +1,118 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
import BlackDynamite as BD
import os
import sys
import socket
from types import ModuleType
################################################################
def main(argv=None):
if (type(argv) == str):
argv = argv.split()
parser = BD.BDParser()
parser.register_params(
group="launchRuns.py",
params={
"outpath": str,
"generator": ModuleType,
"nruns": int,
"state": str,
"machine_name": str},
defaults={
"machine_name": socket.gethostname(),
"nruns": -1,
"generator": "bashCoat"})
parser.help.update({
"nruns": ('Specify the number of runs to launch. '
'This is useful when we want to launch '
'only the first run from the stack.'),
"generator": "Specify the launcher generator"
})
params = parser.parseBDParameters(argv=argv)
mybase = BD.Base(**params)
if ("outpath" not in params):
print('A directory where to create temp files '
'should be provided. use --outpath ')
sys.exit(-1)
mydir = os.path.join(params["outpath"], "BD-" + params["study"] + "-runs")
if not os.path.exists(mydir):
os.makedirs(mydir)
os.chdir(mydir)
runSelector = BD.RunSelector(mybase)
constraints = []
if ("constraints" in params):
constraints = params["constraints"]
def item_matcher(name, item):
return item.lower().lstrip().startswith(name)
if not any([item_matcher("state", item) for item in constraints]):
constraints.append("state = CREATED")
if not any([item_matcher("machine_name", item)
for item in constraints]):
constraints.append("machine_name = {0}".format(
params["machine_name"]))
run_list = runSelector.selectRuns(constraints)
if (params["nruns"] > 0):
run_list = [run_list[i]
for i in range(0, min(params["nruns"],
len(run_list)))]
if (len(run_list) == 0):
print("No runs to be launched")
for r, j in run_list:
print("Dealing with job {0.id}, run {1.id}".format(j, r))
r["run_path"] = os.path.join(mydir, "run-" + str(r.id))
print(j.types)
j.update()
if not os.path.exists("run-" + str(r.id)):
os.makedirs("run-" + str(r.id))
os.chdir("run-" + str(r.id))
conffiles = r.getConfigFiles()
for conf in conffiles:
print("create file " + conf["filename"])
f = open(conf["filename"], 'w')
f.write(conf["file"])
f.close()
print("launch in '" + mydir + "/" + "run-" + str(r.id) + "/'")
mymod = params["generator"]
print(mymod)
mymod.launch(r, params)
os.chdir("../")
if (params["truerun"] is True):
mybase.commit()
if __name__ == '__main__':
main()
diff --git a/scripts/mvRuns.py b/scripts/mvRuns.py
index af5992d..011a028 100755
--- a/scripts/mvRuns.py
+++ b/scripts/mvRuns.py
@@ -1,90 +1,103 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
################################################################
from __future__ import print_function
################################################################
import BlackDynamite as BD
import os
import subprocess
import socket
################################################################
def move_run(r,
machine_src, machine_dst,
path_src, path_dst):
if path_src == path_dst and machine_dst == machine_src:
return
if not os.path.exists(path_dst):
os.makedirs(path_dst)
# print(machine_src, machine_dst)
# print(path_src, path_dst)
url_src = machine_src + ':' + path_src
if url_src[-1] == '/':
url_src = url_src[:-1]
print('mv run,job:', r.id, j.id)
url_dst = path_dst
if machine_src == machine_dst:
url_src = path_src
else:
url_src = machine_src + ':' + path_src
if url_src[-1] == '/':
url_src = url_src[:-1]
if url_dst[-1] == '/':
url_dst = url_dst[:-1]
rsync_command = 'rsync --remove-source-files -auP {0} {1}'.format(
url_src, url_dst)
print(rsync_command)
if params['truerun'] is True:
ret = subprocess.call(rsync_command, shell=True)
else:
ret = True
if ret:
return
r['run_path'] = path_dst
r['machine_name'] = machine_dst
r.update()
if params['truerun'] is True:
mybase.commit()
################################################################
parser = BD.BDParser()
parser.register_params(
group="mvRuns",
params={"path": str},
mandatory={'path': True},
help={"path": "Path to the local machine where to store the run outputs"})
params = parser.parseBDParameters()
mybase = BD.Base(**params)
runSelector = BD.RunSelector(mybase)
run_list = runSelector.selectRuns(params, params, quiet=True)
for r, j in run_list:
machine_dst = socket.gethostname()
machine_src = r['machine_name']
path_src = r['run_path']
if path_src is None:
continue
f, p = os.path.split(path_src)
while p == '':
f, p = os.path.split(f)
run_subdir = p
path_dst = os.path.join(params['path'],
"BD-" + params["study"] + "-runs",
run_subdir)
move_run(r, machine_src, machine_dst, path_src, path_dst)
diff --git a/scripts/pushQuantity.py b/scripts/pushQuantity.py
index a01994d..c5e4d58 100755
--- a/scripts/pushQuantity.py
+++ b/scripts/pushQuantity.py
@@ -1,69 +1,81 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
def main(argv=None):
parser = BD.BDParser()
parser.register_params(
group="pushQuantity",
params={"runid": int,
"quantity_id": str,
"value": str,
"values": [str],
"step": int,
"is_float": bool},
defaults={"step": 0},
help={"runid": "The id of the run to update",
"quantity_id": "ID of the Quantity to push",
"value": "Value tu push for the quantity",
"values": "Vectorial value tu push for the quantity",
"step": "Step at which the data is generated",
"is_float": "0 if the quantity is a float 1 other why"}
)
params = parser.parseBDParameters()
if("runid" not in params):
raise Exception("The run id should be set")
if("quantity_id" not in params):
raise Exception("The quantity id should be set")
if("value" not in params and "values" not in params):
raise Exception("The value should be set")
is_vector = False
if("values" in params):
is_vector = True
if("value" in params):
raise Exception(
"You cannot define values and value at the same time")
base = BD.Base(**params)
if ("runid" in params):
if "run_constraints" not in params:
params["run_constraints"] = []
params["run_constraints"].append("id = " + str(params["runid"]))
runSelector = BD.RunSelector(base)
run_list = runSelector.selectRuns(params, params)
if (not len(run_list) == 1):
raise Exception("No or too many runs selected")
r, j = run_list[0]
if params["truerun"] is True:
if is_vector is False:
r.pushScalarQuantity(
params["value"], params["step"], params["quantity_id"],
params["is_float"] is False)
else:
r.pushVectorQuantity(
params["values"], params["step"], params["quantity_id"],
params["is_float"] is False)
base.commit()
if __name__ == '__main__':
main()
diff --git a/scripts/saveBDStudy.py b/scripts/saveBDStudy.py
index a726644..78c0244 100755
--- a/scripts/saveBDStudy.py
+++ b/scripts/saveBDStudy.py
@@ -1,61 +1,73 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
import os
import sys
import subprocess
def saveSchema(params):
if ("out_file" not in params):
out_file = os.path.join("./", params["study"] + ".db")
else:
out_file = params["out_file"]
r, ext = os.path.splitext(out_file)
if (not ext == ".gz"):
out_file += ".gz"
print("Saving study " + params["study"] + " to file " + out_file)
command = ("pg_dump --host " + params["host"] + " --schema=" +
params["study"] + " -C -f " + out_file + " --compress=9")
if params["verbose"] is True:
command += " --verbose"
command = command.strip().split(" ")
ret = subprocess.call(command)
if not ret == 0:
sys.exit("pg_dump error")
def main(argv=None):
if isinstance(argv, str):
argv = argv.split()
parser = BD.BDParser()
parser.register_params(
group="saveBDStudy.py",
params={"out_file": str, "verbose": bool, "study": str},
help={"out_file": "Specify the filename where to save the study",
"verbose": "Activate the verbose mode of pg_dump",
"study": "specify the study to backup. \
If none provided all studies are backed up"})
params = parser.parseBDParameters(argv=argv)
params["should_not_check_study"] = True
mybase = BD.Base(**params)
if ("study" in params):
saveSchema(params)
else:
if "out_file" in params:
del params["out_file"]
sch_list = mybase.getSchemaList()
for s in sch_list:
params["study"] = s
saveSchema(params)
if __name__ == '__main__':
main()
diff --git a/scripts/studyInfo.py b/scripts/studyInfo.py
index a7cc71b..7b33d27 100755
--- a/scripts/studyInfo.py
+++ b/scripts/studyInfo.py
@@ -1,92 +1,104 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
from __future__ import print_function
################################################################
import BlackDynamite as BD
################################################################
def printDataBaseInfo(base, params):
curs = base.connection.cursor()
curs.execute("select current_database()")
datname = curs.fetchone()[0]
curs.execute("""
select d.datname,
pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname))
from pg_catalog.pg_database d where (d.datname) = ('{0}')
""".format(datname))
datsize = curs.fetchone()[1]
print("Database:", datname, datsize)
def printUserInfo(base, params):
users = base.getUserList()
print('registered Users: {0}'.format(', '.join(users)))
def printStudyInfo(base, study, params):
study_size = base.getStudySize(study)
owner = base.getStudyOwner(study)
granted_users = base.getGrantedUsers(study)
print('{0}'.format(study))
print(' Owner:{:>30} '.format(owner))
print(' #jobs:{:>30} '.format(study_size['njobs']))
print(' #runs:{:>30} '.format(study_size['nruns']))
print(' size: {:>30}'.format(study_size['size']))
print(' grants: {0}'.format(','.join(granted_users)))
def fetchInfo(base, params):
printUserInfo(base, params)
if "study" not in params:
study_list = base.getSchemaList(filter_names=False)
else:
study_list = [base.schema]
printDataBaseInfo(base, params)
for s in study_list:
printStudyInfo(base, s, params)
def fetchStudy(base, params):
runSelector = BD.RunSelector(base)
run_list = runSelector.selectRuns(params, params, quiet=True)
print(run_list)
def main(argv=None):
if (type(argv) == str):
argv = argv.split()
parser = BD.BDParser()
parser.register_params(
group="studyInfo.py",
params={"full": bool, "study": str,
"grant": str, "revoke": str,
"fetch": bool},
help={"full": "Say that you want details (can be costful)",
"study": "specify a study to analyse",
"grant": "specify an user to grant read permission",
"revoke": "specify an user to revoke read permission",
"fetch": "fetch the specified study as one of yours"})
params = parser.parseBDParameters(argv=argv)
params["should_not_check_study"] = True
mybase = BD.Base(**params)
if 'grant' in params:
mybase.grantAccess(params['study'], params['grant'])
if 'revoke' in params:
mybase.revokeAccess(params['study'], params['revoke'])
if params['fetch'] is True:
fetchStudy(mybase, params)
fetchInfo(mybase, params)
if __name__ == '__main__':
main()
diff --git a/scripts/updateRuns.py b/scripts/updateRuns.py
index 1a0c40f..ec904cf 100755
--- a/scripts/updateRuns.py
+++ b/scripts/updateRuns.py
@@ -1,42 +1,54 @@
#!/usr/bin/env python3
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
import BlackDynamite as BD
import os
import sys
import socket
parser = BD.BDParser()
parser.register_params(
group="updateRuns",
params={"run_id": int,
"updates": [str]},
defaults={"machine_name": socket.gethostname()},
help={"run_id": "The id of the run to update",
"updates": "The updates to perform. Syntax should be 'key = newval'"}
)
params = parser.parseBDParameters()
if "user" not in params.keys():
params["user"] = os.getlogin()
base = BD.Base(**params)
if "run_id" in params:
if "constraints" not in params:
params["constraints"] = []
params["constraints"].append("runs.id = " + str(params["run_id"]))
runSelector = BD.RunSelector(base)
run_list = runSelector.selectRuns(params)
if len(run_list) == 0:
print("No runs to be updated")
if "updates" not in params:
print("No update to be performed: use --updates option")
sys.exit(-1)
for r, j in run_list:
r.setFields(params["updates"])
if params["truerun"] is True:
r.update()
base.commit()
diff --git a/setup.py b/setup.py
index 5f63dda..cb7c0e4 100644
--- a/setup.py
+++ b/setup.py
@@ -1,39 +1,52 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
from setuptools import setup
setup(name="blackdynamite",
packages=['BlackDynamite', 'BlackDynamite.coating'],
version="0.0.1",
author="Guillaume Anciaux",
author_email="guillaume.anciaux@epfl.ch",
description=("Compliant parametric study tool"),
package_data={
'BlackDynamite': ['build_tables.sql']},
scripts=['scripts/canYouDigIt.py',
'scripts/cleanRuns.py',
'scripts/createUser.py',
'scripts/enterRun.py',
'scripts/getRunInfo.py',
'scripts/launchRuns.py',
'scripts/mvRuns.py',
'scripts/pushQuantity.py',
'scripts/saveBDStudy.py',
'scripts/updateRuns.py',
'scripts/studyInfo.py',
'scripts/bash_completion.sh'],
install_requires=["psycopg2-binary", "numpy",
"argcomplete", "pyparsing"],
test_suite="pythontests",
license="""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see .
""")