diff --git a/BlackDynamite/BDstat.py b/BlackDynamite/BDstat.py index 9ed56bf..e005a35 100644 --- a/BlackDynamite/BDstat.py +++ b/BlackDynamite/BDstat.py @@ -1,151 +1,163 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function from . import job import numpy as np class BDStat(object): """ """ def average(self, quantity, run_list, entries_to_average): result = dict() run_info = dict() cpt = dict() run_counter = 0 nb_runs = len(run_list) entries_to_average.append("id") entries_to_consider = [] if (run_list is not []): for e in run_list[0][1].entries: if e not in entries_to_average: entries_to_consider.append(e) entries_to_consider = sorted(entries_to_consider) # print (entries_to_consider) for r, j in run_list: q = r.getScalarQuantity(quantity) run_counter += 1 if (q is None): continue print("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format( r.id, q.shape, run_counter, nb_runs, 1.*run_counter/nb_runs*100)) ent = [j[i] for i in entries_to_consider] ent = tuple(ent) if (ent not in result.keys()): # print (ent) result[ent] = np.zeros([q.shape[0], 3]) cpt[ent] = 0 # print (result[ent].shape) # print (q.shape) sz1 = q.shape[0] sz2 = result[ent].shape[0] if (sz1 > sz2): q = q[:sz2] elif (sz2 > sz1): result[ent] = result[ent][:sz1] result[ent][:, 0] += q[:, 0] result[ent][:, 1] += q[:, 1] result[ent][:, 2] += q[:, 1]**2 cpt[ent] += 1 for ent in result.keys(): # print (ent) myjob = job.Job(self.base) for i in range(0, len(entries_to_consider)): myjob.entries[entries_to_consider[i]] = ent[i] result[ent] /= cpt[ent] result[ent][:, 2] -= result[ent][:, 1]**2 result[ent][:, 2] = np.maximum( np.zeros(result[ent][:, 2].shape[0]), result[ent][:, 2]) result[ent][:, 2] = np.sqrt(result[ent][:, 2]) result[ent] = {"ref_job": myjob, "averaged_number": cpt[ent], "data": result[ent] } return result def averageVector(self, quantity, run_list, entries_to_average): result = dict() all_steps = dict() run_info = dict() cpt = dict() run_counter = 0 nb_runs = len(run_list) entries_to_average.append("id") entries_to_consider = [] if (run_list is not []): for e in run_list[0][1].entries: if e not in entries_to_average: entries_to_consider.append(e) for r, j in run_list: steps, data = r.getAllVectorQuantity(quantity) run_counter += 1 if (data is None): continue print("{0:<5} {1:<15} {2:>5}/{3:<5} {4:.1f}%".format( r.id, data.shape, run_counter, nb_runs, 1.*run_counter/nb_runs*100)) ent = [j[i] for i in entries_to_consider] ent = tuple(ent) if (ent not in result.keys()): # print (ent) result[ent] = np.zeros([data.shape[0], data.shape[1], 2]) all_steps[ent] = steps cpt[ent] = 0 # print (result[ent].shape) # print (q.shape) sz1 = data.shape[0] sz2 = result[ent].shape[0] if (sz1 > sz2): data = data[:sz2] steps = steps[:sz2] elif (sz2 > sz1): result[ent] = result[ent][:sz1] all_steps[ent] = all_steps[ent][:sz1] result[ent][:, :, 0] += data[:, :] result[ent][:, :, 1] += data[:, :]**2 cpt[ent] += 1 for ent in result.keys(): # print (ent) myjob = job.Job(self.base) for i in range(0, len(entries_to_consider)): myjob.entries[entries_to_consider[i]] = ent[i] result[ent] /= cpt[ent] result[ent][:, :, 1] -= result[ent][:, :, 0]**2 result[ent][:, :, 1] = np.sqrt(result[ent][:, :, 1]) result[ent] = {"ref_job": myjob, "averaged_number": cpt[ent], "steps": all_steps[ent], "data": result[ent]} return result def __init__(self, base, **params): self.base = base ################################################################ __all__ = ["BDStat"] diff --git a/BlackDynamite/__init__.py b/BlackDynamite/__init__.py index a665e6d..a46bde6 100644 --- a/BlackDynamite/__init__.py +++ b/BlackDynamite/__init__.py @@ -1,38 +1,50 @@ #!/usr/bin/env python3 # flake8: noqa +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from . import bdparser from . import run from . import job from . import base from . import runselector from . import jobselector from . import conffile from . import BDstat from .bdparser import * from .run import * from .job import * from .base import * from .runselector import * from .jobselector import * from .conffile import * from .BDstat import * __all__ = ["run", "job", "base"] __all__.extend(bdparser.__all__) __all__.extend(run.__all__) __all__.extend(job.__all__) __all__.extend(base.__all__) __all__.extend(runselector.__all__) __all__.extend(jobselector.__all__) __all__.extend(conffile.__all__) __all__.extend(BDstat.__all__) # try: # import graphhelper # __all__.append("graphhelper") # from graphhelper import * # __all__.extend(graphhelper.__all__) # except: # print "graphhelper not loaded" diff --git a/BlackDynamite/bdlogging.py b/BlackDynamite/bdlogging.py index d00616e..09c5d33 100644 --- a/BlackDynamite/bdlogging.py +++ b/BlackDynamite/bdlogging.py @@ -1,84 +1,96 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function import logging import traceback import os import sys from . import __name__ as global_name # Base level logger root_logger = logging.getLogger(global_name) root_logger.setLevel(logging.DEBUG) # Avoid hard-filtering # Logging format BD_FORMAT = "%(levelname)s:%(funcName)s [%(filename)s:%(lineno)d]: %(message)s" sh = logging.StreamHandler(sys.stderr) sh.setLevel(logging.WARNING) # Only show warnings to screen sh.setFormatter(logging.Formatter(BD_FORMAT)) root_logger.addHandler(sh) class ExtraContext: """Adds some context to logging""" _wire = { 'foo': lambda x: x.foo(), 'l': lambda x: x.lololo(), 'f': lambda x: x.fname() } @staticmethod def getTr(): return traceback.extract_stack(limit=20)[11] def fname(self): return os.path.basename(self.getTr()[0]) def lololo(self): return self.getTr()[1] def foo(self): return self.getTr()[2] def __getitem__(self, name): return self._wire[name](self) def __iter__(self): d = {k: self._wire[k](self) for k in self._wire} return iter(d) def invalidPrint(x): raise Exception('print should not be used in that class: ' 'use the logging system instead: "{0}"'.format(x)) def activateFileLogging(): """Activate logging to file (if not already enabled)""" # formatter = logging.Formatter(fmt='%(levelname)s:%(foo)50s:%(f)15s:%(l)s:' # + ' '*10 + '%(message)s') formatter = logging.Formatter(BD_FORMAT) # Handler for file bd_file_handler = logging.FileHandler('bd.log', mode='a+') bd_file_handler.setFormatter(formatter) bd_file_handler.setLevel(logging.DEBUG) # Log everything to file if '_has_file_handler' not in globals() \ or not globals()['_has_file_handler']: logger = logging.getLogger(global_name) logger.debug("Activating logging to file") logger.addHandler(bd_file_handler) # This should be the first line logged in file logger.debug("Activated logging to file") globals()['_has_file_handler'] = True def getLogger(name): logger = logging.getLogger(name) logger.propagate = True return logger diff --git a/BlackDynamite/bdparser.py b/BlackDynamite/bdparser.py index f7d8753..e673e1b 100755 --- a/BlackDynamite/bdparser.py +++ b/BlackDynamite/bdparser.py @@ -1,658 +1,670 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function from . import __path__ as BD_path from . import base from . import run from . import bdlogging import sys import re import os import stat import pwd import argcomplete import argparse from argcomplete.completers import EnvironCompleter import traceback from types import ModuleType import imp ################################################################ print = bdlogging.invalidPrint logger = bdlogging.getLogger(__name__) ################################################################ class BDParser(object): def listPossibleHosts(self): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir, 'hosts') hosts = [] try: hosts += [h.strip() for h in open(bd_hosts)] except Exception: pass return hosts def listPossibleModules(self, pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args): paths += pre_args["module_path"].split(':') paths += BD_path paths += [path + "/coating" for path in BD_path] module_list = [] paths = [p.strip() for p in paths if not p.strip() == ''] for p in paths: files = os.listdir(p) files = [f for f in files if os.path.splitext(f)[1] == '.py'] files = [f for f in files if not f[0] == '_'] matching_string = ".*blackdynamite.*" files = [os.path.splitext(f)[0] for f in files if re.match( matching_string, open(os.path.join(p, f)).read().replace('\n', ' '), flags=re.IGNORECASE)] module_list += files logger.debug("found these files " + str(module_list)) return module_list def updatePossibleHosts(self, new_host): logger.debug("in") bd_dir = os.path.expanduser("~/.blackdynamite") bd_hosts = os.path.join(bd_dir, 'hosts') hosts = set(self.listPossibleHosts()) hosts.add(new_host) f = open(bd_hosts, 'w') for h in hosts: f.write(h+'\n') def completer(self, prefix, **kwargs): # bdlogging.activateFileLogging() try: params = vars(kwargs["parsed_args"]) if params['logging'] is True: bdlogging.activateFileLogging() logger.debug("in") logger.debug("BDparser prefix " + str(prefix) + "\n") for k, v in kwargs.items(): logger.debug("kwargs[" + str(k) + "] = " + str(v) + "\n") logger.debug("dest " + str(vars(kwargs["action"])["dest"]) + "\n") for k in params.keys(): if params[k] is None: del params[k] key = vars(kwargs["action"])["dest"] logger.debug("key " + str(key) + "\n") if (key == "BDconf"): return self.listPossibleConf() if 'BDconf' in params: self.readConfFiles(params, params['BDconf']) if 'host' in params: self.readConfFile(params, params['host']+'.bd') logger.debug("key " + str(key) + "\n") for k in params.keys(): logger.debug("params = " + str(k)) if (key == "host"): return self.listPossibleHosts() if (key == "study"): params["should_not_check_study"] = True mybase = base.Base(**params) return mybase.getSchemaList() if (key == 'quantity'): mybase = base.Base(**params) myrun = run.Run(mybase) return myrun.listQuantities() if (key in self.admissible_params and self.admissible_params[key] == ModuleType): logger.debug( "trying to complete module list for '{}'".format(key)) return self.listPossibleModules(params) except Exception as e: logger.debug(traceback.format_exc()) logger.debug(str(e)) return [] def listPossibleConf(self): logger.debug("in") files = [] for dir in ["./", os.path.expanduser("~/.blackdynamite")]: for filename in os.listdir(dir): fileName, fileExtension = os.path.splitext(filename) if (fileExtension == ".bd"): files.append(filename) return files return files def readConfFiles(self, read_params, fnames): logger.debug("in") for f in fnames: self.readConfFile(read_params, f) def readConfFile(self, read_params, fname): logger.debug("in") logger.debug("readConfFileList {0}".format(self.readConfFileList)) if fname in self.readConfFileList: return self.readConfFileList.append(fname) if type(fname) == list: raise Exception( 'cannot use list in that function: ' + str(type(fname))) pre_args = {} for dir in ["./", os.path.expanduser("~/.blackdynamite")]: fullpath = os.path.join(dir, fname) if (os.path.isfile(fullpath)): fname = fullpath break try: with open(fname) as fh: logger.debug("loading file '{0}'".format(fname)) os.chmod(fname, stat.S_IREAD | stat.S_IWRITE) lines = [line.strip() for line in fh] regex = "(.*)=(.*)" for line in lines: match = re.match(regex, line) if (not match): print("malformed line:" + line) sys.exit(-1) param = match.group(1).strip() val = match.group(2).strip() pre_args[param] = val self.argv.append("--"+param) self.argv.append(val) logger.debug("read parameters: '{0}'".format(self.argv)) logger.debug("pre args : '{0}'".format(pre_args)) read_params.update(self.createParamsMap(pre_args)) except Exception as e: logger.debug("cannot open file " + fname + '\n' + str(e) + '\n' + str(traceback.format_exc())) logger.debug("out") def checkParam(self, p, dico): logger.debug("in") print("****************") print("Obsolete: should use the mandatory argument" " for the declare_params function") print("It was used by object " + str(self) + " for keyword " + p) print("FATAL => ABORT") print("****************") sys.exit(-1) def loadModule(self, read_params, myscript, pre_args): logger.debug("in") paths = [] if ("PYTHONPATH" in os.environ): paths = os.environ["PYTHONPATH"].split(':') if ("module_path" in pre_args): paths += pre_args["module_path"].split(':') paths += BD_path paths += [path + "/coating" for path in BD_path] mymod = None for p in paths: try: modfile = os.path.join(p, myscript+".py") # print("loading file " + modfile) mymod = imp.load_source(myscript, modfile) break except IOError as io_err: logger.debug("loadModule " + str(io_err)) logger.debug("loadModule " + str(mymod)) if (mymod is None): logger.debug("cannot find module '" + myscript + "' from paths " + str(paths)) logger.debug("trace :" + traceback.format_exc() + '\n') raise Exception("cannot find module '" + myscript + "' from paths " + str(paths)) return mymod def createParamsMap(self, pre_args): logger.debug("in") read_params = {} if ('logging' in pre_args) and (pre_args['logging'] is True): bdlogging.activateFileLogging() for opt, args in pre_args.items(): logger.debug("createParamsMap1 " + str(opt) + " : " + str(args)) if (args is None): continue if (not type(args) == list): args = [args] if (type(args) == str): args = [args] logger.debug("createParamsMap2 " + str(opt) + " : " + str(args)) for arg in args: if (arg is None): continue if (opt == 'BDconf'): if (not type(arg) == list): arg = [arg] self.readConfFiles(read_params, arg) continue if (opt == 'host'): self.updatePossibleHosts(arg) self.readConfFile(read_params, arg+'.bd') for param, typ in self.admissible_params.items(): if opt == param: logger.debug("createParamsMap3 " + str(param) + " : " + str(typ)) if (typ == ModuleType): read_params[param] = self.loadModule( read_params, arg, pre_args) logger.debug("createParamsMap4 " + str(param) + " : " + str(typ)) elif (type(typ) == list): args = arg.split(",") if (param not in read_params): read_params[param] = [] if (not len(typ) == 1): subtype = str else: subtype = typ[0] for i in range(0, len(args)): read_params[param].append(subtype(args[i])) else: read_params[param] = typ(arg) break return read_params def addModulesAdmissibleParameters(self, read_params): logger.debug("in") for k, v in read_params.items(): if self.admissible_params[k] == ModuleType: mymod = read_params[k] modname = mymod.__name__ if "admissible_params" in mymod.__dict__: self.admissible_params.update( mymod.__dict__["admissible_params"]) self.group_params["'" + modname + "' module options"] = ( mymod.__dict__["admissible_params"].keys()) if "default_params" in mymod.__dict__: self.default_params.update( mymod.__dict__["default_params"]) if "help" in mymod.__dict__: self.help.update(mymod.__dict__["help"]) if "mandatory" in mymod.__dict__: self.mandatory.update(mymod.__dict__["mandatory"]) logger.debug(str(self.admissible_params)) def addModulesAdmissibleParametersForComplete(self, read_params): logger.debug("in") if "_ARGCOMPLETE" not in os.environ: return logger.debug("arg complete ? " + os.environ["_ARGCOMPLETE"]) # breaks = os.environ["COMP_WORDBREAKS"] tmp_read_params = {} breaks = " |=|&|<|>|;" logger.debug("break line " + os.environ["COMP_LINE"]) all_args = re.split(breaks, os.environ["COMP_LINE"]) logger.debug("break line " + str(all_args)) for i in range(0, len(all_args)): a = all_args[i] res = re.match("--(.*)", a) if res is None: continue a = res.group(1) logger.debug("treating a " + str(a)) if a in self.admissible_params: if self.admissible_params[a] == ModuleType: logger.debug("here treating a " + str(a)) if i+1 >= len(all_args): continue b = all_args[i+1] logger.debug("treating b " + str(b)) res = re.match("--(.*)", b) if res is not None: continue if not b.strip() == '': tmp_read_params[a] = b if ("module_path" in read_params): tmp_read_params["module_path"] = read_params["module_path"] logger.debug("tmp_read_params " + str(tmp_read_params)) try: tmp_read_params = self.createParamsMap(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc() + '\n' + str(e)) logger.debug("AAAAAAAAA " + str(tmp_read_params)) try: self.addModulesAdmissibleParameters(tmp_read_params) except Exception as e: logger.debug("trace :" + traceback.format_exc() + '\n' + str(e)) logger.debug("CCCCCCCCCC" + str(self.admissible_params)) def constructArgParser(self, add_help=True, add_mandatory=True): logger.debug("in") parser = argparse.ArgumentParser( description="BlackDynamite option parser", formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=add_help) self.params_group = {} group = parser.add_argument_group("General") self.params_group["General"] = group for g, param_list in self.group_params.items(): group = parser.add_argument_group(g) for p in param_list: self.params_group[p] = group for param, typ in self.admissible_params.items(): # print("param {0}: {1}".format(param,typ) ) p_help = "help TODO" is_mandatory = (param in self.mandatory.keys() and self.mandatory[param] is True and add_mandatory) if (param in self.help): p_help = self.help[param] if (param in self.params_group): grp = self.params_group[param] else: grp = self.params_group["General"] if (typ is None): raise Exception("Deprectated option type for " + param + " : should be changed to 'bool'") if (typ is bool): if (param in self.default_params and self.default_params[param] is True): grp.add_argument("--" + param, help=p_help, dest=param, action='store_false', required=is_mandatory) else: grp.add_argument("--" + param, help=p_help, dest=param, action='store_true', required=is_mandatory) elif (typ is list or typ == [str]): grp.add_argument( "--" + param, action='append', dest=param, help=p_help, required=is_mandatory).completer = self.completer else: grp.add_argument( "--" + param, dest=param, help=p_help, required=is_mandatory).completer = self.completer parser.set_defaults(**self.default_params) return parser def register_params(self, group="General", params=None, defaults=None, help=None, mandatory=None): logger.debug("in") if (params is not None): self.admissible_params.update(params) if group not in self.group_params: self.group_params[group] = [] self.group_params[group] += params.keys() if (defaults is not None): self.default_params.update(defaults) for key in defaults.keys(): self.mandatory[key] = False if (help is not None): self.help.update(help) if (mandatory is not None): self.mandatory.update(mandatory) for param, typ in self.admissible_params.items(): if typ == bool and param not in self.default_params: self.default_params[param] = False def addEnvBDArguments(self, parser): logger.debug("in") parser = self.constructArgParser(add_help=False, add_mandatory=False) pre_args = vars(parser.parse_known_args(args=self.argv)[0]) for name, value in os.environ.items(): m = re.match("BLACKDYNAMITE_(.*)", name) if (m): var = m.group(1).lower() if (var not in pre_args or pre_args[var] is None): if (var in self.admissible_params): self.argv.append("--" + var) self.argv.append(value) def parseBDParameters(self, argv=None): logger.debug("in") if argv is None: self.argv = list(sys.argv[1:]) else: self.argv = list(argv) logger.debug("program called with " + str(len(self.argv)) + " args " + str(self.argv) + "\n") logger.debug("env is\n\n") for k, v in os.environ.items(): logger.debug("export " + k + "='" + v + "'\n") logger.debug("constructArgParser\n") parser = self.constructArgParser(add_help=False, add_mandatory=False) self.addEnvBDArguments(parser) logger.debug("parse_known_args\n") pre_args = parser.parse_known_args(args=self.argv)[0] logger.debug("createParamsMap\n") read_params = self.createParamsMap(vars(pre_args)) logger.debug("addModuleAdmissibleParameters\n") self.addModulesAdmissibleParameters(read_params) logger.debug("addModulesAdmissibleParametersForComplete\n") try: self.addModulesAdmissibleParametersForComplete(read_params) except KeyError as e: logger.debug("trace :" + traceback.format_exc()) logger.debug("constructArgParser\n") parser = self.constructArgParser() argcomplete.autocomplete(parser) pre_args = parser.parse_args(args=self.argv) read_params = self.createParamsMap(vars(pre_args)) if "user" not in read_params: read_params["user"] = pwd.getpwuid(os.getuid())[0] return read_params def __init__(self): logger.debug("in") self.admissible_params = {} self.help = {} self.default_params = {} self.group_params = {} self.mandatory = {} self.admissible_params["study"] = str self.help["study"] = ( "Specify the study from the BlackDynamite database" ". This refers to the schemas in PostgreSQL language") self.admissible_params["host"] = str self.help["host"] = "Specify data base server address" self.admissible_params["port"] = int self.help["port"] = "Specify data base server port" self.admissible_params["user"] = str self.help["user"] = "Specify user name to connect to data base server" self.admissible_params["password"] = str self.help["password"] = "Provides the password" self.admissible_params["BDconf"] = list self.help["BDconf"] = ( "Path to a BlackDynamite file (*.bd) configuring current optons") self.admissible_params["truerun"] = bool self.help["truerun"] = ( "Set this flag if you want to truly perform the" " action on base. If not set all action are mainly dryrun") self.default_params["truerun"] = False self.admissible_params["constraints"] = [str] self.help["constraints"] = ( "This allows to constraint run/job selections by properties") self.default_params["constraints"] = None self.admissible_params["binary_operator"] = str self.default_params["binary_operator"] = 'and' self.help["binary_operator"] = ( 'Set the default binary operator to make requests to database') self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = ( "Request to list the possible job/run parameters") self.admissible_params["yes"] = bool self.default_params["yes"] = False self.help["yes"] = "Answer all questions to yes." self.admissible_params["logging"] = bool self.help["logging"] = "Activate the file logging system" self.group_params["BDParser"] = ["study", "host", "port", "user", "password", "BDconf", "truerun", "constraints", "job_constraints", "run_constraints", "list_parameters"] self.readConfFileList = [] ################################################################ def validate_question(question, params, default_validated=True): logger.debug("in") if (default_validated): default_str = "(Y/n)" else: default_str = "(y/N)" if params["yes"] is False: validated = input("{0}? {1} ".format(question, default_str)) # print (validated) if (validated == "\n" or validated == ""): validated = default_validated elif(validated == "Y" or validated == "y"): validated = True else: validated = False else: logger.info("{0}? {1} Forced Y".format(question, default_str)) validated = True return validated ################################################################ def filterParams(sub_list, total_list): logger.debug("in") new_list = {} for p in sub_list: if (p in total_list and total_list[p] is not False): new_list[p] = total_list[p] return new_list ################################################################ class RunParser(BDParser): """ """ def parseBDParameters(self): logger.debug("in") params = BDParser.parseBDParameters(self) params['run_name'], nb_subs = re.subn('\s', '_', params['run_name']) return params def __init__(self): logger.debug("in") BDParser.__init__(self) self.mandatory["machine_name"] = True self.mandatory["nproc"] = True self.mandatory["run_name"] = True self.admissible_params["machine_name"] = str self.help["machine_name"] = ("Specify the name of the machine where" " the job is to be launched") self.admissible_params["nproc"] = int self.help["nproc"] = ("Specify the number of processors onto which" " this run is supposed to be launched") self.admissible_params["run_name"] = str self.help["run_name"] = ( 'User friendly name given to this run.' ' This is usually helpful to recall a run kind') self.default_params = {} self.default_params["job_constraints"] = None self.group_params["RunParser"] = [ "machine_name", "nproc", "run_name"] ################################################################ __all__ = ["BDParser", "RunParser"] diff --git a/BlackDynamite/coating/__init__.py b/BlackDynamite/coating/__init__.py index a9694fb..7913597 100644 --- a/BlackDynamite/coating/__init__.py +++ b/BlackDynamite/coating/__init__.py @@ -1,6 +1,18 @@ #!/bin/env python +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import bashCoat import pbsCoat import sgeCoat import slurmCoat diff --git a/BlackDynamite/coating/bashCoat.py b/BlackDynamite/coating/bashCoat.py index 5ca88cb..82f654d 100755 --- a/BlackDynamite/coating/bashCoat.py +++ b/BlackDynamite/coating/bashCoat.py @@ -1,63 +1,75 @@ #!/usr/bin/env python +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # from BlackDynamite import * import os import stat import subprocess admissible_params = {"stdout": bool, "stop_on_error": bool} default_params = {"stop_on_error": False} help = {"stdout": "Specify if you want the standard output instead of a file", "stop_on_error": "Specify if should raise an error in case " "of an error in the bash script"} def launch(run, params): _exec = run.getExecFile() head = \ """#!/bin/bash export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__ export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__ export BLACKDYNAMITE_USER={0} """.format(params["user"]) _exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"] f = open(_exec["filename"], 'w') f.write(_exec["file"]) f.close() os.chmod(_exec["filename"], stat.S_IRWXU) print("execute ./" + _exec["filename"]) if params["truerun"] is True: run["state"] = "launched" run.update() run.commit() filename = run["run_name"] + ".o" + str(run.id) filename_err = run["run_name"] + ".e" + str(run.id) if params["stdout"] is True: ret = subprocess.call("./" + _exec["filename"]) else: with open(filename, "w") as outfile: with open(filename_err, "w") as errfile: ret = subprocess.call( "./" + _exec["filename"], stdout=outfile, stderr=errfile) if ret == 0: run["state"] = "FINISHED" else: run["state"] = "BASH error" run.update() run.commit() if params["stop_on_error"] is True and not ret == 0: raise Exception( "The underlying bash script returned " "with the error code {0}.".format(ret)) diff --git a/BlackDynamite/coating/pbsCoat.py b/BlackDynamite/coating/pbsCoat.py index b0d95be..3e154f0 100755 --- a/BlackDynamite/coating/pbsCoat.py +++ b/BlackDynamite/coating/pbsCoat.py @@ -1,123 +1,135 @@ #!/usr/bin/env python +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from BlackDynamite import * import os import stat import subprocess import re import socket admissible_params = {"walltime": str, "email": str, "nproc": int, "pbs_option": [str], "module": [str], "mpiprocs": int, "ncpus": int, "cwd": bool} default_params = {"walltime": "48:00:00", "cwd": True} help = {"walltime": "Specify the wall time for the runs", "email": "Specify the email to notify", "nproc": "Force the number of processors and update the run", "pbs_option": "Allow to provide additional PBS options", "module": "List of module to load", "ncpus": "Number of cpus per nodes", "mpiprocs": "Number of mpi processus per nodes", "cwd": "Run by default in the run folder"} def launch(run, params): _exec = run.getExecFile() head = \ """#!/bin/bash #PBS -l walltime={0} """.format(params["walltime"]) if ("email" in params): head += "#PBS -m abe\n" head += "#PBS -M {0}\n".format(params["email"]) pbs_head_name = "#PBS -N {0}_{1}\n".format(run["run_name"], run.id) head += pbs_head_name run["state"] = "PBS submit" if ("nproc" in params): run["nproc"] = params["nproc"] nproc = run["nproc"] if ("mpiprocs" in params or "npcus" in params): args = [] if "ncpus" in params: npernode = params["ncpus"] args.append("ncpus={0}".format(npernode)) if "mpiprocs" in params: npernode = min(params["mpiprocs"], nproc) args.append("mpiprocs={0}".format(npernode)) select = max(1, nproc/npernode) args.insert(0, "#PBS -l select={0}".format(select)) select_str = ":".join(args) print select_str head += select_str + "\n" else: head += "#PBS -l nodes=" + str(nproc) + "\n" if ("pbs_option" in params): for i in params["pbs_option"]: head += "#PBS {0}\n".format(i) if ("module" in params): for i in params["module"]: head += "module load {0}\n".format(i) run.update() head += """ export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__ export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__ export BLACKDYNAMITE_USER=""" + params["user"] + """ on_kill() { updateRuns.py --updates \"state = PBS killed\" --truerun exit 0 } on_stop() { updateRuns.py --updates \"state = PBS stopped\" --truerun exit 0 } # Execute function on_die() receiving TERM signal # trap on_stop SIGUSR1 trap on_stop SIGTERM trap on_kill SIGUSR2 trap on_kill SIGKILL """ if (params["cwd"]): head += """ cd __BLACKDYNAMITE__run_path__ """ _exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"] f = open(_exec["filename"], 'w') f.write(_exec["file"]) f.close() # os.chmod(_exec["filename"], stat.S_IRWXU) print("execute qsub ./" + _exec["filename"]) print("in dir ") subprocess.call("pwd") if params["truerun"] is True: ret = subprocess.call("qsub " + _exec["filename"], shell=True) diff --git a/BlackDynamite/coating/sgeCoat.py b/BlackDynamite/coating/sgeCoat.py index 0cab3b0..56f84bf 100755 --- a/BlackDynamite/coating/sgeCoat.py +++ b/BlackDynamite/coating/sgeCoat.py @@ -1,99 +1,111 @@ #!/usr/bin/env python +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from BlackDynamite import * import subprocess admissible_params = {"walltime": str, "email": str, "nproc": int, "sge_option": [str], "module": [str]} # default_params = {"walltime":"00:05:00"} help = {"walltime": "Specify the wall time for the runs", "email": "Specify the email to notify", "nproc": "Force the number of processors and update the run", "sge_option": "Allow to provide additional SGE options", "module": "List of module to load"} def launch(run, params): _exec = run.getExecFile() if ("walltime" not in params): raise Exception("walltime not set for this job {0}".format(run.id)) head = \ """#!/bin/bash #$ -S /bin/sh #$ -cwd #$ -j y #$ -notify #$ -l walltime={0} """.format(params["walltime"]) if ("email" in params): head += "#$ -m eas -M {0}\n".format(params["email"]) sge_head_name = "#$ -N " + "run" + \ str(run.id) + "-" + run["run_name"] + "\n" sge_head_name = sge_head_name.replace(":", "_") head += sge_head_name run["state"] = "SGE submit" if ("nproc" in params): run["nproc"] = params["nproc"] nproc = run["nproc"] if (nproc % 12 == 0 and nproc % 8 == 0): head += "#$ -pe orte* " + str(nproc) + "\n" elif (nproc % 12 == 0): head += "#$ -pe orte12 " + str(nproc) + "\n" elif (nproc % 8 == 0): head += "#$ -pe orte8 " + str(nproc) + "\n" if ("sge_option" in params): for i in params["sge_option"]: head += "#$ {0}\n".format(i) if ("module" in params): for i in params["module"]: head += "module load {0}\n".format(i) run.update() head += """ export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__ export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__ export BLACKDYNAMITE_USER=""" + params["user"] + """ on_kill() { updateRuns.py --updates \"state = SGE killed\" --truerun exit 0 } on_stop() { updateRuns.py --updates \"state = SGE stopped\" --truerun exit 0 } # Execute function on_die() receiving TERM signal # trap on_stop SIGUSR1 trap on_kill SIGUSR2 """ _exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"] f = open(_exec["filename"], 'w') f.write(_exec["file"]) f.close() # os.chmod(_exec["filename"], stat.S_IRWXU) print("execute qsub ./" + _exec["filename"]) print("in dir ") subprocess.call("pwd") if params["truerun"] is True: ret = subprocess.call("qsub " + _exec["filename"], shell=True) diff --git a/BlackDynamite/coating/slurmCoat.py b/BlackDynamite/coating/slurmCoat.py index 7abfe84..4fd740a 100755 --- a/BlackDynamite/coating/slurmCoat.py +++ b/BlackDynamite/coating/slurmCoat.py @@ -1,127 +1,139 @@ #!/usr/bin/env python +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function from BlackDynamite import * import os import stat import subprocess import re import socket admissible_params = {"walltime": str, "email": str, "nodes": int, "module": [str], "cpus-per-task": int, "cpus-per-node": int, "cwd": bool, "slurm_option": [str]} default_params = {"walltime": "48:00:00", "cwd": True, "nodes": 1, "cpus-per-task": 1} help = {"walltime": "Specify the wall time for the runs", "email": "Specify the email to notify", "nodes": "Number of nodes for the job", "slurm_option": "Allow to provide additional SLURM options", "module": "List of modules to load", "cpus-per-node": "Number of CPU per nodes", "cpus-per-task": "Number of thread per MPI process", "cwd": "Run by default in the run folder"} mandatory = {"cpus-per-node": True} def launch(run, params): _exec = run.getExecFile() head = "#!/bin/bash\n\n" head += "#SBATCH --time={0}\n".format(params["walltime"]) if ("email" in params): head += "#SBATCH --mail-type=ALL\n" head += "#SBATCH --mail-user={0}\n".format(params["email"]) slurm_head_name = "#SBATCH --job-name={0}_{1}\n".format( run["run_name"], run.id) head += slurm_head_name run["state"] = "SLURM submit" if ("nproc" in params): run["nproc"] = params["nproc"] nproc = run["nproc"] try: nodes = max(nproc * params["cpus-per-task"] // params["cpus-per-node"], 1) except Exception as e: print(params.keys()) print(e) raise e head += "#SBATCH --nodes={0}\n".format(nodes) head += "#SBATCH --ntasks={0}\n".format(nproc) head += "#SBATCH --cpus-per-task={0}\n".format(params["cpus-per-task"]) if "slurm_option" in params: for option in params["slurm_option"]: # To get consistent behavior between --slurm_option="" # and --slurm_option "" m = re.match(r'^--(\S+)$', option) if m: option = m.group(1) head += "#SBATCH --{}\n".format(option) if (params["cwd"]): head += "#SBATCH --workdir=__BLACKDYNAMITE__run_path__\n" if ("module" in params): head += "\nmodule purge\n" for i in params["module"]: head += "module load {0}\n".format(i) run.update() head += """ export BLACKDYNAMITE_HOST=__BLACKDYNAMITE__dbhost__ export BLACKDYNAMITE_SCHEMA=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_STUDY=__BLACKDYNAMITE__study__ export BLACKDYNAMITE_RUN_ID=__BLACKDYNAMITE__run_id__ export BLACKDYNAMITE_USER=""" + params["user"] + """ on_kill() { updateRuns.py --updates \"state = SLURM killed\" --truerun exit 0 } on_stop() { updateRuns.py --updates \"state = SLURM stopped\" --truerun exit 0 } # Execute function on_die() receiving TERM signal # trap on_stop SIGUSR1 trap on_stop SIGTERM trap on_kill SIGUSR2 trap on_kill SIGKILL """ _exec["file"] = run.replaceBlackDynamiteVariables(head) + _exec["file"] f = open(_exec["filename"], 'w') f.write(_exec["file"]) f.close() # os.chmod(_exec["filename"], stat.S_IRWXU) print("execute sbatch ./" + _exec["filename"]) print("in dir ") subprocess.call("pwd") if params["truerun"] is True: ret = subprocess.call("sbatch " + _exec["filename"], shell=True) diff --git a/BlackDynamite/conffile.py b/BlackDynamite/conffile.py index d2099a5..6f63675 100755 --- a/BlackDynamite/conffile.py +++ b/BlackDynamite/conffile.py @@ -1,80 +1,92 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function ################################################################ from . import sqlobject from . import selector import re import os ################################################################ __all__ = ["ConfFile", "addFile"] ################################################################ class ConfFile(sqlobject.SQLObject): """ """ table_name = "configfiles" def addFile(self, filename, params=None, regex_params=None, content=None): print("adding file " + filename) self.entries["filename"] = os.path.basename(filename) if content: self.entries["file"] = content else: self.entries["file"] = open(filename, 'r').read() if regex_params: for p in params: # lowerp = p.lower() rp = "(" + regex_params rp = rp.replace("%p", ")(" + p) rp = rp.replace("%v", ")(.*)") rr = "\\1\\2__BLACKDYNAMITE__" + p + "__" # print (rp) # print (rr) self.entries["file"] = re.sub(rp, rr, self.entries["file"], flags=re.IGNORECASE) # print (self.entries["file"]) file_select = selector.Selector(self.base) filelist = file_select.select(ConfFile, self) if (len(filelist) == 0): tmp_conffile = ConfFile(self.base) tmp_conffile.entries = dict(self.entries) del tmp_conffile.entries['filename'] md5filelist = tmp_conffile.getMatchedObjectList() if len(md5filelist) != 0: import md5 for f in md5filelist: raise Exception(""" There is already another file with same content but different name: this is an impossible situation for BlackDynamite. The file concerned is '{0}' md5:{1} ** If you want keep going, please rename the file before insertion ** """.format(f['filename'], md5.new(f['file']).hexdigest())) self.base.insert(self) elif (len(filelist) == 1): self.entries = filelist[0].entries self.id = filelist[0].id def __init__(self, connection): sqlobject.SQLObject.__init__(self, connection) self.types["filename"] = str self.types["file"] = str def addFile(filename, base, **kwargs): cnffile = ConfFile(base) cnffile.addFile(filename, **kwargs) return cnffile diff --git a/BlackDynamite/constraints.py b/BlackDynamite/constraints.py index f56e1d8..2005148 100644 --- a/BlackDynamite/constraints.py +++ b/BlackDynamite/constraints.py @@ -1,220 +1,233 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from __future__ import print_function ################################################################ from . import sqlobject from . import job from . import run from . import bdlogging import pyparsing as pp ################################################################ print = bdlogging.invalidPrint logger = bdlogging.getLogger(__name__) ################################################################ class BDconstraints(object): "" def __iadd__(self, constraints): if not isinstance(constraints, BDconstraints): # print('cons2', constraints) constraints = BDconstraints(self.base, constraints) self.constraints += constraints.constraints return self def __init__(self, base, constraints): # print('cons', constraints) self.constraints = constraints self.base = base self.conditions = None if isinstance(constraints, BDconstraints): self.constraints = constraints.constraints elif isinstance(constraints, dict): if "constraints" in constraints: self.constraints = constraints["constraints"] elif "run_id" in constraints: self.constraints = [ 'runs.id = {0}'.format(constraints['run_id'])] elif "job_id" in constraints: self.constraints = [ 'jobs.id = {0}'.format(constraints['job_id'])] else: self.constraints = [] if not isinstance(self.constraints, list): self.constraints = [self.constraints] self.constraint_parser = BDconstraintsParser(self.base) # print('cons(end) ', self.constraints) def _pushCondition(self, _cond, _params): if self.conditions != '': self.conditions += ' and ' self.conditions += _cond self.params += _params def pushConditionFromSQLObject(self, _cstr): _cond = [] _params = [] sql_obj = _cstr for k, v in sql_obj.entries.items(): _cond.append('({0}.{1} = %s)'.format( sql_obj.table_name, k)) _params.append(v) _cond = ' and '.join(_cond) self._pushCondition(_cond, _params) def pushConditionFromString(self, _cstr): _cond, _params = self.constraint_parser.parse(_cstr) self._pushCondition(_cond, _params) def getMatchingCondition(self): self.conditions = "" self.params = [] for _cstr in self.constraints: if isinstance(_cstr, str): self.pushConditionFromString(_cstr) if isinstance(_cstr, sqlobject.SQLObject): self.pushConditionFromSQLObject(_cstr) logger.debug(self.conditions) logger.debug(self.params) return self.conditions, self.params ################################################################ class BDconstraintsParser(object): def __init__(self, base): self.base = base self._params = [] self.ref_run = run.Run(self.base) self.ref_job = job.Job(self.base) # rule for entry in the sqlobject var = pp.Word(pp.alphanums+'_') prefix = (pp.Literal('runs') | pp.Literal('jobs')) + pp.Literal('.') entry = pp.Optional(prefix) + var def check_varname(tokens): # print(tokens) res = pp.ParseResults(''.join(tokens)) logger.debug(res) if len(tokens) == 3: obj_type = tokens[0] var_name = tokens[2].lower() else: obj_type = None var_name = tokens[0].lower() if obj_type is None: job_var = var_name in self.ref_job.types run_var = var_name in self.ref_run.types if job_var and run_var: raise RuntimeError( 'ambiguous variable: {0}\n{1}'.format( res, self.base.getPossibleParameters())) if job_var: res.type = self.ref_job.types[var_name] elif run_var: res.type = self.ref_run.types[var_name] else: raise RuntimeError( 'unknown variable: {0}\n{1}'.format( res[0], self.base.getPossibleParameters())) else: if obj_type == 'runs': ref_obj = self.ref_run elif obj_type == 'jobs': ref_obj = self.ref_job if var_name not in ref_obj.types: raise RuntimeError( 'unknown variable: "{0}"\n{1}'.format( var_name, ref_obj.types)) res.type = ref_obj.types[var_name] return res entry = entry.setParseAction(check_varname) # rule to parse the operators operators = [ # '+', # addition 2 + 3 5 # '-', # subtraction 2 - 3 -1 # '*', # multiplication 2 * 3 6 # '/', # division (integer division truncates the result) # '%', # modulo (remainder) 5 % 4 1 # '^', # exponentiation 2.0 ^ 3.0 8 '<', # less than '>', # greater than '<=', # less than or equal to '>=', # greater than or equal to '=', # equal '!=', # not equal '~', # Matches regular expression, case sensitive '~*', # Matches regular expression, case insensitive '!~', # Does not match regular expression, case sensitive '!~*' # Does not match regular expression, case insensitive ] ops = pp.Literal(operators[0]) for o in operators[1:]: ops |= pp.Literal(o) # parse a constraint of the form 'var operator value' and flatten it constraint = pp.Group(entry + ops + pp.Word(pp.alphanums+'._')) def regroup_constraints(tokens): expected_type = tokens[0].type key = tokens[0][0] op = tokens[0][1] val = tokens[0][2] try: parse_res = entry.parseString(val) if parse_res.type != expected_type: raise RuntimeError('no the correct type') val = parse_res[0] except Exception: self._params.append(val) val = '%s' res = ('(' + ' '.join([str(key), str(op), str(val)]) + ')') return res constraint = constraint.setParseAction(regroup_constraints) separator = (pp.Literal(',').setParseAction( lambda tokens: 'and') | pp.Literal('and')) self.constraints = (constraint + pp.Optional( pp.OneOrMore(separator + constraint))).setParseAction( lambda tokens: ' '.join(tokens)) def parse(self, _str): self._params = [] logger.debug(_str) res = self.constraints.parseString(_str) res = ' '.join(res) return res, self._params diff --git a/BlackDynamite/graphhelper.py b/BlackDynamite/graphhelper.py index ed15ab1..1caa867 100755 --- a/BlackDynamite/graphhelper.py +++ b/BlackDynamite/graphhelper.py @@ -1,374 +1,387 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from . import run from . import runselector from . import bdparser import re import sys import numpy as np ################################################################ class GraphHelper(object): """ """ def getMeasures(self, run_list): myresults = [] add_req = [] if (self.frequency): add_req += ["step %% {0} = 0".format(self.frequency)] if (self.start): add_req += ["step > {0}".format(self.start)] if (self.end): add_req += ["step < {0}".format(self.end)] for r, j in run_list: # print ("retrieve data from run " + r["run_name"]) res = r.getScalarQuantities(self.quantities, add_req) for key, value in res: if value is None: del res[key] myresults.append([r, j, res]) return myresults def selectGraphs(self): run_list = self.runSelector.selectRuns(self.constraints, self.sort_by) results = self.getMeasures(run_list) return results def show(self): import matplotlib.pyplot as plt plt.show() def makeGraphs(self, fig=None, **kwargs): import matplotlib.pyplot as plt results = self.selectGraphs() if fig is None: fig = plt.figure(figsize=self.figsize) for r, j, data in results: if data: self.makeCurve(data, fig=fig, myrun=r, myjob=j, **kwargs) return fig def replaceRunAndJobsParameters(self, name, myrun, myjob): res = name # print (res) codes = [["%r." + key, myrun[key]] for key in myrun.entries.keys()] codes += [["%j." + key, myjob[key]] for key in myjob.entries.keys()] for code, val in codes: res = res.replace(code, str(val)) return res def generateLabels(self, results, myrun, myjob): labels = [] names = [r[0] for r in results] for i in range(0, len(results)): name = results[i][0] if (not self.legend or i >= len(self.legend) or not self.legend[i]): labels.append( self.replaceRunAndJobsParameters(name, myrun, myjob) ) continue # print (self.legend[i]) head_legend = self.legend[i].replace("{", "{{") head_legend = head_legend.replace("}", "}}") head_legend = re.sub(r"(%)([0-9]+)", r'{\2}', head_legend).format( *names) # print (head_legend) head_legend = self.replaceRunAndJobsParameters( head_legend, myrun, myjob) # print (head_legend) # if (not head_legend.find("%") == -1): # print("unknown variable name. Possible variables are:") # print "\n".join([c[0] for c in codes]) # sys.exit(-1) # print (head_legend) labels.append(head_legend) return labels def makeComposedQuantity(self, results, myrun, myjob): vecs = [r[1] for r in results] names = [r[0] for r in results] # print (vecs[0].shape) new_results = [] for comp in self.using: exprs = comp.split(":") tmp_res = [] for i in [0, 1]: e = re.sub(r"(%)([0-9]+)\.(x)", r"vecs[\2][:,0]", exprs[i]) e = re.sub(r"(%)([0-9]+)\.(y)", r"vecs[\2][:,1]", e) e = self.replaceRunAndJobsParameters(e, myrun, myjob) try: tmp_res.append(eval(e)) except Exception as ex: print(names) print("invalid expression: '" + exprs[i] + "'") print("invalid expression: '" + e + "'") print(ex) i = 1 for v in vecs: print('quantity {0}/{1} shape: {2}'.format( i, len(vecs), v.shape)) i += 1 sys.exit(-1) name = re.sub(r"(%)([0-9]+)\.([x|y])", r'(" + str(names[\2]) + ")', exprs[1]) res = np.zeros((tmp_res[0].shape[0], 2)) res[:, 0] = tmp_res[0] res[:, 1] = tmp_res[1] # print (res.shape) # expr = re.sub(r"(%)([0-9]+)", r"vecs[\2]", comp) # res[0] = eval(expr) # print (name) name = "\"" + name + "\"" # print (name) name = eval(name) # print (name) new_results.append([name, res]) return new_results def decorateGraph(self, fig, myrun, myjob, results): if not results: return if fig is None: import matplotlib.pyplot as plt fig = plt.figure() axe = fig.add_subplot(1, 1, 1) if self.xrange: axe.set_xlim(self.xrange) if self.yrange: axe.set_ylim(self.yrange) if (self.xlabel): axe.set_xlabel(self.xlabel) if (self.ylabel): axe.set_ylabel(self.ylabel) if (self.title): t = self.replaceRunAndJobsParameters(self.title, myrun, myjob) axe.set_title(t) axe.grid(True, linewidth=0.1) if self.using: results = self.makeComposedQuantity(results, myrun, myjob) labels = self.generateLabels(results, myrun, myjob) # print (labels) return fig, axe, results, labels def makeCurve(self, results, myrun=None, myjob=None, fig=None, **kwargs): fig, axe, results, labels = self.decorateGraph( fig, myrun, myjob, results) for count, result in enumerate(results): # name = result[0] vec = result[1] label = labels[count] # print (self.quantities) # print (name) style = dict() if (self.marker is not None): style["marker"] = self.marker if self.blackwhite: width_index = self.cycle_index/len(self.linestyle_cycle) style_index = self.cycle_index % len(self.linestyle_cycle) self.cycle_index += 1 style["linewidth"] = self.linewidth_cycle[width_index] style["linestyle"] = self.linestyle_cycle[style_index] style["color"] = 'k' axe.plot(vec[:, 0]/self.xscale, vec[:, 1]/self.yscale, label=label, **style) axe.legend(loc='best') if (self.fileout): fig.savefig(self.fileout) return fig def setConstraints(self, **params): self.constraints = [] if "constraints" in params: self.constraints = params["constraints"] def setBinaryOperator(self, **params): self.binary_operator = 'and' if ("binary_operator" in params): self.binary_operator = params["binary_operator"] def setQuantity(self, **params): if ("quantity" in params): self.quantities = params["quantity"] else: print("quantity should be provided using option --quantity") self.quantities = "__BLACKDYNAMITE_ERROR__" def __init__(self, base, **params): self.setConstraints(**params) self.setQuantity(**params) self.base = base self.runSelector = runselector.RunSelector(self.base) self.fig = None self.xrange = None self.yrange = None self.xlabel = None self.ylabel = None self.xscale = None self.yscale = None self.fileout = None self.title = None self.using = None self.frequency = None self.start = None self.end = None self.figsize = None self.blackwhite = None self.legend = None self.sort_by = None self.marker = None # set the members if keys are present in params members = set(self.__dict__.keys()) p = set(params.keys()) for key in members & p: setattr(self, key, params[key]) if params["list_quantities"] is True: myrun = run.Run(base) print("list of possible quantities:\n") print("\n".join(myrun.listQuantities())) sys.exit(0) if params["list_parameters"] is True: self.base.getPossibleParameters() sys.exit(0) self.linewidth_cycle = [1, 2, 4] self.linestyle_cycle = ['-', '--', '-.'] self.cycle_index = 0 ################################################################ class GraphParser(bdparser.BDParser): """ """ def __init__(self): bdparser.BDParser.__init__(self) self.admissible_params["quantity"] = [str] self.help["quantity"] = "Specify the quantity to be outputed" self.admissible_params["xrange"] = [float] self.help["xrange"] = "Specify range of values in the X direction" self.admissible_params["yrange"] = [float] self.help["yrange"] = "Specify range of values in the Y direction" self.admissible_params["sort_by"] = [str] self.help["sort_by"] = ( "Specify a study parameter to be used in sorting the curves") self.admissible_params["xlabel"] = str self.help["xlabel"] = "Specify the label for the X axis" self.admissible_params["ylabel"] = str self.help["ylabel"] = "Specify the label for the Y axis" self.admissible_params["xscale"] = float self.default_params["xscale"] = 1. self.help["xscale"] = "Specify a scale factor for the X axis" self.admissible_params["yscale"] = float self.default_params["yscale"] = 1. self.help["yscale"] = "Specify a scale factor for the Y axis" self.admissible_params["title"] = str self.help["title"] = "Specify title for the graph" self.admissible_params["legend"] = [str] self.help["legend"] = ( "Specify a legend for the curves." " The syntax can use %%j.param or %%r.param to use" " get job and run values") self.default_params["legend"] = None self.admissible_params["using"] = [str] self.help["using"] = ( "Allow to combine several quantities. " "The syntax uses python syntax where " "%%quantity1.column1:%%quantity2.column2 is the python " "numpy vector provided by quantity number (provided using the " "--quantities option) and column number (x or y). " "The sytax is comparable to the GNUPlot one in using the ':' " "to separate X from Y axis") self.admissible_params["list_quantities"] = bool self.help["list_quantities"] = ( "Request to list the possible quantities to be plotted") self.admissible_params["list_parameters"] = bool self.help["list_parameters"] = ( "Request to list the possible job/run parameters") self.admissible_params["frequency"] = int self.default_params["frequency"] = 1 self.help["frequency"] = ( "Set a frequency at which the quantity values " "should be retreived " "(helpful when the amount of data is very large)") self.admissible_params["start"] = float self.help["start"] = "Set the start X value for the graph" self.admissible_params["end"] = int self.help["end"] = "Set the end X value for the graph" self.admissible_params["figsize"] = [float] self.admissible_params["blackwhite"] = bool self.default_params["blackwhite"] = False self.help["blackwhite"] = "Request a black and white graph generation" self.default_params["blackwhite"] = False self.help["blackwhite"] = "Request to plot a black and white graph" self.admissible_params["marker"] = str self.help["marker"] = "Request a specific marker (matplotlib option)" self.admissible_params["fileout"] = str self.help["fileout"] = ( 'Request to write a PDF file' ' (given its name) containing the graph') self.group_params["GraphHelper"] = [ "quantity", "xrange", "yrange", "sort_by", "xlabel", "ylabel", "xscale", "yscale", "title", "legend", "using", "list_quantities", "list_parameters", "frequency", "start", "end", "figsize", "blackwhite", "marker", "fileout"] ################################################################ __all__ = ["GraphHelper", "GraphParser"] ################################################################ diff --git a/BlackDynamite/job.py b/BlackDynamite/job.py index 43e7e23..1c5d301 100755 --- a/BlackDynamite/job.py +++ b/BlackDynamite/job.py @@ -1,18 +1,30 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . ################################################################ from . import sqlobject ################################################################ __all__ = ["Job"] ################################################################ class Job(sqlobject.SQLObject): """ """ table_name = 'jobs' def __init__(self, base): sqlobject.SQLObject.__init__(self, base) self.table_name = "jobs" diff --git a/BlackDynamite/jobselector.py b/BlackDynamite/jobselector.py index 38f8185..528eb4a 100755 --- a/BlackDynamite/jobselector.py +++ b/BlackDynamite/jobselector.py @@ -1,31 +1,44 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from . import selector from . import job ################################################################ class JobSelector(selector.Selector): """ """ def selectJobs(self, constraints=None, sort_by=None, quiet=False): job_list = self.select(job.Job, constraints=constraints, sort_by=sort_by) if quiet is False: if not job_list: print("no jobs found") print("Selected jobs are: " + str([j.id for j in job_list])) return job_list def __init__(self, base): selector.Selector.__init__(self, base) ################################################################ __all__ = ["JobSelector"] diff --git a/BlackDynamite/run.py b/BlackDynamite/run.py index 3df54d2..b8c7538 100755 --- a/BlackDynamite/run.py +++ b/BlackDynamite/run.py @@ -1,464 +1,476 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function ################################################################ from . import job from . import runconfig from . import conffile from . import sqlobject from . import bdparser from . import base from . import runselector from . import bdlogging import sys import re import numpy as np import datetime import subprocess import socket import os ################################################################ __all__ = ['Run', 'getRunFromScript'] print = bdlogging.invalidPrint logger = bdlogging.getLogger(__name__) ################################################################ class Run(sqlobject.SQLObject): """ """ table_name = 'runs' def getJob(self): return self.base.getJobFromID(self.entries["job_id"]) def start(self): self.entries['state'] = 'START' logger.debug('starting run') self.update() logger.debug('update done') self.base.commit() logger.debug('commited') def finish(self): self.entries['state'] = 'FINISHED' logger.debug('finish run') self.update() logger.debug('update done') self.base.commit() logger.debug('commited') def attachToJob(self, job): self["job_id"] = job.id self.base.insert(self) self.addConfigFile(self.execfile) for cnffile in self.configfiles: self.addConfigFile(cnffile) def getExecFile(self): return self.getUpdatedConfigFile(self.entries["exec"]) def setExecFile(self, file_name, **kwargs): # check if the file is already in the config files for f in self.configfiles: if f.entries["filename"] == file_name: self.execfile = f self.entries["exec"] = f.id return f.id # the file is not in the current config files # so it has to be added conf = conffile.addFile(file_name, self.base, **kwargs) self.configfiles.append(conf) self.execfile = conf self.entries["exec"] = conf.id return conf.id def listFiles(self, subdir=""): """List files in run directory / specified sub-directory""" command = 'ls {0}'.format(os.path.join(self['run_path'], subdir)) if not self['machine_name'] == socket.gethostname(): command = 'ssh {0} "{1}"'.format(self['machine_name'], command) logger.info(command) p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) out = p.stdout.readlines() out = [o.strip() for o in out] return out def getFile(self, filename, outpath='/tmp'): dest_path = os.path.join( outpath, "BD-" + self.base.schema + "-cache", "run-{0}".format(self.id)) dest_file = os.path.join(dest_path, filename) if self['machine_name'] == socket.gethostname(): return self.getFullFileName(filename) # In case filename contains sub-directories dest_path = os.path.dirname(dest_file) logger.info(dest_path) logger.info(dest_file) # Making directories try: os.makedirs(dest_path, exist_ok=True) except Exception as e: logger.error(e) pass if os.path.isfile(dest_file): return dest_file cmd = 'scp {0}:{1} {2}'.format(self['machine_name'], self.getFullFileName(filename), dest_file) logger.info(cmd) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) logger.info(p.stdout.read()) return dest_file def getFullFileName(self, filename): return os.path.join(self['run_path'], filename) def addConfigFiles(self, file_list, regex_params=None): if not type(file_list) == list: file_list = [file_list] params_list = list(self.types.keys()) myjob = job.Job(self.base) params_list += list(myjob.types.keys()) # logger.debug (regex_params) file_ids = [f.id for f in self.configfiles] files_to_add = [ conffile.addFile( fname, self.base, regex_params=regex_params, params=params_list) for fname in file_list] for f in files_to_add: if (f.id not in file_ids): self.configfiles.append(f) return self.configfiles def addConfigFile(self, configfile): myrun_config = runconfig.RunConfig(self.base) myrun_config.attachToRun(self) myrun_config.addConfigFile(configfile) self.base.insert(myrun_config) def getConfigFiles(self): # myjob = job.Job(self.base) # myjob["id"] = self.entries["job_id"] # myjob = self.getMatchedObjectList()[0] runconf = runconfig.RunConfig(self.base) runconf["run_id"] = self.id runconf_list = runconf.getMatchedObjectList() logger.info(runconf_list) conffiles = [ self.getUpdatedConfigFile(f["configfile_id"]) for f in runconf_list] return conffiles def getConfigFile(self, file_id): # runconf = runconfig.RunConfig(self.base) conf = conffile.ConfFile(self.base) conf["id"] = file_id conf = conf.getMatchedObjectList()[0] return conf def replaceBlackDynamiteVariables(self, text): myjob = job.Job(self.base) myjob["id"] = self.entries["job_id"] myjob = myjob.getMatchedObjectList()[0] for key, val in myjob.entries.items(): tmp = text.replace("__BLACKDYNAMITE__" + key + "__", str(val)) if ((not tmp == text) and val is None): raise Exception("unset job parameter " + key) text = tmp for key, val in self.entries.items(): tmp = text.replace("__BLACKDYNAMITE__" + key + "__", str(val)) if ((not tmp == text) and val is None): logger.debug(self.entries) raise Exception("unset run parameter " + key) text = tmp text = text.replace("__BLACKDYNAMITE__dbhost__", self.base.dbhost) text = text.replace("__BLACKDYNAMITE__study__", self.base.schema) text = text.replace("__BLACKDYNAMITE__run_id__", str(self.id)) return text def getUpdatedConfigFile(self, file_id): conf = self.getConfigFile(file_id) conf["file"] = self.replaceBlackDynamiteVariables(conf["file"]) return conf def listQuantities(self): request = "SELECT id,name FROM {0}.quantities".format(self.base.schema) curs = self.base.performRequest(request) all_quantities = [res[1] for res in curs] return all_quantities def getScalarQuantities(self, names, additional_request=None): request = ("SELECT id,name FROM {0}.quantities " "WHERE (is_vector) = (false)").format( self.base.schema) params = [] if (names): if (not type(names) == list): names = [names] request += " and (" for name in names: similar_op_match = re.match(r"\s*(~)\s*(.*)", name) op = " = " if (similar_op_match): op = " ~ " name = similar_op_match.group(2) request += " name " + op + "%s or" params.append(str(name)) request = request[:len(request)-3] + ")" # logger.debug (request) # logger.debug (params) curs = self.base.performRequest(request, params) quantities = [res[1] for res in curs] if (len(quantities) == 0): logger.debug("No quantity matches " + str(names)) logger.debug("Quantities declared in the database are \n" + "\n".join( [res for res in self.listQuantities()])) sys.exit(-1) return None try: quant_indexes = [quantities.index(n) for n in names] logger.debug(quant_indexes) quantities = names except: # quant_indexes = None pass # logger.debug (quant) results = [] for key in quantities: q = self.getScalarQuantity(key, additional_request) if (q is not None): results.append([key, q]) logger.debug("got Quantity " + str(key) + " : " + str(q.shape[0]) + " values") return results def getLastStep(self): request = """SELECT max(b.max),max(b.time) from ( SELECT max(step) as max,max(computed_at) as time from {0}.scalar_integer where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.scalar_real where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.vector_integer where run_id = {1} union SELECT max(step) as max,max(computed_at) as time from {0}.vector_real where run_id = {1}) as b """.format(self.base.schema, self.id) # logger.debug (request) curs = self.base.performRequest(request, []) item = curs.fetchone() if (item is not None): return item[0], item[1] def getQuantityID(self, name, is_integer=None, is_vector=None): request = """ SELECT id,is_integer,is_vector FROM {0}.quantities WHERE (name) = (%s) """.format(self.base.schema) curs = self.base.performRequest(request, [name]) item = curs.fetchone() if (item is None): raise Exception("unknown quantity \"" + name + "\"") if ((is_integer is not None) and (not is_integer == item[1])): raise Exception("quantity \"" + name + "\" has is_integer = " + str(item[1])) if ((is_vector is not None) and (not is_vector == item[2])): raise Exception("quantity \"" + name + "\" has is_vector = " + str(item[2])) return item[0], item[1], item[2] def getScalarQuantity(self, name, additional_request=None): quantity_id, is_integer, is_vector = self.getQuantityID(name) if is_vector is True: raise Exception("Quantity " + name + " is not scalar") request = """ SELECT step,measurement from {0}.{1} WHERE (run_id,quantity_id) = ({2},{3}) """.format(self.base.schema, "scalar_real" if (is_integer is False) else "scalar_integer", self.id, quantity_id) if (additional_request): request += " and " + " and ".join(additional_request) request += " ORDER BY step" curs = self.base.performRequest(request, [name]) fetch = curs.fetchall() if (not fetch): return None return (np.array([(val[0], val[1]) for val in fetch])) def getVectorQuantity(self, name, step): quantity_id, is_integer, is_vector = self.getQuantityID(name) if (is_vector is False): raise Exception("Quantity " + name + " is not vectorial") request = """ SELECT measurement from {0}.{1} WHERE (run_id,quantity_id,step) = ({2},{3},{4}) """.format(self.base.schema, "vector_real" if (is_integer is False) else "vector_integer", self.id, quantity_id, step) curs = self.base.performRequest(request, [name]) fetch = curs.fetchone() if (fetch): return np.array(fetch[0]) return None def pushVectorQuantity(self, vec, step, name, is_integer, description=None): logger.debug('pushing {0}'.format(name)) try: quantity_id, is_integer, is_vector = self.getQuantityID( name, is_integer=is_integer, is_vector=True) except Exception as e: typecode = "int" if is_integer is False: typecode = "float" typecode += ".vector" quantity_id = self.base.pushQuantity(name, typecode, description) array = [i for i in vec] # if is_integer is True: # array_format = ",".join(["{:d}".format(i) for i in vec]) request = """ INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s) """.format(self.base.schema, "vector_real" if is_integer is False else "vector_integer") curs = self.base.performRequest(request, [self.id, quantity_id, array, step]) logger.debug(curs) logger.debug('ready to commit') self.base.commit() logger.debug('commited') def pushScalarQuantity(self, val, step, name, is_integer, description=None): logger.debug('pushing {0}'.format(name)) try: quantity_id, is_integer, is_vector = self.getQuantityID( name, is_integer=is_integer, is_vector=False) except Exception as e: typecode = "int" if is_integer is False: typecode = "float" quantity_id = self.base.pushQuantity(name, typecode, description) request = """ INSERT INTO {0}.{1} (run_id,quantity_id,measurement,step) VALUES (%s,%s,%s,%s) """.format(self.base.schema, "scalar_real" if is_integer is False else "scalar_integer") curs = self.base.performRequest(request, [self.id, quantity_id, val, step]) logger.debug(curs) logger.debug('ready to commit') self.base.commit() logger.debug('commited') def getAllVectorQuantity(self, name): quantity_id, is_integer, is_vector = self.getQuantityID( name, is_vector=True) request = """ SELECT step,measurement from {0}.{1} WHERE (run_id,quantity_id) = ({2},{3}) order by step """.format(self.base.schema, "vector_real" if is_integer is False else "vector_integer", self.id, quantity_id) curs = self.base.performRequest(request, [name]) fetch = curs.fetchall() if (not fetch): return [None, None] matres = np.array([val[1] for val in fetch]) stepres = np.array([val[0] for val in fetch]) return (stepres, matres) def deleteData(self): request, params = ( "DELETE FROM {0}.scalar_real WHERE run_id={1}".format( self.base.schema, self.id), []) self.base.performRequest(request, params) request, params = ( "DELETE FROM {0}.scalar_integer WHERE run_id={1}".format( self.base.schema, self.id), []) self.base.performRequest(request, params) request, params = ( "DELETE FROM {0}.vector_real WHERE run_id={1}".format( self.base.schema, self.id), []) self.base.performRequest(request, params) request, params = ( "DELETE FROM {0}.vector_integer WHERE run_id={1}".format( self.base.schema, self.id), []) self.base.performRequest(request, params) def __init__(self, base): sqlobject.SQLObject.__init__(self, base) self.foreign_keys["job_id"] = "jobs" self.types["machine_name"] = str self.types["run_path"] = str self.allowNull["run_path"] = True self.types["job_id"] = int self.types["nproc"] = int self.types["run_name"] = str self.types["wait_id"] = int self.allowNull["wait_id"] = True self.types["start_time"] = datetime.datetime self.allowNull["start_time"] = True self.types["state"] = str self.allowNull["state"] = True self.execfile = None self.configfiles = [] self.types["exec"] = str ################################################################ def getRunFromScript(): parser = bdparser.BDParser() parser.register_params(params={"run_id": int}) params = parser.parseBDParameters(argv=[]) mybase = base.Base(**params) runSelector = runselector.RunSelector(mybase) run_list = runSelector.selectRuns(params) if len(run_list) > 1: raise Exception('internal error') if len(run_list) == 0: raise Exception('internal error') myrun, myjob = run_list[0] # myrun.setEntries(params) return myrun, myjob diff --git a/BlackDynamite/runLM.py b/BlackDynamite/runLM.py index 0f441ec..bc1f1ee 100644 --- a/BlackDynamite/runLM.py +++ b/BlackDynamite/runLM.py @@ -1,91 +1,103 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function from . import conffile from . import run from . import bdparser class RunLM(run.Run): """ """ def attachToJob(self, job): if "exec" not in self.entries: self.createLaunchFile() if "lm_conf" in self.entries: self.addConfigFiles([self.entries["lm_conf"]]) run.Run.attachToJob(self, job) def addConfigFiles(self, file_list, regex_params='LET\s+%p\s*=\s*%v'): run.Run.addConfigFiles(self, file_list, regex_params=regex_params) return self.configfiles def createLaunchFile(self): self.execfile = conffile.addFile("launch.sh", self.base, content=""" export HOST=__BLACKDYNAMITE__dbhost__ export SCHEMA=__BLACKDYNAMITE__study__ export RUN_ID=__BLACKDYNAMITE__run_id__ export DEBUG_LEVEL=0 mpirun __BLACKDYNAMITE__mpi_option__ -np __BLACKDYNAMITE__nproc__ \ __BLACKDYNAMITE__amel_path__ __BLACKDYNAMITE__lm_conf__ \ __BLACKDYNAMITE__nsteps__ if [ $? != 0 ]; then updateRuns.py --study=$SCHEMA --host=$HOST --run_constraints="id = $RUN_ID" \ --updates="state = LM FAILED" --truerun fi """) self["exec"] = self.execfile.id return self.execfile def __init__(self, base): run.Run.__init__(self, base) self.types["amel_path"] = str self.types["nsteps"] = int self.types["lm_conf"] = str self.types["mpi_option"] = str self.types["lm_release_info"] = str self.allowNull["lm_release_info"] = True ################################################################ class RunLMParser(bdparser.RunParser): def parseBDParameters(self): params = bdparser.RunParser.parseBDParameters(self) return params def __init__(self): bdparser.RunParser.__init__(self) self.mandatory["nsteps"] = True self.mandatory["amel_path"] = True self.mandatory["lm_conf"] = True self.admissible_params["nsteps"] = int self.help["nsteps"] = "Set the number of steps to run" self.admissible_params["amel_path"] = str self.help["amel_path"] = ( "The path to the libmultiscale client executable AMEL") self.admissible_params["lm_conf"] = str self.help["lm_conf"] = ( "The name of the configuration file for libmultiscale") self.admissible_params["mpi_option"] = str self.help["mpi_option"] = "optional MPI option" self.default_params["mpi_option"] = " " self.group_params["RunLM"] = [ "amel_path", "nsteps", "lm_conf", "mpi_option", "lm_release_info"] ################################################################ diff --git a/BlackDynamite/runconfig.py b/BlackDynamite/runconfig.py index 88a1b21..fd68871 100755 --- a/BlackDynamite/runconfig.py +++ b/BlackDynamite/runconfig.py @@ -1,26 +1,38 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function from . import sqlobject class RunConfig(sqlobject.SQLObject): """ """ table_name = 'runconfig' def attachToRun(self, run): self["run_id"] = run.id def addConfigFile(self, configfile): self["configfile_id"] = configfile.id def __init__(self, base): sqlobject.SQLObject.__init__(self, base) self.table_name = "runconfig" self.foreign_keys["run_id"] = "runs" self.foreign_keys["configfile_id"] = "configfiles" self.types["run_id"] = int self.types["configfile_id"] = int diff --git a/BlackDynamite/runselector.py b/BlackDynamite/runselector.py index c32cf11..6cd536a 100755 --- a/BlackDynamite/runselector.py +++ b/BlackDynamite/runselector.py @@ -1,43 +1,56 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from . import run from . import job from . import selector from . import constraints as BDconstraints ################################################################ class RunSelector(selector.Selector): """ """ def selectRuns(self, constraints, sort_by=None, quiet=False): run_constraints = BDconstraints.BDconstraints(self.base, 'runs.job_id = jobs.id') run_constraints += constraints run_list = self.select([run.Run, job.Job], constraints=run_constraints, sort_by=sort_by) if quiet is False: if not run_list: print("no runs found") print("Selected runs are: " + str([r[0].id for r in run_list])) print("Selected jobs are: " + str([r[0]["job_id"] for r in run_list])) return run_list def __init__(self, base): selector.Selector.__init__(self, base) ################################################################ __all__ = ["RunSelector"] diff --git a/BlackDynamite/selector.py b/BlackDynamite/selector.py index 9132d09..449199b 100644 --- a/BlackDynamite/selector.py +++ b/BlackDynamite/selector.py @@ -1,90 +1,103 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from __future__ import print_function ################################################################ import copy from . import constraints as BDcons ################################################################ from . import bdlogging ################################################################ print = bdlogging.invalidPrint logger = bdlogging.getLogger(__name__) ################################################################ class Selector(object): def __init__(self, base): self.base = base def buildList(self, curs, sqlobjs): logger.debug(sqlobjs) if not isinstance(sqlobjs, list): sqlobjs = [sqlobjs] col_infos = [] sqlobjs2 = [] for sqlobj in sqlobjs: if isinstance(sqlobj, type): sqlobj = sqlobj(self.base) sqlobjs2.append(sqlobj) col_infos.append(self.base.getColumnProperties(sqlobj)) sqlobjs = sqlobjs2 list_objects = [] for entries in curs: # print(entries) objs = [] offset = 0 logger.debug(sqlobjs) for index, sqlobj in enumerate(sqlobjs): obj = copy.deepcopy(sqlobj) for col_name, size in col_infos[index]: logger.debug((col_name, entries[offset])) obj[col_name] = entries[offset] offset += 1 objs.append(obj) if len(objs) == 1: list_objects.append(objs[0]) else: list_objects.append(tuple(objs)) return list_objects def select(self, _types, constraints=None, sort_by=None): if (sort_by is not None) and (not isinstance(sort_by, str)): raise RuntimeError( 'sort_by argument is not correct: {0}'.format(sort_by)) const = BDcons.BDconstraints(self.base, constraints) condition, params = const.getMatchingCondition() if not isinstance(_types, list): _types = [_types] selected_tables = ['{0}.{1}'.format(self.base.schema, t.table_name) for t in _types] selected_tables = ','.join(selected_tables) request = "SELECT * FROM {0}".format(selected_tables) if condition: request += " WHERE " + condition # print (sort_by) if sort_by: request += " ORDER BY " + sort_by logger.debug(request) logger.debug(params) curs = self.base.performRequest(request, params) obj_list = self.buildList(curs, _types) return obj_list diff --git a/BlackDynamite/sqlobject.py b/BlackDynamite/sqlobject.py index 6644879..5c2eb4c 100644 --- a/BlackDynamite/sqlobject.py +++ b/BlackDynamite/sqlobject.py @@ -1,232 +1,245 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from __future__ import print_function ################################################################ import copy import datetime import psycopg2 import re import sys ################################################################ class LowerCaseDict(object): def __init__(self): self.entries = {} def __getattr__(self, attr): if 'entries' not in self.__dict__: raise AttributeError(attr) key = attr.lower() if key in self.entries: return self.entries[key] else: raise AttributeError(attr) def __setattr__(self, attr, value): key = attr.lower() if key == 'entries': object.__setattr__(self, key, value) entries = self.entries if key in entries: self.__setitem__(attr, value) else: object.__setattr__(self, attr, value) def __getitem__(self, index): return self.entries[index.lower()] def keys(self): return self.entries.keys() def __iter__(self): return self.entries.__iter__() def __setitem__(self, index, value): self.entries[index.lower()] = value def items(self): return self.entries.items() def copy(self): cp = LowerCaseDict() cp.entries = self.entries return cp def setEntries(self, params): for p, val in params.items(): if p in self.types: self.entries[p] = val ################################################################ class SQLObject(LowerCaseDict): " The generic object related to entries in the database " def __str__(self): keys = set(self.entries.keys()) if 'id' in self.entries: keys.remove('id') keys = list(keys) if 'id' in self.entries: keys = ['id'] + keys outputs = [] for k in keys: v = self.entries[k] outputs += [k + ": " + str(v)] return "\n".join(outputs) def commit(self): self.base.connection.commit() def setFields(self, constraints): for cons in constraints: _regex = "(\w*)\s*=\s*(.*)" match = re.match(_regex, cons) if (not match or (not len(match.groups()) == 2)): print("malformed assignment: " + cons) sys.exit(-1) key = match.group(1).lower().strip() val = match.group(2) if key not in self.types: print("unknown key '{0}'".format(key)) print("possible keys are:") for k in self.types.keys(): print("\t" + k) sys.exit(-1) val = self.types[key](val) self.entries[key] = val def __init__(self, base): LowerCaseDict.__init__(self) self.foreign_keys = {} self.allowNull = {} self.types = LowerCaseDict() self.base = base self.operators = {} self._prepare() def __deepcopy__(self, memo): _cp = type(self)(self.base) _cp.types = copy.deepcopy(self.types.copy(), memo) _cp.entries = copy.deepcopy(self.entries.copy(), memo) # _cp.id = self.id _cp.foreign_keys = copy.deepcopy(self.foreign_keys, memo) _cp.allowNull = copy.deepcopy(self.foreign_keys, memo) _cp.connection = self.base return _cp def _prepare(self): try: self.base.setObjectItemTypes(self) except psycopg2.ProgrammingError: self.base.connection.rollback() def insert(self): params = list() # print (self.types) ex_msg = "" for key, value in self.types.items(): if key == "id": continue if ((key not in self.entries) and (key not in self.allowNull)): ex_msg += ( "key '" + key + "' must be given a value before proceeding insertion\n") if (not ex_msg == ""): raise Exception("\n****************\n"+ex_msg+"****************\n") for key, value in self.entries.items(): # print (key) # print (self.types[key]) # print (value) params.append(self.types[key.lower()](value)) request = """ INSERT INTO {0}.{1} ({2}) VALUES ({3}) RETURNING id """.format(self.base.schema, self.table_name, ','.join(self.entries.keys()), ','.join(["%s" for item in params])), params return request def delete(self): request, params = "DELETE FROM {0}.{1} WHERE id={2}".format( self.base.schema, self.table_name, self.id), [] self.base.performRequest(request, params) def update(self): params = list() keys = list() for key, value in self.entries.items(): if (value is None): continue _type = self.types[key] # print (_type) # print (key) # print (type(value)) if (_type == datetime.datetime): continue # _type = str keys.append(key) params.append(_type(value)) request = "UPDATE {0}.{1} SET ({2}) = ({3}) WHERE id = {4}".format( self.base.schema, self.table_name, ','.join(keys), ','.join(["%s" for item in params]), self.id) self.base.performRequest(request, params) def getquoted(self): raise RuntimeError('code needs review') # objs = [sql_adapt(member) for member in self._sql_members()] # for obj in objs: # if hasattr(obj, 'prepare'): # obj.prepare(self._conn) # quoted_objs = [obj.getquoted() for obj in objs] # return '(' + ', '.join(quoted_objs) + ')' def createTableRequest(self): query_string = "CREATE TABLE {0}.{1} ( id SERIAL PRIMARY KEY,".format( self.base.schema, self.table_name) for key, value in self.types.items(): if key == 'id': continue if (value == float): type_string = "DOUBLE PRECISION" elif (value == int): type_string = "INTEGER" elif (value == str): type_string = "TEXT" elif (value == bool): type_string = "BOOLEAN" elif (value == datetime.datetime): type_string = "TIMESTAMP" else: print(value) raise Exception("type '{0}' not handled".format(value)) query_string += "{0} {1} ".format(key, type_string) if (key not in self.allowNull): query_string += " NOT NULL" query_string += "," for key, value in self.foreign_keys.items(): query_string += "FOREIGN KEY ({0}) REFERENCES {1}.{2},".format( key, self.base.schema, value) return query_string[:-1] + ");" def getMatchedObjectList(self): from . import selector sel = selector.Selector(self.base) return sel.select(self, self) diff --git a/example/createDB.py b/example/createDB.py index 12edbe6..b9bd76d 100755 --- a/example/createDB.py +++ b/example/createDB.py @@ -1,26 +1,38 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # First we need to set the python headers and # to import the blackdynamite modules import BlackDynamite as BD # Then you have to create a generic black dynamite parser # and parse the system (including the connection parameters and credentials) parser = BD.bdparser.BDParser() params = parser.parseBDParameters() # Then we can connect to the black dynamite database base = BD.base.Base(**params, creation=True) # Then you have to define the parametric space (the job pattern) myjob_desc = BD.job.Job(base) myjob_desc.types["param1"] = int myjob_desc.types["param2"] = float myjob_desc.types["param3"] = str # Then you have to define the run pattern myruns_desc = BD.run.Run(base) myruns_desc.types["compiler"] = str # Then we request for the creation of the database base.createBase(myjob_desc, myruns_desc, **params) diff --git a/example/createJobs.py b/example/createJobs.py index 7cb84a7..e4bfdec 100755 --- a/example/createJobs.py +++ b/example/createJobs.py @@ -1,24 +1,36 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # First we need to set the python headers # and to import the \blackdynamite modules import BlackDynamite as BD # Then you have to create a generic black dynamite parser # and parse the system (including the connection parameters and credentials) parser = BD.bdparser.BDParser() params = parser.parseBDParameters() # Then we can connect to the black dynamite database base = BD.base.Base(**params) # create of job object job = BD.job.Job(base) # specify a range of jobs job["param1"] = 10 job["param2"] = [3.14, 1., 2.] job["param3"] = 'toto' # creation of the jobs on the database base.createParameterSpace(job) diff --git a/example/createRuns.py b/example/createRuns.py index 9e89a73..92a7ce3 100755 --- a/example/createRuns.py +++ b/example/createRuns.py @@ -1,39 +1,51 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . # First we need to set the python headers # and to import the blackdynamite modules import BlackDynamite as BD # import a runparser (instead of a generic BD parser) parser = BD.RunParser() params = parser.parseBDParameters() # Then we can connect to the black dynamite database base = BD.Base(**params) # create a run object myrun = BD.Run(base) # set the run parameters from the parsed entries myrun.setEntries(params) # add a configuration file myrun.addConfigFiles("doIt.py") # set the entry point (executable) file myrun.setExecFile("launch.sh") # create a job selector jobSelector = BD.JobSelector(base) # select the jobs that should be associated with the runs about to be created job_list = jobSelector.selectJobs(params) # create the runs for j in job_list: myrun['compiler'] = 'gcc' myrun.attachToJob(j) # if truerun, commit the changes to the base if (params["truerun"] is True): base.commit() diff --git a/example/doIt.py b/example/doIt.py index 6939cf1..5c5f998 100644 --- a/example/doIt.py +++ b/example/doIt.py @@ -1,16 +1,28 @@ #!/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD myrun, myjob = BD.getRunFromScript() print(myjob) myrun.start() for step in range(0, 10): _quantity = myrun.id*step myrun.pushScalarQuantity(_quantity, step, "ekin", is_integer=False) myrun.pushScalarQuantity(_quantity*2, step, "epot", is_integer=False) myrun.finish() diff --git a/example/post_treatment.py b/example/post_treatment.py index 99c882b..c639278 100644 --- a/example/post_treatment.py +++ b/example/post_treatment.py @@ -1,44 +1,57 @@ #!/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ import BlackDynamite as BD import matplotlib.pyplot as plt ################################################################ # basic connection parser = BD.BDParser() params = parser.parseBDParameters( '--host lsmssrv1.epfl.ch --study mystudy'.split()) mybase = BD.Base(**params) ################################################################ # function to plot things (user's job) def plot(run_list): for r, j in run_list: ekin = r.getScalarQuantity('ekin') if ekin is None: continue print(j) list_files = r.listFiles() print(list_files) fname = r.getFile(list_files[3]) print(fname + ':') _file = open(fname) print(_file.read()) plt.plot(ekin[:, 0], ekin[:, 1], 'o-', label='$p_2 = {0}$'.format(j['param2'])) plt.legend(loc='best') plt.show() ################################################################ # selecting some runs runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, params) plot(run_list) # selecting some other runs params['run_constraints'] = ['run_name =~ test', 'state = FINISHED'] params['job_constraints'] = ['param2 > 1'] run_list = runSelector.selectRuns(params, params) plot(run_list) diff --git a/pythontests/__init__.py b/pythontests/__init__.py index 5bfd17e..11f0c97 100644 --- a/pythontests/__init__.py +++ b/pythontests/__init__.py @@ -1,2 +1,14 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/pythontests/coding_convention.py b/pythontests/coding_convention.py index 321e3ec..03115b8 100644 --- a/pythontests/coding_convention.py +++ b/pythontests/coding_convention.py @@ -1,63 +1,54 @@ #!/usr/bin/env python3 -""" -@file coding_convention.py - -@author Till Junge - -@date 11 Aug 2016 - -@brief Test whether the source files of game_engine follow our coding - conventions - -@section LICENCE - - Copyright (C) 2016 Till Junge - -coding_convention.py is part of zegame and proprietary -software; you can neither redistribute nor modify coding_convention.py. - -Only members of the zegame association are allowed to view, run, -modify and copy coding_convention.py under the terms of the zegame -association statutes. -""" +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . try: import unittest import os import pep8 import BlackDynamite except ImportError as err: import sys print(err) sys.exit(str(err)) class Pep8Test(unittest.TestCase): """ Test for pep8 conformity """ def setUp(self): """ builds a list of source files. If you find a smarter way, please let me know """ print() # for emacs to evaluate the first line of errors self.mod_files = list() bd_path = os.path.join(BlackDynamite.__path__[0], "..") for dirpath, _, filenames in os.walk(bd_path): self.mod_files += [os.path.join(dirpath, filename) for filename in filenames if filename.endswith((".py", ".pyx"))] for dirpath, _, filenames in os.walk(os.path.join(bd_path, 'scripts')): self.mod_files += [os.path.join(dirpath, filename) for filename in filenames if not filename.endswith(".sh")] def test_pep8_conformity(self): """ check all files for pep8 conformity """ pep8style = pep8.StyleGuide() pep8style.check_files((mod_file for mod_file in self.mod_files)) diff --git a/scripts/canYouDigIt.py b/scripts/canYouDigIt.py index e85d95a..19484ed 100755 --- a/scripts/canYouDigIt.py +++ b/scripts/canYouDigIt.py @@ -1,19 +1,31 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . ################################################################ import BlackDynamite as BD from BlackDynamite.graphhelper import GraphParser from BlackDynamite.graphhelper import GraphHelper ################################################################ parser = GraphParser() params = parser.parseBDParameters() base = BD.Base(**params) gH = GraphHelper(base, **params) gH.makeGraphs(**params) gH.show() diff --git a/scripts/cleanRuns.py b/scripts/cleanRuns.py index a878848..1918632 100755 --- a/scripts/cleanRuns.py +++ b/scripts/cleanRuns.py @@ -1,120 +1,132 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD import os import sys import socket import re import shutil ################################################################ def validate(question): if params["truerun"] is True: validated = BD.bdparser.validate_question(question, params) else: print("{0}? Forced N".format(question)) validated = False return validated parser = BD.BDParser() parser.register_params("clearRun", params={ "runid": int, "clean_orphans": str, "machine_name": str, "constraints": [str], "delete": bool}, defaults={ "machine_name": socket.gethostname(), "delete": False, }) params = parser.parseBDParameters() if "machine_name" in params: if "constraints" in params: params["constraints"].append( "machine_name = " + params["machine_name"]) else: params["constraints"] = ["machine_name = " + params["machine_name"]] base = BD.Base(**params) runSelector = BD.RunSelector(base) if "clean_orphans" in params: run_list = runSelector.selectRuns([]) run_ids = [r.id for r, j in run_list] resdir = params["clean_orphans"] + "/BD-" + params["study"] + "-runs" print("clean orphans from " + resdir) if not os.path.exists(resdir): print("Directory '" + resdir + "' do not exists") sys.exit(-1) to_delete = {} for filename in os.listdir(resdir): fullname = os.path.join(resdir, filename) # print(fullname) if (os.path.isdir(fullname)): match = re.match("run-([0-9]+)", filename) if (match): # print(filename) id = int(match.group(1)) if (id not in run_ids): to_delete[id] = fullname if (len(to_delete.keys()) == 0): print("No orphans found") sys.exit(0) validated = validate("Delete output from runs " + str(to_delete.keys())) if (validated): for id, fullname in to_delete.items(): print("Delete output from run " + str(id)) shutil.rmtree(fullname) sys.exit(0) runSelector = BD.RunSelector(base) run_list = runSelector.selectRuns(params) if (len(run_list) == 0): print("No runs to be cleared") validated = validate("Delete runs " + str([r[0].id for r in run_list])) for r, j in run_list: delete_flag = params["delete"] run_path = r["run_path"] if run_path: if os.path.exists(run_path): if (validated): print("Deleting directory: " + run_path) shutil.rmtree(run_path) else: print("Simulate deletion of directory: " + run_path) else: print("output directory: '" + run_path + "' not found: are we on the right machine ?") if (delete_flag): if validated: print("Deleting run " + str(r.id) + " from base") r.delete() base.commit() else: print("Simulate deletion of run " + str(r.id) + " from base") else: if validated: print("Deleting data associated with run " + str(r.id)) r.deleteData() r["STATE"] = "CREATED" r["start_time"] = None r.update() base.commit() else: print("Simulate deletion of data associated with run " + str(r.id)) diff --git a/scripts/createUser.py b/scripts/createUser.py index 295b151..da375af 100755 --- a/scripts/createUser.py +++ b/scripts/createUser.py @@ -1,90 +1,103 @@ #!/usr/bin/python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ # import getpass import os import stat import psycopg2 import string import argparse import getpass import os ################################################################ from random import randint, choice from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT ################################################################ def generatePassword(): characters = string.ascii_letters + string.digits password = "".join(choice(characters) for x in range(randint(8, 16))) return password ################################################################ def createUser(user, host): connection_params = dict() connection_params["user"] = user if host is not None: connection_params["host"] = host if host is None: host = 'localhost' connection_params["password"] = getpass.getpass( '{0}@{1} password: '.format(connection_params['user'], host)) try: connection = psycopg2.connect(**connection_params) except Exception as e: raise Exception(str(e)+'\n'+'*'*30 + '\ncannot connect to database\n' + '*'*30) new_user = input('new login: ') connection.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) curs = connection.cursor() try: curs.execute('create user {0}'.format(new_user)) except Exception as e: print(e) print('Setting new password') curs.execute('grant create on database blackdynamite to {0}'.format( new_user)) password = generatePassword() curs.execute('alter role {0} with password \'{1}\' '.format( new_user, password)) fname = '{0}.bd'.format(new_user) print('Saving information to {0}'.format(fname)) try: os.remove(fname) except Exception: pass bdconf = open(fname, 'w') bdconf.write('password = {0}\n'.format(password)) bdconf.write('host = {0}'.format(host)) bdconf.close() os.chmod(fname, stat.S_IREAD) ################################################################ parser = argparse.ArgumentParser( description='User creation tool for blackdynamite') parser.add_argument("--user", type=str, help="name of the user to create", required=True) parser.add_argument("--host", type=str, help="host to connect where to create the user", default=None) args = parser.parse_args() args = vars(args) new_user = args['user'] host = args['host'] createUser(new_user, host) diff --git a/scripts/enterRun.py b/scripts/enterRun.py index 2cc58a6..a64839c 100755 --- a/scripts/enterRun.py +++ b/scripts/enterRun.py @@ -1,74 +1,87 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ import BlackDynamite as BD import subprocess import os import sys import socket ################################################################ parser = BD.BDParser() parser.register_params(group="getRunInfo", params={"run_id": int, "order": str}, help={"run_id": "Select a run_id for switching to it"}) params = parser.parseBDParameters() mybase = BD.Base(**params) if 'run_id' in params: params['run_constraints'] = ['id = {0}'.format(params['run_id'])] try: del params['job_constraints'] except: pass runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, quiet=True) mybase.close() if (len(run_list) == 0): print("no run found") sys.exit(1) run, job = run_list[0] run_id = run['id'] separator = '-'*30 print(separator) print("JOB INFO") print(separator) print(job) print(separator) print("RUN INFO") print(separator) print(run) print(separator) print("LOGGING TO '{0}'".format(run['machine_name'])) print(separator) if run['state'] == 'CREATED': print("Cannot enter run: not yet started") sys.exit(-1) bashrc_filename = os.path.join( '/tmp', 'bashrc.user{0}.study{1}.run{2}'.format(params['user'], params['study'], run_id)) bashrc = open(bashrc_filename, 'w') bashrc.write('export PS1="\\u@\\h:<{0}|RUN-{1}> $ "\n'.format( params['study'], run_id)) bashrc.write('cd {0}\n'.format(run['run_path'])) bashrc.write('echo ' + separator) bashrc.close() command_login = 'bash --rcfile {0} -i'.format(bashrc_filename) if not run['machine_name'] == socket.gethostname(): command1 = 'scp -q {0} {1}:{0}'.format(bashrc_filename, run['machine_name']) subprocess.call(command1, shell=True) command_login = 'ssh -X -A -t {0} "{1}"'.format( run['machine_name'], command_login) # print command_login subprocess.call(command_login, shell=True) diff --git a/scripts/getRunInfo.py b/scripts/getRunInfo.py index e319aa4..e43b755 100755 --- a/scripts/getRunInfo.py +++ b/scripts/getRunInfo.py @@ -1,242 +1,254 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD import sys import datetime ################################################################ def printSummary(mybase, params): runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, quiet=True) print("*"*6 + " run summary => study {0} ".format(mybase.schema) + "*"*6) if len(run_list) == 0: print("no runs found") sys.exit(0) request = "SELECT run_name,state,count(state) from {0}.runs ".format( mybase.schema) if (len(run_list) > 0): request += "where id in (" + ",".join( [str(r.id) for r, j in run_list]) + ")" request += " group by state,run_name order by run_name,state" # print (request) curs = mybase.performRequest(request, []) stats = {} for i in curs: if i[0] not in stats: stats[i[0]] = [] stats[i[0]].append([i[1], int(i[2])]) for run_name, st in stats.items(): tot = 0 for n, count in st: tot += count for n, count in st: print("{:20} {:>20} => {:5} ({:>5.1f}%)".format( run_name, n, count, 100.*count/tot)) print("") sys.exit(0) ################################################################ def getRunInfo(run_id, mybase): myrun = BD.Run(mybase) myrun["id"] = run_id myrun.id = run_id run_list = myrun.getMatchedObjectList() if (len(run_list) == 0): print("no run found with id " + str(run_id)) sys.exit(1) myrun = run_list[0] myjob = BD.Job(mybase) myjob.id = myrun["job_id"] myjob["id"] = myrun["job_id"] job_list = myjob.getMatchedObjectList() if (len(job_list) == 0): print("no job found with id " + myjob.id) sys.exit(1) myjob = job_list[0] list_entries = myjob.entries.keys() print("*"*6 + " job info " + "*"*6) for entry in list_entries: if (myjob[entry]): print(entry + ": " + str(myjob[entry])) print("*"*6 + " run info " + "*"*6) list_entries = myrun.entries.keys() regular_run_entries = ("run_name", "job_id", "state", "start_time", "machine_name", "exec", "nproc", "wait_id") for entry in regular_run_entries: if (myrun[entry]): print(entry + ": " + str(myrun[entry])) list_entries.remove(entry) for entry in list_entries: if (myrun[entry]): print(entry + ": " + str(myrun[entry])) conffiles = myrun.getConfigFiles() for conf in conffiles: print("*"*6) print("file #" + str(conf.id) + ": " + conf["filename"]) print("*"*6) print(conf["file"]) ################################################################ def getInfoNames(): infos = [] infos.append("run_name") infos.append("id") infos.append("job_id") if "infos" in params: infos += params['infos'] else: infos += ["state", "nproc", "machine_name"] infos.append("start_time") infos.append("last step") infos.append("last update") infos.append("Time/step") infos.append("Total Time") return infos ################################################################ def getFormatString(infos): format_string = " {:<20} | {:^6} | {:^6} |" if "infos" in params: format_string += " {:^10} |" * len(params['infos']) else: format_string += " {:<15} | {:^5} | {:<20} |" format_string += " {:14} | {:>9} | {:>16} | {:>10} | {:>16} |" return format_string ################################################################ def formatTimeDelta(t): if (t < datetime.timedelta(seconds=1)): if (t < datetime.timedelta(microseconds=1000)): t = str(t.microseconds) + u'\u00B5'.encode('UTF-8') + "s" else: t = str(1./1000.*t.microseconds) + 'ms' else: ms = t.microseconds t -= datetime.timedelta(microseconds=ms) t = str(t) return t ################################################################ def getTimeInfos(r): step, steptime = r.getLastStep() start_time = r['start_time'] time_perstep = None total_time = None if (step is not None and steptime and start_time): time_perstep = (steptime-start_time)/(step+1) total_time = steptime-start_time time_perstep = formatTimeDelta(time_perstep) total_time = formatTimeDelta(total_time) if start_time: start_time = start_time.strftime("%H:%M %d/%m/%y") if steptime: steptime = steptime.strftime("%H:%M %d/%m/%y") run_infos = [start_time, step, steptime, time_perstep, total_time] return run_infos ################################################################ def getRunInfos(r, j): run_infos = [] for col in info_names[:-5]: key_run = col.replace('%r.', '').strip() if not key_run == 'start_time': if key_run in r.entries: run_infos.append(r[key_run]) else: key_job = col.replace('%j.', '').strip() if key_job in j.entries: run_infos.append(j[key_job]) else: raise Exception('Key {0} is not a valid parameter'.format( key_run)) run_infos += getTimeInfos(r) return run_infos ################################################################ parser = BD.BDParser() parser.register_params( group="getRunInfo", params={"run_id": int, "order": str, "summary": bool, "infos": [str]}, defaults={"order": "id"}, help={"run_id": "Select a run_id for complete output", "summary": "Output a summary of the completeness of the study", "order": "specify the column which serves to order the lines"}) params = parser.parseBDParameters() mybase = BD.Base(**params) if params["summary"] is True: printSummary(mybase, params) if ("run_id" in params): getRunInfo(params["run_id"], mybase) else: info_names = getInfoNames() format_string = getFormatString(info_names) header = format_string.format(*info_names) separator = "-" * len(header) print(separator) print(header) print(separator) runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, sort_by="runs." + params["order"], quiet=True) for r, j in run_list: try: infos = getRunInfos(r, j) def transform_None(x): if x is None: return 'None' else: return x infos = [transform_None(x) for x in infos] line = format_string.format(*infos) print(line) except Exception as e: print(getRunInfos(r, j)) print(e) diff --git a/scripts/launchRuns.py b/scripts/launchRuns.py index 015f70e..f0ce038 100755 --- a/scripts/launchRuns.py +++ b/scripts/launchRuns.py @@ -1,105 +1,118 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ import BlackDynamite as BD import os import sys import socket from types import ModuleType ################################################################ def main(argv=None): if (type(argv) == str): argv = argv.split() parser = BD.BDParser() parser.register_params( group="launchRuns.py", params={ "outpath": str, "generator": ModuleType, "nruns": int, "state": str, "machine_name": str}, defaults={ "machine_name": socket.gethostname(), "nruns": -1, "generator": "bashCoat"}) parser.help.update({ "nruns": ('Specify the number of runs to launch. ' 'This is useful when we want to launch ' 'only the first run from the stack.'), "generator": "Specify the launcher generator" }) params = parser.parseBDParameters(argv=argv) mybase = BD.Base(**params) if ("outpath" not in params): print('A directory where to create temp files ' 'should be provided. use --outpath ') sys.exit(-1) mydir = os.path.join(params["outpath"], "BD-" + params["study"] + "-runs") if not os.path.exists(mydir): os.makedirs(mydir) os.chdir(mydir) runSelector = BD.RunSelector(mybase) constraints = [] if ("constraints" in params): constraints = params["constraints"] def item_matcher(name, item): return item.lower().lstrip().startswith(name) if not any([item_matcher("state", item) for item in constraints]): constraints.append("state = CREATED") if not any([item_matcher("machine_name", item) for item in constraints]): constraints.append("machine_name = {0}".format( params["machine_name"])) run_list = runSelector.selectRuns(constraints) if (params["nruns"] > 0): run_list = [run_list[i] for i in range(0, min(params["nruns"], len(run_list)))] if (len(run_list) == 0): print("No runs to be launched") for r, j in run_list: print("Dealing with job {0.id}, run {1.id}".format(j, r)) r["run_path"] = os.path.join(mydir, "run-" + str(r.id)) print(j.types) j.update() if not os.path.exists("run-" + str(r.id)): os.makedirs("run-" + str(r.id)) os.chdir("run-" + str(r.id)) conffiles = r.getConfigFiles() for conf in conffiles: print("create file " + conf["filename"]) f = open(conf["filename"], 'w') f.write(conf["file"]) f.close() print("launch in '" + mydir + "/" + "run-" + str(r.id) + "/'") mymod = params["generator"] print(mymod) mymod.launch(r, params) os.chdir("../") if (params["truerun"] is True): mybase.commit() if __name__ == '__main__': main() diff --git a/scripts/mvRuns.py b/scripts/mvRuns.py index af5992d..011a028 100755 --- a/scripts/mvRuns.py +++ b/scripts/mvRuns.py @@ -1,90 +1,103 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ################################################################ from __future__ import print_function ################################################################ import BlackDynamite as BD import os import subprocess import socket ################################################################ def move_run(r, machine_src, machine_dst, path_src, path_dst): if path_src == path_dst and machine_dst == machine_src: return if not os.path.exists(path_dst): os.makedirs(path_dst) # print(machine_src, machine_dst) # print(path_src, path_dst) url_src = machine_src + ':' + path_src if url_src[-1] == '/': url_src = url_src[:-1] print('mv run,job:', r.id, j.id) url_dst = path_dst if machine_src == machine_dst: url_src = path_src else: url_src = machine_src + ':' + path_src if url_src[-1] == '/': url_src = url_src[:-1] if url_dst[-1] == '/': url_dst = url_dst[:-1] rsync_command = 'rsync --remove-source-files -auP {0} {1}'.format( url_src, url_dst) print(rsync_command) if params['truerun'] is True: ret = subprocess.call(rsync_command, shell=True) else: ret = True if ret: return r['run_path'] = path_dst r['machine_name'] = machine_dst r.update() if params['truerun'] is True: mybase.commit() ################################################################ parser = BD.BDParser() parser.register_params( group="mvRuns", params={"path": str}, mandatory={'path': True}, help={"path": "Path to the local machine where to store the run outputs"}) params = parser.parseBDParameters() mybase = BD.Base(**params) runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, params, quiet=True) for r, j in run_list: machine_dst = socket.gethostname() machine_src = r['machine_name'] path_src = r['run_path'] if path_src is None: continue f, p = os.path.split(path_src) while p == '': f, p = os.path.split(f) run_subdir = p path_dst = os.path.join(params['path'], "BD-" + params["study"] + "-runs", run_subdir) move_run(r, machine_src, machine_dst, path_src, path_dst) diff --git a/scripts/pushQuantity.py b/scripts/pushQuantity.py index a01994d..c5e4d58 100755 --- a/scripts/pushQuantity.py +++ b/scripts/pushQuantity.py @@ -1,69 +1,81 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD def main(argv=None): parser = BD.BDParser() parser.register_params( group="pushQuantity", params={"runid": int, "quantity_id": str, "value": str, "values": [str], "step": int, "is_float": bool}, defaults={"step": 0}, help={"runid": "The id of the run to update", "quantity_id": "ID of the Quantity to push", "value": "Value tu push for the quantity", "values": "Vectorial value tu push for the quantity", "step": "Step at which the data is generated", "is_float": "0 if the quantity is a float 1 other why"} ) params = parser.parseBDParameters() if("runid" not in params): raise Exception("The run id should be set") if("quantity_id" not in params): raise Exception("The quantity id should be set") if("value" not in params and "values" not in params): raise Exception("The value should be set") is_vector = False if("values" in params): is_vector = True if("value" in params): raise Exception( "You cannot define values and value at the same time") base = BD.Base(**params) if ("runid" in params): if "run_constraints" not in params: params["run_constraints"] = [] params["run_constraints"].append("id = " + str(params["runid"])) runSelector = BD.RunSelector(base) run_list = runSelector.selectRuns(params, params) if (not len(run_list) == 1): raise Exception("No or too many runs selected") r, j = run_list[0] if params["truerun"] is True: if is_vector is False: r.pushScalarQuantity( params["value"], params["step"], params["quantity_id"], params["is_float"] is False) else: r.pushVectorQuantity( params["values"], params["step"], params["quantity_id"], params["is_float"] is False) base.commit() if __name__ == '__main__': main() diff --git a/scripts/saveBDStudy.py b/scripts/saveBDStudy.py index a726644..78c0244 100755 --- a/scripts/saveBDStudy.py +++ b/scripts/saveBDStudy.py @@ -1,61 +1,73 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD import os import sys import subprocess def saveSchema(params): if ("out_file" not in params): out_file = os.path.join("./", params["study"] + ".db") else: out_file = params["out_file"] r, ext = os.path.splitext(out_file) if (not ext == ".gz"): out_file += ".gz" print("Saving study " + params["study"] + " to file " + out_file) command = ("pg_dump --host " + params["host"] + " --schema=" + params["study"] + " -C -f " + out_file + " --compress=9") if params["verbose"] is True: command += " --verbose" command = command.strip().split(" ") ret = subprocess.call(command) if not ret == 0: sys.exit("pg_dump error") def main(argv=None): if isinstance(argv, str): argv = argv.split() parser = BD.BDParser() parser.register_params( group="saveBDStudy.py", params={"out_file": str, "verbose": bool, "study": str}, help={"out_file": "Specify the filename where to save the study", "verbose": "Activate the verbose mode of pg_dump", "study": "specify the study to backup. \ If none provided all studies are backed up"}) params = parser.parseBDParameters(argv=argv) params["should_not_check_study"] = True mybase = BD.Base(**params) if ("study" in params): saveSchema(params) else: if "out_file" in params: del params["out_file"] sch_list = mybase.getSchemaList() for s in sch_list: params["study"] = s saveSchema(params) if __name__ == '__main__': main() diff --git a/scripts/studyInfo.py b/scripts/studyInfo.py index a7cc71b..7b33d27 100755 --- a/scripts/studyInfo.py +++ b/scripts/studyInfo.py @@ -1,92 +1,104 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . from __future__ import print_function ################################################################ import BlackDynamite as BD ################################################################ def printDataBaseInfo(base, params): curs = base.connection.cursor() curs.execute("select current_database()") datname = curs.fetchone()[0] curs.execute(""" select d.datname, pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname)) from pg_catalog.pg_database d where (d.datname) = ('{0}') """.format(datname)) datsize = curs.fetchone()[1] print("Database:", datname, datsize) def printUserInfo(base, params): users = base.getUserList() print('registered Users: {0}'.format(', '.join(users))) def printStudyInfo(base, study, params): study_size = base.getStudySize(study) owner = base.getStudyOwner(study) granted_users = base.getGrantedUsers(study) print('{0}'.format(study)) print(' Owner:{:>30} '.format(owner)) print(' #jobs:{:>30} '.format(study_size['njobs'])) print(' #runs:{:>30} '.format(study_size['nruns'])) print(' size: {:>30}'.format(study_size['size'])) print(' grants: {0}'.format(','.join(granted_users))) def fetchInfo(base, params): printUserInfo(base, params) if "study" not in params: study_list = base.getSchemaList(filter_names=False) else: study_list = [base.schema] printDataBaseInfo(base, params) for s in study_list: printStudyInfo(base, s, params) def fetchStudy(base, params): runSelector = BD.RunSelector(base) run_list = runSelector.selectRuns(params, params, quiet=True) print(run_list) def main(argv=None): if (type(argv) == str): argv = argv.split() parser = BD.BDParser() parser.register_params( group="studyInfo.py", params={"full": bool, "study": str, "grant": str, "revoke": str, "fetch": bool}, help={"full": "Say that you want details (can be costful)", "study": "specify a study to analyse", "grant": "specify an user to grant read permission", "revoke": "specify an user to revoke read permission", "fetch": "fetch the specified study as one of yours"}) params = parser.parseBDParameters(argv=argv) params["should_not_check_study"] = True mybase = BD.Base(**params) if 'grant' in params: mybase.grantAccess(params['study'], params['grant']) if 'revoke' in params: mybase.revokeAccess(params['study'], params['revoke']) if params['fetch'] is True: fetchStudy(mybase, params) fetchInfo(mybase, params) if __name__ == '__main__': main() diff --git a/scripts/updateRuns.py b/scripts/updateRuns.py index 1a0c40f..ec904cf 100755 --- a/scripts/updateRuns.py +++ b/scripts/updateRuns.py @@ -1,42 +1,54 @@ #!/usr/bin/env python3 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . import BlackDynamite as BD import os import sys import socket parser = BD.BDParser() parser.register_params( group="updateRuns", params={"run_id": int, "updates": [str]}, defaults={"machine_name": socket.gethostname()}, help={"run_id": "The id of the run to update", "updates": "The updates to perform. Syntax should be 'key = newval'"} ) params = parser.parseBDParameters() if "user" not in params.keys(): params["user"] = os.getlogin() base = BD.Base(**params) if "run_id" in params: if "constraints" not in params: params["constraints"] = [] params["constraints"].append("runs.id = " + str(params["run_id"])) runSelector = BD.RunSelector(base) run_list = runSelector.selectRuns(params) if len(run_list) == 0: print("No runs to be updated") if "updates" not in params: print("No update to be performed: use --updates option") sys.exit(-1) for r, j in run_list: r.setFields(params["updates"]) if params["truerun"] is True: r.update() base.commit() diff --git a/setup.py b/setup.py index 5f63dda..cb7c0e4 100644 --- a/setup.py +++ b/setup.py @@ -1,39 +1,52 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + from setuptools import setup setup(name="blackdynamite", packages=['BlackDynamite', 'BlackDynamite.coating'], version="0.0.1", author="Guillaume Anciaux", author_email="guillaume.anciaux@epfl.ch", description=("Compliant parametric study tool"), package_data={ 'BlackDynamite': ['build_tables.sql']}, scripts=['scripts/canYouDigIt.py', 'scripts/cleanRuns.py', 'scripts/createUser.py', 'scripts/enterRun.py', 'scripts/getRunInfo.py', 'scripts/launchRuns.py', 'scripts/mvRuns.py', 'scripts/pushQuantity.py', 'scripts/saveBDStudy.py', 'scripts/updateRuns.py', 'scripts/studyInfo.py', 'scripts/bash_completion.sh'], install_requires=["psycopg2-binary", "numpy", "argcomplete", "pyparsing"], test_suite="pythontests", license=""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """)