diff --git a/BlackDynamite/constraints.py b/BlackDynamite/constraints.py index 8c4baa2..6526774 100644 --- a/BlackDynamite/constraints.py +++ b/BlackDynamite/constraints.py @@ -1,189 +1,197 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import print_function import sqlobject import job import run import pyparsing as pp class BDconstraints(object): "" - def __init__(self, base, ref_obj): - + def __init__(self, base): self.base = base - self.ref_obj = ref_obj - if isinstance(ref_obj, type): - self.ref_obj = ref_obj(base) - - self.ref_obj.prepare() def getMatchingCondition(self, constraints): - _constraints = [] + _constraints = "" _params = [] + if isinstance(constraints, str): + return constraints, [] + if isinstance(constraints, list): - _constraints = ''.join(constraints) + print(constraints) + _constr = [self.getMatchingCondition(c) for c in constraints] + for c, p in _constr: + _constraints += c + _params += p + + print ('aaa', _constraints) + print (_params) + _constraints = ') and ('.join(_constraints) + if _constraints: + return ('(' + _constraints + ')'), _params if isinstance(constraints, dict): if "constraints" in constraints: constraints = constraints["constraints"] if "run_id" in constraints: constraints = [ 'runs.id = {0}'.format(constraints['run_id'])] if "job_id" in constraints: constraints = [ 'jobs.id = {0}'.format(constraints['job_id'])] if isinstance(constraints, sqlobject.SQLObject): + _constraints = "" sql_obj = constraints for k, v in sql_obj.entries.iteritems(): _constraints.append('({0}.{1} = %s)'.format( sql_obj.table_name, k)) _params.append(v) _constraints = ' and '.join(_constraints) return _constraints, _params def makeConstraint(self, sqlobject): # print (constraints) for cons in constraints: _regex = "\s*(\w*)\s*((?:\=~)|(?:!=)|<|>|=)\s*(.*)" match = re.match(_regex, cons) # print (match.groups()) if (not match or (not len(match.groups()) == 3)): print ("malformed constraint: " + cons) sys.exit(-1) key = match.group(1).lower().strip() op = match.group(2) if (op == "=~"): op = "~" # print (op) val = match.group(3) # print (key) # print (op) # print (val) if (key not in self.types): print ("unknown key '{0}'".format(key)) print (constraints) print ("possible keys are:") for k in self.types.keys(): print ("\t" + k) sys.exit(-1) # print (val) # print (key) # print (self.types[key]) val = self.types[key](val) if (key not in self.entries): self.entries[key] = [] self.entries[key].append(val) if (key not in self.operators): self.operators[key] = [] self.operators[key].append(op) def makeMatchingCondition(self, binary_operator="and"): self.prepare() condition = "" params = [] binary_operator = " " + binary_operator + " " for key, values in self.entries.items(): if (values is None): continue if (not type(values) == list): values = [values] for i in range(0, len(values)): try: if (not isinstance(self.types[key], type)): raise Exception("Key " + key + " needs to be given a type and not " + self.types[key]) except KeyError as err: raise KeyError( "{0}, keys are {1}".format(err, self.types.keys())) # print (str(key) + "type:" + str(self.types[key]) + " " # + str(values[i])) params.append(self.types[key](values[i])) op = "=" if (key in self.operators): op = self.operators[key][i] if (condition): condition += binary_operator if (isinstance(values[i], float) and op == "=" and values[i] != 0): condition += "@({0}.{1} / %s -1) < 1e-10".format( self.table_name, key) else: condition += self.table_name + "." + key condition += " " + op + " " condition += " %s " return condition, params def joinConditions(self, condition_job, condition_run, params_job, params_run): condition = '' if condition_job: condition = '(' + condition_job + ')' if (condition and condition_run): condition += ' and ' if (condition_run): condition += '(' + condition_run + ')' if (condition): condition += ' and ' condition += " runs.job_id = jobs.id " params = params_job + params_run return params ################################################################ # try parsing logic expressions ################################################################ # rule for entry in the sqlobject var = pp.Word(pp.alphas) prefix = (pp.Literal('runs') | pp.Literal('jobs')) + '.' entry = (pp.Optional(prefix) + var).setResultsName('entry') entry.setParseAction(lambda tokens: ''.join(tokens)) # rule to parse the operators operators = [ '+', # addition 2 + 3 5 '-', # subtraction 2 - 3 -1 '*', # multiplication 2 * 3 6 '/', # division (integer division truncates the result) 4 / 2 2 '%', # modulo (remainder) 5 % 4 1 '^', # exponentiation 2.0 ^ 3.0 8 '<', # less than '>', # greater than '<=', # less than or equal to '>=', # greater than or equal to '=', # equal '!=', # not equal '~', # Matches regular expression, case sensitive '~*', # Matches regular expression, case insensitive '!~', # Does not match regular expression, case sensitive '!~*' # Does not match regular expression, case insensitive ] ops = pp.Literal(operators[0]) for o in operators[1:]: ops |= pp.Literal(o) # parse a constraint of the form 'var operator value' and flatten it constraint = pp.Group(entry + ops + pp.Word(pp.alphanums)).setParseAction( lambda tokens: '(' + ' '.join([str(t) for t in tokens[0]]) + ')') separator = pp.Literal(',').setParseAction(lambda tokens: 'and') constraints = (constraint + pp.Optional(pp.OneOrMore(separator + constraint)))\ .setParseAction(lambda tokens: ' '.join(tokens)) hello = "runs.toto > 2, tata = 2, jobs.toto != 3" res = constraints.parseString(hello)[0] print (res) diff --git a/BlackDynamite/jobselector.py b/BlackDynamite/jobselector.py index e6ee26b..26973da 100755 --- a/BlackDynamite/jobselector.py +++ b/BlackDynamite/jobselector.py @@ -1,28 +1,28 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- ################################################################ __all__ = ["JobSelector"] import constraints as BDcons import selector import job ################################################################ class JobSelector(selector.Selector): """ """ def selectJobs(self, constraints=None, sort_by=None, quiet=False): job_list = self.select(job.Job, constraints=constraints, sort_by=sort_by) - if not job_list: - print ("no jobs found") if quiet is False: + if not job_list: + print ("no jobs found") print ("Selected jobs are: " + str([j.id for j in job_list])) return job_list def __init__(self, base): selector.Selector.__init__(self, base) diff --git a/BlackDynamite/runselector.py b/BlackDynamite/runselector.py index 88a67d9..5a87145 100755 --- a/BlackDynamite/runselector.py +++ b/BlackDynamite/runselector.py @@ -1,113 +1,86 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- __all__ = ["RunSelector"] import copy -# import run -# import job +import run +import job +import selector import constraints as BDcons ################################################################ -class RunSelector(object): +class RunSelector(selector.Selector): """ """ def selectRuns(self, constraints, sort_by=None, quiet=False): - request, params = self.selectRunsRequest( - constraints, sort_by=sort_by) + run_list = self.select([run.Run, job.Job], + constraints=['run.job_id = job.id', constraints], + sort_by=sort_by) - curs = self.base.performRequest(request, params) - run_list = self.buildJoinedList(myjob, myrun, curs) - - if (not quiet): + if quiet is False: + if not run_list: + print ("no runs found") print("Selected runs are: " + str([r[0].id for r in run_list])) print("Selected jobs are: " + str([r[0]["job_id"] for r in run_list])) - return run_list - - def selectRunsRequest(self, constraints, sort_by=None): - - const = BDcons.BDconstraints(self.base, constraints) - - raise - if (not type(sort_by) == list and sort_by is not None): - sort_by = [sort_by] - order_condition = "" - if (sort_by is not None): - order_condition = " ORDER BY (" - for cond in sort_by: - order_condition += cond + "," - order_condition = order_condition[:len(order_condition)-1] + ")" - - # print ("order condition: " + order_condition) - request = "SELECT * FROM {0}.jobs,{0}.runs ".format(self.base.schema) - request += " WHERE " + condition - request += order_condition - - # print (condition) - # print (request) - # print (params) - raise + # run_list = self.buildJoinedList(myjob, myrun, curs) + return run_list def buildJoinedList(self, job_object, run_object, curs): coljob_info = self.base.getColumnProperties(job_object) colrun_info = self.base.getColumnProperties(run_object) run_list = [] for entries in curs: jobj = copy.deepcopy(job_object) robj = copy.deepcopy(run_object) ncoljob = len(coljob_info) ncolrun = len(colrun_info) for i in range(0, ncoljob): col_name = coljob_info[i][0] jobj[col_name] = entries[i] jobj.id = jobj["id"] for i in range(0, ncolrun): col_name = colrun_info[i][0] robj[col_name] = entries[i+ncoljob] robj.id = robj["id"] run_list.append([robj, jobj]) return run_list def getObjectList(self, order="id"): self.prepare() request = "SELECT * FROM {0}.{1} ORDER BY ".format( self.base.schema, self.table_name) + order curs = self.base.performRequest(request) return self.buildList(curs) def getMatchedObjectList(self, order_condition=""): condition, params = self.makeMatchingCondition() request = self.makeSelectRequest() # print (condition) # print (order_condition) if (condition): request += "WHERE " + condition if (order_condition): request += order_condition # print (request) # print (params) curs = self.base.performRequest(request, params) return self.buildList(curs) - - def __init__(self, base): - self.base = base - - - + selector.Selector.__init__(self, base) diff --git a/BlackDynamite/selector.py b/BlackDynamite/selector.py index 1c141f2..17ccfc2 100644 --- a/BlackDynamite/selector.py +++ b/BlackDynamite/selector.py @@ -1,54 +1,75 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- ################################################################ import copy import constraints as BDcons ################################################################ class Selector(object): def __init__(self, base): self.base = base def buildList(self, curs, sqlobj): if isinstance(sqlobj, type): sqlobj = sqlobj(self.base) col_info = self.base.getColumnProperties(sqlobj) list_objects = [] for entries in curs: obj = copy.deepcopy(sqlobj) for i in range(0, len(col_info)): col_name = col_info[i][0] obj[col_name] = entries[i] list_objects.append(obj) # print len(list_objects) return list_objects - def select(self, _type, constraints=None, sort_by=None): + def select(self, _types, constraints=None, sort_by=None): - const = BDcons.BDconstraints(self.base, _type) + const = BDcons.BDconstraints(self.base) condition, params = const.getMatchingCondition(constraints) - if sort_by: - condition += " ORDER BY " + sort_by + +################################################################ +#review +#if not isinstance(sort_by, list) and sort_by is not None): +# sort_by = [sort_by] +# +# order_condition = "" +# if (sort_by is not None): +# order_condition = " ORDER BY (" +# for cond in sort_by: +# order_condition += cond + "," +# order_condition = order_condition[:len(order_condition)-1] + ")" +################################################################ + + if not isinstance(_types, list): + _types = [_types] - request = "SELECT * FROM {0}.{1} ".format(self.base.schema, - _type.table_name) + selected_tables = ['{0}.{1}'.format(self.base.schema, t.table_name) + for t in _types] + selected_tables = ','.join(selected_tables) + + request = "SELECT * FROM {0}".format(selected_tables) if condition: request += " WHERE " + condition - # print(request) - # print(params) + if sort_by: + request += " ORDER BY " + sort_by + + + print(request) + print(params) curs = self.base.performRequest(request, params) - obj_list = self.buildList(curs, _type) + obj_list = self.buildList(curs, _types[0]) return obj_list diff --git a/scripts/getRunInfo.py b/scripts/getRunInfo.py index 7eb7500..65ef153 100755 --- a/scripts/getRunInfo.py +++ b/scripts/getRunInfo.py @@ -1,236 +1,236 @@ #!/usr/bin/env python import BlackDynamite as BD import sys import datetime ################################################################ def printSummary(mybase, params): runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, params, quiet=True) print ("*"*6 + " run summary => study {0} ".format(mybase.schema) + "*"*6) if (len(run_list) == 0): print ("no runs found") sys.exit(0) request = "SELECT run_name,state,count(state) from {0}.runs ".format( mybase.schema) if (len(run_list) > 0): request += "where id in (" + ",".join( [str(r.id) for r, j in run_list]) + ")" request += " group by state,run_name order by run_name,state" # print (request) curs = mybase.performRequest(request, []) stats = {} for i in curs: if i[0] not in stats: stats[i[0]] = [] stats[i[0]].append([i[1], int(i[2])]) for run_name, st in stats.iteritems(): tot = 0 for n, count in st: tot += count for n, count in st: print ("{:20} {:>20} => {:5} ({:>5.1f}%)".format( run_name, n, count, 100.*count/tot)) print ("") sys.exit(0) ################################################################ def getRunInfo(run_id, mybase): myrun = BD.Run(mybase) myrun["id"] = run_id myrun.id = run_id run_list = myrun.getMatchedObjectList() if (len(run_list) == 0): print ("no run found with id " + str(run_id)) sys.exit(1) myrun = run_list[0] myjob = BD.Job(mybase) myjob.id = myrun["job_id"] myjob["id"] = myrun["job_id"] job_list = myjob.getMatchedObjectList() if (len(job_list) == 0): print ("no job found with id " + myjob.id) sys.exit(1) myjob = job_list[0] list_entries = myjob.entries.keys() print ("*"*6 + " job info " + "*"*6) for entry in list_entries: if (myjob[entry]): print (entry + ": " + str(myjob[entry])) print ("*"*6 + " run info " + "*"*6) list_entries = myrun.entries.keys() regular_run_entries = ("run_name", "job_id", "state", "start_time", "machine_name", "exec", "nproc", "wait_id") for entry in regular_run_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) list_entries.remove(entry) for entry in list_entries: if (myrun[entry]): print (entry + ": " + str(myrun[entry])) conffiles = myrun.getConfigFiles() for conf in conffiles: print ("*"*6) print ("file #" + str(conf.id) + ": " + conf["filename"]) print ("*"*6) print (conf["file"]) ################################################################ def getInfoNames(): infos = [] infos.append("run_name") infos.append("id") infos.append("job_id") if "infos" in params: infos += params['infos'] else: infos += ["state", "nproc", "machine_name"] infos.append("start_time") infos.append("last step") infos.append("last update") infos.append("Time/step") infos.append("Total Time") return infos ################################################################ def getFormatString(infos): format_string = " {:<20} | {:^6} | {:^6} |" if "infos" in params: format_string += " {:^10} |" * len(params['infos']) else: format_string += " {:<15} | {:^5} | {:<20} |" format_string += " {:14} | {:>9} | {:>16} | {:>10} | {:>16} |" return format_string ################################################################ def formatTimeDelta(t): if (t < datetime.timedelta(seconds=1)): if (t < datetime.timedelta(microseconds=1000)): t = str(t.microseconds) + u'\u00B5'.encode('UTF-8') + "s" else: t = str(1./1000.*t.microseconds) + 'ms' else: ms = t.microseconds t -= datetime.timedelta(microseconds=ms) t = str(t) return t ################################################################ def getTimeInfos(r): step, steptime = r.getLastStep() start_time = r['start_time'] time_perstep = None total_time = None if (step is not None and steptime and start_time): time_perstep = (steptime-start_time)/(step+1) total_time = steptime-start_time time_perstep = formatTimeDelta(time_perstep) total_time = formatTimeDelta(total_time) if start_time: start_time = start_time.strftime("%H:%M %d/%m/%y") if steptime: steptime = steptime.strftime("%H:%M %d/%m/%y") run_infos = [start_time, step, steptime, time_perstep, total_time] return run_infos ################################################################ def getRunInfos(r, j): run_infos = [] for col in info_names[:-5]: key_run = col.replace('%r.', '').strip() if not key_run == 'start_time': if key_run in r.entries: run_infos.append(r[key_run]) else: key_job = col.replace('%j.', '').strip() if key_job in j.entries: run_infos.append(j[key_job]) else: raise Exception('Key {0} is not a valid parameter'.format( key_run)) run_infos += getTimeInfos(r) return run_infos ################################################################ parser = BD.BDParser() parser.register_params( group="getRunInfo", params={"run_id": int, "order": str, "summary": bool, "infos": [str]}, defaults={"order": "id"}, help={"run_id": "Select a run_id for complete output", "summary": "Output a summary of the completeness of the study", "order": "specify the column which serves to order the lines"}) params = parser.parseBDParameters() mybase = BD.Base(**params) if params["summary"] is True: printSummary(mybase, params) if ("run_id" in params): getRunInfo(params["run_id"], mybase) else: info_names = getInfoNames() format_string = getFormatString(info_names) header = format_string.format(*info_names) separator = "-" * len(header) print (separator) print (header) print (separator) run_constraints = [] job_constraints = [] runSelector = BD.RunSelector(mybase) - run_list = runSelector.selectRuns(params, params, + run_list = runSelector.selectRuns(params, sort_by="runs." + params["order"], quiet=True) for r, j in run_list: try: line = format_string.format(*getRunInfos(r, j)) print (line) except Exception as e: print getRunInfos(r, j) print e