diff --git a/.gitignore b/.gitignore index 6add511..8b0c839 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc -*~ \ No newline at end of file +*~ +blackdynamite.egg-info \ No newline at end of file diff --git a/scripts/cleanRuns.py b/scripts/cleanRuns.py index f074b60..a878848 100755 --- a/scripts/cleanRuns.py +++ b/scripts/cleanRuns.py @@ -1,124 +1,120 @@ #!/usr/bin/env python3 import BlackDynamite as BD import os import sys import socket import re import shutil ################################################################ def validate(question): if params["truerun"] is True: validated = BD.bdparser.validate_question(question, params) else: print("{0}? Forced N".format(question)) validated = False return validated parser = BD.BDParser() parser.register_params("clearRun", params={ "runid": int, "clean_orphans": str, "machine_name": str, - "run_constraints": [str], - "job_constraints": [str], + "constraints": [str], "delete": bool}, defaults={ "machine_name": socket.gethostname(), "delete": False, }) params = parser.parseBDParameters() -run_constraints = [] -job_constraints = [] - -if ("machine_name" in params): +if "machine_name" in params: if "constraints" in params: params["constraints"].append( "machine_name = " + params["machine_name"]) else: params["constraints"] = ["machine_name = " + params["machine_name"]] base = BD.Base(**params) runSelector = BD.RunSelector(base) if "clean_orphans" in params: run_list = runSelector.selectRuns([]) run_ids = [r.id for r, j in run_list] resdir = params["clean_orphans"] + "/BD-" + params["study"] + "-runs" print("clean orphans from " + resdir) if not os.path.exists(resdir): print("Directory '" + resdir + "' do not exists") sys.exit(-1) to_delete = {} for filename in os.listdir(resdir): fullname = os.path.join(resdir, filename) # print(fullname) if (os.path.isdir(fullname)): match = re.match("run-([0-9]+)", filename) if (match): # print(filename) id = int(match.group(1)) if (id not in run_ids): to_delete[id] = fullname if (len(to_delete.keys()) == 0): print("No orphans found") sys.exit(0) validated = validate("Delete output from runs " + str(to_delete.keys())) if (validated): for id, fullname in to_delete.items(): print("Delete output from run " + str(id)) shutil.rmtree(fullname) sys.exit(0) runSelector = BD.RunSelector(base) run_list = runSelector.selectRuns(params) if (len(run_list) == 0): print("No runs to be cleared") validated = validate("Delete runs " + str([r[0].id for r in run_list])) for r, j in run_list: delete_flag = params["delete"] run_path = r["run_path"] if run_path: if os.path.exists(run_path): if (validated): print("Deleting directory: " + run_path) shutil.rmtree(run_path) else: print("Simulate deletion of directory: " + run_path) else: print("output directory: '" + run_path + "' not found: are we on the right machine ?") if (delete_flag): if validated: print("Deleting run " + str(r.id) + " from base") r.delete() base.commit() else: print("Simulate deletion of run " + str(r.id) + " from base") else: if validated: print("Deleting data associated with run " + str(r.id)) r.deleteData() r["STATE"] = "CREATED" r["start_time"] = None r.update() base.commit() else: print("Simulate deletion of data associated with run " + str(r.id)) diff --git a/scripts/getRunInfo.py b/scripts/getRunInfo.py index 57c31ee..e319aa4 100755 --- a/scripts/getRunInfo.py +++ b/scripts/getRunInfo.py @@ -1,244 +1,242 @@ #!/usr/bin/env python3 import BlackDynamite as BD import sys import datetime ################################################################ def printSummary(mybase, params): runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, quiet=True) print("*"*6 + " run summary => study {0} ".format(mybase.schema) + "*"*6) if len(run_list) == 0: print("no runs found") sys.exit(0) request = "SELECT run_name,state,count(state) from {0}.runs ".format( mybase.schema) if (len(run_list) > 0): request += "where id in (" + ",".join( [str(r.id) for r, j in run_list]) + ")" request += " group by state,run_name order by run_name,state" # print (request) curs = mybase.performRequest(request, []) stats = {} for i in curs: if i[0] not in stats: stats[i[0]] = [] stats[i[0]].append([i[1], int(i[2])]) for run_name, st in stats.items(): tot = 0 for n, count in st: tot += count for n, count in st: print("{:20} {:>20} => {:5} ({:>5.1f}%)".format( run_name, n, count, 100.*count/tot)) print("") sys.exit(0) ################################################################ def getRunInfo(run_id, mybase): myrun = BD.Run(mybase) myrun["id"] = run_id myrun.id = run_id run_list = myrun.getMatchedObjectList() if (len(run_list) == 0): print("no run found with id " + str(run_id)) sys.exit(1) myrun = run_list[0] myjob = BD.Job(mybase) myjob.id = myrun["job_id"] myjob["id"] = myrun["job_id"] job_list = myjob.getMatchedObjectList() if (len(job_list) == 0): print("no job found with id " + myjob.id) sys.exit(1) myjob = job_list[0] list_entries = myjob.entries.keys() print("*"*6 + " job info " + "*"*6) for entry in list_entries: if (myjob[entry]): print(entry + ": " + str(myjob[entry])) print("*"*6 + " run info " + "*"*6) list_entries = myrun.entries.keys() regular_run_entries = ("run_name", "job_id", "state", "start_time", "machine_name", "exec", "nproc", "wait_id") for entry in regular_run_entries: if (myrun[entry]): print(entry + ": " + str(myrun[entry])) list_entries.remove(entry) for entry in list_entries: if (myrun[entry]): print(entry + ": " + str(myrun[entry])) conffiles = myrun.getConfigFiles() for conf in conffiles: print("*"*6) print("file #" + str(conf.id) + ": " + conf["filename"]) print("*"*6) print(conf["file"]) ################################################################ def getInfoNames(): infos = [] infos.append("run_name") infos.append("id") infos.append("job_id") if "infos" in params: infos += params['infos'] else: infos += ["state", "nproc", "machine_name"] infos.append("start_time") infos.append("last step") infos.append("last update") infos.append("Time/step") infos.append("Total Time") return infos ################################################################ def getFormatString(infos): format_string = " {:<20} | {:^6} | {:^6} |" if "infos" in params: format_string += " {:^10} |" * len(params['infos']) else: format_string += " {:<15} | {:^5} | {:<20} |" format_string += " {:14} | {:>9} | {:>16} | {:>10} | {:>16} |" return format_string ################################################################ def formatTimeDelta(t): if (t < datetime.timedelta(seconds=1)): if (t < datetime.timedelta(microseconds=1000)): t = str(t.microseconds) + u'\u00B5'.encode('UTF-8') + "s" else: t = str(1./1000.*t.microseconds) + 'ms' else: ms = t.microseconds t -= datetime.timedelta(microseconds=ms) t = str(t) return t ################################################################ def getTimeInfos(r): step, steptime = r.getLastStep() start_time = r['start_time'] time_perstep = None total_time = None if (step is not None and steptime and start_time): time_perstep = (steptime-start_time)/(step+1) total_time = steptime-start_time time_perstep = formatTimeDelta(time_perstep) total_time = formatTimeDelta(total_time) if start_time: start_time = start_time.strftime("%H:%M %d/%m/%y") if steptime: steptime = steptime.strftime("%H:%M %d/%m/%y") run_infos = [start_time, step, steptime, time_perstep, total_time] return run_infos ################################################################ def getRunInfos(r, j): run_infos = [] for col in info_names[:-5]: key_run = col.replace('%r.', '').strip() if not key_run == 'start_time': if key_run in r.entries: run_infos.append(r[key_run]) else: key_job = col.replace('%j.', '').strip() if key_job in j.entries: run_infos.append(j[key_job]) else: raise Exception('Key {0} is not a valid parameter'.format( key_run)) run_infos += getTimeInfos(r) return run_infos ################################################################ parser = BD.BDParser() parser.register_params( group="getRunInfo", params={"run_id": int, "order": str, "summary": bool, "infos": [str]}, defaults={"order": "id"}, help={"run_id": "Select a run_id for complete output", "summary": "Output a summary of the completeness of the study", "order": "specify the column which serves to order the lines"}) params = parser.parseBDParameters() mybase = BD.Base(**params) if params["summary"] is True: printSummary(mybase, params) if ("run_id" in params): getRunInfo(params["run_id"], mybase) else: info_names = getInfoNames() format_string = getFormatString(info_names) header = format_string.format(*info_names) separator = "-" * len(header) print(separator) print(header) print(separator) - run_constraints = [] - job_constraints = [] runSelector = BD.RunSelector(mybase) run_list = runSelector.selectRuns(params, sort_by="runs." + params["order"], quiet=True) for r, j in run_list: try: infos = getRunInfos(r, j) def transform_None(x): if x is None: return 'None' else: return x infos = [transform_None(x) for x in infos] line = format_string.format(*infos) print(line) except Exception as e: print(getRunInfos(r, j)) print(e)