diff --git a/python/BlackDynamite/runselector.py b/python/BlackDynamite/runselector.py index a87078c..1108df0 100755 --- a/python/BlackDynamite/runselector.py +++ b/python/BlackDynamite/runselector.py @@ -1,112 +1,114 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -*- py-which-shell: "python"; -*- __all__ = [ "RunSelector" ] import copy import jobselector import base import run import job import sys import re class RunSelector(object): """ """ def selectRuns(self,run_constraints,job_constraints,sort_by = None,quiet=False,binary_operator = "and"): -# print 'AAAAAAAAA ' + binary_operator - if (type(run_constraints) == dict): if ("run_constraints" in run_constraints): run_constraints = run_constraints["run_constraints"] + elif "run_id" in run_constraints: + run_constraints = ['id = {0}'.format(run_constraints['run_id'])] else: run_constraints = [] if (type(job_constraints) == dict): if ("job_constraints" in job_constraints): job_constraints = job_constraints["job_constraints"] + elif "job_id" in job_constraints: + job_constraints = ['id = {0}'.format(job_constraints['job_id'])] else: job_constraints = [] myjob = job.Job(self.base) myjob.prepare() myjob.setConstraint(job_constraints) myrun = run.Run(self.base) myrun.prepare() myrun.setConstraint(run_constraints) if ("machine_name" in myrun.entries): if (len(myrun["machine_name"]) > 1): myrun["machine_name"] = myrun["machine_name"][0] condition_job, params_job = myjob.makeMatchingCondition(binary_operator) condition_run, params_run = myrun.makeMatchingCondition(binary_operator) condition = '' if condition_job: condition = '(' + condition_job + ')' if (condition and condition_run): condition += ' and ' if (condition_run): condition += '(' + condition_run + ')' if (condition): condition += ' and ' condition += " runs.job_id = jobs.id " params = params_job + params_run if (not type(sort_by) == list and sort_by is not None): sort_by = [sort_by] order_condition = "" if (sort_by is not None): order_condition = " ORDER BY (" for cond in sort_by: order_condition += cond + "," order_condition = order_condition[:len(order_condition)-1] + ")" # print ("order condition: " + order_condition) request = "SELECT * FROM {0}.jobs,{0}.runs ".format(self.base.schema) request += " WHERE " + condition request += order_condition # print (condition) # print (request) # print (params) curs = self.base.performRequest(request, params) run_list = self.buildJoinedList(myjob,myrun,curs) #run_list = self.getMatchedObjectList(myrun,myjob,order_condition) if (not quiet): print ("Selected runs are: " + str([r[0].id for r in run_list])) print ("Selected jobs are: " + str([r[0]["job_id"] for r in run_list])) return run_list def buildJoinedList(self, job_object, run_object, curs): coljob_info = self.base.getColumnProperties(job_object) colrun_info = self.base.getColumnProperties(run_object) run_list = [] for entries in curs: jobj = copy.deepcopy(job_object) robj = copy.deepcopy(run_object) ncoljob = len(coljob_info) ncolrun = len(colrun_info) for i in range(0,ncoljob): col_name = coljob_info[i][0] jobj[col_name] = entries[i] jobj.id = jobj["id"] for i in range(0,ncolrun): col_name = colrun_info[i][0] robj[col_name] = entries[i+ncoljob] robj.id = robj["id"] run_list.append([robj,jobj]) return run_list def __init__ (self,base): self.base = base diff --git a/src/run_manager.cc b/src/run_manager.cc index 2c84487..8cb287d 100644 --- a/src/run_manager.cc +++ b/src/run_manager.cc @@ -1,179 +1,180 @@ /* author : Nicolas RICHART author : Guillaume ANCIAUX (anciaux@labri.fr, g.anciaux@laposte.net) author : Till JUNGE */ /* -------------------------------------------------------------------------- */ #include #include "blackdynamite.hh" #include "sql_connection_manager.hh" #include "state_manager.hh" #include "quantities_manager.hh" #include #include #include /* -------------------------------------------------------------------------- */ namespace BlackDynamite { /* ------------------------------------------------------------------------ */ static std::string getenv(const std::string & var, std::string & default_val) { char * ptr = ::getenv(var.c_str()); if (!ptr) return default_val; return std::string(ptr); } static std::string getenv(const std::string & var) { char * ptr = ::getenv(var.c_str()); if (!ptr) FATAL("Undefined environment variable " + var); return std::string(ptr); } /* ------------------------------------------------------------------------ */ std::string RunManager::readPassword(const std::string & host) { std::string home = getenv("HOME"); std::string host_conf = home+ "/.blackdynamite/" + host + ".bd"; std::ifstream fin(host_conf.c_str()); std::string line; std::string password = ""; - std::cerr << "try to read the password" << std::endl; + // std::cerr << "try to read the password" << std::endl; while (fin.good()) { getline(fin, line); std::stringstream sstr(line); std::vector tokens; std::copy(std::istream_iterator(sstr), std::istream_iterator(), std::back_inserter(tokens)); if (tokens.size() == 3) { if (tokens[0] == "password" && tokens[1] == "="){ password = tokens[2]; break; } } } - std::cerr << "I have read the password: " << password << std::endl; + // std::cerr << "I have read the password: " << password << std::endl; + std::cerr << "I have read the password" << std::endl; return password; } /* ------------------------------------------------------------------------ */ RunManager::RunManager() { std::string user = getenv("BLACKDYNAMITE_USER"); std::string dbname = getenv("BLACKDYNAMITE_DBNAME", user); std::string host = getenv("BLACKDYNAMITE_HOST"); std::string schema = getenv("BLACKDYNAMITE_SCHEMA"); UInt run_id = atoi(getenv("BLACKDYNAMITE_RUN_ID").c_str()); std::string password = readPassword(host); init(dbname, user, host, schema, password, run_id); } /* ------------------------------------------------------------------------ */ RunManager::RunManager(const std::string & dbname, const std::string & user, const std::string & host, const std::string & schema, const std::string & password, UInt run_id) { init(dbname, user, host, schema, password, run_id); } /* ------------------------------------------------------------------------ */ RunManager::~RunManager() { delete quantities_manager; delete state_manager; } /* ------------------------------------------------------------------------ */ void RunManager::init(const std::string & dbname, const std::string & user, const std::string & host, const std::string & schema, const std::string & password, UInt run_id) { this->run_id = run_id; if (this->run_id <1) { FATAL("The run id required for the sql dumps has not been specified"); } this->sql_connection_manager = new SQLConnectionManager(dbname, user, host, schema, password); this->sql_schema = schema; this->quantities_manager = new QuantitiesManager(*this->sql_connection_manager, this->sql_schema, this->run_id); this->state_manager = new StateManager(*this->sql_connection_manager, this->sql_schema, this->run_id); } /* ------------------------------------------------------------------------ */ void RunManager::startRun(){ this->state_manager->changeState("START"); } /* ------------------------------------------------------------------------ */ void RunManager::endRun(){ this->state_manager->changeState("FINISHED"); } /* ------------------------------------------------------------------------ */ void RunManager::changeState(const std::string & state){ this->state_manager->changeState(state); } /* ------------------------------------------------------------------------ */ UInt RunManager::getInstanceCounter() { return this->sql_connection_manager->getInstanceCounter(); } /* ------------------------------------------------------------------------ */ template void RunManager::push(const T & value, const std::string & quantity, const UInt& step) { this->quantities_manager->pushQuantity(quantity, value, step); } /*------------------------------------------------------------------------- */ void RunManager::push(const std::string & value, const std::string & quantity) { this->quantities_manager->pushQuantity(quantity, value); } /* ------------------------------------------------------------------------ */ template void RunManager::push (const int & value, const std::string & quantity, const UInt & step); template void RunManager::push (const UInt & value, const std::string & quantity, const UInt & step); template void RunManager::push (const Real & value, const std::string & quantity, const UInt & step); template void RunManager::push>(const std::vector & value, const std::string & quantity, const UInt & step); template void RunManager::push> (const std::vector & value, const std::string & quantity, const UInt & step); /* ------------------------------------------------------------------------ */ unsigned int SQLConnectionManager::connection_counter = 0; pqxx::connection * SQLConnectionManager::sql_connection = NULL; /* ------------------------------------------------------------------------ */ } diff --git a/src/sql_query.hh b/src/sql_query.hh index 83ad21b..3f4ef12 100644 --- a/src/sql_query.hh +++ b/src/sql_query.hh @@ -1,275 +1,290 @@ /* author : Nicolas RICHART author : Till JUNGE */ #ifndef __BLACKDYNAMITE_SQL_QUERY__ #define __BLACKDYNAMITE_SQL_QUERY__ #include #include #include #include #include namespace BlackDynamite { /* -------------------------------------------------------------------------- */ template inline std::string nan_avoider(const num& val) { return pqxx::to_string(val); } /* -------------------------------------------------------------------------- */ template <> inline std::string nan_avoider(const double& val) { if (val != val) { #if PQXX_VERSION_MAJOR < 4 return "('nan')::double precision"; #else return "NaN"; #endif } else { /* Postgres has its own range definitions */ double fval = std::abs(val); if (fval <= 1e-307) { #if PQXX_VERSION_MAJOR < 4 return "(0.0)::double precision"; #else return "0."; #endif } else if (fval >=1e308) { if (val > 0.) { #if PQXX_VERSION_MAJOR < 4 return "('infinity')::double precision"; #else return "Infinity"; #endif } else { #if PQXX_VERSION_MAJOR < 4 return "('-infinity')::double precision"; #else return "-Infinity"; #endif } } std::stringstream val_rep; val_rep.precision(16); val_rep << std::scientific << val; return val_rep.str(); } } /** * Simple query handler as a secured transaction */ template< class transaction = pqxx::transaction > class SQLQuery : public pqxx::transactor { public: SQLQuery(const std::string & sql_query) : sql_query(sql_query) { } protected: SQLQuery() {} public: - virtual void operator() (transaction & trans) { - try { - this->result = this->execute(trans); - } catch (std::runtime_error & e) { - FATAL("Failed to execute query: " - << sql_query << std::endl << " with message: " << e.what()); + virtual void operator() (transaction & trans) throw { + UInt retry = 10000; + bool success = false; + std::string message; + std::runtime_error * error; + while (retry != 0){ + try { + this->result = this->execute(trans); + retry = 0; + success = true; + } catch (std::runtime_error & e) { + --retry; + message = e.what(); + error = &e; + } + } + if (!success){ + std::cerr << "Failed to execute query after " << retry << " retries " + << sql_query << std::endl << " with message: " << message; + throw *error; } } protected: virtual pqxx::result execute(pqxx::transaction_base & trans) { + return trans.exec(sql_query); } private: std::string sql_query; pqxx::result result; }; /* ------------------------------------------------------------------------ */ /** * Simple prepared query this query rely on sql queries previously prepared, * the SQLQueryPrepareHelper is done to help in this process */ template, typename... Arguments> class SQLPreparedQuery : public SQLQuery { public: SQLPreparedQuery(const std::string & query_name, Arguments... parameters) : query_name(query_name), parameters(std::make_tuple(parameters...)) { } /* ---------------------------------------------------------------------- */ /// execute a prepared function protected: virtual pqxx::result execute(pqxx::transaction_base & trans) { pqxx::prepare::invocation invoc = trans.prepared(query_name); return this->prepared<0, Arguments...>(invoc).exec(); } private: template inline pqxx::prepare::invocation & prepared(pqxx::prepare::invocation & invoc) { return invoc; } template inline pqxx::prepare::invocation & prepared(pqxx::prepare::invocation & invoc) { return this->prepared(this->addParameter(invoc, std::get(this->parameters))); } template inline pqxx::prepare::invocation & addParameter(pqxx::prepare::invocation & invoc, const T & param) { return invoc(param); } template struct NanAvoiderIT { const std::string operator()(typename std::vector::iterator & it) { return nan_avoider(*it); } }; template inline pqxx::prepare::invocation & addParameter(pqxx::prepare::invocation & invoc, const std::vector & param) { std::string join = nan_avoider(param[0]); for(auto it = param.begin() + 1; it != param.end(); ++it) { join += ", "; join += nan_avoider(*it); } return invoc("{" + join + "}"); // return invoc("ARRAY [" + pqxx::separated_list(",", param.begin(), param.end(), NanAvoiderIT*() + "]"); } private: std::string query_name; std::tuple parameters; }; /* ------------------------------------------------------------------------ */ #if PQXX_VERSION_MAJOR < 4 /* ---------------------------------------------------------------------- */ // Convertion functions /// Function to convert a c++ type in a SQL type for the preparation template struct InternalPrepareHelper { }; /// convert std::string to "varchar" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("varchar", pqxx::prepare::treat_string); } }; /// convert double to "real" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("double precision"); } }; /// convert std::vector to "double precision[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("double precision[]"); } }; /// convert std::vector to "interger[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer[]"); } }; /// convert std::vector to "interger[]" template<> struct InternalPrepareHelper> { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer[]"); } }; /// convert int to "interger" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer"); } }; /// convert unsigned int to "interger" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("integer"); } }; /// convert bool to "boolean" template<> struct InternalPrepareHelper { static const pqxx::prepare::declaration & addArgumentType(const pqxx::prepare::declaration & decl) { return decl("boolean"); } }; #endif /** * SQLQueryPrepareHelper */ class SQLQueryPrepareHelper { /* ---------------------------------------------------------------------- */ // preparation function public: /// prepare the request in the connection static void prepare(pqxx::connection_base & connection, const std::string & request_name, const std::string & sql_request) { connection.prepare(request_name, sql_request); } /// prepare the request in the connection template static void prepare(pqxx::connection_base & connection, const std::string & request_name, const std::string & sql_request) { #if PQXX_VERSION_MAJOR < 4 pqxx::prepare::declaration decl = connection.prepare(request_name, sql_request); prepare(decl); #else connection.prepare(request_name, sql_request); #endif } static void unprepare(pqxx::connection_base & connection, const std::string & request_name) { connection.unprepare(request_name); } #if PQXX_VERSION_MAJOR < 4 private: template static const pqxx::prepare::declaration & prepare(const pqxx::prepare::declaration & decl) { return decl; } template static const pqxx::prepare::declaration & prepare(const pqxx::prepare::declaration & decl) { return prepare(InternalPrepareHelper::addArgumentType(decl)); } #endif }; } #endif //__BLACKDYNAMITE_SQL_QUERY__