Homec4science
Diffusion LibMultiScale (features/new_parralelism) Browse

libmultiscale/src/common/reference_manager.cc
a4bb12a292b8features/new_parralelism

Last Change
Raw File

reference_manager.cc
View Options

/**
* @file reference_manager.cc
*
* @author Guillaume Anciaux <guillaume.anciaux@epfl.ch>
*
* @date Mon Sep 08 23:40:22 2014
*
* @brief This is the manager of reference and coherency with migrations
*
* @section LICENSE
*
* Copyright INRIA and CEA
*
* The LibMultiScale is a C++ parallel framework for the multiscale
* coupling methods dedicated to material simulations. This framework
* provides an API which makes it possible to program coupled simulations
* and integration of already existing codes.
*
* This Project was initiated in a collaboration between INRIA Futurs Bordeaux
* within ScAlApplix team and CEA/DPTA Ile de France.
* The project is now continued at the Ecole Polytechnique Fédérale de Lausanne
* within the LSMS/ENAC laboratory.
*
* This software is governed by the CeCILL-C license under French law and
* abiding by the rules of distribution of free software. You can use,
* modify and/ or redistribute the software under the terms of the CeCILL-C
* license as circulated by CEA, CNRS and INRIA at the following URL
* "http://www.cecill.info".
*
* As a counterpart to the access to the source code and rights to copy,
* modify and redistribute granted by the license, users are provided only
* with a limited warranty and the software's author, the holder of the
* economic rights, and the successive licensors have only limited
* liability.
*
* In this respect, the user's attention is drawn to the risks associated
* with loading, using, modifying and/or developing or reproducing the
* software by the user in light of its specific status of free software,
* that may mean that it is complicated to manipulate, and that also
* therefore means that it is reserved for developers and experienced
* professionals having in-depth computer knowledge. Users are therefore
* encouraged to load and test the software's suitability as regards their
* requirements in conditions enabling the security of their systems and/or
* data to be ensured and, more generally, to use and operate it in the
* same conditions as regards security.
*
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*
*/
/* -------------------------------------------------------------------------- */
#include <boost/preprocessor.hpp>
/* -------------------------------------------------------------------------- */
#include "lib_continuum.hh"
#include "lib_dd.hh"
#include "lib_md.hh"
#include "lm_common.hh"
#include "lm_communicator.hh"
#include "reference_manager.hh"
/* -------------------------------------------------------------------------- */
__BEGIN_LIBMULTISCALE__
/* -------------------------------------------------------------------------- */
template <typename Ref>
ReferenceManager<Ref>::ReferenceManager(
typename Ref::Domain::ContainerPoints &global_container) {
auto name = global_container.getID();
subsets[name] = make_subset<false>(global_container, 0);
}
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::printPackBuffersStatus() {
#ifndef LM_OPTIMIZED
for (auto &&[proc, buf] : buffers_torecv) {
DUMP("from " << proc << " receive buffer status: " << buf.size() << " "
<< buf.remainingSize(),
DBG_INFO);
}
// same thing for sending buffers
for (auto &&[proc, buf] : buffers_tosend) {
DUMP("to " << proc << " send buffer status: " << buf.size() << " "
<< buf.remainingSize(),
DBG_INFO);
}
#endif // LM_OPTIMIZED
}
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::clearPackBuffers() {
// clear buffers
buffers_tosend.clear();
buffers_torecv.clear();
// create a packbuffer for each proc I should receive from
// and clear it
for (auto &&[proc, ref] : newrefs) {
PackBuffer &buf = buffers_torecv[proc];
buf.clear();
}
// same thing for sending buffers
for (auto &&[proc, ref] : sent_byproc) {
PackBuffer &buf = buffers_tosend[proc];
buf.clear();
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::updateRefSubSets() {
MapRefToUInt masks_send;
MapRefToUInt masks_recv;
//#ifndef LM_OPTIMIZED
comm_group->synchronize(); // MPI_Barrier(worldCom);
//#endif
if (have_changed) {
DUMP("have_changed = " << have_changed, DBG_INFO);
}
// do a bilan and checkup of coherency state
// printBilan();
#ifndef LM_OPTIMIZED
std::map<LMID, UInt> n_tot_subset;
// count number of refs in before subsets
for (auto &&[name, s] : subsets) {
UInt n_tot = s->getContainer().size();
comm_group->reduce(&n_tot, 1, "n_tot", OP_SUM, 0);
// MPI_Reduce(&n, &n_tot, 1, MPI_INT, MPI_SUM, 0, worldCom);
if (lm_my_proc_id == 0)
n_tot_subset[name] = n_tot;
}
#endif
// declare the ref gone to other processors to attached subsets
for (auto &&[name, s] : subsets)
s->declareSentRefs(sent, sent_byproc);
// global_set.declareSentAtoms(sent, sent_byproc);
// build the mask of ownership for each ref
for (auto &&[name, s] : subsets)
s->buildMasksForSentRefs(masks_send);
// initialize the buffers
clearPackBuffers();
// pack the masks to a packbuffer
packMasks(masks_send);
// exchange the masks
exchangeBuffers(buffers_tosend, buffers_torecv);
//#ifndef LM_OPTIMIZED
comm_group->synchronize(); // MPI_Barrier(worldCom);
//#endif
// unpack the masks to a MapRefToInt
unpackMasks(masks_recv);
// translate references
for (auto &&[name, s] : subsets)
s->translateMovingReferences(moved);
// declare the received refs and mask
for (auto &&[name, s] : subsets)
s->declareRecvRefs(masks_recv, newrefs);
// global_set.declareRecvAtoms(newatoms);
// initialize the buffers
clearPackBuffers();
printPackBuffersStatus();
// pack attached data
for (auto &&[name, s] : subsets)
s->packAttachedData(buffers_tosend);
// global_set.packAttachedData(buffers_tosend);
// do the communication
exchangeBuffers(buffers_tosend, buffers_torecv);
//#ifndef LM_OPTIMIZED
comm_group->synchronize(); // MPI_Barrier(worldCom);
//#endif
printPackBuffersStatus();
// unpack attached data
for (auto &&[name, s] : subsets)
s->unpackAttachedData(buffers_torecv);
// global_set.unpackAttachedData(buffers_torecv);
// translate references
for (auto &&[name, s] : subsets)
s->fillRemainingHoles();
UInt current_size [[gnu::unused]];
for (auto &&[name, s] : subsets) {
if (not s->isSubset())
current_size = s->resize();
}
// for (auto &&[name, s] : subsets) {
// if (s->isSubset())
// s->resize(current_size);
// }
#ifndef LM_OPTIMIZED
// count number of refs in subsets
for (auto &&[name, s] : subsets) {
UInt n_tot = s->getContainer().rows();
DUMP("container:" << name << " size: " << n_tot, DBG_INFO);
comm_group->reduce(&n_tot, 1, "n_tot", OP_SUM, 0);
if (lm_my_proc_id == 0)
if (n_tot != n_tot_subset[name])
LM_FATAL("subset " << name << " mismatch of ref count " << n_tot << " "
<< n_tot_subset[name]);
}
#endif
// clear the stl structures
moved.clear();
sent.clear();
sent_byproc.clear();
newrefs.clear();
have_changed = false;
DUMP("end update ref", DBG_INFO);
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::exchangeBuffers(BufferMap &toSend,
BufferMap &toRecv) {
std::map<int, Request> send_requests;
/* ------------------------------------------------------------------------ */
// now send the buffers through network
/* ------------------------------------------------------------------------ */
// sending of the buffers
for (auto &&[proc, buf_send] : toSend) {
if (proc == UINT_MAX)
continue;
UInt size = buf_send.size();
auto buffer = buf_send.buffer();
send_requests[proc] =
comm_group->isend(buffer, size, proc, "ref_manager send");
// MPI_Isend(buffer, size, MPI_CHAR, proc, rank, worldCom, &request);
}
// reception async of the vectors
for (auto &&[proc, buffer_to_recv] : toRecv) {
if (proc == UINT_MAX)
continue;
DUMP("probing from proc " << proc, DBG_INFO);
int size = comm_group->probe<char>(
proc,
"probing ref_manager recv"); // MPI_Probe(proc, proc,
// worldCom, &status);
PackBuffer &buf = toRecv[proc];
buf.resize(size);
auto buffer = buffer_to_recv.buffer();
DUMP("receiving from proc " << proc << " " << size << " bytes", DBG_INFO);
comm_group->receive(
buffer, size, proc,
"recv ref_manager buffer"); //(buffer, size, MPI_CHAR, proc, proc,
// worldCom, &status);
DUMP("received packet from proc " << proc << " of size " << size
<< " bytes",
DBG_DETAIL);
}
// now I wait for requests to be acheived
for (auto &&[proc, request] : send_requests) {
if (proc == int(UINT_MAX))
continue;
#ifndef LM_OPTIMIZED
UInt size = toRecv[proc].size();
DUMP("waiting to send a packet to proc " << proc << " of size " << size
<< " bytes",
DBG_INFO);
#endif // LM_OPTIMIZED
comm_group->wait(request);
DUMP("sent packet to proc " << proc << " of size " << size << " bytes",
DBG_DETAIL);
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::packMasks(MapRefToUInt &masks) {
// pack the masks
for (auto &&[proc, sendlist] : sent_byproc) {
for (UInt i = 0; i < sendlist.size(); ++i) {
Ref at = sendlist[i];
buffers_tosend[proc] << masks[at];
}
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::unpackMasks(MapRefToUInt &masks) {
for (auto &&[proc, recvlist] : newrefs) {
for (UInt i = 0; i < recvlist.size(); ++i) {
Ref at = recvlist[i];
UInt m;
buffers_torecv[proc] >> m;
masks[at] = m;
}
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref> void ReferenceManager<Ref>::printBilan() {
std::map<UInt, UInt> nb_sent_by_proc;
if (moved.size())
DUMP(moved.size() << " refs moved in memory", DBG_WARNING);
for (auto &&[proc, refs] : newrefs)
DUMP(refs.size() << " refs received from proc " << proc, DBG_WARNING);
for (auto &&[proc, sent] : sent_byproc)
DUMP(sent.size() << " refs sent to proc " << proc, DBG_WARNING);
for (auto &&it : sent)
++nb_sent_by_proc[it.second];
for (auto &&[proc, sentlist] : sent_byproc) {
if (sentlist.size() != nb_sent_by_proc[proc])
LM_FATAL("sent_by_proc registered "
<< sentlist.size() << " refs as sent but sent registered "
<< nb_sent_by_proc[proc]
<< " fatal inconstistency in send maps");
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
template <typename Cont>
std::enable_if_t<std::is_same_v<typename Cont::Ref, Ref>>
ReferenceManager<Ref>::addSubset(Cont &container) {
auto name = container.getID();
auto it = subsets.find(name);
if (it != subsets.end()) {
DUMP(name << " was already registered", DBG_INFO);
return;
}
// UInt new_subset_id = subsets.size();
auto subset = make_subset<true>(container, subsets.size());
DUMP("adding a subset to managed subset named " << name << " " << &container,
DBG_INFO_STARTUP);
subsets[name] = subset;
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::removeSubset(const std::string &name) {
subsets.erase(name);
#ifndef LM_OPTIMIZED
UInt size = subsets.size();
DUMP("removing a subset from managed subset at position "
<< size << " and named " << name,
DBG_INFO);
#endif
}
/* ------------------------------------------------------------------------ */
// template <typename Ref>
// void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
// ContainerInterface &cont) {
// DUMP("try to attach vector<Real> " << &v << " to subset " << &cont,
// DBG_MESSAGE);
// UInt i = 0;
// for (auto &&[name, subset] : subsets) {
// if (&subset->getContainer() == &cont) {
// auto atV = std::make_shared<AttachedVector<Real>>(v);
// if (subset->attachData(*atV))
// DUMP("attaching vector<Real> " << &v << " to subset " << cont.getID()
// << " " << &cont,
// DBG_MESSAGE);
// break;
// }
// ++i;
// }
// if (i == subsets.size()) {
// DUMP("attaching vector<Real> could not be done : did not find its subset
// "
// << &cont << "\nRegistered vectors are ",
// DBG_MESSAGE);
// for (auto &s : subsets) {
// DUMP(s.first << " : " << &s.second->getContainer() << "\n",
// DBG_MESSAGE);
// }
// LM_FATAL("abort");
// }
// }
/* ------------------------------------------------------------------------ */
// template <typename Ref>
// void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
// GlobalContainer &) {
// AttachedVector<Real> *atV = new AttachedVector<Real>(v);
// global_set.attachData(*atV);
// }
/* --------------------------------------------------------------------- */
// template <typename Ref>
// void ReferenceManager<Ref>::detachVector(ContainerArray<Real> &,
// ContainerInterface &) {
// LM_TOIMPLEMENT;
// }
// /* --------------------------------------------------------------------------
// */
template <typename Ref>
void ReferenceManager<Ref>::setCommGroup(CommGroup &group) {
comm_group = &group;
}
// worldCom = comm.getMPIComm();
// if (worldCom != MPI_COMM_NULL)
// MPI_Comm_rank(worldCom, &rank);
// }
#define DECLARE_REF_MANAGER(n, data, obj) \
template class ReferenceManager<BOOST_PP_TUPLE_ELEM(3, 0, obj)::RefPoint>;
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::acquireContext(const LMObject &obj) {
this->setCommGroup(obj.getCommGroup());
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::addSubset(ContainerInterface &cont) {
this->addSubset<dispatch>(cont);
}
/* -------------------------------------------------------------------------- */
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_ATOM_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_DD_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_CONTINUUM_MODEL)
#undef DECLARE_REF_MANAGER
__END_LIBMULTISCALE__
c4science · Help