Homec4science
Diffusion LibMultiScale (features/coupler_component) Browse

libmultiscale/src/common/reference_manager.cc
f46e4477a0fcfeatures/coupler_compon…

Last Change
Raw File

reference_manager.cc
View Options

/**
* @file reference_manager.cc
*
* @author Guillaume Anciaux <guillaume.anciaux@epfl.ch>
*
* @date Mon Sep 08 23:40:22 2014
*
* @brief This is the manager of reference and coherency with migrations
*
* @section LICENSE
*
* Copyright INRIA and CEA
*
* The LibMultiScale is a C++ parallel framework for the multiscale
* coupling methods dedicated to material simulations. This framework
* provides an API which makes it possible to program coupled simulations
* and integration of already existing codes.
*
* This Project was initiated in a collaboration between INRIA Futurs Bordeaux
* within ScAlApplix team and CEA/DPTA Ile de France.
* The project is now continued at the Ecole Polytechnique Fédérale de Lausanne
* within the LSMS/ENAC laboratory.
*
* This software is governed by the CeCILL-C license under French law and
* abiding by the rules of distribution of free software. You can use,
* modify and/ or redistribute the software under the terms of the CeCILL-C
* license as circulated by CEA, CNRS and INRIA at the following URL
* "http://www.cecill.info".
*
* As a counterpart to the access to the source code and rights to copy,
* modify and redistribute granted by the license, users are provided only
* with a limited warranty and the software's author, the holder of the
* economic rights, and the successive licensors have only limited
* liability.
*
* In this respect, the user's attention is drawn to the risks associated
* with loading, using, modifying and/or developing or reproducing the
* software by the user in light of its specific status of free software,
* that may mean that it is complicated to manipulate, and that also
* therefore means that it is reserved for developers and experienced
* professionals having in-depth computer knowledge. Users are therefore
* encouraged to load and test the software's suitability as regards their
* requirements in conditions enabling the security of their systems and/or
* data to be ensured and, more generally, to use and operate it in the
* same conditions as regards security.
*
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*
*/
/* -------------------------------------------------------------------------- */
#include <boost/preprocessor.hpp>
/* -------------------------------------------------------------------------- */
#include "lib_continuum.hh"
#include "lib_dd.hh"
#include "lib_md.hh"
#include "lm_common.hh"
#include "reference_manager.hh"
/* -------------------------------------------------------------------------- */
__BEGIN_LIBMULTISCALE__
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::printPackBuffersStatus() {
typename BufferMap::iterator it;
typename BufferMap::iterator end;
// create a packbuffer for each proc I should receive from
// and clear it
for (it = buffers_torecv.begin(), end = buffers_torecv.end(); it != end;
++it) {
#ifndef LM_OPTIMIZED
UInt proc = (*it).first;
PackBuffer &buf = (*it).second;
#endif
DUMP("from " << proc << " receive buffer status: " << buf.size() << " "
<< buf.remainingSize(),
DBG_INFO);
}
// same thing for sending buffers
for (it = buffers_tosend.begin(), end = buffers_tosend.end(); it != end;
++it) {
#ifndef LM_OPTIMIZED
UInt proc = (*it).first;
PackBuffer &buf = (*it).second;
#endif // LM_OPTIMIZED
DUMP("to " << proc << " send buffer status: " << buf.size() << " "
<< buf.remainingSize(),
DBG_INFO);
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::clearPackBuffers() {
// clear buffers
buffers_tosend.clear();
buffers_torecv.clear();
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
// create a packbuffer for each proc I should receive from
// and clear it
for (it = newatoms.begin(), end = newatoms.end(); it != end; ++it) {
PackBuffer &buf = buffers_torecv[(*it).first];
buf.clear();
}
// same thing for sending buffers
for (it = sent_byproc.begin(), end = sent_byproc.end(); it != end; ++it) {
PackBuffer &buf = buffers_tosend[(*it).first];
buf.clear();
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref> void ReferenceManager<Ref>::updateRefSubSets() {
MapRefToUInt masks_send;
MapRefToUInt masks_recv;
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
if (have_changed)
DUMP("have_changed = " << have_changed, DBG_INFO);
// do a bilan and checkup of coherency state
// printBilan();
/* --------------------------------------------------------------------------
*/
// new stufff
/* --------------------------------------------------------------------------
*/
#ifndef LM_OPTIMIZED
std::map<LMID, UInt> n_tot_subset;
// count number of atoms in before subsets
for (auto &s : subsets) {
UInt n = s.second->getContainer().size();
UInt n_tot;
MPI_Reduce(&n, &n_tot, 1, MPI_INT, MPI_SUM, 0, worldCom);
if (lm_my_proc_id == 0)
n_tot_subset[s.first] = n_tot;
}
#endif
// declare the atom gone to other processors to attached subsets
for (auto &s : subsets)
s.second->declareSentAtoms(sent, sent_byproc);
global_set.declareSentAtoms(sent, sent_byproc);
// build the mask of ownership for each atom
for (auto &s : subsets)
s.second->buildMasksForSentAtoms(masks_send);
// initialize the buffers
clearPackBuffers();
// pack the masks to a packbuffer
packMasks(masks_send);
// exchange the masks
exchangeBuffers(buffers_tosend, buffers_torecv);
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
// unpack the masks to a MapRefToInt
unpackMasks(masks_recv);
// translate references
for (auto &s : subsets)
s.second->translateMovingReferences(moved);
// declare the received atoms and mask
for (auto &s : subsets)
s.second->declareRecvAtoms(masks_recv, newatoms);
global_set.declareRecvAtoms(newatoms);
// initialize the buffers
clearPackBuffers();
printPackBuffersStatus();
// pack attached data
for (auto &s : subsets)
s.second->packAttachedData(buffers_tosend);
global_set.packAttachedData(buffers_tosend);
// do the communication
exchangeBuffers(buffers_tosend, buffers_torecv);
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
printPackBuffersStatus();
// unpack attached data
for (auto &s : subsets)
s.second->unpackAttachedData(buffers_torecv);
global_set.unpackAttachedData(buffers_torecv);
// translate references
for (auto &s : subsets)
s.second->fillRemainingHoles();
global_set.resize();
#ifndef LM_OPTIMIZED
// count number of atoms in subsets
for (auto &s : subsets) {
UInt n = s.second->getContainer().size();
UInt n_tot;
MPI_Reduce(&n, &n_tot, 1, MPI_INT, MPI_SUM, 0, worldCom);
if (lm_my_proc_id == 0)
if (n_tot != n_tot_subset[s.first])
LM_FATAL("subset " << s.first << " mismatch of atom count " << n_tot
<< " " << n_tot_subset[s.first]);
}
#endif
// clear the stl structures
moved.clear();
sent.clear();
sent_byproc.clear();
newatoms.clear();
have_changed = false;
DUMP("end update ref", DBG_INFO);
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::exchangeBuffers(BufferMap &toSend,
BufferMap &toRecv) {
// the size (in bytes) to be received map with the proc I receive from
// std::map<UInt,UInt> recv_sizes;
// the mpi_requests map by processors
std::map<int, MPI_Request> send_requests;
// iterators on buffers
BufferMap::iterator it;
BufferMap::iterator end;
// iterators on MPI requests
std::map<int, MPI_Request>::iterator it_req;
std::map<int, MPI_Request>::iterator end_req;
// MPIstatus object useful for the wait operations
MPI_Status status;
/* --------------------------------------------------------------------------
*/
// now send the buffers through network
/* --------------------------------------------------------------------------
*/
// sending of the buffers
for (end = toSend.end(), it = toSend.begin(); it != end; ++it) {
UInt proc = (*it).first;
if (proc == UINT_MAX)
continue;
UInt size = (*it).second.size();
char *buffer = (*it).second.buffer();
MPI_Request &request = send_requests[proc];
MPI_Isend(buffer, size, MPI_CHAR, proc, rank, worldCom, &request);
}
// reception async of the vectors
for (end = toRecv.end(), it = toRecv.begin(); it != end; ++it) {
UInt proc = (*it).first;
if (proc == UINT_MAX)
continue;
DUMP("probing from proc " << proc, DBG_INFO);
MPI_Probe(proc, proc, worldCom, &status);
int size = 0;
MPI_Get_count(&status, MPI_CHAR, &size);
PackBuffer &buf = toRecv[proc];
buf.resize(size);
char *buffer = toRecv[proc].buffer();
DUMP("receiving from proc " << proc << " " << size << " bytes", DBG_INFO);
MPI_Recv(buffer, size, MPI_CHAR, proc, proc, worldCom, &status);
DUMP("received packet from proc " << proc << " of size " << size
<< " bytes",
DBG_DETAIL);
}
// now I wait for requests to be acheived
for (end_req = send_requests.end(), it_req = send_requests.begin();
it_req != end_req; ++it_req) {
UInt proc = (*it_req).first;
if (proc == UINT_MAX)
continue;
#ifndef LM_OPTIMIZED
UInt size = toRecv[proc].size();
#endif // LM_OPTIMIZED
DUMP("waiting to send a packet to proc " << proc << " of size " << size
<< " bytes",
DBG_INFO);
MPI_Wait(&((*it_req).second), &status);
DUMP("sent packet to proc " << proc << " of size " << size << " bytes",
DBG_DETAIL);
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::packMasks(MapRefToUInt &masks) {
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
// pack the masks
for (it = sent_byproc.begin(), end = sent_byproc.end(); it != end; ++it) {
UInt proc = (*it).first;
std::vector<Ref> &sendlist = (*it).second;
for (UInt i = 0; i < sendlist.size(); ++i) {
Ref at = sendlist[i];
buffers_tosend[proc] << masks[at];
}
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::unpackMasks(MapRefToUInt &masks) {
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
for (it = newatoms.begin(), end = newatoms.end(); it != end; ++it) {
UInt proc = (*it).first;
std::vector<Ref> &recvlist = (*it).second;
for (UInt i = 0; i < recvlist.size(); ++i) {
Ref at = recvlist[i];
UInt m;
buffers_torecv[proc] >> m;
masks[at] = m;
}
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref> void ReferenceManager<Ref>::printBilan() {
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
typename MapRefToUInt::iterator it2;
typename MapRefToUInt::iterator end2;
std::map<UInt, UInt> nb_sent_by_proc;
if (moved.size())
DUMP(moved.size() << " atoms moved in memory", DBG_WARNING);
for (end = newatoms.end(), it = newatoms.begin(); it != end; ++it)
DUMP((*it).second.size() << " atoms received from proc " << (*it).first,
DBG_WARNING);
for (end = sent_byproc.end(), it = sent_byproc.begin(); it != end; ++it)
DUMP((*it).second.size() << " atoms sent to proc " << (*it).first,
DBG_WARNING);
for (end2 = sent.end(), it2 = sent.begin(); it2 != end2; ++it2)
++nb_sent_by_proc[(*it2).second];
for (end = sent_byproc.end(), it = sent_byproc.begin(); it != end; ++it) {
if ((*it).second.size() != nb_sent_by_proc[(*it).first])
LM_FATAL("sent_by_proc registered "
<< (*it).second.size() << " atoms as sent but sent registered "
<< nb_sent_by_proc[(*it).first]
<< " fatal inconstistency in send maps");
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::addSubSet(ContainerInterface &s) {
auto &subset = dynamic_cast<ContainerArray<Ref> &>(s);
auto name = subset.getID();
auto it = subsets.find(name);
if (it == subsets.end()) {
DUMP(name << " was already registered", DBG_INFO);
}
for (auto &s : subsets) {
if (&s.second->getContainer() == &subset) {
DUMP(name << " was already registered", DBG_INFO);
return;
}
}
UInt new_subset_id = subsets.size();
auto sub = std::make_shared<RefSubset<ContainerArray<Ref>>>(
subset, new_subset_id, name);
DUMP("adding a subset to managed subset named " << name << " " << &subset,
DBG_INFO_STARTUP);
subsets[name] = sub;
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::removeSubSet(const std::string &name) {
subsets.erase(name);
#ifndef LM_OPTIMIZED
UInt size = subsets.size();
DUMP("removing a subset from managed subset at position "
<< size << " and named " << name,
DBG_INFO);
#endif
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
ContainerInterface &cont) {
DUMP("try to attach vector<Real> " << &v << " to subset " << &cont,
DBG_MESSAGE);
UInt i = 0;
for (auto &&[name, subset] : subsets) {
if (&subset->getContainer() == &cont) {
auto atV = std::make_shared<AttachedVector<Real>>(v);
if (subset->attachData(*atV))
DUMP("attaching vector<Real> " << &v << " to subset " << subset->getID()
<< " " << &cont,
DBG_MESSAGE);
break;
}
++i;
}
if (i == subsets.size()) {
DUMP("attaching vector<Real> could not be done : did not find its subset "
<< &cont << "\nRegistered vectors are ",
DBG_MESSAGE);
for (auto &s : subsets) {
DUMP(s.first << " : " << &s.second->getContainer() << "\n", DBG_MESSAGE);
}
LM_FATAL("abort");
}
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
GlobalContainer &) {
AttachedVector<Real> *atV = new AttachedVector<Real>(v);
global_set.attachData(*atV);
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::attachObject(AttachedObject &obj,
ContainerInterface &cont) {
bool found = false;
for (auto &&[name, subset] : subsets) {
if (&subset->getContainer() == &cont) {
DUMP("attaching object " << &obj << " to subset " << subset->getID(),
DBG_INFO);
subset->attachData(obj);
found = true;
break;
}
}
if (not found)
LM_FATAL("attaching vector<Real> could not be done : did not found its "
"subset");
}
/* --------------------------------------------------------------------------
*/
template <typename Ref>
void ReferenceManager<Ref>::detachObject(AttachedObject &obj,
ContainerInterface &cont) {
auto &c = dynamic_cast<ContainerArray<Ref> &>(cont);
bool found = false;
for (auto &&[name, subset] : subsets) {
if (&subset->getContainer() == &c) {
DUMP("detaching object " << &obj << " from subset " << subset->getID(),
DBG_INFO);
subset->detachData(obj);
found = true;
break;
}
}
if (not found)
LM_FATAL("attaching vector<Real> could not be done : did not found its "
"subset");
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::detachVector(ContainerArray<Real> &,
ContainerInterface &) {
LM_TOIMPLEMENT;
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::setCommGroup(CommGroup &comm) {
worldCom = comm.getMPIComm();
if (worldCom != MPI_COMM_NULL)
MPI_Comm_rank(worldCom, &rank);
}
#define DECLARE_REF_MANAGER(n, data, obj) \
template class ReferenceManager<BOOST_PP_TUPLE_ELEM(3, 0, obj)::RefPoint>;
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_ATOM_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_DD_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_CONTINUUM_MODEL)
#undef DECLARE_REF_MANAGER
__END_LIBMULTISCALE__
c4science · Help