Homec4science
Diffusion LibMultiScale (master) Browse

libmultiscale/src/common/reference_manager.cc
b05d5f2aba78master

Last Change
Raw File

reference_manager.cc
View Options

/**
* @file reference_manager.cc
*
* @author Guillaume Anciaux <guillaume.anciaux@epfl.ch>
*
* @date Mon Sep 08 23:40:22 2014
*
* @brief This is the manager of reference and coherency with migrations
*
* @section LICENSE
*
* Copyright INRIA and CEA
*
* The LibMultiScale is a C++ parallel framework for the multiscale
* coupling methods dedicated to material simulations. This framework
* provides an API which makes it possible to program coupled simulations
* and integration of already existing codes.
*
* This Project was initiated in a collaboration between INRIA Futurs Bordeaux
* within ScAlApplix team and CEA/DPTA Ile de France.
* The project is now continued at the Ecole Polytechnique Fédérale de Lausanne
* within the LSMS/ENAC laboratory.
*
* This software is governed by the CeCILL-C license under French law and
* abiding by the rules of distribution of free software. You can use,
* modify and/ or redistribute the software under the terms of the CeCILL-C
* license as circulated by CEA, CNRS and INRIA at the following URL
* "http://www.cecill.info".
*
* As a counterpart to the access to the source code and rights to copy,
* modify and redistribute granted by the license, users are provided only
* with a limited warranty and the software's author, the holder of the
* economic rights, and the successive licensors have only limited
* liability.
*
* In this respect, the user's attention is drawn to the risks associated
* with loading, using, modifying and/or developing or reproducing the
* software by the user in light of its specific status of free software,
* that may mean that it is complicated to manipulate, and that also
* therefore means that it is reserved for developers and experienced
* professionals having in-depth computer knowledge. Users are therefore
* encouraged to load and test the software's suitability as regards their
* requirements in conditions enabling the security of their systems and/or
* data to be ensured and, more generally, to use and operate it in the
* same conditions as regards security.
*
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*
*/
/* -------------------------------------------------------------------------- */
#include <boost/preprocessor.hpp>
/* -------------------------------------------------------------------------- */
#include "lm_common.hh"
#include "lib_md.hh"
#include "lib_continuum.hh"
#include "lib_dd.hh"
#include "reference_manager.hh"
/* -------------------------------------------------------------------------- */
__BEGIN_LIBMULTISCALE__
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::printPackBuffersStatus(){
typename BufferMap::iterator it;
typename BufferMap::iterator end;
// create a packbuffer for each proc I should receive from
// and clear it
for (it = buffers_torecv.begin(), end = buffers_torecv.end();it != end ; ++it){
#ifndef LM_OPTIMIZED
UInt proc = (*it).first;
PackBuffer & buf = (*it).second;
#endif
DUMP("from " << proc << " receive buffer status: "
<< buf.size() << " " << buf.remainingSize(),DBG_INFO);
}
// same thing for sending buffers
for (it = buffers_tosend.begin(), end = buffers_tosend.end();it != end ; ++it){
#ifndef LM_OPTIMIZED
UInt proc = (*it).first;
PackBuffer & buf = (*it).second;
#endif// LM_OPTIMIZED
DUMP("to " << proc << " send buffer status: "
<< buf.size() << " " << buf.remainingSize(),DBG_INFO);
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::clearPackBuffers(){
//clear buffers
buffers_tosend.clear();
buffers_torecv.clear();
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
// create a packbuffer for each proc I should receive from
// and clear it
for (it = newatoms.begin(), end = newatoms.end();it != end ; ++it){
PackBuffer & buf = buffers_torecv[(*it).first];
buf.clear();
}
// same thing for sending buffers
for (it = sent_byproc.begin(), end = sent_byproc.end();it != end ; ++it){
PackBuffer & buf = buffers_tosend[(*it).first];
buf.clear();
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::updateRefSubSets(){
MapRefToUInt masks_send;
MapRefToUInt masks_recv;
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
if (have_changed) DUMP("have_changed = " << have_changed,DBG_INFO);
// do a bilan and checkup of coherency state
//printBilan();
/* -------------------------------------------------------------------------- */
//new stufff
/* -------------------------------------------------------------------------- */
//get the number of subsets
UInt n_subsets = subsets.size();
#ifndef LM_OPTIMIZED
std::map<UInt,UInt> n_tot_subset;
//count number of atoms in before subsets
for (UInt i = 0 ; i < n_subsets ; ++i){
UInt n = subsets[i]->getContainer().nbElem();
UInt n_tot;
MPI_Reduce(&n,&n_tot,1,MPI_INT,MPI_SUM,0,worldCom);
if (lm_my_proc_id == 0 )
n_tot_subset[i] = n_tot;
}
#endif
// declare the atom gone to other processors to attached subsets
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->declareSentAtoms(sent,sent_byproc);
global_set.declareSentAtoms(sent,sent_byproc);
//build the mask of ownership for each atom
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->buildMasksForSentAtoms(masks_send);
//initialize the buffers
clearPackBuffers();
//pack the masks to a packbuffer
packMasks(masks_send);
//exchange the masks
exchangeBuffers(buffers_tosend,buffers_torecv);
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
//unpack the masks to a MapRefToInt
unpackMasks(masks_recv);
//translate references
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->translateMovingReferences(moved);
//declare the received atoms and mask
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->declareRecvAtoms(masks_recv,newatoms);
global_set.declareRecvAtoms(newatoms);
//initialize the buffers
clearPackBuffers();
printPackBuffersStatus();
//pack attached data
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->packAttachedData(buffers_tosend);
global_set.packAttachedData(buffers_tosend);
//do the communication
exchangeBuffers(buffers_tosend,buffers_torecv);
//#ifndef LM_OPTIMIZED
MPI_Barrier(worldCom);
//#endif
printPackBuffersStatus();
//unpack attached data
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->unpackAttachedData(buffers_torecv);
global_set.unpackAttachedData(buffers_torecv);
//translate references
for (UInt i = 0 ; i < n_subsets ; ++i)
subsets[i]->fillRemainingHoles();
global_set.resize();
#ifndef LM_OPTIMIZED
//count number of atoms in subsets
for (UInt i = 0 ; i < n_subsets ; ++i){
UInt n = subsets[i]->getContainer().nbElem();
UInt n_tot;
MPI_Reduce(&n,&n_tot,1,MPI_INT,MPI_SUM,0,worldCom);
if (lm_my_proc_id == 0 )
if (n_tot != n_tot_subset[i])
LM_FATAL("subset " << i
<< " mismatch of atom count "
<< n_tot << " " << n_tot_subset[i]);
}
#endif
//clear the stl structures
moved.clear();
sent.clear();
sent_byproc.clear();
newatoms.clear();
have_changed = false;
DUMP("end update ref",DBG_INFO);
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::exchangeBuffers(BufferMap & toSend,BufferMap & toRecv){
// the size (in bytes) to be received map with the proc I receive from
// std::map<UInt,UInt> recv_sizes;
// the mpi_requests map by processors
std::map<int,MPI_Request> send_requests;
//iterators on buffers
BufferMap::iterator it;
BufferMap::iterator end;
//iterators on MPI requests
std::map<int,MPI_Request>::iterator it_req;
std::map<int,MPI_Request>::iterator end_req;
//MPIstatus object useful for the wait operations
MPI_Status status;
/* -------------------------------------------------------------------------- */
// now send the buffers through network
/* -------------------------------------------------------------------------- */
//sending of the buffers
for (end = toSend.end(),it = toSend.begin();it != end ; ++it){
UInt proc = (*it).first;
if (proc == UINT_MAX) continue;
UInt size = (*it).second.size();
char * buffer = (*it).second.buffer();
MPI_Request & request = send_requests[proc];
MPI_Isend(buffer,size,MPI_CHAR,proc,rank,worldCom,&request);
}
//reception async of the vectors
for (end = toRecv.end(),it = toRecv.begin();it != end ; ++it){
UInt proc = (*it).first;
if (proc == UINT_MAX) continue;
DUMP("probing from proc " << proc,DBG_INFO);
MPI_Probe(proc,proc,worldCom,&status);
int size = 0;
MPI_Get_count(&status,MPI_CHAR,&size);
PackBuffer & buf = toRecv[proc];
buf.resize(size);
char * buffer = toRecv[proc].buffer();
DUMP("receiving from proc " << proc << " " << size << " bytes",DBG_INFO);
MPI_Recv(buffer,size,MPI_CHAR,proc,proc,worldCom,&status);
DUMP("received packet from proc " << proc << " of size " << size
<< " bytes",DBG_DETAIL);
}
//now I wait for requests to be acheived
for (end_req = send_requests.end(),it_req = send_requests.begin();
it_req != end_req ; ++it_req){
UInt proc = (*it_req).first;
if (proc == UINT_MAX) continue;
#ifndef LM_OPTIMIZED
UInt size = toRecv[proc].size();
#endif // LM_OPTIMIZED
DUMP("waiting to send a packet to proc " << proc
<< " of size " << size << " bytes",DBG_INFO);
MPI_Wait(&((*it_req).second),&status);
DUMP("sent packet to proc " << proc << " of size " << size
<< " bytes",DBG_DETAIL);
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::packMasks(MapRefToUInt & masks){
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
//pack the masks
for (it = sent_byproc.begin(), end = sent_byproc.end();
it != end ; ++it){
UInt proc = (*it).first;
std::vector<Ref> & sendlist = (*it).second;
for (UInt i = 0 ; i < sendlist.size() ; ++i){
Ref at = sendlist[i];
buffers_tosend[proc] << masks[at];
}
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::unpackMasks(MapRefToUInt & masks){
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
for (it = newatoms.begin(), end = newatoms.end();
it != end ; ++it){
UInt proc = (*it).first;
std::vector<Ref> & recvlist = (*it).second;
for (UInt i = 0 ; i < recvlist.size() ; ++i){
Ref at = recvlist[i];
UInt m;
buffers_torecv[proc] >> m;
masks[at] = m;
}
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::printBilan(){
typename MapUIntToRefList::iterator it;
typename MapUIntToRefList::iterator end;
typename MapRefToUInt::iterator it2;
typename MapRefToUInt::iterator end2;
std::map<UInt ,UInt > nb_sent_by_proc;
if (moved.size()) DUMP(moved.size() << " atoms moved in memory",DBG_WARNING);
for (end = newatoms.end(), it = newatoms.begin();it != end;++it)
DUMP((*it).second.size() << " atoms received from proc " << (*it).first,DBG_WARNING);
for(end = sent_byproc.end(), it = sent_byproc.begin();it != end;++it)
DUMP((*it).second.size() << " atoms sent to proc " << (*it).first,DBG_WARNING);
for(end2 = sent.end(), it2 = sent.begin();it2 != end2;++it2)
++nb_sent_by_proc[(*it2).second];
for(end = sent_byproc.end(),it = sent_byproc.begin();it != end;++it){
if((*it).second.size() != nb_sent_by_proc[(*it).first])
LM_FATAL("sent_by_proc registered " << (*it).second.size()
<< " atoms as sent but sent registered "
<< nb_sent_by_proc[(*it).first]
<< " fatal inconstistency in send maps");
}
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::addSubSet(const std::string & name,
ContainerArray<Ref> & subset){
UInt size = subsets.size();
UInt i=0;
for (i = 0; i < size ; ++i){
if (&subsets[i]->getContainer() == &subset){
DUMP(name << " was already registered",DBG_INFO);
return;
}
}
RefSubset<ContainerArray<Ref> > * sub =
new RefSubset<ContainerArray<Ref> >(subset,size,name);
DUMP("adding a subset to managed subset at position " << size
<< " and named " << name << " " << &subset,DBG_INFO);
subsets.push_back(sub);
}
/* -------------------------------------------------------------------------- */
//template <typename Ref>
//void ReferenceManager<Ref>::removeSubSet(const std::string & name){
//
// UInt size = subsets.size();
// RefSubset<ContainerArray<Ref> > * sub = subsets[name];
// delete sub;
// subsets.erase(name);
// DUMP("removing a subset from managed subset at position " << size
// << " and named " << name,DBG_INFO);
//}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::attachVector(std::vector<Real> & v,ContainerArray<Ref> & c,
UInt nb_components){
UInt nb = subsets.size();
UInt i=0;
for (i = 0; i < nb ; ++i){
if (&subsets[i]->getContainer() == &c){
AttachedVector<Real> * atV = new AttachedVector<Real>(v,nb_components);
if (!subsets[i]->attachData(atV)) delete atV;
else DUMP("attaching vector<Real> " << &v << " to subset "
<< i << " " << subsets[i]->getID() << " " << &c,DBG_INFO);
break;
}
}
if (i == nb)
LM_FATAL("attaching vector<Real> could not be done : did not find its subset " << &c);
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::attachVector(std::vector<Real> & v,
GlobalContainer & cont,
UInt nb_components){
AttachedVector<Real> * atV = new AttachedVector<Real>(v,nb_components);
global_set.attachData(atV);
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::attachObject(AttachedObject & obj,
ContainerArray<Ref> & c){
UInt nb = subsets.size();
UInt i;
for (i = 0; i < nb ; ++i){
if (&subsets[i]->getContainer() == &c){
DUMP("attaching object " << &obj << " to subset " << i,DBG_INFO);
subsets[i]->attachData(&obj);
break;
}
}
if (i == nb)
LM_FATAL("attaching vector<Real> could not be done : did not found its subset");
}
/* -------------------------------------------------------------------------- */
template <typename Ref>
void ReferenceManager<Ref>::detachObject(AttachedObject & obj,
ContainerArray<Ref> & c){
UInt nb = subsets.size();
UInt i;
for (i = 0; i < nb ; ++i){
if (&subsets[i]->getContainer() == &c){
DUMP("detaching object " << &obj << " from subset " << i,DBG_INFO);
subsets[i]->detachData(&obj);
break;
}
}
if (i == nb)
LM_FATAL("attaching vector<Real> could not be done : did not found its subset");
}
/* -------------------------------------------------------------------------- */
#define DECLARE_REF_MANAGER(n,data,obj) \
template class ReferenceManager<BOOST_PP_TUPLE_ELEM(3,0,obj)::RefPoint >;
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER,f,LIST_ATOM_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER,f,LIST_DD_MODEL)
BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER,f,LIST_CONTINUUM_MODEL)
#undef DECLARE_REF_MANAGER
__END_LIBMULTISCALE__
c4science · Help