reference_manager.cc
No OneTemporary
Actions

Subscribers

None

File Metadata

Created: Tue, Aug 6, 09:16

reference_manager.cc
View Options

	/**
	* @file reference_manager.cc
	*
	* @author Guillaume Anciaux <guillaume.anciaux@epfl.ch>
	*
	* @date Mon Sep 08 23:40:22 2014
	*
	* @brief This is the manager of reference and coherency with migrations
	*
	* @section LICENSE
	*
	* Copyright INRIA and CEA
	*
	* The LibMultiScale is a C++ parallel framework for the multiscale
	* coupling methods dedicated to material simulations. This framework
	* provides an API which makes it possible to program coupled simulations
	* and integration of already existing codes.
	*
	* This Project was initiated in a collaboration between INRIA Futurs Bordeaux
	* within ScAlApplix team and CEA/DPTA Ile de France.
	* The project is now continued at the Ecole Polytechnique Fédérale de Lausanne
	* within the LSMS/ENAC laboratory.
	*
	* This software is governed by the CeCILL-C license under French law and
	* abiding by the rules of distribution of free software. You can use,
	* modify and/ or redistribute the software under the terms of the CeCILL-C
	* license as circulated by CEA, CNRS and INRIA at the following URL
	* "http://www.cecill.info".
	*
	* As a counterpart to the access to the source code and rights to copy,
	* modify and redistribute granted by the license, users are provided only
	* with a limited warranty and the software's author, the holder of the
	* economic rights, and the successive licensors have only limited
	* liability.
	*
	* In this respect, the user's attention is drawn to the risks associated
	* with loading, using, modifying and/or developing or reproducing the
	* software by the user in light of its specific status of free software,
	* that may mean that it is complicated to manipulate, and that also
	* therefore means that it is reserved for developers and experienced
	* professionals having in-depth computer knowledge. Users are therefore
	* encouraged to load and test the software's suitability as regards their
	* requirements in conditions enabling the security of their systems and/or
	* data to be ensured and, more generally, to use and operate it in the
	* same conditions as regards security.
	*
	* The fact that you are presently reading this means that you have had
	* knowledge of the CeCILL-C license and that you accept its terms.
	*
	*/

	/* -------------------------------------------------------------------------- */
	#include <boost/preprocessor.hpp>
	/* -------------------------------------------------------------------------- */
	#include "lib_continuum.hh"
	#include "lib_dd.hh"
	#include "lib_md.hh"
	#include "lm_common.hh"
	#include "lm_communicator.hh"
	#include "reference_manager.hh"
	/* -------------------------------------------------------------------------- */
	__BEGIN_LIBMULTISCALE__

	/* -------------------------------------------------------------------------- */
	template <typename Ref>
	ReferenceManager<Ref>::ReferenceManager(
	typename Ref::Domain::ContainerPoints &global_container) {

	auto name = global_container.getID();
	subsets[name] = make_subset<false>(global_container, 0);
	}

	/* -------------------------------------------------------------------------- */
	template <typename Ref> void ReferenceManager<Ref>::printPackBuffersStatus() {

	#ifndef LM_OPTIMIZED
	for (auto &&[proc, buf] : buffers_torecv) {
	DUMP("from " << proc << " receive buffer status: " << buf.size() << " "
	<< buf.remainingSize(),
	DBG_INFO);
	}

	// same thing for sending buffers
	for (auto &&[proc, buf] : buffers_tosend) {
	DUMP("to " << proc << " send buffer status: " << buf.size() << " "
	<< buf.remainingSize(),
	DBG_INFO);
	}
	#endif // LM_OPTIMIZED
	}

	/* -------------------------------------------------------------------------- */

	template <typename Ref> void ReferenceManager<Ref>::clearPackBuffers() {
	// clear buffers
	buffers_tosend.clear();
	buffers_torecv.clear();

	// create a packbuffer for each proc I should receive from
	// and clear it
	for (auto &&[proc, ref] : newrefs) {
	PackBuffer &buf = buffers_torecv[proc];
	buf.clear();
	}
	// same thing for sending buffers
	for (auto &&[proc, ref] : sent_byproc) {
	PackBuffer &buf = buffers_tosend[proc];
	buf.clear();
	}
	}
	/* -------------------------------------------------------------------------- */

	template <typename Ref> void ReferenceManager<Ref>::updateRefSubSets() {
	MapRefToUInt masks_send;
	MapRefToUInt masks_recv;

	//#ifndef LM_OPTIMIZED
	comm_group->synchronize(); // MPI_Barrier(worldCom);
	//#endif

	if (have_changed) {
	DUMP("have_changed = " << have_changed, DBG_INFO);
	}

	// do a bilan and checkup of coherency state
	// printBilan();

	#ifndef LM_OPTIMIZED
	std::map<LMID, UInt> n_tot_subset;
	// count number of refs in before subsets
	for (auto &&[name, s] : subsets) {
	UInt n_tot = s->getContainer().size();
	comm_group->reduce(&n_tot, 1, "n_tot", OP_SUM, 0);
	// MPI_Reduce(&n, &n_tot, 1, MPI_INT, MPI_SUM, 0, worldCom);
	if (lm_my_proc_id == 0)
	n_tot_subset[name] = n_tot;
	}
	#endif

	// declare the ref gone to other processors to attached subsets
	for (auto &&[name, s] : subsets)
	s->declareSentRefs(sent, sent_byproc);
	// global_set.declareSentAtoms(sent, sent_byproc);

	// build the mask of ownership for each ref
	for (auto &&[name, s] : subsets)
	s->buildMasksForSentRefs(masks_send);
	// initialize the buffers
	clearPackBuffers();
	// pack the masks to a packbuffer
	packMasks(masks_send);
	// exchange the masks
	exchangeBuffers(buffers_tosend, buffers_torecv);
	//#ifndef LM_OPTIMIZED
	comm_group->synchronize(); // MPI_Barrier(worldCom);
	//#endif
	// unpack the masks to a MapRefToInt
	unpackMasks(masks_recv);
	// translate references
	for (auto &&[name, s] : subsets)
	s->translateMovingReferences(moved);
	// declare the received refs and mask
	for (auto &&[name, s] : subsets)
	s->declareRecvRefs(masks_recv, newrefs);
	// global_set.declareRecvAtoms(newatoms);

	// initialize the buffers
	clearPackBuffers();
	printPackBuffersStatus();
	// pack attached data
	for (auto &&[name, s] : subsets)
	s->packAttachedData(buffers_tosend);
	// global_set.packAttachedData(buffers_tosend);

	// do the communication
	exchangeBuffers(buffers_tosend, buffers_torecv);
	//#ifndef LM_OPTIMIZED
	comm_group->synchronize(); // MPI_Barrier(worldCom);
	//#endif
	printPackBuffersStatus();
	// unpack attached data
	for (auto &&[name, s] : subsets)
	s->unpackAttachedData(buffers_torecv);
	// global_set.unpackAttachedData(buffers_torecv);

	// translate references
	for (auto &&[name, s] : subsets)
	s->fillRemainingHoles();

	UInt current_size [[gnu::unused]];
	for (auto &&[name, s] : subsets) {
	if (not s->isSubset())
	current_size = s->resize();
	}
	// for (auto &&[name, s] : subsets) {
	// if (s->isSubset())
	// s->resize(current_size);
	// }

	#ifndef LM_OPTIMIZED
	// count number of refs in subsets
	for (auto &&[name, s] : subsets) {
	UInt n_tot = s->getContainer().rows();
	DUMP("container:" << name << " size: " << n_tot, DBG_INFO);
	comm_group->reduce(&n_tot, 1, "n_tot", OP_SUM, 0);

	if (lm_my_proc_id == 0)
	if (n_tot != n_tot_subset[name])
	LM_FATAL("subset " << name << " mismatch of ref count " << n_tot << " "
	<< n_tot_subset[name]);
	}
	#endif

	// clear the stl structures
	moved.clear();
	sent.clear();
	sent_byproc.clear();
	newrefs.clear();

	have_changed = false;
	DUMP("end update ref", DBG_INFO);
	}

	/* --------------------------------------------------------------------------
	*/
	template <typename Ref>
	void ReferenceManager<Ref>::exchangeBuffers(BufferMap &toSend,
	BufferMap &toRecv) {

	std::map<int, Request> send_requests;

	/* ------------------------------------------------------------------------ */
	// now send the buffers through network
	/* ------------------------------------------------------------------------ */

	// sending of the buffers
	for (auto &&[proc, buf_send] : toSend) {
	if (proc == UINT_MAX)
	continue;
	UInt size = buf_send.size();
	auto buffer = buf_send.buffer();
	send_requests[proc] =
	comm_group->isend(buffer, size, proc, "ref_manager send");
	// MPI_Isend(buffer, size, MPI_CHAR, proc, rank, worldCom, &request);
	}
	// reception async of the vectors
	for (auto &&[proc, buffer_to_recv] : toRecv) {
	if (proc == UINT_MAX)
	continue;
	DUMP("probing from proc " << proc, DBG_INFO);
	int size = comm_group->probe<char>(
	proc,
	"probing ref_manager recv"); // MPI_Probe(proc, proc,
	// worldCom, &status);

	PackBuffer &buf = toRecv[proc];
	buf.resize(size);
	auto buffer = buffer_to_recv.buffer();

	DUMP("receiving from proc " << proc << " " << size << " bytes", DBG_INFO);
	comm_group->receive(
	buffer, size, proc,
	"recv ref_manager buffer"); //(buffer, size, MPI_CHAR, proc, proc,
	// worldCom, &status);

	DUMP("received packet from proc " << proc << " of size " << size
	<< " bytes",
	DBG_DETAIL);
	}
	// now I wait for requests to be acheived
	for (auto &&[proc, request] : send_requests) {
	if (proc == int(UINT_MAX))
	continue;
	#ifndef LM_OPTIMIZED
	UInt size = toRecv[proc].size();
	DUMP("waiting to send a packet to proc " << proc << " of size " << size
	<< " bytes",
	DBG_INFO);
	#endif // LM_OPTIMIZED

	comm_group->wait(request);
	DUMP("sent packet to proc " << proc << " of size " << size << " bytes",
	DBG_DETAIL);
	}
	}
	/* --------------------------------------------------------------------------
	*/
	template <typename Ref>
	void ReferenceManager<Ref>::packMasks(MapRefToUInt &masks) {
	// pack the masks
	for (auto &&[proc, sendlist] : sent_byproc) {
	for (UInt i = 0; i < sendlist.size(); ++i) {
	Ref at = sendlist[i];
	buffers_tosend[proc] << masks[at];
	}
	}
	}
	/* --------------------------------------------------------------------------
	*/
	template <typename Ref>
	void ReferenceManager<Ref>::unpackMasks(MapRefToUInt &masks) {

	for (auto &&[proc, recvlist] : newrefs) {
	for (UInt i = 0; i < recvlist.size(); ++i) {
	Ref at = recvlist[i];
	UInt m;
	buffers_torecv[proc] >> m;
	masks[at] = m;
	}
	}
	}

	/* --------------------------------------------------------------------------
	*/

	template <typename Ref> void ReferenceManager<Ref>::printBilan() {
	std::map<UInt, UInt> nb_sent_by_proc;

	if (moved.size())
	DUMP(moved.size() << " refs moved in memory", DBG_WARNING);

	for (auto &&[proc, refs] : newrefs)
	DUMP(refs.size() << " refs received from proc " << proc, DBG_WARNING);

	for (auto &&[proc, sent] : sent_byproc)
	DUMP(sent.size() << " refs sent to proc " << proc, DBG_WARNING);

	for (auto &&it : sent)
	++nb_sent_by_proc[it.second];

	for (auto &&[proc, sentlist] : sent_byproc) {
	if (sentlist.size() != nb_sent_by_proc[proc])
	LM_FATAL("sent_by_proc registered "
	<< sentlist.size() << " refs as sent but sent registered "
	<< nb_sent_by_proc[proc]
	<< " fatal inconstistency in send maps");
	}
	}

	/* --------------------------------------------------------------------------
	*/
	template <typename Ref>
	template <typename Cont>
	std::enable_if_t<std::is_same_v<typename Cont::Ref, Ref>>
	ReferenceManager<Ref>::addSubset(Cont &container) {

	auto name = container.getID();
	auto it = subsets.find(name);
	if (it != subsets.end()) {
	DUMP(name << " was already registered", DBG_INFO);
	return;
	}

	// UInt new_subset_id = subsets.size();
	auto subset = make_subset<true>(container, subsets.size());

	DUMP("adding a subset to managed subset named " << name << " " << &container,
	DBG_INFO_STARTUP);

	subsets[name] = subset;
	}

	/* --------------------------------------------------------------------------
	*/
	template <typename Ref>
	void ReferenceManager<Ref>::removeSubset(const std::string &name) {

	subsets.erase(name);
	#ifndef LM_OPTIMIZED
	UInt size = subsets.size();
	DUMP("removing a subset from managed subset at position "
	<< size << " and named " << name,
	DBG_INFO);
	#endif
	}

	/* ------------------------------------------------------------------------ */
	// template <typename Ref>
	// void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
	// ContainerInterface &cont) {

	// DUMP("try to attach vector<Real> " << &v << " to subset " << &cont,
	// DBG_MESSAGE);

	// UInt i = 0;
	// for (auto &&[name, subset] : subsets) {
	// if (&subset->getContainer() == &cont) {
	// auto atV = std::make_shared<AttachedVector<Real>>(v);
	// if (subset->attachData(*atV))
	// DUMP("attaching vector<Real> " << &v << " to subset " << cont.getID()
	// << " " << &cont,
	// DBG_MESSAGE);

	// break;
	// }
	// ++i;
	// }

	// if (i == subsets.size()) {
	// DUMP("attaching vector<Real> could not be done : did not find its subset
	// "
	// << &cont << "\nRegistered vectors are ",
	// DBG_MESSAGE);
	// for (auto &s : subsets) {
	// DUMP(s.first << " : " << &s.second->getContainer() << "\n",
	// DBG_MESSAGE);
	// }
	// LM_FATAL("abort");
	// }
	// }
	/* ------------------------------------------------------------------------ */

	// template <typename Ref>
	// void ReferenceManager<Ref>::attachVector(ContainerArray<Real> &v,
	// GlobalContainer &) {

	// AttachedVector<Real> *atV = new AttachedVector<Real>(v);
	// global_set.attachData(*atV);
	// }

	/* --------------------------------------------------------------------- */

	// template <typename Ref>
	// void ReferenceManager<Ref>::detachVector(ContainerArray<Real> &,
	// ContainerInterface &) {
	// LM_TOIMPLEMENT;
	// }

	// /* --------------------------------------------------------------------------
	// */

	template <typename Ref>
	void ReferenceManager<Ref>::setCommGroup(CommGroup &group) {
	comm_group = &group;
	}

	// worldCom = comm.getMPIComm();
	// if (worldCom != MPI_COMM_NULL)
	// MPI_Comm_rank(worldCom, &rank);
	// }

	#define DECLARE_REF_MANAGER(n, data, obj) \
	template class ReferenceManager<BOOST_PP_TUPLE_ELEM(3, 0, obj)::RefPoint>;
	/* -------------------------------------------------------------------------- */

	template <typename Ref>
	void ReferenceManager<Ref>::acquireContext(const LMObject &obj) {
	this->setCommGroup(obj.getCommGroup());
	}

	/* -------------------------------------------------------------------------- */

	template <typename Ref>
	void ReferenceManager<Ref>::addSubset(ContainerInterface &cont) {
	this->addSubset<dispatch>(cont);
	}

	/* -------------------------------------------------------------------------- */

	BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_ATOM_MODEL)
	BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_DD_MODEL)
	BOOST_PP_SEQ_FOR_EACH(DECLARE_REF_MANAGER, f, LIST_CONTINUUM_MODEL)

	#undef DECLARE_REF_MANAGER

	__END_LIBMULTISCALE__

reference_manager.ccNo OneTemporaryActions

File Metadata

reference_manager.ccView Options

Event Timeline

reference_manager.cc
No OneTemporary
Actions

reference_manager.cc
View Options