Diffusion akantu (bugfixes/65-Making-the-integrators-available-in-Python) Browse

65-Making-the…

master_element_info_per_processor.cc
View Options

			/**
			* @file master_element_info_per_processor.cc
			*
			* @author Nicolas Richart <nicolas.richart@epfl.ch>
			*
			* @date creation: Wed Mar 16 2016
			* @date last modification: Thu Nov 12 2020
			*
			* @brief Helper class to distribute a mesh
			*
			*
			* @section LICENSE
			*
			* Copyright (©) 2016-2021 EPFL (Ecole Polytechnique Fédérale de Lausanne)
			* Laboratory (LSMS - Laboratoire de Simulation en Mécanique des Solides)
			*
			* Akantu is free software: you can redistribute it and/or modify it under the
			* terms of the GNU Lesser General Public License as published by the Free
			* Software Foundation, either version 3 of the License, or (at your option) any
			* later version.
			*
			* Akantu is distributed in the hope that it will be useful, but WITHOUT ANY
			* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
			* A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
			* details.
			*
			* You should have received a copy of the GNU Lesser General Public License
			* along with Akantu. If not, see <http://www.gnu.org/licenses/>.
			*
			*/

			/* -------------------------------------------------------------------------- */
			#include "aka_iterators.hh"
			#include "communicator.hh"
			#include "element_group.hh"
			#include "element_info_per_processor.hh"
			#include "element_synchronizer.hh"
			#include "mesh_iterators.hh"
			#include "mesh_utils.hh"
			/* -------------------------------------------------------------------------- */
			#include <algorithm>
			#include <iostream>
			#include <map>
			#include <tuple>
			/* -------------------------------------------------------------------------- */

			namespace akantu {

			/* -------------------------------------------------------------------------- */
			MasterElementInfoPerProc::MasterElementInfoPerProc(
			ElementSynchronizer & synchronizer, UInt message_cnt, UInt root,
			ElementType type, const MeshPartition & partition)
			: ElementInfoPerProc(synchronizer, message_cnt, root, type),
			partition(partition), all_nb_local_element(nb_proc, 0),
			all_nb_ghost_element(nb_proc, 0), all_nb_element_to_send(nb_proc, 0) {
			Vector<UInt> size(5);
			size(0) = (UInt)type;

			if (type != _not_defined) {
			nb_nodes_per_element = Mesh::getNbNodesPerElement(type);
			nb_element = mesh.getNbElement(type);
			const auto & partition_num =
			this->partition.getPartition(this->type, _not_ghost);
			const auto & ghost_partition =
			this->partition.getGhostPartitionCSR()(this->type, _not_ghost);

			for (UInt el = 0; el < nb_element; ++el) {
			this->all_nb_local_element[partition_num(el)]++;
			for (auto part = ghost_partition.begin(el);
			part != ghost_partition.end(el); ++part) {
			this->all_nb_ghost_element[*part]++;
			}
			this->all_nb_element_to_send[partition_num(el)] +=
			ghost_partition.getNbCols(el) + 1;
			}

			/// tag info
			auto && tag_names = this->mesh.getTagNames(type);
			this->nb_tags = tag_names.size();

			size(4) = nb_tags;

			for (UInt p = 0; p < nb_proc; ++p) {
			if (p != root) {
			size(1) = this->all_nb_local_element[p];
			size(2) = this->all_nb_ghost_element[p];
			size(3) = this->all_nb_element_to_send[p];
			AKANTU_DEBUG_INFO(
			"Sending connectivities informations to proc "
			<< p << " TAG("
			<< Tag::genTag(this->rank, this->message_count, Tag::_sizes)
			<< ")");
			comm.send(size, p,
			Tag::genTag(this->rank, this->message_count, Tag::_sizes));
			} else {
			this->nb_local_element = this->all_nb_local_element[p];
			this->nb_ghost_element = this->all_nb_ghost_element[p];
			}
			}
			} else {
			for (UInt p = 0; p < this->nb_proc; ++p) {
			if (p != this->root) {
			AKANTU_DEBUG_INFO(
			"Sending empty connectivities informations to proc "
			<< p << " TAG("
			<< Tag::genTag(this->rank, this->message_count, Tag::_sizes)
			<< ")");
			comm.send(size, p,
			Tag::genTag(this->rank, this->message_count, Tag::_sizes));
			}
			}
			}
			}

			/* ------------------------------------------------------------------------ */
			void MasterElementInfoPerProc::synchronizeConnectivities() {
			const auto & partition_num =
			this->partition.getPartition(this->type, _not_ghost);
			const auto & ghost_partition =
			this->partition.getGhostPartitionCSR()(this->type, _not_ghost);

			std::vector<Array<UInt>> buffers(this->nb_proc);

			const auto & connectivities =
			this->mesh.getConnectivity(this->type, _not_ghost);

			/// copying the local connectivity
			for (auto && part_conn :
			zip(partition_num,
			make_view(connectivities, this->nb_nodes_per_element))) {
			auto && part = std::get<0>(part_conn);
			auto && conn = std::get<1>(part_conn);
			for (UInt i = 0; i < conn.size(); ++i) {
			buffers[part].push_back(conn[i]);
			}
			}

			/// copying the connectivity of ghost element
			for (auto && tuple :
			enumerate(make_view(connectivities, this->nb_nodes_per_element))) {
			auto && el = std::get<0>(tuple);
			auto && conn = std::get<1>(tuple);
			for (auto part = ghost_partition.begin(el); part != ghost_partition.end(el);
			++part) {
			UInt proc = *part;
			for (UInt i = 0; i < conn.size(); ++i) {
			buffers[proc].push_back(conn[i]);
			}
			}
			}

			#ifndef AKANTU_NDEBUG
			for (auto p : arange(this->nb_proc)) {
			UInt size = this->nb_nodes_per_element *
			(this->all_nb_local_element[p] + this->all_nb_ghost_element[p]);
			AKANTU_DEBUG_ASSERT(
			buffers[p].size() == size,
			"The connectivity data packed in the buffer are not correct");
			}
			#endif

			/// send all connectivity and ghost information to all processors
			std::vector<CommunicationRequest> requests;
			for (auto p : arange(this->nb_proc)) {
			if (p == this->root) {
			continue;
			}
			auto && tag =
			Tag::genTag(this->rank, this->message_count, Tag::_connectivity);
			AKANTU_DEBUG_INFO("Sending connectivities to proc " << p << " TAG(" << tag
			<< ")");
			requests.push_back(comm.asyncSend(buffers[p], p, tag));
			}

			Array<UInt> & old_nodes = this->getNodesGlobalIds();

			/// create the renumbered connectivity
			AKANTU_DEBUG_INFO("Renumbering local connectivities");
			MeshUtils::renumberMeshNodes(mesh, buffers[root], all_nb_local_element[root],
			all_nb_ghost_element[root], type, old_nodes);

			Communicator::waitAll(requests);
			Communicator::freeCommunicationRequest(requests);
			}

			/* ------------------------------------------------------------------------ */
			void MasterElementInfoPerProc::synchronizePartitions() {
			const auto & partition_num =
			this->partition.getPartition(this->type, _not_ghost);
			const auto & ghost_partition =
			this->partition.getGhostPartitionCSR()(this->type, _not_ghost);

			std::vector<Array<UInt>> buffers(this->partition.getNbPartition());

			/// splitting the partition information to send them to processors
			Vector<UInt> count_by_proc(nb_proc, 0);
			for (UInt el = 0; el < nb_element; ++el) {
			UInt proc = partition_num(el);
			buffers[proc].push_back(ghost_partition.getNbCols(el));

			UInt i(0);
			for (auto part = ghost_partition.begin(el); part != ghost_partition.end(el);
			++part, ++i) {
			buffers[proc].push_back(*part);
			}
			}

			for (UInt el = 0; el < nb_element; ++el) {
			UInt i(0);
			for (auto part = ghost_partition.begin(el); part != ghost_partition.end(el);
			++part, ++i) {
			buffers[*part].push_back(partition_num(el));
			}
			}

			#ifndef AKANTU_NDEBUG
			for (UInt p = 0; p < this->nb_proc; ++p) {
			AKANTU_DEBUG_ASSERT(buffers[p].size() == (this->all_nb_ghost_element[p] +
			this->all_nb_element_to_send[p]),
			"Data stored in the buffer are most probably wrong");
			}
			#endif

			std::vector<CommunicationRequest> requests;
			/// last data to compute the communication scheme
			for (UInt p = 0; p < this->nb_proc; ++p) {
			if (p == this->root) {
			continue;
			}

			auto && tag =
			Tag::genTag(this->rank, this->message_count, Tag::_partitions);
			AKANTU_DEBUG_INFO("Sending partition informations to proc " << p << " TAG("
			<< tag << ")");
			requests.push_back(comm.asyncSend(buffers[p], p, tag));
			}

			if (Mesh::getSpatialDimension(this->type) ==
			this->mesh.getSpatialDimension()) {
			AKANTU_DEBUG_INFO("Creating communications scheme");
			this->fillCommunicationScheme(buffers[this->rank]);
			}

			Communicator::waitAll(requests);
			Communicator::freeCommunicationRequest(requests);
			}

			/* -------------------------------------------------------------------------- */
			void MasterElementInfoPerProc::synchronizeTags() {
			AKANTU_DEBUG_IN();

			if (this->nb_tags == 0) {
			AKANTU_DEBUG_OUT();
			return;
			}

			/// tag info
			auto tag_names = mesh.getTagNames(type);

			// Make sure the tags are sorted (or at least not in random order),
			// because they come from a map !!
			std::sort(tag_names.begin(), tag_names.end());

			// Sending information about the tags in mesh_data: name, data type and
			// number of components of the underlying array associated to the current
			// type
			DynamicCommunicationBuffer mesh_data_sizes_buffer;
			for (auto && tag_name : tag_names) {
			mesh_data_sizes_buffer << tag_name;
			mesh_data_sizes_buffer << mesh.getTypeCode(tag_name);
			mesh_data_sizes_buffer << mesh.getNbComponent(tag_name, type);
			}

			AKANTU_DEBUG_INFO(
			"Broadcasting the size of the information about the mesh data tags: ("
			<< mesh_data_sizes_buffer.size() << ").");
			AKANTU_DEBUG_INFO(
			"Broadcasting the information about the mesh data tags, addr "
			<< (void *)mesh_data_sizes_buffer.storage());

			comm.broadcast(mesh_data_sizes_buffer, root);

			if (mesh_data_sizes_buffer.empty()) {
			return;
			}

			// Sending the actual data to each processor
			std::vector<DynamicCommunicationBuffer> buffers(nb_proc);
			// Loop over each tag for the current type
			for (auto && tag_name : tag_names) {
			// Type code of the current tag (i.e. the tag named *names_it)
			this->fillTagBuffer(buffers, tag_name);
			}

			std::vector<CommunicationRequest> requests;
			for (UInt p = 0; p < nb_proc; ++p) {
			if (p == root) {
			continue;
			}

			auto && tag = Tag::genTag(this->rank, this->message_count, Tag::_mesh_data);
			AKANTU_DEBUG_INFO("Sending " << buffers[p].size()
			<< " bytes of mesh data to proc " << p
			<< " TAG(" << tag << ")");

			requests.push_back(comm.asyncSend(buffers[p], p, tag));
			}

			// Loop over each tag for the current type
			for (auto && tag_name : tag_names) {
			// Reinitializing the mesh data on the master
			this->fillMeshData(buffers[root], tag_name, mesh.getTypeCode(tag_name),
			mesh.getNbComponent(tag_name, type));
			}

			Communicator::waitAll(requests);
			Communicator::freeCommunicationRequest(requests);
			requests.clear();

			AKANTU_DEBUG_OUT();
			}

			/* -------------------------------------------------------------------------- */
			template <typename T>
			void MasterElementInfoPerProc::fillTagBufferTemplated(
			std::vector<DynamicCommunicationBuffer> & buffers,
			const std::string & tag_name) {
			const auto & data = mesh.getElementalDataArray<T>(tag_name, type);
			const auto & partition_num =
			this->partition.getPartition(this->type, _not_ghost);
			const auto & ghost_partition =
			this->partition.getGhostPartitionCSR()(this->type, _not_ghost);

			// Not possible to use the iterator because it potentially triggers the
			// creation of complex
			// type templates (such as akantu::Vector< std::vector<Element> > which don't
			// implement the right interface
			// (e.g. operator<< in that case).
			// typename Array<T>::template const_iterator< Vector<T> > data_it =
			// data.begin(data.getNbComponent());
			// typename Array<T>::template const_iterator< Vector<T> > data_end =
			// data.end(data.getNbComponent());

			const T * data_it = data.storage();
			const T * data_end = data.storage() + data.size() * data.getNbComponent();
			const UInt * part = partition_num.storage();

			/// copying the data, element by element
			for (; data_it != data_end; ++part) {
			for (UInt j(0); j < data.getNbComponent(); ++j, ++data_it) {
			buffers[part] << data_it;
			}
			}

			data_it = data.storage();
			/// copying the data for the ghost element
			for (UInt el(0); data_it != data_end;
			data_it += data.getNbComponent(), ++el) {
			auto it = ghost_partition.begin(el);
			auto end = ghost_partition.end(el);
			for (; it != end; ++it) {
			UInt proc = *it;
			for (UInt j(0); j < data.getNbComponent(); ++j) {
			buffers[proc] << data_it[j];
			}
			}
			}
			}

			/* -------------------------------------------------------------------------- */
			void MasterElementInfoPerProc::fillTagBuffer(
			std::vector<DynamicCommunicationBuffer> & buffers,
			const std::string & tag_name) {
			#define AKANTU_DISTRIBUTED_SYNHRONIZER_TAG_DATA(r, extra_param, elem) \
			case MeshDataTypeCode::BOOST_PP_TUPLE_ELEM(2, 0, elem): { \
			this->fillTagBufferTemplated<BOOST_PP_TUPLE_ELEM(2, 1, elem)>(buffers, \
			tag_name); \
			break; \
			}

			MeshDataTypeCode data_type_code = mesh.getTypeCode(tag_name);
			switch (data_type_code) {
			BOOST_PP_SEQ_FOR_EACH(AKANTU_DISTRIBUTED_SYNHRONIZER_TAG_DATA, ,
			AKANTU_MESH_DATA_TYPES)
			default:
			AKANTU_ERROR("Could not obtain the type of tag" << tag_name << "!");
			break;
			}
			#undef AKANTU_DISTRIBUTED_SYNHRONIZER_TAG_DATA
			}

			/* -------------------------------------------------------------------------- */
			void MasterElementInfoPerProc::synchronizeGroups() {
			AKANTU_DEBUG_IN();

			std::vector<DynamicCommunicationBuffer> buffers(nb_proc);

			using ElementToGroup = std::vector<std::vector<std::string>>;
			ElementToGroup element_to_group(nb_element);

			for (auto & eg : mesh.iterateElementGroups()) {
			const auto & name = eg.getName();

			for (const auto & element : eg.getElements(type, _not_ghost)) {
			element_to_group[element].push_back(name);
			}

			eg.clear(type, _not_ghost);
			}

			const auto & partition_num =
			this->partition.getPartition(this->type, _not_ghost);
			const auto & ghost_partition =
			this->partition.getGhostPartitionCSR()(this->type, _not_ghost);

			/// copying the data, element by element
			for (auto && pair : zip(partition_num, element_to_group)) {
			buffers[std::get<0>(pair)] << std::get<1>(pair);
			}

			/// copying the data for the ghost element
			for (auto && pair : enumerate(element_to_group)) {
			auto && el = std::get<0>(pair);
			auto it = ghost_partition.begin(el);
			auto end = ghost_partition.end(el);
			for (; it != end; ++it) {
			UInt proc = *it;
			buffers[proc] << std::get<1>(pair);
			}
			}

			std::vector<CommunicationRequest> requests;
			for (UInt p = 0; p < this->nb_proc; ++p) {
			if (p == this->rank) {
			continue;
			}

			auto && tag = Tag::genTag(this->rank, p, Tag::_element_group);
			AKANTU_DEBUG_INFO("Sending element groups to proc " << p << " TAG(" << tag
			<< ")");
			requests.push_back(comm.asyncSend(buffers[p], p, tag));
			}

			this->fillElementGroupsFromBuffer(buffers[this->rank]);

			Communicator::waitAll(requests);
			Communicator::freeCommunicationRequest(requests);
			requests.clear();

			AKANTU_DEBUG_OUT();
			}

			/* -------------------------------------------------------------------------- */

			} // namespace akantu

rAKA/src/synchronizer/master_element_info_per_processor.cc522a8942da5dbugfixes/65-Making-the…

master_element_info_per_processor.ccView Options

rAKA/src/synchronizer/master_element_info_per_processor.cc
522a8942da5dbugfixes/65-Making-the…

master_element_info_per_processor.cc
View Options