Homec4science
Diffusion tamaas (archive/legacy_mpi) Browse

tamaas/src/core/grid.cpp
63f2a04e3716archive/legacy_mpi

Last Change
Raw File

grid.cpp
View Options

/**
* @file
*
* @author Lucas Frérot <lucas.frerot@epfl.ch>
*
* @section LICENSE
*
* Copyright (©) 2016 EPFL (Ecole Polytechnique Fédérale de
* Lausanne) Laboratory (LSMS - Laboratoire de Simulation en Mécanique des
* Solides)
*
* Tamaas is free software: you can redistribute it and/or modify it under the
* terms of the GNU Lesser General Public License as published by the Free
* Software Foundation, either version 3 of the License, or (at your option) any
* later version.
*
* Tamaas is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
* A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
* details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Tamaas. If not, see <http://www.gnu.org/licenses/>.
*
*/
/* -------------------------------------------------------------------------- */
#include "grid.hh"
#include "tamaas.hh"
#include <algorithm>
#include <complex>
#include <cstring>
#include <fftw3-mpi.h>
#include <mpi.h>
/* -------------------------------------------------------------------------- */
__BEGIN_TAMAAS__
/* -------------------------------------------------------------------------- */
extern int mpi_rank;
extern int mpi_size;
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
Grid<T, dim>::Grid() : GridBase<T>() {
this->n.fill(0);
this->strides.fill(1);
this->nb_components = 1;
this->distributed = false;
}
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
Grid<T, dim>::Grid(const std::array<UInt, dim>& n, UInt nb_components)
: GridBase<T>() {
this->init(n, nb_components);
}
template <typename T, UInt dim>
Grid<T, dim>::Grid(const std::vector<UInt>& n, UInt nb_components)
: GridBase<T>() {
if (n.size() != dim)
TAMAAS_EXCEPTION("Provided sizes for grid do not match dimension");
this->init(n, nb_components);
}
template <typename T, UInt dim>
Grid<T, dim>::Grid(const std::initializer_list<UInt>& n, UInt nb_components)
: GridBase<T>() {
if (n.size() != dim)
TAMAAS_EXCEPTION("Provided sizes for grid do not match dimension");
this->init(n, nb_components);
}
template <typename T, UInt dim>
Grid<T, dim>::Grid(const Grid<T, dim>& o)
: GridBase<T>(o), distributed(o.distributed), n(o.n), strides(o.strides) {}
template <typename T, UInt dim>
Grid<T, dim>::Grid(Grid<T, dim>&& o)
: GridBase<T>(o), distributed(o.distributed), n(std::move(o.n)),
strides(std::move(o.strides)) {}
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
void Grid<T, dim>::resize(const std::array<UInt, dim>& n) {
this->n = n;
UInt size = this->computeSize();
this->data.resize(size);
this->data.assign(size, T(0.));
this->computeStrides();
}
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
void Grid<T, dim>::computeStrides() {
std::copy(n.begin() + 1, n.end(), strides.rbegin() + 2);
strides[dim] = 1;
strides[dim - 1] = this->nb_components;
std::partial_sum(strides.rbegin(), strides.rend(), strides.rbegin(),
std::multiplies<UInt>());
}
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
void Grid<T, dim>::printself(std::ostream& str) const {
str << "Grid(" << dim << ", " << this->nb_components << ") {";
for (auto& val : *this) {
str << val << ", ";
}
str << "\b\b}";
}
/* -------------------------------------------------------------------------- */
/* Parallel(MPI) operations */
/* -------------------------------------------------------------------------- */
template <typename T, UInt dim>
void Grid<T, dim>::distribute(int root_rank) {
if (dim == 2) {
ptrdiff_t local_n0, local_0_start;
fftw_mpi_local_size_2d(n[0], n[1], MPI_COMM_WORLD, &local_n0,
&local_0_start);
// auto alloc_local = fftw_mpi_local_size_2d(n[0], n[1], MPI_COMM_WORLD,
// &local_n0, &local_0_start);
// std::cout << "mpi_size " << mpi_size << std::endl;
// for (UInt i = 0; i < dim; ++i)
// std::cout << "n[" << i << "] = " << n[i] << std::endl;
// std::cout << mpi_rank << " alloc_local " << alloc_local << std::endl;
// std::cout << mpi_rank << " local_n0 " << local_n0 << std::endl;
// std::cout << mpi_rank << " local_0_start " << local_0_start << std::endl;
// saving the total size
ntotal = n;
n[0] = local_n0;
std::vector<int> data_per_proc;
int* recv_buf = NULL;
T* ptr_data_copy = NULL;
Array<T>* data_copy = NULL;
if (mpi_rank == root_rank) {
data_copy = new Array<T>(this->data);
data_per_proc.resize(mpi_size);
recv_buf = &data_per_proc[0];
ptr_data_copy = data_copy->data();
}
this->resize(n);
int local_data_size = this->data.type_sizeof() * this->dataSize();
MPI_Gather(&local_data_size, 1, MPI_INT, recv_buf, 1, MPI_INT, root_rank,
MPI_COMM_WORLD);
std::vector<int> offset_per_proc;
if (mpi_rank == root_rank) {
// std::cout << "AAAAAAAA" << std::endl;
// std::for_each(ptr_data_copy + local_0_start*n[1],
// ptr_data_copy + (local_0_start + local_n0)*n[1],
// [](T x) { std::cout << x << std::endl; });
std::copy(ptr_data_copy + local_0_start * n[1],
ptr_data_copy + (local_0_start + local_n0) * n[1],
this->data.data());
offset_per_proc.assign(mpi_size, 0);
for (int i = 0; i < mpi_size; ++i) {
if (i > 0)
offset_per_proc[i] += data_per_proc[i - 1];
// std::cout << "data_per_proc[" << i << "] = " << data_per_proc[i]
// << std::endl;
// std::cout << "offset_per_proc[" << i << "] = " << offset_per_proc[i]
// << std::endl;
}
}
// std::cout << "BBBBBB" << std::endl;
// std::for_each(this->data.data(), this->data.data() + this->data.size(),
// [](T x) { std::cout << x << std::endl; });
// std::cout << "done" << std::endl;
MPI_Barrier(MPI_COMM_WORLD);
// std::cout << mpi_rank << ": local_data_size " << local_data_size
// << std::endl;
// std::cout << mpi_rank << ": local_data_size2 "
// << this->data.type_sizeof() * this->dataSize() << std::endl;
// std::cout << mpi_rank << ": n " << n[0] << " " << n[1] << std::endl;
// std::cout << mpi_rank << ": n2 " << n[0] * n[1] *
// this->data.type_sizeof()
// << std::endl;
// std::cout << mpi_rank << " ptr " << this->data.data() << std::endl;
// std::cout << mpi_rank << " ptr_copy " << ptr_data_copy << std::endl;
MPI_Barrier(MPI_COMM_WORLD);
MPI_Scatterv(ptr_data_copy, &data_per_proc[0], &offset_per_proc[0],
MPI_BYTE, this->data.data(), local_data_size, MPI_BYTE,
root_rank, MPI_COMM_WORLD);
// std::cout << mpi_rank << " CCCCC" << std::endl;
// std::for_each(this->data.data(), this->data.data() + this->data.size(),
// [](T x) { std::cout << x << std::endl; });
MPI_Barrier(MPI_COMM_WORLD);
// std::cout << mpi_rank << " done" << std::endl;
MPI_Barrier(MPI_COMM_WORLD);
delete data_copy;
}
this->distributed = true;
}
template <typename T, UInt dim>
void Grid<T, dim>::gather(int root_rank) {
TAMAAS_EXCEPTION("TOIMPLEMENT: missing dimension");
}
/* -------------------------------------------------------------------------- */
/// Class instanciation
#define GRID_INSTANCIATE_TYPE(type) \
template class Grid<type, 1>; \
template class Grid<type, 2>; \
template class Grid<type, 3>
GRID_INSTANCIATE_TYPE(Real);
GRID_INSTANCIATE_TYPE(UInt);
GRID_INSTANCIATE_TYPE(Complex);
GRID_INSTANCIATE_TYPE(int);
GRID_INSTANCIATE_TYPE(bool);
GRID_INSTANCIATE_TYPE(unsigned long);
#undef GRID_INSTANCIATE_TYPE
__END_TAMAAS__
c4science · Help