Homec4science
Diffusion Dynamic loop scheduling using MPI passive-target remote memory access (master) Browse

dls_MPI_RMA/PSIA_one_sided_memoryless.c
4672720499a7master

Last Change
Raw File

PSIA_one_sided_memoryless.c
View Options

/*********************************************************************************
* Copyright (c) 2018 *
* Ahmed Eleliemy <ahmed.eleliemy@unibas.ch> *
* University of Basel, Switzerland *
* All rights reserved. *
* This code is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. *
* This program is free software; you can redistribute it and/or modify it *
* under the terms of the license (GNU LGPL) which comes with this package. *
*********************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <math.h>
#include "dls_core.h"
#include "SpinDataStructures.h"
#include "SpinCalculator.h"
#include "ObjectLoader.h"
#define MIN(a,b) (((a)<(b))?(a):(b))
#define DATAOWNER 0
#ifndef CHECK_PERIOD
#define CHECK_PERIOD 1
#endif
#define REQ_WORK 1000
#define ASSI_WORK 1001
#define TERMINATE 1002
//-------- function headers -------
void parallel_one_sided_memory(int argc, char** argv);
//--------- main -------
int main (int argc, char** argv)
{
parallel_one_sided_memory(argc,argv);
return 0;
}
//------- master-worker functions ---------------
void all_processors_do(int argc, char ** argv, int number_of_workers, int id)
{
// variables for scheduling
int flag=0;
MPI_Status status;
int terminated=0;
int master_is_terminated=0;
int my_start_index=0;
int my_end_index=0;
struct Scheduling_params params;
char dummy='\0';
if(argc<7)
{
printf("Usgae: ./ObjectTester.o iSpinPoints iImageWidth fBinSize iSupportAngle(PI/x) iDLSMethods(0-STATIC 1-SS 2-FSC 3-GSS 4-FAC 5-TSS) sFileName\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
int number_of_tasks=atoi(argv[1]);
int method=atoi(argv[5]);
if(id==DATAOWNER)
{
printf("-------------------\n");
printf("Experiment one-sided stateful - based on step\n");
printf("Number of workers %d \n", number_of_workers);
printf("Number of tasks %d \n", number_of_tasks);
printf("Scheduling method %d \n", method);
printf("------------------- \n");
}
// variables for the PSIA
int image_width=atoi(argv[2]);
double bin_size=atof(argv[3]);
double support_angle=M_PI/atof(argv[4]);
char * filename=argv[6];
MeshObject* target_object = NULL;
double * images = NULL;
double * results=NULL;
double * all_double_data=NULL;
int * all_int_data=NULL;
params.scheduled_tasks=0;
params.total_tasks=number_of_tasks;
params.total_workers=number_of_workers;
params.step=0;
params.method=method;
params.last_chunk=0;
if(method == 2 && argc == 9)
{
double h = atof(argv[7]);
double sigma = atof(argv[8]);
params.constant=helper_FSC(number_of_workers,number_of_tasks,h,sigma);
}
else if(method ==5 && argc == 9)
{
int F = atoi(argv[7]);
int L = atoi(argv[8]);
int D = helper_TSS(number_of_workers,number_of_tasks, F, L);
params.last_chunk = F + D;
params.constant = D;
params.minimum_chunk=L;
}
else if(method ==5 || method ==2 )
{
printf("Usgae: ./ObjectTester.o iSpinPoints iImageWidth fBinSize iSupportAngle(PI/x) iDLSMethods(2-FSC 5-TSS) sFileName (h,sigma) or (F,L) \n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
else if(method==6)
{
char processor_name[30];
int name_len;
MPI_Get_processor_name(processor_name, &name_len);
params.knl_count= atoi(argv[7]);
params.my_weight= WF_helper(params.knl_count, processor_name, number_of_workers);
//printf("%s my weight %lf\n",processor_name,params.my_weight );
}
double start_time= MPI_Wtime();
images= (double *) calloc(number_of_tasks * image_width * image_width, sizeof(double));
results = (double *) calloc(number_of_tasks*image_width*image_width,sizeof(double));
if(id == DATAOWNER)
{
target_object=loadofffile(filename);
calcPointsNormal(target_object);
}
else
{
target_object = (MeshObject *) malloc (sizeof(MeshObject));
}
MPI_Bcast(&target_object->iPointCount,1,MPI_INT,DATAOWNER,MPI_COMM_WORLD);
int position=0;
int size_of_double_buffer_in_bytes=12*sizeof(double)*target_object->iPointCount;
int size_of_int_buffer_in_bytes=sizeof(int)*target_object->iPointCount;
all_double_data = (double*) malloc(size_of_double_buffer_in_bytes);
all_int_data = (int *) malloc(size_of_int_buffer_in_bytes);
//packing all data
if(id == DATAOWNER)
{
MPI_Pack(target_object->dptrXPoints, target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
MPI_Pack(target_object->dptrYPoints, target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
MPI_Pack(target_object->dptrZPoints, target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
MPI_Pack(target_object->dptrXNPoints, 3*target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
MPI_Pack(target_object->dptrYNPoints, 3*target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
MPI_Pack(target_object->dptrZNPoints, 3*target_object->iPointCount, MPI_DOUBLE, all_double_data, size_of_double_buffer_in_bytes,&position,MPI_COMM_WORLD);
position=0;
MPI_Pack(target_object->facesPerPoint, target_object->iPointCount, MPI_INT, all_int_data, size_of_int_buffer_in_bytes,&position,MPI_COMM_WORLD);
}
//sending data
MPI_Bcast(all_double_data,12*target_object->iPointCount,MPI_DOUBLE,DATAOWNER,MPI_COMM_WORLD);
MPI_Bcast(all_int_data,target_object->iPointCount,MPI_INT,DATAOWNER,MPI_COMM_WORLD);
if(id!=DATAOWNER)
{
target_object->dptrXPoints =malloc(sizeof(double)*target_object->iPointCount);
target_object->dptrYPoints =malloc(sizeof(double)*target_object->iPointCount);
target_object->dptrZPoints =malloc(sizeof(double)*target_object->iPointCount);
target_object->facesPerPoint =malloc(sizeof(int)*target_object->iPointCount);
target_object->dptrXNPoints =malloc(3*sizeof(double)*target_object->iPointCount);
target_object->dptrYNPoints =malloc(3*sizeof(double)*target_object->iPointCount);
target_object->dptrZNPoints =malloc(3*sizeof(double)*target_object->iPointCount);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrXPoints,target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrYPoints,target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrZPoints,target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrXNPoints,3*target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrYNPoints,3*target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
MPI_Unpack(all_double_data,size_of_double_buffer_in_bytes, &position,target_object->dptrZNPoints,3*target_object->iPointCount,MPI_DOUBLE,MPI_COMM_WORLD);
position=0;
MPI_Unpack(all_int_data,size_of_int_buffer_in_bytes, &position,target_object->facesPerPoint,target_object->iPointCount ,MPI_INT,MPI_COMM_WORLD);
}
free(all_double_data);
free(all_int_data);
double start_calc_time=0;
double end_calc_time=0;
double total_calc_time=0;
double total_request_time=0;
double max_loop_time=0;
double min_loop_time=0;
int check_request=0;
int to_execute=0;
int I_executed=0;
int I_executed_chunks=0;
double execution_times[number_of_workers];
int chunk_size;
int start=0;
int end=0;
MPI_Win win_global_scheduling_step;
MPI_Win win_global_scheduled_tasks;
int scheduling_step =0;
int scheduled_tasks=0;
int local_scheduling_step =0;
int local_scheduled_tasks=0;
MPI_Win_create(&scheduling_step, sizeof(int),sizeof(int), MPI_INFO_NULL,MPI_COMM_WORLD, &win_global_scheduling_step);
MPI_Win_create(&scheduled_tasks, sizeof(int),sizeof(int), MPI_INFO_NULL,MPI_COMM_WORLD, &win_global_scheduled_tasks);
const int one=1;
start_calc_time = MPI_Wtime();
while(start!=-1)
{
//printf("Id %d local_scheduled_tasks %d\n", id,local_scheduled_tasks);
MPI_Win_lock(MPI_LOCK_SHARED,0,0,win_global_scheduling_step);
MPI_Get_accumulate(&one,1,MPI_INT,&local_scheduling_step,1,MPI_INT,0,0,1,MPI_INT,MPI_SUM,win_global_scheduling_step);
MPI_Win_unlock(0,win_global_scheduling_step);
params.step = local_scheduling_step;
calculate_next_chunk_stateful(params,&chunk_size);
MPI_Win_lock(MPI_LOCK_SHARED,0,0,win_global_scheduled_tasks);
MPI_Get_accumulate(&chunk_size,1,MPI_INT,&local_scheduled_tasks,1,MPI_INT,0,0,1,MPI_INT,MPI_SUM,win_global_scheduled_tasks);
MPI_Win_unlock(0,win_global_scheduled_tasks);
if(local_scheduled_tasks>=params.total_tasks)
break;
start = local_scheduled_tasks;
end = start + MIN(params.total_tasks-local_scheduled_tasks, chunk_size);
//if(start==end)
// printf("Id %d start %d end %d local_scheduled_tasks %d\n", id , start, end, local_scheduled_tasks);
I_executed += end - start;
I_executed_chunks++;
calcSpinImages(images,target_object,bin_size, image_width, support_angle,false, params.total_tasks,start,end);
//printf("rank %d from %d to %d chunksize %d\n", id, end,start, end-start);
if(params.method==0)
start=-1;
}
end_calc_time = MPI_Wtime();
total_calc_time += end_calc_time - start_calc_time;
// get data from workers
// printf("master %d spent %f in loop\n",id ,total_calc_time);
MPI_Win_free(&win_global_scheduling_step);
MPI_Win_free(&win_global_scheduled_tasks);
MPI_Reduce(images,results,number_of_tasks*image_width*image_width,MPI_DOUBLE,MPI_SUM,DATAOWNER,MPI_COMM_WORLD);
double end_time= MPI_Wtime();
MPI_Gather(&total_calc_time,1,MPI_DOUBLE,&execution_times,1,MPI_DOUBLE,DATAOWNER,MPI_COMM_WORLD);
int * executed_at_id = (int *) calloc(params.total_workers,sizeof(int));
int * executed_chunks_at_id=(int *) calloc(params.total_workers,sizeof(int));
MPI_Gather(&I_executed,1,MPI_INT,executed_at_id,1,MPI_INT,DATAOWNER,MPI_COMM_WORLD);
MPI_Gather(&I_executed_chunks,1,MPI_INT,executed_chunks_at_id,1,MPI_INT,DATAOWNER,MPI_COMM_WORLD);
//MPI_Reduce(&total_calc_time, &max_loop_time, 1, MPI_DOUBLE, MPI_MAX, MASTER,MPI_COMM_WORLD);
//MPI_Reduce(&total_calc_time, &min_loop_time, 1, MPI_DOUBLE, MPI_MIN, MASTER,MPI_COMM_WORLD);
/* int i,j, c;
for (c = 0; c < 0; c++)
{
printf("image %d: \n",c);
for (i = 0; i < image_width; i++)
{
for (j = 0; j < image_width; j++)
printf("%f ", images[ (c * image_width * image_width) +( i * image_width) + j]);
printf("\n");
}
printf("\n");
}*/
max_loop_time = execution_times[0];
min_loop_time = execution_times[0];
if(id==DATAOWNER)
{
printf("----- ranks calculation times and total scheduled tasks -------\n");
printf("id time(s) total_tasks\n");
printf("%d %lf %d %d\n",id, execution_times[0],executed_at_id[0],executed_chunks_at_id[0]);
for (int i = 1; i < number_of_workers; i++)
{
if(max_loop_time < execution_times[i] )
max_loop_time = execution_times[i];
else if(min_loop_time > execution_times[i])
min_loop_time = execution_times[i];
printf("%d %lf %d %d\n",id, execution_times[i],executed_at_id[i],executed_chunks_at_id[i]);
}
printf("Program execution time %lf\n", end_time-start_time);
printf("Paralell max calculation time %lf\n", max_loop_time);
printf("Paralell min calculation time %lf\n", min_loop_time);
int i,j, c;
//number_of_tasks=0;
/*for (c = 0; c < number_of_tasks ; c++)
{
printf("image %d: \n",c);
for (i = 0; i < image_width; i++)
{
for (j = 0; j < image_width; j++)
printf("%f ", results[ (c * image_width * image_width) +( i * image_width) + j]);
printf("\n");
}
printf("\n");
}*/
if(executed_at_id!=NULL)
free(executed_at_id);
if(executed_chunks_at_id!=NULL)
free(executed_chunks_at_id);
//printf("average waiting time %lf\n", end_time-start_time);
// free resources
if(results!=NULL)
free(results);
if(images!=NULL)
free(images);
if(target_object!=NULL)
{
free(target_object->facesPerPoint);
free(target_object->dptrXPoints);
free(target_object->dptrYPoints);
free(target_object->dptrZPoints);
free(target_object->dptrXNPoints);
free(target_object->dptrYNPoints);
free(target_object->dptrZNPoints);
free(target_object);
}
}
}
//------- testing functions ----------
void parallel_one_sided_memory(int argc, char ** argv)
{
int id, number_of_workers;
MPI_Init(NULL,NULL);
MPI_Comm_size(MPI_COMM_WORLD, &number_of_workers);
MPI_Comm_rank(MPI_COMM_WORLD,&id);
all_processors_do(argc,argv,number_of_workers, id);
MPI_Finalize();
}
c4science · Help