This file is larger than 256 KB, so syntax highlighting was skipped.
This document is not UTF8. It was detected as ISO-8859-1 (Latin 1) and converted to UTF8 for display.
diff --git a/Readme.md b/Readme.md
new file mode 100644
index 0000000..5c58ff5
--- /dev/null
+++ b/Readme.md
@@ -0,0 +1,26 @@
+SCITAS Application Benchmark suite
+==================================
+
+This directory contains the benchmarks that SCITAS uses to evaluate new architectures. It includes
+
+* users applications benchmarks
+
+
+How to use
+----------
+
+### Files to modify for the whole setup
+
+* `clusters/cores.txt` contains the number of cores on which the tests must run.
+* `clusters/cluster.job` is a link to an appropriate file in `local-configs`. We suggest you create your own and link it. It contains specific configurations to the setup that is used by all the runs, when submitted to a SLURM scheduler. For example, modules to be loaded, varables to be set, or specific partition/QOS setting.
+
+### For more information
+
+More configuration has to be done at the application level. Please read the `clusters/user/Readme.md` file for more information.
+
+### License
+
+Unless otherwise stated:
+
+* source code (including bash and R scripts) is licensed under the GPLv2
+* the documentation is licensed under the GFDL 1.3
+ * \brief provides instances of all global variables.
+ */
+
+#include <stdio.h>
+#include <gsl/gsl_rng.h>
+#include "tags.h"
+#include "allvars.h"
+
+int SetMinTimeStepForActives=0;
+
+int ThisTask; /*!< the rank of the local processor */
+int NTask; /*!< number of processors */
+int PTask; /*!< smallest integer such that NTask <= 2^PTask */
+
+int NumPart; /*!< number of particles on the LOCAL processor */
+int N_gas; /*!< number of gas particles on the LOCAL processor */
+#if defined(SFR) || defined(STELLAR_PROP)
+int N_stars; /*!< number of stars particle on the LOCAL processor */
+#endif
+#ifdef MULTIPHASE
+int N_sph;
+int N_sticky;
+int N_stickyflaged;
+int N_dark;
+int NumColPotLocal; /*!< local number of potentially collisional particles */
+int NumColPot; /*!< total number of potentially collisional particles */
+int NumColLocal; /*!< local number of collisions */
+int NumCol; /*!< total number of collisions */
+int NumNoColLocal;
+int NumNoCol;
+#endif
+long long Ntype[6]; /*!< total number of particles of each type */
+int NtypeLocal[6]; /*!< local number of particles of each type */
+
+int NumForceUpdate; /*!< number of active particles on local processor in current timestep */
+int NumSphUpdate; /*!< number of active SPH particles on local processor in current timestep */
+#ifdef CHIMIE
+int NumStUpdate;
+#endif
+
+double CPUThisRun; /*!< Sums the CPU time for the process (current submission only) */
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+double CPU_Gravity;
+#endif
+
+int RestartFlag; /*!< taken from command line used to start code. 0 is normal start-up from
+ initial conditions, 1 is resuming a run from a set of restart files, while 2
+ marks a restart from a snapshot file. */
+
+char *Exportflag; /*!< Buffer used for flagging whether a particle needs to be exported to another process */
+
+int *Ngblist; /*!< Buffer to hold indices of neighbours retrieved by the neighbour search routines */
+
+int TreeReconstructFlag; /*!< Signals that a new tree needs to be constructed */
+#ifdef SFR
+int RearrangeParticlesFlag;/*!< Signals that particles must be rearanged */
+#endif
+
+
+int Flag_FullStep; /*!< This flag signals that the current step involves all particles */
+
+
+gsl_rng *random_generator; /*!< the employed random number generator of the GSL library */
+
+double RndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+
+#ifdef SFR
+double StarFormationRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef FEEDBACK_WIND
+double FeedbackWindRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE
+double ChimieRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+double ChimieKineticFeedbackRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+
+double DomainCorner[3]; /*!< gives the lower left corner of simulation volume */
+double DomainCenter[3]; /*!< gives the center of simulation volume */
+double DomainLen; /*!< gives the (maximum) side-length of simulation volume */
+double DomainFac; /*!< factor used for converting particle coordinates to a Peano-Hilbert mesh covering the simulation volume */
+int DomainMyStart; /*!< first domain mesh cell that resides on the local processor */
+int DomainMyLast; /*!< last domain mesh cell that resides on the local processor */
+int *DomainStartList; /*!< a table that lists the first domain mesh cell for all processors */
+int *DomainEndList; /*!< a table that lists the last domain mesh cell for all processors */
+double *DomainWork; /*!< a table that gives the total "work" due to the particles stored by each processor */
+int *DomainCount; /*!< a table that gives the total number of particles held by each processor */
+int *DomainCountSph; /*!< a table that gives the total number of SPH particles held by each processor */
+
+int *DomainTask; /*!< this table gives for each leaf of the top-level tree the processor it was assigned to */
+int *DomainNodeIndex; /*!< this table gives for each leaf of the top-level tree the corresponding node of the gravitational tree */
+FLOAT *DomainTreeNodeLen; /*!< this table gives for each leaf of the top-level tree the side-length of the corresponding node of the gravitational tree */
+FLOAT *DomainHmax; /*!< this table gives for each leaf of the top-level tree the maximum SPH smoothing length among the particles of the corresponding node of the gravitational tree */
+
+struct DomainNODE
+ *DomainMoment; /*!< this table stores for each node of the top-level tree corresponding node data from the gravitational tree */
+
+peanokey *DomainKeyBuf; /*!< this points to a buffer used during the exchange of particle data */
+
+peanokey *Key; /*!< a table used for storing Peano-Hilbert keys for particles */
+peanokey *KeySorted; /*!< holds a sorted table of Peano-Hilbert keys for all particles, used to construct top-level tree */
+
+
+int NTopnodes; /*!< total number of nodes in top-level tree */
+int NTopleaves; /*!< number of leaves in top-level tree. Each leaf can be assigned to a different processor */
+
+struct topnode_data
+ *TopNodes; /*!< points to the root node of the top-level tree */
+
+
+double TimeOfLastTreeConstruction; /*!< holds what it says, only used in connection with FORCETEST */
+
+
+
+
+/* variables for input/output, usually only used on process 0 */
+
+char ParameterFile[MAXLEN_FILENAME]; /*!< file name of parameterfile used for starting the simulation */
+
+FILE *FdInfo; /*!< file handle for info.txt log-file. */
+FILE *FdLog ; /*!< file handle for log.txt log-file. */
+FILE *FdEnergy; /*!< file handle for energy.txt log-file. */
+#ifdef SYSTEMSTATISTICS
+FILE *FdSystem;
+#endif
+FILE *FdTimings; /*!< file handle for timings.txt log-file. */
+FILE *FdCPU; /*!< file handle for cpu.txt log-file. */
+
+#ifdef FORCETEST
+FILE *FdForceTest; /*!< file handle for forcetest.txt log-file. */
+#endif
+
+#ifdef SFR
+FILE *FdSfr; /*!< file handle for sfr.txt log-file. */
+#endif
+
+#ifdef CHIMIE
+FILE *FdChimie; /*!< file handle for chimie log-file. */
+#endif
+
+#ifdef MULTIPHASE
+FILE *FdPhase; /*!< file handle for phase.txt log-file. */
+FILE *FdSticky; /*!< file handle for sticky.txt log-file. */
+#endif
+
+#ifdef AGN_ACCRETION
+FILE *FdAccretion; /*!< file handle for accretion.txt log-file. */
+#endif
+
+#ifdef BONDI_ACCRETION
+FILE *FdBondi; /*!< file handle for bondi.txt log-file. */
+#endif
+
+#ifdef BUBBLES
+FILE *FdBubble; /*!< file handle for bubble.txt log-file. */
+#endif
+
+double DriftTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological drift factors */
+double GravKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for gravitational forces */
+double HydroKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for hydrodynmical forces */
+
+#ifdef COSMICTIME
+double CosmicTimeTable[COSMICTIME_TABLE_LENGTH]; /*!< table for the computation of cosmic time */
+#endif
+
+void *CommBuffer; /*!< points to communication buffer, which is used in the domain decomposition, the
+ parallel tree-force computation, the SPH routines, etc. */
+
+
+/*! This structure contains data which is the SAME for all tasks (mostly code parameters read from the
+ * parameter file). Holding this data in a structure is convenient for writing/reading the restart file, and
+ * it allows the introduction of new global variables in a simple way. The only thing to do is to introduce
+ * them into this structure.
+ */
+struct global_data_all_processes
+ All;
+
+
+
+/*! This structure holds all the information that is
+ * stored for each particle of the simulation.
+ */
+struct particle_data
+ *P, /*!< holds particle data on local processor */
+ *DomainPartBuf; /*!< buffer for particle data used in domain decomposition */
+
+
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+struct sph_particle_data
+ *SphP, /*!< holds SPH particle data on local processor */
+ *DomainSphBuf; /*!< buffer for SPH particle data in domain decomposition */
+
+
+#ifdef STELLAR_PROP
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+struct st_particle_data
+ *StP, /*!< holds ST particle data on local processor */
+ *DomainStBuf; /*!< buffer for ST particle data in domain decomposition */
+#endif
+
+
+/* Variables for Tree
+ */
+
+int MaxNodes; /*!< maximum allowed number of internal nodes */
+int Numnodestree; /*!< number of (internal) nodes in each tree */
+
+struct NODE
+ *Nodes_base, /*!< points to the actual memory allocted for the nodes */
+ *Nodes; /*!< this is a pointer used to access the nodes which is shifted such that Nodes[All.MaxPart]
+ gives the first allocated node */
+
+
+int *Nextnode; /*!< gives next node in tree walk */
+int *Father; /*!< gives parent node in tree */
+
+
+struct extNODE /*!< this structure holds additional tree-node information which is not needed in the actual gravity computation */
+ *Extnodes_base, /*!< points to the actual memory allocted for the extended node information */
+ *Extnodes; /*!< provides shifted access to extended node information, parallel to Nodes/Nodes_base */
+
+
+
+
+
+/*! Header for the standard file format.
+ */
+struct io_header
+ header; /*!< holds header for snapshot files */
+
+#ifdef CHIMIE_EXTRAHEADER
+/*! Header for the chimie part.
+ */
+struct io_chimie_extraheader
+ chimie_extraheader;
+#endif
+
+
+char Tab_IO_Labels[IO_NBLOCKS][4]; /*<! This table holds four-byte character tags used for fileformat 2 */
+
+
+
+/* global state of system, used for global statistics
+ */
+struct state_of_system
+ SysState; /*<! Structure for storing some global statistics about the simulation. */
+
+/*! This structure contains data related to the energy budget.
+ These values are different for each task. It need to be stored
+ in the restart flag.
+ */
+struct local_state_of_system
+ LocalSysState; /*<! Structure for storing some local statistics about the simulation. */
+
+
+
+/* Various structures for communication
+ */
+struct gravdata_in
+ *GravDataIn, /*!< holds particle data to be exported to other processors */
+ *GravDataGet, /*!< holds particle data imported from other processors */
+ *GravDataResult, /*!< holds the partial results computed for imported particles. Note: We use GravDataResult = GravDataGet, such that the result replaces the imported data */
+ *GravDataOut; /*!< holds partial results received from other processors. This will overwrite the GravDataIn array */
+
+struct gravdata_index
+ *GravDataIndexTable; /*!< the particles to be exported are grouped by task-number. This table allows the results to be disentangled again and to be assigned to the correct particle */
+
+
+
+struct densdata_in
+ *DensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *DensDataGet; /*!< holds imported particle data for SPH density computation */
+
+struct densdata_out
+ *DensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *DensDataPartialResult; /*!< imported partial SPH density results from other processors */
+
+
+
+struct hydrodata_in
+ *HydroDataIn, /*!< holds particle data for SPH hydro-force computation to be exported to other processors */
+ *HydroDataGet; /*!< holds imported particle data for SPH hydro-force computation */
+
+struct hydrodata_out
+ *HydroDataResult, /*!< stores the locally computed SPH hydro results for imported particles */
+ *HydroDataPartialResult; /*!< imported partial SPH hydro-force results from other processors */
+
+
+#ifdef MULTIPHASE
+
+struct stickydata_in
+ *StickyDataIn,
+ *StickyDataGet;
+
+struct stickydata_out
+ *StickyDataResult,
+ *StickyDataPartialResult;
+
+struct Sticky_index *StickyIndex;
+#endif
+
+#ifdef CHIMIE
+struct chimiedata_in
+ *ChimieDataIn, /*!< holds particle data for Chimie computation to be exported to other processors */
+ *ChimieDataGet; /*!< holds imported particle data for Chimie computation */
+
+struct chimiedata_out
+ *ChimieDataResult, /*!< stores the locally computed Chimie results for imported particles */
+ *ChimieDataPartialResult; /*!< imported partial Chimie results from other processors */
+
+
+struct starsdensdata_in
+ *StarsDensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *StarsDensDataGet; /*!< holds imported particle data for SPH density computation */
+
+struct starsdensdata_out
+ *StarsDensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *StarsDensDataPartialResult; /*!< imported partial SPH density results from other processors */
+#define NORM_COEFF M_PI /*!< Coefficient for kernel normalization. */
+#endif
+
+
+#ifdef MULTIPHASE
+#define GAS_SPH 0
+#define GAS_STICKY 1
+#define GAS_DARK 2
+#endif
+
+#if defined(SFR) || defined(STELLAR_PROP)
+#define ST 1
+#endif
+
+
+
+#ifdef CHIMIE
+#define NELEMENTS 5
+#define MAXNELEMENTS 64
+#define FIRST_ELEMENT "Fe"
+#define FE 0
+#endif
+
+#ifdef COOLING
+#define COOLING_NMETALICITIES 9
+#define COOLING_NTEMPERATURES 171
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+#define VELOCITY_DISPERSION_SIZE 3
+#endif
+
+extern int SetMinTimeStepForActives;
+
+extern int ThisTask; /*!< the rank of the local processor */
+extern int NTask; /*!< number of processors */
+extern int PTask; /*!< smallest integer such that NTask <= 2^PTask */
+
+extern int NumPart; /*!< number of particles on the LOCAL processor */
+extern int N_gas; /*!< number of gas particles on the LOCAL processor */
+#if defined(SFR) || defined(STELLAR_PROP)
+extern int N_stars; /*!< number of stars particle on the LOCAL processor */
+#endif
+#ifdef MULTIPHASE
+extern int N_sph;
+extern int N_sticky;
+extern int N_stickyflaged;
+extern int N_dark;
+extern int NumColPotLocal; /*!< local number of potentially collisional particles */
+extern int NumColPot; /*!< total number of potentially collisional particles */
+extern int NumColLocal; /*!< local number of collisions */
+extern int NumCol; /*!< total number of collisions */
+extern int NumNoColLocal;
+extern int NumNoCol;
+#endif
+extern long long Ntype[6]; /*!< total number of particles of each type */
+extern int NtypeLocal[6]; /*!< local number of particles of each type */
+
+extern int NumForceUpdate; /*!< number of active particles on local processor in current timestep */
+extern int NumSphUpdate; /*!< number of active SPH particles on local processor in current timestep */
+#ifdef CHIMIE
+extern int NumStUpdate;
+#endif
+
+extern double CPUThisRun; /*!< Sums the CPU time for the process (current submission only) */
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+extern double CPU_Gravity;
+#endif
+
+extern int RestartFlag; /*!< taken from command line used to start code. 0 is normal start-up from
+ initial conditions, 1 is resuming a run from a set of restart files, while 2
+ marks a restart from a snapshot file. */
+
+extern char *Exportflag; /*!< Buffer used for flagging whether a particle needs to be exported to another process */
+
+extern int *Ngblist; /*!< Buffer to hold indices of neighbours retrieved by the neighbour search routines */
+
+extern int TreeReconstructFlag; /*!< Signals that a new tree needs to be constructed */
+#ifdef SFR
+extern int RearrangeParticlesFlag;/*!< Signals that particles must be rearanged */
+#endif
+
+extern int Flag_FullStep; /*!< This flag signals that the current step involves all particles */
+
+
+extern gsl_rng *random_generator; /*!< the employed random number generator of the GSL library */
+
+extern double RndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+
+#ifdef SFR
+extern double StarFormationRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+#ifdef FEEDBACK_WIND
+extern double FeedbackWindRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE
+extern double ChimieRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+extern double ChimieKineticFeedbackRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+extern double DomainCorner[3]; /*!< gives the lower left corner of simulation volume */
+extern double DomainCenter[3]; /*!< gives the center of simulation volume */
+extern double DomainLen; /*!< gives the (maximum) side-length of simulation volume */
+extern double DomainFac; /*!< factor used for converting particle coordinates to a Peano-Hilbert mesh covering the simulation volume */
+extern int DomainMyStart; /*!< first domain mesh cell that resides on the local processor */
+extern int DomainMyLast; /*!< last domain mesh cell that resides on the local processor */
+extern int *DomainStartList; /*!< a table that lists the first domain mesh cell for all processors */
+extern int *DomainEndList; /*!< a table that lists the last domain mesh cell for all processors */
+extern double *DomainWork; /*!< a table that gives the total "work" due to the particles stored by each processor */
+extern int *DomainCount; /*!< a table that gives the total number of particles held by each processor */
+extern int *DomainCountSph; /*!< a table that gives the total number of SPH particles held by each processor */
+
+extern int *DomainTask; /*!< this table gives for each leaf of the top-level tree the processor it was assigned to */
+extern int *DomainNodeIndex; /*!< this table gives for each leaf of the top-level tree the corresponding node of the gravitational tree */
+extern FLOAT *DomainTreeNodeLen; /*!< this table gives for each leaf of the top-level tree the side-length of the corresponding node of the gravitational tree */
+extern FLOAT *DomainHmax; /*!< this table gives for each leaf of the top-level tree the maximum SPH smoothing length among the particles of the corresponding node of the gravitational tree */
+
+extern struct DomainNODE
+{
+ FLOAT s[3]; /*!< center-of-mass coordinates */
+ FLOAT vs[3]; /*!< center-of-mass velocities */
+ FLOAT mass; /*!< mass of node */
+#ifdef STELLAR_FLUX
+ FLOAT starlum; /*!< star luminosity of node */
+#endif
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ int bitflags; /*!< this bit-field encodes the particle type with the largest softening among the particles of the nodes, and whether there are particles with different softening in the node */
+#else
+ FLOAT maxsoft; /*!< hold the maximum gravitational softening of particles in the
+ node if the ADAPTIVE_GRAVSOFT_FORGAS option is selected */
+#endif
+#endif
+}
+ *DomainMoment; /*!< this table stores for each node of the top-level tree corresponding node data from the gravitational tree */
+
+extern peanokey *DomainKeyBuf; /*!< this points to a buffer used during the exchange of particle data */
+
+extern peanokey *Key; /*!< a table used for storing Peano-Hilbert keys for particles */
+extern peanokey *KeySorted; /*!< holds a sorted table of Peano-Hilbert keys for all particles, used to construct top-level tree */
+
+
+extern int NTopnodes; /*!< total number of nodes in top-level tree */
+extern int NTopleaves; /*!< number of leaves in top-level tree. Each leaf can be assigned to a different processor */
+
+extern struct topnode_data
+{
+ int Daughter; /*!< index of first daughter cell (out of 8) of top-level node */
+ int Pstart; /*!< for the present top-level node, this gives the index of the first node in the concatenated list of topnodes collected from all processors */
+ int Blocks; /*!< for the present top-level node, this gives the number of corresponding nodes in the concatenated list of topnodes collected from all processors */
+ int Leaf; /*!< if the node is a leaf, this gives its number when all leaves are traversed in Peano-Hilbert order */
+ peanokey Size; /*!< number of Peano-Hilbert mesh-cells represented by top-level node */
+ peanokey StartKey; /*!< first Peano-Hilbert key in top-level node */
+ long long Count; /*!< counts the number of particles in this top-level node */
+}
+ *TopNodes; /*!< points to the root node of the top-level tree */
+
+
+extern double TimeOfLastTreeConstruction; /*!< holds what it says, only used in connection with FORCETEST */
+
+
+
+/* variables for input/output, usually only used on process 0 */
+
+extern char ParameterFile[MAXLEN_FILENAME]; /*!< file name of parameterfile used for starting the simulation */
+
+extern FILE *FdInfo; /*!< file handle for info.txt log-file. */
+extern FILE *FdLog; /*!< file handle for log.txt log-file. */
+extern FILE *FdEnergy; /*!< file handle for energy.txt log-file. */
+#ifdef SYSTEMSTATISTICS
+extern FILE *FdSystem;
+#endif
+extern FILE *FdTimings; /*!< file handle for timings.txt log-file. */
+extern FILE *FdCPU; /*!< file handle for cpu.txt log-file. */
+
+#ifdef FORCETEST
+extern FILE *FdForceTest; /*!< file handle for forcetest.txt log-file. */
+#endif
+
+#ifdef SFR
+extern FILE *FdSfr; /*!< file handle for sfr.txt log-file. */
+#endif
+
+#ifdef CHIMIE
+extern FILE *FdChimie; /*!< file handle for chimie log-file. */
+#endif
+
+#ifdef MULTIPHASE
+extern FILE *FdPhase; /*!< file handle for pase.txt log-file. */
+extern FILE *FdSticky; /*!< file handle for sticky.txt log-file. */
+#endif
+
+#ifdef AGN_ACCRETION
+extern FILE *FdAccretion; /*!< file handle for accretion.txt log-file. */
+#endif
+
+#ifdef BONDI_ACCRETION
+extern FILE *FdBondi; /*!< file handle for bondi.txt log-file. */
+#endif
+
+#ifdef BUBBLES
+extern FILE *FdBubble; /*!< file handle for bubble.txt log-file. */
+#endif
+
+extern double DriftTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological drift factors */
+extern double GravKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for gravitational forces */
+extern double HydroKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for hydrodynmical forces */
+
+#ifdef COSMICTIME
+extern double CosmicTimeTable[COSMICTIME_TABLE_LENGTH]; /*!< table for the computation of cosmic time */
+#endif
+
+extern void *CommBuffer; /*!< points to communication buffer, which is used in the domain decomposition, the
+ parallel tree-force computation, the SPH routines, etc. */
+
+
+
+/*! This structure contains data which is the SAME for all tasks (mostly code parameters read from the
+ * parameter file). Holding this data in a structure is convenient for writing/reading the restart file, and
+ * it allows the introduction of new global variables in a simple way. The only thing to do is to introduce
+ * them into this structure.
+ */
+extern struct global_data_all_processes
+{
+ long long TotNumPart; /*!< total particle numbers (global value) */
+ long long TotN_gas; /*!< total gas particle number (global value) */
+#if defined(SFR) || defined(STELLAR_PROP)
+ long long TotN_stars; /*!< total stars particle number (global value) */
+#endif
+#ifdef MULTIPHASE
+ long long TotN_sph; /*!< total sph particle number (global value) */
+ long long TotN_sticky; /*!< total sticky particle number (global value) */
+ long long TotN_stickyflaged; /*!< total sticky flaged particle number (global value) */
+ long long TotN_stickyactive; /*!< total sticky active particle number (global value) */
+ long long TotN_dark; /*!< total dark particle number (global value) */
+#endif
+
+ int MaxPart; /*!< This gives the maxmimum number of particles that can be stored on one processor. */
+ int MaxPartSph; /*!< This gives the maxmimum number of SPH particles that can be stored on one processor. */
+#ifdef STELLAR_PROP
+ int MaxPartStars; /*!< This gives the maxmimum number of Star particles that can be stored on one processor. */
+#endif
+ double BoxSize; /*!< Boxsize in case periodic boundary conditions are used */
+
+ int ICFormat; /*!< selects different versions of IC file-format */
+
+ int SnapFormat; /*!< selects different versions of snapshot file-formats */
+
+ int NumFilesPerSnapshot; /*!< number of files in multi-file snapshot dumps */
+ int NumFilesWrittenInParallel;/*!< maximum number of files that may be written simultaneously when
+ writing/reading restart-files, or when writing snapshot files */
+
+ int BufferSize; /*!< size of communication buffer in MB */
+ int BunchSizeForce; /*!< number of particles fitting into the buffer in the parallel tree-force algorithm */
+ int BunchSizeDensity; /*!< number of particles fitting into the communication buffer in the density computation */
+ int BunchSizeHydro; /*!< number of particles fitting into the communication buffer in the SPH hydrodynamical force computation */
+ int BunchSizeDomain; /*!< number of particles fitting into the communication buffer in the domain decomposition */
+#ifdef MULTIPHASE
+ int BunchSizeSticky; /*!< number of particles fitting into the communication buffer in the Chimie computation */
+#endif
+#ifdef CHIMIE
+ int BunchSizeChimie; /*!< number of particles fitting into the communication buffer in the Chimie computation */
+ int BunchSizeStarsDensity; /*!< number of particles fitting into the communication buffer in the star density computation */
+#endif
+
+
+ double PartAllocFactor; /*!< in order to maintain work-load balance, the particle load will usually
+ NOT be balanced. Each processor allocates memory for PartAllocFactor times
+ the average number of particles to allow for that */
+
+ double TreeAllocFactor; /*!< Each processor allocates a number of nodes which is TreeAllocFactor times
+ the maximum(!) number of particles. Note: A typical local tree for N
+ particles needs usually about ~0.65*N nodes. */
+
+#ifdef SFR
+ double StarsAllocFactor; /*!< Estimated fraction of gas particles that will form stars during the simulation
+ This allow to reduce the memory stored for stellar particles */
+
+#endif
+
+ /* some SPH parameters */
+
+ double DesNumNgb; /*!< Desired number of SPH neighbours */
+ double MaxNumNgbDeviation; /*!< Maximum allowed deviation neighbour number */
+
+ double ArtBulkViscConst; /*!< Sets the parameter \f$\alpha\f$ of the artificial viscosity */
+#ifdef ART_CONDUCTIVITY
+ double ArtCondConst; /*!< Sets the parameter \f$\alpha\f$ of the artificial conductivity */
+ double ArtCondThreshold;
+#endif
+ double InitGasTemp; /*!< may be used to set the temperature in the IC's */
+ double MinGasTemp; /*!< may be used to set a floor for the gas temperature */
+ double MinEgySpec; /*!< the minimum allowed temperature expressed as energy per unit mass */
+ long long TotNumOfForces; /*!< counts total number of force computations */
+ long long NumForcesSinceLastDomainDecomp; /*!< count particle updates since last domain decomposition */
+
+ /* system of units */
+
+ double G; /*!< Gravity-constant in internal units */
+ double UnitTime_in_s; /*!< factor to convert internal time unit to seconds/h */
+ double UnitMass_in_g; /*!< factor to convert internal mass unit to grams/h */
+ double UnitVelocity_in_cm_per_s; /*!< factor to convert intqernal velocity unit to cm/sec */
+ double UnitLength_in_cm; /*!< factor to convert internal length unit to cm/h */
+ double UnitPressure_in_cgs; /*!< factor to convert internal pressure unit to cgs units (little 'h' still around!) */
+ double UnitDensity_in_cgs; /*!< factor to convert internal length unit to g/cm^3*h^2 */
+ double UnitCoolingRate_in_cgs; /*!< factor to convert internal cooling rate to cgs units */
+ double UnitEnergy_in_cgs; /*!< factor to convert internal energy to cgs units */
+ double UnitTime_in_Megayears; /*!< factor to convert internal time to megayears/h */
+ double GravityConstantInternal; /*!< If set to zero in the parameterfile, the internal value of the
+ gravitational constant is set to the Newtonian value based on the system of
+ units specified. Otherwise the value provided is taken as internal gravity constant G. */
+
+
+ /* Cosmological parameters */
+
+ double Hubble; /*!< Hubble-constant in internal units */
+ double Omega0; /*!< matter density in units of the critical density (at z=0)*/
+ double OmegaLambda; /*!< vaccum energy density relative to crictical density (at z=0) */
+ double OmegaBaryon; /*!< baryon density in units of the critical density (at z=0)*/
+ double HubbleParam; /*!< little `h', i.e. Hubble constant in units of 100 km/s/Mpc. Only needed to get absolute physical values for cooling physics */
+
+
+ /* Code options */
+
+ int ComovingIntegrationOn; /*!< flags that comoving integration is enabled */
+ int PeriodicBoundariesOn; /*!< flags that periodic boundaries are enabled */
+ int ResubmitOn; /*!< flags that automatic resubmission of job to queue system is enabled */
+ int TypeOfOpeningCriterion; /*!< determines tree cell-opening criterion: 0 for Barnes-Hut, 1 for relative criterion */
+ int TypeOfTimestepCriterion; /*!< gives type of timestep criterion (only 0 supported right now - unlike gadget-1.1) */
+ int OutputListOn; /*!< flags that output times are listed in a specified file */
+
+
+ /* Parameters determining output frequency */
+
+ int SnapshotFileCount; /*!< number of snapshot that is written next */
+ double TimeBetSnapshot; /*!< simulation time interval between snapshot files */
+ double TimeOfFirstSnapshot; /*!< simulation time of first snapshot files */
+ double TimeLastRestartFile; /*!< cpu-time when last restart-file was written */
+ double TimeBetStatistics; /*!< simulation time interval between computations of energy statistics */
+ double TimeLastStatistics; /*!< simulation time when the energy statistics was computed the last time */
+ int NumCurrentTiStep; /*!< counts the number of system steps taken up to this point */
+
+
+ /* Current time of the simulation, global step, and end of simulation */
+
+ double Time; /*!< current time of the simulation */
+ double TimeBegin; /*!< time of initial conditions of the simulation */
+ double TimeStep; /*!< difference between current times of previous and current timestep */
+ double TimeMax; /*!< marks the point of time until the simulation is to be evolved */
+
+
+ /* variables for organizing discrete timeline */
+
+ double Timebase_interval; /*!< factor to convert from floating point time interval to integer timeline */
+ int Ti_Current; /*!< current time on integer timeline */
+ int Ti_nextoutput; /*!< next output time on integer timeline */
+#ifdef FLEXSTEPS
+ int PresentMinStep; /*!< If FLEXSTEPS is used, particle timesteps are chosen as multiples of the present minimum timestep. */
+ int PresentMaxStep; /*!< If FLEXSTEPS is used, this is the maximum timestep in timeline units, rounded down to the next power 2 division */
+#endif
+#ifdef PMGRID
+ int PM_Ti_endstep; /*!< begin of present long-range timestep */
+ int PM_Ti_begstep; /*!< end of present long-range timestep */
+#endif
+
+
+ /* Placement of PM grids */
+
+#ifdef PMGRID
+ double Asmth[2]; /*!< Gives the scale of the long-range/short-range split (in mesh-cells), both for the coarse and the high-res mesh */
+ double Rcut[2]; /*!< Gives the maximum radius for which the short-range force is evaluated with the tree (in mesh-cells), both for the coarse and the high-res mesh */
+ double Corner[2][3]; /*!< lower left corner of coarse and high-res PM-mesh */
+ double UpperCorner[2][3]; /*!< upper right corner of coarse and high-res PM-mesh */
+ double Xmintot[2][3]; /*!< minimum particle coordinates both for coarse and high-res PM-mesh */
+ double Xmaxtot[2][3]; /*!< maximum particle coordinates both for coarse and high-res PM-mesh */
+ double TotalMeshSize[2]; /*!< total extension of coarse and high-res PM-mesh */
+#endif
+
+
+ /* Variables that keep track of cumulative CPU consumption */
+
+ double TimeLimitCPU; /*!< CPU time limit as defined in parameterfile */
+ double CPU_TreeConstruction; /*!< time spent for constructing the gravitational tree */
+ double CPU_TreeWalk; /*!< actual time spent for pure tree-walks */
+ double CPU_Gravity; /*!< cumulative time used for gravity computation (tree-algorithm only) */
+ double CPU_Potential; /*!< time used for computing gravitational potentials */
+ double CPU_Domain; /*!< cumulative time spent for domain decomposition */
+ double CPU_Snapshot; /*!< time used for writing snapshot files */
+ double CPU_Total; /*!< cumulative time spent for domain decomposition */
+ double CPU_CommSum; /*!< accumulated time used for communication, and for collecting partial results, in tree-gravity */
+ double CPU_Imbalance; /*!< cumulative time lost accross all processors as work-load imbalance in gravitational tree */
+ double CPU_HydCompWalk; /*!< time used for actual SPH computations, including neighbour search */
+ double CPU_HydCommSumm; /*!< cumulative time used for communication in SPH, and for collecting partial results */
+ double CPU_HydImbalance; /*!< cumulative time lost due to work-load imbalance in SPH */
+ double CPU_Hydro; /*!< cumulative time spent for SPH related computations */
+#ifdef SFR
+ double CPU_StarFormation; /*!< cumulative time spent for star formation computations */
+#endif
+#ifdef CHIMIE
+ double CPU_Chimie; /*!< cumulative time spent for chimie computations */
+ double CPU_ChimieDensCompWalk;
+ double CPU_ChimieDensCommSumm;
+ double CPU_ChimieDensImbalance;
+ double CPU_ChimieDensEnsureNgb;
+ double CPU_ChimieCompWalk;
+ double CPU_ChimieCommSumm;
+ double CPU_ChimieImbalance;
+#endif
+#ifdef MULTIPHASE
+ double CPU_Sticky; /*!< cumulative time spent for sticky computations */
+#endif
+ double CPU_EnsureNgb; /*!< time needed to iterate on correct neighbour numbers */
+ double CPU_Predict; /*!< cumulative time to drift the system forward in time, including dynamic tree updates */
+ double CPU_TimeLine; /*!< time used for determining new timesteps, and for organizing the timestepping, including kicks of active particles */
+ double CPU_PM; /*!< time used for long-range gravitational force */
+ double CPU_Peano; /*!< time required to establish Peano-Hilbert order */
+
+
+#ifdef DETAILED_CPU_DOMAIN
+ double CPU_Domain_findExtend;
+ double CPU_Domain_determineTopTree;
+ double CPU_Domain_sumCost;
+ double CPU_Domain_findSplit;
+ double CPU_Domain_shiftSplit;
+ double CPU_Domain_countToGo;
+ double CPU_Domain_exchange;
+#endif
+#ifdef DETAILED_CPU_GRAVITY
+ double CPU_Gravity_TreeWalk1;
+ double CPU_Gravity_TreeWalk2;
+ double CPU_Gravity_CommSum1;
+ double CPU_Gravity_CommSum2;
+ double CPU_Gravity_Imbalance1;
+ double CPU_Gravity_Imbalance2;
+#endif
+
+#ifdef COOLING
+ double CPU_Cooling;
+#endif
+
+#ifdef DETAILED_CPU
+ double CPU_Leapfrog;
+ double CPU_Physics;
+ double CPU_Residual;
+ double CPU_Accel;
+ double CPU_Begrun;
+#endif
+
+
+
+
+
+
+ /* tree code opening criterion */
+
+ double ErrTolTheta; /*!< BH tree opening angle */
+ double ErrTolForceAcc; /*!< parameter for relative opening criterion in tree walk */
+
+
+ /* adjusts accuracy of time-integration */
+
+ double ErrTolIntAccuracy; /*!< accuracy tolerance parameter \f$ \eta \f$ for timestep criterion. The
+ timestep is \f$ \Delta t = \sqrt{\frac{2 \eta eps}{a}} \f$ */
+
+ double MinSizeTimestep; /*!< minimum allowed timestep. Normally, the simulation terminates if the
+ timestep determined by the timestep criteria falls below this limit. */
+ double MaxSizeTimestep; /*!< maximum allowed timestep */
+
+ double MaxRMSDisplacementFac; /*!< this determines a global timestep criterion for cosmological simulations
+ in comoving coordinates. To this end, the code computes the rms velocity
+ of all particles, and limits the timestep such that the rms displacement
+ is a fraction of the mean particle separation (determined from the
+ particle mass and the cosmological parameters). This parameter specifies
+ this fraction. */
+
+ double CourantFac; /*!< SPH-Courant factor */
+
+
+ /* frequency of tree reconstruction/domain decomposition */
+
+ double TreeDomainUpdateFrequency; /*!< controls frequency of domain decompositions */
+
+
+ /* Gravitational and hydrodynamical softening lengths (given in terms of an `equivalent' Plummer softening length).
+ * Five groups of particles are supported 0="gas", 1="halo", 2="disk", 3="bulge", 4="stars", 5="bndry"
+ */
+
+ double MinGasHsmlFractional; /*!< minimum allowed SPH smoothing length in units of SPH gravitational softening length */
+ *GravDataIn, /*!< holds particle data to be exported to other processors */
+ *GravDataGet, /*!< holds particle data imported from other processors */
+ *GravDataResult, /*!< holds the partial results computed for imported particles. Note: We use GravDataResult = GravDataGet, such that the result replaces the imported data */
+ *GravDataOut; /*!< holds partial results received from other processors. This will overwrite the GravDataIn array */
+
+extern struct gravdata_index
+{
+ int Task;
+ int Index;
+ int SortIndex;
+}
+ *GravDataIndexTable; /*!< the particles to be exported are grouped by task-number. This table allows the results to be disentangled again and to be assigned to the correct particle */
+
+
+
+extern struct densdata_in
+{
+ FLOAT Pos[3];
+ FLOAT Vel[3];
+ FLOAT Hsml;
+#ifdef MULTIPHASE
+ int Phase;
+#endif
+ int Index;
+ int Task;
+#ifdef ART_CONDUCTIVITY
+ FLOAT EnergyIntPred;
+#endif
+}
+ *DensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *DensDataGet; /*!< holds imported particle data for SPH density computation */
+
+extern struct densdata_out
+{
+ FLOAT Rho;
+ FLOAT Div, Rot[3];
+ FLOAT DhsmlDensity;
+ FLOAT Ngb;
+#ifdef ART_CONDUCTIVITY
+ FLOAT GradEnergyInt[3];
+#endif
+}
+ *DensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *DensDataPartialResult; /*!< imported partial SPH density results from other processors */
+ //All.StarFormationNStarsFromGas = all.StarFormationNStarsFromGas; /* do not change the param. if restarting, else, StarFormationStarMass will be wrong */
+ // printf("(%d) %g the particle 65546 has a max livetime of %g (metal=%g Mmin=%g)\n",ThisTask,All.Time,maxlivetime,StP[m].Metal[NELEMENTS-1],Cp->Mmin);
+ * This function initializes the MPI communication packages, and sets
+ * cpu-time counters to 0. Then begrun() is called, which sets up
+ * the simulation either from IC's or from restart files. Finally,
+ * run() is started, the main simulation loop, which iterates over
+ * the timesteps.
+ */
+int main(int argc, char **argv)
+{
+ double t0, t1;
+
+ MPI_Init(&argc, &argv);
+ MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask);
+ MPI_Comm_size(MPI_COMM_WORLD, &NTask);
+
+ if(NTask <= 1)
+ {
+ if(ThisTask == 0)
+ printf
+ ("Note: This is a massively parallel code, but you are running with 1 processor only.\nCompared to an equivalent serial code, there is some unnecessary overhead.\n");
+ }
+
+ for(PTask = 0; NTask > (1 << PTask); PTask++);
+
+ if(argc < 2)
+ {
+ if(ThisTask == 0)
+ {
+ printf("Parameters are missing.\n");
+ printf("Call with <ParameterFile> [<RestartFlag>]\n");
+ fprintf(FdTimings,"Step = %06d Time = %g \n\n",All.NumCurrentTiStep,All.Time);
+
+
+ fprintf(FdTimings,"%g %g : Total number of active particles : %d%09d\n\n",All.Time,All.TimeStep,(int) (NumActivePatricles / 1000000000), (int) (NumActivePatricles % 1000000000));
+ //printf("(%d) new star (a) i=%d id=%d (StPi=%d PIdx=%d)\n",ThisTask,N_gas+N_stars+i,P[N_gas+N_stars+i].ID,P[N_gas+N_stars+i].StPIdx,StP[P[N_gas+N_stars+i].StPIdx].PIdx);
+#endif
+
+ }
+ }
+ else /* move other particles outwards */
+ {
+
+ if (NumPart-N_gas-N_stars>0) /* there is other particles */
+ {
+ for(i=N_gas+N_stars;i<NumPart;i++)
+ {
+ Psave = P[i];
+ P[i] = P[i+NumNewStars];
+ P[i+NumNewStars] = Psave;
+#ifdef STELLAR_PROP
+ /* set new index for StP */
+ StP[P[i].StPIdx].PIdx = i;
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[i].StPIdx].ID = P[i].ID;
+#endif
+
+#endif
+ //printf("(%d) new star (b) i=%d id=%d\n",ThisTask,N_gas+N_stars+i,P[N_gas+N_stars+i].ID);
+ }
+
+
+
+ /* now, set new index for the StP that where untouched */
+ for(i=N_gas+N_stars+1;i<N_gas+N_stars+NumNewStars;i++) /* here, we should move up to <N_gas+N_stars+NumNewStars */
+ { /* however, if there is only one particle, it will not be counted */
+#ifdef STELLAR_PROP
+ /* set new index for StP */
+ StP[P[i].StPIdx].PIdx = i;
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[i].StPIdx].ID = P[i].ID;
+#endif
+#endif
+ }
+
+
+
+
+ }
+ else /* there is no other particles outside, i.e, new particles are already the last ones */
+This page describes the users benchmakrs than SCITAS runs on their machines and when testing new architectures. These benchmarks are also used for tenders.
+
+You need Intel Compilers and MKL, as well as Intel MPI to run these benchmarks.
+
+### Settings, general
+
+Settings are recorded in the file `cores.txt`. List there on how many cores you want to run.
+
+### Settings, scheduler
+
+Settings for the scheduler can be found in `../cluster.job`. Put in this file what is between the SLURM directives and the actual run command. An example is provided.
+
+
+
+Quantum Espresso
+----------------
+
+Quantum Espresso (hereafter abbreviated by QE) stands for opEn Source Package for Research in Electronic Structure, Simulation, and Optimization. It is an integrated suite of computer codes for electronic-structure calculations and materials modeling at the nanoscale. It is based on Density-Functional Theory (DFT), using a Plane-Wave basis set and pseudo-potentials.
+
+More details can be found at http://www.quantum-espresso.org
+
+The code is written in Fortran 95.
+
+Two kinds of simulation are performed in this benchmark :
+It is important to be sure that the program finds the MKL libraries. Have a look at the make.sys file.
+
+* You can compile only the parts of Quantum Espresso which are needed for this benchmark with
+
+ make cp pw
+
+* After the compilation successfully exits, you can find the binaries in the `espresso-5.3.0/bin` directory
+
+### Execution
+
+* Go to the QE directory
+
+ cd ../QE
+
+* if you want to change the input of the program (problem size for example), you should modify `PW/generate-pw-test` and `CP/generate-cp-test`. For example, you can change the size of the problem by setting the `ecutwfc` to 70.0 for a run using 64 GB of RAM.
+
+* have a look at the `runme.sh` (for interactive runs) or `runme_slurm.sh` (for runs with the SLURM scheduler). The only part you should want to edit are the environment variables at the top.
+
+* depending if you are on a cluster of on a machine in interactive mode, run `runme.sh` or `runme_slurm.sh`.
+
+GEAR
+----
+
+GEAR is a fully parallel code for cosmological N-body simulations. It is based on the freely available code GADGET-2 (version 2.0) that it supplements with the complex treatment of the baryon physics (Y. Revaz and P. Jablonka, Astronomy & Astrophysics, 2012). It computes gravitational forces with a hierarchical tree algorithm (optionally in combination with a particle-mesh scheme for long-range gravitational forces) and represents fluids by means of smoothed particle hydrodynamics (SPH).
+
+It is written in C and parallelized with the MPI communication interface.
+
+For this benchmark, higher CPU frequency provides higher performance. Our test cases produce a large amount of data. Performances also depend on the speed of the storage.
+
+Please note that the GEAR benchmark depends on the FFTW and GSL libraries.
+
+### Compiling
+
+We will use in this example the `gsl-1.15` and `fftw-2.1.5` libraries. In order to compile these pre-requisites, do the following:
+Now, you can compile GEAR. Enter the GEAR directory. In the src directory, edit the `Makefile` in the following way:
+
+* look for the string `modify to suit your machine` and verify the variables
+* type make
+* you should end up with a binary called `Gadget2`
+
+### Execution
+
+* choose the input file. A small problem (32 GB of RAM) is available here: http://scitas-data.epfl.ch/benchmarks/inputs/snap-118.dat.bz2. A larger problem (64 GB of RAM) is available here: http://scitas-data.epfl.ch/benchmarks/inputs/snap-284.dat.bz2. Download the input file to the GEAR directory, decompress it and rename it (or link it to) `snap.dat`.
+
+* have a look at the `runme.sh` (for interactive runs) or `runme_slurm.sh` (for runs with the SLURM scheduler). The only part you should want to edit are the environment variables at the top.
+
+* make sure the GEAR directory is clean. No `dIrr-*` directory from previous runs must be present
+
+* execute `runme.sh` or `runme_slurm.sh`
+
+CPMD
+----
+
+CPMD (Car-Parrinello Molecular Dynamics) is an ab-initio molecular dynamics simulation software that provides a way to perform molecular dynamic simulations from first principles, using a plane wave/pseudopotential implementation of Density Functional Theory. Its development is coordinated by the CPMD consortium (http://www.cpmd.org/).
+
+It is written in Fortran 90 and parallelized with MPI.
+
+The CPMD kernel can perform:
+
+* Car-Parrinello Molecular Dynamics
+* Ab-initio Molecular Dynamics
+* Response calculations
+* Metadynamics
+* QM/MM
+
+This code is memory bound.
+
+### Compilation
+
+* Enter the CPMD directory
+* Type `make`
+
+### Execution
+
+* have a look at the `runme.sh` (for interactive runs) or `runme_slurm.sh` (for runs with the SLURM scheduler). The only part you should want to edit are the environment variables at the top.
+
+* execute `runme.sh` or `runme_slurm.sh`
+
+MiniFE
+------
+
+MiniFE is a Finite Element mini-application which implements a couple of kernels representative of implicit finite-element applications. It assembles a sparse linear-system from the steady-state conduction equation on a brick-shaped problem domain of linear 8-node hex elements. It then solves the linear-system using a simple un-preconditioned conjugate-gradient algorithm.
+
+MiniFE contains kernels responsible for :
+
+* computation of element-operators (diffusion matrix, source vector)
+* assembly (scattering element-operators into sparse matrix and vector)
+More details can be found at http://www.nersc.gov/users/computational-systems/cori/nersc-8-procurement/trinity-nersc-8-rfp/nersc-8-trinity-benchmarks/minife
+
+The code is written in C++ and parallelized with MPI.
+
+### Compilation
+
+Enter the `miniFE-1.4` directory and compile with:
+
+ make -f makefile.intel.mpi
+
+### Execution
+
+The problem sizes you want to run must be specified in the `../miniFE-sizes.txt` file. Then, you can have a look at the `runme.sh` (for interactive runs) or `runme_slurm.sh` (for runs with the SLURM scheduler), and run with either `runme.sh` or `runme_slurm.sh`.
+
+Post-processing
+---------------
+
+A post-processing script `post.sh` is available in each benchmark directory. A post-processing script `post.sh` in this directory takes care of calling all post-processing scripts in the benchmarks directories. This means, that after you have completed all the benchmarks, you can come back here and run
+
+ sh post.sh
+
+This will generate all the data needed to plot graphs and will copy all the results files to the `Results` directory. If you need to take home only the final results, you can copy over only this directory. You can find there a `onecluster.R` file, which takes care of reading all the data and outputting to the `user.Rdata` file.
+
+About this file
+---------------
+
+Convert to HTML with
+ pandoc Readme.md -f markdown -t html -s -o Readme.html
+grid.text("BENCHMARKS", y = unit(1, "npc"), x = unit(0.5, "npc"), vjust = 1, hjust = .5, gp = gpar(fontfamily = "Impact", col = col3, cex = 11.2, alpha = 0.3))
+grid.text("TEXT", y = unit(0.95, "npc"), gp = gpar(fontfamily = "Impact", col = col4, cex = 6.4))
+grid.text("SOME TEXT", vjust = 0, y = unit(0.936, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 0.8))
+grid.text("SOME OTHER TEXT", vjust = 0, y = unit(0.930, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 0.8))
+grid.text("maybe.an.url", vjust = 0, y = unit(0.924, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 0.8))
+grid.text("Benchmarks for the evaluation of some stuff", vjust = 0, hjust = 0, x = unit(0.01, "npc"), y = unit(0.905, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 1.2))
+# version intelMPI et icc.
+# orange rect for "APPLICATIONS"
+grid.rect(gp = gpar(fill = col2, col = col2), x = unit(0.5, "npc"), y = unit(0.854, "npc"), width = unit(1, "npc"), height = unit(0.085, "npc"))
+grid.text("APPLICATIONS", y = unit(0.854, "npc"), x = unit(0.5, "npc"), vjust = .5, hjust = .5, gp = gpar(fontfamily = "Impact", col = col3, cex = 10.9, alpha = 0.3))
+grid.text("MACHINE INFO", vjust = 0, hjust = 0, x = unit(0.01, "npc"), y = unit(0.885, "npc"), gp = gpar(fontfamily = "Impact", col = "white", cex = 1.2))
+grid.text(paste(
+ "Machine ID",
+ "Number of nodes",
+ "Sockets per node",
+ "CPU model",
+ "Cores per socket",
+ "Frequency",
+ "Temporal Date", sep = "\n"), vjust = 0, hjust = 0, x = unit(0.01, "npc"), y = unit(0.83, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 0.8))
+grid.text(paste(
+ args,
+ 1,
+ 2,
+ "Intel(R) Xeon(R) CPU E5-2660 v4@ 2.00GHz",
+ 14,
+ "2.0 GHz",
+ "April 7, 2016", sep = "\n"), vjust = 0, hjust = 0, x = unit(0.15, "npc"), y = unit(0.83, "npc"), gp = gpar(fontfamily = "Impact", col = mycol, cex = 0.8))