Page MenuHomec4science
Files F112287730

No OneTemporary
Actions

File Metadata

Created
Fri, May 9, 16:29

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
This document is not UTF8. It was detected as ISO-8859-1 (Latin 1) and converted to UTF8 for display.
diff --git a/Gadget2 b/Gadget2
new file mode 100755
index 0000000..8b26513
Binary files /dev/null and b/Gadget2 differ
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..b0e712f
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,832 @@
+
+#----------------------------------------------------------------------
+# From the list below, please activate/deactivate the options that
+# apply to your run. If you modify any of these options, make sure
+# that you recompile the whole code by typing "make clean; make".
+#
+# Look at end of file for a brief guide to the compile-time options.
+#----------------------------------------------------------------------
+
+
+#--------------------------------------- Basic operation mode of code
+#OPT += -DPERIODIC
+OPT += -DUNEQUALSOFTENINGS
+
+
+#--------------------------------------- Things that are always recommended
+OPT += -DPEANOHILBERT
+OPT += -DWALLCLOCK
+
+
+#--------------------------------------- TreePM Options
+#OPT += -DPMGRID=128
+#OPT += -DPLACEHIGHRESREGION=3
+#OPT += -DENLARGEREGION=1.2
+#OPT += -DASMTH=1.25
+#OPT += -DRCUT=4.5
+
+
+#--------------------------------------- Single/Double Precision
+#OPT += -DDOUBLEPRECISION
+#OPT += -DDOUBLEPRECISION_FFTW
+
+
+#--------------------------------------- Time integration options
+OPT += -DSYNCHRONIZATION
+#OPT += -DFLEXSTEPS
+#OPT += -DPSEUDOSYMMETRIC
+OPT += -DNOSTOP_WHEN_BELOW_MINTIMESTEP
+#OPT += -DNOPMSTEPADJUSTMENT
+
+
+#--------------------------------------- Output
+OPT += -DADVANCEDSTATISTICS
+OPT += -DADVANCEDCPUSTATISTICS
+OPT += -DSYSTEMSTATISTICS
+OPT += -DBLOCK_SKIPPING
+#OPT += -DHAVE_HDF5
+#OPT += -DOUTPUTPOTENTIAL
+#OPT += -DOUTPUTACCELERATION
+#OPT += -DOUTPUTCHANGEOFENTROPY
+#OPT += -DOUTPUTTIMESTEP
+#OPT += -DOUTPUTERADSTICKY
+#OPT += -DOUTPUTERADFEEDBACK
+#OPT += -DOUTPUTENERGYFLUX
+#OPT += -DOUTPUTOPTVAR
+OPT += -DOUTPUTSTELLAR_PROP
+
+#--------------------------------------- Things for special behaviour
+#OPT += -DNOGRAVITY
+#OPT += -DNOTREERND
+#OPT += -DNOTYPEPREFIX_FFTW
+#OPT += -DLONG_X=60
+#OPT += -DLONG_Y=5
+#OPT += -DLONG_Z=0.2
+#OPT += -DTWODIMS
+#OPT += -DSPH_BND_PARTICLES
+#OPT += -DNOVISCOSITYLIMITER
+OPT += -DCOMPUTE_POTENTIAL_ENERGY
+#OPT += -DLONGIDS
+#OPT += -DISOTHERM_EQS
+#OPT += -DADAPTIVE_GRAVSOFT_FORGAS
+#OPT += -DSELECTIVE_NO_GRAVITY=2+4+8+16
+#OPT += -DAVOIDNUMNGBPROBLEM
+#OPT += -DLIMIT_DVEL=1.0
+#OPT += -DOTHERINFO
+#OPT += -DDOMAIN_AT_ORIGIN
+OPT += -DNO_NEGATIVE_PRESSURE
+#OPT += -DCOMPUTE_VELOCITY_DISPERSION
+#OPT += -DCYLINDRICAL_SYMMETRY
+
+OPT += -DWRITE_ALL_MASSES
+OPT += -DENTROPYPRED
+OPT += -DCOUNT_ACTIVE_PARTICLES
+OPT += -DRANDOMSEED_AS_PARAMETER
+OPT += -DDETAILED_CPU
+OPT += -DDETAILED_CPU_GRAVITY
+OPT += -DDETAILED_CPU_DOMAIN
+OPT += -DDETAILED_OUTPUT_IN_GRAVTREE
+#OPT += -DSPLIT_DOMAIN_USING_TIME
+
+#--------------------------------------- Physical processes
+OPT += -DCOOLING
+#OPT += -DIMPLICIT_COOLING_INTEGRATION
+#OPT += -DDO_NO_USE_HYDROGEN_MASSFRAC_IN_COOLING
+
+#OPT += -DHEATING
+#OPT += -DHEATING_PE # photo-electric heating
+
+OPT += -DSFR
+OPT += -DCOMPUTE_SFR_ENERGY
+OPT += -DSFR_NEG_DIV_ONLY
+
+OPT += -DSTELLAR_PROP
+
+OPT += -DCHIMIE # need stellar prop
+OPT += -DCHIMIE_THERMAL_FEEDBACK
+OPT += -DCHIMIE_COMPUTE_THERMAL_FEEDBACK_ENERGY
+#OPT += -DCHIMIE_KINETIC_FEEDBACK
+#OPT += -DCHIMIE_COMPUTE_KINETIC_FEEDBACK_ENERGY
+OPT += -DCHIMIE_EXTRAHEADER
+
+#OPT += -DFEEDBACK
+#OPT += -DFEEDBACK_WIND
+
+#--------------------------------------- multiphase
+#OPT += -DMULTIPHASE
+#OPT += -DNO_HYDRO_FOR_GAS # do not use hydro routine (at all)
+#OPT += -DNO_DENSITY_FOR_STICKY # do not compute density in sticky (need to be done in sfr)
+
+#OPT += -DPHASE_MIXING # need MULTIPHASE : enable phase mixing
+#OPT += -DCOLDGAS_CYCLE # need MULTIPHASE and PHASE_MIXING
+#OPT += -DEXTERNAL_FLUX
+#OPT += -DSTELLAR_FLUX
+#OPT += -DCOUNT_COLLISIONS # count sticky collisions
+
+
+#--------------------------------------- Outer potential
+#OPT += -DOUTERPOTENTIAL
+#OPT += -DNFW
+#OPT += -DPISOTHERM
+#OPT += -DPLUMMER
+#OPT += -DCORIOLIS
+
+#--------------------------------------- Testing and Debugging options
+#OPT += -DFORCETEST=0.1
+#OPT += -DWITH_ID_IN_HYDRA
+#OPT += -DPARTICLE_FLAG
+#OPT += -DOUTPUT_EVERY_TIMESTEP
+#OPT += -DOUTPUT_COOLING_FUNCTION
+
+OPT += -DCHECK_BLOCK_ORDER
+OPT += -DCHECK_ENTROPY_SIGN
+OPT += -DCHECK_TYPE_DURING_IO
+OPT += -DCHECK_ID_CORRESPONDENCE
+
+
+#--------------------------------------- Glass making
+#OPT += -DMAKEGLASS=262144
+
+#--------------------------------------- Agn
+#OPT += -DBUBBLES
+#OPT += -DAGN_ACCRETION
+#OPT += -DAGN_FEEDBACK
+#OPT += -DAGN_USE_ANGULAR_MOMENTUM
+#OPT += -DAGN_HEATING
+#OPT += -DBONDI_ACCRETION
+#OPT += -DUSE_BONDI_POWER
+
+#----------------------------------------------------------------------
+# Here, select compile environment for the target machine. This may need
+# adjustment, depending on your local system. Follow the examples to add
+# additional target platforms, and to get things properly compiled.
+#----------------------------------------------------------------------
+
+#--------------------------------------- Select some defaults
+
+CC = mpicc # sets the C-compiler
+OPTIMIZE = -O2 -Wall -g # sets optimization and warning flags
+MPICHLIB = -lmpich
+
+
+#--------------------------------------- Select target computer
+
+#SYSTYPE="callisto-intel"
+#SYSTYPE="bg1"
+#SYSTYPE="obsds"
+SYSTYPE="leo_openmpi"
+#SYSTYPE="leo_mpich2shm"
+#SYSTYPE="graphor0"
+#SYSTYPE="obsrevaz"
+#SYSTYPE="regor_openmpigcc"
+#SYSTYPE="regor_mvapich2gcc"
+#SYSTYPE="meso_mpich2"
+#SYSTYPE="meso"
+#SYSTYPE="revaz/local"
+#SYSTYPE="revaz/local_mpich2"
+#SYSTYPE="horizon3_mpich1"
+#SYSTYPE="horizon3_mpich2"
+#SYSTYPE="horizon3"
+#SYSTYPE="LUXOR"
+#SYSTYPE="MPA"
+#SYSTYPE="Mako"
+#SYSTYPE="Regatta"
+#SYSTYPE="RZG_LinuxCluster"
+#SYSTYPE="RZG_LinuxCluster-gcc"
+#SYSTYPE="OpteronMPA"
+#SYSTYPE="OPA-Cluster32"
+#SYSTYPE="OPA-Cluster64"
+
+
+#--------------------------------------- Adjust settings for target computer
+
+ifeq ($(SYSTYPE),"callisto-intel")
+CC = mpicc
+OPTIMIZE =
+GSL_INCL = -I/u1/yrevaz/local/gsl-intel/include
+GSL_LIBS = -L/u1/yrevaz/local/gsl-intel/lib
+FFTW_INCL= -I/u1/yrevaz/local/fftw-2.1.5-intel/include
+FFTW_LIBS= -L/u1/yrevaz/local/fftw-2.1.5-intel/lib
+MPICHLIB =
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"bg1")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+GSL_INCL = -I/home/yrevaz/local/include
+GSL_LIBS = -L/home/yrevaz/local/lib
+FFTW_INCL=
+FFTW_LIBS=
+MPICHLIB =
+HDF5INCL =
+HDF5LIB =
+NO_FFTW_LIB = "yes"
+endif
+
+ifeq ($(SYSTYPE),"obsds")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+GSL_INCL =
+GSL_LIBS =
+FFTW_INCL=
+FFTW_LIBS=
+MPICHLIB =
+HDF5INCL =
+HDF5LIB =
+NO_FFTW_LIB = "yes"
+endif
+
+ifeq ($(SYSTYPE),"graphor0")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+GSL_INCL = -I/home/epfl/revaz/local/include
+GSL_LIBS = -L/home/epfl/revaz/local/lib
+FFTW_INCL= -I/home/epfl/revaz/local/include
+FFTW_LIBS= -L/home/epfl/revaz/local/lib
+MPICHLIB = -L/home/epfl/revaz/local/openmpi/lib -lmpi
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"obsrevaz")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -fpack-struct
+GSL_INCL =
+GSL_LIBS =
+FFTW_INCL= -I/home/revaz/local/include/
+FFTW_LIBS= -L/home/revaz/local/lib/
+MPICHLIB = -lmpi
+HDF5INCL =
+HDF5LIB =
+endif
+
+
+ifeq ($(SYSTYPE),"regor_openmpigcc")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -fpack-struct
+GSL_INCL = -I/usr/include
+GSL_LIBS = -L/usr/lib64/
+FFTW_INCL= -I/home/revaz/local_mvapich2gcc/include/
+FFTW_LIBS= -L/home/revaz/local_mvapich2gcc/lib/
+MPICHLIB = -lmpi
+HDF5INCL =
+HDF5LIB =
+OPT += -DMESOMACHINE
+endif
+
+ifeq ($(SYSTYPE),"regor_mpich2")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -fpack-struct
+GSL_INCL = -I/usr/include
+GSL_LIBS = -L/usr/lib64/
+FFTW_INCL= -I/home/revaz/local_mvapich2gcc/include/
+FFTW_LIBS= -L/home/revaz/local_mvapich2gcc/lib/
+MPICHLIB = -L/home/revaz/local/mpich2-1.0.6nemesis/lib/ -lmpich
+HDF5INCL =
+HDF5LIB =
+OPT += -DMESOMACHINE
+endif
+
+
+ifeq ($(SYSTYPE),"regor_mvapich2gcc")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -fpack-struct
+GSL_INCL = -I/usr/include
+GSL_LIBS = -L/usr/lib64/
+FFTW_INCL= -I/home/revaz/local_mvapich2gcc/include/
+FFTW_LIBS= -L/home/revaz/local_mvapich2gcc/lib/
+MPICHLIB = -L/cvos/shared/apps/ofed/1.2.5.3/mpi/gcc/mvapich2-0.9.8-15/lib/ -lmpich
+HDF5INCL =
+HDF5LIB =
+OPT += -DMESOMACHINE
+endif
+
+ifeq ($(SYSTYPE),"leo_openmpi")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -fpack-struct
+GSL_INCL = -I/export/revaz/local/include
+GSL_LIBS = -L/export/revaz/local/lib
+FFTW_INCL= -I/export/revaz/local/include
+FFTW_LIBS= -L/export/revaz/local/lib
+MPICHLIB = -L/usr/local/mpich2-pgi/lib -lmpi
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"leo_mpich2shm")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g -fpack-struct
+GSL_INCL = -I/export/revaz/local/include
+GSL_LIBS = -L/export/revaz/local/lib
+FFTW_INCL= -I/export/revaz/local/include
+FFTW_LIBS= -L/export/revaz/local/lib
+MPICHLIB = -L/usr/local/mpich2-pgi/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"meso_mpich2")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g -fpack-struct
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/horizon1/x86_64_sl4/fftw/2.1.5/include/
+FFTW_LIBS= -L/horizon1/x86_64_sl4/fftw/2.1.5/lib/
+MPICHLIB = -L/home/revaz/local/mpich2-1.0.3/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"meso")
+CC = mpicc
+OPTIMIZE = -O3 -g
+GSL_INCL =
+GSL_LIBS =
+FFTW_INCL= -I/horizon1/x86_64_sl4/fftw/2.1.5/include/
+FFTW_LIBS= -L/horizon1/x86_64_sl4/fftw/2.1.5/lib/
+MPICHLIB =
+HDF5INCL =
+HDF5LIB =
+OPT += -DMESOMACHINE
+endif
+
+ifeq ($(SYSTYPE),"revaz/local")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/home/revaz/local/include
+FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -L/home/revaz/local/mpich-1.2.5/ch_p4/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"revaz/local_mpich2")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/home/revaz/local/include
+FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -L/home/revaz/local/mpich2-1.0.3/lib/ -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"LUXOR")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g
+#GSL_INCL = -I/home/revaz/local/include
+#GSL_LIBS = -L/home/revaz/local/lib
+#FFTW_INCL= -I/home/revaz/local/include
+#FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -L/home/revaz/local/mpich-1.2.7/ch_p4/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+
+
+ifeq ($(SYSTYPE),"horizon3")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g -fpack-struct
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/home/revaz/local/include
+FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -llam
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"horizon3_mpich1")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g -fpack-struct
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/home/revaz/local/include
+FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -L/home/revaz/local/mpich-1.2.7/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+
+ifeq ($(SYSTYPE),"horizon3_mpich2")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -g -fpack-struct
+GSL_INCL = -I/home/revaz/local/include
+GSL_LIBS = -L/home/revaz/local/lib
+FFTW_INCL= -I/home/revaz/local/include
+FFTW_LIBS= -L/home/revaz/local/lib
+MPICHLIB = -L/usr/local/mpich2-pgi/lib -lmpich
+HDF5INCL =
+HDF5LIB =
+endif
+
+ifeq ($(SYSTYPE),"MPA")
+CC = mpicc
+OPTIMIZE = -O3 -Wall
+GSL_INCL = -I/usr/common/pdsoft/include
+GSL_LIBS = -L/usr/common/pdsoft/lib -Wl,"-R /usr/common/pdsoft/lib"
+FFTW_INCL=
+FFTW_LIBS=
+MPICHLIB =
+HDF5INCL =
+HDF5LIB = -lhdf5 -lz
+endif
+
+
+ifeq ($(SYSTYPE),"OpteronMPA")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -m64
+GSL_INCL = -L/usr/local/include
+GSL_LIBS = -L/usr/local/lib
+FFTW_INCL=
+FFTW_LIBS=
+MPICHLIB =
+HDF5INCL = -I/opt/hdf5/include
+HDF5LIB = -L/opt/hdf5/lib -lhdf5 -lz -Wl,"-R /opt/hdf5/lib"
+endif
+
+
+ifeq ($(SYSTYPE),"OPA-Cluster32")
+CC = mpicc
+OPTIMIZE = -O3 -Wall
+GSL_INCL = -I/afs/rzg/bc-b/vrs/opteron32/include
+GSL_LIBS = -L/afs/rzg/bc-b/vrs/opteron32/lib -Wl,"-R /afs/rzg/bc-b/vrs/opteron32/lib"
+FFTW_INCL= -I/afs/rzg/bc-b/vrs/opteron32/include
+FFTW_LIBS= -L/afs/rzg/bc-b/vrs/opteron32/lib
+MPICHLIB =
+HDF5INCL =
+HDF5LIB = -lhdf5 -lz
+endif
+
+
+ifeq ($(SYSTYPE),"OPA-Cluster64")
+CC = mpicc
+OPTIMIZE = -O3 -Wall -m64
+GSL_INCL = -I/afs/rzg/bc-b/vrs/opteron64/include
+GSL_LIBS = -L/afs/rzg/bc-b/vrs/opteron64/lib -Wl,"-R /afs/rzg/bc-b/vrs/opteron64/lib"
+FFTW_INCL= -I/afs/rzg/bc-b/vrs/opteron64/include
+FFTW_LIBS= -L/afs/rzg/bc-b/vrs/opteron64/lib
+MPICHLIB =
+HDF5INCL =
+HDF5LIB = -lhdf5 -lz
+endif
+
+
+ifeq ($(SYSTYPE),"Mako")
+CC = mpicc # sets the C-compiler
+OPTIMIZE = -O3 -march=athlon-mp -mfpmath=sse
+GSL_INCL =
+GSL_LIBS =
+FFTW_INCL=
+FFTW_LIBS=
+MPICHLIB =
+HDF5INCL =
+HDF5LIB = -lhdf5 -lz
+endif
+
+
+ifeq ($(SYSTYPE),"Regatta")
+CC = mpcc_r
+OPTIMIZE = -O5 -qstrict -qipa -q64
+GSL_INCL = -I/afs/rzg/u/vrs/gsl_psi64/include
+GSL_LIBS = -L/afs/rzg/u/vrs/gsl_psi64/lib
+FFTW_INCL= -I/afs/rzg/u/vrs/fftw_psi64/include
+FFTW_LIBS= -L/afs/rzg/u/vrs/fftw_psi64/lib -q64 -qipa
+MPICHLIB =
+HDF5INCL = -I/afs/rzg/u/vrs/hdf5_psi64/include
+HDF5LIB = -L/afs/rzg/u/vrs/hdf5_psi64/lib -lhdf5 -lz
+endif
+
+
+ifeq ($(SYSTYPE),"RZG_LinuxCluster")
+CC = mpicci
+OPTIMIZE = -O3 -ip # Note: Don't use the "-rcd" optimization of Intel's compiler! (causes code crashes)
+GSL_INCL = -I/afs/rzg/u/vrs/gsl_linux/include
+GSL_LIBS = -L/afs/rzg/u/vrs/gsl_linux/lib -Wl,"-R /afs/rzg/u/vrs/gsl_linux/lib"
+FFTW_INCL= -I/afs/rzg/u/vrs/fftw_linux/include
+FFTW_LIBS= -L/afs/rzg/u/vrs/fftw_linux/lib
+HDF5INCL = -I/afs/rzg/u/vrs/hdf5_linux/include
+HDF5LIB = -L/afs/rzg/u/vrs/hdf5_linux/lib -lhdf5 -lz -Wl,"-R /afs/rzg/u/vrs/hdf5_linux/lib"
+endif
+
+
+ifeq ($(SYSTYPE),"RZG_LinuxCluster-gcc")
+CC = mpiccg
+OPTIMIZE = -Wall -g -O3 -march=pentium4
+GSL_INCL = -I/afs/rzg/u/vrs/gsl_linux_gcc3.2/include
+GSL_LIBS = -L/afs/rzg/u/vrs/gsl_linux_gcc3.2/lib -Wl,"-R /afs/rzg/u/vrs/gsl_linux_gcc3.2/lib"
+FFTW_INCL= -I/afs/rzg/u/vrs/fftw_linux_gcc3.2/include
+FFTW_LIBS= -L/afs/rzg/u/vrs/fftw_linux_gcc3.2/lib
+HDF5INCL = -I/afs/rzg/u/vrs/hdf5_linux/include
+HDF5LIB = -L/afs/rzg/u/vrs/hdf5_linux/lib -lhdf5 -lz -Wl,"-R /afs/rzg/u/vrs/hdf5_linux/lib"
+endif
+
+
+ifneq (HAVE_HDF5,$(findstring HAVE_HDF5,$(OPT)))
+HDF5INCL =
+HDF5LIB =
+endif
+
+
+OPTIONS = $(OPTIMIZE) $(OPT)
+
+EXEC = Gadget2
+
+OBJS = main.o run.o predict.o begrun.o endrun.o global.o \
+ timestep.o init.o restart.o io.o \
+ accel.o read_ic.o ngb.o \
+ system.o allocate.o density.o \
+ gravtree.o hydra.o driftfac.o \
+ domain.o allvars.o potential.o \
+ forcetree.o peano.o gravtree_forcetest.o \
+ pm_periodic.o pm_nonperiodic.o longrange.o \
+ cooling.o agn_heating.o phase.o sticky.o outerpotential.o starformation.o \
+ agn_feedback.o bubbles.o bondi_accretion.o chimie.o stars_density.o
+
+INCL = allvars.h proto.h tags.h Makefile
+
+
+CFLAGS = $(OPTIONS) $(GSL_INCL) $(FFTW_INCL) $(HDF5INCL)
+
+
+ifeq (NOTYPEPREFIX_FFTW,$(findstring NOTYPEPREFIX_FFTW,$(OPT))) # fftw installed with type prefix?
+ FFTW_LIB = $(FFTW_LIBS) -lrfftw_mpi -lfftw_mpi -lrfftw -lfftw
+else
+ifeq (DOUBLEPRECISION_FFTW,$(findstring DOUBLEPRECISION_FFTW,$(OPT)))
+ FFTW_LIB = $(FFTW_LIBS) -ldrfftw_mpi -ldfftw_mpi -ldrfftw -ldfftw
+else
+ FFTW_LIB = $(FFTW_LIBS) -lsrfftw_mpi -lsfftw_mpi -lsrfftw -lsfftw
+endif
+endif
+
+ifeq ($(NO_FFTW_LIB),"yes")
+ FFTW_LIB =
+endif
+
+LIBS = $(HDF5LIB) -g $(MPICHLIB) $(GSL_LIBS) -lgsl -lgslcblas -lm $(FFTW_LIB)
+
+$(EXEC): $(OBJS)
+ $(CC) $(OBJS) $(LIBS) -o $(EXEC)
+
+$(OBJS): $(INCL)
+
+
+clean:
+ rm -f $(OBJS) $(EXEC)
+
+
+#-----------------------------------------------------------------------
+#
+# Brief guide to compile-time options of the code. More information
+# can be found in the code documentation.
+#
+# - PERIODIC:
+# Set this if you want to have periodic boundary conditions.
+#
+# - UNEQUALSOFTENINGS:
+# Set this if you use particles with different gravitational
+# softening lengths.
+#
+# - PEANOHILBERT:
+# This is a tuning option. When set, the code will bring the
+# particles after each domain decomposition into Peano-Hilbert
+# order. This improves cache utilization and performance.
+#
+# - WALLCLOCK:
+# If set, a wallclock timer is used by the code to measure internal
+# time consumption (see cpu-log file). Otherwise, a timer that
+# measures consumed processor ticks is used.
+#
+# - PMGRID:
+# This enables the TreePM method, i.e. the long-range force is
+# computed with a PM-algorithm, and the short range force with the
+# tree. The parameter has to be set to the size of the mesh that
+# should be used, (e.g. 64, 96, 128, etc). The mesh dimensions need
+# not necessarily be a power of two. Note: If the simulation is
+# not in a periodic box, then a FFT method for vacuum boundaries is
+# employed, using an actual mesh with dimension twice(!) that
+# specified by PMGRID.
+#
+# - PLACEHIGHRESREGION:
+# If this option is set (will only work together with PMGRID), then
+# the long range force is computed in two stages: One Fourier-grid
+# is used to cover the whole simulation volume, allowing the
+# computation of the longe-range force. A second Fourier mesh is
+# placed on the region occupied by "high-resolution" particles,
+# allowing the computation of an intermediate scale force. Finally,
+# the force on short scales is computed with the tree. This
+# procedure can be useful for "zoom-simulations", provided the
+# majority of particles (the high-res particles) are occupying only
+# a small fraction of the volume. To activate this option, the
+# parameter needs to be set to an integer bit mask that encodes the
+# particle types that make up the high-res particles.
+# For example, if types 0, 1, and 4 form the high-res
+# particles, set the parameter to PLACEHIGHRESREGION=19, because
+# 2^0 + 2^1 + 2^4 = 19. The spatial region covered by the high-res
+# grid is determined automatically from the initial conditions.
+# Note: If a periodic box is used, the high-res zone may not intersect
+# the box boundaries.
+#
+# - ENLARGEREGION:
+# The spatial region covered by the high-res zone has a fixed size
+# during the simulation, which initially is set to the smallest
+# region that encompasses all high-res particles. Normally, the
+# simulation will be interrupted if high-res particles leave this
+# region in the course of the run. However, by setting this
+# parameter to a value larger than one, the size of the high-res
+# region can be expanded, providing a buffer region. For example,
+# setting it to 1.4 will enlarge its side-length by 40% (it remains
+# centered on the high-res particles). Hence, with this setting, the
+# high-res region may expand or move by a limited amount.
+# Note: If SYNCHRONIZATION is activated, the code will be able to
+# continue even if high-res particles leave the initial high-res
+# grid. In this case, the code will update the size and position of
+# the grid that is placed onto the high-resolution region
+# automatically. To prevent that this potentially happens every
+# single PM step, one should nevertheless assign a value slightly
+# larger than 1 to ENLARGEREGION.
+#
+# - ASMTH:
+# This can be used to override the value assumed for the scale that
+# defines the long-range/short-range force-split in the TreePM
+# algorithm. The default value is 1.25, in mesh-cells.
+#
+# - RCUT:
+# This can be used to override the maximum radius in which the
+# short-range tree-force is evaluated (in case the TreePM algorithm
+# is used). The default value is 4.5, given in mesh-cells.
+#
+# - DOUBLEPRECISION:
+# This makes the code store and compute internal particle data in
+# double precision. Note that output files are nevertheless written
+# by converting the particle data to single precision.
+#
+# - DDOUBLEPRECISION_FFTW:
+# If this is set, the code will use the double-precision version of
+# FTTW, provided the latter has been explicitly installed with a
+# "d" prefix, and NOTYPEPREFIX_FFTW is not set. Otherwise the
+# single precision version ("s" prefix) is used.
+#
+# - SYNCHRONIZATION:
+# When this is set, particles are kept in a binary hierarchy of
+# timesteps and may only increase their timestep if the new
+# timestep will put them into synchronization with the higher time
+# level.
+#
+# - FLEXSTEPS:
+# This is an alternative to SYNCHRONIZATION. Particle timesteps are
+# here allowed to be integer multiples of the minimum timestep that
+# occurs among the particles, which in turn is rounded down to the
+# nearest power-of-two devision of the total simulated
+# timespan. This option distributes particles more evenly over
+# individual system timesteps, particularly once a simulation has
+# run for a while, and may then result in a reduction of work-load
+# imbalance losses.
+#
+# - PSEUDOSYMMETRIC:
+# When this option is set, the code will try to "anticipate"
+# timestep changes by extrapolating the change of the acceleration
+# into the future. This can in certain idealized cases improve the
+# long-term integration behaviour of periodic orbits, but should
+# make little or no difference in most real-world applications. May
+# only be used together with SYNCHRONIZATION.
+#
+# - NOSTOP_WHEN_BELOW_MINTIMESTEP:
+# If this is activated, the code will not terminate when the
+# timestep falls below the value of MinSizeTimestep specified in
+# the parameterfile. This is useful for runs where one wants to
+# enforce a constant timestep for all particles. This can be done
+# by activating this option, and by setting MinSizeTimestep and
+# MaxSizeTimestep to an equal value.
+#
+# - NOPMSTEPADJUSTMENT:
+# When this is set, the long-range timestep for the PM-force
+# computation (when the TreePM algorithm is used) is always
+# determined by MaxSizeTimeStep. Otherwise, it is determined by
+# the MaxRMSDisplacement parameter, or MaxSizeTimeStep, whichever
+# gives the smaller step.
+#
+# - HAVE_HDF5:
+# If this is set, the code will be compiled with support for input
+# and output in the HDF5 format. You need to have the HDF5
+# libraries and headers installed on your computer for this option
+# to work. The HDF5 format can then be selected as format "3" in
+# Gadget's parameterfile.
+#
+# - OUTPUTPOTENTIAL:
+# This will make the code compute gravitational potentials for
+# all particles each time a snapshot file is generated. The values
+# are then included in the snapshot file. Note that the computation
+# of the values of the gravitational potential costs additional CPU.
+#
+# - OUTPUTACCELERATION:
+# This will include the physical acceleration of each particle in
+# snapshot files.
+#
+# - OUTPUTCHANGEOFENTROPY:
+# This will include the rate of change of entropy of gas particles
+# in snapshot files.
+#
+# - OUTPUTTIMESTEP:
+# This will include the current timesteps of all particles in the
+# snapshot files.
+#
+# - NOGRAVITY
+# This switches off gravity. Useful only for pure SPH simulations
+# in non-expanding space.
+#
+# - NOTREERND:
+# If this is not set, the tree construction will succeed even when
+# there are a few particles at identical locations. This is done by
+# `rerouting' particles once the node-size has fallen below 1.0e-3
+# of the softening length. When this option is activated, this will
+# be surpressed and the tree construction will always fail if there
+# are particles at extremely close coordinates.
+#
+# - NOTYPEPREFIX_FFTW:
+# This is an option that signals that FFTW has been compiled
+# without the type-prefix option, i.e. no leading "d" or "s"
+# characters are used to access the library.
+#
+# - LONG_X/Y/Z:
+# These options can be used together with PERIODIC and NOGRAVITY only.
+# When set, the options define numerical factors that can be used to
+# distorts the periodic simulation cube into a parallelepiped of
+# arbitrary aspect ratio. This can be useful for idealized SPH tests.
+#
+# - TWODIMS:
+# This effectively switches of one dimension in SPH, i.e. the code
+# follows only 2d hydrodynamics in the xy-, yz-, or xz-plane. This
+# only works with NOGRAVITY, and if all coordinates of the third
+# axis are exactly equal. Can be useful for idealized SPH tests.
+#
+# - SPH_BND_PARTICLES:
+# If this is set, particles with a particle-ID equal to zero do not
+# receive any SPH acceleration. This can be useful for idealized
+# SPH tests, where these particles represent fixed "walls".
+#
+# - NOVISCOSITYLIMITER:
+# If this is set, the code will not try to put an upper limit on
+# the viscous force in case an implausibly high pair-wise viscous
+# force (which may lead to a particle 'reflection' in case of poor
+# timestepping) should arise. Note: For proper settings of the
+# timestep parameters, this situation should not arise.
+#
+# - COMPUTE_POTENTIAL_ENERGY:
+# When this option is set, the code will compute the gravitational
+# potential energy each time a global statistics is computed. This
+# can be useful for testing global energy conservation.
+#
+# - LONGIDS:
+# If this is set, the code assumes that particle-IDs are stored as
+# 64-bit long integers. This is only really needed if you want to
+# go beyond ~2 billion particles.
+#
+# - ISOTHERM_EQS:
+# This special option makes the gas behave like an isothermal gas
+# with equation of state P = cs^2 * rho. The sound-speed cs is set by
+# the thermal energy per unit mass in the intial conditions,
+# i.e. cs^2=u. If the value for u is zero, then the initial gas
+# temperature in the parameter file is used to define the sound speed
+# according to cs^2 = 3/2 kT/mp, where mp is the proton mass.
+#
+# - ADAPTIVE_GRAVSOFT_FORGAS:
+# When this option is set, the gravitational softening lengths used for
+# gas particles is tied to their SPH smoothing length. This can be useful
+# for dissipative collapse simulations. The option requires the setting
+# of UNEQUALSOFTENINGS.
+#
+# - SELECTIVE_NO_GRAVITY:
+# This can be used for special computations where one wants to
+# exclude certain particle types from receiving gravitational
+# forces. The particle types that are excluded in this fashion are
+# specified by a bit mask, in the same as for the PLACEHIGHRESREGION
+# option.
+#
+# - FORCETEST:
+# This can be set to check the force accuracy of the code. The
+# option needs to be set to a number between 0 and 1 (e.g. 0.01),
+# which is taken to specify a random fraction of particles for
+# which at each timestep forces by direct summation are
+# computed. The normal tree-forces and the correct direct
+# summation forces are collected in a file. Note that the
+# simulation itself is unaffected by this option, but it will of
+# course run much(!) slower, especially if
+# FORCETEST*NumPart*NumPart >> NumPart. Note: Particle IDs must
+# be set to numbers >=1 for this to work.
+#
+# - MAKEGLASS
+# This option can be used to generate a glass-like particle
+# configuration. The value assigned gives the particle load,
+# which is initially generated as a Poisson sample and then
+# evolved towards a glass with the sign of gravity reversed.
+#
+#-----------------------------------------------------------------------
+
diff --git a/Readme b/Readme
new file mode 100644
index 0000000..d80d5a2
--- /dev/null
+++ b/Readme
@@ -0,0 +1,561 @@
+ven oct 20 09:42:19 CEST 2006
+
+ correction dans cooling.c, multiplication de ne par h.
+
+Sun Oct 22 16:06:46 CEST 2006
+
+ *
+ dans timestep.c, on ajoute la variable :
+ adotinv = 1./(hubble_a*All.Time);
+
+ dtEntropyRad -> dtEntropyRad*adotinv
+
+
+ *
+ dans starformation.c,, on ajoute la variable :
+ adotinv = 1./(hubble_a*All.Time);
+
+ *
+ dans cooling.c
+ on corrige de h
+ l = l * All.HubbleParam;
+
+
+Wed Nov 1 17:26:54 CET 2006
+ * Makefile -> option OUTPUT_EVERY_TIMESTEP
+ * run.c
+ #ifdef OUTPUT_EVERY_TIMESTEP
+ savepositions(All.SnapshotFileCount++); /* write snapshot file */
+ #endif
+ * !!! correction importante !!!
+ if((SphP[i].DtEntropy + dtEntropyRad) * dt_entr > 0.5 * SphP[i].Entropy)
+ dtEntropyRad = +0.5*SphP[i].Entropy/dt_entr - SphP[i].DtEntropy;
+
+ * !!! correction mega importante !!!
+
+ /* finally compute the entropy */
+ SphP[i].Entropy += (SphP[i].DtEntropy+dtEntropyRad) * dt_entr;
+ /* merge DtEntropy with dtEntropyRad (important for predicted temperature) */
+ SphP[i].DtEntropy = SphP[i].DtEntropy+dtEntropyRad;
+
+Wed Jan 24 15:23:24 CET 2007
+
+ * forcetree.c updated with version 2.0.3
+ * timestep.c updated with version 2.0.3
+ * longrange.c updated with version 2.0.3
+ * init.c updated with version 2.0.3
+ * run.c updated with version 2.0.3
+ * potential.c updated with version 2.0.3
+ * pm_nonperiodic.c updated with version 2.0.3
+ * read_ic.c updated with version 2.0.3
+ * proto.h updated with version 2.0.3
+ * allvars.h updated with version 2.0.3
+
+ * predict.c
+ dt_entr = (All.Ti_Current - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+ -->
+ dt_entr = (time1 - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+
+Thu Jan 25 17:51:35 CET 2007
+
+ * cooling.c + some related modif in timestep.c
+ * agn_heating.c + some related modif in timestep.c
+
+Wed Jan 31 12:39:55 CET 2007
+ * hydra.c
+
+ hfc_feedback = sqrt(0.5*(ekinspec/dt)*wk *0.5*(mass + P[j].Mass) /rho_ij/dt)/r;
+ hfc_feedback = dmin(hfc_feedback, 0.25*(sqrt(dvx*dvx+dvy*dvy+dvz*dvz)/dt));
+
+
+Thu Feb 15 14:11:47 CET 2007
+ * hydra.c
+
+ correction of feedback in comoving integration
+
+ dt = imax(timestep, (P[j].Ti_endstep - P[j].Ti_begstep)) * All.Timebase_interval/ hubble_a;
+ hfc_feedback = sqrt(0.5*(ekinspec/dt)*wk *0.5*(mass + P[j].Mass) /rho_ij/dt)/r * atime*fac_egy;
+ hfc_feedback = dmin(hfc_feedback, 0.25*fac_egy*(sqrt(dvx*dvx+dvy*dvy+dvz*dvz)/dt/r));
+ uintspec = 2*(uintspec/dt) *wk *0.5*(mass + P[j].Mass) /rho_ij *fac_pow ;
+
+Tue Mar 27 11:15:35 CEST 2007
+ * cooling.c
+
+ add the resolution of ionisation equilibrium equations
+ -> cooling + heating
+
+ * new parameter :
+ CoolingType : 0=Sutherland 1=+heating
+
+ * begrun.c
+ All.SupernovaTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s;
+
+ SupernovaTime is now given in Myr
+
+Tue Apr 3 09:06:10 CEST 2007
+
+ * accel.c
+ Cooling after heating
+ heating();
+ agn_heating();
+ cooling();
+
+ * cooling.c
+ Add isochoric cooling :
+ double DoCooling(...)
+
+ * starformation.c
+
+ tstar = 1./ All.StarFormationParameter / pow(SphP[i].Density,0.5);
+ >
+ tstar = 1./ All.StarFormationParameter / pow(SphP[i].Density*a3inv,0.5);
+
+
+ WARNING : THE COOLING_RAD IS PROBABLY NOT TAKEN INTO ACCOUNT
+ THE VOLKER IMPLEMENTATION IS USED IN timestep.c
+
+Thu Apr 5 09:06:10 CEST 2007
+
+ add the possibility to choose the starformation type
+ Rasera - Springel
+
+ * starformation.c
+
+ switch (All.StarFormationType)
+ {
+ case 0:
+ ThresholdDensity = All.StarFormationDensity;
+ break;
+
+ case 1:
+ MeanDensity = All.Omega0 * (3 * All.Hubble * All.Hubble / (8 * M_PI * All.G));
+ ThresholdDensity = dmin(All.StarFormationDensity/a3inv,200*MeanDensity);
+ break;
+ }
+
+
+ if (SphP[i].Density > ThresholdDensity)
+
+ tstar = All.StarFormationTime / pow(SphP[i].Density*a3inv/All.StarFormationDensity,0.5);
+
+ if (T < All.StarFormationTemperature)
+
+
+ * begrun.c
+
+ All.StarFormationType = all.StarFormationType;
+ All.StarFormationDensity = all.StarFormationDensity;
+ All.StarFormationTemperature = all.StarFormationTemperature;
+
+ All.StarFormationTime = All.StarFormationTime/All.UnitTime_in_s * 3.1536e16;
+ All.StarFormationDensity = All.StarFormationDensity/All.UnitDensity_in_cgs;
+
+Wed Apr 25 08:11:42 CEST 2007
+
+ * hydra.c
+
+ fac_pow = 1.0;
+
+ * starformation.c
+
+ - some corrections of the starformation threashold
+ - merge the code (one phase/multiphase)
+
+Wed Apr 25 08:45:17 CEST 2007
+
+ change
+ #ifdef STICKY
+ in
+ #ifdef MULTIPHASE
+
+
+Fri May 25 10:52:08 CEST 2007
+
+ * a huge number of changes
+
+ * modification of the feedback, feedback_wind
+ * modification of the sticky
+ * add the cold cycle
+ * other small
+
+
+!!! faire un flag pour
+meanweight = 4.0 / (1 + 3 * HYDROGEN_MASSFRAC); /* note: we assume neutral gas here */
+/*meanweight = 4 / (8 - 5 * (1 - HYDROGEN_MASSFRAC));*/ /* note: we assume FULL ionized gas here */
+
+Wed Jul 4 13:28:45 CEST 2007
+
+ * outer plummer potential (PLUMMER)
+ * possibility to compute velocity dispersion (COMPUTE_VELOCITY_DISPERSION)
+
+
+Wed Oct 31 09:53:39 CET 2007
+
+ This code has been used to compute the multiphase simulations
+
+ * new implementation of sticky
+
+ * ngb.c
+ /* GAS_SPH sees only GAS_SPH */
+ /* GAS_STICKY sees GAS_SPH, GAS_STICKY but not GAS_DARK */
+ /* GAS_DARK sees everybody */
+
+ * cooling.c
+
+ heating_compton : z dependency
+
+ compute_cooling_from_Egyspec_and_Density :
+ take the dmax(0,val)
+
+ * hydra.c : add sticky_evaluate
+
+ * phase.c : add cold gas
+ add possibility of collision
+
+ * timestep.c : modification of sticky energy computation
+
+
+Tue Apr 15 15:31:23 CEST 2008
+
+ Merging of different diverging versions
+
+ * updated with respect to the "multiphase" version used for the paper
+
+
+ * add regor and leo in the Makefile
+
+ * definitively disable compton heating (heating_compton())
+
+ * update outerpotential : potential can now be summed
+ add plummer potential
+
+ + proto.h
+ + outerpotential.c
+ + begrun.c
+
+Tue May 27 14:21:50 CEST 2008
+
+ * lot's of cleaning
+ * sph as been rewriten, but not finnished (do not use it)
+
+Wed May 28 19:48:32 CEST 2008
+
+ * sticky.c replace sticky_evaluate in hydra
+ -> new implementation of sticky
+ -> when need to collide, each particle try to find a particle in the same proc
+ -> ngb_treefind_sticky_collision
+
+ * in hydra, we have removed sticky_evaluate and multi_hydro_evaluate
+
+
+Tue Jun 3 13:59:59 CEST 2008
+
+ * coriolis added in outerpotential.c
+ * in phase.c
+
+ for(i = 0; i < N_gas; i++)
+ instead of
+ for(i = 0; i < NumPart; i++)
+
+
+Mon Jun 16 21:08:09 CEST 2008
+
+ * modifications in sticky.c
+ sticky collisions are computed using a grid (Bournaud) or neighbors.
+
+Fri Jun 20 15:52:43 CEST 2008
+
+ * add a velocity condition (StickyMinVelocity) for sticky collisions
+
+Sun Sep 14 12:08:41 CEST 2008
+
+ * new cooling function based on "treeasph" format file
+ the cooling function may now depends on metallicity
+
+ * for the old cooling function, we no longer use the MetallicityIndex,
+ but give the Metallicity using the InitGasMetallicity parameter
+
+Tue Sep 16 17:25:24 CEST 2008
+
+ * bug correction in cooling.c
+ add case in function of the cooling type
+
+
+Tue Nov 11 13:57:34 CET 2008
+
+ * in restart.c use again system function
+ system(buf_mv); /* move old restart files to .bak files */
+
+
+ * move all initializations at the end of begrun() in begrun.c
+ this is important to restart the simulation properly
+
+ /* other initialization for special behavior */
+ init_cooling();
+ init_outer_potential();
+ init_bubble();
+ init_sticky();
+
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ !!! we should also check that : star formation, feedback, bondi_accretion, agn_feedback, agn_heating, phase
+ !!! are correctly initialized in restart mode
+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
+Wed Jan 28 13:04:25 CET 2009
+
+ * The cooling is now computed in timestep.c and not in accel.c
+ This has the advantage to use the real timestep used to advance
+ the velocities. The energy lost is better taken into account.
+
+ The entropy variation due to cooling is computed with an adaptative
+ timestep integration. However, it appears not to be so usefull.
+
+ It is now possible to use the option IMPLICIT_COOLING_INTEGRATION
+ if one need to integrated implicitely the cooling. Usefull in
+ the case of UV heating.
+
+
+ A new cooling function has been introduced : CoolingForOne
+
+
+Wed Jan 28 13:39:46 CET 2009
+
+ * use StarFormationNStarsFromGas instead of NStarsFromGas
+
+ !!! this was done before, but not mentionned !!!
+ now, we can use StarFormationType=2 and specify StarFormationCstar
+ then, StarFormationTime
+
+ All.ThresholdDensity = All.StarFormationDensity;
+ All.StarFormationTime = 1./All.StarFormationCstar / pow(4*PI*All.G*All.StarFormationDensity,0.5);
+
+
+
+Tue Feb 3 17:38:55 CET 2009
+
+ * introduce StP
+
+ - allvars.h
+ - allvars.c
+ - allocate.c
+ - domain.c
+ - peano.c
+ reorder_gas
+ reorder_particles
+ - starformation.c
+ - restart.c
+ - io.c # OUTPUTSTELLAR_PROP
+
+
+
+ * introduction of : do_chimie()
+
+ - chimie.c
+ - run.c
+
+ but the chimie is not implemented
+
+
+ * tests of the new star formation routine
+
+ the energy is correctly conserved if TreeDomainUpdateFrequency=0
+ when StarFormationNStarsFromGas > 1.
+
+
+Wed Jul 29 19:03:55 CEST 2009
+
+ * work of several weeks.
+
+ - modification of sfr (energy is now conserved)
+ - implementation of chimie (seems to work)
+ - implementation of chimie_thermalfeedback
+
+ - energy is more or less well conserved when chimie is
+ switched on with thermal feedback
+
+ - However, we observe a slight derive of the energy if sfr is strong,
+ even, with a small CourantFac=0.01
+ The increase in energy is due to feedback. The increased specific
+ energy is badly converted in kinetic energy... This has to be fixed.
+
+ - Without kinetic feedback, the energy is well conserved.
+
+
+Fri Jul 31 11:43:11 CEST 2009
+
+ * little change in cooling.c to be compatible with treeasph.
+ The new option DO_NO_USE_HYDROGEN_MASSFRAC_IN_COOLING
+ allow to run the cooling without using HYDROGEN_MASSFRAC (assumed to be 1).
+ This is wrong, but compatible with treeasph.
+ The temperature is still computed correctly (as in treeasph).
+
+Thu Aug 13 19:22:37 CEST 2009
+
+ * the conservation in energy is now fixed for CHIMIE_KINETIC_FEEDBACK.
+ This is solved by anticipating the wind injection and setting the timesteps
+ of all actives particles to the minimal value (ti_step=1);
+
+ Some changes is also done in timestep.c in order to allow the timesteps to
+ quickly increase.
+
+
+ while(((TIMEBASE - P[i].Ti_endstep) % ti_step) > 0)
+ ti_step = ti_step/2;
+
+ instead of
+
+ //if(((TIMEBASE - P[i].Ti_endstep) % ti_step) > 0)
+ // ti_step = P[i].Ti_endstep - P[i].Ti_begstep; /* leave at old step */
+
+
+ * fix some problems during restar
+
+
+Wed Aug 19 12:43:10 CEST 2009
+
+ * remove some printf in domain.c and starformation.c
+
+ * !!! add force_update_hmax(); after density() in run.c
+ hmax is needed for ngb search, in :
+ ngb_treefind_pairs : hydra.c, chimie.c
+ ngb_treefind_phase_pairs : bondi_accretion.c:
+
+
+ * remove : ngb_treefind_pairs_for_chimie
+ in chimie.c, we use ngb_treefind_pairs
+
+ NOTE : in stars_density.c we use ngb_treefind_variable_for_chimie
+ but it is similar to ngb_treefind_variable
+
+
+Mon Aug 31 18:18:27 CEST 2009
+
+
+ * some modification of sticky
+ - add CPU_Sticky
+ - check energy conservation
+ - check restart
+
+Thu Sep 3 11:20:17 CEST 2009
+
+ * some minor modification of sticky
+
+
+ Thu Sep 10 11:17:16 CEST 2009
+
+ * work on the work-load balance problem
+ In some cases, where the number of procs is too high comparing
+ to the number of particles, the load balance is bad. It means that
+ some procs needs much less time to computre gravity forces comparing
+ to others and they have to wait after.
+ I have tried to improve the load balance by spliting the domains not
+ base on gravcost but on the time spend to computre the gravity forces.
+
+ Unfortunately, as the forces are computed in two parts (local particles
+ and exported particles) it is difficult to reduce the imbalance for the
+ two parts. The gain of the new method is thus limited.
+
+
+ If the option SPLIT_DOMAIN_USING_TIME is enabled,
+ domain_findSplityr
+ and
+ domain_shiftSplityr
+ are used instead of
+ domain_findSplit
+ and
+ domain_shiftSplit
+
+ WARNING
+
+ - In domain_findSplityr we do not check the memory constaints.
+ - Not tested with adaptative timesteps
+
+
+Fri Sep 11 11:50:42 CEST 2009
+
+ * io.c
+
+ /* set fill to " " : yr Thu Aug 13 17:34:07 CEST 2009*/
+ for (i=0;i<sizeof(header.fill);i++)
+ strcpy(&header.fill[i]," ");
+
+ causes memory overflow. We have replaced it by :
+
+ memset(header.fill,' ',sizeof(header.fill));
+
+Thu Sep 17 14:58:00 CEST 2009
+
+ Enhancement of the CPU count (DETAILED_CPU)
+
+
+Tue Jan 5 16:56:12 CET 2010
+
+ * sticky.c
+
+ new version of sticky (double loop) now independents of the
+ number of procs
+
+ * sfr.c chimie.c
+
+ small corrections, implementation of delay of cooling after feedback
+
+Tue Mar 9 15:13:06 CET 201
+
+ * CHIMIE_EXTRAHEADER, io.c, allvars.h,read_ic.c
+ in the case of chemical evolution, we add an extraheader of 256 octets after the normal header
+ to store information on the chimie parameters.
+
+ * begrun.c
+ the cooling and chimie is now initialized before reading the first snapshot
+
+ * chimie.c
+ it is now possible to add several elements, all defined in the chimie file.
+
+ * cooling.c
+ the SolarAbundance of the cooling is now the one of the chimie.
+
+ * init.c, io.c
+ it is now possible to start a simulation with gas particles having metallicities different from 0.
+
+ * read_ic.c
+ a bug has been corrected when reading IO_STAR_IDPROJ, IO_STAR_RHO and IO_STAR_HSML
+
+ * peano.c : now we can order the particles following the peano keys, when STELLAR_PROP is on.
+ In a simulation including the chemical evolution, the improvement is not so strong (10% ?)
+
+ First, gas particles are sorted, then the stars and finally the other particles.
+ StP is also stored based on P.
+
+
+Thu Mar 18 14:33:25 CET 2010
+
+ * domain.c
+
+ if(TopNodes[sub].Count > All.TotNumPart / (TOPNODEFACTOR * NTask))
+
+ instead of
+
+ if(TopNodes[sub].Count > All.TotNumPart / (TOPNODEFACTOR * NTask * NTask))
+
+ This should improve the time needed for domain decomposition.
+ http://www.mpa-garching.mpg.de/gadget/gadget-list/0197.html
+
+
+Thu May 20 16:16:11 CEST 2010
+
+ * merge correction performed on the sticky part
+ * a lot of changes conserning the feedback and feedback_wind (tested with sticky)
+
+
+
+------------------------------------------------------------------------------------
+
+
+
+
+
+
+
+
+
diff --git a/TIME b/TIME
new file mode 100644
index 0000000..3285072
--- /dev/null
+++ b/TIME
@@ -0,0 +1,122 @@
+
+
+
+
+#ifdef DETAILED_CPU
+ double tstart,tend;
+#endif
+
+#ifdef DETAILED_CPU
+ tstart = second();
+#endif
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Leapfrog += timediff(tstart, tend);
+#endif
+
+
+
+
+
+All.CPU_Total
+
+ ################################ CPU_Leapfrog/CPU_Predict ok
+
+ find_next_sync_point_and_drift();
+
+ ################################ CPU_Residual ok
+
+ every_timestep_stuff();
+
+ ################################ CPU_Snapshot ok
+
+ savepositions(All.SnapshotFileCount++);
+
+ ################################ CPU_Physics ok
+
+ compute_agn_accretion();
+
+ make_bubble();
+
+ update_phase();
+
+ set_outer_potential_coriolis();
+
+ chimie();
+
+ density();
+ sfr_compute_energy_int(1);
+ star_formation();
+
+ sticky();
+
+ ################################ CPU_Domain+CPU_Peano ok
+
+ domain_Decomposition(); /* do domain decomposition if needed */
+
+ ################################ CPU_Accel ok
+
+ compute_accelerations(0); /* compute accelerations for
+
+
+ ################################ CPU_Physics ok
+
+ sfr_compute_energy_int(2);
+
+
+ ################################ CPU_Potential ok
+
+ compute_potential();
+
+ ################################ CPU_Residual ok
+
+ energy_statistics();
+ advanced_energy_statistics();
+
+ ################################ CPU_Leapfrog/CPU_TimeLine
+
+ advance_and_find_timesteps();
+
+ ################################ CPU_Residual
+
+ autres...
+
+ ################################
+
+
+
+
+
+CPU_Accel
+ CPU_Gravity, CPU_Hydro,CPU_PM ok
+CPU_Physics
+ CPU_StarFormation x
+ CPU_Chimie
+ CPU_Sticky
+CPU_Leapfrog +-ok
+ CPU_Predict
+ CPU_TimeLine
+CPU_Potential ok
+ CPU_Potential
+CPU_Snapshot ok
+ CPU_Snapshot
+CPU_Domain ok
+ CPU_Domain
+CPU_Peano ok
+ CPU_Peano
+CPU_Residual neg
+
+CPU_Begrun neg
+
+
+
+
+CPU_Gravity
+CPU_Hydro
+CPU_Domain
+
+
+
+
+
diff --git a/TODO b/TODO
new file mode 100644
index 0000000..aadea5a
--- /dev/null
+++ b/TODO
@@ -0,0 +1,687 @@
+
+
+
+
+
+
+
+
+new cooling :
+----------
+
+--- tester le cooling avec M=4, on devrait avoir Tcool << Tdyn à faire
+
+
+- ajouter implicit cooling integration ok
+
+ + tester ok, semble bien
+
+
+- verifier ComovingIntegrationOn à tester
+
+- tester sur cluster à faire
+
+- tester sur boite cosmo à faire
+
+
+
+on peut enlever :
+
+DtEgySpecRadSph
+DtEntropyRadSph
+
+
+
+new sfr :
+----------
+
+
+- crée des étoiles de plus petite masse, et donc, garder la part. de gaz. ok
+
+- utiliser tous les critères, jeans aussi... à faire
+
+- tester un model statique --> loi exp décroissante ok
+
+- transformer toute la particule lorsque elle est trops petite ok
+
+- ecrire les output de StP ok
+
+- prendre en compte l'énergie des étoiles... ok
+
+- conservation de l'énergie ok
+
+ 1 proc, StarFormationNStarsFromGas = 1 snap00
+ 4 proc, StarFormationNStarsFromGas = 1 snap01 mini diff.
+
+ 1 proc, StarFormationNStarsFromGas = 4 snap02 sfr correcte, mais
+ !!! il y a des oscillations...
+
+ 1 proc, StarFormationNStarsFromGas = 4 snap03 ok, on corrige avec TreeDomainUpdateFrequency = 0
+ 4 proc, StarFormationNStarsFromGas = 4
+
+
+- permettre : TreeDomainUpdateFrequency > 0 à faire
+
+
+
+domain :
+----------
+
+ - forcer les particules stellaire a etre arrangées dans le block 1 ok
+
+
+
+ domain.c à faire (difficile)
+ maxloadsph -> maxloadstars
+
+ pour PEANOHILBERT : trier les particules star à faire
+ peano.c à faire
+
+ domain_sumCost -> add local_DomainCountSt à faire (difficile ?)
+
+
+
+
+ - modifier StP ok
+
+
+ All.TotN_stars ok
+
+ All.MaxPartStars = All.StarFormationNStarsFromGas*All.MaxPartSph ok
+ + All.PartAllocFactor * (All.TotN_stars / NTask)
+
+
+ allocate.c ok
+ All.MaxPartStars
+ restart.c ok
+
+
+ init.c ok
+
+ domain.c ok
+ starformation.c ok
+
+ rearrange_particle_sequence ok
+
+ io.c
+ read_ic.c ok
+ ok
+
+
+
+ - vérifier les output ok
+
+ int flag_sfr
+ int flag_feedback
+ int flag_cooling
+ int flag_stellarage
+ int flag_metals
+
+ typelist[6] ok
+ --> 0 si aucune particule de ce type est présente dans le block
+
+ get_particles_in_block ok
+
+
+
+ - vérifier les input ok
+
+ - partir avec un fichier qui contiend déjà des étoiles
+ --> conditions initiales (lit qu'une partie...) ok
+ --> redémarrage
+
+ - partir avec un fichier qui contiend déjà la métalicité du gaz
+
+ --> conditions initiales
+ --> redémarrage
+
+
+ - tester All.TotN_stars ok
+ - N_stars ok
+
+
+
+
+
+
+
+
+
+
+
+chimie :
+----------
+
+
+- pour une part. d'étoile, trouver les plus proches particules de gaz ok
+
+
+ - il faut calculer hsml/density (gaz) pour les étoiles... ok
+
+ - dans do_chimie, commencer la boucle par la première étoile ok
+ et non particule
+
+ - verifier le bon nombre de voisins pour les étoiles ok
+
+
+
+
+ stars_density.c ok
+ chimie.c ok
+
+
+ tests :
+
+ - vérifier la densité (écrire) ok
+ - vérifier hsml (écrire) ok
+
+ -> on écrit dans le fichier de sortie Density,Hsml ok
+
+
+
+ - vérifier les plus proches voisins ok
+ --> dans chimie,
+ -> vérifier : Sum m/rho wij = 1, par example... ok
+
+
+
+- initialisation
+
+ utiliser get_nelts pour avoir NELEMENTS, ou alors, check.... à faire
+
+
+
+
+
+- pour une particule stellaire, calculer à faire
+
+ - la masse totale ejectée ok
+ - les métaux éjectés ok
+ - l'énergie éjectée ok
+
+
+- injecter la masse au plus proches voisins ok
+- enlever mass et elt ejecté par une étoile ok
+
+ !!! conservation de l'énergie kin,pot, int
+ !!! conservation de l'impulsion...
+ !!! conservation de la masse ok
+ ok, a 1%pres... a vérifier par la suite...
+ dans des conditions moins difficiles...
+
+ <-------- Wed Jul 22 15:09:25 CEST 2009
+
+- mieux conserver l'énergie lors de feedback thermqie (rendre part. active ?) à faire !!! (ou feedback cinétique...)
+
+- faire un fichier statistique qui comptes ce qui est crée... à faire
+
+- injecter l'énergie thermique ok
+
+- injecter l'énergie cinetique à faire
+
+- utiliser la metallicité pour le cooling ok
+
+
+
+
+
+
+
+
+
+- utiliser TreeDomainUpdateFrequency != 0.0 à faire
+
+
+- vérifier dans chimie.c -> utilisation de vel ou velpred...
+
+
+- unifier à faire
+
+
+SolarAbun_Fe = 0.001771 pNbody
+SolarAbun_Mg = 0.00091245 pNbody
+
+#define FEH_SOLAR 0.001771 Gadget /* 0.00181 */
+#define MGH_SOLAR 0.00091245 Gadget
+
+FeHSolar = 0.00181 Gadget cooling !!!
+
+
+
+
+
+- vérifier restart -> particulièrement les parametres à faire
+ ??? All.ChimieSupernovaEnergy = all.ChimieSupernovaEnergy
+
+
+
+- revoir sticky (faire attention au leap frog, par ex.) à faire
+- cooling : on peut completement sortie docooling de timestep, non ?
+
+
+
+!! initialisation correcte de : StarEnergyInt,StarEnergyRadSph, à faire
+ StarEnergyRadSticky...
+ --> aussi lors d'un restart !!!
+
+ --> should be in All.
+
+
+
+
+
+
+
+
+
+
+
+
+--------------------------------------------------------------------------------
+CPU
+--------------------------------------------------------------------------------
+
+timediff(t0, t1) = t1-t0
+
+All.CPU_Total,
+ run.c : on somme les diff de la boucle principale
+
+All.CPU_Gravity,
+ accel.c :
+ gravity_tree()
+
+All.CPU_Hydro,
+ accel.c :
+ density()
+ hydro_force();
+
+
+All.CPU_Domain,
+ domain.c :
+ domain_Decomposition() sans peano_hilbert_order
+
+
+All.CPU_Potential, potential.c :
+ compute_potential()
+
+All.CPU_Predict, accel.c :
+ force_update_hmax(); ???
+ predic.c :
+ move_particles()
+ run.c :
+ find_next_sync_point_and_drift()
+
+All.CPU_TimeLine, timestep.c :
+ advance_and_find_timesteps()
+
+All.CPU_Snapshot, io.c
+ savepositions()
+
+
+All.CPU_TreeConstruction potential.c
+ force_treebuild()
+ gravtree.c
+ force_treebuild()
+
+All.CPU_TreeWalk, gravtree.c !!!
+All.CPU_CommSum, gravtree.c
+All.CPU_Imbalance, gravtree.c
+ All.CPU_TreeWalk += sumt / NTask;
+ All.CPU_Imbalance += sumimbalance / NTask;
+ All.CPU_CommSum += sumcomm / NTask;
+
+
+All.CPU_HydCompWalk, chimie.c density.c hydra.c stars_density.c !!!
+All.CPU_HydCommSumm, chimie.c density.c hydra.c stars_density.c
+All.CPU_HydImbalance, chimie.c density.c hydra.c stars_density.c
+ All.CPU_HydImbalance += sumimbalance / NTask;
+ All.CPU_HydCommSumm += sumcomm / NTask;
+ All.CPU_HydCompWalk += sumt / NTask;
+
+All.CPU_EnsureNgb, density.c !!!
+ All.CPU_EnsureNgb += sumtimengb / NTask;
+
+ stars_density.c
+ All.CPU_EnsureNgb += sumtimengb / NTask;
+
+
+All.CPU_PM, accel.c
+ long_range_force()
+
+All.CPU_Peano
+ domain.c : peano_hilbert_order()
+
+
+on pourait rajouter :
+-------------------
+
+All.CPU_Accel
+
+if faut rajouter :
+
+All.CPU_Chimie
+All.CPU_StarFormation
+
+
+
+
+
+
+
+
+
+
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+
+
+- sticky bournaud
+
+ - faire des tests
+ - choix du sticky --> pour cluster, mieux dans rayon sph...
+
+- éviter que la densité soit calculée ??? vraiment utiles ?
+
+
+- conservation de l'énergie pour sticky
+
+
+
+
+----------------------------------------
+
+- improve multiphase computation : only with MULTIPHASE switched on
+
+
+ - test sticky only
+
+ * for each particle, find one particle to collide with
+ * try to do that in the same function than the main hydra (???)
+
+
+
+
+
+
+
+
+
+
+ - test sph only
+
+ - test dark gas
+
+ - test combination of all
+
+
+
+ check
+
+
+ * indep. of number of proc
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+##########################################3
+MULTIPHASE
+##########################################
+
+ - part of particles behave differently (sticky)
+
+
+GAS_SPH -> normal sph
+GAS_STICKY -> sticky
+GAS_DARK -> no (weak) interaction
+
+
+
+
+
+functions
+
+multi_hydro_evaluate
+sticky_evaluate
+ngb_treefind_phase_pairs
+
+
+io.c ok
+init.c ok
+read_ic.c ok
+
+run.c ok
+
+
+density.c ok (rien de particulier)
+ngb.c
+ ngb_treefind_pairs hydra.c
+ every body sees every body
+
+
+ ngb_treefind_variable density.c
+ find density and hsml !!!! hsml is then needed to find colliding particules
+ !!!! but may be problematic if only sph part. are present in hsml !!!
+
+
+ ngb_treefind_phase_pairs bondi_accretion.c
+
+
+hydra.c !!!!!!!!!!!!
+ multi_hydro_evaluate(i, 0);
+ hydro_evaluate(i, 0);
+ sticky_evaluate(i, 0);
+
+
+
+
+
+
+timestep.c
+global.c
+
+cooling.c
+starformation.c
+bubbles.c
+
+
+
+
+
+
+
+##########################################
+PHASE_MIXING
+##########################################
+
+proto.h
+ phase_mixing
+
+run.c
+phase.c
+
+
+
+
+
+
+
+##########################################
+COLDGAS_CYCLE (need MULTIPHASE)
+##########################################
+
+ - compute cycling or not
+
+
+ ---------------------------------------
+
+ COLDGAS_CYCLE :parameters
+ -------------------------
+
+ ColdGasCycleTransitionTime;
+ ColdGasCycleTransitionParameter;
+
+ allvars.h ok
+ begrun.c ok
+ phase.c ok
+
+
+##########################################
+EXTERNAL_FLUX : really only external flux
+##########################################
+
+ begrun.c
+ cooling.c
+ phase.c
+ allvars.h
+
+
+ HeatingPeSolarEnergyDensity
+ HeatingExternalFLuxEnergyDensity
+
+##########################################
+STELLAR_FLUX : only stellar flux
+##########################################
+
+ allvars.h
+ init.c
+ gravtree.c
+ forcetree.c
+ begrun.c
+
+ HeatingPeSolarEnergyDensity;
+ HeatingPeLMRatioGas;
+ HeatingPeLMRatioHalo;
+ HeatingPeLMRatioDisk;
+ HeatingPeLMRatioBulge;
+ HeatingPeLMRatioStars;
+ HeatingPeLMRatioBndry;
+ HeatingPeLMRatio[6];
+
+
+
+
+
+####################################################
+# sticky_evaluate(i, 0)
+####################################################
+
+
+
+1) check who can inteact with who ?
+
+
+ - loop over active particles
+ - ngb_treefind_pairs ! find all particles in h_i ... check !!!
+ here we could use different fct, depending on the type
+
+ - if(P[j].Ti_endstep == All.Ti_Current) only active particles ! ensure symetry,
+ really necessary ???
+
+ - if(SphP[j].Phase == GAS_STICKY) ok, but may be done with "ngb_treefind_pairs"
+
+
+ - if(SphP[j].StickyFlag) SphP[i].StickyFlag = 1; in init.c
+ SphP[i].StickyFlag is determined in phase.c
+
+
+
+
+
+2) what is modified by sticky_evaluate
+
+
+ P[target].Vel[k] <------ here, we change the velocity !!!!!!!
+ P[j].Vel[k]
+
+
+
+ SphP[target].EgySpecRadSticky <------ here, we count the energy !!!!!!!
+ SphP[j].EgySpecRadSticky
+
+
+ SphP[target].StickyCollisionNumber++;
+ SphP[j].StickyCollisionNumber++;
+
+ SphP[target].HydroAccel[k] = 0;
+
+ SphP[target].StickyFlag = 0;
+ SphP[target].DtEntropy = 0; /* force dt entropy to zero */
+
+
+ SphP[j].DtEgySpecFeedback = 0; /* should not be there */
+
+
+
+ !!!! we change the velocity and count energy further
+
+
+
+
+
+
+
+
+
+tests:
+
+ - only sticky : be sure that particles interact symetrically...
+
+
+
+
+
+
+
+
+####################################################
+# problemes
+####################################################
+
+
+(!!) si Entropy < 0, le gaz est considéré GAS_DARK (output)
+ -> bien vérifier...
+
+
+(!!) si une particule oscille frequemment entre dark et visible, elle risque
+ de colisonner trops souvent... StickyFlag=1 automatique lors du retour au sticky
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/accel.c b/accel.c
new file mode 100644
index 0000000..2215515
--- /dev/null
+++ b/accel.c
@@ -0,0 +1,122 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file accel.c
+ * \brief driver routine to carry out force computation
+ */
+
+
+/*! This routine computes the accelerations for all active particles.
+ * First, the long-range PM force is computed if the TreePM algorithm is
+ * used and a "big" PM step is done. Next, the gravitational tree forces
+ * are computed. This also constructs the tree, if needed.
+ *
+ * If gas particles are present, the density-loop for active SPH particles
+ * is carried out. This includes an iteration on the correct number of
+ * neighbours. Finally, the hydrodynamical forces are added.
+ */
+void compute_accelerations(int mode)
+{
+ double tstart, tend;
+
+#ifdef DETAILED_CPU
+ double t0,t1;
+ t0 = second();
+#endif
+
+
+
+
+ if(ThisTask == 0)
+ {
+ printf("Start force computation...\n");
+ fflush(stdout);
+ }
+
+#ifdef PMGRID
+ if(All.PM_Ti_endstep == All.Ti_Current)
+ {
+ tstart = second();
+ long_range_force();
+ tend = second();
+ All.CPU_PM += timediff(tstart, tend);
+ }
+#endif
+
+ tstart = second(); /* measure the time for the full force computation */
+
+ gravity_tree(); /* computes gravity accel. */
+
+ if(All.TypeOfOpeningCriterion == 1 && All.Ti_Current == 0)
+ gravity_tree(); /* For the first timestep, we redo it
+ * to allow usage of relative opening
+ * criterion for consistent accuracy.
+ */
+ tend = second();
+ All.CPU_Gravity += timediff(tstart, tend);
+
+#ifdef FORCETEST
+ gravity_forcetest();
+#endif
+
+
+
+ if(All.TotN_gas > 0)
+ {
+
+ if(ThisTask == 0)
+ {
+ printf("Start density computation...\n");
+ fflush(stdout);
+ }
+
+ tstart = second();
+ density(0); /* computes density, and pressure */
+ tend = second();
+ All.CPU_Hydro += timediff(tstart, tend);
+
+
+ tstart = second();
+ force_update_hmax(); /* tell the tree nodes the new SPH smoothing length such that they are guaranteed to hold the correct max(Hsml) */
+ tend = second();
+ All.CPU_Predict += timediff(tstart, tend);
+
+
+#ifndef NO_HYDRO_FOR_GAS
+
+ if(ThisTask == 0)
+ {
+ printf("Start hydro-force computation...\n");
+ fflush(stdout);
+ }
+
+ tstart = second();
+ hydro_force(); /* adds hydrodynamical accelerations and computes viscous entropy injection */
+ tend = second();
+ All.CPU_Hydro += timediff(tstart, tend);
+
+#endif
+ }
+
+
+
+
+ if(ThisTask == 0)
+ {
+ printf("force computation done.\n");
+ fflush(stdout);
+ }
+
+#ifdef DETAILED_CPU
+ t1 = second();
+ All.CPU_Accel += timediff(t0, t1);
+#endif
+
+
+}
diff --git a/agn_feedback.c b/agn_feedback.c
new file mode 100644
index 0000000..4031926
--- /dev/null
+++ b/agn_feedback.c
@@ -0,0 +1,214 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef AGN_ACCRETION
+
+
+/*! \file bubble.c
+ * \brief Init bubble
+ *
+*/
+
+
+/*! compute bubble
+ *
+ */
+void compute_agn_accretion()
+{
+ int i,k;
+ double AccretionRadius2;
+ double Ma[7],Ma_sum[7];
+ double r2,r;
+#ifdef AGN_USE_ANGULAR_MOMENTUM
+ double Lvec[3],Lvec_sum[3];
+ double theta,phi;
+#endif
+
+ /*double a3inv=1.;*/ /* !!!!!!!!!!!! */
+
+
+
+ if((All.Time - All.TimeLastAccretion) >= All.TimeBetAccretion || All.NumCurrentTiStep==0)
+ {
+
+ All.TimeLastAccretion += All.TimeBetAccretion;
+
+ if(ThisTask == 0)
+ printf("computing accretion\n");
+
+
+ AccretionRadius2 = All.AccretionRadius*All.AccretionRadius;
+ for (k = 0; k < 7; k++)
+ {
+ Ma[k]=0.;
+ Ma_sum[k]=0.;
+ }
+
+#ifdef AGN_USE_ANGULAR_MOMENTUM
+ Lvec[0] = Lvec[1] = Lvec[2] = 0;
+#endif
+ /*
+ loop over all particles
+ */
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+
+ r2 = P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2];
+
+ if (r2<AccretionRadius2)
+ {
+
+#ifdef MULTIPHASE
+
+ if (P[i].Type==0)
+ {
+
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ Ma[0] += P[i].Mass;
+ break;
+
+ case GAS_STICKY:
+ case GAS_DARK:
+ Ma[6] += P[i].Mass;
+ break;
+ }
+
+ }
+ else
+ {
+ Ma[P[i].Type] += P[i].Mass;
+ }
+
+
+#else
+ Ma[P[i].Type] += P[i].Mass;
+#endif
+
+#ifdef AGN_USE_ANGULAR_MOMENTUM
+ Lvec[0] += P[i].Pos[1]*P[i].Vel[2] - P[i].Pos[2]*P[i].Vel[1];
+ Lvec[1] += P[i].Pos[2]*P[i].Vel[0] - P[i].Pos[0]*P[i].Vel[2];
+ Lvec[2] += P[i].Pos[0]*P[i].Vel[1] - P[i].Pos[1]*P[i].Vel[0];
+#endif
+ }
+
+ }
+
+
+ /* share results */
+ MPI_Allreduce(Ma, Ma_sum, 7, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+#ifdef AGN_USE_ANGULAR_MOMENTUM
+ MPI_Allreduce(&Lvec, &Lvec_sum, 3, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+#endif
+
+ /*
+ compute parameters for feedback
+
+ All.LastMTotInRa : total Mass in Ra at last feedback
+ All.MTotInRa : total Mass in Ra
+ All.dMTotInRa : diff between total Mass in Ra now and total Mass in Ra last feedback
+
+ All.C : speed of light
+
+ All.AGNFactor : factor efficiency
+ All.MinMTotInRa : minimum accumulated mass in Ra needed to generate a buble
+
+ */
+
+
+ /* store results Ma_sum[6]=sticky */
+ All.MTotInRa = Ma_sum[0]+Ma_sum[6]+Ma_sum[1]+Ma_sum[2]+Ma_sum[3]+Ma_sum[4];
+
+ if (All.NumCurrentTiStep==0)
+ {
+ All.LastMTotInRa = All.MTotInRa;
+ All.dMTotInRa = 0;
+ }
+ else
+ {
+ All.dMTotInRa = All.MTotInRa - All.LastMTotInRa;
+ }
+
+ /*
+ if(ThisTask == 0)
+ printf("--> time=%g dMTotInRa=%g MTotInRa=%g LastMTotInRa=%g MinMTotInRa=%g AGNFactor=%g\n",All.Time,All.dMTotInRa,All.MTotInRa,All.LastMTotInRa,All.MinMTotInRa,All.AGNFactor);
+ */
+
+#ifdef AGN_FEEDBACK
+
+ /* check if it is time for a bubble */
+ if (All.dMTotInRa > All.MinMTotInRa)
+ {
+
+ /* compute jet energy */
+ All.BubblesE[All.BubblesIndex] = All.AGNFactor * All.dMTotInRa * All.LightSpeed* All.LightSpeed;
+ All.LastMTotInRa = All.MTotInRa;
+
+
+#ifdef AGN_USE_ANGULAR_MOMENTUM
+ /* compute jet direction */
+ r = sqrt( Lvec_sum[0]*Lvec_sum[0] + Lvec_sum[1]*Lvec_sum[1] + Lvec_sum[2]*Lvec_sum[2] );
+ theta = acos(Lvec_sum[2]/r);
+
+ r = sqrt( Lvec_sum[0]*Lvec_sum[0] + Lvec_sum[1]*Lvec_sum[1] );
+ phi = acos(Lvec_sum[0]/r);
+
+ All.BubblesA[All.BubblesIndex] = theta;
+ All.BubblesB[All.BubblesIndex] = phi;
+#endif
+ /* set time for the bubbles */
+ All.BubblesTime[All.BubblesIndex]= 0.;
+
+ }
+
+#endif
+
+
+ /* write output */
+ if(ThisTask == 0)
+ {
+
+#ifdef MULTIPHASE
+ fprintf(FdAccretion, "Step: %d, Time: %g, M1a: %g, M1b: %g, M2: %g, M3: %g, M4: %g, M5: %g, M6: %g\n", All.NumCurrentTiStep, All.Time,
+ Ma_sum[0],Ma_sum[6],Ma_sum[1],Ma_sum[2],Ma_sum[3],Ma_sum[4],Ma_sum[5]);
+#else
+ fprintf(FdAccretion, "Step: %d, Time: %g, M1: %g, M2: %g, M3: %g, M4: %g, M5: %g, M6: %g\n", All.NumCurrentTiStep, All.Time,
+ Ma_sum[0],Ma_sum[1],Ma_sum[2],Ma_sum[3],Ma_sum[4],Ma_sum[5]);
+#endif
+ fflush(FdAccretion);
+ }
+
+
+
+ }
+
+}
+
+
+
+
+
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/agn_heating.c b/agn_heating.c
new file mode 100644
index 0000000..1514508
--- /dev/null
+++ b/agn_heating.c
@@ -0,0 +1,191 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef AGN_HEATING
+
+
+/*! \file agn_heating.c
+ * \brief Compute gas heating due to agn
+ *
+*/
+
+static double hubble_a, a3inv;
+
+
+
+void agn_heating()
+ {
+
+
+ int i;
+ double SpecPower;
+ double r,fm,fm_sum,pn,pn_sum;
+
+
+ if(All.ComovingIntegrationOn)
+ {
+
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ }
+ else
+ a3inv = hubble_a = 1;
+
+
+ /* here, we want to inject a power P */
+
+ /************************************/
+ /* first loop : determine SpecPower */ /* here, we should take into account non active part */
+ /************************************/
+
+ fm = 0;
+ pn = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ if(P[i].Type == 0) /* gas */
+ {
+ r = sqrt( P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2] );
+
+ if (r < All.AGNHeatingRmax)
+ fm += HeatingRadialDependency(r)*P[i].Mass;
+
+ }
+ }
+ else
+ {
+ if(P[i].Type == 0) /* gas */
+ {
+ pn += -SphP[i].DtEgySpecAGNHeat*P[i].Mass; /* power of non active particles */
+ }
+ }
+
+ }
+
+
+
+ /* share results */
+ MPI_Allreduce(&fm, &fm_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&pn, &pn_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+#ifdef USE_BONDI_POWER
+ All.AGNHeatingPower = All.BondiPower;
+#endif
+
+ SpecPower = (All.AGNHeatingPower-pn_sum)/fm_sum ;
+
+
+ /***************/
+ /* second loop */
+ /***************/
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+ if(P[i].Type == 0) /* gas */
+ {
+
+ r = sqrt( P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2] );
+
+ if (r < All.AGNHeatingRmax)
+ {
+
+ /* first reset dtentropy */
+ SphP[i].DtEntropyAGNHeat = 0;
+
+ /* first, we simply stop the radiative cooling */
+ //SphP[i].DtEntropyAGNHeat = -SphP[i].DtEntropyRadSph;
+ //SphP[i].DtEntropyRadSph = 0;
+
+ /* then, heat with agn*/
+ SphP[i].DtEntropyAGNHeat += GAMMA_MINUS1*pow(SphP[i].Density * a3inv,-GAMMA)*gamma_fct(SphP[i].Density *a3inv,r,SpecPower)/hubble_a;
+ SphP[i].DtEgySpecAGNHeat = - 1/GAMMA_MINUS1 * pow(SphP[i].Density * a3inv,GAMMA_MINUS1) * (SphP[i].DtEntropyAGNHeat);
+
+
+#ifdef MULTIPHASE
+ switch(SphP[i].Phase)
+ {
+ case GAS_SPH: /* here we increase the entropy */
+ SphP[i].DtEntropy += SphP[i].DtEntropyAGNHeat;
+ break;
+ case GAS_STICKY: /* here we increase the energy */
+ case GAS:DARK:
+ SphP[i].DtEntropy += -SphP[i].DtEgySpecAGNHeat;
+ break;
+ }
+#else
+ SphP[i].DtEntropy += SphP[i].DtEntropyAGNHeat;
+#endif
+
+ }
+ else /* no heating */
+ {
+ SphP[i].DtEntropyAGNHeat = 0; /* if the part is inactive ??? */
+ }
+
+ }
+ }
+ }
+
+
+ }
+
+
+/*! heating function
+ *
+ */
+double gamma_fct(FLOAT density,double r, double SpecPower)
+{
+ double g;
+ g = SpecPower * HeatingRadialDependency(r) * density;
+ return g;
+}
+
+
+
+
+/*! heating radial dependency
+ *
+ */
+double HeatingRadialDependency(double r)
+{
+ double f;
+ /* radius dependency */
+ //f = (1 - r/All.AGNHeatingRmax);
+ //f = f*f;
+
+ if (r<All.AGNHeatingRmax)
+ f = 1;
+ else
+ f = 0;
+
+ return f;
+}
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/allocate.c b/allocate.c
new file mode 100644
index 0000000..dd4c615
--- /dev/null
+++ b/allocate.c
@@ -0,0 +1,218 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file allocate.c
+ * \brief routines for allocating particle and tree storage
+ */
+
+/*! Allocates a number of small buffers and arrays, the largest one being
+ * the communication buffer. The communication buffer itself is mapped
+ * onto various tables used in the different parts of the force
+ * algorithms. We further allocate space for the top-level tree nodes, and
+ * auxiliary arrays for the domain decomposition algorithm.
+ */
+void allocate_commbuffers(void)
+{
+ size_t bytes;
+
+ Exportflag = malloc(NTask * sizeof(char));
+ DomainStartList = malloc(NTask * sizeof(int));
+ DomainEndList = malloc(NTask * sizeof(int));
+
+ TopNodes = malloc(MAXTOPNODES * sizeof(struct topnode_data));
+
+ DomainWork = malloc(MAXTOPNODES * sizeof(double));
+ DomainCount = malloc(MAXTOPNODES * sizeof(int));
+ DomainCountSph = malloc(MAXTOPNODES * sizeof(int));
+ DomainTask = malloc(MAXTOPNODES * sizeof(int));
+ DomainNodeIndex = malloc(MAXTOPNODES * sizeof(int));
+ DomainTreeNodeLen = malloc(MAXTOPNODES * sizeof(FLOAT));
+ DomainHmax = malloc(MAXTOPNODES * sizeof(FLOAT));
+ DomainMoment = malloc(MAXTOPNODES * sizeof(struct DomainNODE));
+
+ if(!(CommBuffer = malloc(bytes = All.BufferSize * 1024 * 1024)))
+ {
+ printf("failed to allocate memory for `CommBuffer' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(2);
+ }
+
+ All.BunchSizeForce =
+ (All.BufferSize * 1024 * 1024) / (sizeof(struct gravdata_index) + 2 * sizeof(struct gravdata_in));
+
+ if(All.BunchSizeForce & 1)
+ All.BunchSizeForce -= 1; /* make sure that All.BunchSizeForce is an even number
+ --> 8-byte alignment for 64bit processors */
+
+ GravDataIndexTable = (struct gravdata_index *) CommBuffer;
+ GravDataIn = (struct gravdata_in *) (GravDataIndexTable + All.BunchSizeForce);
+ GravDataGet = GravDataIn + All.BunchSizeForce;
+ GravDataOut = GravDataIn; /* this will overwrite the GravDataIn-Table */
+ GravDataResult = GravDataGet; /* this will overwrite the GravDataGet-Table */
+
+
+ All.BunchSizeDensity =
+ (All.BufferSize * 1024 * 1024) / (2 * sizeof(struct densdata_in) + 2 * sizeof(struct densdata_out));
+
+ DensDataIn = (struct densdata_in *) CommBuffer;
+ DensDataGet = DensDataIn + All.BunchSizeDensity;
+ DensDataResult = (struct densdata_out *) (DensDataGet + All.BunchSizeDensity);
+ DensDataPartialResult = DensDataResult + All.BunchSizeDensity;
+
+ All.BunchSizeHydro =
+ (All.BufferSize * 1024 * 1024) / (2 * sizeof(struct hydrodata_in) + 2 * sizeof(struct hydrodata_out));
+
+ HydroDataIn = (struct hydrodata_in *) CommBuffer;
+ HydroDataGet = HydroDataIn + All.BunchSizeHydro;
+ HydroDataResult = (struct hydrodata_out *) (HydroDataGet + All.BunchSizeHydro);
+ HydroDataPartialResult = HydroDataResult + All.BunchSizeHydro;
+
+#ifdef MULTIPHASE
+ All.BunchSizeSticky =
+ (All.BufferSize * 1024 * 1024) / (2 * sizeof(struct stickydata_in) + 2 * sizeof(struct stickydata_out));
+
+ StickyDataIn = (struct stickydata_in *) CommBuffer;
+ StickyDataGet = StickyDataIn + All.BunchSizeSticky;
+ StickyDataResult = (struct stickydata_out *) (StickyDataGet + All.BunchSizeSticky);
+ StickyDataPartialResult = StickyDataResult + All.BunchSizeSticky;
+
+#endif
+
+#ifdef CHIMIE
+ All.BunchSizeChimie =
+ (All.BufferSize * 1024 * 1024) / (2 * sizeof(struct chimiedata_in) + 2 * sizeof(struct chimiedata_out));
+
+ ChimieDataIn = (struct chimiedata_in *) CommBuffer;
+ ChimieDataGet = ChimieDataIn + All.BunchSizeChimie;
+ ChimieDataResult = (struct chimiedata_out *) (ChimieDataGet + All.BunchSizeChimie);
+ ChimieDataPartialResult = ChimieDataResult + All.BunchSizeChimie;
+
+ All.BunchSizeStarsDensity =
+ (All.BufferSize * 1024 * 1024) / (2 * sizeof(struct starsdensdata_in) + 2 * sizeof(struct starsdensdata_out));
+
+ StarsDensDataIn = (struct starsdensdata_in *) CommBuffer;
+ StarsDensDataGet = StarsDensDataIn + All.BunchSizeStarsDensity;
+ StarsDensDataResult = (struct starsdensdata_out *) (StarsDensDataGet + All.BunchSizeStarsDensity);
+ StarsDensDataPartialResult = StarsDensDataResult + All.BunchSizeStarsDensity;
+
+
+#endif
+
+
+
+#ifdef STELLAR_PROP
+ All.BunchSizeDomain =
+ (All.BufferSize * 1024 * 1024) / (sizeof(struct particle_data) + sizeof(struct sph_particle_data) +
+ sizeof(struct st_particle_data) + sizeof(peanokey));
+#else
+ All.BunchSizeDomain =
+ (All.BufferSize * 1024 * 1024) / (sizeof(struct particle_data) + sizeof(struct sph_particle_data) +
+ sizeof(peanokey));
+#endif
+
+ if(All.BunchSizeDomain & 1)
+ All.BunchSizeDomain -= 1; /* make sure that All.BunchSizeDomain is even
+ --> 8-byte alignment of DomainKeyBuf for 64bit processors */
+
+ DomainPartBuf = (struct particle_data *) CommBuffer;
+ DomainSphBuf = (struct sph_particle_data *) (DomainPartBuf + All.BunchSizeDomain);
+ DomainKeyBuf = (peanokey *) (DomainSphBuf + All.BunchSizeDomain);
+#ifdef STELLAR_PROP
+ DomainStBuf = (struct st_particle_data *) (DomainKeyBuf + All.BunchSizeDomain);
+#endif
+
+ if(ThisTask == 0)
+ {
+ printf("\nAllocated %d MByte communication buffer per processor.\n\n", All.BufferSize);
+ printf("Communication buffer has room for %d particles in gravity computation\n", All.BunchSizeForce);
+ printf("Communication buffer has room for %d particles in density computation\n", All.BunchSizeDensity);
+ printf("Communication buffer has room for %d particles in hydro computation\n", All.BunchSizeHydro);
+ printf("Communication buffer has room for %d particles in domain decomposition\n", All.BunchSizeDomain);
+ printf("\n");
+ }
+
+}
+
+
+
+/*! This routine allocates memory for particle storage, both the
+ * collisionless and the SPH particles.
+ */
+void allocate_memory(void)
+{
+ size_t bytes;
+ double bytes_tot = 0;
+
+ if(All.MaxPart > 0)
+ {
+ if(!(P = malloc(bytes = All.MaxPart * sizeof(struct particle_data))))
+ {
+ printf("failed to allocate memory for `P' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(ThisTask == 0)
+ printf("\nAllocated %g MByte for particle storage. %d\n\n", bytes_tot / (1024.0 * 1024.0), sizeof(struct particle_data));
+ }
+
+ if(All.MaxPartSph > 0)
+ {
+ bytes_tot = 0;
+
+ if(!(SphP = malloc(bytes = All.MaxPartSph * sizeof(struct sph_particle_data))))
+ {
+ printf("failed to allocate memory for `SphP' (%g MB) %d.\n", bytes / (1024.0 * 1024.0), sizeof(struct sph_particle_data));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(ThisTask == 0)
+ printf("Allocated %g MByte for storage of SPH data. %d\n\n", bytes_tot / (1024.0 * 1024.0), sizeof(struct sph_particle_data));
+ }
+
+#ifdef STELLAR_PROP
+ if(All.MaxPartStars > 0)
+ {
+ bytes_tot = 0;
+
+ if(!(StP = malloc(bytes = All.MaxPartStars * sizeof(struct st_particle_data))))
+ {
+ printf("failed to allocate memory for `StP' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(ThisTask == 0)
+ printf("\nAllocated %g MByte for star properties storage. %d\n\n", bytes_tot / (1024.0 * 1024.0), sizeof(struct st_particle_data));
+ }
+#endif
+
+}
+
+
+
+
+/*! This routine frees the memory for the particle storage. Note: We don't
+ * actually bother to call it in the code... When the program terminats,
+ * the memory will be automatically freed by the operating system.
+ */
+void free_memory(void)
+{
+
+#ifdef STELLAR_PROP
+ if(All.MaxPartStars > 0)
+ free(StP);
+#endif
+
+ if(All.MaxPartSph > 0)
+ free(SphP);
+
+ if(All.MaxPart > 0)
+ free(P);
+}
+
diff --git a/allvars.c b/allvars.c
new file mode 100644
index 0000000..895ed9e
--- /dev/null
+++ b/allvars.c
@@ -0,0 +1,329 @@
+/*! \file allvars.c
+ * \brief provides instances of all global variables.
+ */
+
+#include <stdio.h>
+#include <gsl/gsl_rng.h>
+#include "tags.h"
+#include "allvars.h"
+
+int SetMinTimeStepForActives=0;
+
+int ThisTask; /*!< the rank of the local processor */
+int NTask; /*!< number of processors */
+int PTask; /*!< smallest integer such that NTask <= 2^PTask */
+
+int NumPart; /*!< number of particles on the LOCAL processor */
+int N_gas; /*!< number of gas particles on the LOCAL processor */
+#if defined(SFR) || defined(STELLAR_PROP)
+int N_stars; /*!< number of stars particle on the LOCAL processor */
+#endif
+#ifdef MULTIPHASE
+int N_sph;
+int N_sticky;
+int N_stickyflaged;
+int N_dark;
+int NumColPotLocal; /*!< local number of potentially collisional particles */
+int NumColPot; /*!< total number of potentially collisional particles */
+int NumColLocal; /*!< local number of collisions */
+int NumCol; /*!< total number of collisions */
+int NumNoColLocal;
+int NumNoCol;
+#endif
+long long Ntype[6]; /*!< total number of particles of each type */
+int NtypeLocal[6]; /*!< local number of particles of each type */
+
+int NumForceUpdate; /*!< number of active particles on local processor in current timestep */
+int NumSphUpdate; /*!< number of active SPH particles on local processor in current timestep */
+#ifdef CHIMIE
+int NumStUpdate;
+#endif
+
+double CPUThisRun; /*!< Sums the CPU time for the process (current submission only) */
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+double CPU_Gravity;
+#endif
+
+int RestartFlag; /*!< taken from command line used to start code. 0 is normal start-up from
+ initial conditions, 1 is resuming a run from a set of restart files, while 2
+ marks a restart from a snapshot file. */
+
+char *Exportflag; /*!< Buffer used for flagging whether a particle needs to be exported to another process */
+
+int *Ngblist; /*!< Buffer to hold indices of neighbours retrieved by the neighbour search routines */
+
+int TreeReconstructFlag; /*!< Signals that a new tree needs to be constructed */
+#ifdef SFR
+int RearrangeParticlesFlag;/*!< Signals that particles must be rearanged */
+#endif
+
+
+int Flag_FullStep; /*!< This flag signals that the current step involves all particles */
+
+
+gsl_rng *random_generator; /*!< the employed random number generator of the GSL library */
+
+double RndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+
+#ifdef SFR
+double StarFormationRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef FEEDBACK_WIND
+double FeedbackWindRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE
+double ChimieRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+double ChimieKineticFeedbackRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+
+double DomainCorner[3]; /*!< gives the lower left corner of simulation volume */
+double DomainCenter[3]; /*!< gives the center of simulation volume */
+double DomainLen; /*!< gives the (maximum) side-length of simulation volume */
+double DomainFac; /*!< factor used for converting particle coordinates to a Peano-Hilbert mesh covering the simulation volume */
+int DomainMyStart; /*!< first domain mesh cell that resides on the local processor */
+int DomainMyLast; /*!< last domain mesh cell that resides on the local processor */
+int *DomainStartList; /*!< a table that lists the first domain mesh cell for all processors */
+int *DomainEndList; /*!< a table that lists the last domain mesh cell for all processors */
+double *DomainWork; /*!< a table that gives the total "work" due to the particles stored by each processor */
+int *DomainCount; /*!< a table that gives the total number of particles held by each processor */
+int *DomainCountSph; /*!< a table that gives the total number of SPH particles held by each processor */
+
+int *DomainTask; /*!< this table gives for each leaf of the top-level tree the processor it was assigned to */
+int *DomainNodeIndex; /*!< this table gives for each leaf of the top-level tree the corresponding node of the gravitational tree */
+FLOAT *DomainTreeNodeLen; /*!< this table gives for each leaf of the top-level tree the side-length of the corresponding node of the gravitational tree */
+FLOAT *DomainHmax; /*!< this table gives for each leaf of the top-level tree the maximum SPH smoothing length among the particles of the corresponding node of the gravitational tree */
+
+struct DomainNODE
+ *DomainMoment; /*!< this table stores for each node of the top-level tree corresponding node data from the gravitational tree */
+
+peanokey *DomainKeyBuf; /*!< this points to a buffer used during the exchange of particle data */
+
+peanokey *Key; /*!< a table used for storing Peano-Hilbert keys for particles */
+peanokey *KeySorted; /*!< holds a sorted table of Peano-Hilbert keys for all particles, used to construct top-level tree */
+
+
+int NTopnodes; /*!< total number of nodes in top-level tree */
+int NTopleaves; /*!< number of leaves in top-level tree. Each leaf can be assigned to a different processor */
+
+struct topnode_data
+ *TopNodes; /*!< points to the root node of the top-level tree */
+
+
+double TimeOfLastTreeConstruction; /*!< holds what it says, only used in connection with FORCETEST */
+
+
+
+
+/* variables for input/output, usually only used on process 0 */
+
+char ParameterFile[MAXLEN_FILENAME]; /*!< file name of parameterfile used for starting the simulation */
+
+FILE *FdInfo; /*!< file handle for info.txt log-file. */
+FILE *FdLog ; /*!< file handle for log.txt log-file. */
+FILE *FdEnergy; /*!< file handle for energy.txt log-file. */
+#ifdef SYSTEMSTATISTICS
+FILE *FdSystem;
+#endif
+FILE *FdTimings; /*!< file handle for timings.txt log-file. */
+FILE *FdCPU; /*!< file handle for cpu.txt log-file. */
+
+#ifdef FORCETEST
+FILE *FdForceTest; /*!< file handle for forcetest.txt log-file. */
+#endif
+
+#ifdef SFR
+FILE *FdSfr; /*!< file handle for sfr.txt log-file. */
+#endif
+
+#ifdef CHIMIE
+FILE *FdChimie; /*!< file handle for chimie log-file. */
+#endif
+
+#ifdef MULTIPHASE
+FILE *FdPhase; /*!< file handle for phase.txt log-file. */
+FILE *FdSticky; /*!< file handle for sticky.txt log-file. */
+#endif
+
+#ifdef AGN_ACCRETION
+FILE *FdAccretion; /*!< file handle for accretion.txt log-file. */
+#endif
+
+#ifdef BONDI_ACCRETION
+FILE *FdBondi; /*!< file handle for bondi.txt log-file. */
+#endif
+
+#ifdef BUBBLES
+FILE *FdBubble; /*!< file handle for bubble.txt log-file. */
+#endif
+
+double DriftTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological drift factors */
+double GravKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for gravitational forces */
+double HydroKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for hydrodynmical forces */
+
+void *CommBuffer; /*!< points to communication buffer, which is used in the domain decomposition, the
+ parallel tree-force computation, the SPH routines, etc. */
+
+
+/*! This structure contains data which is the SAME for all tasks (mostly code parameters read from the
+ * parameter file). Holding this data in a structure is convenient for writing/reading the restart file, and
+ * it allows the introduction of new global variables in a simple way. The only thing to do is to introduce
+ * them into this structure.
+ */
+struct global_data_all_processes
+ All;
+
+
+
+/*! This structure holds all the information that is
+ * stored for each particle of the simulation.
+ */
+struct particle_data
+ *P, /*!< holds particle data on local processor */
+ *DomainPartBuf; /*!< buffer for particle data used in domain decomposition */
+
+
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+struct sph_particle_data
+ *SphP, /*!< holds SPH particle data on local processor */
+ *DomainSphBuf; /*!< buffer for SPH particle data in domain decomposition */
+
+
+#ifdef STELLAR_PROP
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+struct st_particle_data
+ *StP, /*!< holds ST particle data on local processor */
+ *DomainStBuf; /*!< buffer for ST particle data in domain decomposition */
+#endif
+
+
+/* Variables for Tree
+ */
+
+int MaxNodes; /*!< maximum allowed number of internal nodes */
+int Numnodestree; /*!< number of (internal) nodes in each tree */
+
+struct NODE
+ *Nodes_base, /*!< points to the actual memory allocted for the nodes */
+ *Nodes; /*!< this is a pointer used to access the nodes which is shifted such that Nodes[All.MaxPart]
+ gives the first allocated node */
+
+
+int *Nextnode; /*!< gives next node in tree walk */
+int *Father; /*!< gives parent node in tree */
+
+
+struct extNODE /*!< this structure holds additional tree-node information which is not needed in the actual gravity computation */
+ *Extnodes_base, /*!< points to the actual memory allocted for the extended node information */
+ *Extnodes; /*!< provides shifted access to extended node information, parallel to Nodes/Nodes_base */
+
+
+
+
+
+/*! Header for the standard file format.
+ */
+struct io_header
+ header; /*!< holds header for snapshot files */
+
+#ifdef CHIMIE_EXTRAHEADER
+/*! Header for the chimie part.
+ */
+struct io_chimie_extraheader
+ chimie_extraheader;
+#endif
+
+
+char Tab_IO_Labels[IO_NBLOCKS][4]; /*<! This table holds four-byte character tags used for fileformat 2 */
+
+
+
+/* global state of system, used for global statistics
+ */
+struct state_of_system
+ SysState; /*<! Structure for storing some global statistics about the simulation. */
+
+/*! This structure contains data related to the energy budget.
+ These values are different for each task. It need to be stored
+ in the restart flag.
+ */
+struct local_state_of_system
+ LocalSysState; /*<! Structure for storing some local statistics about the simulation. */
+
+
+
+/* Various structures for communication
+ */
+struct gravdata_in
+ *GravDataIn, /*!< holds particle data to be exported to other processors */
+ *GravDataGet, /*!< holds particle data imported from other processors */
+ *GravDataResult, /*!< holds the partial results computed for imported particles. Note: We use GravDataResult = GravDataGet, such that the result replaces the imported data */
+ *GravDataOut; /*!< holds partial results received from other processors. This will overwrite the GravDataIn array */
+
+struct gravdata_index
+ *GravDataIndexTable; /*!< the particles to be exported are grouped by task-number. This table allows the results to be disentangled again and to be assigned to the correct particle */
+
+
+
+struct densdata_in
+ *DensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *DensDataGet; /*!< holds imported particle data for SPH density computation */
+
+struct densdata_out
+ *DensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *DensDataPartialResult; /*!< imported partial SPH density results from other processors */
+
+
+
+struct hydrodata_in
+ *HydroDataIn, /*!< holds particle data for SPH hydro-force computation to be exported to other processors */
+ *HydroDataGet; /*!< holds imported particle data for SPH hydro-force computation */
+
+struct hydrodata_out
+ *HydroDataResult, /*!< stores the locally computed SPH hydro results for imported particles */
+ *HydroDataPartialResult; /*!< imported partial SPH hydro-force results from other processors */
+
+
+#ifdef MULTIPHASE
+
+struct stickydata_in
+ *StickyDataIn,
+ *StickyDataGet;
+
+struct stickydata_out
+ *StickyDataResult,
+ *StickyDataPartialResult;
+
+struct Sticky_index *StickyIndex;
+#endif
+
+#ifdef CHIMIE
+struct chimiedata_in
+ *ChimieDataIn, /*!< holds particle data for Chimie computation to be exported to other processors */
+ *ChimieDataGet; /*!< holds imported particle data for Chimie computation */
+
+struct chimiedata_out
+ *ChimieDataResult, /*!< stores the locally computed Chimie results for imported particles */
+ *ChimieDataPartialResult; /*!< imported partial Chimie results from other processors */
+
+
+struct starsdensdata_in
+ *StarsDensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *StarsDensDataGet; /*!< holds imported particle data for SPH density computation */
+
+struct starsdensdata_out
+ *StarsDensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *StarsDensDataPartialResult; /*!< imported partial SPH density results from other processors */
+
+#endif
diff --git a/allvars.h b/allvars.h
new file mode 100644
index 0000000..3d6cc5d
--- /dev/null
+++ b/allvars.h
@@ -0,0 +1,1614 @@
+
+/*! \file allvars.h
+ * \brief declares global variables.
+ *
+ * This file declares all global variables. Further variables should be added here, and declared as
+ * 'extern'. The actual existence of these variables is provided by the file 'allvars.c'. To produce
+ * 'allvars.c' from 'allvars.h', do the following:
+ *
+ * - Erase all #define's, typedef's, and enum's
+ * - add #include "allvars.h", delete the #ifndef ALLVARS_H conditional
+ * - delete all keywords 'extern'
+ * - delete all struct definitions enclosed in {...}, e.g.
+ * "extern struct global_data_all_processes {....} All;"
+ * becomes "struct global_data_all_processes All;"
+ */
+
+#ifndef ALLVARS_H
+#define ALLVARS_H
+
+#include <stdio.h>
+#include <gsl/gsl_rng.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+#include <gsl/gsl_integration.h>
+#include "tags.h"
+
+#define GADGETVERSION "2.0" /*!< code version string */
+
+#define TIMEBASE (1<<28) /*!< The simulated timespan is mapped onto the integer interval [0,TIMESPAN],
+ * where TIMESPAN needs to be a power of 2. Note that (1<<28) corresponds to 2^29
+ */
+
+#define MAXTOPNODES 200000 /*!< Maximum number of nodes in the top-level tree used for domain decomposition */
+
+
+typedef long long peanokey; /*!< defines the variable type used for Peano-Hilbert keys */
+
+#define BITS_PER_DIMENSION 18 /*!< Bits per dimension available for Peano-Hilbert order.
+ Note: If peanokey is defined as type int, the allowed maximum is 10.
+ If 64-bit integers are used, the maximum is 21 */
+
+#define PEANOCELLS (((peanokey)1)<<(3*BITS_PER_DIMENSION)) /*!< The number of different Peano-Hilbert cells */
+
+
+#define RNDTABLE 3000 /*!< gives the length of a table with random numbers, refreshed at every timestep.
+ This is used to allow application of random numbers to a specific particle
+ in a way that is independent of the number of processors used. */
+#define MAX_REAL_NUMBER 1e37
+#define MIN_REAL_NUMBER 1e-37
+
+#define MAXLEN_FILENAME 100 /*!< Maximum number of characters for filenames (including the full path) */
+
+#ifdef ISOTHERM_EQS
+#define GAMMA (1.0) /*!< index for isothermal gas */
+#else
+#define GAMMA (5.0/3) /*!< adiabatic index of simulated gas */
+#endif
+
+#define GAMMA_MINUS1 (GAMMA-1)
+
+#define HYDROGEN_MASSFRAC 0.76 /*!< mass fraction of hydrogen, relevant only for radiative cooling */
+
+/* Some physical constants in cgs units */
+
+#define GRAVITY 6.672e-8 /*!< Gravitational constant (in cgs units) */
+#define SOLAR_MASS 1.989e33
+#define SOLAR_LUM 3.826e33
+#define RAD_CONST 7.565e-15
+#define AVOGADRO 6.0222e23
+#define BOLTZMANN 1.3806e-16
+#define GAS_CONST 8.31425e7
+#define C 2.9979e10
+#define PLANCK 6.6262e-27
+#define CM_PER_MPC 3.085678e24
+#define PROTONMASS 1.6726e-24
+#define ELECTRONMASS 9.10953e-28
+#define THOMPSON 6.65245e-25
+#define ELECTRONCHARGE 4.8032e-10
+#define HUBBLE 3.2407789e-18 /* in h/sec */
+#define YEAR_IN_SECOND 31536000.0 /* year in sec */
+
+
+#define PI 3.1415926535897931
+#define TWOPI 6.2831853071795862
+
+/* Some conversion factors */
+
+#define SEC_PER_MEGAYEAR 3.155e13
+#define SEC_PER_YEAR 3.155e7
+
+#ifndef ASMTH
+#define ASMTH 1.25 /*!< ASMTH gives the scale of the short-range/long-range force split in units of FFT-mesh cells */
+#endif
+
+#ifndef RCUT
+#define RCUT 4.5 /*!< RCUT gives the maximum distance (in units of the scale used for the force split) out to
+ which short-range forces are evaluated in the short-range tree walk. */
+#endif
+
+#define MAX_NGB 20000 /*!< defines maximum length of neighbour list */
+
+#define MAXLEN_OUTPUTLIST 500 /*!< maxmimum number of entries in list of snapshot output times */
+
+#define DRIFT_TABLE_LENGTH 1000 /*!< length of the lookup table used to hold the drift and kick factors */
+
+#define MAXITER 1000 /*!< maxmimum number of steps for SPH neighbour iteration */
+
+
+#ifdef DOUBLEPRECISION /*!< If defined, the variable type FLOAT is set to "double", otherwise to FLOAT */
+#define FLOAT double
+#else
+#define FLOAT float
+#endif
+
+
+#ifndef TWODIMS
+#define NUMDIMS 3 /*!< For 3D-normalized kernel */
+#define KERNEL_COEFF_1 2.546479089470 /*!< Coefficients for SPH spline kernel and its derivative */
+#define KERNEL_COEFF_2 15.278874536822
+#define KERNEL_COEFF_3 45.836623610466
+#define KERNEL_COEFF_4 30.557749073644
+#define KERNEL_COEFF_5 5.092958178941
+#define KERNEL_COEFF_6 (-15.278874536822)
+#define NORM_COEFF 4.188790204786 /*!< Coefficient for kernel normalization. Note: 4.0/3 * PI = 4.188790204786 */
+#else
+#define NUMDIMS 2 /*!< For 2D-normalized kernel */
+#define KERNEL_COEFF_1 (5.0/7*2.546479089470) /*!< Coefficients for SPH spline kernel and its derivative */
+#define KERNEL_COEFF_2 (5.0/7*15.278874536822)
+#define KERNEL_COEFF_3 (5.0/7*45.836623610466)
+#define KERNEL_COEFF_4 (5.0/7*30.557749073644)
+#define KERNEL_COEFF_5 (5.0/7*5.092958178941)
+#define KERNEL_COEFF_6 (5.0/7*(-15.278874536822))
+#define NORM_COEFF M_PI /*!< Coefficient for kernel normalization. */
+#endif
+
+
+#ifdef MULTIPHASE
+#define GAS_SPH 0
+#define GAS_STICKY 1
+#define GAS_DARK 2
+#endif
+
+#if defined(SFR) || defined(STELLAR_PROP)
+#define ST 1
+#endif
+
+
+
+#ifdef CHIMIE
+#define NELEMENTS 5
+#define MAXNELEMENTS 64
+#define FIRST_ELEMENT "Fe"
+#define FE 0
+#endif
+
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+#define VELOCITY_DISPERSION_SIZE 3
+#endif
+
+extern int SetMinTimeStepForActives;
+
+extern int ThisTask; /*!< the rank of the local processor */
+extern int NTask; /*!< number of processors */
+extern int PTask; /*!< smallest integer such that NTask <= 2^PTask */
+
+extern int NumPart; /*!< number of particles on the LOCAL processor */
+extern int N_gas; /*!< number of gas particles on the LOCAL processor */
+#if defined(SFR) || defined(STELLAR_PROP)
+extern int N_stars; /*!< number of stars particle on the LOCAL processor */
+#endif
+#ifdef MULTIPHASE
+extern int N_sph;
+extern int N_sticky;
+extern int N_stickyflaged;
+extern int N_dark;
+extern int NumColPotLocal; /*!< local number of potentially collisional particles */
+extern int NumColPot; /*!< total number of potentially collisional particles */
+extern int NumColLocal; /*!< local number of collisions */
+extern int NumCol; /*!< total number of collisions */
+extern int NumNoColLocal;
+extern int NumNoCol;
+#endif
+extern long long Ntype[6]; /*!< total number of particles of each type */
+extern int NtypeLocal[6]; /*!< local number of particles of each type */
+
+extern int NumForceUpdate; /*!< number of active particles on local processor in current timestep */
+extern int NumSphUpdate; /*!< number of active SPH particles on local processor in current timestep */
+#ifdef CHIMIE
+extern int NumStUpdate;
+#endif
+
+extern double CPUThisRun; /*!< Sums the CPU time for the process (current submission only) */
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+extern double CPU_Gravity;
+#endif
+
+extern int RestartFlag; /*!< taken from command line used to start code. 0 is normal start-up from
+ initial conditions, 1 is resuming a run from a set of restart files, while 2
+ marks a restart from a snapshot file. */
+
+extern char *Exportflag; /*!< Buffer used for flagging whether a particle needs to be exported to another process */
+
+extern int *Ngblist; /*!< Buffer to hold indices of neighbours retrieved by the neighbour search routines */
+
+extern int TreeReconstructFlag; /*!< Signals that a new tree needs to be constructed */
+#ifdef SFR
+extern int RearrangeParticlesFlag;/*!< Signals that particles must be rearanged */
+#endif
+
+extern int Flag_FullStep; /*!< This flag signals that the current step involves all particles */
+
+
+extern gsl_rng *random_generator; /*!< the employed random number generator of the GSL library */
+
+extern double RndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+
+#ifdef SFR
+extern double StarFormationRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+#ifdef FEEDBACK_WIND
+extern double FeedbackWindRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE
+extern double ChimieRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+extern double ChimieKineticFeedbackRndTable[RNDTABLE]; /*!< Hold a table with random numbers, refreshed every timestep */
+#endif
+
+
+extern double DomainCorner[3]; /*!< gives the lower left corner of simulation volume */
+extern double DomainCenter[3]; /*!< gives the center of simulation volume */
+extern double DomainLen; /*!< gives the (maximum) side-length of simulation volume */
+extern double DomainFac; /*!< factor used for converting particle coordinates to a Peano-Hilbert mesh covering the simulation volume */
+extern int DomainMyStart; /*!< first domain mesh cell that resides on the local processor */
+extern int DomainMyLast; /*!< last domain mesh cell that resides on the local processor */
+extern int *DomainStartList; /*!< a table that lists the first domain mesh cell for all processors */
+extern int *DomainEndList; /*!< a table that lists the last domain mesh cell for all processors */
+extern double *DomainWork; /*!< a table that gives the total "work" due to the particles stored by each processor */
+extern int *DomainCount; /*!< a table that gives the total number of particles held by each processor */
+extern int *DomainCountSph; /*!< a table that gives the total number of SPH particles held by each processor */
+
+extern int *DomainTask; /*!< this table gives for each leaf of the top-level tree the processor it was assigned to */
+extern int *DomainNodeIndex; /*!< this table gives for each leaf of the top-level tree the corresponding node of the gravitational tree */
+extern FLOAT *DomainTreeNodeLen; /*!< this table gives for each leaf of the top-level tree the side-length of the corresponding node of the gravitational tree */
+extern FLOAT *DomainHmax; /*!< this table gives for each leaf of the top-level tree the maximum SPH smoothing length among the particles of the corresponding node of the gravitational tree */
+
+extern struct DomainNODE
+{
+ FLOAT s[3]; /*!< center-of-mass coordinates */
+ FLOAT vs[3]; /*!< center-of-mass velocities */
+ FLOAT mass; /*!< mass of node */
+#ifdef STELLAR_FLUX
+ FLOAT starlum; /*!< star luminosity of node */
+#endif
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ int bitflags; /*!< this bit-field encodes the particle type with the largest softening among the particles of the nodes, and whether there are particles with different softening in the node */
+#else
+ FLOAT maxsoft; /*!< hold the maximum gravitational softening of particles in the
+ node if the ADAPTIVE_GRAVSOFT_FORGAS option is selected */
+#endif
+#endif
+}
+ *DomainMoment; /*!< this table stores for each node of the top-level tree corresponding node data from the gravitational tree */
+
+extern peanokey *DomainKeyBuf; /*!< this points to a buffer used during the exchange of particle data */
+
+extern peanokey *Key; /*!< a table used for storing Peano-Hilbert keys for particles */
+extern peanokey *KeySorted; /*!< holds a sorted table of Peano-Hilbert keys for all particles, used to construct top-level tree */
+
+
+extern int NTopnodes; /*!< total number of nodes in top-level tree */
+extern int NTopleaves; /*!< number of leaves in top-level tree. Each leaf can be assigned to a different processor */
+
+extern struct topnode_data
+{
+ int Daughter; /*!< index of first daughter cell (out of 8) of top-level node */
+ int Pstart; /*!< for the present top-level node, this gives the index of the first node in the concatenated list of topnodes collected from all processors */
+ int Blocks; /*!< for the present top-level node, this gives the number of corresponding nodes in the concatenated list of topnodes collected from all processors */
+ int Leaf; /*!< if the node is a leaf, this gives its number when all leaves are traversed in Peano-Hilbert order */
+ peanokey Size; /*!< number of Peano-Hilbert mesh-cells represented by top-level node */
+ peanokey StartKey; /*!< first Peano-Hilbert key in top-level node */
+ long long Count; /*!< counts the number of particles in this top-level node */
+}
+ *TopNodes; /*!< points to the root node of the top-level tree */
+
+
+extern double TimeOfLastTreeConstruction; /*!< holds what it says, only used in connection with FORCETEST */
+
+
+
+/* variables for input/output, usually only used on process 0 */
+
+extern char ParameterFile[MAXLEN_FILENAME]; /*!< file name of parameterfile used for starting the simulation */
+
+extern FILE *FdInfo; /*!< file handle for info.txt log-file. */
+extern FILE *FdLog; /*!< file handle for log.txt log-file. */
+extern FILE *FdEnergy; /*!< file handle for energy.txt log-file. */
+#ifdef SYSTEMSTATISTICS
+extern FILE *FdSystem;
+#endif
+extern FILE *FdTimings; /*!< file handle for timings.txt log-file. */
+extern FILE *FdCPU; /*!< file handle for cpu.txt log-file. */
+
+#ifdef FORCETEST
+extern FILE *FdForceTest; /*!< file handle for forcetest.txt log-file. */
+#endif
+
+#ifdef SFR
+extern FILE *FdSfr; /*!< file handle for sfr.txt log-file. */
+#endif
+
+#ifdef CHIMIE
+extern FILE *FdChimie; /*!< file handle for chimie log-file. */
+#endif
+
+#ifdef MULTIPHASE
+extern FILE *FdPhase; /*!< file handle for pase.txt log-file. */
+extern FILE *FdSticky; /*!< file handle for sticky.txt log-file. */
+#endif
+
+#ifdef AGN_ACCRETION
+extern FILE *FdAccretion; /*!< file handle for accretion.txt log-file. */
+#endif
+
+#ifdef BONDI_ACCRETION
+extern FILE *FdBondi; /*!< file handle for bondi.txt log-file. */
+#endif
+
+#ifdef BUBBLES
+extern FILE *FdBubble; /*!< file handle for bubble.txt log-file. */
+#endif
+
+extern double DriftTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological drift factors */
+extern double GravKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for gravitational forces */
+extern double HydroKickTable[DRIFT_TABLE_LENGTH]; /*!< table for the cosmological kick factor for hydrodynmical forces */
+
+extern void *CommBuffer; /*!< points to communication buffer, which is used in the domain decomposition, the
+ parallel tree-force computation, the SPH routines, etc. */
+
+
+
+/*! This structure contains data which is the SAME for all tasks (mostly code parameters read from the
+ * parameter file). Holding this data in a structure is convenient for writing/reading the restart file, and
+ * it allows the introduction of new global variables in a simple way. The only thing to do is to introduce
+ * them into this structure.
+ */
+extern struct global_data_all_processes
+{
+ long long TotNumPart; /*!< total particle numbers (global value) */
+ long long TotN_gas; /*!< total gas particle number (global value) */
+#if defined(SFR) || defined(STELLAR_PROP)
+ long long TotN_stars; /*!< total stars particle number (global value) */
+#endif
+#ifdef MULTIPHASE
+ long long TotN_sph; /*!< total sph particle number (global value) */
+ long long TotN_sticky; /*!< total sticky particle number (global value) */
+ long long TotN_stickyflaged; /*!< total sticky flaged particle number (global value) */
+ long long TotN_dark; /*!< total dark particle number (global value) */
+#endif
+
+ int MaxPart; /*!< This gives the maxmimum number of particles that can be stored on one processor. */
+ int MaxPartSph; /*!< This gives the maxmimum number of SPH particles that can be stored on one processor. */
+#ifdef STELLAR_PROP
+ int MaxPartStars; /*!< This gives the maxmimum number of Star particles that can be stored on one processor. */
+#endif
+ double BoxSize; /*!< Boxsize in case periodic boundary conditions are used */
+
+ int ICFormat; /*!< selects different versions of IC file-format */
+
+ int SnapFormat; /*!< selects different versions of snapshot file-formats */
+
+ int NumFilesPerSnapshot; /*!< number of files in multi-file snapshot dumps */
+ int NumFilesWrittenInParallel;/*!< maximum number of files that may be written simultaneously when
+ writing/reading restart-files, or when writing snapshot files */
+
+ int BufferSize; /*!< size of communication buffer in MB */
+ int BunchSizeForce; /*!< number of particles fitting into the buffer in the parallel tree-force algorithm */
+ int BunchSizeDensity; /*!< number of particles fitting into the communication buffer in the density computation */
+ int BunchSizeHydro; /*!< number of particles fitting into the communication buffer in the SPH hydrodynamical force computation */
+ int BunchSizeDomain; /*!< number of particles fitting into the communication buffer in the domain decomposition */
+#ifdef MULTIPHASE
+ int BunchSizeSticky; /*!< number of particles fitting into the communication buffer in the Chimie computation */
+#endif
+#ifdef CHIMIE
+ int BunchSizeChimie; /*!< number of particles fitting into the communication buffer in the Chimie computation */
+ int BunchSizeStarsDensity; /*!< number of particles fitting into the communication buffer in the star density computation */
+#endif
+
+
+ double PartAllocFactor; /*!< in order to maintain work-load balance, the particle load will usually
+ NOT be balanced. Each processor allocates memory for PartAllocFactor times
+ the average number of particles to allow for that */
+
+ double TreeAllocFactor; /*!< Each processor allocates a number of nodes which is TreeAllocFactor times
+ the maximum(!) number of particles. Note: A typical local tree for N
+ particles needs usually about ~0.65*N nodes. */
+
+ /* some SPH parameters */
+
+ double DesNumNgb; /*!< Desired number of SPH neighbours */
+ double MaxNumNgbDeviation; /*!< Maximum allowed deviation neighbour number */
+
+ double ArtBulkViscConst; /*!< Sets the parameter \f$\alpha\f$ of the artificial viscosity */
+ double InitGasTemp; /*!< may be used to set the temperature in the IC's */
+ double MinGasTemp; /*!< may be used to set a floor for the gas temperature */
+ double MinEgySpec; /*!< the minimum allowed temperature expressed as energy per unit mass */
+
+ /* Usefull constants */
+ double Boltzmann;
+ double ProtonMass;
+ double mumh;
+
+#ifdef COOLING
+ /* Cooling parameters */
+ double *logT;
+ double *logL;
+
+ gsl_interp_accel *acc_cooling_spline;
+ gsl_spline *cooling_spline;
+
+ double CoolingType;
+ char CoolingFile[MAXLEN_FILENAME]; /*!< cooling file */
+ double CutofCoolingTemperature;
+ double InitGasMetallicity;
+
+ /*
+ new metal dependent cooling
+ */
+
+ double CoolingParameters_zmin;
+ double CoolingParameters_zmax;
+ double CoolingParameters_slz;
+ double CoolingParameters_tmin;
+ double CoolingParameters_tmax;
+ double CoolingParameters_slt;
+ double CoolingParameters_FeHSolar;
+ double CoolingParameters_cooling_data_max;
+ double CoolingParameters_cooling_data[9][171];
+ int CoolingParameters_p;
+ int CoolingParameters_q;
+
+#endif
+
+
+#ifdef CHIMIE
+ char ChimieParameterFile[MAXLEN_FILENAME]; /*!< chimie parameter file */
+ double ChimieSupernovaEnergy;
+ double ChimieKineticFeedbackFraction;
+ double ChimieWindSpeed;
+ double ChimieWindTime;
+ double ChimieThermalTime;
+ double ChimieMaxSizeTimestep;
+#endif
+
+
+
+
+
+#if !defined (HEATING_PE)
+ double HeatingPeElectronFraction;
+#endif
+#if !defined (HEATING_PE) || defined (STELLAR_FLUX) || defined (EXTERNAL_FLUX)
+ double HeatingPeSolarEnergyDensity;
+#endif
+#if !defined (HEATING_PE) || defined (STELLAR_FLUX)
+ double HeatingPeLMRatioGas;
+ double HeatingPeLMRatioHalo;
+ double HeatingPeLMRatioDisk;
+ double HeatingPeLMRatioBulge;
+ double HeatingPeLMRatioStars;
+ double HeatingPeLMRatioBndry;
+ double HeatingPeLMRatio[6];
+#endif
+
+#ifdef EXTERNAL_FLUX
+ double HeatingExternalFLuxEnergyDensity;
+#endif
+
+#ifdef MULTIPHASE
+ double CriticalTemperature;
+ double CriticalEgySpec;
+ double CriticalNonCollisionalTemperature;
+ double CriticalNonCollisionalEgySpec;
+#ifdef COLDGAS_CYCLE
+ double ColdGasCycleTransitionTime;
+ double ColdGasCycleTransitionParameter;
+#endif
+#endif
+
+
+#ifdef MULTIPHASE
+ /* some STICKY parameters */
+ int StickyUseGridForCollisions;
+ double StickyTime; /*!< Cooling time of sticky particle collision */
+ double StickyCollisionTime;
+ double StickyLastCollisionTime;
+ double StickyIdleTime;
+ double StickyMinVelocity;
+ double StickyMaxVelocity;
+ int StickyGridNx;
+ int StickyGridNy;
+ int StickyGridNz;
+ double StickyGridXmin;
+ double StickyGridXmax;
+ double StickyGridYmin;
+ double StickyGridYmax;
+ double StickyGridZmin;
+ double StickyGridZmax;
+ double StickyLambda;
+ double StickyDensity;
+ double StickyDensityPower;
+ double StickyBetaR;
+ double StickyBetaT;
+ double StickyRsphFact; /*!< Fraction of the sph radius used in sticky particle */
+#endif
+
+#ifdef OUTERPOTENTIAL
+
+#ifdef NFW
+ double HaloConcentration;
+ double HaloMass;
+ double GasMassFraction;
+ double NFWPotentialCte;
+ double Rs;
+#endif
+
+#ifdef PLUMMER
+ double PlummerMass;
+ double PlummerSoftenning;
+ double PlummerPotentialCte;
+#endif
+
+
+#ifdef PISOTHERM
+ double Rho0;
+ double Rc;
+ double PisothermPotentialCte;
+ double GasMassFraction;
+ double PotentialInf;
+ gsl_function PotentialF;
+ gsl_integration_workspace *Potentialw;
+#endif
+
+#ifdef CORIOLIS
+ double CoriolisOmegaX;
+ double CoriolisOmegaY;
+ double CoriolisOmegaZ;
+ double CoriolisOmegaX0;
+ double CoriolisOmegaY0;
+ double CoriolisOmegaZ0;
+#endif
+
+#endif
+
+#ifdef SFR
+ int StarFormationNStarsFromGas;
+ double StarFormationStarMass;
+ double StarFormationMgMsFraction;
+ int StarFormationType;
+ double StarFormationCstar;
+ double StarFormationTime;
+ double StarFormationDensity;
+ double StarFormationTemperature;
+ double ThresholdDensity;
+#endif
+
+#ifdef FEEDBACK
+ double SupernovaTime;
+#endif
+
+#ifdef FEEDBACK_WIND
+ double SupernovaWindEgySpecPerMassUnit;
+ double SupernovaWindFractionInEgyKin;
+ double SupernovaWindParameter;
+ double SupernovaWindSpeed;
+ double SupernovaWindIntAccuracy;
+#endif
+
+#ifdef AGN_ACCRETION
+ double TimeBetAccretion;
+ double AccretionRadius;
+ double AGNFactor;
+ double MinMTotInRa;
+
+
+ double TimeLastAccretion;
+ double LastMTotInRa;
+ double MTotInRa;
+ double dMTotInRa;
+#endif
+
+#ifdef BUBBLES
+ char BubblesInitFile[MAXLEN_FILENAME]; /*!< bubble file */
+ double *BubblesTime;
+ double *BubblesD;
+ double *BubblesR;
+ double *BubblesE;
+ double *BubblesA;
+ double *BubblesB;
+ int BubblesIndex;
+
+ double BubblesAlpha;
+ double BubblesBeta;
+ double BubblesDelta;
+ double BubblesRadiusFactor;
+ double EnergyBubbles;
+#endif
+
+#ifdef AGN_HEATING
+ double AGNHeatingPower;
+ double AGNHeatingRmax;
+#endif
+
+#ifdef BONDI_ACCRETION
+ double BondiEfficiency;
+ double BondiBlackHoleMass;
+ double BondiHsmlFactor;
+ double BondiPower;
+ double BondiTimeBet;
+ double BondiTimeLast;
+#endif
+
+#if defined (AGN_ACCRETION) || defined (BONDI_ACCRETION)
+ double LightSpeed;
+#endif
+
+
+ /* some force counters */
+
+ long long TotNumOfForces; /*!< counts total number of force computations */
+ long long NumForcesSinceLastDomainDecomp; /*!< count particle updates since last domain decomposition */
+
+ /* system of units */
+
+ double G; /*!< Gravity-constant in internal units */
+ double UnitTime_in_s; /*!< factor to convert internal time unit to seconds/h */
+ double UnitMass_in_g; /*!< factor to convert internal mass unit to grams/h */
+ double UnitVelocity_in_cm_per_s; /*!< factor to convert intqernal velocity unit to cm/sec */
+ double UnitLength_in_cm; /*!< factor to convert internal length unit to cm/h */
+ double UnitPressure_in_cgs; /*!< factor to convert internal pressure unit to cgs units (little 'h' still around!) */
+ double UnitDensity_in_cgs; /*!< factor to convert internal length unit to g/cm^3*h^2 */
+ double UnitCoolingRate_in_cgs; /*!< factor to convert internal cooling rate to cgs units */
+ double UnitEnergy_in_cgs; /*!< factor to convert internal energy to cgs units */
+ double UnitTime_in_Megayears; /*!< factor to convert internal time to megayears/h */
+ double GravityConstantInternal; /*!< If set to zero in the parameterfile, the internal value of the
+ gravitational constant is set to the Newtonian value based on the system of
+ units specified. Otherwise the value provided is taken as internal gravity constant G. */
+
+
+ /* Cosmological parameters */
+
+ double Hubble; /*!< Hubble-constant in internal units */
+ double Omega0; /*!< matter density in units of the critical density (at z=0)*/
+ double OmegaLambda; /*!< vaccum energy density relative to crictical density (at z=0) */
+ double OmegaBaryon; /*!< baryon density in units of the critical density (at z=0)*/
+ double HubbleParam; /*!< little `h', i.e. Hubble constant in units of 100 km/s/Mpc. Only needed to get absolute physical values for cooling physics */
+
+
+ /* Code options */
+
+ int ComovingIntegrationOn; /*!< flags that comoving integration is enabled */
+ int PeriodicBoundariesOn; /*!< flags that periodic boundaries are enabled */
+ int ResubmitOn; /*!< flags that automatic resubmission of job to queue system is enabled */
+ int TypeOfOpeningCriterion; /*!< determines tree cell-opening criterion: 0 for Barnes-Hut, 1 for relative criterion */
+ int TypeOfTimestepCriterion; /*!< gives type of timestep criterion (only 0 supported right now - unlike gadget-1.1) */
+ int OutputListOn; /*!< flags that output times are listed in a specified file */
+
+
+ /* Parameters determining output frequency */
+
+ int SnapshotFileCount; /*!< number of snapshot that is written next */
+ double TimeBetSnapshot; /*!< simulation time interval between snapshot files */
+ double TimeOfFirstSnapshot; /*!< simulation time of first snapshot files */
+ double CpuTimeBetRestartFile; /*!< cpu-time between regularly generated restart files */
+ double TimeLastRestartFile; /*!< cpu-time when last restart-file was written */
+ double TimeBetStatistics; /*!< simulation time interval between computations of energy statistics */
+ double TimeLastStatistics; /*!< simulation time when the energy statistics was computed the last time */
+ int NumCurrentTiStep; /*!< counts the number of system steps taken up to this point */
+
+
+ /* Current time of the simulation, global step, and end of simulation */
+
+ double Time; /*!< current time of the simulation */
+ double TimeBegin; /*!< time of initial conditions of the simulation */
+ double TimeStep; /*!< difference between current times of previous and current timestep */
+ double TimeMax; /*!< marks the point of time until the simulation is to be evolved */
+
+
+ /* variables for organizing discrete timeline */
+
+ double Timebase_interval; /*!< factor to convert from floating point time interval to integer timeline */
+ int Ti_Current; /*!< current time on integer timeline */
+ int Ti_nextoutput; /*!< next output time on integer timeline */
+#ifdef FLEXSTEPS
+ int PresentMinStep; /*!< If FLEXSTEPS is used, particle timesteps are chosen as multiples of the present minimum timestep. */
+ int PresentMaxStep; /*!< If FLEXSTEPS is used, this is the maximum timestep in timeline units, rounded down to the next power 2 division */
+#endif
+#ifdef PMGRID
+ int PM_Ti_endstep; /*!< begin of present long-range timestep */
+ int PM_Ti_begstep; /*!< end of present long-range timestep */
+#endif
+
+
+ /* Placement of PM grids */
+
+#ifdef PMGRID
+ double Asmth[2]; /*!< Gives the scale of the long-range/short-range split (in mesh-cells), both for the coarse and the high-res mesh */
+ double Rcut[2]; /*!< Gives the maximum radius for which the short-range force is evaluated with the tree (in mesh-cells), both for the coarse and the high-res mesh */
+ double Corner[2][3]; /*!< lower left corner of coarse and high-res PM-mesh */
+ double UpperCorner[2][3]; /*!< upper right corner of coarse and high-res PM-mesh */
+ double Xmintot[2][3]; /*!< minimum particle coordinates both for coarse and high-res PM-mesh */
+ double Xmaxtot[2][3]; /*!< maximum particle coordinates both for coarse and high-res PM-mesh */
+ double TotalMeshSize[2]; /*!< total extension of coarse and high-res PM-mesh */
+#endif
+
+
+ /* Variables that keep track of cumulative CPU consumption */
+
+ double TimeLimitCPU; /*!< CPU time limit as defined in parameterfile */
+ double CPU_TreeConstruction; /*!< time spent for constructing the gravitational tree */
+ double CPU_TreeWalk; /*!< actual time spent for pure tree-walks */
+ double CPU_Gravity; /*!< cumulative time used for gravity computation (tree-algorithm only) */
+ double CPU_Potential; /*!< time used for computing gravitational potentials */
+ double CPU_Domain; /*!< cumulative time spent for domain decomposition */
+ double CPU_Snapshot; /*!< time used for writing snapshot files */
+ double CPU_Total; /*!< cumulative time spent for domain decomposition */
+ double CPU_CommSum; /*!< accumulated time used for communication, and for collecting partial results, in tree-gravity */
+ double CPU_Imbalance; /*!< cumulative time lost accross all processors as work-load imbalance in gravitational tree */
+ double CPU_HydCompWalk; /*!< time used for actual SPH computations, including neighbour search */
+ double CPU_HydCommSumm; /*!< cumulative time used for communication in SPH, and for collecting partial results */
+ double CPU_HydImbalance; /*!< cumulative time lost due to work-load imbalance in SPH */
+ double CPU_Hydro; /*!< cumulative time spent for SPH related computations */
+#ifdef SFR
+ double CPU_StarFormation; /*!< cumulative time spent for star formation computations */
+#endif
+#ifdef CHIMIE
+ double CPU_Chimie; /*!< cumulative time spent for chimie computations */
+#endif
+#ifdef MULTIPHASE
+ double CPU_Sticky; /*!< cumulative time spent for sticky computations */
+#endif
+ double CPU_EnsureNgb; /*!< time needed to iterate on correct neighbour numbers */
+ double CPU_Predict; /*!< cumulative time to drift the system forward in time, including dynamic tree updates */
+ double CPU_TimeLine; /*!< time used for determining new timesteps, and for organizing the timestepping, including kicks of active particles */
+ double CPU_PM; /*!< time used for long-range gravitational force */
+ double CPU_Peano; /*!< time required to establish Peano-Hilbert order */
+
+
+#ifdef DETAILED_CPU_DOMAIN
+ double CPU_Domain_findExtend;
+ double CPU_Domain_determineTopTree;
+ double CPU_Domain_sumCost;
+ double CPU_Domain_findSplit;
+ double CPU_Domain_shiftSplit;
+ double CPU_Domain_countToGo;
+ double CPU_Domain_exchange;
+#endif
+#ifdef DETAILED_CPU_GRAVITY
+ double CPU_Gravity_TreeWalk1;
+ double CPU_Gravity_TreeWalk2;
+ double CPU_Gravity_CommSum1;
+ double CPU_Gravity_CommSum2;
+ double CPU_Gravity_Imbalance1;
+ double CPU_Gravity_Imbalance2;
+#endif
+
+#ifdef COOLING
+ double CPU_Cooling;
+#endif
+
+#ifdef DETAILED_CPU
+ double CPU_Leapfrog;
+ double CPU_Physics;
+ double CPU_Residual;
+ double CPU_Accel;
+ double CPU_Begrun;
+#endif
+
+
+
+
+
+
+ /* tree code opening criterion */
+
+ double ErrTolTheta; /*!< BH tree opening angle */
+ double ErrTolForceAcc; /*!< parameter for relative opening criterion in tree walk */
+
+
+ /* adjusts accuracy of time-integration */
+
+ double ErrTolIntAccuracy; /*!< accuracy tolerance parameter \f$ \eta \f$ for timestep criterion. The
+ timestep is \f$ \Delta t = \sqrt{\frac{2 \eta eps}{a}} \f$ */
+
+ double MinSizeTimestep; /*!< minimum allowed timestep. Normally, the simulation terminates if the
+ timestep determined by the timestep criteria falls below this limit. */
+ double MaxSizeTimestep; /*!< maximum allowed timestep */
+
+ double MaxRMSDisplacementFac; /*!< this determines a global timestep criterion for cosmological simulations
+ in comoving coordinates. To this end, the code computes the rms velocity
+ of all particles, and limits the timestep such that the rms displacement
+ is a fraction of the mean particle separation (determined from the
+ particle mass and the cosmological parameters). This parameter specifies
+ this fraction. */
+
+ double CourantFac; /*!< SPH-Courant factor */
+
+
+ /* frequency of tree reconstruction/domain decomposition */
+
+ double TreeDomainUpdateFrequency; /*!< controls frequency of domain decompositions */
+
+
+ /* Gravitational and hydrodynamical softening lengths (given in terms of an `equivalent' Plummer softening length).
+ * Five groups of particles are supported 0="gas", 1="halo", 2="disk", 3="bulge", 4="stars", 5="bndry"
+ */
+
+ double MinGasHsmlFractional; /*!< minimum allowed SPH smoothing length in units of SPH gravitational softening length */
+ double MinGasHsml; /*!< minimum allowed SPH smoothing length */
+
+
+ double SofteningGas; /*!< comoving gravitational softening lengths for type 0 */
+ double SofteningHalo; /*!< comoving gravitational softening lengths for type 1 */
+ double SofteningDisk; /*!< comoving gravitational softening lengths for type 2 */
+ double SofteningBulge; /*!< comoving gravitational softening lengths for type 3 */
+ double SofteningStars; /*!< comoving gravitational softening lengths for type 4 */
+ double SofteningBndry; /*!< comoving gravitational softening lengths for type 5 */
+
+ double SofteningGasMaxPhys; /*!< maximum physical softening length for type 0 */
+ double SofteningHaloMaxPhys; /*!< maximum physical softening length for type 1 */
+ double SofteningDiskMaxPhys; /*!< maximum physical softening length for type 2 */
+ double SofteningBulgeMaxPhys; /*!< maximum physical softening length for type 3 */
+ double SofteningStarsMaxPhys; /*!< maximum physical softening length for type 4 */
+ double SofteningBndryMaxPhys; /*!< maximum physical softening length for type 5 */
+
+ double SofteningTable[6]; /*!< current (comoving) gravitational softening lengths for each particle type */
+ double ForceSoftening[6]; /*!< the same, but multiplied by a factor 2.8 - at that scale the force is Newtonian */
+
+
+ double MassTable[6]; /*!< Table with particle masses for particle types with equal mass.
+ If particle masses are all equal for one type, the corresponding entry in MassTable
+ is set to this value, allowing the size of the snapshot files to be reduced. */
+
+
+
+ /* some filenames */
+
+ char InitCondFile[MAXLEN_FILENAME]; /*!< filename of initial conditions */
+ char OutputDir[MAXLEN_FILENAME]; /*!< output directory of the code */
+ char SnapshotFileBase[MAXLEN_FILENAME]; /*!< basename to construct the names of snapshotf files */
+ char EnergyFile[MAXLEN_FILENAME]; /*!< name of file with energy statistics */
+#ifdef SYSTEMSTATISTICS
+ char SystemFile[MAXLEN_FILENAME];
+#endif
+ char CpuFile[MAXLEN_FILENAME]; /*!< name of file with cpu-time statistics */
+ char InfoFile[MAXLEN_FILENAME]; /*!< name of log-file with a list of the timesteps taken */
+ char LogFile[MAXLEN_FILENAME]; /*!< name of log-file with varied info */
+#ifdef SFR
+ char SfrFile[MAXLEN_FILENAME]; /*!< name of file with sfr records */
+#endif
+#ifdef CHIMIE
+ char ChimieFile[MAXLEN_FILENAME]; /*!< name of file with chimie records */
+#endif
+#ifdef MULTIPHASE
+ char PhaseFile[MAXLEN_FILENAME]; /*!< name of file with phase records */
+ char StickyFile[MAXLEN_FILENAME]; /*!< name of file with sticky records */
+#endif
+#ifdef AGN_ACCRETION
+ char AccretionFile[MAXLEN_FILENAME]; /*!< name of file with accretion records */
+#endif
+#ifdef BONDI_ACCRETION
+ char BondiFile[MAXLEN_FILENAME]; /*!< name of file with bondi records */
+#endif
+#ifdef BUBBLES
+ char BubbleFile[MAXLEN_FILENAME]; /*!< name of file with bubble records */
+#endif
+ char TimingsFile[MAXLEN_FILENAME]; /*!< name of file with performance metrics of gravitational tree algorithm */
+ char RestartFile[MAXLEN_FILENAME]; /*!< basename of restart-files */
+ char ResubmitCommand[MAXLEN_FILENAME]; /*!< name of script-file that will be executed for automatic restart */
+ char OutputListFilename[MAXLEN_FILENAME]; /*!< name of file with list of desired output times */
+
+ double OutputListTimes[MAXLEN_OUTPUTLIST]; /*!< table with desired output times */
+ int OutputListLength; /*!< number of output times stored in the table of desired output times */
+
+#ifdef RANDOMSEED_AS_PARAMETER
+ int RandomSeed; /*!< initial random seed >*/
+#endif
+
+
+}
+ All; /*!< a container variable for global variables that are equal on all processors */
+
+
+
+/*! This structure holds all the information that is
+ * stored for each particle of the simulation.
+ */
+extern struct particle_data
+{
+ FLOAT Pos[3]; /*!< particle position at its current time */
+ FLOAT Mass; /*!< particle mass */
+ FLOAT Vel[3]; /*!< particle velocity at its current time */
+ FLOAT GravAccel[3]; /*!< particle acceleration due to gravity */
+#ifdef PMGRID
+ FLOAT GravPM[3]; /*!< particle acceleration due to long-range PM gravity force*/
+#endif
+#ifdef FORCETEST
+ FLOAT GravAccelDirect[3]; /*!< particle acceleration when computed with direct summation */
+#endif
+ FLOAT Potential; /*!< gravitational potential */
+ FLOAT OldAcc; /*!< magnitude of old gravitational force. Used in relative opening criterion */
+#ifndef LONGIDS
+ unsigned int ID; /*!< particle identifier */
+#else
+ unsigned long long ID; /*!< particle identifier */
+#endif
+
+ int Type; /*!< flags particle type. 0=gas, 1=halo, 2=disk, 3=bulge, 4=stars, 5=bndry */
+ int Ti_endstep; /*!< marks start of current timestep of particle on integer timeline */
+ int Ti_begstep; /*!< marks end of current timestep of particle on integer timeline */
+#ifdef FLEXSTEPS
+ int FlexStepGrp; /*!< a random 'offset' on the timeline to create a smooth groouping of particles */
+#endif
+ float GravCost; /*!< weight factor used for balancing the work-load */
+#ifdef PSEUDOSYMMETRIC
+ float AphysOld; /*!< magnitude of acceleration in last timestep. Used to make a first order
+ prediction of the change of acceleration expected in the future, thereby
+ allowing to guess whether a decrease/increase of the timestep should occur
+ in the timestep that is started. */
+#endif
+#ifdef PARTICLE_FLAG
+ float Flag;
+#endif
+
+#ifdef STELLAR_PROP
+ unsigned int StPIdx; /*!< index to the corresponding StP particle */
+#endif
+
+}
+ *P, /*!< holds particle data on local processor */
+ *DomainPartBuf; /*!< buffer for particle data used in domain decomposition */
+
+
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+extern struct sph_particle_data
+{
+ FLOAT Entropy; /*!< current value of entropy (actually entropic function) of particle */
+ FLOAT Density; /*!< current baryonic mass density of particle */
+ FLOAT Hsml; /*!< current smoothing length */
+ FLOAT Left; /*!< lower bound in iterative smoothing length search */
+ FLOAT Right; /*!< upper bound in iterative smoothing length search */
+ FLOAT NumNgb; /*!< weighted number of neighbours found */
+#ifdef AVOIDNUMNGBPROBLEM
+ FLOAT OldNumNgb;
+#endif
+ FLOAT Pressure; /*!< current pressure */
+ FLOAT DtEntropy; /*!< rate of change of entropy */
+#ifdef STELLAR_FLUX
+ FLOAT EnergyFlux; /*!< current value of local energy flux - Sph particles */
+#endif
+#ifdef AGN_HEATING
+ FLOAT EgySpecAGNHeat; /*!< current value of specific energy radiated of particle - Sph particles */
+ FLOAT DtEgySpecAGNHeat; /*!< rate of change of specific radiated energy - Sph particles */
+ FLOAT DtEntropyAGNHeat;
+#endif
+#ifdef MULTIPHASE
+ FLOAT StickyTime;
+ int StickyFlag;
+#ifdef COUNT_COLLISIONS
+ float StickyCollisionNumber;
+#endif
+#endif
+#ifdef FEEDBACK
+ FLOAT EgySpecFeedback;
+ FLOAT DtEgySpecFeedback;
+ FLOAT EnergySN;
+ FLOAT EnergySNrem;
+ FLOAT TimeSN;
+ FLOAT FeedbackVel[3]; /*!< kick due to feedback force */
+#endif
+#ifdef FEEDBACK_WIND
+ FLOAT FeedbackWindVel[3]; /*!< kick due to feedback force */
+#endif
+ FLOAT HydroAccel[3]; /*!< acceleration due to hydrodynamical force */
+ FLOAT VelPred[3]; /*!< predicted SPH particle velocity at the current time */
+ FLOAT DivVel; /*!< local velocity divergence */
+ FLOAT CurlVel; /*!< local velocity curl */
+ FLOAT Rot[3]; /*!< local velocity curl */
+ FLOAT DhsmlDensityFactor; /*!< correction factor needed in the equation of motion of the conservative entropy formulation of SPH */
+ FLOAT MaxSignalVel; /*!< maximum "signal velocity" occuring for this particle */
+#ifdef MULTIPHASE
+ int Phase;
+ int StickyIndex;
+ int StickyNgb;
+ int StickyMaxID;
+ float StickyMaxFs;
+ FLOAT StickyNewVel[3];
+#endif
+#ifdef OUTPUTOPTVAR
+ FLOAT OptVar; /*!< optional variable */
+#endif
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ FLOAT VelocityDispersion[VELOCITY_DISPERSION_SIZE]; /*!< velocity dispersion */
+#endif
+
+#ifdef CHIMIE
+ FLOAT Metal[NELEMENTS];
+ FLOAT dMass; /*!< mass variation due to mass transfere */
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ FLOAT DeltaEgySpec;
+ FLOAT ThermalTime; /*!< flag particles that got energy from SN */
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ FLOAT WindTime; /*!< flag particles that belongs to the wind */
+ unsigned int WindFlag; /*!< flag particles that will be part of the wind */
+#endif
+#endif /*CHIMIE*/
+
+#ifdef ENTROPYPRED
+FLOAT EntropyPred; /*!< predicted entropy at the current time */
+#endif
+}
+ *SphP, /*!< holds SPH particle data on local processor */
+ *DomainSphBuf; /*!< buffer for SPH particle data in domain decomposition */
+
+
+#ifdef STELLAR_PROP
+/* the following struture holds data that is stored for each SPH particle in addition to the collisionless
+ * variables.
+ */
+extern struct st_particle_data
+{
+#ifdef CHECK_ID_CORRESPONDENCE
+ unsigned int ID; /*!< particle identifier (must be the same as P[].ID) only used to check ID correspondance */
+#endif
+ FLOAT FormationTime; /*!< star formation time of particle */
+ FLOAT InitialMass; /*!< initial stellar mass */
+#ifndef LONGIDS
+ unsigned int IDProj; /*!< id of projenitor particle */
+#else
+ unsigned long long IDProj; /*!< id of projenitor particle */
+#endif
+ FLOAT Metal[NELEMENTS];
+ FLOAT Density; /*!< current baryonic mass density of particle */
+ FLOAT Volume; /*!< current volume of particle */
+ FLOAT Hsml; /*!< current smoothing length */
+ FLOAT Left; /*!< lower bound in iterative smoothing length search */
+ FLOAT Right; /*!< upper bound in iterative smoothing length search */
+ FLOAT NumNgb; /*!< weighted number of neighbours found */
+ unsigned int PIdx; /*!< index to the corresponding particle */
+
+#ifdef AVOIDNUMNGBPROBLEM
+ FLOAT OldNumNgb;
+#endif
+ FLOAT DhsmlDensityFactor; /*!< correction factor needed in the equation of motion of the conservative entropy formulation of SPH */
+
+ double TotalEjectedGasMass;
+ double TotalEjectedEltMass[NELEMENTS];
+ double TotalEjectedEgySpec;
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double NgbMass; /*!< mass of neighbours */
+#endif
+#ifdef CHIMIE
+ unsigned int Flag;
+#endif
+}
+ *StP, /*!< holds ST particle data on local processor */
+ *DomainStBuf; /*!< buffer for ST particle data in domain decomposition */
+#endif
+
+/* Variables for Tree
+ */
+
+extern int MaxNodes; /*!< maximum allowed number of internal nodes */
+extern int Numnodestree; /*!< number of (internal) nodes in each tree */
+
+extern struct NODE
+{
+ FLOAT len; /*!< sidelength of treenode */
+ FLOAT center[3]; /*!< geometrical center of node */
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ FLOAT maxsoft; /*!< hold the maximum gravitational softening of particles in the
+ node if the ADAPTIVE_GRAVSOFT_FORGAS option is selected */
+#endif
+#ifdef STELLAR_FLUX
+ FLOAT starlum ; /*!< star luminosity of node */
+#endif
+
+ union
+ {
+ int suns[8]; /*!< temporary pointers to daughter nodes */
+ struct
+ {
+ FLOAT s[3]; /*!< center of mass of node */
+ FLOAT mass; /*!< mass of node */
+ int bitflags; /*!< a bit-field with various information on the node */
+ int sibling; /*!< this gives the next node in the walk in case the current node can be used */
+ int nextnode; /*!< this gives the next node in case the current node needs to be opened */
+ int father; /*!< this gives the parent node of each node (or -1 if we have the root node) */
+ }
+ d;
+ }
+ u;
+}
+ *Nodes_base, /*!< points to the actual memory allocted for the nodes */
+ *Nodes; /*!< this is a pointer used to access the nodes which is shifted such that Nodes[All.MaxPart]
+ gives the first allocated node */
+
+
+extern int *Nextnode; /*!< gives next node in tree walk */
+extern int *Father; /*!< gives parent node in tree */
+
+
+extern struct extNODE /*!< this structure holds additional tree-node information which is not needed in the actual gravity computation */
+{
+ FLOAT hmax; /*!< maximum SPH smoothing length in node. Only used for gas particles */
+ FLOAT vs[3]; /*!< center-of-mass velocity */
+}
+ *Extnodes_base, /*!< points to the actual memory allocted for the extended node information */
+ *Extnodes; /*!< provides shifted access to extended node information, parallel to Nodes/Nodes_base */
+
+
+
+
+
+/*! Header for the standard file format.
+ */
+extern struct io_header
+{
+ int npart[6]; /*!< number of particles of each type in this file */
+ double mass[6]; /*!< mass of particles of each type. If 0, then the masses are explicitly
+ stored in the mass-block of the snapshot file, otherwise they are omitted */
+ double time; /*!< time of snapshot file */
+ double redshift; /*!< redshift of snapshot file */
+ int flag_sfr; /*!< flags whether the simulation was including star formation */
+ int flag_feedback; /*!< flags whether feedback was included (obsolete) */
+ unsigned int npartTotal[6]; /*!< total number of particles of each type in this snapshot. This can be
+ different from npart if one is dealing with a multi-file snapshot. */
+ int flag_cooling; /*!< flags whether cooling was included */
+ int num_files; /*!< number of files in multi-file snapshot */
+ double BoxSize; /*!< box-size of simulation in case periodic boundaries were used */
+ double Omega0; /*!< matter density in units of critical density */
+ double OmegaLambda; /*!< cosmological constant parameter */
+ double HubbleParam; /*!< Hubble parameter in units of 100 km/sec/Mpc */
+ int flag_stellarage; /*!< flags whether the file contains formation times of star particles */
+ int flag_metals; /*!< flags whether the file contains metallicity values for gas and star particles */
+ unsigned int npartTotalHighWord[6]; /*!< High word of the total number of particles of each type */
+ int flag_entropy_instead_u; /*!< flags that IC-file contains entropy instead of u */
+ int flag_chimie_extraheader; /*!< flags that IC-file contains extra-header for chimie */
+#ifdef MULTIPHASE
+ double critical_energy_spec;
+#ifdef MESOMACHINE
+ char fill[38];
+#else
+ char fill[48]; /* use 42 with regor... */
+#endif
+#else
+ char fill[56]; /*!< fills to 256 Bytes */
+#endif
+}
+ header; /*!< holds header for snapshot files */
+
+
+
+#ifdef CHIMIE_EXTRAHEADER
+/*! Header for the chimie part.
+ */
+extern struct io_chimie_extraheader
+ {
+ int nelts; /*!< number of chemical element followed */
+ float SolarAbundances[NELEMENTS];
+ char labels[256-4-4*(NELEMENTS)];
+ }
+ chimie_extraheader;
+#endif
+
+
+
+#define IO_NBLOCKS 23 /*!< total number of defined information blocks for snapshot files.
+ Must be equal to the number of entries in "enum iofields" */
+
+enum iofields /*!< this enumeration lists the defined output blocks in snapshot files. Not all of them need to be present. */
+{
+ IO_POS,
+ IO_VEL,
+ IO_ID,
+ IO_MASS,
+ IO_U,
+ IO_RHO,
+ IO_HSML,
+ IO_POT,
+ IO_ACCEL,
+ IO_DTENTR,
+ IO_TSTP,
+ IO_ERADSPH,
+ IO_ERADSTICKY,
+ IO_ERADFEEDBACK,
+ IO_ENERGYFLUX,
+ IO_OPTVAR,
+ IO_METALS,
+ IO_STAR_FORMATIONTIME,
+ IO_INITIAL_MASS,
+ IO_STAR_IDPROJ,
+ IO_STAR_RHO,
+ IO_STAR_HSML,
+ IO_STAR_METALS
+};
+
+
+extern char Tab_IO_Labels[IO_NBLOCKS][4]; /*<! This table holds four-byte character tags used for fileformat 2 */
+
+
+/* global state of system, used for global statistics
+ */
+extern struct state_of_system
+{
+ double Mass;
+ double EnergyKin;
+ double EnergyPot;
+ double EnergyInt;
+#ifdef COOLING
+ double EnergyRadSph;
+#endif
+#ifdef AGN_HEATING
+ double EnergyAGNHeat;
+#endif
+#ifdef MULTIPHASE
+ double EnergyRadSticky;
+#endif
+#ifdef FEEDBACK_WIND
+ double EnergyFeedbackWind;
+#endif
+#ifdef BUBBLES
+ double EnergyBubbles;
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ double EnergyThermalFeedback;
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double EnergyKineticFeedback;
+#endif
+ double EnergyTot;
+ double Momentum[4];
+ double AngMomentum[4];
+ double CenterOfMass[4];
+ double MassComp[6];
+ double EnergyKinComp[6];
+ double EnergyPotComp[6];
+ double EnergyIntComp[6];
+#ifdef COOLING
+ double EnergyRadSphComp[6];
+#endif
+#ifdef AGN_HEATING
+ double EnergyAGNHeatComp[6];
+#endif
+#ifdef MULTIPHASE
+ double EnergyRadStickyComp[6];
+#endif
+#ifdef FEEDBACK_WIND
+ double EnergyFeedbackWindComp[6];
+#endif
+#ifdef BUBBLES
+ double EnergyBubblesComp[6];
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ double EnergyThermalFeedbackComp[6];
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double EnergyKineticFeedbackComp[6];
+#endif
+
+ double EnergyTotComp[6];
+ double MomentumComp[6][4];
+ double AngMomentumComp[6][4];
+ double CenterOfMassComp[6][4];
+}
+ SysState; /*<! Structure for storing some global statistics about the simulation. */
+
+
+
+
+
+/*! This structure contains data related to the energy budget.
+ These values are different for each task. It need to be stored
+ in the restart flag.
+ */
+extern struct local_state_of_system
+{
+
+ double EnergyTest;
+ double EnergyInt1;
+ double EnergyInt2;
+ double EnergyKin1;
+ double EnergyKin2;
+
+
+#ifdef COOLING
+ double RadiatedEnergy;
+#endif
+
+#ifdef SFR
+ double StarEnergyInt;
+#ifdef FEEDBACK
+ double StarEnergyFeedback;
+#endif
+#endif
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ double EnergyThermalFeedback;
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double EnergyKineticFeedback;
+#endif
+
+#ifdef MULTIPHASE
+ double EnergyRadSticky;
+#endif
+
+#ifdef FEEDBACK_WIND
+ double EnergyFeedbackWind;
+#endif
+
+
+}
+ LocalSysState; /*<! Structure for storing some local statistics about the simulation. */
+
+
+
+
+
+
+
+
+
+/* Various structures for communication
+ */
+extern struct gravdata_in
+{
+ union
+ {
+ FLOAT Pos[3];
+ FLOAT Acc[3];
+ FLOAT Potential;
+ }
+ u;
+#if defined(UNEQUALSOFTENINGS) || defined(STELLAR_FLUX)
+ int Type;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ FLOAT Soft;
+#endif
+#endif
+#ifdef STELLAR_FLUX
+ FLOAT EnergyFlux;
+#endif
+ union
+ {
+ FLOAT OldAcc;
+ int Ninteractions;
+ }
+ w;
+}
+ *GravDataIn, /*!< holds particle data to be exported to other processors */
+ *GravDataGet, /*!< holds particle data imported from other processors */
+ *GravDataResult, /*!< holds the partial results computed for imported particles. Note: We use GravDataResult = GravDataGet, such that the result replaces the imported data */
+ *GravDataOut; /*!< holds partial results received from other processors. This will overwrite the GravDataIn array */
+
+extern struct gravdata_index
+{
+ int Task;
+ int Index;
+ int SortIndex;
+}
+ *GravDataIndexTable; /*!< the particles to be exported are grouped by task-number. This table allows the results to be disentangled again and to be assigned to the correct particle */
+
+
+
+extern struct densdata_in
+{
+ FLOAT Pos[3];
+ FLOAT Vel[3];
+ FLOAT Hsml;
+#ifdef MULTIPHASE
+ int Phase;
+#endif
+ int Index;
+ int Task;
+}
+ *DensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *DensDataGet; /*!< holds imported particle data for SPH density computation */
+
+extern struct densdata_out
+{
+ FLOAT Rho;
+ FLOAT Div, Rot[3];
+ FLOAT DhsmlDensity;
+ FLOAT Ngb;
+}
+ *DensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *DensDataPartialResult; /*!< imported partial SPH density results from other processors */
+
+
+
+extern struct hydrodata_in
+{
+ FLOAT Pos[3];
+ FLOAT Vel[3];
+ FLOAT Hsml;
+#ifdef FEEDBACK
+ FLOAT EnergySN;
+#endif
+#ifdef MULTIPHASE
+ int Phase;
+ FLOAT Entropy;
+ int StickyFlag;
+#endif
+ FLOAT Mass;
+ FLOAT Density;
+ FLOAT Pressure;
+ FLOAT F1;
+ FLOAT DhsmlDensityFactor;
+ int Timestep;
+ int Task;
+ int Index;
+#ifdef WITH_ID_IN_HYDRA
+ int ID;
+#endif
+}
+ *HydroDataIn, /*!< holds particle data for SPH hydro-force computation to be exported to other processors */
+ *HydroDataGet; /*!< holds imported particle data for SPH hydro-force computation */
+
+extern struct hydrodata_out
+{
+ FLOAT Acc[3];
+ FLOAT DtEntropy;
+#ifdef FEEDBACK
+ FLOAT DtEgySpecFeedback;
+ FLOAT FeedbackAccel[3]; /*!< acceleration due to feedback force */
+#endif
+ FLOAT MaxSignalVel;
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ FLOAT VelocityDispersion[VELOCITY_DISPERSION_SIZE];
+#endif
+#ifdef MULTIPHASE
+ FLOAT StickyDVel[3]; /*!< differences in velocities induced by sticky collisions */
+#endif
+}
+ *HydroDataResult, /*!< stores the locally computed SPH hydro results for imported particles */
+ *HydroDataPartialResult; /*!< imported partial SPH hydro-force results from other processors */
+
+
+
+#ifdef MULTIPHASE
+
+
+
+
+extern struct stickydata_in
+{
+ FLOAT Pos[3];
+ FLOAT Vel[3];
+ FLOAT Mass;
+ FLOAT Hsml;
+ int ID;
+ int StickyMaxID;
+ int StickyNgb;
+ float StickyMaxFs;
+ int Task;
+ int Index;
+}
+ *StickyDataIn, /*!< holds particle data for sticky computation to be exported to other processors */
+ *StickyDataGet; /*!< holds imported particle data for sticky computation */
+
+extern struct stickydata_out
+{
+ int StickyMaxID;
+ int StickyNgb;
+ float StickyMaxFs;
+ FLOAT StickyNewVel[3];
+}
+ *StickyDataResult, /*!< stores the locally computed sticky results for imported particles */
+ *StickyDataPartialResult; /*!< imported partial sticky results from other processors */
+
+
+
+
+
+
+
+
+extern struct Sticky_index
+{
+ int Index;
+ int CellIndex;
+ int Flag;
+}
+ *StickyIndex;
+#endif
+
+
+
+#ifdef CHIMIE
+
+extern struct chimiedata_in
+{
+ FLOAT Pos[3];
+ FLOAT Vel[3];
+
+#ifndef LONGIDS
+ unsigned int ID; /*!< particle identifier */
+#else
+ unsigned long long ID; /*!< particle identifier */
+#endif
+
+ FLOAT Hsml;
+#ifdef FEEDBACK
+ FLOAT EnergySN;
+#endif
+#ifdef MULTIPHASE
+ int Phase;
+ FLOAT Entropy;
+ int StickyFlag;
+#endif
+ FLOAT Density;
+ FLOAT Volume;
+ FLOAT Pressure;
+ FLOAT F1;
+ FLOAT DhsmlDensityFactor;
+ int Timestep;
+ int Task;
+ int Index;
+#ifdef WITH_ID_IN_HYDRA
+ int ID;
+#endif
+ double TotalEjectedGasMass;
+ double TotalEjectedEltMass[NELEMENTS];
+ double TotalEjectedEgySpec;
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ FLOAT NgbMass;
+#endif
+
+}
+ *ChimieDataIn, /*!< holds particle data for Chimie computation to be exported to other processors */
+ *ChimieDataGet; /*!< holds imported particle data for Chimie computation */
+
+extern struct chimiedata_out
+{
+ FLOAT Acc[3];
+ FLOAT DtEntropy;
+#ifdef FEEDBACK
+ FLOAT DtEgySpecFeedback;
+ FLOAT FeedbackAccel[3]; /*!< acceleration due to feedback force */
+#endif
+ FLOAT MaxSignalVel;
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ FLOAT VelocityDispersion[VELOCITY_DISPERSION_SIZE];
+#endif
+#ifdef MULTIPHASE
+ FLOAT StickyDVel[3]; /*!< differences in velocities induced by sticky collisions */
+#endif
+}
+ *ChimieDataResult, /*!< stores the locally computed Chimie results for imported particles */
+ *ChimieDataPartialResult; /*!< imported partial Chimie results from other processors */
+
+
+
+
+
+
+extern struct starsdensdata_in
+{
+ FLOAT Pos[3];
+ FLOAT Hsml;
+ int Index;
+ int Task;
+}
+ *StarsDensDataIn, /*!< holds particle data for SPH density computation to be exported to other processors */
+ *StarsDensDataGet; /*!< holds imported particle data for SPH density computation */
+
+extern struct starsdensdata_out
+{
+ FLOAT Rho;
+ FLOAT Volume;
+ FLOAT DhsmlDensity;
+ FLOAT Ngb;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ FLOAT NgbMass;
+#endif
+}
+ *StarsDensDataResult, /*!< stores the locally computed SPH density results for imported particles */
+ *StarsDensDataPartialResult; /*!< imported partial SPH density results from other processors */
+
+
+
+
+#endif
+
+
+
+
+#endif
+
diff --git a/begrun.c b/begrun.c
new file mode 100644
index 0000000..11c6c1c
--- /dev/null
+++ b/begrun.c
@@ -0,0 +1,2015 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <gsl/gsl_rng.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file begrun.c
+ * \brief initial set-up of a simulation run
+ *
+ * This file contains various functions to initialize a simulation run. In
+ * particular, the parameterfile is read in and parsed, the initial
+ * conditions or restart files are read, and global variables are
+ * initialized to their proper values.
+ */
+
+
+/*! This function performs the initial set-up of the simulation. First, the
+ * parameterfile is set, then routines for setting units, reading
+ * ICs/restart-files are called, auxialiary memory is allocated, etc.
+ */
+void begrun(void)
+{
+
+ struct global_data_all_processes all;
+
+#ifdef DETAILED_CPU
+ double tstart,tend;
+ tstart = second();
+#endif
+
+
+ if(ThisTask == 0)
+ {
+ printf("\nThis is Gadget, version `%s'.\n", GADGETVERSION);
+ printf("\nRunning on %d processors.\n", NTask);
+ }
+
+ read_parameter_file(ParameterFile); /* ... read in parameters for this run */
+
+ allocate_commbuffers(); /* ... allocate buffer-memory for particle
+ exchange during force computation */
+ set_units();
+
+
+#if defined(PERIODIC) && (!defined(PMGRID) || defined(FORCETEST))
+ ewald_init();
+#endif
+
+ open_outputfiles();
+
+ random_generator = gsl_rng_alloc(gsl_rng_ranlxd1);
+#ifdef RANDOMSEED_AS_PARAMETER
+ if(ThisTask == 0)
+ printf("Using %d as initial random seed\n",All.RandomSeed);
+ gsl_rng_set(random_generator, All.RandomSeed); /* start-up seed */
+#else
+ printf("Using %d as initial random seed\n",42);
+ gsl_rng_set(random_generator, 42); /* start-up seed */
+#endif
+
+#ifdef PMGRID
+ long_range_init();
+#endif
+
+ All.TimeLastRestartFile = CPUThisRun;
+
+#ifdef MULTIPHASE
+ All.StickyLastCollisionTime = -1;
+#endif
+
+
+/* other physics initialization */
+
+
+#ifdef COOLING
+ if (All.CoolingType==0) /* sutherland */
+ {
+ if(ThisTask == 0) printf("Initialize cooling function...\n");
+ init_cooling(0);
+ if(ThisTask == 0) printf("Initialize cooling function done.\n");
+ }
+
+
+ if (All.CoolingType==2) /* cooling with metals */
+ {
+ if(ThisTask == 0) printf("Initialize cooling function...\n");
+ init_cooling_with_metals();
+ if(ThisTask == 0) printf("Initialize cooling function done.\n");
+ }
+#endif
+
+
+#ifdef CHIMIE
+ int i;
+ if(ThisTask == 0) printf("Initialize chimie...\n");
+
+ init_chimie();
+ check_chimie();
+
+ if(ThisTask == 0)
+ {
+ for (i=0;i<get_nelts();i++)
+ printf("solar abundance %s\t= %g\n",get_Element(i),get_SolarAbundance(i));
+ }
+ if(ThisTask == 0) printf("Initialize chimie done...\n");
+
+#ifdef COOLING
+ All.CoolingParameters_FeHSolar = get_SolarAbundance(FE); /* for consitency, use the value defined in chimie file */
+#endif
+#endif
+
+
+
+
+
+
+
+ if(RestartFlag == 0 || RestartFlag == 2)
+ {
+ set_random_numbers();
+
+ init(); /* ... read in initial model */
+
+ init_local_sys_state();
+ }
+ else
+ {
+ all = All; /* save global variables. (will be read from restart file) */
+
+ restart(RestartFlag); /* ... read restart file. Note: This also resets
+ all variables in the struct `All'.
+ However, during the run, some variables in the parameter
+ file are allowed to be changed, if desired. These need to
+ copied in the way below.
+ Note: All.PartAllocFactor is treated in restart() separately.
+ */
+
+ /* yr
+ if we want a parameter to be taken as the one written in the parameter file,
+ we have to save it below,
+ instead, the value of the restart file will be taken.
+
+ This is usefull, for example, if stop a run and want it to be restarted with
+ different parameters.
+ */
+
+
+ All.MinSizeTimestep = all.MinSizeTimestep;
+ All.MaxSizeTimestep = all.MaxSizeTimestep;
+ All.BufferSize = all.BufferSize;
+ All.BunchSizeForce = all.BunchSizeForce;
+ All.BunchSizeDensity = all.BunchSizeDensity;
+ All.BunchSizeHydro = all.BunchSizeHydro;
+ All.BunchSizeDomain = all.BunchSizeDomain;
+#ifdef MULTIPHASE
+ All.BunchSizeSticky = all.BunchSizeSticky;
+#endif
+#ifdef CHIMIE
+ All.BunchSizeChimie = all.BunchSizeChimie;
+#endif
+
+ All.TimeLimitCPU = all.TimeLimitCPU;
+ All.ResubmitOn = all.ResubmitOn;
+ All.TimeBetSnapshot = all.TimeBetSnapshot;
+ All.TimeBetStatistics = all.TimeBetStatistics;
+ All.CpuTimeBetRestartFile = all.CpuTimeBetRestartFile;
+ All.ErrTolIntAccuracy = all.ErrTolIntAccuracy;
+ All.MaxRMSDisplacementFac = all.MaxRMSDisplacementFac;
+
+ All.ErrTolForceAcc = all.ErrTolForceAcc;
+
+ All.TypeOfTimestepCriterion = all.TypeOfTimestepCriterion;
+ All.TypeOfOpeningCriterion = all.TypeOfOpeningCriterion;
+ All.NumFilesWrittenInParallel = all.NumFilesWrittenInParallel;
+ All.TreeDomainUpdateFrequency = all.TreeDomainUpdateFrequency;
+
+ All.SnapFormat = all.SnapFormat;
+ All.NumFilesPerSnapshot = all.NumFilesPerSnapshot;
+ All.MaxNumNgbDeviation = all.MaxNumNgbDeviation;
+ All.ArtBulkViscConst = all.ArtBulkViscConst;
+
+
+ All.OutputListOn = all.OutputListOn;
+ All.CourantFac = all.CourantFac;
+
+ All.OutputListLength = all.OutputListLength;
+ memcpy(All.OutputListTimes, all.OutputListTimes, sizeof(double) * All.OutputListLength);
+
+#ifdef RANDOMSEED_AS_PARAMETER
+ All.RandomSeed = all.RandomSeed;
+#endif
+
+#ifdef MULTIPHASE
+ All.CriticalTemperature = all.CriticalTemperature;
+ All.CriticalNonCollisionalTemperature = all.CriticalNonCollisionalTemperature;
+
+ All.StickyUseGridForCollisions = all.StickyUseGridForCollisions;
+ All.StickyTime = all.StickyTime;
+ All.StickyCollisionTime = all.StickyCollisionTime;
+ All.StickyIdleTime = all.StickyIdleTime;
+ All.StickyMinVelocity = all.StickyMinVelocity;
+ All.StickyMaxVelocity = all.StickyMaxVelocity;
+ All.StickyLambda = all.StickyLambda;
+ All.StickyDensity = all.StickyDensity;
+ All.StickyDensityPower = all.StickyDensityPower;
+ All.StickyRsphFact = all.StickyRsphFact;
+ All.StickyBetaR = all.StickyBetaR;
+ All.StickyBetaT = all.StickyBetaT;
+ All.StickyGridNx = all.StickyGridNx;
+ All.StickyGridNy = all.StickyGridNy;
+ All.StickyGridNz = all.StickyGridNz;
+ All.StickyGridXmin = all.StickyGridXmin;
+ All.StickyGridXmax = all.StickyGridXmax;
+ All.StickyGridYmin = all.StickyGridYmin;
+ All.StickyGridYmax = all.StickyGridYmax;
+ All.StickyGridZmin = all.StickyGridZmin;
+ All.StickyGridZmax = all.StickyGridZmax;
+#ifdef COLDGAS_CYCLE
+ All.ColdGasCycleTransitionTime = all.ColdGasCycleTransitionTime;
+ All.ColdGasCycleTransitionParameter = all.ColdGasCycleTransitionParameter;
+#endif
+#endif
+
+#ifdef OUTERPOTENTIAL
+
+#ifdef NFW
+ All.HaloConcentration = all.HaloConcentration;
+ All.HaloMass = all.HaloMass;
+ All.GasMassFraction = all.GasMassFraction;
+#endif
+
+#ifdef PLUMMER
+ All.PlummerMass = all.PlummerMass;
+ All.PlummerSoftenning = all.PlummerSoftenning;
+#endif
+
+#ifdef PISOTHERM
+ All.Rho0 = all.Rho0;
+ All.Rc = all.Rc;
+ All.GasMassFraction = all.GasMassFraction;
+#endif
+
+#ifdef CORIOLIS
+ All.CoriolisOmegaX0 = all.CoriolisOmegaX0;
+ All.CoriolisOmegaY0 = all.CoriolisOmegaY0;
+ All.CoriolisOmegaZ0 = all.CoriolisOmegaZ0;
+#endif
+
+#endif
+
+#ifdef SFR
+ //All.StarFormationNStarsFromGas = all.StarFormationNStarsFromGas; /* do not change the param. if restarting, else, StarFormationStarMass will be wrong */
+ //All.StarFormationStarMass = all.StarFormationStarMass;
+ All.StarFormationMgMsFraction = all.StarFormationMgMsFraction;
+ All.StarFormationType = all.StarFormationType;
+ All.StarFormationCstar = all.StarFormationCstar;
+ All.StarFormationTime = all.StarFormationTime;
+ All.StarFormationDensity = all.StarFormationDensity;
+ All.StarFormationTemperature = all.StarFormationTemperature;
+ All.ThresholdDensity = all.ThresholdDensity;
+#endif
+
+#ifdef COOLING
+ All.CoolingType = all.CoolingType;
+ All.CutofCoolingTemperature = all.CutofCoolingTemperature;
+ All.InitGasMetallicity = all.InitGasMetallicity;
+#endif
+
+#ifdef CHIMIE
+ All.ChimieSupernovaEnergy = all.ChimieSupernovaEnergy; /* do not use this value, use the restartfile one */
+ All.ChimieKineticFeedbackFraction = all.ChimieKineticFeedbackFraction;
+ All.ChimieWindSpeed = all.ChimieWindSpeed;
+ All.ChimieWindTime = all.ChimieWindTime;
+ All.ChimieThermalTime = all.ChimieThermalTime;
+ All.ChimieMaxSizeTimestep = all.ChimieMaxSizeTimestep;
+#endif
+
+#if defined (HEATING_PE)
+ All.HeatingPeElectronFraction = all.HeatingPeElectronFraction;
+#endif
+#if defined (HEATING_PE) || defined (STELLAR_FLUX) || defined (EXTERNAL_FLUX)
+ All.HeatingPeSolarEnergyDensity = all.HeatingPeSolarEnergyDensity;
+#endif
+#if defined (HEATING_PE) || defined (STELLAR_FLUX)
+ All.HeatingPeLMRatioGas = all.HeatingPeLMRatioGas;
+ All.HeatingPeLMRatioHalo = all.HeatingPeLMRatioHalo;
+ All.HeatingPeLMRatioDisk = all.HeatingPeLMRatioDisk;
+ All.HeatingPeLMRatioBulge = all.HeatingPeLMRatioBulge;
+ All.HeatingPeLMRatioStars = all.HeatingPeLMRatioStars;
+ All.HeatingPeLMRatioBndry = all.HeatingPeLMRatioBndry;
+ All.HeatingPeLMRatio[0] = all.HeatingPeLMRatio[0];
+ All.HeatingPeLMRatio[1] = all.HeatingPeLMRatio[1];
+ All.HeatingPeLMRatio[2] = all.HeatingPeLMRatio[2];
+ All.HeatingPeLMRatio[3] = all.HeatingPeLMRatio[3];
+ All.HeatingPeLMRatio[4] = all.HeatingPeLMRatio[4];
+ All.HeatingPeLMRatio[5] = all.HeatingPeLMRatio[5];
+#endif
+
+#ifdef EXTERNAL_FLUX
+ All.HeatingExternalFLuxEnergyDensity = all.HeatingExternalFLuxEnergyDensity;
+#endif
+
+#ifdef FEEDBACK
+ All.SupernovaEgySpecPerMassUnit = all.SupernovaEgySpecPerMassUnit;
+ All.SupernovaFractionInEgyKin = all.SupernovaFractionInEgyKin;
+ All.SupernovaTime = all.SupernovaTime;
+#endif
+
+#ifdef FEEDBACK_WIND
+ All.SupernovaWindEgySpecPerMassUnit = all.SupernovaWindEgySpecPerMassUnit;
+ All.SupernovaWindFractionInEgyKin = all.SupernovaWindFractionInEgyKin;
+ All.SupernovaWindParameter = all.SupernovaWindParameter;
+ All.SupernovaWindIntAccuracy = all.SupernovaWindIntAccuracy;
+#endif
+
+#ifdef BUBBLES
+ All.BubblesDelta = all.BubblesDelta;
+ All.BubblesAlpha = all.BubblesAlpha;
+ All.BubblesRadiusFactor = all.BubblesRadiusFactor;
+ All.BubblesR = all.BubblesR;
+#endif
+
+#ifdef AGN_HEATING
+ All.AGNHeatingPower = all.AGNHeatingPower;
+ All.AGNHeatingRmax = all.AGNHeatingRmax;
+#endif
+
+#ifdef AGN_ACCRETION
+ All.TimeBetAccretion = all.TimeBetAccretion;
+ All.AccretionRadius = all.AccretionRadius;
+ All.AGNFactor = all.AGNFactor;
+ All.MinMTotInRa = all.MinMTotInRa;
+#endif
+
+#ifdef BONDI_ACCRETION
+ All.BondiEfficiency = all.BondiEfficiency;
+ All.BondiBlackHoleMass = all.BondiBlackHoleMass;
+ All.BondiHsmlFactor = all.BondiHsmlFactor;
+ All.BondiPower = all.BondiPower;
+ All.BondiTimeBet = all.BondiTimeBet;
+#endif
+
+ strcpy(All.ResubmitCommand, all.ResubmitCommand);
+ strcpy(All.OutputListFilename, all.OutputListFilename);
+ strcpy(All.OutputDir, all.OutputDir);
+ strcpy(All.RestartFile, all.RestartFile);
+ strcpy(All.EnergyFile, all.EnergyFile);
+#ifdef SYSTEMSTATISTICS
+ strcpy(All.SystemFile, all.SystemFile);
+#endif
+ strcpy(All.InfoFile, all.InfoFile);
+ strcpy(All.CpuFile, all.CpuFile);
+ strcpy(All.LogFile, all.LogFile);
+#ifdef SFR
+ strcpy(All.SfrFile, all.SfrFile);
+#endif
+#ifdef CHIMIE
+ strcpy(All.ChimieFile, all.ChimieFile);
+#endif
+#ifdef MULTIPHASE
+ strcpy(All.PhaseFile, all.PhaseFile);
+ strcpy(All.StickyFile, all.StickyFile);
+#endif
+#ifdef AGN_ACCRETION
+ strcpy(All.AccretionFile, all.AccretionFile);
+#endif
+#ifdef BONDI_ACCRETION
+ strcpy(All.BondiFile, all.BondiFile);
+#endif
+#ifdef BUBBLES
+ strcpy(All.BubbleFile, all.BubbleFile);
+#endif
+
+ strcpy(All.TimingsFile, all.TimingsFile);
+ strcpy(All.SnapshotFileBase, all.SnapshotFileBase);
+
+ if(All.TimeMax != all.TimeMax)
+ readjust_timebase(All.TimeMax, all.TimeMax);
+ }
+
+#ifdef PMGRID
+ long_range_init_regionsize();
+#endif
+
+ if(All.ComovingIntegrationOn)
+ init_drift_table();
+
+ if(RestartFlag == 2)
+ All.Ti_nextoutput = find_next_outputtime(All.Ti_Current + 1);
+ else
+ All.Ti_nextoutput = find_next_outputtime(All.Ti_Current);
+
+
+ All.TimeLastRestartFile = CPUThisRun;
+
+
+ /* other initialization for special behavior */
+
+
+#ifdef SFR
+ if (ThisTask == 0)
+ printf("StarFormationStarMass = %g\n\n",All.StarFormationStarMass);
+#endif
+
+
+#ifdef OUTERPOTENTIAL
+ if(ThisTask == 0) printf("Initialize outer potential...\n");
+ init_outer_potential();
+ if(ThisTask == 0) printf("Initialize outer potential done.\n");
+#endif
+
+
+#ifdef BUBBLES
+ if(ThisTask == 0) printf("Initialize bubble function...\n");
+ init_bubble();
+ if(ThisTask == 0) printf("Initialize bubble function done.\n");
+#endif
+
+
+#ifdef MULTIPHASE
+ if(ThisTask == 0) printf("Initialize sticky...\n");
+ header.critical_energy_spec = All.CriticalEgySpec;
+ init_sticky();
+ if(ThisTask == 0) printf("Initialize sticky done.\n");
+#endif
+
+
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Begrun += timediff(tstart, tend);
+ All.CPU_Begrun -= All.CPU_Leapfrog;
+ All.CPU_Begrun -= All.CPU_Domain;
+ All.CPU_Begrun -= All.CPU_Snapshot;
+#endif
+
+
+}
+
+
+
+
+/*! Computes conversion factors between internal code units and the
+ * cgs-system.
+ */
+void set_units(void)
+{
+ double meanweight;
+
+ All.UnitTime_in_s = All.UnitLength_in_cm / All.UnitVelocity_in_cm_per_s;
+ All.UnitTime_in_Megayears = All.UnitTime_in_s / SEC_PER_MEGAYEAR;
+
+ if(All.GravityConstantInternal == 0)
+ All.G = GRAVITY / pow(All.UnitLength_in_cm, 3) * All.UnitMass_in_g * pow(All.UnitTime_in_s, 2);
+ else
+ All.G = All.GravityConstantInternal;
+
+ All.UnitDensity_in_cgs = All.UnitMass_in_g / pow(All.UnitLength_in_cm, 3);
+ All.UnitPressure_in_cgs = All.UnitMass_in_g / All.UnitLength_in_cm / pow(All.UnitTime_in_s, 2);
+ All.UnitCoolingRate_in_cgs = All.UnitPressure_in_cgs / All.UnitTime_in_s;
+ All.UnitEnergy_in_cgs = All.UnitMass_in_g * pow(All.UnitLength_in_cm, 2) / pow(All.UnitTime_in_s, 2);
+
+ /* convert some physical input parameters to internal units */
+ All.Hubble = HUBBLE * All.UnitTime_in_s;
+
+ meanweight = 4.0 / (1 + 3 * HYDROGEN_MASSFRAC); /* note: we assume neutral gas here */
+ /*meanweight = 4 / (8 - 5 * (1 - HYDROGEN_MASSFRAC));*/ /* note: we assume FULL ionized gas here */
+
+
+ All.Boltzmann = BOLTZMANN /All.UnitEnergy_in_cgs;
+ All.ProtonMass = PROTONMASS/All.UnitMass_in_g;
+ All.mumh = All.ProtonMass*meanweight;
+
+
+#ifdef MULTIPHASE
+
+ if (All.ComovingIntegrationOn)
+ All.StickyTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s; /* Myr to code unit */
+ else
+ All.StickyTime *= 3.1536e+13/All.UnitTime_in_s; /* Myr to code unit */
+
+ if (All.ComovingIntegrationOn)
+ All.StickyCollisionTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s; /* Myr to code unit */
+ else
+ All.StickyCollisionTime *= 3.1536e+13/All.UnitTime_in_s; /* Myr to code unit */
+
+ if (All.ComovingIntegrationOn)
+ All.StickyIdleTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s; /* Myr to code unit */
+ else
+ All.StickyIdleTime *= 3.1536e+13/All.UnitTime_in_s; /* Myr to code unit */
+
+
+ if (All.ComovingIntegrationOn)
+ {
+ All.StickyMinVelocity *= 100000/All.UnitTime_in_s; /* km/s to code unit */
+ printf("here, you must check the unit conversion !");
+ endrun(9876);
+ }
+ else
+ All.StickyMinVelocity *= 100000/All.UnitVelocity_in_cm_per_s; /* km/s to code unit */
+
+ if (All.ComovingIntegrationOn)
+ {
+ All.StickyMaxVelocity *= 100000/All.UnitTime_in_s; /* km/s to code unit */
+ printf("here, you must check the unit conversion !");
+ endrun(9876);
+ }
+ else
+ All.StickyMaxVelocity *= 100000/All.UnitVelocity_in_cm_per_s; /* km/s to code unit */
+
+
+ if (All.StickyTime==0)
+ All.StickyLambda = 0;
+ else
+ All.StickyLambda = 1./All.StickyTime;
+
+ All.CriticalEgySpec = 1./GAMMA_MINUS1 * All.Boltzmann/All.mumh * All.CriticalTemperature;
+ All.CriticalNonCollisionalEgySpec = 1./GAMMA_MINUS1 * All.Boltzmann/All.mumh * All.CriticalNonCollisionalTemperature;
+
+ All.StickyDensity = All.StickyDensity/All.UnitDensity_in_cgs;
+
+
+// if((All.StickyLambda > 0.1/All.MaxSizeTimestep)&&(ThisTask==0))
+// {
+// printf("\nStickyLambda is to big and you may experiment numerical problems !\n");
+// printf("You should either decrease StickyLambda or decrease MaxSizeTimestep.\n");
+// printf("(StickyLambda=%g,maxStickyLambda=%g)\n",All.StickyLambda,0.01/All.MaxSizeTimestep);
+// printf("try \n");
+// printf("StickyLambda <= %g or MaxSizeTimestep <= %g \n",(0.01/All.MaxSizeTimestep),(0.01/All.StickyLambda));
+// fflush(stdout);
+// endrun(121212);
+// }
+
+
+#ifdef COLDGAS_CYCLE
+ if (All.ComovingIntegrationOn)
+ All.ColdGasCycleTransitionTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s; /* Myr to code unit */
+ else
+ All.ColdGasCycleTransitionTime *= 3.1536e+13/All.UnitTime_in_s; /* Myr to code unit */
+#endif
+#endif
+
+#ifdef SFR
+ All.StarFormationTime = All.StarFormationTime/All.UnitTime_in_s * 3.1536e16;
+ All.StarFormationDensity = All.StarFormationDensity/All.UnitDensity_in_cgs;
+#endif
+
+#if defined (HEATING_PE) || defined (STELLAR_FLUX)
+ All.HeatingPeLMRatio[0] = All.HeatingPeLMRatioGas;
+ All.HeatingPeLMRatio[1] = All.HeatingPeLMRatioHalo;
+ All.HeatingPeLMRatio[2] = All.HeatingPeLMRatioDisk;
+ All.HeatingPeLMRatio[3] = All.HeatingPeLMRatioBulge;
+ All.HeatingPeLMRatio[4] = All.HeatingPeLMRatioStars;
+ All.HeatingPeLMRatio[5] = All.HeatingPeLMRatioBndry;
+
+ int k;
+ for (k=0;k<6;k++)
+ {
+ All.HeatingPeLMRatio[k] *= 1./SOLAR_MASS; /* erg/s/Msol to erg/s/g */
+ All.HeatingPeLMRatio[k] *= All.UnitMass_in_g*All.UnitTime_in_s / All.UnitEnergy_in_cgs; /* erg/s/g to code unit */
+ }
+#endif
+
+#ifdef FEEDBACK
+ All.SupernovaEgySpecPerMassUnit *= All.UnitMass_in_g / All.UnitEnergy_in_cgs;
+ if (All.ComovingIntegrationOn)
+ All.SupernovaTime *= 3.1536e+13*All.HubbleParam/All.UnitTime_in_s; /* Myr to code unit */
+ else
+ All.SupernovaTime *= 3.1536e+13/All.UnitTime_in_s; /* Myr to code unit */
+#endif
+
+#ifdef FEEDBACK_WIND
+ All.SupernovaWindEgySpecPerMassUnit *= All.UnitMass_in_g / All.UnitEnergy_in_cgs;
+ All.SupernovaWindSpeed = sqrt( 2*All.SupernovaWindFractionInEgyKin * All.SupernovaWindEgySpecPerMassUnit / All.SupernovaWindParameter );
+#endif
+
+#if defined (AGN_ACCRETION) || defined (BONDI_ACCRETION)
+ All.LightSpeed = C/All.UnitVelocity_in_cm_per_s;
+#endif
+
+
+#ifdef CHIMIE
+ All.ChimieSupernovaEnergy = All.ChimieSupernovaEnergy/All.UnitMass_in_g/pow(All.UnitVelocity_in_cm_per_s,2);
+ All.ChimieWindSpeed = All.ChimieWindSpeed*1e5/All.UnitVelocity_in_cm_per_s;
+ All.ChimieWindTime = All.ChimieWindTime*3.1536e13/All.UnitTime_in_s;
+ All.ChimieThermalTime = All.ChimieThermalTime*3.1536e13/All.UnitTime_in_s;
+ All.ChimieMaxSizeTimestep = All.ChimieMaxSizeTimestep*3.1536e13/All.UnitTime_in_s;
+#endif
+
+
+
+ if(ThisTask == 0)
+ {
+ printf("\nHubble (internal units) = %g\n", All.Hubble);
+ printf("G (internal units) = %g\n", All.G);
+ printf("Boltzmann = %g \n", All.Boltzmann);
+ printf("ProtonMass = %g \n", All.ProtonMass);
+ printf("mumh = %g \n", All.mumh);
+ printf("UnitMass_in_g = %g \n", All.UnitMass_in_g);
+ printf("UnitTime_in_s = %g \n", All.UnitTime_in_s);
+ printf("UnitVelocity_in_cm_per_s = %g \n", All.UnitVelocity_in_cm_per_s);
+ printf("UnitDensity_in_cgs = %g \n", All.UnitDensity_in_cgs);
+ printf("UnitEnergy_in_cgs = %g \n", All.UnitEnergy_in_cgs);
+ printf("\n");
+#ifdef SFR
+ printf("StarFormationDensity (internal units) = %g \n", All.StarFormationDensity);
+ printf("StarFormationTime (internal units) = %g \n", All.StarFormationTime);
+#endif
+#ifdef FEEDBACK
+ printf("SupernovaTime (internal units) = %g \n", All.SupernovaTime);
+ printf("SupernovaEgySpecPerMassUnit (internal units) = %g \n", All.SupernovaEgySpecPerMassUnit);
+#endif
+
+#ifdef FEEDBACK_WIND
+ printf("SupernovaWindEgySpecPerMassUnit (internal units) = %g \n", All.SupernovaWindEgySpecPerMassUnit);
+ printf("SupernovaWindSpeed (internal units) = %g \n", All.SupernovaWindSpeed);
+#endif
+
+#ifdef MULTIPHASE
+ printf("CriticalEgySpec (internal units) = %g \n", All.CriticalEgySpec);
+ printf("CriticalNonCollisionalEgySpec (internal units) = %g \n", All.CriticalNonCollisionalEgySpec);
+ printf("StickyCollisionTime (internal units) = %g \n", All.StickyCollisionTime);
+ printf("StickyIdleTime (internal units) = %g \n", All.StickyIdleTime);
+ printf("StickyDensity (internal units) = %g \n", All.StickyDensity);
+ printf("StickyTime (internal units) = %g \n", All.StickyTime);
+ printf("StickyMinVelocity (internal units) = %g \n", All.StickyMinVelocity);
+ printf("StickyMaxVelocity (internal units) = %g \n", All.StickyMaxVelocity);
+#endif
+
+#ifdef COLDGAS_CYCLE
+ printf("ColdGasCycleTransitionTime (internal units) = %g \n", All.ColdGasCycleTransitionTime);
+#endif
+
+#ifdef CHIMIE
+ printf("ChimieSupernovaEnergy (internal units) = %g \n", All.ChimieSupernovaEnergy);
+ printf("ChimieWindSpeed (internal units) = %g \n", All.ChimieWindSpeed);
+ printf("ChimieWindTime (internal units) = %g \n", All.ChimieWindTime);
+ printf("ChimieThermalTime (internal units) = %g \n", All.ChimieThermalTime);
+ printf("ChimieMaxSizeTimestep (internal units) = %g \n", All.ChimieMaxSizeTimestep);
+#endif
+
+ printf("\n");
+ }
+
+#ifdef ISOTHERM_EQS
+ All.MinEgySpec = 0;
+#else
+ All.MinEgySpec = 1 / meanweight * (1.0 / GAMMA_MINUS1) * (BOLTZMANN / PROTONMASS) * All.MinGasTemp;
+ All.MinEgySpec *= All.UnitMass_in_g / All.UnitEnergy_in_cgs;
+#endif
+
+}
+
+
+
+
+
+/*! Initialize local system state variables
+ */
+void init_local_sys_state(void)
+{
+
+#ifdef SFR
+ LocalSysState.StarEnergyInt = 0.;
+#ifdef COOLING
+ LocalSysState.RadiatedEnergy = 0.;
+#endif
+#endif
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ LocalSysState.EnergyThermalFeedback = 0.;
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ LocalSysState.EnergyKineticFeedback = 0.;
+#endif
+
+#ifdef MULTIPHASE
+ LocalSysState.EnergyRadSticky = 0.;
+#endif
+
+#ifdef FEEDBACK_WIND
+ LocalSysState.EnergyFeedbackWind = 0.;
+#endif
+}
+
+
+
+/*! This function opens various log-files that report on the status and
+ * performance of the simulstion. On restart from restart-files
+ * (start-option 1), the code will append to these files.
+ */
+void open_outputfiles(void)
+{
+ char mode[2], buf[200];
+
+#ifdef ADVANCEDSTATISTICS
+ int i=0;
+#endif
+
+ if(ThisTask != 0) /* only the root processor writes to the log files */
+ return;
+
+ if(RestartFlag == 0)
+ strcpy(mode, "w");
+ else
+ strcpy(mode, "a");
+
+
+ sprintf(buf, "%s%s", All.OutputDir, All.CpuFile);
+ if(!(FdCPU = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#ifdef ADVANCEDCPUSTATISTICS
+ else
+ {
+ if(RestartFlag == 0) /* write the header */
+ {
+ fprintf(FdCPU,"# Step ");
+ fprintf(FdCPU,"Time ");
+ fprintf(FdCPU,"nCPUs ");
+ fprintf(FdCPU,"CPU_Total ");
+#ifdef DETAILED_CPU
+ fprintf(FdCPU,"CPU_Leapfrog ");
+ fprintf(FdCPU,"CPU_Physics ");
+ fprintf(FdCPU,"CPU_Residual ");
+ fprintf(FdCPU,"CPU_Accel ");
+ fprintf(FdCPU,"CPU_Begrun ");
+#endif
+ fprintf(FdCPU,"CPU_Gravity ");
+ fprintf(FdCPU,"CPU_Hydro ");
+#ifdef COOLING
+ fprintf(FdCPU,"CPU_Cooling ");
+#endif
+#ifdef SFR
+ fprintf(FdCPU,"CPU_StarFormation ");
+#endif
+#ifdef CHIMIE
+ fprintf(FdCPU,"CPU_Chimie ");
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdCPU,"CPU_Sticky ");
+#endif
+ fprintf(FdCPU,"CPU_Domain ");
+ fprintf(FdCPU,"CPU_Potential ");
+ fprintf(FdCPU,"CPU_Predict ");
+ fprintf(FdCPU,"CPU_TimeLine ");
+ fprintf(FdCPU,"CPU_Snapshot ");
+ fprintf(FdCPU,"CPU_TreeWalk ");
+ fprintf(FdCPU,"CPU_TreeConstruction ");
+ fprintf(FdCPU,"CPU_CommSum ");
+ fprintf(FdCPU,"CPU_Imbalance ");
+ fprintf(FdCPU,"CPU_HydCompWalk ");
+ fprintf(FdCPU,"CPU_HydCommSumm ");
+ fprintf(FdCPU,"CPU_HydImbalance ");
+ fprintf(FdCPU,"CPU_EnsureNgb ");
+ fprintf(FdCPU,"CPU_PM ");
+ fprintf(FdCPU,"CPU_Peano ");
+#ifdef DETAILED_CPU_DOMAIN
+ fprintf(FdCPU,"CPU_Domain_findExtend ");
+ fprintf(FdCPU,"CPU_Domain_determineTopTree ");
+ fprintf(FdCPU,"CPU_Domain_sumCost ");
+ fprintf(FdCPU,"CPU_Domain_findSplit ");
+ fprintf(FdCPU,"CPU_Domain_shiftSplit ");
+ fprintf(FdCPU,"CPU_Domain_countToGo ");
+ fprintf(FdCPU,"CPU_Domain_exchange ");
+#endif
+#ifdef DETAILED_CPU_GRAVITY
+ fprintf(FdCPU,"CPU_Gravity_TreeWalk1 ");
+ fprintf(FdCPU,"CPU_Gravity_TreeWalk2 ");
+ fprintf(FdCPU,"CPU_Gravity_CommSum1 ");
+ fprintf(FdCPU,"CPU_Gravity_CommSum2 ");
+ fprintf(FdCPU,"CPU_Gravity_Imbalance1 ");
+ fprintf(FdCPU,"CPU_Gravity_Imbalance2 ");
+#endif
+ /* return */
+ fprintf(FdCPU,"\n");
+ fflush(FdCPU);
+ }
+ }
+#endif
+
+
+
+
+
+ sprintf(buf, "%s%s", All.OutputDir, All.InfoFile);
+ if(!(FdInfo = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+
+ sprintf(buf, "%s%s", All.OutputDir, All.LogFile);
+ if(!(FdLog = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+
+ sprintf(buf, "%s%s", All.OutputDir, All.EnergyFile);
+ if(!(FdEnergy = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#ifdef ADVANCEDSTATISTICS
+ else
+ {
+ if(RestartFlag == 0) /* write the header */
+ {
+ fprintf(FdEnergy,"# Time EnergyInt EnergyPot EnergyKin ");
+#ifdef COOLING
+ fprintf(FdEnergy,"EnergyRadSph ");
+#endif
+#ifdef AGN_HEATING
+ fprintf(FdEnergy,"EnergyAGNHeat ");
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdEnergy,"EnergyRadSticky ");
+#endif
+#ifdef FEEDBACK_WIND
+ fprintf(FdEnergy,"EnergyFeedbackWind ");
+#endif
+#ifdef BUBBLES
+ fprintf(FdEnergy,"EnergyBubbles ");
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ fprintf(FdEnergy,"EnergyThermalFeedback ");
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ fprintf(FdEnergy,"EnergyKineticFeedback ");
+#endif
+ for (i=0;i<6;i++)
+ {
+ fprintf(FdEnergy,"EnergyIntComp%d EnergyPotComp%d EnergyKinComp%d ",i+1,i+1,i+1);
+#ifdef COOLING
+ fprintf(FdEnergy,"EnergyRadSphComp%d ",i+1);
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdEnergy,"EnergyRadStickyComp%d ",i+1);
+#endif
+#ifdef FEEDBACK_WIND
+ fprintf(FdEnergy,"EnergyFeedbackWindComp%d ",i+1);
+#endif
+#ifdef BUBBLES
+ fprintf(FdEnergy,"EnergyBubblesComp%d ",i+1);
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ fprintf(FdEnergy,"EnergyThermalFeedbackComp%d ",i+1);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ fprintf(FdEnergy,"EnergyKineticFeedbackComp%d ",i+1);
+#endif
+ }
+
+ for (i=0;i<6;i++)
+ fprintf(FdEnergy,"MassComp%d ",i+1);
+
+ /* return */
+ fprintf(FdEnergy,"\n");
+ fflush(FdEnergy);
+ }
+ }
+#endif
+
+#ifdef SYSTEMSTATISTICS
+ sprintf(buf, "%s%s", All.OutputDir, All.SystemFile);
+ if(!(FdSystem = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+ sprintf(buf, "%s%s", All.OutputDir, All.TimingsFile);
+ if(!(FdTimings = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+
+#ifdef FORCETEST
+ if(RestartFlag == 0)
+ {
+ sprintf(buf, "%s%s", All.OutputDir, "forcetest.txt");
+ if(!(FdForceTest = fopen(buf, "w")))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+ fclose(FdForceTest);
+ }
+#endif
+
+#ifdef SFR
+ sprintf(buf, "%s%s", All.OutputDir, All.SfrFile);
+ if(!(FdSfr = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+#ifdef CHIMIE
+ sprintf(buf, "%s%s", All.OutputDir, All.ChimieFile);
+ if(!(FdChimie = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+#ifdef MULTIPHASE
+ sprintf(buf, "%s%s", All.OutputDir, All.PhaseFile);
+ if(!(FdPhase = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+ sprintf(buf, "%s%s", All.OutputDir, All.StickyFile);
+ if(!(FdSticky = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+#ifdef AGN_ACCRETION
+ sprintf(buf, "%s%s", All.OutputDir, All.AccretionFile);
+ if(!(FdAccretion = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+#ifdef BONDI_ACCRETION
+ sprintf(buf, "%s%s", All.OutputDir, All.BondiFile);
+ if(!(FdBondi = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+
+#ifdef BUBBLES
+ sprintf(buf, "%s%s", All.OutputDir, All.BubbleFile);
+ if(!(FdBubble = fopen(buf, mode)))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(1);
+ }
+#endif
+}
+
+
+/*! This function closes the global log-files.
+ */
+void close_outputfiles(void)
+{
+ if(ThisTask != 0) /* only the root processor writes to the log files */
+ return;
+
+ fclose(FdCPU);
+ fclose(FdInfo);
+ fclose(FdLog);
+ fclose(FdEnergy);
+#ifdef SYSTEMSTATISTICS
+ fclose(FdSystem);
+#endif
+ fclose(FdTimings);
+#ifdef FORCETEST
+ fclose(FdForceTest);
+#endif
+
+#ifdef SFR
+ fclose(FdSfr);
+#endif
+
+#ifdef MULTIPHASE
+ fclose(FdPhase);
+ fclose(FdSticky);
+#endif
+
+#ifdef AGN_ACCRETION
+ fclose(FdAccretion);
+#endif
+
+#ifdef BONDI_ACCRETION
+ fclose(FdBondi);
+#endif
+
+#ifdef BUBBLES
+ fclose(FdBubble);
+#endif
+}
+
+
+
+
+/*! This function parses the parameterfile in a simple way. Each paramater
+ * is defined by a keyword (`tag'), and can be either of type double, int,
+ * or character string. The routine makes sure that each parameter
+ * appears exactly once in the parameterfile, otherwise error messages are
+ * produced that complain about the missing parameters.
+ */
+void read_parameter_file(char *fname)
+{
+#define DOUBLE 1
+#define STRING 2
+#define INT 3
+#define MAXTAGS 300
+
+ FILE *fd, *fdout;
+ char buf[200], buf1[200], buf2[200], buf3[400];
+ int i, j, nt;
+ int id[MAXTAGS];
+ void *addr[MAXTAGS];
+ char tag[MAXTAGS][50];
+ int errorFlag = 0;
+
+ if(sizeof(long long) != 8)
+ {
+ if(ThisTask == 0)
+ printf("\nType `long long' is not 64 bit on this platform. Stopping.\n\n");
+ endrun(0);
+ }
+
+ if(sizeof(int) != 4)
+ {
+ if(ThisTask == 0)
+ printf("\nType `int' is not 32 bit on this platform. Stopping.\n\n");
+ endrun(0);
+ }
+
+ if(sizeof(float) != 4)
+ {
+ if(ThisTask == 0)
+ printf("\nType `float' is not 32 bit on this platform. Stopping.\n\n");
+ endrun(0);
+ }
+
+ if(sizeof(double) != 8)
+ {
+ if(ThisTask == 0)
+ printf("\nType `double' is not 64 bit on this platform. Stopping.\n\n");
+ endrun(0);
+ }
+
+
+ if(ThisTask == 0) /* read parameter file on process 0 */
+ {
+ nt = 0;
+
+ strcpy(tag[nt], "InitCondFile");
+ addr[nt] = All.InitCondFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "OutputDir");
+ addr[nt] = All.OutputDir;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "SnapshotFileBase");
+ addr[nt] = All.SnapshotFileBase;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "EnergyFile");
+ addr[nt] = All.EnergyFile;
+ id[nt++] = STRING;
+#ifdef SYSTEMSTATISTICS
+ strcpy(tag[nt], "SystemFile");
+ addr[nt] = All.SystemFile;
+ id[nt++] = STRING;
+#endif
+ strcpy(tag[nt], "CpuFile");
+ addr[nt] = All.CpuFile;
+ id[nt++] = STRING;
+#ifdef SFR
+ strcpy(tag[nt], "SfrFile");
+ addr[nt] = All.SfrFile;
+ id[nt++] = STRING;
+#endif
+#ifdef CHIMIE
+ strcpy(tag[nt], "ChimieFile");
+ addr[nt] = All.ChimieFile;
+ id[nt++] = STRING;
+#endif
+#ifdef MULTIPHASE
+ strcpy(tag[nt], "PhaseFile");
+ addr[nt] = All.PhaseFile;
+ id[nt++] = STRING;
+ strcpy(tag[nt], "StickyFile");
+ addr[nt] = All.StickyFile;
+ id[nt++] = STRING;
+#endif
+#ifdef AGN_ACCRETION
+ strcpy(tag[nt], "AccretionFile");
+ addr[nt] = All.AccretionFile;
+ id[nt++] = STRING;
+#endif
+#ifdef BONDI_ACCRETION
+ strcpy(tag[nt], "BondiFile");
+ addr[nt] = All.BondiFile;
+ id[nt++] = STRING;
+#endif
+#ifdef BUBBLES
+ strcpy(tag[nt], "BubbleFile");
+ addr[nt] = All.BubbleFile;
+ id[nt++] = STRING;
+#endif
+ strcpy(tag[nt], "InfoFile");
+ addr[nt] = All.InfoFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "LogFile");
+ addr[nt] = All.LogFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "TimingsFile");
+ addr[nt] = All.TimingsFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "RestartFile");
+ addr[nt] = All.RestartFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "ResubmitCommand");
+ addr[nt] = All.ResubmitCommand;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "OutputListFilename");
+ addr[nt] = All.OutputListFilename;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "OutputListOn");
+ addr[nt] = &All.OutputListOn;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "Omega0");
+ addr[nt] = &All.Omega0;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "OmegaBaryon");
+ addr[nt] = &All.OmegaBaryon;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "OmegaLambda");
+ addr[nt] = &All.OmegaLambda;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HubbleParam");
+ addr[nt] = &All.HubbleParam;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BoxSize");
+ addr[nt] = &All.BoxSize;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "PeriodicBoundariesOn");
+ addr[nt] = &All.PeriodicBoundariesOn;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "TimeOfFirstSnapshot");
+ addr[nt] = &All.TimeOfFirstSnapshot;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CpuTimeBetRestartFile");
+ addr[nt] = &All.CpuTimeBetRestartFile;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TimeBetStatistics");
+ addr[nt] = &All.TimeBetStatistics;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TimeBegin");
+ addr[nt] = &All.TimeBegin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TimeMax");
+ addr[nt] = &All.TimeMax;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TimeBetSnapshot");
+ addr[nt] = &All.TimeBetSnapshot;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "UnitVelocity_in_cm_per_s");
+ addr[nt] = &All.UnitVelocity_in_cm_per_s;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "UnitLength_in_cm");
+ addr[nt] = &All.UnitLength_in_cm;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "UnitMass_in_g");
+ addr[nt] = &All.UnitMass_in_g;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TreeDomainUpdateFrequency");
+ addr[nt] = &All.TreeDomainUpdateFrequency;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ErrTolIntAccuracy");
+ addr[nt] = &All.ErrTolIntAccuracy;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ErrTolTheta");
+ addr[nt] = &All.ErrTolTheta;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ErrTolForceAcc");
+ addr[nt] = &All.ErrTolForceAcc;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MinGasHsmlFractional");
+ addr[nt] = &All.MinGasHsmlFractional;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MaxSizeTimestep");
+ addr[nt] = &All.MaxSizeTimestep;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MinSizeTimestep");
+ addr[nt] = &All.MinSizeTimestep;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MaxRMSDisplacementFac");
+ addr[nt] = &All.MaxRMSDisplacementFac;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ArtBulkViscConst");
+ addr[nt] = &All.ArtBulkViscConst;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CourantFac");
+ addr[nt] = &All.CourantFac;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "DesNumNgb");
+ addr[nt] = &All.DesNumNgb;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MaxNumNgbDeviation");
+ addr[nt] = &All.MaxNumNgbDeviation;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ComovingIntegrationOn");
+ addr[nt] = &All.ComovingIntegrationOn;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "ICFormat");
+ addr[nt] = &All.ICFormat;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "SnapFormat");
+ addr[nt] = &All.SnapFormat;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "NumFilesPerSnapshot");
+ addr[nt] = &All.NumFilesPerSnapshot;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "NumFilesWrittenInParallel");
+ addr[nt] = &All.NumFilesWrittenInParallel;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "ResubmitOn");
+ addr[nt] = &All.ResubmitOn;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "TypeOfTimestepCriterion");
+ addr[nt] = &All.TypeOfTimestepCriterion;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "TypeOfOpeningCriterion");
+ addr[nt] = &All.TypeOfOpeningCriterion;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "TimeLimitCPU");
+ addr[nt] = &All.TimeLimitCPU;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningHalo");
+ addr[nt] = &All.SofteningHalo;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningDisk");
+ addr[nt] = &All.SofteningDisk;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningBulge");
+ addr[nt] = &All.SofteningBulge;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningGas");
+ addr[nt] = &All.SofteningGas;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningStars");
+ addr[nt] = &All.SofteningStars;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningBndry");
+ addr[nt] = &All.SofteningBndry;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningHaloMaxPhys");
+ addr[nt] = &All.SofteningHaloMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningDiskMaxPhys");
+ addr[nt] = &All.SofteningDiskMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningBulgeMaxPhys");
+ addr[nt] = &All.SofteningBulgeMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningGasMaxPhys");
+ addr[nt] = &All.SofteningGasMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningStarsMaxPhys");
+ addr[nt] = &All.SofteningStarsMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SofteningBndryMaxPhys");
+ addr[nt] = &All.SofteningBndryMaxPhys;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BufferSize");
+ addr[nt] = &All.BufferSize;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "PartAllocFactor");
+ addr[nt] = &All.PartAllocFactor;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "TreeAllocFactor");
+ addr[nt] = &All.TreeAllocFactor;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "GravityConstantInternal");
+ addr[nt] = &All.GravityConstantInternal;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "InitGasTemp");
+ addr[nt] = &All.InitGasTemp;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MinGasTemp");
+ addr[nt] = &All.MinGasTemp;
+ id[nt++] = DOUBLE;
+
+#ifdef RANDOMSEED_AS_PARAMETER
+ strcpy(tag[nt], "RandomSeed");
+ addr[nt] = &All.RandomSeed;
+ id[nt++] = INT;
+#endif
+
+#ifdef COOLING
+ strcpy(tag[nt], "CoolingFile");
+ addr[nt] = All.CoolingFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "CutofCoolingTemperature");
+ addr[nt] = &All.CutofCoolingTemperature;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "InitGasMetallicity");
+ addr[nt] = &All.InitGasMetallicity;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CoolingType");
+ addr[nt] = &All.CoolingType;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef CHIMIE
+ strcpy(tag[nt], "ChimieParameterFile");
+ addr[nt] = All.ChimieParameterFile;
+ id[nt++] = STRING;
+
+ strcpy(tag[nt], "ChimieSupernovaEnergy");
+ addr[nt] = &All.ChimieSupernovaEnergy;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ChimieKineticFeedbackFraction");
+ addr[nt] = &All.ChimieKineticFeedbackFraction;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ChimieWindSpeed");
+ addr[nt] = &All.ChimieWindSpeed;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ChimieWindTime");
+ addr[nt] = &All.ChimieWindTime;
+ id[nt++] = DOUBLE;
+ strcpy(tag[nt], "ChimieThermalTime");
+ addr[nt] = &All.ChimieThermalTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ChimieMaxSizeTimestep");
+ addr[nt] = &All.ChimieMaxSizeTimestep;
+ id[nt++] = DOUBLE;
+#endif
+
+
+#if defined (HEATING_PE)
+ strcpy(tag[nt], "HeatingPeElectronFraction");
+ addr[nt] = &All.HeatingPeElectronFraction;
+ id[nt++] = DOUBLE;
+#endif
+#if defined (HEATING_PE) || defined (STELLAR_FLUX) || defined (EXTERNAL_FLUX)
+ strcpy(tag[nt], "HeatingPeSolarEnergyDensity");
+ addr[nt] = &All.HeatingPeSolarEnergyDensity;
+ id[nt++] = DOUBLE;
+#endif
+#if defined (HEATING_PE) || defined (STELLAR_FLUX)
+ strcpy(tag[nt], "HeatingPeLMRatioGas");
+ addr[nt] = &All.HeatingPeLMRatioGas;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HeatingPeLMRatioHalo");
+ addr[nt] = &All.HeatingPeLMRatioHalo;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HeatingPeLMRatioDisk");
+ addr[nt] = &All.HeatingPeLMRatioDisk;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HeatingPeLMRatioBulge");
+ addr[nt] = &All.HeatingPeLMRatioBulge;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HeatingPeLMRatioStars");
+ addr[nt] = &All.HeatingPeLMRatioStars;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HeatingPeLMRatioBndry");
+ addr[nt] = &All.HeatingPeLMRatioBndry;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef EXTERNAL_FLUX
+ strcpy(tag[nt], "HeatingExternalFLuxEnergyDensity");
+ addr[nt] = &All.HeatingExternalFLuxEnergyDensity;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef MULTIPHASE
+ strcpy(tag[nt], "CriticalTemperature");
+ addr[nt] = &All.CriticalTemperature;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CriticalNonCollisionalTemperature");
+ addr[nt] = &All.CriticalNonCollisionalTemperature;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyUseGridForCollisions");
+ addr[nt] = &All.StickyUseGridForCollisions;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StickyTime");
+ addr[nt] = &All.StickyTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyCollisionTime");
+ addr[nt] = &All.StickyCollisionTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyIdleTime");
+ addr[nt] = &All.StickyIdleTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyMinVelocity");
+ addr[nt] = &All.StickyMinVelocity;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyMaxVelocity");
+ addr[nt] = &All.StickyMaxVelocity;
+ id[nt++] = DOUBLE;
+
+
+ strcpy(tag[nt], "StickyBetaR");
+ addr[nt] = &All.StickyBetaR;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyBetaT");
+ addr[nt] = &All.StickyBetaT;
+ id[nt++] = DOUBLE;
+
+
+ strcpy(tag[nt], "StickyGridNx");
+ addr[nt] = &All.StickyGridNx;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StickyGridNy");
+ addr[nt] = &All.StickyGridNy;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StickyGridNz");
+ addr[nt] = &All.StickyGridNz;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StickyGridXmin");
+ addr[nt] = &All.StickyGridXmin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyGridXmax");
+ addr[nt] = &All.StickyGridXmax;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyGridYmin");
+ addr[nt] = &All.StickyGridYmin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyGridYmax");
+ addr[nt] = &All.StickyGridYmax;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyGridZmin");
+ addr[nt] = &All.StickyGridZmin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyGridZmax");
+ addr[nt] = &All.StickyGridZmax;
+ id[nt++] = DOUBLE;
+
+
+ strcpy(tag[nt], "StickyDensity");
+ addr[nt] = &All.StickyDensity;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyDensityPower");
+ addr[nt] = &All.StickyDensityPower;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StickyRsphFact");
+ addr[nt] = &All.StickyRsphFact;
+ id[nt++] = DOUBLE;
+
+#ifdef COLDGAS_CYCLE
+ strcpy(tag[nt], "ColdGasCycleTransitionTime");
+ addr[nt] = &All.ColdGasCycleTransitionTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "ColdGasCycleTransitionParameter");
+ addr[nt] = &All.ColdGasCycleTransitionParameter;
+ id[nt++] = DOUBLE;
+
+#endif
+#endif
+
+#ifdef OUTERPOTENTIAL
+
+#ifdef NFW
+ strcpy(tag[nt], "HaloConcentration");
+ addr[nt] = &All.HaloConcentration;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "HaloMass");
+ addr[nt] = &All.HaloMass;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "GasMassFraction");
+ addr[nt] = &All.GasMassFraction;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef PLUMMER
+ strcpy(tag[nt], "PlummerMass");
+ addr[nt] = &All.PlummerMass;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "PlummerSoftenning");
+ addr[nt] = &All.PlummerSoftenning;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef PISOTHERM
+ strcpy(tag[nt], "Rho0");
+ addr[nt] = &All.Rho0;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "Rc");
+ addr[nt] = &All.Rc;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "GasMassFraction");
+ addr[nt] = &All.GasMassFraction;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef CORIOLIS
+ strcpy(tag[nt], "CoriolisOmegaX0");
+ addr[nt] = &All.CoriolisOmegaX0;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CoriolisOmegaY0");
+ addr[nt] = &All.CoriolisOmegaY0;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "CoriolisOmegaZ0");
+ addr[nt] = &All.CoriolisOmegaZ0;
+ id[nt++] = DOUBLE;
+#endif
+
+#endif
+
+#ifdef SFR
+ strcpy(tag[nt], "StarFormationNStarsFromGas");
+ addr[nt] = &All.StarFormationNStarsFromGas;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StarFormationMgMsFraction");
+ addr[nt] = &All.StarFormationMgMsFraction;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StarFormationStarMass");
+ addr[nt] = &All.StarFormationStarMass;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StarFormationType");
+ addr[nt] = &All.StarFormationType;
+ id[nt++] = INT;
+
+ strcpy(tag[nt], "StarFormationCstar");
+ addr[nt] = &All.StarFormationCstar;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StarFormationTime");
+ addr[nt] = &All.StarFormationTime;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StarFormationDensity");
+ addr[nt] = &All.StarFormationDensity;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "StarFormationTemperature");
+ addr[nt] = &All.StarFormationTemperature;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef FEEDBACK
+ strcpy(tag[nt], "SupernovaEgySpecPerMassUnit");
+ addr[nt] = &All.SupernovaEgySpecPerMassUnit;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SupernovaFractionInEgyKin");
+ addr[nt] = &All.SupernovaFractionInEgyKin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SupernovaTime");
+ addr[nt] = &All.SupernovaTime;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef FEEDBACK_WIND
+ strcpy(tag[nt], "SupernovaWindEgySpecPerMassUnit");
+ addr[nt] = &All.SupernovaWindEgySpecPerMassUnit;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SupernovaWindFractionInEgyKin");
+ addr[nt] = &All.SupernovaWindFractionInEgyKin;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SupernovaWindParameter");
+ addr[nt] = &All.SupernovaWindParameter;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "SupernovaWindIntAccuracy");
+ addr[nt] = &All.SupernovaWindIntAccuracy;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef AGN_ACCRETION
+ strcpy(tag[nt], "TimeBetAccretion");
+ addr[nt] = &All.TimeBetAccretion;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "AccretionRadius");
+ addr[nt] = &All.AccretionRadius;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "AGNFactor");
+ addr[nt] = &All.AGNFactor;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "MinMTotInRa");
+ addr[nt] = &All.MinMTotInRa;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef BUBBLES
+ strcpy(tag[nt], "BubblesDelta");
+ addr[nt] = &All.BubblesDelta;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BubblesAlpha");
+ addr[nt] = &All.BubblesAlpha;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BubblesRadiusFactor");
+ addr[nt] = &All.BubblesRadiusFactor;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BubblesInitFile");
+ addr[nt] = All.BubblesInitFile;
+ id[nt++] = STRING;
+#endif
+
+#ifdef AGN_HEATING
+ strcpy(tag[nt], "AGNHeatingPower");
+ addr[nt] = &All.AGNHeatingPower;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "AGNHeatingRmax");
+ addr[nt] = &All.AGNHeatingRmax;
+ id[nt++] = DOUBLE;
+#endif
+
+#ifdef BONDI_ACCRETION
+ strcpy(tag[nt], "BondiEfficiency");
+ addr[nt] = &All.BondiEfficiency;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BondiBlackHoleMass");
+ addr[nt] = &All.BondiBlackHoleMass;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BondiHsmlFactor");
+ addr[nt] = &All.BondiHsmlFactor;
+ id[nt++] = DOUBLE;
+
+ strcpy(tag[nt], "BondiTimeBet");
+ addr[nt] = &All.BondiTimeBet;
+ id[nt++] = DOUBLE;
+#endif
+
+
+ if((fd = fopen(fname, "r")))
+ {
+ sprintf(buf, "%s%s", fname, "-usedvalues");
+ if(!(fdout = fopen(buf, "w")))
+ {
+ printf("error opening file '%s' \n", buf);
+ errorFlag = 1;
+ }
+ else
+ {
+ while(!feof(fd))
+ {
+ *buf = 0;
+ fgets(buf, 200, fd);
+ if(sscanf(buf, "%s%s%s", buf1, buf2, buf3) < 2)
+ continue;
+
+ if(buf1[0] == '%')
+ continue;
+
+ for(i = 0, j = -1; i < nt; i++)
+ if(strcmp(buf1, tag[i]) == 0)
+ {
+ j = i;
+ tag[i][0] = 0;
+ break;
+ }
+
+ if(j >= 0)
+ {
+ switch (id[j])
+ {
+ case DOUBLE:
+ *((double *) addr[j]) = atof(buf2);
+ fprintf(fdout, "%-35s%g\n", buf1, *((double *) addr[j]));
+ break;
+ case STRING:
+ strcpy(addr[j], buf2);
+ fprintf(fdout, "%-35s%s\n", buf1, buf2);
+ break;
+ case INT:
+ *((int *) addr[j]) = atoi(buf2);
+ fprintf(fdout, "%-35s%d\n", buf1, *((int *) addr[j]));
+ break;
+ }
+ }
+ else
+ {
+ fprintf(stdout, "Error in file %s: Tag '%s' not allowed or multiple defined.\n",
+ fname, buf1);
+ errorFlag = 1;
+ }
+ }
+ fclose(fd);
+ fclose(fdout);
+
+ i = strlen(All.OutputDir);
+ if(i > 0)
+ if(All.OutputDir[i - 1] != '/')
+ strcat(All.OutputDir, "/");
+
+
+ /* copy parameters-usedvalues file*/
+ sprintf(buf1, "%s%s", fname, "-usedvalues");
+ sprintf(buf2, "%s%s", All.OutputDir, "parameters-usedvalues");
+
+ fd = fopen(buf1,"r");
+ fdout = fopen(buf2,"w");
+
+ while(1)
+ {
+ fgets(buf, 200, fd);
+ if (feof(fd)) break;
+ fprintf(fdout, buf, 200);
+ }
+
+ fclose(fd);
+ fclose(fdout);
+
+
+ }
+ }
+ else
+ {
+ printf("\nParameter file %s not found.\n\n", fname);
+ errorFlag = 2;
+ }
+
+ if(errorFlag != 2)
+ for(i = 0; i < nt; i++)
+ {
+ if(*tag[i])
+ {
+ printf("Error. I miss a value for tag '%s' in parameter file '%s'.\n", tag[i], fname);
+ errorFlag = 1;
+ }
+ }
+
+ if(All.OutputListOn && errorFlag == 0)
+ errorFlag += read_outputlist(All.OutputListFilename);
+ else
+ All.OutputListLength = 0;
+ }
+
+ MPI_Bcast(&errorFlag, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if(errorFlag)
+ {
+ MPI_Finalize();
+ exit(0);
+ }
+
+ /* now communicate the relevant parameters to the other processes */
+ MPI_Bcast(&All, sizeof(struct global_data_all_processes), MPI_BYTE, 0, MPI_COMM_WORLD);
+
+
+ if(All.NumFilesWrittenInParallel < 1)
+ {
+ if(ThisTask == 0)
+ printf("NumFilesWrittenInParallel MUST be at least 1\n");
+ endrun(0);
+ }
+
+ if(All.NumFilesWrittenInParallel > NTask)
+ {
+ if(ThisTask == 0)
+ printf("NumFilesWrittenInParallel MUST be smaller than number of processors\n");
+ endrun(0);
+ }
+
+#ifdef PERIODIC
+ if(All.PeriodicBoundariesOn == 0)
+ {
+ if(ThisTask == 0)
+ {
+ printf("Code was compiled with periodic boundary conditions switched on.\n");
+ printf("You must set `PeriodicBoundariesOn=1', or recompile the code.\n");
+ }
+ endrun(0);
+ }
+#else
+ if(All.PeriodicBoundariesOn == 1)
+ {
+ if(ThisTask == 0)
+ {
+ printf("Code was compiled with periodic boundary conditions switched off.\n");
+ printf("You must set `PeriodicBoundariesOn=0', or recompile the code.\n");
+ }
+ endrun(0);
+ }
+#endif
+
+
+ if(All.TypeOfTimestepCriterion >= 1)
+ {
+ if(ThisTask == 0)
+ {
+ printf("The specified timestep criterion\n");
+ printf("is not valid\n");
+ }
+ endrun(0);
+ }
+
+#if defined(LONG_X) || defined(LONG_Y) || defined(LONG_Z)
+#ifndef NOGRAVITY
+ if(ThisTask == 0)
+ {
+ printf("Code was compiled with LONG_X/Y/Z, but not with NOGRAVITY.\n");
+ printf("Stretched periodic boxes are not implemented for gravity yet.\n");
+ }
+ endrun(0);
+#endif
+#endif
+
+#undef DOUBLE
+#undef STRING
+#undef INT
+#undef MAXTAGS
+}
+
+
+/*! this function reads a table with a list of desired output times. The
+ * table does not have to be ordered in any way, but may not contain more
+ * than MAXLEN_OUTPUTLIST entries.
+ */
+int read_outputlist(char *fname)
+{
+ FILE *fd;
+
+ if(!(fd = fopen(fname, "r")))
+ {
+ printf("can't read output list in file '%s'\n", fname);
+ return 1;
+ }
+
+ All.OutputListLength = 0;
+ do
+ {
+ if(fscanf(fd, " %lg ", &All.OutputListTimes[All.OutputListLength]) == 1)
+ All.OutputListLength++;
+ else
+ break;
+ }
+ while(All.OutputListLength < MAXLEN_OUTPUTLIST);
+
+ fclose(fd);
+
+ printf("\nfound %d times in output-list.\n", All.OutputListLength);
+
+ return 0;
+}
+
+
+/*! If a restart from restart-files is carried out where the TimeMax
+ * variable is increased, then the integer timeline needs to be
+ * adjusted. The approach taken here is to reduce the resolution of the
+ * integer timeline by factors of 2 until the new final time can be
+ * reached within TIMEBASE.
+ */
+void readjust_timebase(double TimeMax_old, double TimeMax_new)
+{
+ int i;
+ long long ti_end;
+
+ if(ThisTask == 0)
+ {
+ printf("\nAll.TimeMax has been changed in the parameterfile\n");
+ printf("Need to adjust integer timeline\n\n\n");
+ }
+
+ if(TimeMax_new < TimeMax_old)
+ {
+ if(ThisTask == 0)
+ printf("\nIt is not allowed to reduce All.TimeMax\n\n");
+ endrun(556);
+ }
+
+ if(All.ComovingIntegrationOn)
+ ti_end = log(TimeMax_new / All.TimeBegin) / All.Timebase_interval;
+ else
+ ti_end = (TimeMax_new - All.TimeBegin) / All.Timebase_interval;
+
+ while(ti_end > TIMEBASE)
+ {
+ All.Timebase_interval *= 2.0;
+
+ ti_end /= 2;
+ All.Ti_Current /= 2;
+
+#ifdef PMGRID
+ All.PM_Ti_begstep /= 2;
+ All.PM_Ti_endstep /= 2;
+#endif
+
+ for(i = 0; i < NumPart; i++)
+ {
+ P[i].Ti_begstep /= 2;
+ P[i].Ti_endstep /= 2;
+ }
+ }
+
+ All.TimeMax = TimeMax_new;
+}
+
diff --git a/bondi_accretion.c b/bondi_accretion.c
new file mode 100644
index 0000000..be6c99e
--- /dev/null
+++ b/bondi_accretion.c
@@ -0,0 +1,205 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef BONDI_ACCRETION
+
+
+/*! \file bondi_accretion.c
+ * \brief Compute bondi accretion on central black hole
+ *
+*/
+
+static double hubble_a, a3inv;
+
+
+
+void compute_bondi_accretion()
+ {
+
+
+ int i,n,j;
+ double pressure,density;
+ //double pressure_max,density_max;
+ //double *density_list,*pressure_list,*radius_list;
+ double pressure_sum,density_sum;
+ double BondiMdot;
+
+ int numngb,startnode,phase,ngb,ngb_sum;
+ FLOAT pos[3],h_i,h_i2;
+ double dx, dy, dz;
+ double h_j,r,r2;
+ double u,wk,hinv,hinv3;
+
+ if(All.ComovingIntegrationOn)
+ {
+
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ }
+ else
+ a3inv = hubble_a = 1;
+
+
+
+
+
+ if((All.Time - All.BondiTimeLast) >= All.BondiTimeBet || All.NumCurrentTiStep==0)
+ {
+
+ All.BondiTimeLast += All.BondiTimeBet;
+
+ density = 0;
+ pressure = 0;
+ ngb=0;
+
+ /* find neighbors around the center */
+ pos[0] = 0;
+ pos[1] = 0;
+ pos[2] = 0;
+ h_i = All.SofteningTable[0]*All.BondiHsmlFactor;
+ h_i2 = h_i * h_i;
+
+ hinv = 1.0 / h_i;
+ hinv3 = hinv * hinv * hinv;
+
+
+ startnode = All.MaxPart;
+#ifdef MULTIPHASE
+ phase = GAS_SPH;
+ numngb = ngb_treefind_phase_pairs(&pos[0],h_i,phase, &startnode);
+#else
+ phase = 0;
+ numngb = ngb_treefind_pairs(&pos[0],h_i,phase, &startnode);
+#endif
+
+
+ for(n = 0; n < numngb; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+ r2 = dx * dx + dy * dy + dz * dz;
+ h_j = SphP[j].Hsml;
+
+ if(r2 < h_i2)
+ {
+ r = sqrt(r2);
+ u = r * hinv;
+
+ ngb++;
+
+ if(u < 0.5)
+ {
+ wk = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ }
+ else
+ {
+ wk = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ }
+
+ density += P[j].Mass * wk;
+ pressure += SphP[j].Pressure * P[j].Mass/SphP[j].Density * wk;
+
+ }
+ }
+
+
+ /* sum contribution from other nodes */
+ MPI_Allreduce(&density, &density_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&pressure, &pressure_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&ngb, &ngb_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ density = density_sum;
+ pressure = pressure_sum;
+
+ /************************************/
+ /* loop over all particles */
+ /************************************/
+/*
+ density_max = 0;
+ pressure_max = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+
+ if(P[i].Type == 0)
+ {
+
+ if (density_max < SphP[i].Density)
+ {
+ density_max = SphP[i].Density;
+ pressure_max = SphP[i].Pressure;
+ }
+
+
+ }
+
+
+ }
+
+ density_list = malloc(NTask * sizeof(double));
+ pressure_list = malloc(NTask * sizeof(double));
+ radius_list = malloc(NTask * sizeof(double));
+
+
+ MPI_Allgather(&density_max, 1, MPI_DOUBLE, density_list, 1, MPI_DOUBLE, MPI_COMM_WORLD);
+ MPI_Allgather(&pressure_max, 1, MPI_DOUBLE, pressure_list, 1, MPI_DOUBLE, MPI_COMM_WORLD);
+ MPI_Allgather(&radius_max, 1, MPI_DOUBLE, radius_list, 1, MPI_DOUBLE, MPI_COMM_WORLD);
+
+ density = 0;
+ pressure = 0;
+ for(i = 0; i < NTask; i++)
+ if (density < density_list[i])
+ {
+ density = density_list[i];
+ pressure= pressure_list[i];
+ radius= radius_list[i];
+ }
+
+ free(density_list);
+ free(pressure_list);
+ free(radius_list);
+*/
+
+
+
+ /* compute bondi accretion */
+ BondiMdot = 4*PI * pow( All.G*All.BondiBlackHoleMass ,2)
+ * density * pow(GAMMA*pressure/density ,-3./2.);
+
+ /* compute bondi power */
+ All.BondiPower = All.BondiEfficiency * (All.LightSpeed*All.LightSpeed) * BondiMdot;
+
+ /* output info */
+ if(ThisTask == 0)
+ {
+ fprintf(FdBondi, "Step: %d, Time: %g, BondiMdot: %g, BondiPower: %g, BondiDensity: %g, BondiPressure: %g NumNgb: %d\n",All.NumCurrentTiStep,All.Time,BondiMdot,All.BondiPower,density,pressure,ngb_sum);
+ fflush(FdBondi);
+ }
+
+ }
+ }
+
+#endif
+
+
+
+
+
+
diff --git a/bubbles.c b/bubbles.c
new file mode 100644
index 0000000..20c7606
--- /dev/null
+++ b/bubbles.c
@@ -0,0 +1,439 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef BUBBLES
+
+
+/*! \file bubble.c
+ * \brief Init bubble
+ *
+*/
+
+
+/*! compute bubble
+ *
+ */
+void init_bubble()
+{
+ FILE *fd;
+ int n,i;
+ char line[58];
+ float Time,D,R,E,A,B;
+
+
+ fd = fopen(All.BubblesInitFile,"r");
+ fscanf(fd, "# %6d\n", &n);
+
+ /* allocate memory */
+ All.BubblesTime = malloc(n * sizeof(double));
+ All.BubblesD = malloc(n * sizeof(double));
+ All.BubblesR = malloc(n * sizeof(double));
+ All.BubblesE = malloc(n * sizeof(double));
+ All.BubblesA = malloc(n * sizeof(double));
+ All.BubblesB = malloc(n * sizeof(double));
+
+ /* read empty line */
+ fgets(line, sizeof(line), fd);
+
+
+ for (i=0;i<n;i++){
+
+ fscanf(fd, "%f %f %f %f %f %f\n",&Time,&D,&R,&E,&A,&B);
+
+ All.BubblesTime[i] = (double)Time;
+ All.BubblesD[i] = (double)D;
+ All.BubblesR[i] = (double)R;
+ All.BubblesE[i] = (double)E;
+ All.BubblesA[i] = (double)A;
+ All.BubblesB[i] = (double)B;
+
+ if(ThisTask == 0)
+ printf("Time = %8.3f D=%8.3f R=%8.3f E=%8.3f A=%8.3f B=%8.3f\n",All.BubblesTime[i],All.BubblesD[i],All.BubblesR[i],All.BubblesE[i],All.BubblesA[i],All.BubblesB[i]);
+
+ }
+
+ fclose(fd);
+
+ All.BubblesIndex = 0;
+
+ /* set the right index */
+ for (i=0;i<n;i++){
+
+ if (All.BubblesTime[All.BubblesIndex]<All.Time)
+ All.BubblesIndex++;
+ else
+ break;
+
+ }
+
+
+#ifdef AGN_FEEDBACK
+ All.BubblesTime[All.BubblesIndex]=1e30;
+ if(ThisTask == 0)
+ printf("Time for next bubble is not defined\n");
+#else
+ if(ThisTask == 0)
+ printf("Time for next bubble = %f (Tnow = %f)\n",All.BubblesTime[All.BubblesIndex],All.Time);
+#endif
+
+
+
+}
+
+/*! compute bubble
+ *
+ */
+void make_bubble()
+{
+
+
+ if(All.Time>All.BubblesTime[All.BubblesIndex])
+ {
+
+ if(ThisTask == 0)
+ {
+ printf("Its time to make a bubble.\n");
+ }
+
+
+ /* first, compute potential energy for all particles */
+ compute_potential();
+
+ /* make first bubble */
+ create_bubble(+1);
+
+ /* make second bubble */
+ create_bubble(-1);
+
+ if(ThisTask == 0)
+ printf("Two bubbles have been created.\n");
+
+ /* move to the next bubble */
+ All.BubblesIndex++;
+
+#ifdef AGN_FEEDBACK
+ /* ensure that the next bubble will not be lunched without agn_feedback authorisation */
+ All.BubblesTime[All.BubblesIndex]= 1e30;
+#else
+ if(ThisTask == 0)
+ printf("Next bubble at t=%g.\n",All.BubblesTime[All.BubblesIndex]);
+#endif
+
+
+
+ }
+
+}
+
+
+/*! create bubble
+ *
+ */
+void create_bubble(int sign)
+{
+
+ int i;
+ FLOAT x,y,z,r,ee;
+ double D,R,A,B;
+ double u_old=0,u_new=0;
+ double alpha,delta;
+ double xs,cs,ss;
+ double UiInt,UiInt_sum,UeInt,UeInt_sum;
+ double EiPot,EiPot_sum,EePot,EePot_sum;
+ double Mi,Me,Mi_sum,Me_sum;
+ double dEgyPot,dEgyInt;
+
+ double a3inv=1.; /* !!!!!!!!!!!! */
+
+ {
+
+ /*
+ compute total energies and masses of particles that will becomes bubbles or not
+ */
+
+ A = All.BubblesA[All.BubblesIndex];
+ B = All.BubblesB[All.BubblesIndex];
+ D = All.BubblesD[All.BubblesIndex];
+ R = All.BubblesR[All.BubblesIndex];
+
+ UiInt = UiInt_sum = UeInt = UeInt_sum = 0;
+ EiPot = EiPot_sum = EePot = EePot_sum = 0;
+ Mi = Mi_sum = Me = Me_sum = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+
+#ifdef MULTIPHASE
+ if((P[i].Type == 0) && (SphP[i].Phase == GAS_SPH))
+#else
+ if(P[i].Type == 0)
+#endif
+ {
+
+ x = P[i].Pos[0];
+ y = P[i].Pos[1];
+ z = P[i].Pos[2];
+
+ r = sqrt(x*x+y*y+z*z);
+
+ /* (1) rotate the point */
+
+ /* Rz(B) */
+ cs = cos(B);
+ ss = sin(B);
+ xs = cs*x - ss*y;
+ y = ss*x + cs*y;
+ x = xs;
+
+ /* Ry(A) */
+ cs = cos(A);
+ ss = sin(A);
+ xs = cs*x + ss*z;
+ z = -ss*x + cs*z;
+ x = xs;
+
+ /* (2) translate+- */
+
+
+ /* sign */
+ z = z-D*sign;
+ r = sqrt(x*x+y*y+z*z);
+
+#ifdef MULTIPHASE
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ /* egyspec from entropy */
+ u_old = (1./GAMMA_MINUS1) * SphP[i].Entropy *pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+ break;
+
+ case GAS_STICKY:
+ case GAS_DARK:
+ /* egyspec from specific energy */
+ u_old = SphP[i].Entropy;
+ break;
+ }
+
+#else
+ u_old = (1./GAMMA_MINUS1) * SphP[i].Entropy *pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+#endif
+
+ if (r<R) /* the point is in the bubble */
+ {
+ UiInt += u_old*P[i].Mass;
+ EiPot += P[i].Potential*P[i].Mass;
+ Mi += P[i].Mass;
+ }
+ else /* the point is outside the bubble */
+ {
+ ee = exp( (R-r)/(R*All.BubblesRadiusFactor) );
+ UeInt += u_old*P[i].Mass*ee;
+ EePot += P[i].Potential*P[i].Mass*ee;
+ Me += P[i].Mass*ee;
+ }
+
+
+ }
+
+
+ }
+
+ /* share results */
+ MPI_Allreduce(&UiInt, &UiInt_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&EiPot, &EiPot_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&UeInt, &UeInt_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&EePot, &EePot_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&Mi, &Mi_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&Me, &Me_sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+
+ /* if Alpha and Delta are given */
+ if ((All.BubblesAlpha!=0)&&(All.BubblesDelta!=0))
+ {
+ All.BubblesBeta = (1-All.BubblesDelta)* Mi_sum/Me_sum ;
+ All.BubblesE[All.BubblesIndex] = -(All.BubblesDelta-1)*((Mi_sum/Me_sum)*EePot_sum - EiPot_sum) + (All.BubblesAlpha-1)*(UiInt_sum + UeInt_sum);
+ }
+ else
+ {
+ /*if (All.BubblesAlpha==0)*/ /* compute BubblesAlpha from BubblesDelta */
+ if (1) /* !!! if alpha is computed once, it is no longer 0 !!! */
+ {
+ /* we can compute beta */
+ All.BubblesBeta = (1-All.BubblesDelta)* Mi_sum/Me_sum ;
+
+ /* now, we can compute alpha */
+ All.BubblesAlpha = (All.BubblesE[All.BubblesIndex] - (All.BubblesDelta-1)*EiPot_sum - All.BubblesBeta*EePot_sum)/(UiInt_sum + UeInt_sum) + 1;
+ }
+ else /* compute BubblesDelta from BubblesAlpha */
+ {
+ /* we can compute delta */
+ All.BubblesDelta = ( (All.BubblesAlpha-1)*(UiInt_sum + UeInt_sum)- All.BubblesE[All.BubblesIndex])
+ / ( (Mi_sum/Me_sum)*EePot_sum - EiPot_sum) + 1;
+
+ /* now we can compute beta */
+ All.BubblesBeta = (1-All.BubblesDelta)* Mi_sum/Me_sum ;
+ }
+
+ /*
+ if(ThisTask == 0)
+ printf("==> UInt=%g; EiPot=%g; EePot=%g; Mi=%g; Me=%g\n",(UiInt_sum + UeInt_sum),EiPot_sum,EePot_sum,Mi_sum,Me_sum);
+ */
+
+ }
+
+ /* count energy */
+ All.EnergyBubbles -= All.BubblesE[All.BubblesIndex];
+
+ if(ThisTask == 0)
+ {
+ dEgyInt = (All.BubblesAlpha-1)*(UiInt_sum + UeInt_sum);
+ dEgyPot = (1-All.BubblesDelta)*((Mi_sum/Me_sum)*EePot_sum - EiPot_sum);
+ fprintf(FdBubble,"Step: %d, Time: %g, EgyBubble: %g, dEgyInt: %g, dEgyPot: %g, EgyInt: %g, Alpha: %g, Beta: %g, Delta: %g, D: %g, R: %g, A: %g, B: %g \n",
+ All.NumCurrentTiStep,All.Time,All.BubblesE[All.BubblesIndex],dEgyInt,dEgyPot,(UiInt_sum + UeInt_sum),
+ All.BubblesAlpha,All.BubblesBeta,All.BubblesDelta,D,R,A,B);
+ fflush(FdBubble);
+ }
+
+
+ /*
+ now, apply Alpha,Beta,Delta
+ */
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+#ifdef MULTIPHASE
+ if((P[i].Type == 0) && (SphP[i].Phase == GAS_SPH))
+#else
+ if(P[i].Type == 0)
+#endif
+ {
+
+ x = P[i].Pos[0];
+ y = P[i].Pos[1];
+ z = P[i].Pos[2];
+
+ /* (1) rotate the point */
+
+ /* Rz(B) */
+ cs = cos(B);
+ ss = sin(B);
+ xs = cs*x - ss*y;
+ y = ss*x + cs*y;
+ x = xs;
+
+ /* Ry(A) */
+ cs = cos(A);
+ ss = sin(A);
+ xs = cs*x + ss*z;
+ z = -ss*x + cs*z;
+ x = xs;
+
+ /* (2) translate+- */
+
+ D = All.BubblesD[All.BubblesIndex];
+ R = All.BubblesR[All.BubblesIndex];
+
+ /* sign */
+ z = z-D*sign;
+ r = sqrt(x*x+y*y+z*z);
+
+
+#ifdef MULTIPHASE
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ /* egyspec from entropy */
+ u_old = (1./GAMMA_MINUS1) * SphP[i].Entropy *pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+ break;
+
+ case GAS_STICKY:
+ case GAS_DARK:
+ /* egyspec from specific energy */
+ u_old = SphP[i].Entropy;
+ break;
+ }
+
+
+#else
+ u_old = (1./GAMMA_MINUS1) * SphP[i].Entropy *pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+#endif
+
+
+ if (r<R) /* the point is in the bubble */
+ {
+ alpha = All.BubblesAlpha;
+ delta = All.BubblesDelta;
+ }
+ else /* the point is outside the bubble */
+ {
+ ee = exp( (R-r)/(R*All.BubblesRadiusFactor) );
+ alpha = (All.BubblesAlpha-1)*ee + 1;
+ delta = All.BubblesBeta*ee + 1;
+ }
+
+ u_new = (alpha/delta) *u_old;
+ P[i].Mass *= delta;
+ SphP[i].Density *= delta;
+
+
+
+#ifdef MULTIPHASE
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ /* entropy from energy */
+ SphP[i].Entropy = GAMMA_MINUS1 * u_new /pow(SphP[i].Density * a3inv, GAMMA_MINUS1) ;
+ break;
+
+ case GAS_STICKY:
+ case GAS_DARK:
+ /* egyspec from energy */
+ SphP[i].Entropy = u_new;
+ break;
+ }
+
+#else
+ SphP[i].Entropy = GAMMA_MINUS1 * u_new /pow(SphP[i].Density * a3inv, GAMMA_MINUS1) ;
+#endif
+
+
+
+ }
+
+ }
+
+
+
+
+
+ }
+
+
+}
+
+
+
+
+
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/chimie.c b/chimie.c
new file mode 100644
index 0000000..bf02a61
--- /dev/null
+++ b/chimie.c
@@ -0,0 +1,2470 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_math.h>
+#include "allvars.h"
+#include "proto.h"
+
+#ifdef CHIMIE
+
+/*! \file hydra.c
+ * \brief Computation of SPH forces and rate of entropy generation
+ *
+ * This file contains the "second SPH loop", where the SPH forces are
+ * computed, and where the rate of change of entropy due to the shock heating
+ * (via artificial viscosity) is computed.
+ */
+
+
+static double hubble_a, atime, hubble_a2, fac_mu, fac_vsic_fix, a3inv, fac_egy;
+#ifdef FEEDBACK
+static double fac_pow;
+#endif
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+
+
+/****************************************************************************************/
+/*
+/*
+/*
+/* GADGET CHIMIE PART
+/*
+/*
+/*
+/****************************************************************************************/
+
+
+
+#define MAXPTS 10
+#define MAXDATASIZE 200
+#define KPC_IN_CM 3.085e+21
+
+
+static int verbose=0;
+
+static double *MassFracSNII;
+static double *SingleMassFracSNII;
+static double *EjectedMass;
+static double *SingleEjectedMass;
+
+
+
+
+/* intern global variables */
+
+static struct local_params_chimie
+{
+ float coeff_z[3][3];
+ float Mmin,Mmax;
+ int n;
+ float ms[MAXPTS];
+ float as[MAXPTS+1];
+ float bs[MAXPTS+1];
+ float fs[MAXPTS];
+ double imf_Ntot;
+
+ float SNII_Mmin;
+ float SNII_Mmax;
+ float SNII_cte;
+ float SNII_a;
+
+ float SNIa_Mpl;
+ float SNIa_Mpu;
+ float SNIa_a;
+ float SNIa_cte;
+
+ float SNIa_Mdl1;
+ float SNIa_Mdu1;
+ float SNIa_a1;
+ float SNIa_b1;
+ float SNIa_cte1;
+ float SNIa_bb1;
+
+ float SNIa_Mdl2;
+ float SNIa_Mdu2;
+ float SNIa_a2;
+ float SNIa_b2;
+ float SNIa_cte2;
+ float SNIa_bb2;
+
+ float Mco;
+
+ int npts;
+ int nelts;
+
+}
+ Cp;
+
+
+static struct local_elts_chimie
+{
+ float Mmin; /* minimal mass */
+ float Step; /* log of mass step */
+ float Array[MAXDATASIZE]; /* data */
+ float Metal[MAXDATASIZE]; /* data */
+ float MSNIa;
+ float SolarAbundance;
+ char label[72];
+}
+ *Elt;
+
+
+
+void read_chimie()
+ {
+
+ char line[72];
+ FILE *fd;
+ int i,j;
+
+ fd = fopen(All.ChimieParameterFile,"r");
+
+ /* read Lifetime */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g %g %g\n", &Cp.coeff_z[0][0],&Cp.coeff_z[0][1],&Cp.coeff_z[0][2]);
+ fscanf(fd, "%g %g %g\n", &Cp.coeff_z[1][0],&Cp.coeff_z[1][1],&Cp.coeff_z[1][2]);
+ fscanf(fd, "%g %g %g\n", &Cp.coeff_z[2][0],&Cp.coeff_z[2][1],&Cp.coeff_z[2][2]);
+ fgets(line, sizeof(line), fd);
+
+ /* IMF Parameters */
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g %g\n",&Cp.Mmin,&Cp.Mmax);
+ fscanf(fd, "%d\n",&Cp.n);
+
+ if (Cp.n>0)
+ for (i=0;i<Cp.n;i++)
+ fscanf(fd,"%g",&Cp.ms[i]);
+ else
+ fgets(line, sizeof(line), fd);
+
+ for (i=0;i<Cp.n+1;i++)
+ fscanf(fd,"%g",&Cp.as[i]);
+
+ fgets(line, sizeof(line), fd);
+
+ /* Parameters for SNII Rates */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g \n",&Cp.SNII_Mmin);
+ fgets(line, sizeof(line), fd);
+
+ /* Parameters for SNIa Rates */
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g %g\n",&Cp.SNIa_Mpl,&Cp.SNIa_Mpu);
+ fscanf(fd, "%g \n",&Cp.SNIa_a);
+ fscanf(fd, "%g %g %g\n",&Cp.SNIa_Mdl1,&Cp.SNIa_Mdu1,&Cp.SNIa_bb1);
+ fscanf(fd, "%g %g %g\n",&Cp.SNIa_Mdl2,&Cp.SNIa_Mdu2,&Cp.SNIa_bb2);
+ fgets(line, sizeof(line), fd);
+
+
+
+
+ /* Metal injection SNII */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%d %d\n",&Cp.npts,&Cp.nelts);
+
+ /* allocate memory for elts */
+ if (Cp.npts<=MAXDATASIZE)
+ {
+ Elt = malloc((Cp.nelts+2) * sizeof(struct local_elts_chimie));
+ }
+ else
+ {
+ printf("\n Cp.npts = %d > MAXDATASIZE = %d !!!\n\n",Cp.npts,MAXDATASIZE);
+ endrun(88800);
+ }
+
+ /* injected metals */
+
+ for (i=0;i<Cp.nelts+2;i++)
+ {
+ fgets(line, sizeof(line), fd);
+
+ /* strip trailing line */
+ for (j = 0; j < strlen(line); j++)
+ if ( line[j] == '\n' || line[j] == '\r' )
+ line[j] = '\0';
+ /* copy labels */
+ strcpy(Elt[i].label,line);
+ strcpy(Elt[i].label,&Elt[i].label[2]);
+
+ fgets(line, sizeof(line), fd);
+
+ fscanf(fd, "%g %g\n",&Elt[i].Mmin,&Elt[i].Step);
+
+ for (j=0;j<Cp.npts;j++)
+ {
+ fscanf(fd, "%g\n",&Elt[i].Metal[j]);
+ }
+ }
+
+
+
+
+
+ /* integrals of injected metals */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%d %d\n",&Cp.npts,&Cp.nelts);
+
+
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+
+
+ /* integrals of injected metals */
+
+ for (i=0;i<Cp.nelts+2;i++)
+ {
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+
+ fscanf(fd, "%g %g\n",&Elt[i].Mmin,&Elt[i].Step);
+
+ for (j=0;j<Cp.npts;j++)
+ {
+ fscanf(fd, "%g\n",&Elt[i].Array[j]);
+ }
+
+ }
+
+
+
+
+
+
+ /* Metal injection SNIa */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g\n",&Cp.Mco);
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+
+ int nelts;
+ char label[72];
+ fscanf(fd, "%d\n",&nelts);
+ /* check */
+ if (nelts != Cp.nelts)
+ {
+ printf("\nThe number of elements in SNII (=%d) is not identical to the on of SNIa (=%d) !!!\n\n",Cp.nelts,nelts);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88805);
+ }
+
+
+ for (i=0;i<Cp.nelts+2;i++)
+ {
+
+ fgets(line, sizeof(line), fd); /* label */
+
+ /* check label */
+ /* strip trailing line */
+ for (j = 0; j < strlen(line); j++)
+ if ( line[j] == '\n' || line[j] == '\r' )
+ line[j] = '\0';
+
+ strcpy(label,line);
+ strcpy(label,&label[2]);
+ if (strcmp(label,Elt[i].label)!=0)
+ {
+ printf("\nLabel of SNII element %d (=%s) is different from the SNIa one (=%s) !!!\n\n",i,Elt[i].label,label);
+ endrun(88806);
+ }
+
+ //fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g\n",&Elt[i].MSNIa);
+
+ }
+
+
+
+
+ /* Solar Abundances */
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+ fgets(line, sizeof(line), fd);
+
+ fscanf(fd, "%d\n",&nelts);
+ /* check */
+ if (nelts != Cp.nelts)
+ {
+ printf("\nThe number of elements in SolarAbundances (=%d) is not identical to the on of SNIa (=%d) !!!\n\n",Cp.nelts,nelts);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88805);
+ }
+
+
+ for (i=0;i<Cp.nelts;i++)
+ {
+
+ fgets(line, sizeof(line), fd); /* label */
+
+ /* check label */
+ /* strip trailing line */
+ for (j = 0; j < strlen(line); j++)
+ if ( line[j] == '\n' || line[j] == '\r' )
+ line[j] = '\0';
+
+ strcpy(label,line);
+ strcpy(label,&label[2]);
+ if (strcmp(label,Elt[i+2].label)!=0)
+ {
+ printf("\nLabel of SNII element %d (=%s) is different from the SNIa one (=%s) !!!\n\n",i,Elt[i+2].label,label);
+ endrun(88806);
+ }
+
+ //fgets(line, sizeof(line), fd);
+ fscanf(fd, "%g\n",&Elt[i+2].SolarAbundance);
+ }
+
+
+
+ fclose(fd);
+
+
+
+
+ if (verbose && ThisTask==0)
+ {
+
+ printf("%g %g %g\n", Cp.coeff_z[0][0],Cp.coeff_z[0][1],Cp.coeff_z[0][2]);
+ printf("%g %g %g\n", Cp.coeff_z[1][0],Cp.coeff_z[1][1],Cp.coeff_z[1][2]);
+ printf("%g %g %g\n", Cp.coeff_z[2][0],Cp.coeff_z[2][1],Cp.coeff_z[2][2]);
+ printf("\n");
+
+ printf("\nIMF\n");
+ printf("%g %g\n",Cp.Mmin,Cp.Mmax);
+ printf("%d\n",Cp.n);
+ for (i=0;i<Cp.n;i++)
+ printf( "ms : %g ",Cp.ms[i]);
+ printf("\n");
+ for (i=0;i<Cp.n+1;i++)
+ printf( "as : %g ",Cp.as[i]);
+ printf("\n");
+
+ printf("\nRate SNII\n");
+ printf("%g ",Cp.SNII_Mmin);
+ printf("\n");
+
+ printf("\nRate SNIa\n");
+ printf("%g %g\n",Cp.SNIa_Mpl,Cp.SNIa_Mpu);
+ printf("%g \n",Cp.SNIa_a);
+ printf("%g %g %g\n",Cp.SNIa_Mdl1,Cp.SNIa_Mdu1,Cp.SNIa_b1);
+ printf("%g %g %g\n",Cp.SNIa_Mdl2,Cp.SNIa_Mdu2,Cp.SNIa_b2);
+ printf("\n");
+
+
+ for (i=0;i<Cp.nelts+2;i++)
+ {
+
+ printf("> %g %g\n",Elt[i].Mmin,Elt[i].Step);
+
+ for (j=0;j<Cp.npts;j++)
+ {
+ printf(" %g\n",Elt[i].Array[j]);
+ }
+
+ }
+
+
+ printf("\n");
+ printf("%g\n",Cp.Mco);
+ for (i=0;i<Cp.nelts+2;i++)
+ printf("%g\n",Elt[i].MSNIa);
+
+ printf("\n");
+
+ }
+
+ }
+
+
+
+
+
+
+
+
+/*
+
+ This function returns the mass fraction of a star of mass m
+ using the current IMF
+
+*/
+
+
+static float get_imf(double m)
+ {
+
+ int i;
+ int n;
+
+ n = Cp.n;
+
+ /* convert m in msol */
+ m = m*All.UnitMass_in_g / SOLAR_MASS;
+
+
+ if (n==0)
+ return Cp.bs[0]* pow(m,Cp.as[0]);
+ else
+ {
+
+ for (i=0;i<n;i++)
+ if (m < Cp.ms[i])
+ return Cp.bs[i]* pow(m,Cp.as[i]);
+
+ return Cp.bs[n-1]* pow(m,Cp.as[n-1]);
+
+ }
+
+ }
+
+
+
+
+/*
+
+ This function returns the mass fraction between m1 and m2
+ per mass unit, using the current IMF
+
+*/
+
+
+static double get_imf_M(double m1, double m2)
+ {
+
+ int i;
+ int n;
+ double p;
+ double integral=0;
+ double mmin,mmax;
+
+ n = Cp.n;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+ m2 = m2*All.UnitMass_in_g / SOLAR_MASS;
+
+
+ if (n==0)
+ {
+ p = Cp.as[0]+1;
+ integral = (Cp.bs[0]/p) * ( pow(m2,p) - pow(m1,p) );
+ }
+
+ else
+ {
+
+ integral = 0;
+
+ /* first */
+ if (m1<Cp.ms[0])
+ {
+ mmin = m1;
+ mmax = dmin(Cp.ms[0],m2);
+ p = Cp.as[0] + 1;
+ integral += (Cp.bs[0]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+
+ /* last */
+ if (m2>Cp.ms[n-1])
+ {
+ mmin = dmax(Cp.ms[n-1],m1);
+ mmax = m2;
+ p = Cp.as[n] + 1;
+ integral += (Cp.bs[n]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+
+ /* loop over other segments */
+ for (i=0;i<n-1;i++)
+ {
+ mmin = dmax(Cp.ms[i ],m1);
+ mmax = dmin(Cp.ms[i+1],m2);
+ if (mmin<mmax)
+ {
+ p = Cp.as[i+1] + 1;
+ integral += (Cp.bs[i+1]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+ }
+
+
+ }
+
+ /* convert into mass unit mass unit */
+ /* integral = integral * SOLAR_MASS/All.UnitMass_in_g;*/
+
+ return integral;
+
+ }
+
+
+
+/*
+
+ This function returns the number fraction between m1 and m2
+ per mass unit, using the current IMF
+
+*/
+
+
+static double get_imf_N(double m1, double m2)
+ {
+
+ int i;
+ int n;
+ double p;
+ double integral=0;
+ double mmin,mmax;
+
+ n = Cp.n;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+ m2 = m2*All.UnitMass_in_g / SOLAR_MASS;
+
+
+ if (n==0)
+ {
+ p = Cp.as[0];
+ integral = (Cp.bs[0]/p) * ( pow(m2,p) - pow(m1,p) );
+ }
+
+ else
+ {
+
+ integral = 0;
+
+ /* first */
+ if (m1<Cp.ms[0])
+ {
+ mmin = m1;
+ mmax = dmin(Cp.ms[0],m2);
+ p = Cp.as[0];
+ integral += (Cp.bs[0]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+
+ /* last */
+ if (m2>Cp.ms[n-1])
+ {
+ mmin = dmax(Cp.ms[n-1],m1);
+ mmax = m2;
+ p = Cp.as[n];
+ integral += (Cp.bs[n]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+
+ /* loop over other segments */
+ for (i=0;i<n-1;i++)
+ {
+ mmin = dmax(Cp.ms[i ],m1);
+ mmax = dmin(Cp.ms[i+1],m2);
+ if (mmin<mmax)
+ {
+ p = Cp.as[i+1];
+ integral += (Cp.bs[i+1]/p) * ( pow(mmax,p) - pow(mmin,p) );
+ }
+ }
+
+ }
+
+ /* convert into mass unit mass unit */
+ integral = integral / SOLAR_MASS*All.UnitMass_in_g;
+
+ return integral;
+
+ }
+
+
+
+
+
+/*
+
+ This function returns the number fraction between m1 and m2
+ per mass unit, using the current IMF
+
+*/
+
+
+static double imf_sampling()
+ {
+
+ int i;
+ int n;
+ double m;
+ double f;
+ double pmin,pmax;
+
+ n = Cp.n;
+
+ /* init random */
+ //srandom(irand);
+
+ f = (double)random()/(double)RAND_MAX;
+
+
+ if (n==0)
+ {
+ pmin = pow(Cp.Mmin,Cp.as[0]);
+ pmax = pow(Cp.Mmax,Cp.as[0]);
+ m = pow(f*(pmax - pmin) + pmin ,1./Cp.as[0]);
+ return m* SOLAR_MASS/All.UnitMass_in_g;
+ }
+
+ else
+ {
+
+
+ if (f<Cp.fs[0])
+ {
+ pmin = pow(Cp.Mmin ,Cp.as[0]);
+ m = pow(Cp.imf_Ntot*Cp.as[0]/Cp.bs[0]* (f-0) + pmin ,1./Cp.as[0]);
+ return m* SOLAR_MASS/All.UnitMass_in_g;
+
+ }
+
+ for (i=0;i<n-1;i++)
+ {
+
+ if (f<Cp.fs[i+1])
+ {
+ pmin = pow(Cp.ms[i] ,Cp.as[i+1]);
+ m = pow(Cp.imf_Ntot*Cp.as[i+1]/Cp.bs[i+1]* (f-Cp.fs[i]) + pmin ,1./Cp.as[i+1]);
+ return m* SOLAR_MASS/All.UnitMass_in_g;
+ }
+
+
+ }
+
+
+ /* last portion */
+ pmin = pow(Cp.ms[n-1] ,Cp.as[n]);
+ m = pow(Cp.imf_Ntot*Cp.as[n]/Cp.bs[n]* (f-Cp.fs[n-1]) + pmin ,1./Cp.as[n]);
+ return m* SOLAR_MASS/All.UnitMass_in_g;
+
+
+ }
+
+
+
+ }
+
+
+
+
+
+
+
+
+/*
+
+ This function initialized the imf parameters
+ defined in the chimie file
+
+*/
+
+
+void init_imf(void)
+ {
+
+ float integral = 0;
+ float p;
+ float cte;
+ int i,n;
+ double mmin,mmax;
+
+ n = Cp.n;
+
+
+ if (n==0)
+ {
+ p = Cp.as[0]+1;
+ integral = integral + ( pow(Cp.Mmax,p)-pow(Cp.Mmin,p))/(p) ;
+ Cp.bs[0] = 1./integral ;
+ }
+ else
+ {
+ cte = 1.0;
+
+ if (Cp.Mmin < Cp.ms[0])
+ {
+ p = Cp.as[0]+1;
+ integral = integral + (pow(Cp.ms[0],p) - pow(Cp.Mmin,p))/p;
+ }
+
+
+ for (i=0;i<n-1;i++)
+ {
+ cte = cte* pow( Cp.ms[i],( Cp.as[i] - Cp.as[i+1] ));
+ p = Cp.as[i+1]+1;
+ integral = integral + cte*(pow(Cp.ms[i+1],p) - pow(Cp.ms[i],p))/p;
+ }
+
+
+ if (Cp.Mmax > Cp.ms[-1])
+ {
+ cte = cte* pow( Cp.ms[n-1] , ( Cp.as[n-1] - Cp.as[n] ) );
+ p = Cp.as[n]+1;
+ integral = integral + cte*(pow(Cp.Mmax,p) - pow(Cp.ms[n-1],p))/p;
+ }
+
+ /* compute all b */
+ Cp.bs[0] = 1./integral;
+
+ for (i=0;i<n;i++)
+ {
+ Cp.bs[i+1] = Cp.bs[i] * pow( Cp.ms[i],( Cp.as[i] - Cp.as[i+1] ));
+ }
+
+ }
+
+
+ if (verbose && ThisTask==0)
+ {
+ printf("-- bs -- \n");
+ for (i=0;i<n+1;i++)
+ printf("%g ",Cp.bs[i]);
+ printf("\n");
+ }
+
+
+
+ mmin = Cp.Mmin / All.UnitMass_in_g * SOLAR_MASS; /* in mass unit */
+ mmax = Cp.Mmax / All.UnitMass_in_g * SOLAR_MASS; /* in mass unit */
+ Cp.imf_Ntot = get_imf_N(mmin,mmax) *SOLAR_MASS/All.UnitMass_in_g;
+
+ /* init fs : mass fraction at ms */
+ if (n>0)
+ {
+
+ for (i=0;i<n+1;i++)
+ {
+ mmax = Cp.ms[i] / All.UnitMass_in_g * SOLAR_MASS; /* in mass unit */
+ Cp.fs[i] = SOLAR_MASS/All.UnitMass_in_g*get_imf_N(mmin,mmax)/Cp.imf_Ntot;
+ }
+
+ }
+
+
+
+ }
+
+
+
+
+
+
+/*
+
+ This function init the chime parameters
+
+*/
+
+
+void init_chimie(void)
+ {
+
+ int i;
+ double u_lt;
+ double UnitLength_in_kpc;
+ double UnitMass_in_Msol;
+
+
+ UnitLength_in_kpc = All.UnitLength_in_cm / KPC_IN_CM;
+ UnitMass_in_Msol = All.UnitMass_in_g / SOLAR_MASS;
+
+ u_lt = -log10( 4.7287e11*sqrt(pow(UnitLength_in_kpc,3)/UnitMass_in_Msol));
+
+
+ read_chimie();
+
+ /* Conversion into program time unit */
+ Cp.coeff_z[2][2] = Cp.coeff_z[2][2] + u_lt;
+
+ for (i=0;i<3;i++)
+ Cp.coeff_z[1][i] = Cp.coeff_z[1][i]/2.0;
+
+
+ /* init imf parameters */
+ init_imf();
+
+
+ /* init SNII parameters */
+ if (Cp.n==0)
+ {
+ //Cp.SNII_cte[0] = Cp.bs[0]/Cp.as[0];
+ Cp.SNII_cte = Cp.bs[0]/Cp.as[0];
+ Cp.SNII_a = Cp.as[0];
+ }
+ else
+ {
+ //for (i=0;i<Cp.n+1;i++) /* if multiple power law in the SNII mass range */
+ // Cp.SNII_cte[i] = Cp.bs[i]/Cp.as[i];
+ Cp.SNII_cte = Cp.bs[Cp.n]/Cp.as[Cp.n];
+ Cp.SNII_a = Cp.as[Cp.n];
+ }
+
+
+ /* init SNIa parameters */
+ Cp.SNIa_a1 = Cp.SNIa_a;
+ Cp.SNIa_b1 = (Cp.SNIa_a1+1)/(pow(Cp.SNIa_Mdu1,Cp.SNIa_a1+1)-pow(Cp.SNIa_Mdl1,Cp.SNIa_a1+1));
+ Cp.SNIa_cte1 = Cp.SNIa_b1/Cp.SNIa_a1;
+
+ Cp.SNIa_a2 = Cp.SNIa_a;
+ Cp.SNIa_b2 = (Cp.SNIa_a2+1)/(pow(Cp.SNIa_Mdu2,Cp.SNIa_a2+1)-pow(Cp.SNIa_Mdl2,Cp.SNIa_a2+1));
+ Cp.SNIa_cte2 = Cp.SNIa_b2/Cp.SNIa_a2;
+
+
+ /* init SNII parameters */
+ if (Cp.n==0)
+ {
+ Cp.SNIa_cte = Cp.bs[0]/Cp.as[0];
+ Cp.SNIa_a = Cp.as[0];
+ }
+ else
+ {
+ Cp.SNIa_cte = Cp.bs[Cp.n]/Cp.as[Cp.n];
+ Cp.SNIa_a = Cp.as[Cp.n];
+ }
+
+
+
+ Cp.SNII_Mmax = Cp.Mmax;
+
+
+
+ /* init metals ejection parameters */
+ MassFracSNII = malloc((Cp.nelts+2) * sizeof(double));
+ EjectedMass = malloc((Cp.nelts+2) * sizeof(double));
+ SingleMassFracSNII = malloc((Cp.nelts+2) * sizeof(double));
+ SingleEjectedMass = malloc((Cp.nelts+2) * sizeof(double));
+
+ for (i=0;i<Cp.nelts+2;i++)
+ Elt[i].Mmin = log10(Elt[i].Mmin);
+
+
+
+
+
+ /* output info */
+ if (verbose && ThisTask==0)
+ {
+ printf("-- SNII_cte -- \n");
+ //for (i=0;i<Cp.n+1;i++)
+ // printf("%g ",Cp.SNII_cte[i]);
+ printf("%g ",Cp.SNII_cte);
+
+ printf("\n");
+ }
+
+
+
+
+ /* check that the masses are higher than the last IMF elbow */
+ if (Cp.n>0)
+ {
+ if (Cp.SNIa_Mpl < Cp.ms[Cp.n-1])
+ {
+ printf("\nSNIa_Mpl = %g < ms[n-1] = %g !!!\n\n",Cp.SNIa_Mpl,Cp.ms[Cp.n-1]);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88801);
+ }
+
+ if (Cp.SNIa_Mpu < Cp.ms[Cp.n-1])
+ {
+ printf("\nSNIa_Mpu = %g < ms[n-1] = %g !!!\n\n",Cp.SNIa_Mpu,Cp.ms[Cp.n-1]);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88802);
+ }
+
+ if (Cp.SNII_Mmin < Cp.ms[Cp.n-1])
+ {
+ printf("\nSNII_Mmin = %g < ms[n-1] = %g !!!\n\n",Cp.SNII_Mmin,Cp.ms[Cp.n-1]);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88803);
+ }
+
+ if (Cp.SNII_Mmax < Cp.ms[Cp.n-1])
+ {
+ printf("\nSNII_Mmax = %g < ms[n-1] = %g !!!\n\n",Cp.SNII_Mmax,Cp.ms[Cp.n-1]);
+ printf("This is not supported by the current implementation !!!\n");
+ endrun(88804);
+ }
+ }
+ }
+
+
+
+void check_chimie(void)
+ {
+ int i;
+
+ printf("(Taks=%d) Number of elts : %d\n",ThisTask,Cp.nelts);
+ for(i=2;i<Cp.nelts+2;i++)
+ printf("%s ",&Elt[i].label);
+ printf("\n");
+
+ /* check number of elements */
+ if (NELEMENTS != Cp.nelts)
+ {
+ printf("(Taks=%d) NELEMENTS (=%d) != Cp.nelts (=%d) : please check !!!\n\n",ThisTask,NELEMENTS,Cp.nelts);
+ endrun(88807);
+ }
+
+ /* check that iron is the first element */
+
+ if ((strcmp("Fe",Elt[2].label))!=0)
+ {
+ printf("(Taks=%d) first element (=%s) is not %s !!!\n\n",ThisTask,Elt[2].label,FIRST_ELEMENT);
+ endrun(88808);
+ }
+
+
+ }
+
+int get_nelts()
+ {
+ return Cp.nelts;
+ }
+
+float get_SolarAbundance(i)
+ {
+ return Elt[i+2].SolarAbundance;
+ }
+
+char* get_Element(i)
+ {
+ return Elt[i+2].label;
+ }
+
+
+
+double star_lifetime(double z,double m)
+{
+
+ /* z is the mass fraction of metals, ie, the metallicity */
+ /* m is the stellar mass in code unit */
+ /* Return t in code time unit */
+
+ int i;
+ double a,b,c;
+ double coeff[3];
+ double logm,twologm,logm2,time;
+
+
+ /* convert m in msol */
+ m = m*All.UnitMass_in_g / SOLAR_MASS;
+
+
+ for (i=0;i<3;i++)
+ coeff[i] = ( Cp.coeff_z[i][0]*z+Cp.coeff_z[i][1] )*z+Cp.coeff_z[i][2];
+
+ a = coeff[0];
+ b = coeff[1];
+ c = coeff[2];
+
+ logm = log10(m);
+ twologm = 2.0 * logm;
+ logm2 = logm*logm;
+
+
+ time = pow(10.,(a*logm2+b*twologm+c));
+
+ return time;
+
+}
+
+
+double star_mass_from_age(double z,double t)
+{
+
+ /* z is the mass fraction of metals, ie, the metallicity */
+ /* t is the star life time */
+ /* return the stellar mass (in code unit) that has a lifetime equal to t */
+ /* this is the inverse of star_lifetime */
+
+
+ int i;
+ double a,b,c;
+ double coeff[3];
+ double m;
+
+
+ for (i=0;i<3;i++)
+ coeff[i] = ( Cp.coeff_z[i][0]*z+Cp.coeff_z[i][1] )*z+Cp.coeff_z[i][2];
+
+ a = coeff[0];
+ b = coeff[1];
+ c = coeff[2];
+
+ m = -(b+sqrt(b*b-a*(c-log10(t))))/a;
+
+ m = pow(10,m); /* here, m is in solar mass */
+ m = m*SOLAR_MASS/All.UnitMass_in_g; /* Msol to mass unit */
+
+ return m;
+
+}
+
+
+/****************************************************************************************/
+/*
+/* Supernova rate : number of supernova per mass unit
+/*
+/****************************************************************************************/
+
+
+double SNII_rate(double m1,double m2)
+{
+
+ /*
+ masses in code unit
+ */
+
+ double RSNII;
+ double md,mu;
+
+ RSNII = 0.0;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+ m2 = m2*All.UnitMass_in_g / SOLAR_MASS;
+
+ /* (1) find md, mu */
+ md = dmax(m1,Cp.SNII_Mmin);
+ mu = dmin(m2,Cp.SNII_Mmax);
+
+ if (mu<=md) /* no SNII in that mass range */
+ return 0.0;
+
+
+ RSNII = Cp.SNII_cte * (pow(mu,Cp.SNII_a)-pow(md,Cp.SNII_a)); /* number per solar mass */
+
+ /* convert in number per solar mass to number per mass unit */
+ RSNII = RSNII *All.UnitMass_in_g / SOLAR_MASS;
+
+ return RSNII;
+}
+
+
+
+double SNIa_rate(double m1,double m2)
+{
+
+ /*
+ masses in code unit
+ */
+
+ double RSNIa;
+ double md,mu;
+
+
+ RSNIa = 0.0;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+ m2 = m2*All.UnitMass_in_g / SOLAR_MASS;
+
+ /* RG contribution */
+ md = dmax(m1,Cp.SNIa_Mdl1);
+ mu = dmin(m2,Cp.SNIa_Mdu1);
+
+ if (md<mu)
+ RSNIa = RSNIa + Cp.SNIa_bb1 * Cp.SNIa_cte1 * (pow(mu,Cp.SNIa_a1)-pow(md,Cp.SNIa_a1));
+
+
+ /* MS contribution */
+ md = dmax(m1,Cp.SNIa_Mdl2);
+ mu = dmin(m2,Cp.SNIa_Mdu2);
+
+ if (md<mu)
+ RSNIa = RSNIa + Cp.SNIa_bb2 * Cp.SNIa_cte2 * (pow(mu,Cp.SNIa_a2)-pow(md,Cp.SNIa_a2));
+
+
+
+ /* WD contribution */
+ md = dmax(m1,Cp.SNIa_Mpl); /* select stars that have finished their life -> WD */
+ mu = Cp.SNIa_Mpu; /* no upper bond */
+
+
+ if (mu<=md) /* no SNIa in that mass range */
+ return 0.0;
+
+ RSNIa = RSNIa * Cp.SNIa_cte * (pow(mu,Cp.SNIa_a)-pow(md,Cp.SNIa_a)); /* number per solar mass */
+
+ /* convert in number per solar mass to number per mass unit */
+ RSNIa = RSNIa *All.UnitMass_in_g / SOLAR_MASS;
+
+
+
+ return RSNIa;
+}
+
+
+
+
+void SNII_mass_ejection(double m1,double m2)
+{
+
+ double l1,l2;
+ int i1,i2,i1p,i2p,j;
+ double f1,f2;
+ double v1,v2;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+ m2 = m2*All.UnitMass_in_g / SOLAR_MASS;
+
+ j = 0;
+
+ l1 = ( log10(m1) - Elt[j].Mmin) / Elt[j].Step ;
+ l2 = ( log10(m2) - Elt[j].Mmin) / Elt[j].Step ;
+
+ if (l1 < 0.0) l1 = 0.0;
+ if (l2 < 0.0) l2 = 0.0;
+
+ i1 = (int)l1;
+ i2 = (int)l2;
+
+ i1p = i1 + 1;
+ i2p = i2 + 1;
+
+ f1 = l1 - i1;
+ f2 = l2 - i2;
+
+ /* check (yr) */
+ if (i1<0) i1=0;
+ if (i2<0) i2=0;
+
+
+ /* --------- TOTAL GAS ---------- */
+ j = 0;
+ v1 = f1 * ( Elt[j].Array[i1p] - Elt[j].Array[i1] ) + Elt[j].Array[i1];
+ v2 = f2 * ( Elt[j].Array[i2p] - Elt[j].Array[i2] ) + Elt[j].Array[i2];
+ MassFracSNII[j] = v2-v1;
+
+ /* --------- He core therm ---------- */
+ j = 1;
+ v1 = f1 * ( Elt[j].Array[i1p] - Elt[j].Array[i1] ) + Elt[j].Array[i1];
+ v2 = f2 * ( Elt[j].Array[i2p] - Elt[j].Array[i2] ) + Elt[j].Array[i2];
+ MassFracSNII[j] = v2-v1;
+
+
+
+
+ /* ---------------------------- */
+ /* --------- Metals ---------- */
+ /* ---------------------------- */
+
+ j = 2;
+
+ l1 = ( log10(m1) - Elt[j].Mmin) / Elt[j].Step ;
+ l2 = ( log10(m2) - Elt[j].Mmin) / Elt[j].Step ;
+
+ if (l1 < 0.0) l1 = 0.0;
+ if (l2 < 0.0) l2 = 0.0;
+
+ i1 = (int)l1;
+ i2 = (int)l2;
+
+ i1p = i1 + 1;
+ i2p = i2 + 1;
+
+ f1 = l1 - i1;
+ f2 = l2 - i2;
+
+ /* check (yr) */
+ if (i1<0) i1=0;
+ if (i2<0) i2=0;
+
+
+ for (j=2;j<Cp.nelts+2;j++)
+ {
+ v1 = f1 * ( Elt[j].Array[i1p] - Elt[j].Array[i1] ) + Elt[j].Array[i1];
+ v2 = f2 * ( Elt[j].Array[i2p] - Elt[j].Array[i2] ) + Elt[j].Array[i2];
+ MassFracSNII[j] = v2-v1;
+ }
+
+}
+
+
+
+
+
+
+void SNII_single_mass_ejection(double m1)
+{
+
+ double l1;
+ int i1,i1p,j;
+ double f1;
+ double v1;
+
+ /* convert m in msol */
+ m1 = m1*All.UnitMass_in_g / SOLAR_MASS;
+
+ j = 0;
+
+ l1 = ( log10(m1) - Elt[j].Mmin) / Elt[j].Step ;
+
+ if (l1 < 0.0) l1 = 0.0;
+
+ i1 = (int)l1;
+
+ i1p = i1 + 1;
+
+ f1 = l1 - i1;
+
+ /* check (yr) */
+ if (i1<0) i1=0;
+
+
+ /* --------- TOTAL GAS ---------- */
+ j = 0;
+ v1 = f1 * ( Elt[j].Metal[i1p] - Elt[j].Metal[i1] ) + Elt[j].Metal[i1];
+ SingleMassFracSNII[j] = v1;
+
+ /* --------- He core therm ---------- */
+ j = 1;
+ v1 = f1 * ( Elt[j].Metal[i1p] - Elt[j].Metal[i1] ) + Elt[j].Metal[i1];
+ SingleMassFracSNII[j] = v1;
+
+
+
+
+ /* ---------------------------- */
+ /* --------- Metals ---------- */
+ /* ---------------------------- */
+
+ j = 2;
+
+ l1 = ( log10(m1) - Elt[j].Mmin) / Elt[j].Step ;
+
+ if (l1 < 0.0) l1 = 0.0;
+
+ i1 = (int)l1;
+
+ i1p = i1 + 1;
+
+ f1 = l1 - i1;
+
+ /* check (yr) */
+ if (i1<0) i1=0;
+
+
+ for (j=2;j<Cp.nelts+2;j++)
+ {
+ v1 = f1 * ( Elt[j].Metal[i1p] - Elt[j].Metal[i1] ) + Elt[j].Metal[i1];
+ SingleMassFracSNII[j] = v1;
+ }
+
+}
+
+
+
+
+
+
+void Total_mass_ejection(double m1,double m2,double M0,double *z)
+{
+
+ int j;
+ double NSNIa;
+
+ /* compute SNII mass ejection -> MassFracSNII */
+ SNII_mass_ejection(m1,m2);
+
+
+ /* number of SNIa per mass unit between time and time+dt */
+ NSNIa = SNIa_rate(m1,m2)*M0;
+
+ /* number of SNII per mass unit between time and time+dt */
+ //NSNII = SNII_rate(m1,m2)*M0; /* useless (only for energy) */
+
+ /* total ejected gas mass */
+ EjectedMass[0] = M0 * MassFracSNII[0] + Cp.Mco/All.UnitMass_in_g*SOLAR_MASS * NSNIa;
+
+
+ /* ejected mass per element */
+ for (j=2;j<Cp.nelts+2;j++)
+ EjectedMass[j] = M0*(MassFracSNII[j] +z[j-2]*MassFracSNII[1]) + NSNIa* Elt[j].MSNIa/All.UnitMass_in_g*SOLAR_MASS;
+
+
+ /* not used */
+ EjectedMass[1] = -1;
+
+
+}
+
+
+
+
+void Total_single_mass_ejection(double m1,double *z)
+{
+
+ /*
+
+ !!! we do not take into account SNIa
+ */
+
+ int j;
+ float M0;
+
+ M0 = m1;
+
+ /* compute SNII mass ejection -> SingleMassFracSNII */
+ SNII_single_mass_ejection(m1);
+
+
+ /* total ejected gas mass */
+ SingleEjectedMass[0] = M0 * SingleMassFracSNII[0]; /* + Cp.Mco/All.UnitMass_in_g*SOLAR_MASS * NSNIa; */
+
+
+ /* ejected mass per element */
+ for (j=2;j<Cp.nelts+2;j++)
+ SingleEjectedMass[j] = M0*(SingleMassFracSNII[j] +z[j-2]*SingleMassFracSNII[1]); /* + NSNIa* Elt[j].MSNIa/All.UnitMass_in_g*SOLAR_MASS; */
+
+ /* not used */
+ SingleEjectedMass[1] = -1;
+
+}
+
+
+
+
+
+
+
+
+
+/**********************************************************************************************
+
+ END OF CHIMIE FUNCTIONS
+
+**********************************************************************************************/
+
+
+
+
+
+
+
+
+
+#if defined(CHIMIE_THERMAL_FEEDBACK) && defined(CHIMIE_COMPUTE_THERMAL_FEEDBACK_ENERGY)
+
+void chimie_compute_energy_int(int mode)
+{
+ int i;
+ double DeltaEgyInt;
+ double Tot_DeltaEgyInt;
+
+ DeltaEgyInt = 0;
+ Tot_DeltaEgyInt = 0;
+
+ if (mode==1)
+ {
+ LocalSysState.EnergyInt1 = 0;
+ LocalSysState.EnergyInt2 = 0;
+ }
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+
+ if (mode==1)
+ LocalSysState.EnergyInt1 += P[i].Mass * SphP[i].EntropyPred / (GAMMA_MINUS1) * pow(SphP[i].Density, GAMMA_MINUS1);
+ else
+ LocalSysState.EnergyInt2 += P[i].Mass * SphP[i].EntropyPred / (GAMMA_MINUS1) * pow(SphP[i].Density, GAMMA_MINUS1);
+ }
+ }
+
+ if (mode==2)
+ {
+ DeltaEgyInt = LocalSysState.EnergyInt2 - LocalSysState.EnergyInt1;
+ MPI_Reduce(&DeltaEgyInt, &Tot_DeltaEgyInt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ LocalSysState.EnergyThermalFeedback -= DeltaEgyInt;
+ }
+
+
+}
+
+#endif
+
+
+
+#if defined(CHIMIE_KINETIC_FEEDBACK) && defined(CHIMIE_COMPUTE_KINETIC_FEEDBACK_ENERGY)
+
+void chimie_compute_energy_kin(int mode)
+{
+ int i;
+ double DeltaEgyKin;
+ double Tot_DeltaEgyKin;
+
+ DeltaEgyKin = 0;
+ Tot_DeltaEgyKin = 0;
+
+ if (mode==1)
+ {
+ LocalSysState.EnergyKin1 = 0;
+ LocalSysState.EnergyKin2 = 0;
+ }
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+
+ if (mode==1)
+ LocalSysState.EnergyKin1 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ else
+ LocalSysState.EnergyKin2 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ }
+ }
+
+ if (mode==2)
+ {
+ DeltaEgyKin = LocalSysState.EnergyKin2 - LocalSysState.EnergyKin1;
+ MPI_Reduce(&DeltaEgyKin, &Tot_DeltaEgyKin, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ LocalSysState.EnergyKineticFeedback -= DeltaEgyKin;
+ }
+
+
+}
+
+#endif
+
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+void chimie_apply_thermal_feedback(void)
+{
+
+ int i;
+ double EgySpec,NewEgySpec,DeltaEntropy;
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+ if (SphP[i].DeltaEgySpec > 0)
+ {
+
+ /* spec energy at current step */
+ EgySpec = SphP[i].EntropyPred / GAMMA_MINUS1 * pow(SphP[i].Density, GAMMA_MINUS1);
+
+ /* new egyspec */
+ NewEgySpec = EgySpec + SphP[i].DeltaEgySpec;
+
+ LocalSysState.EnergyThermalFeedback -= SphP[i].DeltaEgySpec*P[i].Mass;
+ /* new entropy */
+ DeltaEntropy = GAMMA_MINUS1*NewEgySpec/pow(SphP[i].Density, GAMMA_MINUS1) - SphP[i].EntropyPred;
+
+ SphP[i].EntropyPred += DeltaEntropy;
+ SphP[i].Entropy += DeltaEntropy;
+
+
+ /* reset variable */
+ SphP[i].DeltaEgySpec = 0;
+
+ /* recode time */
+ SphP[i].ThermalTime = All.Time;
+
+ }
+ }
+ }
+
+}
+#endif
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+void chimie_apply_wind(void)
+{
+
+ /* apply wind */
+
+ int i;
+ double e1,e2;
+ double phi,costh,sinth,vx,vy,vz;
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+ if (SphP[i].WindFlag)
+ {
+
+ phi = get_ChimieKineticFeedback_random_number(P[i].ID)*PI*2.;
+ costh = 1.-2.*get_ChimieKineticFeedback_random_number(P[i].ID+1);
+ sinth = sqrt(1.-pow(costh,2));
+
+ vx = All.ChimieWindSpeed*sinth*cos(phi);
+ vy = All.ChimieWindSpeed*sinth*sin(phi);
+ vz = All.ChimieWindSpeed*costh;
+
+ e1 = 0.5*P[i].Mass * ( SphP[i].VelPred[0]*SphP[i].VelPred[0] + SphP[i].VelPred[1]*SphP[i].VelPred[1] + SphP[i].VelPred[2]*SphP[i].VelPred[2]);
+
+ P[i].Vel[0] += vx;
+ P[i].Vel[1] += vy;
+ P[i].Vel[2] += vz;
+
+ SphP[i].VelPred[0] += vx;
+ SphP[i].VelPred[1] += vy;
+ SphP[i].VelPred[2] += vz;
+
+ e2 = 0.5*P[i].Mass * ( SphP[i].VelPred[0]*SphP[i].VelPred[0] + SphP[i].VelPred[1]*SphP[i].VelPred[1] + SphP[i].VelPred[2]*SphP[i].VelPred[2]);
+ LocalSysState.EnergyKineticFeedback -= e2-e1;
+
+ SphP[i].WindFlag = 0;
+ }
+ }
+ }
+
+}
+#endif
+
+
+
+/*! This function is the driver routine for the calculation of chemical evolution
+ */
+void chimie(void)
+{
+ double t0, t1;
+
+ t0 = second(); /* measure the time for the full chimie computation */
+
+
+ if (ThisTask==0)
+ printf("Start Chimie computation.\n");
+
+
+
+/* apply thermal feedback on selected particles */
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ chimie_apply_thermal_feedback();
+#endif
+
+ /* apply wind on selected particles */
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ chimie_apply_wind();
+#endif
+
+
+
+
+ stars_density(); /* compute density */
+ do_chimie(); /* chimie */
+
+
+
+ if (ThisTask==0)
+ printf("Chimie computation done.\n");
+
+
+ t1 = second();
+ All.CPU_Chimie += timediff(t0, t1);
+
+}
+
+
+
+/*! This function is the driver routine for the calculation of chemical evolution
+ */
+void do_chimie(void)
+{
+ long long ntot, ntotleft;
+ int i, j, k, n, m, ngrp, maxfill, source, ndone;
+ int *nbuffer, *noffset, *nsend_local, *nsend, *numlist, *ndonelist;
+ int level, sendTask, recvTask, nexport, place;
+ double tstart, tend, sumt, sumcomm;
+ double timecomp = 0, timecommsumm = 0, timeimbalance = 0, sumimbalance;
+ int flag_chimie;
+ MPI_Status status;
+
+ double t1,t2;
+ double a1,a2;
+ double minlivetime,star_age;
+ double m1,m2,M0;
+ double NSNIa,NSNII;
+ double NSNIa_tot,NSNII_tot,NSNIa_totlocal,NSNII_totlocal;
+ double EgySN,EgySNlocal;
+ double EgySNThermal,EgySNKinetic;
+ int Nchim,Nchimlocal;
+ int Nwind,Nwindlocal;
+ int Nflag,Nflaglocal;
+ int Noldwind,Noldwindlocal;
+ double metals[NELEMENTS];
+
+
+
+
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ double v1m,v2m;
+#endif
+
+ if(All.ComovingIntegrationOn)
+ {
+ /* Factors for comoving integration of hydro */
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+ hubble_a2 = All.Time * All.Time * hubble_a;
+
+ fac_mu = pow(All.Time, 3 * (GAMMA - 1) / 2) / All.Time;
+
+ fac_egy = pow(All.Time, 3 * (GAMMA - 1));
+
+ fac_vsic_fix = hubble_a * pow(All.Time, 3 * GAMMA_MINUS1);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ atime = All.Time;
+
+#ifdef FEEDBACK
+ fac_pow = fac_egy*atime*atime;
+#endif
+
+ }
+ else
+ {
+ hubble_a = hubble_a2 = atime = fac_mu = fac_vsic_fix = a3inv = fac_egy = 1.0;
+#ifdef FEEDBACK
+ fac_pow = 1.0;
+#endif
+ }
+
+ /* `NumStUpdate' gives the number of particles on this processor that want a chimie computation */
+ for(n = 0, NumStUpdate = 0; n < N_gas+N_stars; n++)
+ {
+ if(P[n].Ti_endstep == All.Ti_Current)
+ if(P[n].Type == ST)
+ NumStUpdate++;
+
+ if(P[n].Type == 0)
+ SphP[n].dMass = 0.;
+
+ }
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumStUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+
+ i = 0; /* first gas particle, because stars may be hidden among gas particles */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+
+ NSNIa_tot = 0;
+ NSNII_tot = 0;
+ NSNIa_totlocal = 0;
+ NSNII_totlocal = 0;
+ EgySN = 0;
+ EgySNlocal =0;
+ Nchimlocal = 0;
+ Nchim = 0;
+
+ Nwindlocal = 0;
+ Nwind = 0;
+
+ Noldwindlocal = 0;
+ Noldwind = 0;
+
+ Nflaglocal = 0;
+ Nflag = 0;
+
+ while(ntotleft > 0)
+ {
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < N_gas+N_stars && nexport < All.BunchSizeChimie - NTask; i++)
+
+ /* only active particles and stars */
+ if((P[i].Ti_endstep == All.Ti_Current)&&(P[i].Type == ST))
+ {
+
+
+ if(P[i].Type != ST)
+ {
+ printf("P[i].Type != ST, we better stop.\n");
+ printf("N_gas=%d (type=%d) i=%d (type=%d)\n",N_gas,P[N_gas].Type,i,P[i].Type);
+ printf("Please, check that you do not use PEANOHILBERT\n");
+ endrun(777001);
+ }
+
+
+ m = P[i].StPIdx;
+ flag_chimie = 0;
+
+
+
+
+
+ /******************************************/
+ /* do chimie */
+ /******************************************/
+
+ /*****************************************************/
+ /* look if a SN may have explode during the last step
+ /*****************************************************/
+
+
+ /* minimum live time for a given metallicity */
+ minlivetime = star_lifetime(StP[m].Metal[NELEMENTS-1],Cp.Mmax*SOLAR_MASS/All.UnitMass_in_g);
+
+
+ t1 = P[i].Ti_begstep * All.Timebase_interval / hubble_a; /* t1<t2 */
+ t2 = All.Ti_Current * All.Timebase_interval / hubble_a;
+
+ star_age = t2-StP[m].FormationTime;
+
+
+ if (star_age>minlivetime)
+ {
+
+ Nchimlocal++;
+
+ StP[m].Flag = 1; /* mark it as active */
+
+ //printf("(%d) minlivetime=%g star_age=%g (metalicity=%g Mmax=%g)\n",ThisTask,minlivetime,star_age,StP[m].Metal[NELEMENTS-1],Cp.Mmax);
+ //printf("(%d) t1=%g t2=%g StP[m].FormationTime=%g\n",ThisTask,t1,t2,StP[m].FormationTime);
+
+ a1 = t2-StP[m].FormationTime;
+ a2 = t1-StP[m].FormationTime;
+
+
+ /*
+ find m1 and m2
+ m1 = min mass of stars that may explode between t1 and t2
+ m2 = max mass of stars that may explode between t1 and t2
+ */
+
+ m1 = star_mass_from_age(StP[m].Metal[NELEMENTS-1],a1);
+
+ if (a2 >= minlivetime)
+ m2 = star_mass_from_age(StP[m].Metal[NELEMENTS-1],a2);
+ else
+ m2 = Cp.Mmax*SOLAR_MASS/All.UnitMass_in_g;
+
+
+ //printf("(%d) m1=%g (%g Msol), m2=%g (%g Msol)\n",ThisTask,m1,m1*All.UnitMass_in_g/SOLAR_MASS,m2,m2*All.UnitMass_in_g/SOLAR_MASS);
+
+
+ if (m1>m2)
+ {
+ printf("m1=%g (%g Msol) > m2=%g (%g Msol) !!!\n\n",m1,m1*All.UnitMass_in_g/SOLAR_MASS,m2,m2*All.UnitMass_in_g/SOLAR_MASS);
+ endrun(777002);
+ }
+
+
+ M0 = StP[m].InitialMass;
+ for (k=0;k<NELEMENTS;k++)
+ metals[k] = StP[m].Metal[k];
+
+
+
+ /* number of SNIa */
+ NSNIa = SNIa_rate(m1,m2)*M0;
+ /* number of SNII */
+ NSNII = SNII_rate(m1,m2)*M0;
+
+ NSNIa_totlocal += NSNIa;
+ NSNII_totlocal += NSNII;
+
+
+ /* compute ejectas */
+ Total_mass_ejection(m1,m2,M0,metals);
+
+
+ StP[m].TotalEjectedGasMass = EjectedMass[0]; /* gas mass */
+
+ for (k=0;k<NELEMENTS;k++)
+ StP[m].TotalEjectedEltMass[k] = EjectedMass[k+2]; /* metal mass */
+
+ if (StP[m].TotalEjectedGasMass>0)
+ flag_chimie=1;
+
+ /* compute injected energy */
+ StP[m].TotalEjectedEgySpec = All.ChimieSupernovaEnergy* (NSNIa + NSNII) /StP[m].TotalEjectedGasMass;
+
+ EgySNlocal += All.ChimieSupernovaEnergy* (NSNIa + NSNII);
+
+
+ /* correct mass particle */
+ P[i].Mass = P[i].Mass-StP[m].TotalEjectedGasMass;
+ float Fe,Mg;
+ Fe = StP[m].TotalEjectedEltMass[0];
+ Mg = StP[m].TotalEjectedEltMass[1];
+
+
+ }
+
+
+
+ /******************************************/
+ /* end do chimie */
+ /******************************************/
+
+ ndone++;
+
+
+ if (flag_chimie)
+ {
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ chimie_evaluate(i, 0);
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+
+ for(k = 0; k < 3; k++)
+ {
+ ChimieDataIn[nexport].Pos[k] = P[i].Pos[k];
+ ChimieDataIn[nexport].Vel[k] = P[i].Vel[k];
+ }
+ ChimieDataIn[nexport].ID = P[i].ID;
+ ChimieDataIn[nexport].Timestep = P[i].Ti_endstep - P[i].Ti_begstep;
+
+ ChimieDataIn[nexport].Hsml = StP[m].Hsml;
+ ChimieDataIn[nexport].Density = StP[m].Density;
+ ChimieDataIn[nexport].Volume = StP[m].Volume;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ ChimieDataIn[nexport].NgbMass = StP[m].NgbMass;
+#endif
+
+ ChimieDataIn[nexport].TotalEjectedGasMass = StP[m].TotalEjectedGasMass;
+ for(k = 0; k < NELEMENTS; k++)
+ ChimieDataIn[nexport].TotalEjectedEltMass[k] = StP[m].TotalEjectedEltMass[k];
+ ChimieDataIn[nexport].TotalEjectedEgySpec = StP[m].TotalEjectedEgySpec;
+
+#ifdef WITH_ID_IN_HYDRA
+ ChimieDataIn[nexport].ID = P[i].ID;
+#endif
+ ChimieDataIn[nexport].Index = i;
+ ChimieDataIn[nexport].Task = j;
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+
+ }
+
+ }
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ qsort(ChimieDataIn, nexport, sizeof(struct chimiedata_in), chimie_compare_key);
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeChimie)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&ChimieDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct chimiedata_in), MPI_BYTE,
+ recvTask, TAG_CHIMIE_A,
+ &ChimieDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct chimiedata_in), MPI_BYTE,
+ recvTask, TAG_CHIMIE_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ /* now do the imported particles */
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ chimie_evaluate(j, 1);
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ /* do a block to measure imbalance */
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeChimie)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&ChimieDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct chimiedata_out),
+ MPI_BYTE, recvTask, TAG_CHIMIE_B,
+ &ChimieDataPartialResult[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct chimiedata_out),
+ MPI_BYTE, recvTask, TAG_CHIMIE_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ source = j + noffset[recvTask];
+ place = ChimieDataIn[source].Index;
+
+// for(k = 0; k < 3; k++)
+// SphP[place].HydroAccel[k] += HydroDataPartialResult[source].Acc[k];
+//
+// SphP[place].DtEntropy += HydroDataPartialResult[source].DtEntropy;
+//#ifdef FEEDBACK
+// SphP[place].DtEgySpecFeedback += HydroDataPartialResult[source].DtEgySpecFeedback;
+//#endif
+// if(SphP[place].MaxSignalVel < HydroDataPartialResult[source].MaxSignalVel)
+// SphP[place].MaxSignalVel = HydroDataPartialResult[source].MaxSignalVel;
+//#ifdef COMPUTE_VELOCITY_DISPERSION
+// for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+// SphP[place].VelocityDispersion[k] += HydroDataPartialResult[source].VelocityDispersion[k];
+//#endif
+
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+
+ /* do final operations on results */
+ tstart = second();
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+ P[i].Mass += SphP[i].dMass;
+ SphP[i].dMass = 0.;
+ }
+ }
+
+
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+
+
+
+ /* collect some timing information */
+
+ MPI_Reduce(&timecomp, &sumt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm, &sumcomm, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance, &sumimbalance, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+
+// if(ThisTask == 0)
+// {
+// All.CPU_HydCompWalk += sumt / NTask;
+// All.CPU_HydCommSumm += sumcomm / NTask;
+// All.CPU_HydImbalance += sumimbalance / NTask;
+// }
+
+
+
+ /* collect some chimie informations */
+ MPI_Reduce(&NSNIa_totlocal, &NSNIa_tot, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&NSNII_totlocal, &NSNII_tot, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&EgySNlocal, &EgySN, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&Nchimlocal, &Nchim, 1, MPI_INT , MPI_SUM, 0, MPI_COMM_WORLD);
+
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ EgySNThermal = EgySN*(1-All.ChimieKineticFeedbackFraction);
+#else
+ EgySNThermal = 0;
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ EgySNKinetic = EgySN*All.ChimieKineticFeedbackFraction;
+
+ /* count number of wind particles */
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+ if (SphP[i].WindTime >= (All.Time-All.ChimieWindTime))
+ Nwindlocal++;
+ //else
+ // if (SphP[i].WindTime > All.TimeBegin-2*All.ChimieWindTime)
+ // Noldwindlocal++;
+
+ if (SphP[i].WindFlag)
+ Nflaglocal++;
+ }
+ }
+
+ MPI_Reduce(&Nwindlocal, &Nwind, 1, MPI_INT , MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&Noldwindlocal, &Noldwind, 1, MPI_INT , MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Allreduce(&Nflaglocal, &Nflag, 1, MPI_INT , MPI_SUM, MPI_COMM_WORLD);
+
+#else
+ EgySNKinetic = 0;
+#endif
+
+
+
+ /* write some info */
+ if (ThisTask==0)
+ {
+ fprintf(FdChimie, "%15g %10d %15g %15g %15g %15g %15g %10d %10d %10d\n",All.Time,Nchim,NSNIa_tot,NSNII_tot,EgySN,EgySNThermal,EgySNKinetic,Nwind,Noldwind,Nflag);
+ fflush(FdChimie);
+ }
+
+
+ if (Nflag>0)
+ {
+ SetMinTimeStepForActives=1;
+ if (ThisTask==0)
+ fprintf(FdLog,"%g : !!! set min timestep for active particles !!!\n",All.Time);
+ }
+
+
+
+}
+
+
+/*! This function is the 'core' of the Chemie computation. A target
+ * particle is specified which may either be local, or reside in the
+ * communication buffer.
+ */
+void chimie_evaluate(int target, int mode)
+{
+ int j, n, startnode, numngb,numngb_inbox,k;
+ FLOAT *pos,*vel;
+ //FLOAT *vel;
+ //FLOAT mass;
+ double h, h2;
+ double acc[3];
+ double dx, dy, dz;
+ double wk, r, r2, u=0;
+ double hinv=1, hinv3;
+ int target_stp;
+
+ double density;
+ double volume;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double ngbmass;
+ double p;
+#endif
+
+ double aij;
+ double ejectedGasMass;
+ double ejectedEltMass[NELEMENTS];
+ double ejectedEgySpec;
+
+ double mass_k;
+ double NewMass;
+ double fv,vi2,vj2;
+
+ double EgySpec,NewEgySpec;
+ double DeltaEntropy;
+ double DeltaVel[3];
+
+#ifndef LONGIDS
+ unsigned int id; /*!< particle identifier */
+#else
+ unsigned long long id; /*!< particle identifier */
+#endif
+
+
+ if(mode == 0)
+ {
+ pos = P[target].Pos;
+ vel = P[target].Vel;
+ id = P[target].ID;
+
+ target_stp = P[target].StPIdx;
+ h = StP[target_stp].Hsml;
+ density = StP[target_stp].Density;
+ volume = StP[target_stp].Volume;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ ngbmass = StP[target_stp].NgbMass;
+#endif
+
+ ejectedGasMass = StP[target_stp].TotalEjectedGasMass;
+ for(k=0;k<NELEMENTS;k++)
+ ejectedEltMass[k] = StP[target_stp].TotalEjectedEltMass[k];
+
+ ejectedEgySpec = StP[target_stp].TotalEjectedEgySpec;
+ }
+ else
+ {
+ pos = ChimieDataGet[target].Pos;
+ vel = ChimieDataGet[target].Vel;
+ id = ChimieDataGet[target].ID;
+ h = ChimieDataGet[target].Hsml;
+ density = ChimieDataGet[target].Density;
+ volume = ChimieDataGet[target].Volume;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ ngbmass = ChimieDataGet[target].NgbMass;
+#endif
+
+ ejectedGasMass = ChimieDataGet[target].TotalEjectedGasMass;
+ for(k=0;k<NELEMENTS;k++)
+ ejectedEltMass[k] = ChimieDataGet[target].TotalEjectedEltMass[k];
+
+ ejectedEgySpec = ChimieDataGet[target].TotalEjectedEgySpec;
+
+ }
+
+
+ /* initialize variables before SPH loop is started */
+ acc[0] = acc[1] = acc[2] = 0;
+
+ vi2 = 0;
+ for(k=0;k<3;k++)
+ vi2 += vel[k]*vel[k];
+
+ h2 = h * h;
+ hinv = 1.0 / h;
+#ifndef TWODIMS
+ hinv3 = hinv * hinv * hinv;
+#else
+ hinv3 = hinv * hinv / boxSize_Z;
+#endif
+
+
+ /* Now start the actual SPH computation for this particle */
+ startnode = All.MaxPart;
+ numngb = 0;
+ do
+ {
+ numngb_inbox = ngb_treefind_variable_for_chimie(&pos[0], h, &startnode);
+
+ for(n = 0; n < numngb_inbox; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+#ifdef PERIODIC /* now find the closest image in the given box size */
+ if(dx > boxHalf_X)
+ dx -= boxSize_X;
+ if(dx < -boxHalf_X)
+ dx += boxSize_X;
+ if(dy > boxHalf_Y)
+ dy -= boxSize_Y;
+ if(dy < -boxHalf_Y)
+ dy += boxSize_Y;
+ if(dz > boxHalf_Z)
+ dz -= boxSize_Z;
+ if(dz < -boxHalf_Z)
+ dz += boxSize_Z;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(r2 < h2)
+ {
+ numngb++;
+
+ r = sqrt(r2);
+
+ u = r * hinv;
+
+ if(u < 0.5)
+ {
+ wk = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ }
+ else
+ {
+ wk = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ }
+
+
+
+ /* normalisation using mass */
+ aij = P[j].Mass*wk/density;
+
+ /* normalisation using volume */
+ /* !!! si on utilise, il faut stoquer une nouvelle variable : OldDensity, car density est modifié plus bas... */
+ //aij = P[j].Mass/SphP[j].Density*wk/volume;
+
+
+ /* metal injection */
+ for(k=0;k<NELEMENTS;k++)
+ {
+ mass_k = SphP[j].Metal[k]*P[j].Mass; /* mass of elt k */
+ SphP[j].Metal[k] = ( mass_k + aij*ejectedEltMass[k] )/( P[j].Mass + aij*ejectedGasMass );
+ }
+
+
+ /* new mass */
+ NewMass = P[j].Mass + aij*ejectedGasMass;
+
+
+ /* new velocity */
+ vj2 = 0;
+ for(k=0;k<3;k++)
+ vj2 += SphP[j].VelPred[k]*SphP[j].VelPred[k];
+
+ fv = (P[j].Mass/NewMass) + aij*(ejectedGasMass/NewMass) * (vi2/vj2);
+
+ for(k=0;k<3;k++)
+ {
+ DeltaVel[k] = fv*SphP[j].VelPred[k] - SphP[j].VelPred[k];
+ SphP[j].VelPred[k] += DeltaVel[k];
+ P[j].Vel [k] += DeltaVel[k];
+ }
+
+ /* spec energy at current step */
+ EgySpec = SphP[j].EntropyPred / GAMMA_MINUS1 * pow(SphP[j].Density, GAMMA_MINUS1);
+
+ /* new egyspec */
+ NewEgySpec = (EgySpec )*(P[j].Mass/NewMass);
+
+ /* new density */
+ SphP[j].Density = SphP[j].Density*NewMass/P[j].Mass;
+
+
+ /* new entropy */
+ DeltaEntropy = GAMMA_MINUS1*NewEgySpec/pow(SphP[j].Density, GAMMA_MINUS1) - SphP[j].EntropyPred;
+
+ SphP[j].EntropyPred += DeltaEntropy;
+ SphP[j].Entropy += DeltaEntropy;
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ SphP[j].DeltaEgySpec += (1.-All.ChimieKineticFeedbackFraction)*(ejectedGasMass*ejectedEgySpec)* aij/NewMass;
+#endif
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ p = (All.ChimieKineticFeedbackFraction*ejectedEgySpec*ejectedGasMass)/(0.5*ngbmass*All.ChimieWindSpeed*All.ChimieWindSpeed);
+
+ double r;
+ r = get_Chimie_random_number(P[j].ID+id);
+
+
+ if ( r < p) /* we should maybe have a 2d table here... */
+ {
+
+ if (SphP[j].WindTime < (All.Time-All.ChimieWindTime)) /* not a wind particle */
+ {
+ SphP[j].WindFlag = 1;
+ SphP[j].WindTime = All.Time;
+ }
+
+ }
+#endif
+
+
+
+#ifdef CHECK_ENTROPY_SIGN
+ if ((SphP[j].EntropyPred < 0)||(SphP[j].Entropy < 0))
+ {
+ printf("\ntask=%d: entropy less than zero in chimie_evaluate !\n", ThisTask);
+ printf("ID=%d Entropy=%g EntropyPred=%g DeltaEntropy=%g\n",P[j].ID,SphP[j].Entropy,SphP[j].EntropyPred,DeltaEntropy);
+ fflush(stdout);
+ endrun(777003);
+
+ }
+#endif
+
+
+
+ /* store mass diff. */
+ SphP[j].dMass += NewMass-P[j].Mass;
+
+
+ }
+ }
+ }
+ while(startnode >= 0);
+
+
+
+ /* Now collect the result at the right place */
+ if(mode == 0)
+ {
+// for(k = 0; k < 3; k++)
+// SphP[target].HydroAccel[k] = acc[k];
+// SphP[target].DtEntropy = dtEntropy;
+//#ifdef FEEDBACK
+// SphP[target].DtEgySpecFeedback = dtEgySpecFeedback;
+//#endif
+// SphP[target].MaxSignalVel = maxSignalVel;
+//#ifdef COMPUTE_VELOCITY_DISPERSION
+// for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+// SphP[target].VelocityDispersion[k] = VelocityDispersion[k];
+//#endif
+ }
+ else
+ {
+// for(k = 0; k < 3; k++)
+// HydroDataResult[target].Acc[k] = acc[k];
+// HydroDataResult[target].DtEntropy = dtEntropy;
+//#ifdef FEEDBACK
+// HydroDataResult[target].DtEgySpecFeedback = dtEgySpecFeedback;
+//#endif
+// HydroDataResult[target].MaxSignalVel = maxSignalVel;
+//#ifdef COMPUTE_VELOCITY_DISPERSION
+// for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+// HydroDataResult[target].VelocityDispersion[k] = VelocityDispersion[k];
+//#endif
+ }
+}
+
+
+
+
+
+/*! This is a comparison kernel for a sort routine, which is used to group
+ * particles that are going to be exported to the same CPU.
+ */
+int chimie_compare_key(const void *a, const void *b)
+{
+ if(((struct chimiedata_in *) a)->Task < (((struct chimiedata_in *) b)->Task))
+ return -1;
+ if(((struct chimiedata_in *) a)->Task > (((struct chimiedata_in *) b)->Task))
+ return +1;
+ return 0;
+}
+
+
+#endif
diff --git a/cooling.c b/cooling.c
new file mode 100644
index 0000000..713793b
--- /dev/null
+++ b/cooling.c
@@ -0,0 +1,2039 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+#include <gsl/gsl_math.h>
+#include <gsl/gsl_roots.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef COOLING
+
+
+
+
+/*! initialize cooling function (the metallicity is fixed)
+ *
+ * T = temperature
+ * L0 = m000 primordial metallicity
+ * L1 = m-30
+ * L2 = m-20
+ * L3 = m-10
+ * L4 = m-05
+ * L5 = m-00 solar metallicity
+ * L6 = m+05
+ */
+int init_cooling(FLOAT metallicity)
+{
+
+ FILE *fd;
+ int n,i;
+ char line[72];
+ float T,L0,L1,L2,L3,L4,L5,L6;
+ int MetallicityIndex=4;
+
+
+ /* find the right index */
+ if (All.InitGasMetallicity<-3)
+ MetallicityIndex = 0;
+ else
+ {
+ if (All.InitGasMetallicity<-2)
+ MetallicityIndex = 1;
+ else
+ {
+ if (All.InitGasMetallicity<-1)
+ MetallicityIndex = 2;
+ else
+ {
+ if (All.InitGasMetallicity<-0.5)
+ MetallicityIndex = 3;
+ else
+ {
+ if (All.InitGasMetallicity<0)
+ MetallicityIndex = 4;
+ else
+ {
+ MetallicityIndex = 5;
+ }
+ }
+ }
+ }
+ }
+
+
+
+ fd = fopen(All.CoolingFile,"r");
+ fscanf(fd, "# %6d\n", &n);
+
+ /* allocate memory */
+ All.logT = malloc(n * sizeof(double));
+ All.logL = malloc(n * sizeof(double));
+
+ /* read empty line */
+ fgets(line, sizeof(line), fd);
+
+
+ /* read file */
+
+ for (i=0;i<n;i++){
+
+ fscanf(fd, "%f %f %f %f %f %f %f %f\n",&T,&L0,&L1,&L2,&L3,&L4,&L5,&L6);
+
+ //printf("%8.3f %8.3f\n",T,L0);
+
+ /* keep only solar values */
+ All.logT[i] = (double)T;
+
+ switch (MetallicityIndex)
+ {
+ case 0:
+ All.logL[i] = (double)L0;
+ break;
+ case 1:
+ All.logL[i] = (double)L1;
+ break;
+ case 2:
+ All.logL[i] = (double)L2;
+ break;
+ case 3:
+ All.logL[i] = (double)L3;
+ break;
+ case 4:
+ All.logL[i] = (double)L4;
+ break;
+ case 5:
+ All.logL[i] = (double)L5;
+ break;
+ case 6:
+ All.logL[i] = (double)L6;
+ break;
+ }
+ }
+
+ fclose(fd);
+
+ /* init interpolation */
+ All.acc_cooling_spline = gsl_interp_accel_alloc ();
+ All.cooling_spline = gsl_spline_alloc (gsl_interp_cspline, n);
+ gsl_spline_init (All.cooling_spline, All.logT, All.logL, n);
+
+
+#ifdef OUTPUT_COOLING_FUNCTION
+ /* test cooling */
+ double logT;
+ double l;
+ logT = 1.;
+ while(logT<8)
+ {
+
+ T = pow(10,logT);
+ l = log10(cooling_function(T));
+
+ if(ThisTask == 0)
+ printf("%8.3f %8.3f\n",logT,l);
+
+ logT = logT + 0.05;
+ }
+#endif
+
+ return 0;
+}
+
+
+
+
+/*! This function return the normalized cooling function (no metallicity dependency)
+ */
+double cooling_function(double temperature)
+{
+
+ double logT;
+
+
+ if (temperature >= All.CutofCoolingTemperature)
+ {
+
+ logT = log10(temperature);
+
+ if (logT>8.5)
+ logT = 8.5;
+
+
+ return pow(10,gsl_spline_eval (All.cooling_spline, logT, All.acc_cooling_spline));
+ }
+ else
+ return 1e-100;
+
+
+}
+
+
+
+
+
+
+
+/***************************************************************************
+
+ METALLICITY DEPENDENT COOLING
+
+ **************************************************************************/
+
+
+
+
+int init_cooling_with_metals()
+{
+
+ /*
+
+ zmin zmax slz
+ tmin tmax slt
+ FeHSolar
+ p k
+
+ */
+
+
+ FILE *fd;
+ int p,k,i,j;
+ float zmin,zmax,slz,tmin,tmax,slt,FeHSolar;
+ float lbd;
+
+ fd = fopen(All.CoolingFile,"r");
+ fscanf(fd, "%f %f %f\n", &zmin,&zmax,&slz);
+ fscanf(fd, "%f %f %f\n", &tmin,&tmax,&slt);
+ fscanf(fd, "%f\n" , &FeHSolar);
+ fscanf(fd, "%d %d\n" , &p,&k);
+
+
+ All.CoolingParameters_zmin = zmin;
+ All.CoolingParameters_zmax = zmax;
+ All.CoolingParameters_slz = slz;
+ All.CoolingParameters_tmin = tmin;
+ All.CoolingParameters_tmax = tmax;
+ All.CoolingParameters_slt = slt;
+ //All.CoolingParameters_FeHSolar = FEH_SOLAR; /* instead of FeHSolar*/
+ All.CoolingParameters_cooling_data_max = k-2;
+
+
+ for (i=0;i<p;i++)
+ for (j=0;j<k;j++)
+ {
+ fscanf(fd, "%f\n" ,&lbd);
+ All.CoolingParameters_cooling_data[i][j]=lbd;
+ }
+
+ fclose(fd);
+
+
+
+#ifdef OUTPUT_COOLING_FUNCTION
+ /* test cooling */
+ double logT,T;
+ double l;
+ double metal;
+ logT = 1.;
+
+
+
+ metal = (pow(10,All.InitGasMetallicity)-1e-10)*All.CoolingParameters_FeHSolar;
+ while(logT<8)
+ {
+
+ T = pow(10,logT);
+ l = log10(cooling_function_with_metals(T,metal));
+
+ if(ThisTask == 0)
+ printf("%8.3f %8.3f\n",logT,l);
+
+ logT = logT + 0.05;
+ }
+#endif
+
+
+
+
+
+
+
+ return 0;
+
+
+}
+
+/*! This function return the normalized cooling function, that depends on metallicity
+ */
+double cooling_function_with_metals(double temperature,double metal)
+{
+
+
+ double cooling;
+ double T,Z;
+ double rt, rz, ft, fz, v1, v2, v;
+ int it,iz,itp,izp;
+
+ double zmin,zmax,slz,tmin,tmax,slt,FeHSolar,cooling_data_max;
+
+ zmin = All.CoolingParameters_zmin;
+ zmax = All.CoolingParameters_zmax;
+ slz = All.CoolingParameters_slz;
+ tmin = All.CoolingParameters_tmin;
+ tmax = All.CoolingParameters_tmax;
+ slt = All.CoolingParameters_slt;
+ FeHSolar = All.CoolingParameters_FeHSolar;
+ cooling_data_max = All.CoolingParameters_cooling_data_max;
+
+
+ cooling = 0.0;
+
+
+ T = log10( temperature );
+ Z = log10( metal/FeHSolar + 1.e-10 );
+
+
+ if (Z>zmax)
+ {
+ /*print *,'Warning: Z>Zmax for',i*/
+ Z=zmax;
+ }
+
+ if (Z < zmin)
+ {
+
+ rt = (T-tmin)/slt;
+
+ it = (int)rt;
+
+ if (it < cooling_data_max )
+ it = (int)rt;
+ else
+ it = cooling_data_max;
+
+ itp = it+1;
+
+ ft = rt - it;
+
+ fz = ( 10. + Z )/( 10. + zmin);
+
+
+ v1 = ft*( All.CoolingParameters_cooling_data[1][itp] -All.CoolingParameters_cooling_data[1][it] ) + All.CoolingParameters_cooling_data[1][it];
+ v2 = ft*( All.CoolingParameters_cooling_data[0][itp] -All.CoolingParameters_cooling_data[0][it] ) + All.CoolingParameters_cooling_data[0][it];
+ v = v2 + fz*(v1-v2);
+
+ }
+ else
+ {
+
+ rt = (T-tmin)/slt;
+ rz = (Z-zmin)/slz+1.0;
+
+ it = (int)rt;
+
+ if (it < cooling_data_max )
+ it = (int)rt;
+ else
+ it = cooling_data_max;
+
+
+ iz = (int)rz;
+
+ itp = it+1;
+ izp = iz+1;
+
+ ft = rt - it;
+ fz = rz - iz;
+
+ v1 = ft*(All.CoolingParameters_cooling_data[izp][itp] - All.CoolingParameters_cooling_data[izp][it]) + All.CoolingParameters_cooling_data[izp][it];
+ v2 = ft*(All.CoolingParameters_cooling_data[iz][itp] - All.CoolingParameters_cooling_data[iz][it]) + All.CoolingParameters_cooling_data[iz][it];
+ v = v2 + fz*(v1-v2);
+
+
+ }
+
+ cooling = pow(10,v);
+
+
+
+ return cooling;
+
+}
+
+
+
+
+
+
+
+
+/***************************************************************************
+
+ END OF METALLICITY DEPENDENT COOLING
+
+ **************************************************************************/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+/*! \file cooling.c
+ * \brief Compute gas cooling
+ *
+*/
+
+static double hubble_a, a3inv;
+static double eV = 1.6022000e-12;
+static double normfacJ0 = 0.74627;
+static double J0min = 1.e-29;
+static double alpha = 1.0;
+
+static int Norderweinberg = 7; /* polynom order+1 */
+static double coefweinberg[7][6];
+static double z;
+static double J0;
+
+static double Cte_G_gHI;
+static double Cte_G_gHeI;
+static double Cte_G_gHeII;
+static double Cte_heating_radiative_HI;
+static double Cte_heating_radiative_HeI;
+static double Cte_heating_radiative_HeII;
+
+
+/*
+ * init some variables that depends only on redshift
+ */
+void init_from_new_redshift(double Redshift)
+{
+
+ /* init weinberg coeff */
+
+ coefweinberg[0][0] = -0.31086729929951613e+002;
+ coefweinberg[1][0] = 0.34803667059463761e+001;
+ coefweinberg[2][0] = -0.15145716066316397e+001;
+ coefweinberg[3][0] = 0.54649951450632972e+000;
+ coefweinberg[4][0] = -0.16395924120387340e+000;
+ coefweinberg[5][0] = 0.25197466148524143e-001;
+ coefweinberg[6][0] = -0.15352763785487806e-002;
+
+ coefweinberg[0][1] = -0.31887274113252204e+002;
+ coefweinberg[1][1] = 0.44178493140927095e+001;
+ coefweinberg[2][1] = -0.20158132553082293e+001;
+ coefweinberg[3][1] = 0.64080497292269134e+000;
+ coefweinberg[4][1] = -0.15981267091909040e+000;
+ coefweinberg[5][1] = 0.22056900050237707e-001;
+ coefweinberg[6][1] = -0.12837570029562849e-002;
+
+ coefweinberg[0][2] = -0.35693331167978656e+002;
+ coefweinberg[1][2] = 0.20207245722165794e+001;
+ coefweinberg[2][2] = -0.76856976101363744e-001;
+ coefweinberg[3][2] = -0.75691470654320359e-001;
+ coefweinberg[4][2] = -0.54502220282734729e-001;
+ coefweinberg[5][2] = 0.20633345104660583e-001;
+ coefweinberg[6][2] = -0.18410307456285177e-002;
+
+ coefweinberg[0][3] = -0.56967559787460921e+002;
+ coefweinberg[1][3] = 0.38601174525546353e+001;
+ coefweinberg[2][3] = -0.18318926655684415e+001;
+ coefweinberg[3][3] = 0.67360594266440688e+000;
+ coefweinberg[4][3] = -0.18983466813215341e+000;
+ coefweinberg[5][3] = 0.27768907786915147e-001;
+ coefweinberg[6][3] = -0.16330066969315893e-002;
+
+ coefweinberg[0][4] = -0.56977907250821026e+002;
+ coefweinberg[1][4] = 0.38686249565302266e+001;
+ coefweinberg[2][4] = -0.13330942368518774e+001;
+ coefweinberg[3][4] = 0.33988839029092172e+000;
+ coefweinberg[4][4] = -0.98997915675929332e-001;
+ coefweinberg[5][4] = 0.16781612113050747e-001;
+ coefweinberg[6][4] = -0.11514328893746039e-002;
+
+ coefweinberg[0][5] = -0.59825233828609278e+002;
+ coefweinberg[1][5] = 0.21898162706563347e+001;
+ coefweinberg[2][5] = -0.42982055888598525e+000;
+ coefweinberg[3][5] = 0.50312144291614215e-001;
+ coefweinberg[4][5] = -0.61550639239553132e-001;
+ coefweinberg[5][5] = 0.18017109270959387e-001;
+ coefweinberg[6][5] = -0.15438891584271634e-002;
+
+
+
+ z = Redshift;
+ J0 = J_0();
+
+ /* here, we initialize the ctes that uses J_nu(z) */
+
+ /* Tessier */
+ /*
+ Cte_G_gHI = G_gHI();
+ Cte_G_gHeI = G_gHeI();
+ Cte_G_gHeII = G_gHeII();
+ Cte_heating_radiative_HI = heating_radiative_HI();
+ Cte_heating_radiative_HeI = heating_radiative_HeI();
+ Cte_heating_radiative_HeII = heating_radiative_HeII();
+ */
+
+ /* Theuns */
+ /*
+ Cte_G_gHI = G_gHI_t(J0);
+ Cte_G_gHeI = G_gHeI_t(J0);
+ Cte_G_gHeII = G_gHeII_t(J0);
+ Cte_heating_radiative_HI = heating_radiative_HI_t(J0);
+ Cte_heating_radiative_HeI = heating_radiative_HeI_t(J0);
+ Cte_heating_radiative_HeII = heating_radiative_HeII_t(J0);
+ */
+
+ /* Weinberg */
+ Cte_G_gHI = G_gHI_w();
+ Cte_G_gHeI = G_gHeI_w();
+ Cte_G_gHeII = G_gHeII_w();
+ Cte_heating_radiative_HI = heating_radiative_HI_w();
+ Cte_heating_radiative_HeI = heating_radiative_HeI_w();
+ Cte_heating_radiative_HeII = heating_radiative_HeII_w();
+
+}
+
+/*
+ * J0
+ */
+
+double J_0()
+{
+ double Fz;
+
+ if (z > 6)
+ Fz = 0;
+ else
+ {
+ if (z > 3)
+ Fz = 4/(z+1);
+ else
+ {
+ if (z > 2)
+ Fz = 1;
+ else
+ Fz = pow(((1+z)/3.),3);
+ }
+ }
+
+ return 1.0e-22*Fz;
+}
+
+/*
+ * UV background intensity
+ */
+
+double J_nu(double e)
+{
+ double e_L;
+ e_L = 13.598*eV;
+ return (e_L/e)*J_0();
+}
+
+/*
+ * sigma_rad
+ */
+
+double sigma_rad_HI(double e)
+{
+ double xxx,alph,e_i;
+
+ e_i = 13.598 *eV;
+ xxx = e/e_i;
+ alph = sqrt(xxx-1.0);
+ return 6.30e-18/pow(xxx,4)*exp(4.0-4.0*atan(alph)/alph) /(1.0-exp(-TWOPI/alph));
+}
+
+
+double sigma_rad_HeI(double e)
+{
+ double xxx,alph,e_i;
+
+ e_i = 24.587 *eV;
+ xxx = e/e_i;
+ alph = sqrt(xxx-1.0);
+ return 7.42e-18*(1.660/pow(xxx,2.050)-0.660/pow(xxx,3.050));
+}
+
+double sigma_rad_HeII(double e)
+{
+ double xxx,alph,e_i;
+
+ e_i = 54.416 *eV;
+ xxx = e/e_i;
+ alph = sqrt(xxx-1.0);
+ return 1.58e-18/pow(xxx,4)*exp(4.0-4.0*atan(alph)/alph)/(1.0-exp(-TWOPI/alph));
+}
+
+
+/*
+ * cooling rates
+ */
+
+/* Bremstrahlung */
+double cooling_bremstrahlung_HI(double T)
+{
+ return 1.42e-27*sqrt(T)*(1.10+0.340*exp(-pow((5.50-log10(T)),2) /3.0));
+}
+
+double cooling_bremstrahlung_HeI(double T)
+{
+ return 1.42e-27*sqrt(T)*(1.10+0.340*exp(-pow((5.50-log10(T)),2) /3.0));
+}
+
+double cooling_bremstrahlung_HeII(double T)
+{
+ return 5.68e-27*sqrt(T)*(1.10+0.340*exp(-pow((5.50-log10(T)),2) /3.0));
+}
+
+/* Ionization */
+double cooling_ionization_HI(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 2.54e-21*sqrt(T)*exp(-157809.1/T)/(1+sqrt(T5));
+}
+
+double cooling_ionization_HeI(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 1.88e-21*sqrt(T)*exp(-285335.4/T)/(1+sqrt(T5));
+}
+
+double cooling_ionization_HeII(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 9.90e-22*sqrt(T)*exp(-631515.0/T)/(1+sqrt(T5));
+}
+
+/* Recombination */
+double cooling_recombination_HI(double T)
+{
+ double T3,T6;
+ T3 = T/1e3;
+ T6 = T/1e6;
+ return 8.70e-27*sqrt(T)/pow(T3,0.2)/(1.0+pow(T6,0.7));
+}
+
+double cooling_recombination_HeI(double T)
+{
+ return 1.55e-26*pow(T,0.3647);
+}
+
+double cooling_recombination_HeII(double T)
+{
+ double T3,T6;
+ T3 = T/1e3;
+ T6 = T/1e6;
+ return 3.48e-26*sqrt(T)/pow(T3,0.2)/(1.0+pow(T6,0.7));
+}
+
+/* Dielectric Recombination */
+double cooling_dielectric_recombination(double T)
+{
+ return 1.24e-13*pow(T,-1.5)*exp(-470000.0/T)*(1.0+0.3*exp(-94000.0/T));
+}
+
+/* Ecitation cooling (line cooling) */
+double cooling_excitation_HI(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 7.50e-19*exp(-118348.0/T)/(1+sqrt(T5));
+}
+
+double cooling_excitation_HII(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 5.54e-17/pow(T,0.397)*exp(-473638.0/T)/(1+sqrt(T5));
+}
+
+/* Compton cooling */
+double cooling_compton(double T)
+{
+ return 5.406e-36*(T-2.7*(1+z))*pow((1+z),4);
+}
+
+/*
+ * recombination rates (taux_rec)
+ */
+
+double A_HII(double T)
+{
+ double T3,T6;
+ T3 = T/1e3;
+ T6 = T/1e6;
+ return 6.30e-11/sqrt(T)/pow(T3,0.2)/(1+pow(T6,0.7));
+}
+
+double A_HeIId(double T)
+{
+ return 1.9e-3/pow(T,1.50)*exp(-470000.0/T)*(1.0+0.30*exp(-94000.0/T));
+}
+
+double A_HeII(double T)
+{
+ return 1.5e-10/pow(T,0.6353) + A_HeIId(T);
+}
+
+double A_HeIII(double T)
+{
+ double T3,T6;
+ T3 = T/1e3;
+ T6 = T/1e6;
+ return 3.36e-10/sqrt(T)/pow(T3,0.2)/(1.0+pow(T6,0.7));
+}
+
+/*
+ * collisional rates (taux_ion)
+ */
+
+double G_HI(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 1.17e-10*sqrt(T)*exp(-157809.1/T)/(1.0+sqrt(T5));
+}
+
+double G_HeI(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 2.38e-11*sqrt(T)*exp(-285335.4/T)/(1.0+sqrt(T5));
+}
+
+double G_HeII(double T)
+{
+ double T5;
+ T5 = T/1e5;
+ return 5.68e-12*sqrt(T)*exp(-631515.0/T)/(1.0+sqrt(T5));
+}
+
+/*
+ * photoionisation rates (depend only on z)
+ */
+
+double G_gHI()
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 13.598*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+ while (error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2*TWOPI*J_nu(e)*sigma_rad_HI(e)*de/e;
+ integ = integ + error;
+ error = error/fabs(integ);
+ }
+
+ return integ/PLANCK;
+}
+
+
+double G_gHeI()
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 24.587*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+ while (error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2*TWOPI*J_nu(e)*sigma_rad_HeI(e)*de/e;
+ integ = integ + error;
+ error = error/fabs(integ);
+ }
+
+ return integ/PLANCK;
+}
+
+double G_gHeII()
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 54.416*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+ while (error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2*TWOPI*J_nu(e)*sigma_rad_HeII(e)*de/e;
+ integ = integ + error;
+ error = error/fabs(integ);
+ }
+
+ return integ/PLANCK;
+}
+
+
+double G_gHI_t(double J0)
+{
+ return 1.26e10*J0/(3.0+alpha);
+
+}
+
+double G_gHeI_t(double J0)
+{
+ return 1.48e10*J0*pow(0.5530,alpha) *(1.660/(alpha+2.050)-0.660/(alpha+3.050));
+}
+
+double G_gHeII_t(double J0)
+{
+ return 3.34e9*J0*pow(0.2490,alpha)/(3.0+alpha);
+}
+
+
+
+
+double G_gHI_w()
+{
+ double taux_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][0]*pow(zz,i);
+ taux_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ taux_rad_weinbergint=0.0;
+
+ tt=G_gHI_t(J0min);
+ if (taux_rad_weinbergint < tt)
+ taux_rad_weinbergint=tt;
+
+ return taux_rad_weinbergint;
+}
+
+double G_gHeI_w()
+{
+ double taux_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][1]*pow(zz,i);
+ taux_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ taux_rad_weinbergint=0.0;
+
+ tt=G_gHeI_t(J0min);
+ if (taux_rad_weinbergint < tt)
+ taux_rad_weinbergint=tt;
+
+ return taux_rad_weinbergint;
+}
+
+
+double G_gHeII_w()
+{
+ double taux_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][2]*pow(zz,i);
+ taux_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ taux_rad_weinbergint=0.0;
+
+ tt=G_gHeII_t(J0min);
+ if (taux_rad_weinbergint < tt)
+ taux_rad_weinbergint=tt;
+
+ return taux_rad_weinbergint;
+}
+
+
+
+
+
+/*
+ * heating rates (depend only on z)
+ */
+
+double heating_radiative_HI() /* use J_nu */
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 13.598*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+
+ while(error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2.0*TWOPI*J_nu(e)*sigma_rad_HI(e)*(e/e_i-1.0)*de/e;
+ integ = integ + error;
+ error=error/fabs(integ);
+ }
+ return integ/PLANCK*e_i;
+}
+
+double heating_radiative_HeI() /* use J_nu */
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 24.587*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+
+ while(error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2.0*TWOPI*J_nu(e)*sigma_rad_HeI(e)*(e/e_i-1.0)*de/e;
+ integ = integ + error;
+ error=error/fabs(integ);
+ }
+ return integ/PLANCK*e_i;
+}
+
+
+double heating_radiative_HeII() /* use J_nu */
+{
+ double e_i,integ,e,de,error;
+
+ e_i = 54.416*eV;
+ integ = 0.0;
+ e = e_i;
+ de = e/100.0;
+ error = 1.0;
+
+ while(error>1.e-6)
+ {
+ e = e + de;
+ de = e/100.0;
+ error = 2.0*TWOPI*J_nu(e)*sigma_rad_HeII(e)*(e/e_i-1.0)*de/e;
+ integ = integ + error;
+ error=error/fabs(integ);
+ }
+ return integ/PLANCK*e_i;
+}
+
+
+double heating_radiative_HI_t(double J0) /* use Theuns */
+{
+ return (2.91e-1*J0/(2.0+alpha))/(3.0+alpha);
+}
+
+double heating_radiative_HeI_t(double J0) /* use Theuns */
+{
+ return 5.84e-1*J0*pow(0.5530,alpha)*(1.660/(alpha+1.050)-2.320/(alpha+2.050)+0.660/(alpha+3.050));
+}
+
+double heating_radiative_HeII_t(double J0) /* use Theuns */
+{
+ return (2.92e-1*J0*pow(0.2490,alpha)/(2.0+alpha))/(3.0+alpha);
+}
+
+
+
+
+double heating_radiative_HI_w() /* use weinberg coeff */
+{
+ double heat_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][3]*pow(zz,i);
+ heat_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ heat_rad_weinbergint=0.0;
+
+ tt=heating_radiative_HI_t(J0min);
+ if (heat_rad_weinbergint < tt)
+ heat_rad_weinbergint=tt;
+
+ return heat_rad_weinbergint;
+}
+
+double heating_radiative_HeI_w() /* use weinberg coeff */
+{
+ double heat_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][4]*pow(zz,i);
+ heat_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ heat_rad_weinbergint=0.0;
+
+ tt=heating_radiative_HeI_t(J0min);
+ if (heat_rad_weinbergint < tt)
+ heat_rad_weinbergint=tt;
+
+ return heat_rad_weinbergint;
+}
+
+double heating_radiative_HeII_w() /* use weinberg coeff */
+{
+ double heat_rad_weinbergint;
+ double hh,tt,zz;
+ int i;
+
+ if (z < 8.50)
+ {
+ hh=0.0;
+ zz=dmax(z,1.0e-15);
+ for (i=0;i<Norderweinberg;i++)
+ hh=hh+coefweinberg[i][5]*pow(zz,i);
+ heat_rad_weinbergint=normfacJ0*exp(hh);
+ }
+ else
+ heat_rad_weinbergint=0.0;
+
+ tt=heating_radiative_HeII_t(J0min);
+ if (heat_rad_weinbergint < tt)
+ heat_rad_weinbergint=tt;
+
+ return heat_rad_weinbergint;
+}
+
+
+
+
+
+
+
+double heating_compton()
+{
+ /* Abel, Tom; Haehnelt, Martin G.Apj 520 */
+ //return 5.406e-36*2.726*pow((1+z),5); /* from Ramses */
+
+ //if (z>6)
+ // return 0;
+ //else
+ // return 1.25e-31*pow((1+z),13/3.);
+
+ return 0;
+}
+
+
+
+
+void compute_densities(double T,double X,double *pn_H, double *pn_HI,double *pn_HII,double *pn_HEI,double *pn_HEII,double *pn_HEIII,double *pn_E,double *pmu)
+{
+
+ double Y,yy,x1;
+ double t_rad_HI,t_rec_HI,t_ion_HI;
+ double t_rad_HEI,t_rec_HEI,t_ion_HEI;
+ double t_rad_HEII,t_rec_HEII,t_ion_HEII;
+ double t_ion2_HI,t_ion2_HEI,t_ion2_HEII;
+ double err_nE;
+ double n_T;
+ double n_H,n_HI,n_HII,n_HEI,n_HEII,n_HEIII,n_E,mu;
+
+ Y = 1-X;
+ yy = Y/(4-4*Y);
+
+ t_rad_HI = Cte_G_gHI;
+ t_rec_HI = A_HII(T);
+ t_ion_HI = G_HI(T);
+
+ t_rad_HEI = Cte_G_gHeI;
+ t_rec_HEI = A_HeII(T);
+ t_ion_HEI = G_HeI(T);
+
+ t_rad_HEII = Cte_G_gHeII;
+ t_rec_HEII = A_HeIII(T);
+ t_ion_HEII = G_HeII(T);
+
+ n_H = *pn_H;
+
+ n_E = n_H;
+ err_nE = 1.;
+
+
+ while(err_nE > 1.e-8)
+ {
+
+ /* compute densities (Ramses implementation) */
+ t_ion2_HI = t_ion_HI + t_rad_HI /dmax(n_E,1e-15*n_H);
+ t_ion2_HEI = t_ion_HEI + t_rad_HEI /dmax(n_E,1e-15*n_H);
+ t_ion2_HEII = t_ion_HEII + t_rad_HEII/dmax(n_E,1e-15*n_H);
+
+ n_HI = t_rec_HI/(t_ion2_HI+t_rec_HI)*n_H;
+ n_HII = t_ion2_HI/(t_ion2_HI+t_rec_HI)*n_H;
+
+ x1 = (t_rec_HEII*t_rec_HEI+t_ion2_HEI*t_rec_HEII+t_ion2_HEII*t_ion2_HEI);
+
+ n_HEIII = yy*t_ion2_HEII*t_ion2_HEI/x1*n_H;
+ n_HEII = yy*t_ion2_HEI *t_rec_HEII/x1*n_H;
+ n_HEI = yy*t_rec_HEII *t_rec_HEI /x1*n_H;
+
+ err_nE = fabs((n_E - (n_HII + n_HEII + 2.*n_HEIII))/n_H);
+ n_E = 0.5*n_E+0.5*(n_HII + n_HEII + 2.*n_HEIII);
+
+ }
+
+ n_T = n_HI + n_HII+ n_HEI+ n_HEII+ n_HEIII+ n_E;
+ mu = n_H/X/n_T;
+
+ *pn_H = n_H;
+ *pn_HI = n_HI;
+ *pn_HII = n_HII;
+ *pn_HEI = n_HEI;
+ *pn_HEII = n_HEII;
+ *pn_HEIII = n_HEIII;
+ *pn_E = n_E;
+ *pmu = mu;
+
+
+}
+
+void print_cooling(double T,
+ double c1,double c2,double c3,double c4,double c5,double c6,double c7,double c8,double c9,
+ double c10,double c11,double c12,double c13,double h1, double h2, double h3, double h4)
+{
+
+ double ctot,htot,chtot;
+
+ ctot = c1+c2+c3+c4+c5+c6+c7+c8+c9+c10+c11+c12+c13;
+ htot = h1+h2+h3+h4;
+ chtot= ctot - htot;
+
+ printf("%g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g\n",T,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,h1,h2,h3,h4,ctot,htot,chtot);
+
+}
+
+
+void compute_cooling_from_T_and_Nh(double T,double X,double n_H,
+ double *c1,double *c2,double *c3,double *c4,double *c5,double *c6,double *c7,double *c8,double *c9,
+ double *c10,double *c11,double *c12,double *c13,double *h1, double *h2, double *h3, double *h4)
+{
+
+ double n_HI,n_HII,n_HEI,n_HEII,n_HEIII,n_E,mu;
+ double nH2;
+ //double c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13;
+ //double h1,h2,h3,h4;
+
+ compute_densities(T,X,&n_H,&n_HI,&n_HII,&n_HEI,&n_HEII,&n_HEIII,&n_E,&mu);
+
+ nH2 = n_H*n_H;
+
+ /*
+ * compute cooling
+ */
+
+ /* Bremstrahlung (cool_bre) */
+ *c1 = cooling_bremstrahlung_HI(T) *n_E*n_HII /nH2;
+ *c2 = cooling_bremstrahlung_HeI(T) *n_E*n_HEII /nH2;
+ *c3 = cooling_bremstrahlung_HeII(T) *n_E*n_HEIII/nH2;
+
+
+ /* Ionization cooling (cool_ion) */
+ *c4 = cooling_ionization_HI(T) *n_E*n_HI /nH2;
+ *c5 = cooling_ionization_HeI(T) *n_E*n_HEI /nH2;
+ *c6 = cooling_ionization_HeII(T) *n_E*n_HEII /nH2;
+
+
+ /* Recombination cooling (cool_rec) */
+ *c7 = cooling_recombination_HI(T) *n_E*n_HII /nH2;
+ *c8 = cooling_recombination_HeI(T) *n_E*n_HEII /nH2;
+ *c9 = cooling_recombination_HeII(T) *n_E*n_HEIII/nH2;
+
+ /* Dielectric recombination cooling (cool_die) */
+ *c10 = cooling_dielectric_recombination(T) *n_E*n_HEII /nH2;
+
+ /* Line cooling (cool_exc) */
+ *c11 = cooling_excitation_HI(T) *n_E*n_HI /nH2;
+ *c12 = cooling_excitation_HII(T) *n_E*n_HEII /nH2;
+
+ /* Compton cooling (cool_com) */
+ *c13 = cooling_compton(T) *n_E /nH2; /* !! dep on z */
+
+
+ /*
+ * compute heating
+ */
+
+ /* Radiative heating (h_rad_spec) */
+ *h1 = Cte_heating_radiative_HI *n_HI /nH2;
+ *h2 = Cte_heating_radiative_HeI *n_HEI /nH2;
+ *h3 = Cte_heating_radiative_HeII *n_HEII /nH2;
+
+
+ /* Compton heating (heat_com) */
+ *h4 = heating_compton() *n_E /nH2; /* !! dep on z */
+
+}
+
+
+
+double compute_cooling_from_Egyspec_and_Density(double Egyspec,double Density,double *MeanWeight)
+{
+
+ double T,mu,n_H;
+ double n_HI,n_HII,n_HEI,n_HEII,n_HEIII,n_E;
+
+ double err_mu,mu_left,mu_right,mu_old;
+ int niter;
+
+ double c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13;
+ double h1,h2,h3,h4;
+
+ double nH2;
+
+ /* Hydrogen density (cgs) */
+ n_H = HYDROGEN_MASSFRAC * Density/ PROTONMASS;
+
+
+ /* itterate to find the right mu and T */
+
+ err_mu=1.;
+ mu_left=0.5;
+ mu_right=1.3;
+ niter=0;
+
+ while ( (err_mu > 1.e-4) && (niter <= 50) )
+ {
+
+ mu_old=0.5*(mu_left+mu_right);
+
+ /* compute temperature */
+ T = GAMMA_MINUS1 *mu_old*PROTONMASS/BOLTZMANN *Egyspec;
+
+ /* compute all */
+ compute_densities(T,HYDROGEN_MASSFRAC,&n_H,&n_HI,&n_HII,&n_HEI,&n_HEII,&n_HEIII,&n_E,&mu);
+
+ err_mu = (mu-mu_old)/mu_old;
+
+
+ if(err_mu>0.)
+ {
+ mu_left =0.5*(mu_left+mu_right);
+ mu_right=mu_right;
+ }
+ else
+ {
+ mu_left =mu_left;
+ mu_right=0.5*(mu_left+mu_right);
+ }
+
+ err_mu=fabs(err_mu);
+ niter=niter+1;
+
+ }
+
+ if (niter > 50)
+ printf("ERROR : too many iterations.");
+
+ *MeanWeight = 0.5*(mu_left+mu_right);
+
+ /* now, compute cooling */
+
+ nH2 = n_H*n_H;
+
+ /*
+ * compute cooling
+ */
+
+ /* Bremstrahlung (cool_bre) */
+ c1 = cooling_bremstrahlung_HI(T) *n_E*n_HII /nH2;
+ c2 = cooling_bremstrahlung_HeI(T) *n_E*n_HEII /nH2;
+ c3 = cooling_bremstrahlung_HeII(T) *n_E*n_HEIII/nH2;
+
+
+ /* Ionization cooling (cool_ion) */
+ c4 = cooling_ionization_HI(T) *n_E*n_HI /nH2;
+ c5 = cooling_ionization_HeI(T) *n_E*n_HEI /nH2;
+ c6 = cooling_ionization_HeII(T) *n_E*n_HEII /nH2;
+
+
+ /* Recombination cooling (cool_rec) */
+ c7 = cooling_recombination_HI(T) *n_E*n_HII /nH2;
+ c8 = cooling_recombination_HeI(T) *n_E*n_HEII /nH2;
+ c9 = cooling_recombination_HeII(T) *n_E*n_HEIII/nH2;
+
+ /* Dielectric recombination cooling (cool_die) */
+ c10 = cooling_dielectric_recombination(T) *n_E*n_HEII /nH2;
+
+ /* Line cooling (cool_exc) */
+ c11 = cooling_excitation_HI(T) *n_E*n_HI /nH2;
+ c12 = cooling_excitation_HII(T) *n_E*n_HEII /nH2;
+
+ /* Compton cooling (cool_com) */
+ c13 = cooling_compton(T) *n_E /nH2; /* !! dep on z */
+
+
+ /*
+ * compute heating
+ */
+
+ /* Radiative heating (h_rad_spec) */
+ h1 = Cte_heating_radiative_HI *n_HI /nH2;
+ h2 = Cte_heating_radiative_HeI *n_HEI /nH2;
+ h3 = Cte_heating_radiative_HeII *n_HEII /nH2;
+
+
+ /* Compton heating (heat_com) */
+ h4 = heating_compton() *n_E /nH2; /* !! dep on z */
+
+ /* output info */
+ //print_cooling(T,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,h1,h2,h3,h4);
+
+ c1 = dmax(c1,0);
+ c2 = dmax(c2,0);
+ c3 = dmax(c3,0);
+ c4 = dmax(c4,0);
+ c5 = dmax(c5,0);
+ c6 = dmax(c6,0);
+ c7 = dmax(c7,0);
+ c8 = dmax(c8,0);
+ c9 = dmax(c9,0);
+ c10 = dmax(c10,0);
+ c11 = dmax(c11,0);
+ c12 = dmax(c12,0);
+ c13 = dmax(c13,0);
+ h1 = dmax(h1,0);
+ h2 = dmax(h2,0);
+ h3 = dmax(h3,0);
+ h4 = dmax(h4,0);
+
+ return (c1+c2+c3+c4+c5+c6+c7+c8+c9+c10+c11+c12+c13) - (h1+h2+h3+h4);
+
+}
+
+
+
+struct cooling_solver_params
+{
+ double Entropy;
+ double Density;
+ int Phase;
+ int i;
+ double DtEntropyVisc;
+ double dt;
+ double hubble_a;
+};
+
+double cooling_solver_function(double EntropyVar, void *params)
+{
+ struct cooling_solver_params *p = (struct cooling_solver_params *) params;
+ double Entropy = p->Entropy;
+ double Density = p->Density;
+ int Phase = p->Phase;
+ int i = p->i;
+ double DtEntropyVisc = p->DtEntropyVisc;
+ double dt = p->dt;
+ double hubble_a = p->hubble_a;
+
+ double DtEntropyRadSph=0;
+
+#ifdef MULTIPHASE
+ switch (Phase)
+ {
+ case GAS_SPH:
+ DtEntropyRadSph = -GAMMA_MINUS1*pow(Density,-GAMMA)*lambda(Density,EntropyVar,Phase,i)/hubble_a;
+ break;
+ case GAS_STICKY:
+ case GAS_DARK:
+ DtEntropyRadSph = -1/(Density * a3inv) *lambda(Density,EntropyVar,Phase,i)/hubble_a;
+ break;
+ }
+#else
+ DtEntropyRadSph = -GAMMA_MINUS1*pow(Density,-GAMMA)*lambda(Density,EntropyVar,Phase,i)/hubble_a;
+#endif
+
+
+ return Entropy + (DtEntropyVisc + DtEntropyRadSph)*dt - EntropyVar;
+};
+
+
+
+
+
+/*! This function compute the new Entropy due to isochoric cooling
+ * using an implicit iteration scheme
+ */
+double DoCooling(FLOAT Density,FLOAT Entropy,int Phase,int i,FLOAT DtEntropyVisc, double dt, double hubble_a)
+{
+
+ double EntropyNew;
+ double Entropy_lo=0, Entropy_hi=0;
+ double lo,hi;
+
+ int status;
+ int iter = 0;
+ int max_iter = 100;
+
+ const gsl_root_fsolver_type *T;
+ gsl_root_fsolver *s;
+ gsl_function F;
+ struct cooling_solver_params params = {(double)Entropy,(double)Density,(int)Phase,(int)i,(double)DtEntropyVisc,(double)dt,(double)hubble_a};
+
+
+ F.function = &cooling_solver_function;
+ F.params = &params;
+
+ T = gsl_root_fsolver_brent;
+ s = gsl_root_fsolver_alloc (T);
+
+
+ Entropy_lo = 0.5*Entropy;
+ Entropy_hi = 1.1*Entropy;
+
+
+ lo = cooling_solver_function(Entropy_lo,&params);
+ hi = cooling_solver_function(Entropy_hi,&params);
+
+
+ if (lo*hi>0)
+ {
+
+ do
+ {
+ Entropy_hi = 2* Entropy_hi;
+ Entropy_lo = 0.5*Entropy_lo;
+
+ lo = cooling_solver_function(Entropy_lo,&params);
+ hi = cooling_solver_function(Entropy_hi,&params);
+
+ //printf("here, we need to iterate...\n");
+
+
+ }
+ while (lo*hi>0);
+ }
+
+
+
+
+
+ gsl_root_fsolver_set (s, &F, Entropy_lo, Entropy_hi);
+
+
+ do
+ {
+ iter++;
+ status = gsl_root_fsolver_iterate (s);
+ EntropyNew = gsl_root_fsolver_root (s);
+ Entropy_lo = gsl_root_fsolver_x_lower (s);
+ Entropy_hi = gsl_root_fsolver_x_upper (s);
+ status = gsl_root_test_interval (Entropy_lo, Entropy_hi,0, 0.001);
+
+
+ }
+ while (status == GSL_CONTINUE && iter < max_iter);
+
+ gsl_root_fsolver_free (s);
+
+
+ if (status!=GSL_SUCCESS)
+ {
+ printf("WARNING, HERE WE DO NOT CONVERGE...%g %g\n",Entropy_lo,Entropy_hi);
+ endrun(3737);
+ }
+
+
+
+ return EntropyNew;
+
+}
+
+
+
+/*! This function computes the entropy variation due to the cooling.
+ * Cooling is computed only for sph active particles.
+ */
+void cooling()
+{
+
+ int i;
+ double dt=0;
+ double EntropyNew;
+
+ /* set the right Redshift and compute value indep of Temperature */
+ if (All.CoolingType==1)
+ init_from_new_redshift(1.0 / (All.Time) - 1);
+
+ if(All.ComovingIntegrationOn)
+ {
+
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ }
+ else
+ a3inv = hubble_a = 1;
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current) /* active particles */
+ {
+
+
+ if(P[i].Type == 0) /* SPH stuff */
+ {
+
+
+ //DtEntropyRadSph=0.;
+ //SphP[i].DtEntropyRadSph = 0;
+
+
+#ifdef MULTIPHASE
+ if (SphP[i].Phase == GAS_SPH)
+ {
+#endif
+ /* note : SphP[i].DtEntropyRadSph should not be necessary */
+
+
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval;
+ //SphP[i].DtEntropyRadSph = -GAMMA_MINUS1*pow(SphP[i].Density * a3inv,-GAMMA)*lambda(SphP[i].Density *a3inv,SphP[i].Entropy,0,i)/hubble_a;
+
+ //if (fabs((SphP[i].DtEntropyRadSph+SphP[i].DtEntropy)*dt) > 0.1*fabs(SphP[i].Entropy))
+ {
+ /* do implicit isochoric cooling */
+ EntropyNew = DoCooling(SphP[i].Density*a3inv,SphP[i].Entropy,0,i,SphP[i].DtEntropy,dt,hubble_a);
+
+ if(dt > 0)
+ SphP[i].DtEntropy = (EntropyNew - SphP[i].Entropy)/dt;
+ }
+ //else
+ {
+ // SphP[i].DtEntropy += SphP[i].DtEntropyRadSph;
+ }
+
+ //SphP[i].DtEgySpecRadSph = - 1/GAMMA_MINUS1 * pow(SphP[i].Density * a3inv,GAMMA_MINUS1) * (SphP[i].DtEntropyRadSph);
+
+
+#ifdef MULTIPHASE
+ }
+ else /* STICKY OR DARK */
+ {
+
+ //SphP[i].DtEntropyRadSph = -1/(SphP[i].Density * a3inv)*lambda(SphP[i].Density *a3inv,SphP[i].Entropy,SphP[i].Phase,i)/hubble_a;
+ //SphP[i].DtEntropy += SphP[i].DtEntropyRadSph;
+
+ /* do implicit isochoric cooling */
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval;
+ EntropyNew = DoCooling(SphP[i].Density*a3inv,SphP[i].Entropy,SphP[i].Phase,i,SphP[i].DtEntropy,dt,hubble_a);
+
+ if(dt > 0)
+ SphP[i].DtEntropy = (EntropyNew - SphP[i].Entropy)/dt;
+
+ /* !!! here, we do not take into account the energy variation !!! */
+ /* SphP[i].DtEgySpecRadSph = SphP[i].DtEntropy, no ?*/
+
+ }
+#endif
+
+ /* finally sum to the entropy variation */
+
+ /* WARNING : we do not compute DtEntropy here, it is updated in timestep.c */
+ /* no, because, it is updated juste above, no ? */
+ //SphP[i].DtEntropy += SphP[i].DtEntropyRadSph;
+ //SphP[i].DtEntropy += SphP[i].DtEgySpecRadSph / (-1/GAMMA_MINUS1 * pow(SphP[i].Density * a3inv,GAMMA_MINUS1));
+
+ }
+ }
+ }
+
+}
+
+
+
+
+
+/*! This function computes the new entropy due to the cooling,
+ * between step t0 and t1.
+ */
+void CoolingForOne(int i,int tstart,int tend,double a3inv,double hubble_a)
+{
+
+ double dt,dadt,tcool,dt_entr;
+ double MinSizeTimestep,ErrTolIntAccuracy;
+ int ti_current,istep;
+ int ti_step;
+ double minentropy;
+
+ FLOAT Entropy,DEntropy,DtEntropy,DtEgySpec;
+
+
+
+ if(All.MinEgySpec)
+ minentropy = All.MinEgySpec * GAMMA_MINUS1 / pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+
+
+
+
+ if(All.ComovingIntegrationOn)
+ {
+ printf("CoolingForOne : check dt and its conversion\n");
+ endrun(123321);
+
+ /*
+
+ normally, dt_entr for ComovingIntegrationOn is the same
+ than in the normal case.
+
+
+ dt_entr = (tend - tstart) * All.Timebase_interval;
+
+ */
+
+ }
+
+
+ /* compute dt */
+ dt_entr = (tend - tstart) * All.Timebase_interval;
+
+
+ ErrTolIntAccuracy = 0.02;
+ MinSizeTimestep = 0.01*dt_entr;
+
+
+ /* compute da/dt */
+ //dadt = fabs( -GAMMA_MINUS1*pow(SphP[i].Density * a3inv,-GAMMA)*lambda(SphP[i].Density *a3inv,SphP[i].Entropy,0,i)/hubble_a );
+
+ /* compute cooling time */
+ //tcool = SphP[i].Entropy / dadt;
+
+ //if (ErrTolIntAccuracy*tcool/dt_entr < 1)
+ // printf("** %g %g\n",ErrTolIntAccuracy*tcool,dt_entr); /* --> verifier le cooling time */
+
+
+
+ /***************************************/
+ /* integrate with adaptative timesteps */
+ /***************************************/
+
+
+ Entropy = SphP[i].Entropy;
+ ti_current = tstart;
+ istep = 0;
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ if (SphP[i].ThermalTime > (All.Time-All.ChimieThermalTime))
+ {
+ Entropy = Entropy + SphP[i].DtEntropy* dt_entr;
+
+ /* avoid Entropy to be less than minentropy */
+ if(All.MinEgySpec)
+ if(Entropy < minentropy)
+ Entropy = minentropy;
+
+ /* update particle */
+ SphP[i].DtEntropy = (Entropy-SphP[i].Entropy)/dt_entr;
+ SphP[i].Entropy = Entropy;
+
+ return ;
+ }
+#endif
+
+
+
+ while (ti_current<tend)
+ {
+
+ /* compute da/dt */
+ dadt = fabs( -GAMMA_MINUS1*pow(SphP[i].Density * a3inv,-GAMMA)*lambda(SphP[i].Density *a3inv,Entropy,0,i)/hubble_a );
+
+ /* compute cooling time */
+ tcool = Entropy / dadt;
+
+
+ /* find dt */
+ dt = dmax(MinSizeTimestep, tcool*ErrTolIntAccuracy);
+ dt = dmin(dt,dt_entr);
+
+
+ ti_step = dt / All.Timebase_interval;
+
+ ti_step = imax(1,ti_step);
+ ti_step = imin(ti_step,tend-ti_current);
+
+ dt = ti_step* All.Timebase_interval;
+
+
+#ifndef IMPLICIT_COOLING_INTEGRATION
+ /* normal integration of Entropy */
+ Entropy += SphP[i].DtEntropy* dt; /* viscosity */
+ Entropy += -GAMMA_MINUS1*pow(SphP[i].Density * a3inv,-GAMMA)*lambda(SphP[i].Density *a3inv,Entropy,0,i)/hubble_a *dt; /* cooling */
+#else
+ /* or use implicit integration of Entropy */
+ /* need this if there is also heating like UV */
+ Entropy = DoCooling(SphP[i].Density*a3inv,Entropy,0,i,SphP[i].DtEntropy,dt,hubble_a);
+#endif
+
+
+
+ /* avoid Entropy to be less than minentropy */
+ if(All.MinEgySpec)
+ if(Entropy < minentropy)
+ {
+ Entropy = minentropy;
+ break;
+ }
+
+ ti_current += ti_step;
+ istep = istep+1;
+
+ }
+
+ /* entropy only due to cooling */
+ DEntropy = Entropy-SphP[i].Entropy - SphP[i].DtEntropy* dt_entr;
+ DtEntropy = DEntropy/dt_entr;
+
+
+ //if (istep>1)
+ // printf("* %d %d %d %g %g\n",istep,ti_current,tend,SphP[i].DtEntropy,DtEntropy);
+
+
+
+ /* update particle */
+
+ SphP[i].Entropy = Entropy;
+ SphP[i].DtEntropy = SphP[i].DtEntropy + DtEntropy;
+
+ DtEgySpec = - 1/GAMMA_MINUS1 * pow(SphP[i].Density * a3inv,GAMMA_MINUS1) * (DtEntropy);
+
+
+ LocalSysState.RadiatedEnergy += DtEgySpec * dt_entr * P[i].Mass;
+
+
+
+
+
+
+#ifdef CHECK_ENTROPY_SIGN
+ if (SphP[i].Entropy < 0)
+ {
+ printf("\ntask=%d: entropy less than zero in CoolingForOne !\n", ThisTask);
+ printf("ID=%d Entropy=%g EntropyPred=%g DtEntropy=%g\n",P[i].ID,SphP[i].Entropy,SphP[i].EntropyPred,SphP[i].DtEntropy);
+ fflush(stdout);
+ endrun(444001);
+
+ }
+#endif
+
+
+
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+/*! cooling function
+ *
+ */
+double lambda(FLOAT Density,FLOAT Entropy,int phase,int i)
+{
+ /*
+ * These function returns the Lambda (not the Lambda_n)
+ * Here, we assume that Lambda may also contain the heating term.
+ *
+ * Here, the Entropy and Density are physical, but in h units
+ *
+ * The function is used only in cooling.c
+ *
+ */
+
+ double EgySpec;
+ double MeanWeight;
+ double T=0,nH=0,nH2=0,l=0;
+ double nHcgs=0,nH2cgs=0;
+
+#ifdef HEATING
+ double Gpe=0;
+ double X,XTne,eps,ne,flux_in_cgs;
+#endif
+
+
+
+ /* number of Hydrogen atoms per unit volume (user units, not corrected from h : [nH] = h2/cm^3 ) */
+#ifndef DO_NO_USE_HYDROGEN_MASSFRAC_IN_COOLING
+ nH = HYDROGEN_MASSFRAC*Density/All.ProtonMass;
+#else
+ nH = 1 *Density/All.ProtonMass;
+#endif
+ nH2 = nH*nH;
+
+
+ /* in cgs, corrected from h */
+ nHcgs = nH/pow(All.UnitLength_in_cm, 3);
+ if(All.ComovingIntegrationOn)
+ nHcgs *= (All.HubbleParam*All.HubbleParam);
+ nH2cgs = nHcgs*nHcgs;
+
+
+ /* compute temperature */
+#ifdef MULTIPHASE
+ switch(phase)
+ {
+ case GAS_SPH:
+ T = All.mumh/All.Boltzmann * Entropy * pow(Density,GAMMA_MINUS1);
+ break;
+ case GAS_STICKY:
+ case GAS_DARK:
+ T = All.mumh/All.Boltzmann * GAMMA_MINUS1 * Entropy;
+ break;
+ }
+#else
+ T = All.mumh/All.Boltzmann * Entropy * pow(Density,GAMMA_MINUS1);
+#endif
+
+
+ /*******************
+
+ * * * COOLING * * *
+
+ *******************/
+
+ if (All.CoolingType==0 || All.CoolingType==2)
+ {
+
+ /**************/
+ /* Sutherland */
+ /**************/
+
+#ifdef MULTIPHASE
+ switch(phase)
+ {
+ case GAS_SPH:
+ if (T > All.CutofCoolingTemperature)
+ if (All.CoolingType==0)
+ l = cooling_function(T);
+ else
+#ifdef CHIMIE
+ l = cooling_function_with_metals(T,SphP[i].Metal[FE]);
+#else
+ l = cooling_function_with_metals(T,(pow(10,All.InitGasMetallicity)-1e-10)*All.CoolingParameters_FeHSolar);
+#endif
+ else
+ l = 0;
+ break;
+ case GAS_STICKY:
+ case GAS_DARK:
+ if (T > All.CutofCoolingTemperature)
+ if (All.CoolingType==0)
+ l = cooling_function(T);
+ else
+#ifdef CHIMIE
+ l = cooling_function_with_metals(T,SphP[i].Metal[FE]);
+#else
+ l = cooling_function_with_metals(T,(pow(10,All.InitGasMetallicity)-1e-10)*All.CoolingParameters_FeHSolar);
+#endif
+ else
+ l = 0;
+ break;
+ }
+
+#else
+ /* here, lambda' is in erg*cm^3/s = kg*m^5/s^3 */
+ if (T > All.CutofCoolingTemperature)
+ if (All.CoolingType==0)
+ l = cooling_function(T);
+ else
+#ifdef CHIMIE
+ l = cooling_function_with_metals(T,SphP[i].Metal[FE]);
+#else
+ l = cooling_function_with_metals(T,(pow(10,All.InitGasMetallicity)-1e-10)*All.CoolingParameters_FeHSolar);
+#endif
+ else
+ l = 0;
+#endif
+ }
+ else
+ {
+
+ /******************************/
+ /* cooling with UV background */
+ /******************************/
+
+ /* get the right density and egyspec in cgs */
+ /* entropy and density are already physical */
+
+#ifdef MULTIPHASE
+
+ /* WARNING, HERE, WE MUST DIFERENCIATE ACORDING TO THE PHASE... */
+ printf("WARNING, HERE, WE MUST DIFERENCIATE ACORDING TO THE PHASE...\n");
+ exit(0);
+
+// if (phase == GAS_SPH)
+// EgySpec = Entropy / GAMMA_MINUS1 * pow(Density, GAMMA_MINUS1);
+// else
+// EgySpec = Entropy;
+#else
+ EgySpec = Entropy / GAMMA_MINUS1 * pow(Density, GAMMA_MINUS1);
+#endif
+
+ /* into cgs, corrected from h */
+ EgySpec *= All.UnitEnergy_in_cgs/All.UnitMass_in_g;
+ Density *= All.UnitDensity_in_cgs;
+ if(All.ComovingIntegrationOn)
+ Density *= (All.HubbleParam*All.HubbleParam);
+
+ /* compute cooling from EnergySpec and Density */
+ l = compute_cooling_from_Egyspec_and_Density(EgySpec,Density,&MeanWeight);
+
+ /* compute temperature */
+ /*
+ Temperature = GAMMA_MINUS1 *MeanWeight*PROTONMASS/BOLTZMANN *EgySpec;
+ //printf("%g %g %g\n",Temperature,MeanWeight,Lambda);
+ logT = log10(Temperature);
+ */
+
+ }
+
+
+
+ /*******************
+
+ * * * HEATING * * *
+
+ *******************/
+#ifdef HEATING
+
+
+#ifdef HEATING_PE
+ /**************************/
+ /* Photo-electric heating */ /* all must be in cgs */
+ /**************************/
+ X = 0;
+
+#ifdef STELLAR_FLUX
+ flux_in_cgs = SphP[i].EnergyFlux* All.UnitEnergy_in_cgs/All.UnitTime_in_s/pow(All.UnitLength_in_cm, 2);
+ X = X + flux_in_cgs/C / All.HeatingPeSolarEnergyDensity;
+#endif
+
+#ifdef EXTERNAL_FLUX
+ X = X + All.HeatingExternalFLuxEnergyDensity/All.HeatingPeSolarEnergyDensity ;
+#endif
+
+ ne = nHcgs*All.HeatingPeElectronFraction;
+ XTne = X*sqrt(T)/ne;
+ eps = 4.87e-2/(1+4e-3*pow(XTne,0.73)) + 3.65e-2*(T/1e4)/(1+2e-4*XTne);
+
+ Gpe = (1e-24 * eps * X * nHcgs)/ nH2cgs ;
+
+ l = l - Gpe;
+
+#endif /*HEATING_PE*/
+
+
+
+
+
+
+
+
+#endif
+
+
+
+ /**********************************
+
+ * * * final unit conversions * * *
+
+ ***********************************/
+
+ /* convert lambda' in user units */
+ l = l / All.UnitEnergy_in_cgs /pow(All.UnitLength_in_cm,3) * All.UnitTime_in_s;
+
+ /* correct from h */
+ /*
+ * [ Lambda / H / rho_p ] = [u] = cm^2/s^2
+ *
+ * [H] = h/s
+ * [rho_p] = g/cm^3 * h^2
+ * [Lambda_n] = g * m^5 / s^3
+ * [n] = h^2/m^5
+ *
+ * => Lambda_n must be multiplied by h (in order to remove one h !! not a unit !!)
+ *
+ */
+ if(All.ComovingIntegrationOn)
+ l = l * All.HubbleParam;
+
+ /* get the final lambda by multiplying lambda' by nH2 (all in user units) */
+ l = l*nH2;
+
+ return l;
+}
+
+
+#endif
+
+
+
+
+
+
diff --git a/density.c b/density.c
new file mode 100644
index 0000000..c40a580
--- /dev/null
+++ b/density.c
@@ -0,0 +1,693 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file density.c
+ * \brief SPH density computation and smoothing length determination
+ *
+ * This file contains the "first SPH loop", where the SPH densities and
+ * some auxiliary quantities are computed. If the number of neighbours
+ * obtained falls outside the target range, the correct smoothing
+ * length is determined iteratively, if needed.
+ */
+
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+
+/*! This function computes the local density for each active SPH particle,
+ * the number of neighbours in the current smoothing radius, and the
+ * divergence and curl of the velocity field. The pressure is updated as
+ * well. If a particle with its smoothing region is fully inside the
+ * local domain, it is not exported to the other processors. The function
+ * also detects particles that have a number of neighbours outside the
+ * allowed tolerance range. For these particles, the smoothing length is
+ * adjusted accordingly, and the density computation is executed again.
+ * Note that the smoothing length is not allowed to fall below the lower
+ * bound set by MinGasHsml.
+ */
+void density(int mode)
+{
+ long long ntot, ntotleft;
+ int *noffset, *nbuffer, *nsend, *nsend_local, *numlist, *ndonelist;
+ int i, j, n, ndone, npleft, maxfill, source, iter = 0;
+ int level, ngrp, sendTask, recvTask, place, nexport;
+ double dt_entr, tstart, tend, tstart_ngb = 0, tend_ngb = 0;
+ double sumt, sumcomm, timengb, sumtimengb;
+ double timecomp = 0, timeimbalance = 0, timecommsumm = 0, sumimbalance;
+ MPI_Status status;
+
+#ifdef DETAILED_CPU
+ double t0=0,t1=0;
+#endif
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+#ifdef DETAILED_CPU
+ if (mode==1)
+ t0 = second();
+#endif
+
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+ for(n = 0, NumSphUpdate = 0; n < N_gas; n++)
+ {
+ SphP[n].Left = SphP[n].Right = 0;
+#ifdef AVOIDNUMNGBPROBLEM
+ SphP[n].OldNumNgb = -1;
+#endif
+#ifdef SFR
+ if((P[n].Ti_endstep == All.Ti_Current) && (P[n].Type == 0))
+#else
+ if(P[n].Ti_endstep == All.Ti_Current)
+#endif
+ NumSphUpdate++;
+ }
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumSphUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+
+
+ /* we will repeat the whole thing for those particles where we didn't
+ * find enough neighbours
+ */
+ do
+ {
+ i = 0; /* beginn with this index */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < N_gas && nexport < All.BunchSizeDensity - NTask; i++)
+#ifdef SFR
+ if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+ {
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ density_evaluate(i, 0);
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ DensDataIn[nexport].Pos[0] = P[i].Pos[0];
+ DensDataIn[nexport].Pos[1] = P[i].Pos[1];
+ DensDataIn[nexport].Pos[2] = P[i].Pos[2];
+ DensDataIn[nexport].Vel[0] = SphP[i].VelPred[0];
+ DensDataIn[nexport].Vel[1] = SphP[i].VelPred[1];
+ DensDataIn[nexport].Vel[2] = SphP[i].VelPred[2];
+ DensDataIn[nexport].Hsml = SphP[i].Hsml;
+#ifdef MULTIPHASE
+ DensDataIn[nexport].Phase = SphP[i].Phase;
+#endif
+ DensDataIn[nexport].Index = i;
+ DensDataIn[nexport].Task = j;
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+ }
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ qsort(DensDataIn, nexport, sizeof(struct densdata_in), dens_compare_key);
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeDensity)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&DensDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct densdata_in), MPI_BYTE,
+ recvTask, TAG_DENS_A,
+ &DensDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct densdata_in),
+ MPI_BYTE, recvTask, TAG_DENS_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ density_evaluate(j, 1);
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ /* do a block to explicitly measure imbalance */
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeDensity)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&DensDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct densdata_out),
+ MPI_BYTE, recvTask, TAG_DENS_B,
+ &DensDataPartialResult[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct densdata_out),
+ MPI_BYTE, recvTask, TAG_DENS_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ source = j + noffset[recvTask];
+ place = DensDataIn[source].Index;
+
+ SphP[place].NumNgb += DensDataPartialResult[source].Ngb;
+ SphP[place].Density += DensDataPartialResult[source].Rho;
+ SphP[place].DivVel += DensDataPartialResult[source].Div;
+
+ SphP[place].DhsmlDensityFactor += DensDataPartialResult[source].DhsmlDensity;
+
+ SphP[place].Rot[0] += DensDataPartialResult[source].Rot[0];
+ SphP[place].Rot[1] += DensDataPartialResult[source].Rot[1];
+ SphP[place].Rot[2] += DensDataPartialResult[source].Rot[2];
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+
+
+ /* do final operations on results */
+ tstart = second();
+ for(i = 0, npleft = 0; i < N_gas; i++)
+ {
+#ifdef SFR
+ if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+ {
+
+ {
+ SphP[i].DhsmlDensityFactor =
+ 1 / (1 + SphP[i].Hsml * SphP[i].DhsmlDensityFactor / (NUMDIMS * SphP[i].Density));
+
+ SphP[i].CurlVel = sqrt(SphP[i].Rot[0] * SphP[i].Rot[0] +
+ SphP[i].Rot[1] * SphP[i].Rot[1] +
+ SphP[i].Rot[2] * SphP[i].Rot[2]) / SphP[i].Density;
+
+ SphP[i].DivVel /= SphP[i].Density;
+
+ dt_entr = (All.Ti_Current - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+
+ SphP[i].Pressure =
+ (SphP[i].Entropy + SphP[i].DtEntropy * dt_entr) * pow(SphP[i].Density, GAMMA);
+ }
+
+
+ /* now check whether we had enough neighbours */
+
+
+ if(SphP[i].NumNgb < (All.DesNumNgb - All.MaxNumNgbDeviation) ||
+ (SphP[i].NumNgb > (All.DesNumNgb + All.MaxNumNgbDeviation)
+ && SphP[i].Hsml > (1.01 * All.MinGasHsml)))
+ {
+
+#ifdef AVOIDNUMNGBPROBLEM
+// if((SphP[i].NumNgb>SphP[i].OldNumNgb-All.MaxNumNgbDeviation/10000.)
+// &&(SphP[i].NumNgb<SphP[i].OldNumNgb+All.MaxNumNgbDeviation/10000.))
+ if(SphP[i].NumNgb==SphP[i].OldNumNgb)
+ {
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+ printf("ID=%d NumNgb=%g OldNumNgb=%g\n",P[i].ID,SphP[i].NumNgb,SphP[i].OldNumNgb);
+ continue;
+ }
+
+ SphP[i].OldNumNgb = SphP[i].NumNgb;
+#endif
+
+
+ /* need to redo this particle */
+ npleft++;
+
+ if(SphP[i].Left > 0 && SphP[i].Right > 0)
+ if((SphP[i].Right - SphP[i].Left) < 1.0e-3 * SphP[i].Left)
+ {
+ /* this one should be ok */
+ npleft--;
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1; /* Mark as inactive */
+ continue;
+ }
+
+ if(SphP[i].NumNgb < (All.DesNumNgb - All.MaxNumNgbDeviation))
+ SphP[i].Left = dmax(SphP[i].Hsml, SphP[i].Left);
+ else
+ {
+ if(SphP[i].Right != 0)
+ {
+ if(SphP[i].Hsml < SphP[i].Right)
+ SphP[i].Right = SphP[i].Hsml;
+ }
+ else
+ SphP[i].Right = SphP[i].Hsml;
+ }
+
+ if(iter >= MAXITER - 10)
+ {
+ printf
+ ("i=%d task=%d ID=%d Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
+ i, ThisTask, (int) P[i].ID, SphP[i].Hsml, SphP[i].Left, SphP[i].Right,
+ (float) SphP[i].NumNgb, SphP[i].Right - SphP[i].Left, P[i].Pos[0], P[i].Pos[1],
+ P[i].Pos[2]);
+ fflush(stdout);
+ }
+
+ if(SphP[i].Right > 0 && SphP[i].Left > 0)
+ SphP[i].Hsml = pow(0.5 * (pow(SphP[i].Left, 3) + pow(SphP[i].Right, 3)), 1.0 / 3);
+ else
+ {
+ if(SphP[i].Right == 0 && SphP[i].Left == 0)
+ {
+ printf
+ ("i=%d task=%d ID=%d Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
+ i, ThisTask, (int) P[i].ID, SphP[i].Hsml, SphP[i].Left, SphP[i].Right,
+ (float) SphP[i].NumNgb, SphP[i].Right - SphP[i].Left, P[i].Pos[0], P[i].Pos[1],
+ P[i].Pos[2]);
+ fflush(stdout);
+ endrun(8188); /* can't occur */
+ }
+
+ if(SphP[i].Right == 0 && SphP[i].Left > 0)
+ {
+ if(P[i].Type == 0 && fabs(SphP[i].NumNgb - All.DesNumNgb) < 0.5 * All.DesNumNgb)
+ {
+ SphP[i].Hsml *=
+ 1 - (SphP[i].NumNgb -
+ All.DesNumNgb) / (NUMDIMS * SphP[i].NumNgb) * SphP[i].DhsmlDensityFactor;
+ }
+ else
+ SphP[i].Hsml *= 1.26;
+ }
+
+ if(SphP[i].Right > 0 && SphP[i].Left == 0)
+ {
+ if(P[i].Type == 0 && fabs(SphP[i].NumNgb - All.DesNumNgb) < 0.5 * All.DesNumNgb)
+ {
+ SphP[i].Hsml *=
+ 1 - (SphP[i].NumNgb -
+ All.DesNumNgb) / (NUMDIMS * SphP[i].NumNgb) * SphP[i].DhsmlDensityFactor;
+ }
+ else
+ SphP[i].Hsml /= 1.26;
+ }
+ }
+
+ if(SphP[i].Hsml < All.MinGasHsml)
+ SphP[i].Hsml = All.MinGasHsml;
+ }
+ else
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1; /* Mark as inactive */
+ }
+ }
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&npleft, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+ if(ntot > 0)
+ {
+ if(iter == 0)
+ tstart_ngb = second();
+
+ iter++;
+
+ if(iter > 0 && ThisTask == 0)
+ {
+ printf("ngb iteration %d: need to repeat for %d%09d particles.\n", iter,
+ (int) (ntot / 1000000000), (int) (ntot % 1000000000));
+ fflush(stdout);
+ }
+
+ if(iter > MAXITER)
+ {
+ printf("failed to converge in neighbour iteration in density()\n");
+ fflush(stdout);
+ endrun(1155);
+ }
+ }
+ else
+ tend_ngb = second();
+ }
+ while(ntot > 0);
+
+
+ /* mark as active again */
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+ /* collect some timing information */
+ if(iter > 0)
+ timengb = timediff(tstart_ngb, tend_ngb);
+ else
+ timengb = 0;
+
+ MPI_Reduce(&timengb, &sumtimengb, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecomp, &sumt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm, &sumcomm, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance, &sumimbalance, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+
+
+#ifdef DETAILED_CPU
+ if (mode==1)
+ {
+ t1 = second();
+ All.CPU_StarFormation += timediff(t0, t1);
+ }
+ else
+ if(ThisTask == 0)
+ {
+ All.CPU_HydCompWalk += sumt / NTask;
+ All.CPU_HydCommSumm += sumcomm / NTask;
+ All.CPU_HydImbalance += sumimbalance / NTask;
+ All.CPU_EnsureNgb += sumtimengb / NTask;
+ }
+
+#else
+ if(ThisTask == 0)
+ {
+ All.CPU_HydCompWalk += sumt / NTask;
+ All.CPU_HydCommSumm += sumcomm / NTask;
+ All.CPU_HydImbalance += sumimbalance / NTask;
+ All.CPU_EnsureNgb += sumtimengb / NTask;
+ }
+#endif
+
+}
+
+
+
+/*! This function represents the core of the SPH density computation. The
+ * target particle may either be local, or reside in the communication
+ * buffer.
+ */
+void density_evaluate(int target, int mode)
+{
+ int j, n, startnode, numngb, numngb_inbox;
+ double h, h2, fac, hinv, hinv3, hinv4;
+ double rho, divv, wk, dwk;
+ double dx, dy, dz, r, r2, u, mass_j;
+ double dvx, dvy, dvz, rotv[3];
+ double weighted_numngb, dhsmlrho;
+ FLOAT *pos, *vel;
+ int phase=0;
+
+ if(mode == 0)
+ {
+ pos = P[target].Pos;
+ vel = SphP[target].VelPred;
+ h = SphP[target].Hsml;
+#ifdef MULTIPHASE
+ phase = SphP[target].Phase;
+#endif
+ }
+ else
+ {
+ pos = DensDataGet[target].Pos;
+ vel = DensDataGet[target].Vel;
+ h = DensDataGet[target].Hsml;
+#ifdef MULTIPHASE
+ phase = DensDataGet[target].Phase;
+#endif
+ }
+
+ h2 = h * h;
+ hinv = 1.0 / h;
+#ifndef TWODIMS
+ hinv3 = hinv * hinv * hinv;
+#else
+ hinv3 = hinv * hinv / boxSize_Z;
+#endif
+ hinv4 = hinv3 * hinv;
+
+ rho = divv = rotv[0] = rotv[1] = rotv[2] = 0;
+ weighted_numngb = 0;
+ dhsmlrho = 0;
+
+ startnode = All.MaxPart;
+ numngb = 0;
+ do
+ {
+ numngb_inbox = ngb_treefind_variable(&pos[0], h, phase, &startnode);
+
+ for(n = 0; n < numngb_inbox; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+#ifdef PERIODIC /* now find the closest image in the given box size */
+ if(dx > boxHalf_X)
+ dx -= boxSize_X;
+ if(dx < -boxHalf_X)
+ dx += boxSize_X;
+ if(dy > boxHalf_Y)
+ dy -= boxSize_Y;
+ if(dy < -boxHalf_Y)
+ dy += boxSize_Y;
+ if(dz > boxHalf_Z)
+ dz -= boxSize_Z;
+ if(dz < -boxHalf_Z)
+ dz += boxSize_Z;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(r2 < h2)
+ {
+ numngb++;
+
+ r = sqrt(r2);
+
+ u = r * hinv;
+
+ if(u < 0.5)
+ {
+ wk = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ dwk = hinv4 * u * (KERNEL_COEFF_3 * u - KERNEL_COEFF_4);
+ }
+ else
+ {
+ wk = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ dwk = hinv4 * KERNEL_COEFF_6 * (1.0 - u) * (1.0 - u);
+ }
+
+ mass_j = P[j].Mass;
+
+ rho += mass_j * wk;
+
+ weighted_numngb += NORM_COEFF * wk / hinv3;
+
+ dhsmlrho += -mass_j * (NUMDIMS * hinv * wk + u * dwk);
+
+ if(r > 0)
+ {
+ fac = mass_j * dwk / r;
+
+ dvx = vel[0] - SphP[j].VelPred[0];
+ dvy = vel[1] - SphP[j].VelPred[1];
+ dvz = vel[2] - SphP[j].VelPred[2];
+
+ divv -= fac * (dx * dvx + dy * dvy + dz * dvz);
+
+ rotv[0] += fac * (dz * dvy - dy * dvz);
+ rotv[1] += fac * (dx * dvz - dz * dvx);
+ rotv[2] += fac * (dy * dvx - dx * dvy);
+ }
+ }
+ }
+ }
+ while(startnode >= 0);
+
+ if(mode == 0)
+ {
+ SphP[target].NumNgb = weighted_numngb;
+ SphP[target].Density = rho;
+ SphP[target].DivVel = divv;
+ SphP[target].DhsmlDensityFactor = dhsmlrho;
+ SphP[target].Rot[0] = rotv[0];
+ SphP[target].Rot[1] = rotv[1];
+ SphP[target].Rot[2] = rotv[2];
+ }
+ else
+ {
+ DensDataResult[target].Rho = rho;
+ DensDataResult[target].Div = divv;
+ DensDataResult[target].Ngb = weighted_numngb;
+ DensDataResult[target].DhsmlDensity = dhsmlrho;
+ DensDataResult[target].Rot[0] = rotv[0];
+ DensDataResult[target].Rot[1] = rotv[1];
+ DensDataResult[target].Rot[2] = rotv[2];
+ }
+}
+
+
+
+
+/*! This routine is a comparison kernel used in a sort routine to group
+ * particles that are exported to the same processor.
+ */
+int dens_compare_key(const void *a, const void *b)
+{
+ if(((struct densdata_in *) a)->Task < (((struct densdata_in *) b)->Task))
+ return -1;
+
+ if(((struct densdata_in *) a)->Task > (((struct densdata_in *) b)->Task))
+ return +1;
+
+ return 0;
+}
diff --git a/domain.c b/domain.c
new file mode 100644
index 0000000..e6aab56
--- /dev/null
+++ b/domain.c
@@ -0,0 +1,1818 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+
+/*! \file domain.c
+ * \brief code for domain decomposition
+ *
+ * This file contains the code for the domain decomposition of the
+ * simulation volume. The domains are constructed from disjoint subsets
+ * of the leaves of a fiducial top-level tree that covers the full
+ * simulation volume. Domain boundaries hence run along tree-node
+ * divisions of a fiducial global BH tree. As a result of this method, the
+ * tree force are in principle strictly independent of the way the domains
+ * are cut. The domain decomposition can be carried out for an arbitrary
+ * number of CPUs. Individual domains are not cubical, but spatially
+ * coherent since the leaves are traversed in a Peano-Hilbert order and
+ * individual domains form segments along this order. This also ensures
+ * that each domain has a small surface to volume ratio, which minimizes
+ * communication.
+ */
+
+#define TOPNODEFACTOR 20.0
+
+#define REDUC_FAC 0.98
+
+
+/*! toGo[task*NTask + partner] gives the number of particles in task 'task'
+ * that have to go to task 'partner'
+ */
+static int *toGo, *toGoSph;
+static int *local_toGo, *local_toGoSph;
+static int *list_NumPart;
+static int *list_N_gas;
+static int *list_load;
+static int *list_loadsph;
+static double *list_work;
+
+#ifdef STELLAR_PROP
+static int *toGoSt;
+static int *local_toGoSt;
+static int *list_N_stars;
+#endif
+
+static long long maxload, maxloadsph;
+
+static struct topnode_exchange
+{
+ peanokey Startkey;
+ int Count;
+}
+ *toplist, *toplist_local;
+
+
+
+/*! This is the main routine for the domain decomposition. It acts as a
+ * driver routine that allocates various temporary buffers, maps the
+ * particles back onto the periodic box if needed, and then does the
+ * domain decomposition, and a final Peano-Hilbert order of all particles
+ * as a tuning measure.
+ */
+void domain_Decomposition(void)
+{
+ double t0, t1;
+ int i;
+
+#ifdef PMGRID
+ if(All.PM_Ti_endstep == All.Ti_Current)
+ {
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+ /* to make sure that we do a domain decomposition before the PM-force is evaluated.
+ this is needed to make sure that the particles are wrapped into the box */
+ }
+#endif
+
+ /* Check whether it is really time for a new domain decomposition */
+ if(All.NumForcesSinceLastDomainDecomp > All.TotNumPart * All.TreeDomainUpdateFrequency)
+ {
+ t0 = second();
+
+#ifdef PERIODIC
+ do_box_wrapping(); /* map the particles back onto the box */
+#endif
+ All.NumForcesSinceLastDomainDecomp = 0;
+
+
+#ifdef SFR
+ rearrange_particle_sequence();
+#endif
+
+
+ TreeReconstructFlag = 1; /* ensures that new tree will be constructed */
+
+ if(ThisTask == 0)
+ {
+ printf("start domain decomposition... \n");
+ fflush(stdout);
+ }
+
+ Key = malloc(sizeof(peanokey) * All.MaxPart);
+ KeySorted = malloc(sizeof(peanokey) * All.MaxPart);
+
+ toGo = malloc(sizeof(int) * NTask * NTask);
+ toGoSph = malloc(sizeof(int) * NTask * NTask);
+ local_toGo = malloc(sizeof(int) * NTask);
+ local_toGoSph = malloc(sizeof(int) * NTask);
+ list_NumPart = malloc(sizeof(int) * NTask);
+ list_N_gas = malloc(sizeof(int) * NTask);
+ list_load = malloc(sizeof(int) * NTask);
+ list_loadsph = malloc(sizeof(int) * NTask);
+ list_work = malloc(sizeof(double) * NTask);
+
+ MPI_Allgather(&NumPart, 1, MPI_INT, list_NumPart, 1, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(&N_gas, 1, MPI_INT, list_N_gas, 1, MPI_INT, MPI_COMM_WORLD);
+
+#ifdef STELLAR_PROP
+ toGoSt = malloc(sizeof(int) * NTask * NTask);
+ local_toGoSt = malloc(sizeof(int) * NTask);
+ list_N_stars = malloc(sizeof(int) * NTask);
+ MPI_Allgather(&N_stars, 1, MPI_INT, list_N_stars, 1, MPI_INT, MPI_COMM_WORLD);
+#endif
+
+ maxload = All.MaxPart * REDUC_FAC;
+ maxloadsph = All.MaxPartSph * REDUC_FAC;
+
+
+
+
+
+#ifdef CHECK_ID_CORRESPONDENCE
+ if (ThisTask==0)
+ printf("Check id correspondence before decomposition...\n");
+
+ rearrange_particle_sequence();
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ if( StP[P[i].StPIdx].PIdx != i )
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in domain before) N_stars=%d N_gas=%d i=%d id=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d\n\n",ThisTask,N_stars,N_gas,i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx);
+ endrun(888001);
+ }
+
+
+ if(StP[P[i].StPIdx].ID != P[i].ID)
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in domain before) N_gas=%d N_stars=%d i=%d Type=%d P.Id=%d P[i].StPIdx=%d StP[P[i].StPIdx].ID=%d \n\n",ThisTask,N_gas,N_stars,i,P[i].Type,P[i].ID, P[i].StPIdx, StP[P[i].StPIdx].ID);
+ endrun(888002);
+ }
+ }
+ if (ThisTask==0)
+ printf("Check id correspondence before decomposition done...\n");
+#endif
+
+
+
+ domain_decompose();
+
+ free(list_work);
+ free(list_loadsph);
+ free(list_load);
+ free(list_N_gas);
+ free(list_NumPart);
+ free(local_toGoSph);
+ free(local_toGo);
+ free(toGoSph);
+ free(toGo);
+
+#ifdef STELLAR_PROP
+ free(list_N_stars);
+ free(local_toGoSt);
+ free(toGoSt);
+#endif
+
+#ifdef CHECK_BLOCK_ORDER
+ int typenow;
+ typenow = 0;
+ for(i = 0; i < NumPart; i++)
+ {
+
+ if ((P[i].Type<typenow)&&(typenow<=ST))
+ {
+ printf("\nBlock order error\n");
+ printf("(%d) i=%d Type=%d typenow=%d\n\n",ThisTask,i,P[i].Type,typenow);
+ endrun(888003);
+ }
+
+ typenow = P[i].Type;
+ }
+#endif
+
+#ifdef CHECK_ID_CORRESPONDENCE
+ if (ThisTask==0)
+ printf("Check id correspondence after decomposition...\n");
+
+ rearrange_particle_sequence();
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ if( StP[P[i].StPIdx].PIdx != i )
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in domain after) N_stars=%d N_gas=%d i=%d id=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d\n\n",ThisTask,N_stars,N_gas,i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx);
+ endrun(888004);
+ }
+
+
+ if(StP[P[i].StPIdx].ID != P[i].ID)
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in domain after) N_gas=%d N_stars=%d i=%d Type=%d P.Id=%d P[i].StPIdx=%d StP[P[i].StPIdx].ID=%d \n\n",ThisTask,N_gas,N_stars,i,P[i].Type,P[i].ID, P[i].StPIdx, StP[P[i].StPIdx].ID);
+ endrun(888005);
+ }
+ }
+ if (ThisTask==0)
+ printf("Check id correspondence after decomposition done...\n");
+#endif
+
+
+
+
+
+
+ if(ThisTask == 0)
+ {
+ printf("domain decomposition done. \n");
+ fflush(stdout);
+ }
+
+ t1 = second();
+ All.CPU_Domain += timediff(t0, t1);
+
+#ifdef PEANOHILBERT
+ t0 = second();
+ peano_hilbert_order();
+ t1 = second();
+ All.CPU_Peano += timediff(t0, t1);
+#endif
+
+ free(KeySorted);
+ free(Key);
+ }
+
+}
+
+
+
+/*! This function carries out the actual domain decomposition for all
+ * particle types. It will try to balance the work-load for each domain,
+ * as estimated based on the P[i]-GravCost values. The decomposition will
+ * respect the maximum allowed memory-imbalance given by the value of
+ * PartAllocFactor.
+ */
+void domain_decompose(void)
+{
+ int i, j, status;
+ int ngrp, task, partner, sendcount, recvcount;
+ long long sumtogo, sumload;
+ int maxload, *temp;
+ double sumwork, maxwork;
+
+#ifdef DETAILED_CPU_DOMAIN
+ double t0, t1;
+#endif
+
+ for(i = 0; i < 6; i++)
+ NtypeLocal[i] = 0;
+
+ for(i = 0; i < NumPart; i++)
+ NtypeLocal[P[i].Type]++;
+
+ /* because Ntype[] is of type `long long', we cannot do a simple
+ * MPI_Allreduce() to sum the total particle numbers
+ */
+ temp = malloc(NTask * 6 * sizeof(int));
+ MPI_Allgather(NtypeLocal, 6, MPI_INT, temp, 6, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0; i < 6; i++)
+ {
+ Ntype[i] = 0;
+ for(j = 0; j < NTask; j++)
+ Ntype[i] += temp[j * 6 + i];
+ }
+ free(temp);
+
+#ifndef UNEQUALSOFTENINGS
+ for(i = 0; i < 6; i++)
+ if(Ntype[i] > 0)
+ break;
+
+ for(ngrp = i + 1; ngrp < 6; ngrp++)
+ {
+ if(Ntype[ngrp] > 0)
+ if(All.SofteningTable[ngrp] != All.SofteningTable[i])
+ {
+ if(ThisTask == 0)
+ {
+ fprintf(stdout, "Code was not compiled with UNEQUALSOFTENINGS, but some of the\n");
+ fprintf(stdout, "softening lengths are unequal nevertheless.\n");
+ fprintf(stdout, "This is not allowed.\n");
+ }
+ endrun(0);
+ }
+ }
+#endif
+
+
+ /* determine global dimensions of domain grid */
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+ domain_findExtent();
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_findExtend += timediff(t0, t1);
+#endif
+
+
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+ domain_determineTopTree();
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_determineTopTree += timediff(t0, t1);
+#endif
+
+
+ /* determine cost distribution in domain grid */
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+ domain_sumCost();
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_sumCost += timediff(t0, t1);
+#endif
+
+
+ /* find the split of the domain grid recursively */
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+#ifdef SPLIT_DOMAIN_USING_TIME
+ status = domain_findSplityr(0, NTask, 0, NTopleaves - 1);
+#else
+ status = domain_findSplit(0, NTask, 0, NTopleaves - 1);
+#endif
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_findSplit += timediff(t0, t1);
+#endif
+
+ if(status != 0)
+ {
+ if(ThisTask == 0)
+ printf("\nNo domain decomposition that stays within memory bounds is possible (status=%d).\n",status);
+ endrun(0);
+ }
+
+ /* now try to improve the work-load balance of the split */
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+#ifdef SPLIT_DOMAIN_USING_TIME
+ domain_shiftSplityr();
+#else
+ domain_shiftSplit();
+#endif
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_shiftSplit += timediff(t0, t1);
+#endif
+
+
+ DomainMyStart = DomainStartList[ThisTask];
+ DomainMyLast = DomainEndList[ThisTask];
+
+ if(ThisTask == 0)
+ {
+ sumload = maxload = 0;
+ sumwork = maxwork = 0;
+ for(i = 0; i < NTask; i++)
+ {
+ sumload += list_load[i];
+ sumwork += list_work[i];
+
+ if(list_load[i] > maxload)
+ maxload = list_load[i];
+
+ if(list_work[i] > maxwork)
+ maxwork = list_work[i];
+ }
+
+ printf("work-load balance=%g memory-balance=%g\n",
+ maxwork / (sumwork / NTask), maxload / (((double) sumload) / NTask));
+ }
+
+
+ /* determine for each cpu how many particles have to be shifted to other cpus */
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+ domain_countToGo();
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_countToGo += timediff(t0, t1);
+#endif
+
+
+#ifdef DETAILED_CPU_DOMAIN
+ t0 = second();
+#endif
+
+ for(i = 0, sumtogo = 0; i < NTask * NTask; i++)
+ sumtogo += toGo[i];
+
+ while(sumtogo > 0)
+ {
+ if(ThisTask == 0)
+ {
+ printf("exchange of %d%09d particles\n", (int) (sumtogo / 1000000000),
+ (int) (sumtogo % 1000000000));
+ fflush(stdout);
+ }
+
+ for(ngrp = 1; ngrp < (1 << PTask); ngrp++)
+ {
+ for(task = 0; task < NTask; task++)
+ {
+ partner = task ^ ngrp;
+
+ if(partner < NTask && task < partner)
+ {
+
+
+
+ /* treat SPH separately */
+ if(All.TotN_gas > 0)
+ {
+ domain_findExchangeNumbers(task, partner, 0, &sendcount, &recvcount);
+
+ list_NumPart[task] += recvcount - sendcount;
+ list_NumPart[partner] -= recvcount - sendcount;
+ list_N_gas[task] += recvcount - sendcount;
+ list_N_gas[partner] -= recvcount - sendcount;
+
+ toGo[task * NTask + partner] -= sendcount;
+ toGo[partner * NTask + task] -= recvcount;
+ toGoSph[task * NTask + partner] -= sendcount;
+ toGoSph[partner * NTask + task] -= recvcount;
+
+ //if (sendcount>0)
+ // printf("(%d) gas --> %d (%d)\n",ThisTask,sendcount,N_gas);
+ //if (recvcount>0)
+ // printf("(%d) gas <-- %d (%d)\n",ThisTask,recvcount,N_gas);
+
+ //printf("(%d) N_gas=%d P[N_gas].ID=%d %d (%d %d)\n",ThisTask,N_gas,P[N_gas].ID,P[N_gas].Type,sendcount,recvcount);
+
+ if(task == ThisTask) /* actually carry out the exchange */
+ domain_exchangeParticles(partner, 0, sendcount, recvcount);
+ if(partner == ThisTask)
+ domain_exchangeParticles(task, 0, recvcount, sendcount);
+
+
+ //if (sendcount>0)
+ // printf("(%d) (after) gas --> %d (%d)\n",ThisTask,sendcount,N_gas);
+ //if (recvcount>0)
+ // printf("(%d) (after) gas <-- %d (%d)\n",ThisTask,recvcount,N_gas);
+
+ //printf("(%d) (after) N_gas=%d P[N_gas].ID=%d %d (%d %d)\n",ThisTask,N_gas,P[N_gas].ID,P[N_gas].Type,sendcount,recvcount);
+
+
+ }
+
+
+
+#ifdef STELLAR_PROP
+ /* treat STARS separately */
+ if(All.TotN_stars > 0)
+ {
+ domain_findExchangeNumbers(task, partner, 1, &sendcount, &recvcount);
+
+ list_NumPart[task] += recvcount - sendcount;
+ list_NumPart[partner] -= recvcount - sendcount;
+ list_N_stars[task] += recvcount - sendcount;
+ list_N_stars[partner] -= recvcount - sendcount;
+
+ toGo[task * NTask + partner] -= sendcount;
+ toGo[partner * NTask + task] -= recvcount;
+ toGoSt[task * NTask + partner] -= sendcount;
+ toGoSt[partner * NTask + task] -= recvcount;
+
+ //if (sendcount>0)
+ // printf("(%d) stars --> %d (%d)\n",ThisTask,sendcount,N_stars);
+ //if (recvcount>0)
+ // printf("(%d) stars <-- %d (%d)\n",ThisTask,recvcount,N_stars);
+
+ if(task == ThisTask) /* actually carry out the exchange */
+ domain_exchangeParticles(partner, 1, sendcount, recvcount);
+ if(partner == ThisTask)
+ domain_exchangeParticles(task, 1, recvcount, sendcount);
+
+ //if (sendcount>0)
+ // printf("(%d) stars --> %d (%d)\n",ThisTask,sendcount,N_stars);
+ //if (recvcount>0)
+ // printf("(%d) stars <-- %d (%d)\n",ThisTask,recvcount,N_stars);
+
+ }
+#endif
+
+
+
+
+ domain_findExchangeNumbers(task, partner, -1, &sendcount, &recvcount);
+
+ list_NumPart[task] += recvcount - sendcount;
+ list_NumPart[partner] -= recvcount - sendcount;
+
+ toGo[task * NTask + partner] -= sendcount;
+ toGo[partner * NTask + task] -= recvcount;
+
+ if(task == ThisTask) /* actually carry out the exchange */
+ domain_exchangeParticles(partner, -1, sendcount, recvcount);
+ if(partner == ThisTask)
+ domain_exchangeParticles(task, -1, recvcount, sendcount);
+ }
+ }
+ }
+
+ for(i = 0, sumtogo = 0; i < NTask * NTask; i++)
+ sumtogo += toGo[i];
+ }
+#ifdef SFR
+#ifndef STELLAR_PROP
+ /* count the new N_stars : in case STELLAR_PROP is on,
+ N_stars is already updated during the domain decomposition */
+
+ int nstars;
+
+ for(i = N_gas, nstars = 0; i < NumPart; i++)
+ {
+ if (P[i].Type==ST)
+ nstars++;
+ }
+
+ N_stars = nstars;
+
+#endif
+#endif
+
+
+#ifdef DETAILED_CPU_DOMAIN
+ t1 = second();
+ All.CPU_Domain_exchange += timediff(t0, t1);
+#endif
+
+
+}
+
+/*! This function tries to find a split point in a range of cells in the
+ * domain-grid. The range of cells starts at 'first', and ends at 'last'
+ * (inclusively). The number of cpus that holds the range is 'ncpu', with
+ * the first cpu given by 'cpustart'. If more than 2 cpus are to be split,
+ * the function calls itself recursively. The division tries to achieve a
+ * best particle-load balance under the constraint that 'maxload' and
+ * 'maxloadsph' may not be exceeded, and that each cpu holds at least one
+ * cell from the domaingrid. If such a decomposition cannot be achieved, a
+ * non-zero error code is returned.
+ *
+ * After successful completion, DomainMyStart[] and DomainMyLast[] contain
+ * the first and last cell of the domaingrid assigned to the local task
+ * for the given type. Also, DomainTask[] contains for each cell the task
+ * it was assigned to.
+ */
+int domain_findSplit(int cpustart, int ncpu, int first, int last)
+{
+ int i, split, ok_left, ok_right;
+ long long load, sphload, load_leftOfSplit, sphload_leftOfSplit;
+ int ncpu_leftOfSplit;
+ double maxAvgLoad_CurrentSplit, maxAvgLoad_NewSplit;
+
+
+ ncpu_leftOfSplit = ncpu / 2;
+
+ for(i = first, load = 0, sphload = 0; i <= last; i++)
+ {
+ load += DomainCount[i];
+ sphload += DomainCountSph[i];
+ }
+
+ split = first + ncpu_leftOfSplit;
+
+ for(i = first, load_leftOfSplit = sphload_leftOfSplit = 0; i < split; i++)
+ {
+ load_leftOfSplit += DomainCount[i];
+ sphload_leftOfSplit += DomainCountSph[i];
+ }
+
+ /* find the best split point in terms of work-load balance */
+
+ while(split < last - (ncpu - ncpu_leftOfSplit - 1) && split > 0)
+ {
+ maxAvgLoad_CurrentSplit =
+ dmax(load_leftOfSplit / ncpu_leftOfSplit, (load - load_leftOfSplit) / (ncpu - ncpu_leftOfSplit));
+
+ maxAvgLoad_NewSplit =
+ dmax((load_leftOfSplit + DomainCount[split]) / ncpu_leftOfSplit,
+ (load - load_leftOfSplit - DomainCount[split]) / (ncpu - ncpu_leftOfSplit));
+
+ if(maxAvgLoad_NewSplit <= maxAvgLoad_CurrentSplit)
+ {
+ load_leftOfSplit += DomainCount[split];
+ sphload_leftOfSplit += DomainCountSph[split];
+ split++;
+ }
+ else
+ break;
+ }
+
+
+ /* we will now have to check whether this solution is possible given the restrictions on the maximum load */
+
+ for(i = first, load_leftOfSplit = 0, sphload_leftOfSplit = 0; i < split; i++)
+ {
+ load_leftOfSplit += DomainCount[i];
+ sphload_leftOfSplit += DomainCountSph[i];
+ }
+
+ if(load_leftOfSplit > maxload * ncpu_leftOfSplit ||
+ (load - load_leftOfSplit) > maxload * (ncpu - ncpu_leftOfSplit))
+ {
+ /* we did not find a viable split */
+ printf("(%d) error -1 ",ThisTask);
+ return -1;
+ }
+
+ if(sphload_leftOfSplit > maxloadsph * ncpu_leftOfSplit ||
+ (sphload - sphload_leftOfSplit) > maxloadsph * (ncpu - ncpu_leftOfSplit))
+ {
+ /* we did not find a viable split */
+ printf("(%d) error -2 ",ThisTask);
+ return -2;
+ }
+
+
+
+
+ if(ncpu_leftOfSplit >= 2)
+ ok_left = domain_findSplit(cpustart, ncpu_leftOfSplit, first, split - 1);
+ else
+ ok_left = 0;
+
+ if((ncpu - ncpu_leftOfSplit) >= 2)
+ ok_right = domain_findSplit(cpustart + ncpu_leftOfSplit, ncpu - ncpu_leftOfSplit, split, last);
+ else
+ ok_right = 0;
+
+
+
+ if(ok_left == 0 && ok_right == 0)
+ {
+ /* found a viable split */
+
+ if(ncpu_leftOfSplit == 1)
+ {
+ for(i = first; i < split; i++)
+ DomainTask[i] = cpustart;
+
+ list_load[cpustart] = load_leftOfSplit;
+ list_loadsph[cpustart] = sphload_leftOfSplit;
+ DomainStartList[cpustart] = first;
+ DomainEndList[cpustart] = split - 1;
+ }
+
+ if((ncpu - ncpu_leftOfSplit) == 1)
+ {
+ for(i = split; i <= last; i++)
+ DomainTask[i] = cpustart + ncpu_leftOfSplit;
+
+ list_load[cpustart + ncpu_leftOfSplit] = load - load_leftOfSplit;
+ list_loadsph[cpustart + ncpu_leftOfSplit] = sphload - sphload_leftOfSplit;
+ DomainStartList[cpustart + ncpu_leftOfSplit] = split;
+ DomainEndList[cpustart + ncpu_leftOfSplit] = last;
+ }
+
+ return 0;
+ }
+
+ /* we did not find a viable split */
+ return -3;
+}
+
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+/*! This function tries to find a split point in a range of cells
+ */
+int domain_findSplityr(int cpustart, int ncpu, int first, int last)
+{
+ int i, split, ok_left, ok_right;
+ long long load, sphload, load_leftOfSplit, sphload_leftOfSplit;
+ int ncpu_leftOfSplit;
+ double maxAvgLoad_CurrentSplit, maxAvgLoad_NewSplit;
+
+
+
+
+
+ /************************************/
+ /* find the number of part per proc */
+ /************************************/
+
+
+ double sumCPU_Gravity,meanCPU_Gravity;
+ double imb;
+ double ImbFactor;
+ int NumPartDiff,sumNumPartDiff,DesiredNumPart;
+ int *list_DesiredNumPart;
+
+ MPI_Allreduce(&CPU_Gravity, &sumCPU_Gravity, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ meanCPU_Gravity = sumCPU_Gravity/NTask;
+
+ if (sumCPU_Gravity>0)
+ {
+ imb = CPU_Gravity/meanCPU_Gravity;
+ NumPartDiff = (int) ((meanCPU_Gravity-CPU_Gravity)/meanCPU_Gravity * All.TotNumPart/NTask * 0.5);
+ ImbFactor=0;
+ }
+ else
+ {
+ imb = 0;
+ ImbFactor=0;
+ NumPartDiff=0;
+ }
+
+
+ //NumPartDiff = (int)NumPart*ImbFactor;
+ MPI_Reduce(&NumPartDiff, &sumNumPartDiff, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if (ThisTask==0)
+ {
+ NumPartDiff-= sumNumPartDiff;
+ printf("--------> %d\n",sumNumPartDiff);
+ }
+
+ /* check */
+ MPI_Reduce(&NumPartDiff, &sumNumPartDiff, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ if (ThisTask==0)
+ if (sumNumPartDiff != 0)
+ {
+ printf("we are in trouble here...\n");
+ endrun(89897676);
+ }
+
+ DesiredNumPart = NumPart + NumPartDiff;
+
+
+ /* create a list */
+ list_DesiredNumPart= malloc(sizeof(double) * NTask);
+ MPI_Allgather(&DesiredNumPart, 1, MPI_INT, list_DesiredNumPart, 1, MPI_INT, MPI_COMM_WORLD);
+
+
+
+ printf("(%04d) Step=%04d Imbalance : %12g %12g %12g %12g %12d %12d %12d\n",ThisTask,All.NumCurrentTiStep,CPU_Gravity,sumCPU_Gravity/NTask, imb ,ImbFactor,NumPart,NumPartDiff,DesiredNumPart);
+
+ /************************************/
+ /* find the splits */
+ /************************************/
+
+
+ int task;
+ int domain;
+
+ /* loop over all top leaves */
+ for(task = 0, domain = 0; task < NTask-1; task++)
+ {
+
+
+ DomainStartList[task] = domain;
+
+ list_load[task] =0;
+ list_loadsph[task]=0;
+
+
+ while( (list_load[task]+DomainCount[domain] < list_DesiredNumPart[task]) || ((list_load[task]+DomainCount[domain] - list_DesiredNumPart[task]) < (list_DesiredNumPart[task]-list_load[task])) )
+ {
+
+ /* add the domain to the task */
+ list_load[task] += DomainCount[domain];
+ list_loadsph[task]+= DomainCountSph[domain];
+
+ DomainTask[domain] = task;
+ DomainEndList[task] = domain;
+
+ /* move to next domain */
+ domain++;
+ if (domain==NTopleaves)
+ {
+
+ printf("not enought domains...\n");
+ endrun(77665566);
+ }
+
+
+ }
+
+ }
+
+
+ /* now, do the last task */
+ task = NTask-1;
+
+ DomainStartList[task] = domain;
+
+ list_load[task] =0;
+ list_loadsph[task]=0;
+
+ for(domain = domain; domain < NTopleaves; domain++)
+ {
+ list_load[task] += DomainCount[domain];
+ list_loadsph[task]+= DomainCountSph[domain];
+
+ DomainTask[domain] = task;
+ DomainEndList[task] = domain;
+ }
+
+
+
+
+
+ free(list_DesiredNumPart);
+
+
+ return 0;
+}
+#endif
+
+
+/*! This function tries to improve the domain decomposition found by
+ * domain_findSplit() with respect to work-load balance. To this end, the
+ * boundaries in the existing domain-split solution (which was found by
+ * trying to balance the particle load) are shifted as long as this leads
+ * to better work-load while still remaining within the allowed
+ * memory-imbalance constraints.
+ */
+void domain_shiftSplit(void)
+{
+ int i, task, iter = 0, moved;
+ double maxw, newmaxw;
+
+ for(task = 0; task < NTask; task++)
+ list_work[task] = 0;
+
+ for(i = 0; i < NTopleaves; i++)
+ list_work[DomainTask[i]] += DomainWork[i];
+
+ if (ThisTask==0)
+ {
+ fprintf(FdLog,"1 %12g ",All.Time);
+ for(task = 0; task < NTask; task++)
+ fprintf(FdLog,"%12g ",list_work[task]);
+ fprintf(FdLog,"\n");
+ fflush(FdLog);
+ }
+
+
+ do
+ {
+ for(task = 0, moved = 0; task < NTask - 1; task++)
+ {
+ maxw = dmax(list_work[task], list_work[task + 1]);
+
+ if(list_work[task] < list_work[task + 1])
+ {
+ newmaxw = dmax(list_work[task] + DomainWork[DomainStartList[task + 1]],
+ list_work[task + 1] - DomainWork[DomainStartList[task + 1]]);
+ if(newmaxw <= maxw)
+ {
+ if(list_load[task] + DomainCount[DomainStartList[task + 1]] <= maxload)
+ {
+ if(list_loadsph[task] + DomainCountSph[DomainStartList[task + 1]] > maxloadsph)
+ continue;
+
+ /* ok, we can move one domain cell from right to left */
+ list_work[task] += DomainWork[DomainStartList[task + 1]];
+ list_load[task] += DomainCount[DomainStartList[task + 1]];
+ list_loadsph[task] += DomainCountSph[DomainStartList[task + 1]];
+ list_work[task + 1] -= DomainWork[DomainStartList[task + 1]];
+ list_load[task + 1] -= DomainCount[DomainStartList[task + 1]];
+ list_loadsph[task + 1] -= DomainCountSph[DomainStartList[task + 1]];
+
+ DomainTask[DomainStartList[task + 1]] = task;
+ DomainStartList[task + 1] += 1;
+ DomainEndList[task] += 1;
+
+ moved++;
+ }
+ }
+ }
+ else
+ {
+ newmaxw = dmax(list_work[task] - DomainWork[DomainEndList[task]],
+ list_work[task + 1] + DomainWork[DomainEndList[task]]);
+ if(newmaxw <= maxw)
+ {
+ if(list_load[task + 1] + DomainCount[DomainEndList[task]] <= maxload)
+ {
+ if(list_loadsph[task + 1] + DomainCountSph[DomainEndList[task]] > maxloadsph)
+ continue;
+
+ /* ok, we can move one domain cell from left to right */
+ list_work[task] -= DomainWork[DomainEndList[task]];
+ list_load[task] -= DomainCount[DomainEndList[task]];
+ list_loadsph[task] -= DomainCountSph[DomainEndList[task]];
+ list_work[task + 1] += DomainWork[DomainEndList[task]];
+ list_load[task + 1] += DomainCount[DomainEndList[task]];
+ list_loadsph[task + 1] += DomainCountSph[DomainEndList[task]];
+
+ DomainTask[DomainEndList[task]] = task + 1;
+ DomainEndList[task] -= 1;
+ DomainStartList[task + 1] -= 1;
+
+ moved++;
+ }
+ }
+
+ }
+ }
+
+ iter++;
+ }
+ while(moved > 0 && iter < 10 * NTopleaves);
+
+
+
+ if (ThisTask==0)
+ {
+ fprintf(FdLog,"2 %12g ",All.Time);
+ for(task = 0; task < NTask; task++)
+ fprintf(FdLog,"%12g ",list_work[task]);
+ fprintf(FdLog,"\n");
+ fflush(FdLog);
+ }
+
+
+
+
+
+}
+
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+/*! This function tries to improve the domain decomposition found by
+ * domain_findSplit() with respect to work-load balance. To this end, the
+ * boundaries in the existing domain-split solution (which was found by
+ * trying to balance the particle load) are shifted as long as this leads
+ * to better work-load while still remaining within the allowed
+ * memory-imbalance constraints.
+ */
+void domain_shiftSplityr(void)
+{
+
+}
+#endif
+
+
+/*! This function counts how many particles have to be exchanged between
+ * two CPUs according to the domain split. If the CPUs are already quite
+ * full and hold data from other CPUs as well, not all the particles may
+ * be exchanged at once. In this case the communication phase has to be
+ * repeated, until enough of the third-party particles have been moved
+ * away such that the decomposition can be completed.
+ */
+void domain_findExchangeNumbers(int task, int partner, int sphflag, int *send, int *recv)
+{
+ int numpartA, numpartsphA, ntobesentA, maxsendA, maxsendA_old;
+ int numpartB, numpartsphB, ntobesentB, maxsendB, maxsendB_old;
+#ifdef STELLAR_PROP
+ int numpartstA;
+ int numpartstB;
+#endif
+
+
+ numpartA = list_NumPart[task];
+ numpartsphA = list_N_gas[task];
+#ifdef STELLAR_PROP
+ numpartstA = list_N_stars[task];
+#endif
+
+ numpartB = list_NumPart[partner];
+ numpartsphB = list_N_gas[partner];
+#ifdef STELLAR_PROP
+ numpartstB = list_N_stars[partner];
+#endif
+
+ switch (sphflag)
+ {
+ case 0:
+ ntobesentA = toGoSph[task * NTask + partner];
+ ntobesentB = toGoSph[partner * NTask + task];
+ break;
+#ifdef STELLAR_PROP
+ case 1:
+ ntobesentA = toGoSt[task * NTask + partner];
+ ntobesentB = toGoSt[partner * NTask + task];
+ break;
+#endif
+ default:
+#ifdef STELLAR_PROP
+ ntobesentA = toGo[task * NTask + partner] - toGoSph[task * NTask + partner] - toGoSt[task * NTask + partner];
+ ntobesentB = toGo[partner * NTask + task] - toGoSph[partner * NTask + task] - toGoSt[partner * NTask + task];
+#else
+ ntobesentA = toGo[task * NTask + partner] - toGoSph[task * NTask + partner];
+ ntobesentB = toGo[partner * NTask + task] - toGoSph[partner * NTask + task];
+#endif
+ break;
+ }
+
+
+ maxsendA = imin(ntobesentA, All.BunchSizeDomain);
+ maxsendB = imin(ntobesentB, All.BunchSizeDomain);
+
+ do
+ {
+ maxsendA_old = maxsendA;
+ maxsendB_old = maxsendB;
+
+ maxsendA = imin(All.MaxPart - numpartB + maxsendB, maxsendA);
+ maxsendB = imin(All.MaxPart - numpartA + maxsendA, maxsendB);
+ }
+ while((maxsendA != maxsendA_old) || (maxsendB != maxsendB_old));
+
+
+ /* now make also sure that there is enough space for SPH particles */
+ switch (sphflag)
+ {
+ case 0:
+ do
+ {
+ maxsendA_old = maxsendA;
+ maxsendB_old = maxsendB;
+
+ maxsendA = imin(All.MaxPartSph - numpartsphB + maxsendB, maxsendA);
+ maxsendB = imin(All.MaxPartSph - numpartsphA + maxsendA, maxsendB);
+ }
+ while((maxsendA != maxsendA_old) || (maxsendB != maxsendB_old));
+ break;
+#ifdef STELLAR_PROP
+ case 1:
+ do
+ {
+ maxsendA_old = maxsendA;
+ maxsendB_old = maxsendB;
+
+ maxsendA = imin(All.MaxPartStars - numpartstB + maxsendB, maxsendA);
+ maxsendB = imin(All.MaxPartStars - numpartstA + maxsendA, maxsendB);
+ }
+ while((maxsendA != maxsendA_old) || (maxsendB != maxsendB_old));
+ break;
+#endif
+ default:
+ break;
+ }
+
+
+ *send = maxsendA;
+ *recv = maxsendB;
+}
+
+
+
+
+/*! This function exchanges particles between two CPUs according to the
+ * domain split. In doing this, the memory boundaries which may restrict
+ * the exhange process are observed.
+ */
+void domain_exchangeParticles(int partner, int sphflag, int send_count, int recv_count)
+{
+ int i, no, n, count, rep;
+ MPI_Status status;
+#ifdef STELLAR_PROP
+ int m;
+#endif
+
+ for(n = 0, count = 0; count < send_count && n < NumPart; n++)
+ {
+
+ switch (sphflag)
+ {
+ case 0:
+ if(P[n].Type != 0)
+ continue;
+ break;
+#ifdef STELLAR_PROP
+ case 1:
+ if(P[n].Type != 1)
+ continue;
+ break;
+#endif
+ default:
+#ifdef STELLAR_PROP
+ if((P[n].Type == 0)||(P[n].Type == 1))
+#else
+ if(P[n].Type == 0)
+#endif
+ continue;
+ break;
+ }
+
+ no = 0;
+
+ while(TopNodes[no].Daughter >= 0)
+ no = TopNodes[no].Daughter + (Key[n] - TopNodes[no].StartKey) / (TopNodes[no].Size / 8);
+
+ no = TopNodes[no].Leaf;
+
+ if(DomainTask[no] == partner)
+ {
+
+
+ switch (sphflag)
+ {
+ case 0: /* special reorder routine for SPH particles (need to stay at beginning) */
+#ifndef STELLAR_PROP
+ DomainPartBuf[count] = P[n]; /* copy particle and collect in contiguous memory */
+ DomainKeyBuf[count] = Key[n];
+ DomainSphBuf[count] = SphP[n];
+
+ P[n] = P[N_gas - 1];
+ P[N_gas - 1] = P[NumPart - 1];
+
+ Key[n] = Key[N_gas - 1];
+ Key[N_gas - 1] = Key[NumPart - 1];
+
+ SphP[n] = SphP[N_gas - 1];
+ N_gas--;
+ break;
+#else
+ DomainPartBuf[count] = P[n]; /* copy particle and collect in contiguous memory */
+ DomainKeyBuf[count] = Key[n];
+ DomainSphBuf[count] = SphP[n];
+
+ P[n] = P[N_gas - 1];
+ P[N_gas - 1] = P[N_gas+N_stars - 1];
+ P[N_gas+N_stars - 1] = P[NumPart - 1];
+
+ Key[n] = Key[N_gas - 1];
+ Key[N_gas - 1] = Key[N_gas+N_stars - 1];
+ Key[N_gas+N_stars - 1] = Key[NumPart - 1];
+
+ SphP[n] = SphP[N_gas - 1];
+
+ StP[P[N_gas - 1].StPIdx].PIdx = N_gas - 1;
+
+ N_gas--;
+ break;
+#endif
+#ifdef STELLAR_PROP
+ case 1:
+ DomainPartBuf[count] = P[n]; /* copy particle and collect in contiguous memory */
+ DomainKeyBuf[count] = Key[n];
+ DomainStBuf[count] = StP[P[n].StPIdx];
+
+ m = P[n].StPIdx;
+
+ //printf("(%d) sending P .... n=%d id=%d\n",ThisTask,n,P[n].ID);
+ //printf("(%d) replacing P using .... n=%d id=%d\n",ThisTask,N_gas+N_stars - 1,P[N_gas+N_stars - 1].ID);
+ //printf("(%d) sending Stp .... n=%d id=%d\n",ThisTask,m,StP[m].ID);
+ //printf("(%d) replacing StP using .... n=%d id=%d\n",ThisTask,N_stars - 1,StP[N_stars - 1].ID);
+
+
+ P[n] = P[N_gas+N_stars - 1];
+ StP[P[n].StPIdx].PIdx = n; /* create correct link */
+ P[N_gas+N_stars - 1] = P[NumPart - 1];
+
+
+ Key[n] = Key[N_gas+N_stars - 1];
+ Key[N_gas+N_stars - 1] = Key[NumPart - 1];
+
+
+ if (m!=(N_stars - 1))
+ {
+ StP[m] = StP[N_stars - 1]; /* replace by the last one -> avoid a hole in Stp */
+ P[StP[m].PIdx].StPIdx = m; /* create correct link */
+ }
+
+
+
+
+
+
+ N_stars--;
+ break;
+#endif
+ default:
+ DomainPartBuf[count] = P[n]; /* copy particle and collect in contiguous memory */
+ DomainKeyBuf[count] = Key[n];
+ P[n] = P[NumPart - 1];
+ Key[n] = Key[NumPart - 1];
+ break;
+ }
+
+
+ count++;
+ NumPart--;
+ n--;
+
+ }
+ }
+
+ if(count != send_count)
+ {
+ printf("Houston, we got a problem... (sphflag=%d)\n",sphflag);
+ printf("ThisTask=%d count=%d send_count=%d\n", ThisTask, count, send_count);
+ endrun(88);
+ }
+
+ /* transmit */
+
+ for(rep = 0; rep < 2; rep++)
+ {
+ if((rep == 0 && ThisTask < partner) || (rep == 1 && ThisTask > partner))
+ {
+ if(send_count > 0)
+ {
+ MPI_Ssend(&DomainPartBuf[0], send_count * sizeof(struct particle_data), MPI_BYTE, partner,
+ TAG_PDATA, MPI_COMM_WORLD);
+
+ MPI_Ssend(&DomainKeyBuf[0], send_count * sizeof(peanokey), MPI_BYTE, partner, TAG_KEY,
+ MPI_COMM_WORLD);
+
+ if(sphflag==0)
+ MPI_Ssend(&DomainSphBuf[0], send_count * sizeof(struct sph_particle_data), MPI_BYTE, partner,
+ TAG_SPHDATA, MPI_COMM_WORLD);
+
+#ifdef STELLAR_PROP
+ if(sphflag==ST)
+ MPI_Ssend(&DomainStBuf[0], send_count * sizeof(struct st_particle_data), MPI_BYTE, partner,
+ TAG_STDATA, MPI_COMM_WORLD);
+#endif
+
+ }
+ }
+
+ if((rep == 1 && ThisTask < partner) || (rep == 0 && ThisTask > partner))
+ {
+ if(recv_count > 0)
+ {
+ switch (sphflag)
+ {
+ case 0:
+#ifndef STELLAR_PROP
+
+ if((NumPart - N_gas) > recv_count)
+ {
+ for(i = 0; i < recv_count; i++)
+ {
+ P[NumPart + i] = P[N_gas + i];
+ Key[NumPart + i] = Key[N_gas + i];
+ }
+ }
+ else
+ {
+ for(i = NumPart - 1; i >= N_gas; i--)
+ {
+ P[i + recv_count] = P[i];
+ Key[i + recv_count] = Key[i];
+ }
+ }
+
+#else
+ /* A : move elts of last block */
+ if((NumPart - N_gas - N_stars) > recv_count)
+ {
+ for(i = 0; i < recv_count; i++)
+ {
+ P[NumPart + i] = P[N_gas + N_stars + i];
+ Key[NumPart + i] = Key[N_gas + N_stars + i];
+ }
+ }
+ else
+ {
+ for(i = NumPart - 1; i >= N_gas + N_stars; i--)
+ {
+ P[i + recv_count] = P[i];
+ Key[i + recv_count] = Key[i];
+ }
+ }
+
+
+ /* B : move stars */
+ if (N_stars > 0)
+ {
+ if((N_stars) > recv_count)
+ {
+ for(i = 0; i < recv_count; i++)
+ {
+ P[N_gas + N_stars + i] = P[N_gas + i];
+ Key[N_gas + N_stars + i] = Key[N_gas + i];
+ StP[P[N_gas + N_stars + i].StPIdx].PIdx = N_gas + N_stars + i;
+ }
+ }
+ else
+ {
+ for(i = N_gas + N_stars - 1; i >= N_gas; i--)
+ {
+ P[i + recv_count] = P[i];
+ Key[i + recv_count] = Key[i];
+ StP[P[i + recv_count].StPIdx].PIdx = i + recv_count;
+ }
+ }
+ }
+#endif
+ MPI_Recv(&P[N_gas], recv_count * sizeof(struct particle_data), MPI_BYTE, partner, TAG_PDATA,
+ MPI_COMM_WORLD, &status);
+ MPI_Recv(&Key[N_gas], recv_count * sizeof(peanokey), MPI_BYTE, partner, TAG_KEY,
+ MPI_COMM_WORLD, &status);
+ MPI_Recv(&SphP[N_gas], recv_count * sizeof(struct sph_particle_data), MPI_BYTE, partner,
+ TAG_SPHDATA, MPI_COMM_WORLD, &status);
+
+ N_gas += recv_count;
+ break;
+#ifdef STELLAR_PROP
+ case 1:
+ if((NumPart - N_gas - N_stars) > recv_count)
+ {
+ for(i = 0; i < recv_count; i++)
+ {
+ P[NumPart + i] = P[N_gas + N_stars + i];
+ Key[NumPart + i] = Key[N_gas + N_stars + i];
+ }
+ }
+ else
+ {
+ for(i = NumPart - 1; i >= N_gas + N_stars; i--)
+ {
+ P[i + recv_count] = P[i];
+ Key[i + recv_count] = Key[i];
+ }
+ }
+
+ MPI_Recv(&P[N_gas+N_stars], recv_count * sizeof(struct particle_data), MPI_BYTE, partner, TAG_PDATA,
+ MPI_COMM_WORLD, &status);
+ MPI_Recv(&Key[N_gas+N_stars], recv_count * sizeof(peanokey), MPI_BYTE, partner, TAG_KEY,
+ MPI_COMM_WORLD, &status);
+ MPI_Recv(&StP[N_stars], recv_count * sizeof(struct st_particle_data), MPI_BYTE, partner, TAG_STDATA,
+ MPI_COMM_WORLD, &status);
+
+ /* set right links */
+ for(i = 0; i < recv_count; i++)
+ {
+ P[N_gas + N_stars + i].StPIdx = N_stars + i;
+ StP[N_stars + i].PIdx = N_gas + N_stars + i;
+ }
+
+
+ N_stars += recv_count;
+ break;
+#endif
+ default:
+ MPI_Recv(&P[NumPart], recv_count * sizeof(struct particle_data), MPI_BYTE, partner,
+ TAG_PDATA, MPI_COMM_WORLD, &status);
+ MPI_Recv(&Key[NumPart], recv_count * sizeof(peanokey), MPI_BYTE, partner,
+ TAG_KEY, MPI_COMM_WORLD, &status);
+
+ break;
+ }
+
+
+ NumPart += recv_count;
+ }
+ }
+ }
+}
+
+/*! This function determines how many particles that are currently stored
+ * on the local CPU have to be moved off according to the domain
+ * decomposition.
+ */
+void domain_countToGo(void)
+{
+ int n, no;
+
+ for(n = 0; n < NTask; n++)
+ {
+ local_toGo[n] = 0;
+ local_toGoSph[n] = 0;
+#ifdef STELLAR_PROP
+ local_toGoSt[n] = 0;
+#endif
+ }
+
+ for(n = 0; n < NumPart; n++)
+ {
+ no = 0;
+
+ while(TopNodes[no].Daughter >= 0)
+ no = TopNodes[no].Daughter + (Key[n] - TopNodes[no].StartKey) / (TopNodes[no].Size / 8);
+
+ no = TopNodes[no].Leaf;
+
+ if(DomainTask[no] != ThisTask)
+ {
+ local_toGo[DomainTask[no]] += 1;
+ if(P[n].Type == 0)
+ local_toGoSph[DomainTask[no]] += 1;
+#ifdef STELLAR_PROP
+ if(P[n].Type == 1)
+ local_toGoSt[DomainTask[no]] += 1;
+#endif
+ }
+ }
+
+ MPI_Allgather(local_toGo, NTask, MPI_INT, toGo, NTask, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(local_toGoSph, NTask, MPI_INT, toGoSph, NTask, MPI_INT, MPI_COMM_WORLD);
+#ifdef STELLAR_PROP
+ MPI_Allgather(local_toGoSt, NTask, MPI_INT, toGoSt, NTask, MPI_INT, MPI_COMM_WORLD);
+#endif
+}
+
+
+/*! This function walks the global top tree in order to establish the
+ * number of leaves it has. These leaves are distributed to different
+ * processors.
+ */
+void domain_walktoptree(int no)
+{
+ int i;
+
+ if(TopNodes[no].Daughter == -1)
+ {
+ TopNodes[no].Leaf = NTopleaves;
+ NTopleaves++;
+ }
+ else
+ {
+ for(i = 0; i < 8; i++)
+ domain_walktoptree(TopNodes[no].Daughter + i);
+ }
+}
+
+/*! This routine bins the particles onto the domain-grid, i.e. it sums up the
+ * total number of particles and the total amount of work in each of the
+ * domain-cells. This information forms the basis for the actual decision on
+ * the adopted domain decomposition.
+ */
+void domain_sumCost(void)
+{
+ int i, n, no;
+ double *local_DomainWork;
+ int *local_DomainCount;
+ int *local_DomainCountSph;
+
+ local_DomainWork = malloc(NTopnodes * sizeof(double));
+ local_DomainCount = malloc(NTopnodes * sizeof(int));
+ local_DomainCountSph = malloc(NTopnodes * sizeof(int));
+
+
+
+ NTopleaves = 0;
+
+ domain_walktoptree(0);
+
+ for(i = 0; i < NTopleaves; i++)
+ {
+ local_DomainWork[i] = 0;
+ local_DomainCount[i] = 0;
+ local_DomainCountSph[i] = 0;
+ }
+
+ if(ThisTask == 0)
+ printf("NTopleaves= %d\n", NTopleaves);
+
+
+ for(n = 0; n < NumPart; n++)
+ {
+ no = 0;
+
+ while(TopNodes[no].Daughter >= 0)
+ no = TopNodes[no].Daughter + (Key[n] - TopNodes[no].StartKey) / (TopNodes[no].Size / 8);
+
+ no = TopNodes[no].Leaf;
+
+ if(P[n].Ti_endstep > P[n].Ti_begstep)
+ local_DomainWork[no] += (1.0 + P[n].GravCost) / (P[n].Ti_endstep - P[n].Ti_begstep);
+ else
+ local_DomainWork[no] += (1.0 + P[n].GravCost);
+
+ local_DomainCount[no] += 1;
+ if(P[n].Type == 0)
+ local_DomainCountSph[no] += 1;
+ }
+
+ MPI_Allreduce(local_DomainWork, DomainWork, NTopleaves, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(local_DomainCount, DomainCount, NTopleaves, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(local_DomainCountSph, DomainCountSph, NTopleaves, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+
+ free(local_DomainCountSph);
+ free(local_DomainCount);
+ free(local_DomainWork);
+}
+
+
+/*! This routine finds the extent of the global domain grid.
+ */
+void domain_findExtent(void)
+{
+ int i, j;
+ double len, xmin[3], xmax[3], xmin_glob[3], xmax_glob[3];
+
+ /* determine local extension */
+ for(j = 0; j < 3; j++)
+ {
+ xmin[j] = MAX_REAL_NUMBER;
+ xmax[j] = -MAX_REAL_NUMBER;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ if(xmin[j] > P[i].Pos[j])
+ xmin[j] = P[i].Pos[j];
+
+ if(xmax[j] < P[i].Pos[j])
+ xmax[j] = P[i].Pos[j];
+ }
+ }
+
+ MPI_Allreduce(xmin, xmin_glob, 3, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+ MPI_Allreduce(xmax, xmax_glob, 3, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+
+ len = 0;
+ for(j = 0; j < 3; j++)
+ if(xmax_glob[j] - xmin_glob[j] > len)
+ len = xmax_glob[j] - xmin_glob[j];
+
+ len *= 1.001;
+
+#ifdef DOMAIN_AT_ORIGIN
+ for(j = 0; j < 3; j++)
+ {
+ DomainCenter[j] = 0.0;
+ DomainCorner[j] = - 0.5 * len;
+ }
+#else
+ for(j = 0; j < 3; j++)
+ {
+ DomainCenter[j] = 0.5 * (xmin_glob[j] + xmax_glob[j]);
+ DomainCorner[j] = 0.5 * (xmin_glob[j] + xmax_glob[j]) - 0.5 * len;
+ }
+#endif
+
+ DomainLen = len;
+ DomainFac = 1.0 / len * (((peanokey) 1) << (BITS_PER_DIMENSION));
+
+#ifdef OTHERINFO
+ if(ThisTask == 0)
+ {
+ printf("xmin_glob = %g %g %g\n",xmin_glob[0],xmin_glob[1],xmin_glob[2]);
+ printf("xmax_glob = %g %g %g\n",xmax_glob[0],xmax_glob[1],xmax_glob[2]);
+ printf("DomainCenter = %g %g %g\n",DomainCenter[0], DomainCenter[1], DomainCenter[2]);
+ printf("DomainCorner = %g %g %g\n",DomainCorner[0], DomainCorner[1], DomainCorner[2]);
+ printf("DomainLen = %g\n",DomainLen);
+ }
+#endif
+
+}
+
+
+/*! This function constructs the global top-level tree node that is used
+ * for the domain decomposition. This is done by considering the string of
+ * Peano-Hilbert keys for all particles, which is recursively chopped off
+ * in pieces of eight segments until each segment holds at most a certain
+ * number of particles.
+ */
+void domain_determineTopTree(void)
+{
+ int i, ntop_local, ntop;
+ int *ntopnodelist, *ntopoffset;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ KeySorted[i] = Key[i] = peano_hilbert_key((P[i].Pos[0] - DomainCorner[0]) * DomainFac,
+ (P[i].Pos[1] - DomainCorner[1]) * DomainFac,
+ (P[i].Pos[2] - DomainCorner[2]) * DomainFac,
+ BITS_PER_DIMENSION);
+ }
+
+ qsort(KeySorted, NumPart, sizeof(peanokey), domain_compare_key);
+
+ NTopnodes = 1;
+ TopNodes[0].Daughter = -1;
+ TopNodes[0].Size = PEANOCELLS;
+ TopNodes[0].StartKey = 0;
+ TopNodes[0].Count = NumPart;
+ TopNodes[0].Pstart = 0;
+
+ domain_topsplit_local(0, 0);
+
+ toplist_local = malloc(NTopnodes * sizeof(struct topnode_exchange));
+
+ for(i = 0, ntop_local = 0; i < NTopnodes; i++)
+ {
+ if(TopNodes[i].Daughter == -1) /* only use leaves */
+ {
+ toplist_local[ntop_local].Startkey = TopNodes[i].StartKey;
+ toplist_local[ntop_local].Count = TopNodes[i].Count;
+ ntop_local++;
+ }
+ }
+
+ ntopnodelist = malloc(sizeof(int) * NTask);
+ ntopoffset = malloc(sizeof(int) * NTask);
+
+ MPI_Allgather(&ntop_local, 1, MPI_INT, ntopnodelist, 1, MPI_INT, MPI_COMM_WORLD);
+
+ for(i = 0, ntop = 0, ntopoffset[0] = 0; i < NTask; i++)
+ {
+ ntop += ntopnodelist[i];
+ if(i > 0)
+ ntopoffset[i] = ntopoffset[i - 1] + ntopnodelist[i - 1];
+ }
+
+
+ toplist = malloc(ntop * sizeof(struct topnode_exchange));
+
+ for(i = 0; i < NTask; i++)
+ {
+ ntopnodelist[i] *= sizeof(struct topnode_exchange);
+ ntopoffset[i] *= sizeof(struct topnode_exchange);
+ }
+
+ MPI_Allgatherv(toplist_local, ntop_local * sizeof(struct topnode_exchange), MPI_BYTE,
+ toplist, ntopnodelist, ntopoffset, MPI_BYTE, MPI_COMM_WORLD);
+
+ qsort(toplist, ntop, sizeof(struct topnode_exchange), domain_compare_toplist);
+
+ NTopnodes = 1;
+ TopNodes[0].Daughter = -1;
+ TopNodes[0].Size = PEANOCELLS;
+ TopNodes[0].StartKey = 0;
+ TopNodes[0].Count = All.TotNumPart;
+ TopNodes[0].Pstart = 0;
+ TopNodes[0].Blocks = ntop;
+
+ domain_topsplit(0, 0);
+
+ free(toplist);
+ free(ntopoffset);
+ free(ntopnodelist);
+ free(toplist_local);
+
+}
+
+
+
+/*! This function is responsible for constructing the local top-level
+ * Peano-Hilbert segments. A segment is cut into 8 pieces recursively
+ * until the number of particles in the segment has fallen below
+ * All.TotNumPart / (TOPNODEFACTOR * NTask * NTask).
+ */
+void domain_topsplit_local(int node, peanokey startkey)
+{
+ int i, p, sub, bin;
+
+ if(TopNodes[node].Size >= 8)
+ {
+ TopNodes[node].Daughter = NTopnodes;
+
+ for(i = 0; i < 8; i++)
+ {
+ if(NTopnodes < MAXTOPNODES)
+ {
+ sub = TopNodes[node].Daughter + i;
+ TopNodes[sub].Size = TopNodes[node].Size / 8;
+ TopNodes[sub].Count = 0;
+ TopNodes[sub].Daughter = -1;
+ TopNodes[sub].StartKey = startkey + i * TopNodes[sub].Size;
+ TopNodes[sub].Pstart = TopNodes[node].Pstart;
+
+ NTopnodes++;
+ }
+ else
+ {
+ printf("task=%d: We are out of Topnodes. Increasing the constant MAXTOPNODES might help.\n",
+ ThisTask);
+ fflush(stdout);
+ endrun(13213);
+ }
+ }
+
+ for(p = TopNodes[node].Pstart; p < TopNodes[node].Pstart + TopNodes[node].Count; p++)
+ {
+ bin = (KeySorted[p] - startkey) / (TopNodes[node].Size / 8);
+
+ if(bin < 0 || bin > 7)
+ {
+ printf("task=%d: something odd has happened here. bin=%d\n", ThisTask, bin);
+ fflush(stdout);
+ endrun(13123123);
+ }
+
+ sub = TopNodes[node].Daughter + bin;
+
+ if(TopNodes[sub].Count == 0)
+ TopNodes[sub].Pstart = p;
+
+ TopNodes[sub].Count++;
+ }
+
+ for(i = 0; i < 8; i++)
+ {
+ sub = TopNodes[node].Daughter + i;
+ //if(TopNodes[sub].Count > All.TotNumPart / (TOPNODEFACTOR * NTask * NTask))
+ if(TopNodes[sub].Count > All.TotNumPart / (TOPNODEFACTOR * NTask))
+ domain_topsplit_local(sub, TopNodes[sub].StartKey);
+ }
+ }
+}
+
+
+
+/*! This function is responsible for constructing the global top-level tree
+ * segments. Starting from a joint list of all local top-level segments,
+ * in which mulitple occurences of the same spatial segment have been
+ * combined, a segment is subdivided into 8 pieces recursively until the
+ * number of particles in each segment has fallen below All.TotNumPart /
+ * (TOPNODEFACTOR * NTask).
+ */
+void domain_topsplit(int node, peanokey startkey)
+{
+ int i, p, sub, bin;
+
+ if(TopNodes[node].Size >= 8)
+ {
+ TopNodes[node].Daughter = NTopnodes;
+
+ for(i = 0; i < 8; i++)
+ {
+ if(NTopnodes < MAXTOPNODES)
+ {
+ sub = TopNodes[node].Daughter + i;
+ TopNodes[sub].Size = TopNodes[node].Size / 8;
+ TopNodes[sub].Count = 0;
+ TopNodes[sub].Blocks = 0;
+ TopNodes[sub].Daughter = -1;
+ TopNodes[sub].StartKey = startkey + i * TopNodes[sub].Size;
+ TopNodes[sub].Pstart = TopNodes[node].Pstart;
+ NTopnodes++;
+ }
+ else
+ {
+ printf("Task=%d: We are out of Topnodes. Increasing the constant MAXTOPNODES might help.\n",
+ ThisTask);
+ fflush(stdout);
+ endrun(137213);
+ }
+ }
+
+ for(p = TopNodes[node].Pstart; p < TopNodes[node].Pstart + TopNodes[node].Blocks; p++)
+ {
+ bin = (toplist[p].Startkey - startkey) / (TopNodes[node].Size / 8);
+ sub = TopNodes[node].Daughter + bin;
+
+ if(bin < 0 || bin > 7)
+ endrun(77);
+
+ if(TopNodes[sub].Blocks == 0)
+ TopNodes[sub].Pstart = p;
+
+ TopNodes[sub].Count += toplist[p].Count;
+ TopNodes[sub].Blocks++;
+ }
+
+ for(i = 0; i < 8; i++)
+ {
+ sub = TopNodes[node].Daughter + i;
+ if(TopNodes[sub].Count > All.TotNumPart / (TOPNODEFACTOR * NTask))
+ domain_topsplit(sub, TopNodes[sub].StartKey);
+ }
+ }
+}
+
+
+/*! This is a comparison kernel used in a sort routine.
+ */
+int domain_compare_toplist(const void *a, const void *b)
+{
+ if(((struct topnode_exchange *) a)->Startkey < (((struct topnode_exchange *) b)->Startkey))
+ return -1;
+
+ if(((struct topnode_exchange *) a)->Startkey > (((struct topnode_exchange *) b)->Startkey))
+ return +1;
+
+ return 0;
+}
+
+/*! This is a comparison kernel used in a sort routine.
+ */
+int domain_compare_key(const void *a, const void *b)
+{
+ if(*(peanokey *) a < *(peanokey *) b)
+ return -1;
+
+ if(*(peanokey *) a > *(peanokey *) b)
+ return +1;
+
+ return 0;
+}
diff --git a/driftfac.c b/driftfac.c
new file mode 100644
index 0000000..8487267
--- /dev/null
+++ b/driftfac.c
@@ -0,0 +1,224 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_math.h>
+#include <gsl/gsl_integration.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file driftfac.c
+ * \brief compute loop-up tables for prefactors in cosmological integration
+ */
+
+static double logTimeBegin;
+static double logTimeMax;
+
+
+/*! This function computes look-up tables for factors needed in
+ * cosmological integrations. The (simple) integrations are carried out
+ * with the GSL library. Separate factors are computed for the "drift",
+ * and the gravitational and hydrodynamical "kicks". The lookup-table is
+ * used for reasons of speed.
+ */
+void init_drift_table(void)
+{
+#define WORKSIZE 100000
+ int i;
+ double result, abserr;
+ gsl_function F;
+ gsl_integration_workspace *workspace;
+
+ logTimeBegin = log(All.TimeBegin);
+ logTimeMax = log(All.TimeMax);
+
+ workspace = gsl_integration_workspace_alloc(WORKSIZE);
+
+ for(i = 0; i < DRIFT_TABLE_LENGTH; i++)
+ {
+ F.function = &drift_integ;
+ gsl_integration_qag(&F, exp(logTimeBegin), exp(logTimeBegin + ((logTimeMax - logTimeBegin) / DRIFT_TABLE_LENGTH) * (i + 1)), All.Hubble, /* note: absolute error just a dummy */
+ 1.0e-8, WORKSIZE, GSL_INTEG_GAUSS41, workspace, &result, &abserr);
+ DriftTable[i] = result;
+
+
+ F.function = &gravkick_integ;
+ gsl_integration_qag(&F, exp(logTimeBegin), exp(logTimeBegin + ((logTimeMax - logTimeBegin) / DRIFT_TABLE_LENGTH) * (i + 1)), All.Hubble, /* note: absolute error just a dummy */
+ 1.0e-8, WORKSIZE, GSL_INTEG_GAUSS41, workspace, &result, &abserr);
+ GravKickTable[i] = result;
+
+
+ F.function = &hydrokick_integ;
+ gsl_integration_qag(&F, exp(logTimeBegin), exp(logTimeBegin + ((logTimeMax - logTimeBegin) / DRIFT_TABLE_LENGTH) * (i + 1)), All.Hubble, /* note: absolute error just a dummy */
+ 1.0e-8, WORKSIZE, GSL_INTEG_GAUSS41, workspace, &result, &abserr);
+ HydroKickTable[i] = result;
+ }
+
+ gsl_integration_workspace_free(workspace);
+}
+
+
+/*! This function integrates the cosmological prefactor for a drift step
+ * between time0 and time1. The value returned is * \f[ \int_{a_0}^{a_1}
+ * \frac{{\rm d}a}{H(a)} * \f]
+ */
+double get_drift_factor(int time0, int time1)
+{
+ double a1, a2, df1, df2, u1, u2;
+ int i1, i2;
+
+ /* note: will only be called for cosmological integration */
+
+ a1 = logTimeBegin + time0 * All.Timebase_interval;
+ a2 = logTimeBegin + time1 * All.Timebase_interval;
+
+ u1 = (a1 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i1 = (int) u1;
+ if(i1 >= DRIFT_TABLE_LENGTH)
+ i1 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i1 <= 1)
+ df1 = u1 * DriftTable[0];
+ else
+ df1 = DriftTable[i1 - 1] + (DriftTable[i1] - DriftTable[i1 - 1]) * (u1 - i1);
+
+
+ u2 = (a2 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i2 = (int) u2;
+ if(i2 >= DRIFT_TABLE_LENGTH)
+ i2 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i2 <= 1)
+ df2 = u2 * DriftTable[0];
+ else
+ df2 = DriftTable[i2 - 1] + (DriftTable[i2] - DriftTable[i2 - 1]) * (u2 - i2);
+
+ return df2 - df1;
+}
+
+
+/*! This function integrates the cosmological prefactor for a kick step of
+ * the gravitational force.
+ */
+double get_gravkick_factor(int time0, int time1)
+{
+ double a1, a2, df1, df2, u1, u2;
+ int i1, i2;
+
+ /* note: will only be called for cosmological integration */
+
+ a1 = logTimeBegin + time0 * All.Timebase_interval; /* yr: this gives log(a1) to be compatible with init_drift_table*/
+ a2 = logTimeBegin + time1 * All.Timebase_interval; /* yr: this gives log(a2) to be compatible with init_drift_table*/
+
+ u1 = (a1 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i1 = (int) u1;
+ if(i1 >= DRIFT_TABLE_LENGTH)
+ i1 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i1 <= 1)
+ df1 = u1 * GravKickTable[0];
+ else
+ df1 = GravKickTable[i1 - 1] + (GravKickTable[i1] - GravKickTable[i1 - 1]) * (u1 - i1); /* yr : interpolate */
+
+
+ u2 = (a2 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i2 = (int) u2;
+ if(i2 >= DRIFT_TABLE_LENGTH)
+ i2 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i2 <= 1)
+ df2 = u2 * GravKickTable[0];
+ else
+ df2 = GravKickTable[i2 - 1] + (GravKickTable[i2] - GravKickTable[i2 - 1]) * (u2 - i2);
+
+ return df2 - df1;
+}
+
+/*! This function integrates the cosmological prefactor for a kick step of
+ * the hydrodynamical force.
+ */
+double get_hydrokick_factor(int time0, int time1)
+{
+ double a1, a2, df1, df2, u1, u2;
+ int i1, i2;
+
+ /* note: will only be called for cosmological integration */
+
+ a1 = logTimeBegin + time0 * All.Timebase_interval;
+ a2 = logTimeBegin + time1 * All.Timebase_interval;
+
+ u1 = (a1 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i1 = (int) u1;
+ if(i1 >= DRIFT_TABLE_LENGTH)
+ i1 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i1 <= 1)
+ df1 = u1 * HydroKickTable[0];
+ else
+ df1 = HydroKickTable[i1 - 1] + (HydroKickTable[i1] - HydroKickTable[i1 - 1]) * (u1 - i1);
+
+
+ u2 = (a2 - logTimeBegin) / (logTimeMax - logTimeBegin) * DRIFT_TABLE_LENGTH;
+ i2 = (int) u2;
+ if(i2 >= DRIFT_TABLE_LENGTH)
+ i2 = DRIFT_TABLE_LENGTH - 1;
+
+ if(i2 <= 1)
+ df2 = u2 * HydroKickTable[0];
+ else
+ df2 = HydroKickTable[i2 - 1] + (HydroKickTable[i2] - HydroKickTable[i2 - 1]) * (u2 - i2);
+
+ return df2 - df1;
+}
+
+
+/*! Integration kernel for drift factor computation.
+ */
+double drift_integ(double a, void *param)
+{
+ double h;
+
+ h = All.Omega0 / (a * a * a) + (1 - All.Omega0 - All.OmegaLambda) / (a * a) + All.OmegaLambda;
+ h = All.Hubble * sqrt(h);
+
+ return 1 / (h * a * a * a);
+}
+
+/*! Integration kernel for gravitational kick factor computation.
+ */
+double gravkick_integ(double a, void *param)
+{
+ double h;
+
+ h = All.Omega0 / (a * a * a) + (1 - All.Omega0 - All.OmegaLambda) / (a * a) + All.OmegaLambda;
+ h = All.Hubble * sqrt(h);
+
+ return 1 / (h * a * a);
+}
+
+
+/*! Integration kernel for hydrodynamical kick factor computation.
+ */
+double hydrokick_integ(double a, void *param)
+{
+ double h;
+
+ h = All.Omega0 / (a * a * a) + (1 - All.Omega0 - All.OmegaLambda) / (a * a) + All.OmegaLambda;
+ h = All.Hubble * sqrt(h);
+
+ return 1 / (h * pow(a, 3 * GAMMA_MINUS1) * a);
+}
+
+double growthfactor_integ(double a, void *param)
+{
+ double s;
+
+ s = All.Omega0 + (1 - All.Omega0 - All.OmegaLambda) * a + All.OmegaLambda * a * a * a;
+ s = sqrt(s);
+
+ return pow(sqrt(a) / s, 3);
+}
+
+
diff --git a/endrun.c b/endrun.c
new file mode 100644
index 0000000..3070a7e
--- /dev/null
+++ b/endrun.c
@@ -0,0 +1,43 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file endrun.c
+ * \brief Termination of simulation
+ *
+ * This file contains routines for termination of the simulation.
+ */
+
+/*! This function aborts the simulations. If a single processors wants an
+ * immediate termination, the function needs to be called with ierr>0. A
+ * bunch of MPI-error messages may also appear in this case. For ierr=0,
+ * MPI is gracefully cleaned up, but this requires that all processors
+ * call endrun().
+ */
+void endrun(int ierr)
+{
+ if(ierr)
+ {
+ printf("task %d: endrun called with an error level of %d\n\n\n", ThisTask, ierr);
+ fflush(stdout);
+#ifdef DEBUG
+ terminate_processes();
+ raise(SIGABRT);
+ sleep(60);
+#else
+ MPI_Abort(MPI_COMM_WORLD, ierr);
+#endif
+ exit(0);
+ }
+
+ MPI_Finalize();
+ exit(0);
+}
diff --git a/forcetree.c b/forcetree.c
new file mode 100644
index 0000000..626ebba
--- /dev/null
+++ b/forcetree.c
@@ -0,0 +1,3342 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file forcetree.c
+ * \brief gravitational tree and code for Ewald correction
+ *
+ * This file contains the computation of the gravitational force by means
+ * of a tree. The type of tree implemented is a geometrical oct-tree,
+ * starting from a cube encompassing all particles. This cube is
+ * automatically found in the domain decomposition, which also splits up
+ * the global "top-level" tree along node boundaries, moving the particles
+ * of different parts of the tree to separate processors. Tree nodes can
+ * be dynamically updated in drift/kick operations to avoid having to
+ * reconstruct the tree every timestep.
+ */
+
+/*! auxialiary variable used to set-up non-recursive walk */
+static int last;
+
+
+
+/*! length of lock-up table for short-range force kernel in TreePM algorithm */
+#define NTAB 1000
+/*! variables for short-range lookup table */
+static float tabfac, shortrange_table[NTAB], shortrange_table_potential[NTAB];
+
+/*! toggles after first tree-memory allocation, has only influence on log-files */
+static int first_flag = 0;
+
+
+
+
+#ifdef PERIODIC
+/*! Macro that maps a distance to the nearest periodic neighbour */
+#define NEAREST(x) (((x)>boxhalf)?((x)-boxsize):(((x)<-boxhalf)?((x)+boxsize):(x)))
+/*! Size of 3D lock-up table for Ewald correction force */
+#define EN 64
+/*! 3D lock-up table for Ewald correction to force and potential. Only one
+ * octant is stored, the rest constructed by using the symmetry
+ */
+static FLOAT fcorrx[EN + 1][EN + 1][EN + 1];
+static FLOAT fcorry[EN + 1][EN + 1][EN + 1];
+static FLOAT fcorrz[EN + 1][EN + 1][EN + 1];
+static FLOAT potcorr[EN + 1][EN + 1][EN + 1];
+static double fac_intp;
+#endif
+
+
+
+/*! This function is a driver routine for constructing the gravitational
+ * oct-tree, which is done by calling a small number of other functions.
+ */
+int force_treebuild(int npart)
+{
+ Numnodestree = force_treebuild_single(npart);
+
+ force_update_pseudoparticles();
+
+ force_flag_localnodes();
+
+ TimeOfLastTreeConstruction = All.Time;
+
+ return Numnodestree;
+}
+
+
+
+/*! Constructs the gravitational oct-tree.
+ *
+ * The index convention for accessing tree nodes is the following: the
+ * indices 0...NumPart-1 reference single particles, the indices
+ * All.MaxPart.... All.MaxPart+nodes-1 reference tree nodes. `Nodes_base'
+ * points to the first tree node, while `nodes' is shifted such that
+ * nodes[All.MaxPart] gives the first tree node. Finally, node indices
+ * with values 'All.MaxPart + MaxNodes' and larger indicate "pseudo
+ * particles", i.e. multipole moments of top-level nodes that lie on
+ * different CPUs. If such a node needs to be opened, the corresponding
+ * particle must be exported to that CPU. The 'Extnodes' structure
+ * parallels that of 'Nodes'. Its information is only needed for the SPH
+ * part of the computation. (The data is split onto these two structures
+ * as a tuning measure. If it is merged into 'Nodes' a somewhat bigger
+ * size of the nodes also for gravity would result, which would reduce
+ * cache utilization slightly.
+ */
+int force_treebuild_single(int npart)
+{
+ int i, j, subnode = 0, parent, numnodes;
+ int nfree, th, nn, no;
+ struct NODE *nfreep;
+ double lenhalf, epsilon;
+ peanokey key;
+
+
+ /* create an empty root node */
+ nfree = All.MaxPart; /* index of first free node */
+ nfreep = &Nodes[nfree]; /* select first node */
+
+ nfreep->len = DomainLen;
+ for(j = 0; j < 3; j++)
+ nfreep->center[j] = DomainCenter[j];
+ for(j = 0; j < 8; j++)
+ nfreep->u.suns[j] = -1;
+
+
+ numnodes = 1;
+ nfreep++;
+ nfree++;
+
+ /* create a set of empty nodes corresponding to the top-level domain
+ * grid. We need to generate these nodes first to make sure that we have a
+ * complete top-level tree which allows the easy insertion of the
+ * pseudo-particles at the right place
+ */
+
+ force_create_empty_nodes(All.MaxPart, 0, 1, 0, 0, 0, &numnodes, &nfree);
+
+
+ /* if a high-resolution region in a global tree is used, we need to generate
+ * an additional set empty nodes to make sure that we have a complete
+ * top-level tree for the high-resolution inset
+ */
+
+ nfreep = &Nodes[nfree];
+ parent = -1; /* note: will not be used below before it is changed */
+
+
+ /* now we insert all particles */
+ for(i = 0; i < npart; i++)
+ {
+
+ /* the softening is only used to check whether particles are so close
+ * that the tree needs not to be refined further
+ */
+ epsilon = All.ForceSoftening[P[i].Type];
+
+ key = peano_hilbert_key((P[i].Pos[0] - DomainCorner[0]) * DomainFac,
+ (P[i].Pos[1] - DomainCorner[1]) * DomainFac,
+ (P[i].Pos[2] - DomainCorner[2]) * DomainFac, BITS_PER_DIMENSION);
+
+ no = 0;
+ while(TopNodes[no].Daughter >= 0)
+ no = TopNodes[no].Daughter + (key - TopNodes[no].StartKey) / (TopNodes[no].Size / 8);
+
+ no = TopNodes[no].Leaf;
+ th = DomainNodeIndex[no];
+
+ while(1)
+ {
+ if(th >= All.MaxPart) /* we are dealing with an internal node */
+ {
+ subnode = 0;
+ if(P[i].Pos[0] > Nodes[th].center[0])
+ subnode += 1;
+ if(P[i].Pos[1] > Nodes[th].center[1])
+ subnode += 2;
+ if(P[i].Pos[2] > Nodes[th].center[2])
+ subnode += 4;
+
+ nn = Nodes[th].u.suns[subnode];
+
+ if(nn >= 0) /* ok, something is in the daughter slot already, need to continue */
+ {
+ parent = th;
+ th = nn;
+ }
+ else
+ {
+ /* here we have found an empty slot where we can attach
+ * the new particle as a leaf.
+ */
+ Nodes[th].u.suns[subnode] = i;
+ break; /* done for this particle */
+ }
+ }
+ else
+ {
+ /* We try to insert into a leaf with a single particle. Need
+ * to generate a new internal node at this point.
+ */
+ Nodes[parent].u.suns[subnode] = nfree;
+
+ nfreep->len = 0.5 * Nodes[parent].len;
+ lenhalf = 0.25 * Nodes[parent].len;
+
+ if(subnode & 1)
+ nfreep->center[0] = Nodes[parent].center[0] + lenhalf;
+ else
+ nfreep->center[0] = Nodes[parent].center[0] - lenhalf;
+
+ if(subnode & 2)
+ nfreep->center[1] = Nodes[parent].center[1] + lenhalf;
+ else
+ nfreep->center[1] = Nodes[parent].center[1] - lenhalf;
+
+ if(subnode & 4)
+ nfreep->center[2] = Nodes[parent].center[2] + lenhalf;
+ else
+ nfreep->center[2] = Nodes[parent].center[2] - lenhalf;
+
+ nfreep->u.suns[0] = -1;
+ nfreep->u.suns[1] = -1;
+ nfreep->u.suns[2] = -1;
+ nfreep->u.suns[3] = -1;
+ nfreep->u.suns[4] = -1;
+ nfreep->u.suns[5] = -1;
+ nfreep->u.suns[6] = -1;
+ nfreep->u.suns[7] = -1;
+
+
+ subnode = 0;
+ if(P[th].Pos[0] > nfreep->center[0])
+ subnode += 1;
+ if(P[th].Pos[1] > nfreep->center[1])
+ subnode += 2;
+ if(P[th].Pos[2] > nfreep->center[2])
+ subnode += 4;
+#ifndef NOTREERND
+ if(nfreep->len < 1.0e-3 * epsilon)
+ {
+ /* seems like we're dealing with particles at identical (or extremely close)
+ * locations. Randomize subnode index to allow tree construction. Note: Multipole moments
+ * of tree are still correct, but this will only happen well below gravitational softening
+ * length-scale anyway.
+ */
+ subnode = (int) (8.0 * get_random_number((0xffff & P[i].ID) + P[i].GravCost));
+ P[i].GravCost += 1;
+ if(subnode >= 8)
+ subnode = 7;
+ }
+#endif
+ nfreep->u.suns[subnode] = th;
+
+ th = nfree; /* resume trying to insert the new particle at
+ * the newly created internal node
+ */
+
+ numnodes++;
+ nfree++;
+ nfreep++;
+
+ if((numnodes) >= MaxNodes)
+ {
+ printf("task %d: maximum number %d of tree-nodes reached.\n", ThisTask, MaxNodes);
+ printf("for particle %d\n", i);
+ dump_particles();
+ endrun(1);
+ }
+ }
+ }
+ }
+
+
+ /* insert the pseudo particles that represent the mass distribution of other domains */
+ force_insert_pseudo_particles();
+
+
+ /* now compute the multipole moments recursively */
+ last = -1;
+
+ force_update_node_recursive(All.MaxPart, -1, -1);
+
+ if(last >= All.MaxPart)
+ {
+ if(last >= All.MaxPart + MaxNodes) /* a pseudo-particle */
+ Nextnode[last - MaxNodes] = -1;
+ else
+ Nodes[last].u.d.nextnode = -1;
+ }
+ else
+ Nextnode[last] = -1;
+
+ return numnodes;
+}
+
+
+
+/*! This function recursively creates a set of empty tree nodes which
+ * corresponds to the top-level tree for the domain grid. This is done to
+ * ensure that this top-level tree is always "complete" so that we can
+ * easily associate the pseudo-particles of other CPUs with tree-nodes at
+ * a given level in the tree, even when the particle population is so
+ * sparse that some of these nodes are actually empty.
+*/
+void force_create_empty_nodes(int no, int topnode, int bits, int x, int y, int z, int *nodecount,
+ int *nextfree)
+{
+ int i, j, k, n, sub, count;
+
+ if(TopNodes[topnode].Daughter >= 0)
+ {
+ for(i = 0; i < 2; i++)
+ for(j = 0; j < 2; j++)
+ for(k = 0; k < 2; k++)
+ {
+ sub = 7 & peano_hilbert_key((x << 1) + i, (y << 1) + j, (z << 1) + k, bits);
+
+ count = i + 2 * j + 4 * k;
+
+ Nodes[no].u.suns[count] = *nextfree;
+
+
+ Nodes[*nextfree].len = 0.5 * Nodes[no].len;
+ Nodes[*nextfree].center[0] = Nodes[no].center[0] + (2 * i - 1) * 0.25 * Nodes[no].len;
+ Nodes[*nextfree].center[1] = Nodes[no].center[1] + (2 * j - 1) * 0.25 * Nodes[no].len;
+ Nodes[*nextfree].center[2] = Nodes[no].center[2] + (2 * k - 1) * 0.25 * Nodes[no].len;
+
+ for(n = 0; n < 8; n++)
+ Nodes[*nextfree].u.suns[n] = -1;
+
+ if(TopNodes[TopNodes[topnode].Daughter + sub].Daughter == -1)
+ DomainNodeIndex[TopNodes[TopNodes[topnode].Daughter + sub].Leaf] = *nextfree;
+
+ *nextfree = *nextfree + 1;
+ *nodecount = *nodecount + 1;
+
+ if((*nodecount) >= MaxNodes)
+ {
+ printf("task %d: maximum number %d of tree-nodes reached.\n", ThisTask, MaxNodes);
+ printf("in create empty nodes\n");
+ dump_particles();
+ endrun(11);
+ }
+
+ force_create_empty_nodes(*nextfree - 1, TopNodes[topnode].Daughter + sub,
+ bits + 1, 2 * x + i, 2 * y + j, 2 * z + k, nodecount, nextfree);
+ }
+ }
+}
+
+
+
+/*! this function inserts pseudo-particles which will represent the mass
+ * distribution of the other CPUs. Initially, the mass of the
+ * pseudo-particles is set to zero, and their coordinate is set to the
+ * center of the domain-cell they correspond to. These quantities will be
+ * updated later on.
+ */
+void force_insert_pseudo_particles(void)
+{
+ int i, index, subnode, nn, th;
+
+ for(i = 0; i < NTopleaves; i++)
+ {
+ index = DomainNodeIndex[i];
+
+ DomainMoment[i].mass = 0;
+#ifdef STELLAR_FLUX
+ DomainMoment[i].starlum = 0;
+#endif
+ DomainMoment[i].s[0] = Nodes[index].center[0];
+ DomainMoment[i].s[1] = Nodes[index].center[1];
+ DomainMoment[i].s[2] = Nodes[index].center[2];
+ }
+
+ for(i = 0; i < NTopleaves; i++)
+ {
+ if(i < DomainMyStart || i > DomainMyLast)
+ {
+ th = All.MaxPart; /* select index of first node in tree */
+
+ while(1)
+ {
+ if(th >= All.MaxPart) /* we are dealing with an internal node */
+ {
+ if(th >= All.MaxPart + MaxNodes)
+ endrun(888); /* this can't be */
+
+ subnode = 0;
+ if(DomainMoment[i].s[0] > Nodes[th].center[0])
+ subnode += 1;
+ if(DomainMoment[i].s[1] > Nodes[th].center[1])
+ subnode += 2;
+ if(DomainMoment[i].s[2] > Nodes[th].center[2])
+ subnode += 4;
+
+ nn = Nodes[th].u.suns[subnode];
+
+ if(nn >= 0) /* ok, something is in the daughter slot already, need to continue */
+ {
+ th = nn;
+ }
+ else
+ {
+ /* here we have found an empty slot where we can
+ * attach the pseudo particle as a leaf
+ */
+ Nodes[th].u.suns[subnode] = All.MaxPart + MaxNodes + i;
+
+ break; /* done for this pseudo particle */
+ }
+ }
+ else
+ {
+ endrun(889); /* this can't be */
+ }
+ }
+ }
+ }
+}
+
+
+/*! this routine determines the multipole moments for a given internal node
+ * and all its subnodes using a recursive computation. The result is
+ * stored in the Nodes[] structure in the sequence of this tree-walk.
+ *
+ * Note that the bitflags-variable for each node is used to store in the
+ * lowest bits some special information: Bit 0 flags whether the node
+ * belongs to the top-level tree corresponding to the domain
+ * decomposition, while Bit 1 signals whether the top-level node is
+ * dependent on local mass.
+ *
+ * If UNEQUALSOFTENINGS is set, bits 2-4 give the particle type with
+ * the maximum softening among the particles in the node, and bit 5
+ * flags whether the node contains any particles with lower softening
+ * than that.
+ */
+void force_update_node_recursive(int no, int sib, int father)
+{
+ int j, jj, p, pp, nextsib, suns[8];
+ FLOAT hmax;
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ int maxsofttype, diffsoftflag;
+#else
+ FLOAT maxsoft;
+#endif
+#endif
+ struct particle_data *pa;
+ double s[3], vs[3], mass;
+#ifdef STELLAR_FLUX
+ double starlum;
+#endif
+
+ if(no >= All.MaxPart && no < All.MaxPart + MaxNodes) /* internal node */
+ {
+ for(j = 0; j < 8; j++)
+ suns[j] = Nodes[no].u.suns[j]; /* this "backup" is necessary because the nextnode entry will
+ overwrite one element (union!) */
+ if(last >= 0)
+ {
+ if(last >= All.MaxPart)
+ {
+ if(last >= All.MaxPart + MaxNodes) /* a pseudo-particle */
+ Nextnode[last - MaxNodes] = no;
+ else
+ Nodes[last].u.d.nextnode = no;
+ }
+ else
+ Nextnode[last] = no;
+ }
+
+ last = no;
+
+ mass = 0;
+#ifdef STELLAR_FLUX
+ starlum = 0;
+#endif
+ s[0] = 0;
+ s[1] = 0;
+ s[2] = 0;
+ vs[0] = 0;
+ vs[1] = 0;
+ vs[2] = 0;
+ hmax = 0;
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ maxsofttype = 7;
+ diffsoftflag = 0;
+#else
+ maxsoft = 0;
+#endif
+#endif
+
+ for(j = 0; j < 8; j++)
+ {
+ if((p = suns[j]) >= 0)
+ {
+ /* check if we have a sibling on the same level */
+ for(jj = j + 1; jj < 8; jj++)
+ if((pp = suns[jj]) >= 0)
+ break;
+
+ if(jj < 8) /* yes, we do */
+ nextsib = pp;
+ else
+ nextsib = sib;
+
+ force_update_node_recursive(p, nextsib, no);
+
+
+ if(p >= All.MaxPart) /* an internal node or pseudo particle */
+ {
+ if(p >= All.MaxPart + MaxNodes) /* a pseudo particle */
+ {
+ /* nothing to be done here because the mass of the
+ * pseudo-particle is still zero. This will be changed
+ * later.
+ */
+ }
+ else
+ {
+ mass += Nodes[p].u.d.mass;
+#ifdef STELLAR_FLUX
+ starlum += Nodes[p].starlum;
+#endif
+ s[0] += Nodes[p].u.d.mass * Nodes[p].u.d.s[0];
+ s[1] += Nodes[p].u.d.mass * Nodes[p].u.d.s[1];
+ s[2] += Nodes[p].u.d.mass * Nodes[p].u.d.s[2];
+ vs[0] += Nodes[p].u.d.mass * Extnodes[p].vs[0];
+ vs[1] += Nodes[p].u.d.mass * Extnodes[p].vs[1];
+ vs[2] += Nodes[p].u.d.mass * Extnodes[p].vs[2];
+
+ if(Extnodes[p].hmax > hmax)
+ hmax = Extnodes[p].hmax;
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ diffsoftflag |= (Nodes[p].u.d.bitflags >> 5) & 1;
+
+ if(maxsofttype == 7)
+ {
+ maxsofttype = (Nodes[p].u.d.bitflags >> 2) & 7;
+ }
+ else
+ {
+ if(((Nodes[p].u.d.bitflags >> 2) & 7) != 7)
+ {
+ if(All.ForceSoftening[((Nodes[p].u.d.bitflags >> 2) & 7)] >
+ All.ForceSoftening[maxsofttype])
+ {
+ maxsofttype = ((Nodes[p].u.d.bitflags >> 2) & 7);
+ diffsoftflag = 1;
+ }
+ else
+ {
+ if(All.ForceSoftening[((Nodes[p].u.d.bitflags >> 2) & 7)] <
+ All.ForceSoftening[maxsofttype])
+ diffsoftflag = 1;
+ }
+ }
+ }
+#else
+ if(Nodes[p].maxsoft > maxsoft)
+ maxsoft = Nodes[p].maxsoft;
+#endif
+#endif
+ }
+ }
+ else /* a particle */
+ {
+ pa = &P[p];
+
+ mass += pa->Mass;
+#ifdef STELLAR_FLUX
+ starlum += pa->Mass*All.HeatingPeLMRatio[pa->Type];
+#endif
+
+ s[0] += pa->Mass * pa->Pos[0];
+ s[1] += pa->Mass * pa->Pos[1];
+ s[2] += pa->Mass * pa->Pos[2];
+ vs[0] += pa->Mass * pa->Vel[0];
+ vs[1] += pa->Mass * pa->Vel[1];
+ vs[2] += pa->Mass * pa->Vel[2];
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ if(maxsofttype == 7)
+ {
+ maxsofttype = pa->Type;
+ }
+ else
+ {
+ if(All.ForceSoftening[pa->Type] > All.ForceSoftening[maxsofttype])
+ {
+ maxsofttype = pa->Type;
+ diffsoftflag = 1;
+ }
+ else
+ {
+ if(All.ForceSoftening[pa->Type] < All.ForceSoftening[maxsofttype])
+ diffsoftflag = 1;
+ }
+ }
+#else
+ if(pa->Type == 0)
+ {
+ if(SphP[p].Hsml > maxsoft)
+ maxsoft = SphP[p].Hsml;
+ }
+ else
+ {
+ if(All.ForceSoftening[pa->Type] > maxsoft)
+ maxsoft = All.ForceSoftening[pa->Type];
+ }
+#endif
+#endif
+ if(pa->Type == 0)
+ if(SphP[p].Hsml > hmax)
+ hmax = SphP[p].Hsml;
+ }
+ }
+ }
+
+
+ if(mass)
+ {
+ s[0] /= mass;
+ s[1] /= mass;
+ s[2] /= mass;
+ vs[0] /= mass;
+ vs[1] /= mass;
+ vs[2] /= mass;
+ }
+ else
+ {
+ s[0] = Nodes[no].center[0];
+ s[1] = Nodes[no].center[1];
+ s[2] = Nodes[no].center[2];
+ }
+
+ Nodes[no].u.d.s[0] = s[0];
+ Nodes[no].u.d.s[1] = s[1];
+ Nodes[no].u.d.s[2] = s[2];
+ Nodes[no].u.d.mass = mass;
+#ifdef STELLAR_FLUX
+ Nodes[no].starlum = starlum;
+#endif
+
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ Nodes[no].u.d.bitflags = 4 * maxsofttype + 32 * diffsoftflag;
+#else
+ Nodes[no].u.d.bitflags = 0;
+ Nodes[no].maxsoft = maxsoft;
+#endif
+#else
+ Nodes[no].u.d.bitflags = 0;
+#endif
+
+
+ Extnodes[no].vs[0] = vs[0];
+ Extnodes[no].vs[1] = vs[1];
+ Extnodes[no].vs[2] = vs[2];
+ Extnodes[no].hmax = hmax;
+
+ Nodes[no].u.d.sibling = sib;
+ Nodes[no].u.d.father = father;
+ }
+ else /* single particle or pseudo particle */
+ {
+ if(last >= 0)
+ {
+ if(last >= All.MaxPart)
+ {
+ if(last >= All.MaxPart + MaxNodes) /* a pseudo-particle */
+ Nextnode[last - MaxNodes] = no;
+ else
+ Nodes[last].u.d.nextnode = no;
+ }
+ else
+ Nextnode[last] = no;
+ }
+
+ last = no;
+
+ if(no < All.MaxPart) /* only set it for single particles */
+ Father[no] = father;
+ }
+
+}
+
+
+
+/*! This function updates the multipole moments of the pseudo-particles
+ * that represent the mass distribution on different CPUs. For that
+ * purpose, it first exchanges the necessary data, and then updates the
+ * top-level tree accordingly. The detailed implementation of these two
+ * tasks is done in separate functions.
+ */
+void force_update_pseudoparticles(void)
+{
+ force_exchange_pseudodata();
+
+ force_treeupdate_pseudos();
+}
+
+
+
+/*! This function communicates the values of the multipole moments of the
+ * top-level tree-nodes of the domain grid. This data can then be used to
+ * update the pseudo-particles on each CPU accordingly.
+ */
+void force_exchange_pseudodata(void)
+{
+ int i, no;
+ MPI_Status status;
+ int level, sendTask, recvTask;
+
+ for(i = DomainMyStart; i <= DomainMyLast; i++)
+ {
+ no = DomainNodeIndex[i];
+
+ /* read out the multipole moments from the local base cells */
+ DomainMoment[i].s[0] = Nodes[no].u.d.s[0];
+ DomainMoment[i].s[1] = Nodes[no].u.d.s[1];
+ DomainMoment[i].s[2] = Nodes[no].u.d.s[2];
+ DomainMoment[i].vs[0] = Extnodes[no].vs[0];
+ DomainMoment[i].vs[1] = Extnodes[no].vs[1];
+ DomainMoment[i].vs[2] = Extnodes[no].vs[2];
+ DomainMoment[i].mass = Nodes[no].u.d.mass;
+#ifdef STELLAR_FLUX
+ DomainMoment[i].starlum = Nodes[no].starlum;
+#endif
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ DomainMoment[i].bitflags = Nodes[no].u.d.bitflags;
+#else
+ DomainMoment[i].maxsoft = Nodes[no].maxsoft;
+#endif
+#endif
+ }
+
+ /* share the pseudo-particle data accross CPUs */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ MPI_Sendrecv(&DomainMoment[DomainStartList[sendTask]],
+ (DomainEndList[sendTask] - DomainStartList[sendTask] + 1) * sizeof(struct DomainNODE),
+ MPI_BYTE, recvTask, TAG_DMOM,
+ &DomainMoment[DomainStartList[recvTask]],
+ (DomainEndList[recvTask] - DomainStartList[recvTask] + 1) * sizeof(struct DomainNODE),
+ MPI_BYTE, recvTask, TAG_DMOM, MPI_COMM_WORLD, &status);
+ }
+
+}
+
+/*! This function updates the top-level tree after the multipole moments of
+ * the pseudo-particles have been updated.
+ */
+void force_treeupdate_pseudos(void)
+{
+ int i, k, no;
+ FLOAT sold[3], vsold[3], snew[3], vsnew[3], massold, massnew, mm;
+#ifdef STELLAR_FLUX
+ FLOAT starlumold, starlumnew, starmm;
+#endif
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ int maxsofttype, diffsoftflag;
+#else
+ FLOAT maxsoft;
+#endif
+#endif
+
+ for(i = 0; i < NTopleaves; i++)
+ if(i < DomainMyStart || i > DomainMyLast)
+ {
+ no = DomainNodeIndex[i];
+
+ for(k = 0; k < 3; k++)
+ {
+ sold[k] = Nodes[no].u.d.s[k];
+ vsold[k] = Extnodes[no].vs[k];
+ }
+ massold = Nodes[no].u.d.mass;
+#ifdef STELLAR_FLUX
+ starlumold = Nodes[no].starlum;
+#endif
+
+
+ for(k = 0; k < 3; k++)
+ {
+ snew[k] = DomainMoment[i].s[k];
+ vsnew[k] = DomainMoment[i].vs[k];
+ }
+ massnew = DomainMoment[i].mass;
+#ifdef STELLAR_FLUX
+ starlumnew = DomainMoment[i].starlum;
+#endif
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ maxsofttype = (DomainMoment[i].bitflags >> 2) & 7;
+ diffsoftflag = (DomainMoment[i].bitflags >> 5) & 1;
+#else
+ maxsoft = DomainMoment[i].maxsoft;
+#endif
+#endif
+ do
+ {
+ mm = Nodes[no].u.d.mass + massnew - massold;
+#ifdef STELLAR_FLUX
+ starmm = Nodes[no].starlum + starlumnew - starlumold;
+#endif
+ for(k = 0; k < 3; k++)
+ {
+ if(mm > 0)
+ {
+ Nodes[no].u.d.s[k] =
+ (Nodes[no].u.d.mass * Nodes[no].u.d.s[k] + massnew * snew[k] - massold * sold[k]) / mm;
+ Extnodes[no].vs[k] =
+ (Nodes[no].u.d.mass * Extnodes[no].vs[k] + massnew * vsnew[k] -
+ massold * vsold[k]) / mm;
+ }
+ }
+ Nodes[no].u.d.mass = mm;
+#ifdef STELLAR_FLUX
+ Nodes[no].starlum = starmm;
+#endif
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ diffsoftflag |= (Nodes[no].u.d.bitflags >> 5) & 1;
+
+ if(maxsofttype == 7)
+ maxsofttype = (Nodes[no].u.d.bitflags >> 2) & 7;
+ else
+ {
+ if(((Nodes[no].u.d.bitflags >> 2) & 7) != 7)
+ {
+ if(All.ForceSoftening[((Nodes[no].u.d.bitflags >> 2) & 7)] >
+ All.ForceSoftening[maxsofttype])
+ {
+ maxsofttype = ((Nodes[no].u.d.bitflags >> 2) & 7);
+ diffsoftflag = 1;
+ }
+ else
+ {
+ if(All.ForceSoftening[((Nodes[no].u.d.bitflags >> 2) & 7)] <
+ All.ForceSoftening[maxsofttype])
+ diffsoftflag = 1;
+ }
+ }
+ }
+
+ Nodes[no].u.d.bitflags = (Nodes[no].u.d.bitflags & 3) + 4 * maxsofttype + 32 * diffsoftflag;
+#else
+ if(Nodes[no].maxsoft < maxsoft)
+ Nodes[no].maxsoft = maxsoft;
+ maxsoft = Nodes[no].maxsoft;
+#endif
+#endif
+ no = Nodes[no].u.d.father;
+
+ }
+ while(no >= 0);
+ }
+}
+
+
+
+/*! This function flags nodes in the top-level tree that are dependent on
+ * local particle data.
+ */
+void force_flag_localnodes(void)
+{
+ int no, i;
+
+ /* mark all top-level nodes */
+
+ for(i = 0; i < NTopleaves; i++)
+ {
+ no = DomainNodeIndex[i];
+
+ while(no >= 0)
+ {
+ if((Nodes[no].u.d.bitflags & 1))
+ break;
+
+ Nodes[no].u.d.bitflags |= 1;
+
+ no = Nodes[no].u.d.father;
+ }
+ }
+
+ /* mark top-level nodes that contain local particles */
+
+ for(i = DomainMyStart; i <= DomainMyLast; i++)
+ {
+ /*
+ if(DomainMoment[i].mass > 0)
+ */
+ {
+ no = DomainNodeIndex[i];
+
+ while(no >= 0)
+ {
+ if((Nodes[no].u.d.bitflags & 2))
+ break;
+
+ Nodes[no].u.d.bitflags |= 2;
+
+ no = Nodes[no].u.d.father;
+ }
+ }
+ }
+}
+
+
+
+/*! This function updates the side-length of tree nodes in case the tree is
+ * not reconstructed, but only drifted. The grouping of particles to tree
+ * nodes is not changed in this case, but some tree nodes may need to be
+ * enlarged because particles moved out of their original bounds.
+ */
+void force_update_len(void)
+{
+ int i, no;
+ MPI_Status status;
+ int level, sendTask, recvTask;
+
+ force_update_node_len_local();
+
+ /* first update the side-lengths of all local nodes */
+ for(i = DomainMyStart; i <= DomainMyLast; i++)
+ {
+ no = DomainNodeIndex[i];
+
+ DomainTreeNodeLen[i] = Nodes[no].len;
+ }
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ MPI_Sendrecv(&DomainTreeNodeLen[DomainStartList[sendTask]],
+ (DomainEndList[sendTask] - DomainStartList[sendTask] + 1) * sizeof(FLOAT),
+ MPI_BYTE, recvTask, TAG_NODELEN,
+ &DomainTreeNodeLen[DomainStartList[recvTask]],
+ (DomainEndList[recvTask] - DomainStartList[recvTask] + 1) * sizeof(FLOAT),
+ MPI_BYTE, recvTask, TAG_NODELEN, MPI_COMM_WORLD, &status);
+ }
+
+ /* Finally, we update the top-level tree. */
+ force_update_node_len_toptree();
+}
+
+
+/*! This function recursively enlarges nodes such that they always contain
+ * all their daughter nodes and daughter particles.
+ */
+void force_update_node_len_local(void)
+{
+ int i, p, k, no;
+ FLOAT dist, distmax;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ no = Father[i];
+
+ for(k = 0, distmax = 0; k < 3; k++)
+ {
+ dist = P[i].Pos[k] - Nodes[no].center[k];
+ if(dist < 0)
+ dist = -dist;
+ if(dist > distmax)
+ distmax = dist;
+ }
+
+ if(distmax + distmax > Nodes[no].len)
+ {
+ Nodes[no].len = distmax + distmax;
+ p = Nodes[no].u.d.father;
+
+ while(p >= 0)
+ {
+ distmax = Nodes[p].center[0] - Nodes[no].center[0];
+ if(distmax < 0)
+ distmax = -distmax;
+ distmax = distmax + distmax + Nodes[no].len;
+
+ if(0.999999 * distmax > Nodes[p].len)
+ {
+ Nodes[p].len = distmax;
+ no = p;
+ p = Nodes[p].u.d.father;
+ }
+ else
+ break;
+ }
+ }
+ }
+}
+
+
+/*! This function recursively enlarges nodes of the top-level tree such
+ * that they always contain all their daughter nodes.
+ */
+void force_update_node_len_toptree(void)
+{
+ int i, no, p;
+ FLOAT distmax;
+
+ for(i = 0; i < NTopleaves; i++)
+ if(i < DomainMyStart || i > DomainMyLast)
+ {
+ no = DomainNodeIndex[i];
+
+ if(Nodes[no].len < DomainTreeNodeLen[i])
+ Nodes[no].len = DomainTreeNodeLen[i];
+
+ p = Nodes[no].u.d.father;
+
+ while(p >= 0)
+ {
+ distmax = Nodes[p].center[0] - Nodes[no].center[0];
+ if(distmax < 0)
+ distmax = -distmax;
+ distmax = distmax + distmax + Nodes[no].len;
+
+ if(0.999999 * distmax > Nodes[p].len)
+ {
+ Nodes[p].len = distmax;
+ no = p;
+ p = Nodes[p].u.d.father;
+ }
+ else
+ break;
+ }
+ }
+}
+
+
+
+
+/*! This function updates the hmax-values in tree nodes that hold SPH
+ * particles. These values are needed to find all neighbors in the
+ * hydro-force computation. Since the Hsml-values are potentially changed
+ * in the SPH-denity computation, force_update_hmax() should be carried
+ * out just before the hydrodynamical SPH forces are computed, i.e. after
+ * density().
+ */
+void force_update_hmax(void)
+{
+ int i, no;
+ MPI_Status status;
+ int level, sendTask, recvTask;
+
+ force_update_node_hmax_local();
+
+ for(i = DomainMyStart; i <= DomainMyLast; i++)
+ {
+ no = DomainNodeIndex[i];
+
+ DomainHmax[i] = Extnodes[no].hmax;
+ }
+
+ /* share the hmax-data of the pseudo-particles accross CPUs */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ MPI_Sendrecv(&DomainHmax[DomainStartList[sendTask]],
+ (DomainEndList[sendTask] - DomainStartList[sendTask] + 1) * sizeof(FLOAT),
+ MPI_BYTE, recvTask, TAG_HMAX,
+ &DomainHmax[DomainStartList[recvTask]],
+ (DomainEndList[recvTask] - DomainStartList[recvTask] + 1) * sizeof(FLOAT),
+ MPI_BYTE, recvTask, TAG_HMAX, MPI_COMM_WORLD, &status);
+ }
+
+
+ force_update_node_hmax_toptree();
+}
+
+/*! This routine updates the hmax-values of local tree nodes.
+ */
+void force_update_node_hmax_local(void)
+{
+ int i, p, no;
+
+ for(i = 0; i < N_gas; i++)
+ {
+
+ no = Father[i];
+
+ if(SphP[i].Hsml > Extnodes[no].hmax)
+ {
+
+ Extnodes[no].hmax = SphP[i].Hsml;
+ p = Nodes[no].u.d.father;
+
+ while(p >= 0)
+ {
+ if(Extnodes[no].hmax > Extnodes[p].hmax)
+ {
+ Extnodes[p].hmax = Extnodes[no].hmax;
+ no = p;
+ p = Nodes[p].u.d.father;
+ }
+ else
+ break;
+ }
+ }
+
+ }
+}
+
+
+
+
+/*! This function recursively sets the hmax-values of the top-level tree.
+ */
+void force_update_node_hmax_toptree(void)
+{
+
+ int i, no, p;
+
+
+ for(i = 0; i < NTopleaves; i++)
+ if(i < DomainMyStart || i > DomainMyLast)
+ {
+ no = DomainNodeIndex[i];
+
+ if(Extnodes[no].hmax < DomainHmax[i])
+ Extnodes[no].hmax = DomainHmax[i];
+
+ p = Nodes[no].u.d.father;
+
+ while(p >= 0)
+ {
+ if(Extnodes[no].hmax > Extnodes[p].hmax)
+ {
+ Extnodes[p].hmax = Extnodes[no].hmax;
+ no = p;
+ p = Nodes[p].u.d.father;
+ }
+ else
+ break;
+ }
+ }
+}
+
+
+
+
+
+/*! This routine computes the gravitational force for a given local
+ * particle, or for a particle in the communication buffer. Depending on
+ * the value of TypeOfOpeningCriterion, either the geometrical BH
+ * cell-opening criterion, or the `relative' opening criterion is used.
+ */
+int force_treeevaluate(int target, int mode, double *ewaldcountsum)
+{
+ struct NODE *nop = 0;
+ int no, ninteractions, ptype;
+ double r2, dx, dy, dz, mass, r, fac, u, h, h_inv, h3_inv;
+ double acc_x, acc_y, acc_z, pos_x, pos_y, pos_z, aold;
+#if defined(UNEQUALSOFTENINGS) && !defined(ADAPTIVE_GRAVSOFT_FORGAS)
+ int maxsofttype;
+#endif
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ double soft = 0;
+#endif
+#ifdef PERIODIC
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+#endif
+
+#ifdef STELLAR_FLUX
+ double flux=0,starlum=0;
+ int type=0;
+ double hf;
+#endif
+
+ acc_x = 0;
+ acc_y = 0;
+ acc_z = 0;
+ ninteractions = 0;
+
+ if(mode == 0)
+ {
+ pos_x = P[target].Pos[0];
+ pos_y = P[target].Pos[1];
+ pos_z = P[target].Pos[2];
+ ptype = P[target].Type;
+ aold = All.ErrTolForceAcc * P[target].OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = SphP[target].Hsml;
+#endif
+#ifdef STELLAR_FLUX
+ type = P[target].Type;
+#endif
+ }
+ else
+ {
+ pos_x = GravDataGet[target].u.Pos[0];
+ pos_y = GravDataGet[target].u.Pos[1];
+ pos_z = GravDataGet[target].u.Pos[2];
+#ifdef UNEQUALSOFTENINGS
+ ptype = GravDataGet[target].Type;
+#else
+ ptype = P[0].Type;
+#endif
+ aold = All.ErrTolForceAcc * GravDataGet[target].w.OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = GravDataGet[target].Soft;
+#endif
+#ifdef STELLAR_FLUX
+ type = GravDataGet[target].Type;
+#endif
+ }
+
+
+
+#ifndef UNEQUALSOFTENINGS
+ h = All.ForceSoftening[ptype];
+ h_inv = 1.0 / h;
+ h3_inv = h_inv * h_inv * h_inv;
+#endif
+ no = All.MaxPart; /* root node */
+
+ while(no >= 0)
+ {
+
+ if(no < All.MaxPart) /* single particle */
+ {
+ /* the index of the node is the index of the particle */
+ /* observe the sign */
+
+ dx = P[no].Pos[0] - pos_x;
+ dy = P[no].Pos[1] - pos_y;
+ dz = P[no].Pos[2] - pos_z;
+
+ mass = P[no].Mass;
+#ifdef STELLAR_FLUX
+ starlum = P[no].Mass*All.HeatingPeLMRatio[P[no].Type];
+#endif
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+ nop = &Nodes[no];
+ dx = nop->u.d.s[0] - pos_x;
+ dy = nop->u.d.s[1] - pos_y;
+ dz = nop->u.d.s[2] - pos_z;
+
+ mass = nop->u.d.mass;
+
+#ifdef STELLAR_FLUX
+ starlum = nop->starlum;
+#endif
+
+ }
+#ifdef PERIODIC
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(no < All.MaxPart)
+ {
+
+#ifdef UNEQUALSOFTENINGS
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(P[no].Type == 0)
+ {
+ if(h < SphP[no].Hsml)
+ h = SphP[no].Hsml;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+ }
+#else
+ h = All.ForceSoftening[ptype];
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+#endif
+#endif
+ no = Nextnode[no];
+ }
+ else /* we have an internal node. Need to check opening criterion */
+ {
+ if(mode == 1)
+ {
+ if((nop->u.d.bitflags & 3) == 1) /* if it's a top-level node
+ * which does not contain
+ * local particles we can
+ * continue to do a short-cut */
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+ }
+
+
+ if(All.ErrTolTheta) /* check Barnes-Hut opening criterion */
+ {
+ if(nop->len * nop->len > r2 * All.ErrTolTheta * All.ErrTolTheta)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ else /* check relative opening criterion */
+ {
+ if(mass * nop->len * nop->len > r2 * r2 * aold)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ /* check in addition whether we lie inside the cell */
+
+ if(fabs(nop->center[0] - pos_x) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[1] - pos_y) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[2] - pos_z) < 0.60 * nop->len)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ h = All.ForceSoftening[ptype];
+ maxsofttype = (nop->u.d.bitflags >> 2) & 7;
+ if(maxsofttype == 7) /* may only occur for zero mass top-level nodes */
+ {
+ if(mass > 0)
+ endrun(986);
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[maxsofttype])
+ {
+ h = All.ForceSoftening[maxsofttype];
+ if(r2 < h * h)
+ {
+ if(((nop->u.d.bitflags >> 5) & 1)) /* bit-5 signals that there are particles of different softening in the node */
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+#else
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(h < nop->maxsoft)
+ {
+ h = nop->maxsoft;
+ if(r2 < h * h)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+#endif
+#endif
+
+ no = nop->u.d.sibling; /* ok, node can be used */
+
+ if(mode == 1)
+ {
+ if(((nop->u.d.bitflags) & 1)) /* Bit 0 signals that this node belongs to top-level tree */
+ continue;
+ }
+ }
+
+ r = sqrt(r2);
+
+ if(r >= h)
+ fac = mass / (r2 * r);
+ else
+ {
+#ifdef UNEQUALSOFTENINGS
+ h_inv = 1.0 / h;
+ h3_inv = h_inv * h_inv * h_inv;
+#endif
+ u = r * h_inv;
+ if(u < 0.5)
+ fac = mass * h3_inv * (10.666666666667 + u * u * (32.0 * u - 38.4));
+ else
+ fac =
+ mass * h3_inv * (21.333333333333 - 48.0 * u +
+ 38.4 * u * u - 10.666666666667 * u * u * u - 0.066666666667 / (u * u * u));
+ }
+
+ acc_x += dx * fac;
+ acc_y += dy * fac;
+ acc_z += dz * fac;
+
+#ifdef STELLAR_FLUX
+ if (type==0) /* gas particle */
+ {
+
+ hf = All.SofteningTable[type];
+
+ flux += starlum / (4*PI) / (r2 + hf*hf) ;
+
+ //if ((target==0)&&(starlum!=0))
+ // printf("flux flux=%g\n",flux);
+ //if ((target==0)&&(starlum!=0))
+ // printf("flux r=%g starlum=%g mass=%g h=%g no=%d\n",sqrt(r2),starlum,mass,hf,no);
+
+
+ //if(r >= hf)
+ // flux += starlum/All.HeatingPeLMRatio/ (4*PI) / r2 ;
+ //else
+ // flux += starlum/All.HeatingPeLMRatio/ (4*PI) / (r2 + hf*hf) ;
+ }
+#endif
+
+
+ ninteractions++;
+ }
+
+
+ /* store result at the proper place */
+ if(mode == 0)
+ {
+ P[target].GravAccel[0] = acc_x;
+ P[target].GravAccel[1] = acc_y;
+ P[target].GravAccel[2] = acc_z;
+#ifdef STELLAR_FLUX
+ if (P[target].Type==0) /* only for gas */
+ SphP[target].EnergyFlux = flux;
+#endif
+ P[target].GravCost = ninteractions;
+ }
+ else
+ {
+ GravDataResult[target].u.Acc[0] = acc_x;
+ GravDataResult[target].u.Acc[1] = acc_y;
+ GravDataResult[target].u.Acc[2] = acc_z;
+#ifdef STELLAR_FLUX
+ GravDataResult[target].EnergyFlux = flux;
+#endif
+ GravDataResult[target].w.Ninteractions = ninteractions;
+ }
+
+#ifdef PERIODIC
+ *ewaldcountsum += force_treeevaluate_ewald_correction(target, mode, pos_x, pos_y, pos_z, aold);
+#endif
+
+ return ninteractions;
+}
+
+
+
+
+
+
+#ifdef PMGRID
+/*! In the TreePM algorithm, the tree is walked only locally around the
+ * target coordinate. Tree nodes that fall outside a box of half
+ * side-length Rcut= RCUT*ASMTH*MeshSize can be discarded. The short-range
+ * potential is modified by a complementary error function, multiplied
+ * with the Newtonian form. The resulting short-range suppression compared
+ * to the Newtonian force is tabulated, because looking up from this table
+ * is faster than recomputing the corresponding factor, despite the
+ * memory-access panelty (which reduces cache performance) incurred by the
+ * table.
+ */
+int force_treeevaluate_shortrange(int target, int mode)
+{
+ struct NODE *nop = 0;
+ int no, ptype, ninteractions, tabindex;
+ double r2, dx, dy, dz, mass, r, fac, u, h, h_inv, h3_inv;
+ double acc_x, acc_y, acc_z, pos_x, pos_y, pos_z, aold;
+ double eff_dist;
+ double rcut, asmth, asmthfac, rcut2, dist;
+#if defined(UNEQUALSOFTENINGS) && !defined(ADAPTIVE_GRAVSOFT_FORGAS)
+ int maxsofttype;
+#endif
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ double soft = 0;
+#endif
+#ifdef PERIODIC
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+#endif
+
+
+ acc_x = 0;
+ acc_y = 0;
+ acc_z = 0;
+ ninteractions = 0;
+
+ if(mode == 0)
+ {
+ pos_x = P[target].Pos[0];
+ pos_y = P[target].Pos[1];
+ pos_z = P[target].Pos[2];
+ ptype = P[target].Type;
+ aold = All.ErrTolForceAcc * P[target].OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = SphP[target].Hsml;
+#endif
+ }
+ else
+ {
+ pos_x = GravDataGet[target].u.Pos[0];
+ pos_y = GravDataGet[target].u.Pos[1];
+ pos_z = GravDataGet[target].u.Pos[2];
+#ifdef UNEQUALSOFTENINGS
+ ptype = GravDataGet[target].Type;
+#else
+ ptype = P[0].Type;
+#endif
+ aold = All.ErrTolForceAcc * GravDataGet[target].w.OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = GravDataGet[target].Soft;
+#endif
+ }
+
+ rcut = All.Rcut[0];
+ asmth = All.Asmth[0];
+#ifdef PLACEHIGHRESREGION
+ if(((1 << ptype) & (PLACEHIGHRESREGION)))
+ {
+ rcut = All.Rcut[1];
+ asmth = All.Asmth[1];
+ }
+#endif
+ rcut2 = rcut * rcut;
+
+ asmthfac = 0.5 / asmth * (NTAB / 3.0);
+
+#ifndef UNEQUALSOFTENINGS
+ h = All.ForceSoftening[ptype];
+ h_inv = 1.0 / h;
+ h3_inv = h_inv * h_inv * h_inv;
+#endif
+ no = All.MaxPart; /* root node */
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart)
+ {
+ /* the index of the node is the index of the particle */
+ dx = P[no].Pos[0] - pos_x;
+ dy = P[no].Pos[1] - pos_y;
+ dz = P[no].Pos[2] - pos_z;
+#ifdef PERIODIC
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ mass = P[no].Mass;
+#ifdef UNEQUALSOFTENINGS
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(P[no].Type == 0)
+ {
+ if(h < SphP[no].Hsml)
+ h = SphP[no].Hsml;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+ }
+#else
+ h = All.ForceSoftening[ptype];
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+#endif
+#endif
+ no = Nextnode[no];
+ }
+ else /* we have an internal node */
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ nop = &Nodes[no];
+
+ if(mode == 1)
+ {
+ if((nop->u.d.bitflags & 3) == 1) /* if it's a top-level node
+ * which does not contain
+ * local particles we can
+ * continue at this point
+ */
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+ }
+
+ mass = nop->u.d.mass;
+
+ dx = nop->u.d.s[0] - pos_x;
+ dy = nop->u.d.s[1] - pos_y;
+ dz = nop->u.d.s[2] - pos_z;
+#ifdef PERIODIC
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(r2 > rcut2)
+ {
+ /* check whether we can stop walking along this branch */
+ eff_dist = rcut + 0.5 * nop->len;
+#ifdef PERIODIC
+ dist = NEAREST(nop->center[0] - pos_x);
+#else
+ dist = nop->center[0] - pos_x;
+#endif
+ if(dist < -eff_dist || dist > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+#ifdef PERIODIC
+ dist = NEAREST(nop->center[1] - pos_y);
+#else
+ dist = nop->center[1] - pos_y;
+#endif
+ if(dist < -eff_dist || dist > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+#ifdef PERIODIC
+ dist = NEAREST(nop->center[2] - pos_z);
+#else
+ dist = nop->center[2] - pos_z;
+#endif
+ if(dist < -eff_dist || dist > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+ }
+
+
+ if(All.ErrTolTheta) /* check Barnes-Hut opening criterion */
+ {
+ if(nop->len * nop->len > r2 * All.ErrTolTheta * All.ErrTolTheta)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ else /* check relative opening criterion */
+ {
+ if(mass * nop->len * nop->len > r2 * r2 * aold)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ /* check in addition whether we lie inside the cell */
+
+ if(fabs(nop->center[0] - pos_x) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[1] - pos_y) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[2] - pos_z) < 0.60 * nop->len)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ h = All.ForceSoftening[ptype];
+ maxsofttype = (nop->u.d.bitflags >> 2) & 7;
+ if(maxsofttype == 7) /* may only occur for zero mass top-level nodes */
+ {
+ if(mass > 0)
+ endrun(987);
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[maxsofttype])
+ {
+ h = All.ForceSoftening[maxsofttype];
+ if(r2 < h * h)
+ {
+ if(((nop->u.d.bitflags >> 5) & 1)) /* bit-5 signals that there are particles of different softening in the node */
+ {
+ no = nop->u.d.nextnode;
+
+ continue;
+ }
+ }
+ }
+ }
+#else
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(h < nop->maxsoft)
+ {
+ h = nop->maxsoft;
+ if(r2 < h * h)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+#endif
+#endif
+ no = nop->u.d.sibling; /* ok, node can be used */
+
+ if(mode == 1)
+ {
+ if(((nop->u.d.bitflags) & 1)) /* Bit 0 signals that this node belongs to top-level tree */
+ continue;
+ }
+ }
+
+ r = sqrt(r2);
+
+ if(r >= h)
+ fac = mass / (r2 * r);
+ else
+ {
+#ifdef UNEQUALSOFTENINGS
+ h_inv = 1.0 / h;
+ h3_inv = h_inv * h_inv * h_inv;
+#endif
+ u = r * h_inv;
+ if(u < 0.5)
+ fac = mass * h3_inv * (10.666666666667 + u * u * (32.0 * u - 38.4));
+ else
+ fac =
+ mass * h3_inv * (21.333333333333 - 48.0 * u +
+ 38.4 * u * u - 10.666666666667 * u * u * u - 0.066666666667 / (u * u * u));
+ }
+
+ tabindex = (int) (asmthfac * r);
+
+ if(tabindex < NTAB)
+ {
+ fac *= shortrange_table[tabindex];
+
+ acc_x += dx * fac;
+ acc_y += dy * fac;
+ acc_z += dz * fac;
+
+ ninteractions++;
+ }
+ }
+
+
+ /* store result at the proper place */
+ if(mode == 0)
+ {
+ P[target].GravAccel[0] = acc_x;
+ P[target].GravAccel[1] = acc_y;
+ P[target].GravAccel[2] = acc_z;
+ P[target].GravCost = ninteractions;
+ }
+ else
+ {
+ GravDataResult[target].u.Acc[0] = acc_x;
+ GravDataResult[target].u.Acc[1] = acc_y;
+ GravDataResult[target].u.Acc[2] = acc_z;
+ GravDataResult[target].w.Ninteractions = ninteractions;
+ }
+
+ return ninteractions;
+}
+
+#endif
+
+
+
+#ifdef PERIODIC
+/*! This function computes the Ewald correction, and is needed if periodic
+ * boundary conditions together with a pure tree algorithm are used. Note
+ * that the ordinary tree walk does not carry out this correction directly
+ * as it was done in Gadget-1.1. Instead, the tree is walked a second
+ * time. This is actually faster because the "Ewald-Treewalk" can use a
+ * different opening criterion than the normal tree walk. In particular,
+ * the Ewald correction is negligible for particles that are very close,
+ * but it is large for particles that are far away (this is quite
+ * different for the normal direct force). So we can here use a different
+ * opening criterion. Sufficient accuracy is usually obtained if the node
+ * length has dropped to a certain fraction ~< 0.25 of the
+ * BoxLength. However, we may only short-cut the interaction list of the
+ * normal full Ewald tree walk if we are sure that the whole node and all
+ * daughter nodes "lie on the same side" of the periodic boundary,
+ * i.e. that the real tree walk would not find a daughter node or particle
+ * that was mapped to a different nearest neighbour position when the tree
+ * walk would be further refined.
+ */
+int force_treeevaluate_ewald_correction(int target, int mode, double pos_x, double pos_y, double pos_z,
+ double aold)
+{
+ struct NODE *nop = 0;
+ int no, cost;
+ double dx, dy, dz, mass, r2;
+ int signx, signy, signz;
+ int i, j, k, openflag;
+ double u, v, w;
+ double f1, f2, f3, f4, f5, f6, f7, f8;
+ double acc_x, acc_y, acc_z;
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+
+ acc_x = 0;
+ acc_y = 0;
+ acc_z = 0;
+ cost = 0;
+
+ no = All.MaxPart;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ /* the index of the node is the index of the particle */
+ /* observe the sign */
+
+ dx = P[no].Pos[0] - pos_x;
+ dy = P[no].Pos[1] - pos_y;
+ dz = P[no].Pos[2] - pos_z;
+ mass = P[no].Mass;
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ nop = &Nodes[no];
+ dx = nop->u.d.s[0] - pos_x;
+ dy = nop->u.d.s[1] - pos_y;
+ dz = nop->u.d.s[2] - pos_z;
+ mass = nop->u.d.mass;
+ }
+
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+
+ if(no < All.MaxPart)
+ no = Nextnode[no];
+ else /* we have an internal node. Need to check opening criterion */
+ {
+ openflag = 0;
+
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(All.ErrTolTheta) /* check Barnes-Hut opening criterion */
+ {
+ if(nop->len * nop->len > r2 * All.ErrTolTheta * All.ErrTolTheta)
+ {
+ openflag = 1;
+ }
+ }
+ else /* check relative opening criterion */
+ {
+ if(mass * nop->len * nop->len > r2 * r2 * aold)
+ {
+ openflag = 1;
+ }
+ else
+ {
+ if(fabs(nop->center[0] - pos_x) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[1] - pos_y) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[2] - pos_z) < 0.60 * nop->len)
+ {
+ openflag = 1;
+ }
+ }
+ }
+ }
+ }
+
+ if(openflag)
+ {
+ /* now we check if we can avoid opening the cell */
+
+ u = nop->center[0] - pos_x;
+ if(u > boxhalf)
+ u -= boxsize;
+ if(u < -boxhalf)
+ u += boxsize;
+
+ if(fabs(u) > 0.5 * (boxsize - nop->len))
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ u = nop->center[1] - pos_y;
+ if(u > boxhalf)
+ u -= boxsize;
+ if(u < -boxhalf)
+ u += boxsize;
+
+ if(fabs(u) > 0.5 * (boxsize - nop->len))
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ u = nop->center[2] - pos_z;
+ if(u > boxhalf)
+ u -= boxsize;
+ if(u < -boxhalf)
+ u += boxsize;
+
+ if(fabs(u) > 0.5 * (boxsize - nop->len))
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ /* if the cell is too large, we need to refine
+ * it further
+ */
+ if(nop->len > 0.20 * boxsize)
+ {
+ /* cell is too large */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+
+ no = nop->u.d.sibling; /* ok, node can be used */
+
+ if(mode == 1)
+ {
+ if((nop->u.d.bitflags & 1)) /* Bit 0 signals that this node belongs to top-level tree */
+ continue;
+ }
+ }
+
+ /* compute the Ewald correction force */
+
+ if(dx < 0)
+ {
+ dx = -dx;
+ signx = +1;
+ }
+ else
+ signx = -1;
+
+ if(dy < 0)
+ {
+ dy = -dy;
+ signy = +1;
+ }
+ else
+ signy = -1;
+
+ if(dz < 0)
+ {
+ dz = -dz;
+ signz = +1;
+ }
+ else
+ signz = -1;
+
+ u = dx * fac_intp;
+ i = (int) u;
+ if(i >= EN)
+ i = EN - 1;
+ u -= i;
+ v = dy * fac_intp;
+ j = (int) v;
+ if(j >= EN)
+ j = EN - 1;
+ v -= j;
+ w = dz * fac_intp;
+ k = (int) w;
+ if(k >= EN)
+ k = EN - 1;
+ w -= k;
+
+ /* compute factors for trilinear interpolation */
+
+ f1 = (1 - u) * (1 - v) * (1 - w);
+ f2 = (1 - u) * (1 - v) * (w);
+ f3 = (1 - u) * (v) * (1 - w);
+ f4 = (1 - u) * (v) * (w);
+ f5 = (u) * (1 - v) * (1 - w);
+ f6 = (u) * (1 - v) * (w);
+ f7 = (u) * (v) * (1 - w);
+ f8 = (u) * (v) * (w);
+
+ acc_x += mass * signx * (fcorrx[i][j][k] * f1 +
+ fcorrx[i][j][k + 1] * f2 +
+ fcorrx[i][j + 1][k] * f3 +
+ fcorrx[i][j + 1][k + 1] * f4 +
+ fcorrx[i + 1][j][k] * f5 +
+ fcorrx[i + 1][j][k + 1] * f6 +
+ fcorrx[i + 1][j + 1][k] * f7 + fcorrx[i + 1][j + 1][k + 1] * f8);
+
+ acc_y += mass * signy * (fcorry[i][j][k] * f1 +
+ fcorry[i][j][k + 1] * f2 +
+ fcorry[i][j + 1][k] * f3 +
+ fcorry[i][j + 1][k + 1] * f4 +
+ fcorry[i + 1][j][k] * f5 +
+ fcorry[i + 1][j][k + 1] * f6 +
+ fcorry[i + 1][j + 1][k] * f7 + fcorry[i + 1][j + 1][k + 1] * f8);
+
+ acc_z += mass * signz * (fcorrz[i][j][k] * f1 +
+ fcorrz[i][j][k + 1] * f2 +
+ fcorrz[i][j + 1][k] * f3 +
+ fcorrz[i][j + 1][k + 1] * f4 +
+ fcorrz[i + 1][j][k] * f5 +
+ fcorrz[i + 1][j][k + 1] * f6 +
+ fcorrz[i + 1][j + 1][k] * f7 + fcorrz[i + 1][j + 1][k + 1] * f8);
+ cost++;
+ }
+
+
+ /* add the result at the proper place */
+
+ if(mode == 0)
+ {
+ P[target].GravAccel[0] += acc_x;
+ P[target].GravAccel[1] += acc_y;
+ P[target].GravAccel[2] += acc_z;
+ P[target].GravCost += cost;
+ }
+ else
+ {
+ GravDataResult[target].u.Acc[0] += acc_x;
+ GravDataResult[target].u.Acc[1] += acc_y;
+ GravDataResult[target].u.Acc[2] += acc_z;
+ GravDataResult[target].w.Ninteractions += cost;
+ }
+
+ return cost;
+}
+
+#endif
+
+
+
+
+
+
+/*! This routine computes the gravitational potential by walking the
+ * tree. The same opening criteria is used as for the gravitational force
+ * walk.
+ */
+void force_treeevaluate_potential(int target, int mode)
+{
+ struct NODE *nop = 0;
+ int no, ptype;
+ double r2, dx, dy, dz, mass, r, u, h, h_inv, wp;
+ double pot, pos_x, pos_y, pos_z, aold;
+#if defined(UNEQUALSOFTENINGS) && !defined(ADAPTIVE_GRAVSOFT_FORGAS)
+ int maxsofttype;
+#endif
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ double soft = 0;
+#endif
+#ifdef PERIODIC
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+#endif
+
+ pot = 0;
+
+ if(mode == 0)
+ {
+ pos_x = P[target].Pos[0];
+ pos_y = P[target].Pos[1];
+ pos_z = P[target].Pos[2];
+ ptype = P[target].Type;
+ aold = All.ErrTolForceAcc * P[target].OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = SphP[target].Hsml;
+#endif
+ }
+ else
+ {
+ pos_x = GravDataGet[target].u.Pos[0];
+ pos_y = GravDataGet[target].u.Pos[1];
+ pos_z = GravDataGet[target].u.Pos[2];
+#ifdef UNEQUALSOFTENINGS
+ ptype = GravDataGet[target].Type;
+#else
+ ptype = P[0].Type;
+#endif
+ aold = All.ErrTolForceAcc * GravDataGet[target].w.OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = GravDataGet[target].Soft;
+#endif
+ }
+
+
+#ifndef UNEQUALSOFTENINGS
+ h = All.ForceSoftening[ptype];
+ h_inv = 1.0 / h;
+#endif
+ no = All.MaxPart;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ /* the index of the node is the index of the particle */
+ /* observe the sign */
+
+ dx = P[no].Pos[0] - pos_x;
+ dy = P[no].Pos[1] - pos_y;
+ dz = P[no].Pos[2] - pos_z;
+ mass = P[no].Mass;
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ nop = &Nodes[no];
+ dx = nop->u.d.s[0] - pos_x;
+ dy = nop->u.d.s[1] - pos_y;
+ dz = nop->u.d.s[2] - pos_z;
+ mass = nop->u.d.mass;
+ }
+
+#ifdef PERIODIC
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(no < All.MaxPart)
+ {
+#ifdef UNEQUALSOFTENINGS
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(P[no].Type == 0)
+ {
+ if(h < SphP[no].Hsml)
+ h = SphP[no].Hsml;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+ }
+#else
+ h = All.ForceSoftening[ptype];
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+#endif
+#endif
+ no = Nextnode[no];
+ }
+ else /* we have an internal node. Need to check opening criterion */
+ {
+ if(mode == 1)
+ {
+ if((nop->u.d.bitflags & 3) == 1) /* if it's a top-level node
+ * which does not contain
+ * local particles we can make
+ * a short-cut
+ */
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+ }
+
+ if(All.ErrTolTheta) /* check Barnes-Hut opening criterion */
+ {
+ if(nop->len * nop->len > r2 * All.ErrTolTheta * All.ErrTolTheta)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ else /* check relative opening criterion */
+ {
+ if(mass * nop->len * nop->len > r2 * r2 * aold)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ if(fabs(nop->center[0] - pos_x) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[1] - pos_y) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[2] - pos_z) < 0.60 * nop->len)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ h = All.ForceSoftening[ptype];
+ maxsofttype = (nop->u.d.bitflags >> 2) & 7;
+ if(maxsofttype == 7) /* may only occur for zero mass top-level nodes */
+ {
+ if(mass > 0)
+ endrun(988);
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[maxsofttype])
+ {
+ h = All.ForceSoftening[maxsofttype];
+ if(r2 < h * h)
+ {
+ if(((nop->u.d.bitflags >> 5) & 1)) /* bit-5 signals that there are particles of different softening in the node */
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+#else
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(h < nop->maxsoft)
+ {
+ h = nop->maxsoft;
+ if(r2 < h * h)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+#endif
+#endif
+
+ no = nop->u.d.sibling; /* node can be used */
+
+ if(mode == 1)
+ {
+ if(((nop->u.d.bitflags) & 1)) /* Bit 0 signals that this node belongs to top-level tree */
+ continue;
+ }
+ }
+
+ r = sqrt(r2);
+
+ if(r >= h)
+ pot -= mass / r;
+ else
+ {
+#ifdef UNEQUALSOFTENINGS
+ h_inv = 1.0 / h;
+#endif
+ u = r * h_inv;
+
+ if(u < 0.5)
+ wp = -2.8 + u * u * (5.333333333333 + u * u * (6.4 * u - 9.6));
+ else
+ wp =
+ -3.2 + 0.066666666667 / u + u * u * (10.666666666667 +
+ u * (-16.0 + u * (9.6 - 2.133333333333 * u)));
+
+ pot += mass * h_inv * wp;
+ }
+#ifdef PERIODIC
+ pot += mass * ewald_pot_corr(dx, dy, dz);
+#endif
+ }
+
+ /* store result at the proper place */
+
+ if(mode == 0)
+ P[target].Potential = pot;
+ else
+ GravDataResult[target].u.Potential = pot;
+}
+
+
+
+
+#ifdef PMGRID
+/*! This function computes the short-range potential when the TreePM
+ * algorithm is used. This potential is the Newtonian potential, modified
+ * by a complementary error function.
+ */
+void force_treeevaluate_potential_shortrange(int target, int mode)
+{
+ struct NODE *nop = 0;
+ int no, ptype, tabindex;
+ double r2, dx, dy, dz, mass, r, u, h, h_inv, wp;
+ double pot, pos_x, pos_y, pos_z, aold;
+ double eff_dist, fac, rcut, asmth, asmthfac;
+ double dxx, dyy, dzz;
+#if defined(UNEQUALSOFTENINGS) && !defined(ADAPTIVE_GRAVSOFT_FORGAS)
+ int maxsofttype;
+#endif
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ double soft = 0;
+#endif
+
+#ifdef PERIODIC
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+#endif
+
+ pot = 0;
+
+ if(mode == 0)
+ {
+ pos_x = P[target].Pos[0];
+ pos_y = P[target].Pos[1];
+ pos_z = P[target].Pos[2];
+ ptype = P[target].Type;
+ aold = All.ErrTolForceAcc * P[target].OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = SphP[target].Hsml;
+#endif
+ }
+ else
+ {
+ pos_x = GravDataGet[target].u.Pos[0];
+ pos_y = GravDataGet[target].u.Pos[1];
+ pos_z = GravDataGet[target].u.Pos[2];
+#ifdef UNEQUALSOFTENINGS
+ ptype = GravDataGet[target].Type;
+#else
+ ptype = P[0].Type;
+#endif
+ aold = All.ErrTolForceAcc * GravDataGet[target].w.OldAcc;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ soft = GravDataGet[target].Soft;
+#endif
+ }
+
+
+ rcut = All.Rcut[0];
+ asmth = All.Asmth[0];
+#ifdef PLACEHIGHRESREGION
+ if(((1 << ptype) & (PLACEHIGHRESREGION)))
+ {
+ rcut = All.Rcut[1];
+ asmth = All.Asmth[1];
+ }
+#endif
+ asmthfac = 0.5 / asmth * (NTAB / 3.0);
+
+#ifndef UNEQUALSOFTENINGS
+ h = All.ForceSoftening[ptype];
+ h_inv = 1.0 / h;
+#endif
+
+ no = All.MaxPart;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ /* the index of the node is the index of the particle */
+ /* observe the sign */
+
+ dx = P[no].Pos[0] - pos_x;
+ dy = P[no].Pos[1] - pos_y;
+ dz = P[no].Pos[2] - pos_z;
+ mass = P[no].Mass;
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ nop = &Nodes[no];
+ dx = nop->u.d.s[0] - pos_x;
+ dy = nop->u.d.s[1] - pos_y;
+ dz = nop->u.d.s[2] - pos_z;
+ mass = nop->u.d.mass;
+ }
+
+#ifdef PERIODIC
+ dx = NEAREST(dx);
+ dy = NEAREST(dy);
+ dz = NEAREST(dz);
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(no < All.MaxPart)
+ {
+#ifdef UNEQUALSOFTENINGS
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(P[no].Type == 0)
+ {
+ if(h < SphP[no].Hsml)
+ h = SphP[no].Hsml;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+ }
+#else
+ h = All.ForceSoftening[ptype];
+ if(h < All.ForceSoftening[P[no].Type])
+ h = All.ForceSoftening[P[no].Type];
+#endif
+#endif
+ no = Nextnode[no];
+ }
+ else /* we have an internal node. Need to check opening criterion */
+ {
+ /* check whether we can stop walking along this branch */
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ if(mode == 0)
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ }
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ if(mode == 1)
+ {
+ if((nop->u.d.bitflags & 3) == 1) /* if it's a top-level node which does not contain local particles */
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+ }
+
+ eff_dist = rcut + 0.5 * nop->len;
+
+ dxx = nop->center[0] - pos_x; /* observe the sign ! */
+ dyy = nop->center[1] - pos_y; /* this vector is -y in my thesis notation */
+ dzz = nop->center[2] - pos_z;
+#ifdef PERIODIC
+ dxx = NEAREST(dxx);
+ dyy = NEAREST(dyy);
+ dzz = NEAREST(dzz);
+#endif
+ if(dxx < -eff_dist || dxx > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+
+ if(dyy < -eff_dist || dyy > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+
+ if(dzz < -eff_dist || dzz > eff_dist)
+ {
+ no = nop->u.d.sibling;
+ continue;
+ }
+
+ if(All.ErrTolTheta) /* check Barnes-Hut opening criterion */
+ {
+ if(nop->len * nop->len > r2 * All.ErrTolTheta * All.ErrTolTheta)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ else /* check relative opening criterion */
+ {
+ if(mass * nop->len * nop->len > r2 * r2 * aold)
+ {
+ /* open cell */
+ no = nop->u.d.nextnode;
+ continue;
+ }
+
+ if(fabs(nop->center[0] - pos_x) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[1] - pos_y) < 0.60 * nop->len)
+ {
+ if(fabs(nop->center[2] - pos_z) < 0.60 * nop->len)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+
+#ifdef UNEQUALSOFTENINGS
+#ifndef ADAPTIVE_GRAVSOFT_FORGAS
+ h = All.ForceSoftening[ptype];
+ maxsofttype = (nop->u.d.bitflags >> 2) & 7;
+ if(maxsofttype == 7) /* may only occur for zero mass top-level nodes */
+ {
+ if(mass > 0)
+ endrun(989);
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ else
+ {
+ if(h < All.ForceSoftening[maxsofttype])
+ {
+ h = All.ForceSoftening[maxsofttype];
+ if(r2 < h * h)
+ {
+ /* bit-5 signals that there are particles of
+ * different softening in the node
+ */
+ if(((nop->u.d.bitflags >> 5) & 1))
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+ }
+ }
+#else
+ if(ptype == 0)
+ h = soft;
+ else
+ h = All.ForceSoftening[ptype];
+
+ if(h < nop->maxsoft)
+ {
+ h = nop->maxsoft;
+ if(r2 < h * h)
+ {
+ no = nop->u.d.nextnode;
+ continue;
+ }
+ }
+#endif
+#endif
+ no = nop->u.d.sibling; /* node can be used */
+
+ if(mode == 1)
+ {
+ if(((nop->u.d.bitflags) & 1)) /* Bit 0 signals that this node belongs to top-level tree */
+ continue;
+ }
+ }
+
+ r = sqrt(r2);
+
+ tabindex = (int) (r * asmthfac);
+
+ if(tabindex < NTAB)
+ {
+ fac = shortrange_table_potential[tabindex];
+
+ if(r >= h)
+ pot -= fac * mass / r;
+ else
+ {
+#ifdef UNEQUALSOFTENINGS
+ h_inv = 1.0 / h;
+#endif
+ u = r * h_inv;
+
+ if(u < 0.5)
+ wp = -2.8 + u * u * (5.333333333333 + u * u * (6.4 * u - 9.6));
+ else
+ wp =
+ -3.2 + 0.066666666667 / u + u * u * (10.666666666667 +
+ u * (-16.0 + u * (9.6 - 2.133333333333 * u)));
+ pot += fac * mass * h_inv * wp;
+ }
+ }
+ }
+
+
+ /* store result at the proper place */
+ if(mode == 0)
+ P[target].Potential = pot;
+ else
+ GravDataResult[target].u.Potential = pot;
+}
+
+#endif
+
+
+
+/*! This function allocates the memory used for storage of the tree and of
+ * auxiliary arrays needed for tree-walk and link-lists. Usually,
+ * maxnodes approximately equal to 0.7*maxpart is sufficient to store the
+ * tree for up to maxpart particles.
+ */
+void force_treeallocate(int maxnodes, int maxpart)
+{
+ int i;
+ size_t bytes;
+ double allbytes = 0;
+ double u;
+
+ MaxNodes = maxnodes;
+
+ if(!(Nodes_base = malloc(bytes = (MaxNodes + 1) * sizeof(struct NODE))))
+ {
+ printf("failed to allocate memory for %d tree-nodes (%g MB).\n", MaxNodes, bytes / (1024.0 * 1024.0));
+ endrun(3);
+ }
+ allbytes += bytes;
+
+ if(!(Extnodes_base = malloc(bytes = (MaxNodes + 1) * sizeof(struct extNODE))))
+ {
+ printf("failed to allocate memory for %d tree-extnodes (%g MB).\n", MaxNodes,
+ bytes / (1024.0 * 1024.0));
+ endrun(3);
+ }
+ allbytes += bytes;
+
+ Nodes = Nodes_base - All.MaxPart;
+ Extnodes = Extnodes_base - All.MaxPart;
+
+ if(!(Nextnode = malloc(bytes = (maxpart + MAXTOPNODES) * sizeof(int))))
+ {
+ printf("Failed to allocate %d spaces for 'Nextnode' array (%g MB)\n", maxpart + MAXTOPNODES,
+ bytes / (1024.0 * 1024.0));
+ exit(0);
+ }
+ allbytes += bytes;
+
+ if(!(Father = malloc(bytes = (maxpart) * sizeof(int))))
+ {
+ printf("Failed to allocate %d spaces for 'Father' array (%g MB)\n", maxpart, bytes / (1024.0 * 1024.0));
+ exit(0);
+ }
+ allbytes += bytes;
+
+ if(first_flag == 0)
+ {
+ first_flag = 1;
+
+ if(ThisTask == 0)
+ printf("\nAllocated %g MByte for BH-tree. %d\n\n", allbytes / (1024.0 * 1024.0),
+ sizeof(struct NODE) + sizeof(struct extNODE));
+
+ tabfac = NTAB / 3.0;
+
+ for(i = 0; i < NTAB; i++)
+ {
+ u = 3.0 / NTAB * (i + 0.5);
+ shortrange_table[i] = erfc(u) + 2.0 * u / sqrt(M_PI) * exp(-u * u);
+ shortrange_table_potential[i] = erfc(u);
+ }
+ }
+}
+
+
+/*! This function frees the memory allocated for the tree, i.e. it frees
+ * the space allocated by the function force_treeallocate().
+ */
+void force_treefree(void)
+{
+ free(Father);
+ free(Nextnode);
+ free(Extnodes_base);
+ free(Nodes_base);
+}
+
+
+
+
+/*! This function does the force computation with direct summation for the
+ * specified particle in the communication buffer. This can be useful for
+ * debugging purposes, in particular for explicit checks of the force
+ * accuracy.
+ */
+#ifdef FORCETEST
+int force_treeevaluate_direct(int target, int mode)
+{
+ double epsilon;
+ double h, h_inv, dx, dy, dz, r, r2, u, r_inv, fac;
+ int i, ptype;
+ double pos_x, pos_y, pos_z;
+ double acc_x, acc_y, acc_z;
+
+#ifdef PERIODIC
+ double fcorr[3];
+#endif
+#ifdef PERIODIC
+ double boxsize, boxhalf;
+
+ boxsize = All.BoxSize;
+ boxhalf = 0.5 * All.BoxSize;
+#endif
+
+ acc_x = 0;
+ acc_y = 0;
+ acc_z = 0;
+
+ if(mode == 0)
+ {
+ pos_x = P[target].Pos[0];
+ pos_y = P[target].Pos[1];
+ pos_z = P[target].Pos[2];
+ ptype = P[target].Type;
+ }
+ else
+ {
+ pos_x = GravDataGet[target].u.Pos[0];
+ pos_y = GravDataGet[target].u.Pos[1];
+ pos_z = GravDataGet[target].u.Pos[2];
+#ifdef UNEQUALSOFTENINGS
+ ptype = GravDataGet[target].Type;
+#else
+ ptype = P[0].Type;
+#endif
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ epsilon = dmax(All.ForceSoftening[P[i].Type], All.ForceSoftening[ptype]);
+
+ h = epsilon;
+ h_inv = 1 / h;
+
+ dx = P[i].Pos[0] - pos_x;
+ dy = P[i].Pos[1] - pos_y;
+ dz = P[i].Pos[2] - pos_z;
+
+#ifdef PERIODIC
+ while(dx > boxhalf)
+ dx -= boxsize;
+ while(dy > boxhalf)
+ dy -= boxsize;
+ while(dz > boxhalf)
+ dz -= boxsize;
+ while(dx < -boxhalf)
+ dx += boxsize;
+ while(dy < -boxhalf)
+ dy += boxsize;
+ while(dz < -boxhalf)
+ dz += boxsize;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ r = sqrt(r2);
+
+ u = r * h_inv;
+
+ if(u >= 1)
+ {
+ r_inv = 1 / r;
+
+ fac = P[i].Mass * r_inv * r_inv * r_inv;
+ }
+ else
+ {
+ if(u < 0.5)
+ fac = P[i].Mass * h_inv * h_inv * h_inv * (10.666666666667 + u * u * (32.0 * u - 38.4));
+ else
+ fac =
+ P[i].Mass * h_inv * h_inv * h_inv * (21.333333333333 -
+ 48.0 * u + 38.4 * u * u -
+ 10.666666666667 * u * u *
+ u - 0.066666666667 / (u * u * u));
+ }
+
+ acc_x += dx * fac;
+ acc_y += dy * fac;
+ acc_z += dz * fac;
+
+#ifdef PERIODIC
+ if(u > 1.0e-5)
+ {
+ ewald_corr(dx, dy, dz, fcorr);
+
+ acc_x += P[i].Mass * fcorr[0];
+ acc_y += P[i].Mass * fcorr[1];
+ acc_z += P[i].Mass * fcorr[2];
+ }
+#endif
+ }
+
+
+ if(mode == 0)
+ {
+ P[target].GravAccelDirect[0] = acc_x;
+ P[target].GravAccelDirect[1] = acc_y;
+ P[target].GravAccelDirect[2] = acc_z;
+ }
+ else
+ {
+ GravDataResult[target].u.Acc[0] = acc_x;
+ GravDataResult[target].u.Acc[1] = acc_y;
+ GravDataResult[target].u.Acc[2] = acc_z;
+ }
+
+
+ return NumPart;
+}
+#endif
+
+
+/*! This function dumps some of the basic particle data to a file. In case
+ * the tree construction fails, it is called just before the run
+ * terminates with an error message. Examination of the generated file may
+ * then give clues to what caused the problem.
+ */
+void dump_particles(void)
+{
+ FILE *fd;
+ char buffer[200];
+ int i;
+
+ sprintf(buffer, "particles%d.dat", ThisTask);
+ fd = fopen(buffer, "w");
+ my_fwrite(&NumPart, 1, sizeof(int), fd);
+
+ for(i = 0; i < NumPart; i++)
+ my_fwrite(&P[i].Pos[0], 3, sizeof(FLOAT), fd);
+
+ for(i = 0; i < NumPart; i++)
+ my_fwrite(&P[i].Vel[0], 3, sizeof(FLOAT), fd);
+
+ for(i = 0; i < NumPart; i++)
+ my_fwrite(&P[i].ID, 1, sizeof(int), fd);
+
+ fclose(fd);
+}
+
+
+
+#ifdef PERIODIC
+
+/*! This function initializes tables with the correction force and the
+ * correction potential due to the periodic images of a point mass located
+ * at the origin. These corrections are obtained by Ewald summation. (See
+ * Hernquist, Bouchet, Suto, ApJS, 1991, 75, 231) The correction fields
+ * are used to obtain the full periodic force if periodic boundaries
+ * combined with the pure tree algorithm are used. For the TreePM
+ * algorithm, the Ewald correction is not used.
+ *
+ * The correction fields are stored on disk once they are computed. If a
+ * corresponding file is found, they are loaded from disk to speed up the
+ * initialization. The Ewald summation is done in parallel, i.e. the
+ * processors share the work to compute the tables if needed.
+ */
+void ewald_init(void)
+{
+ int i, j, k, beg, len, size, n, task, count;
+ double x[3], force[3];
+ char buf[200];
+ FILE *fd;
+
+ if(ThisTask == 0)
+ {
+ printf("initialize Ewald correction...\n");
+ fflush(stdout);
+ }
+
+#ifdef DOUBLEPRECISION
+ sprintf(buf, "ewald_spc_table_%d_dbl.dat", EN);
+#else
+ sprintf(buf, "ewald_spc_table_%d.dat", EN);
+#endif
+
+ if((fd = fopen(buf, "r")))
+ {
+ if(ThisTask == 0)
+ {
+ printf("\nreading Ewald tables from file `%s'\n", buf);
+ fflush(stdout);
+ }
+
+ my_fread(&fcorrx[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fread(&fcorry[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fread(&fcorrz[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fread(&potcorr[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ fclose(fd);
+ }
+ else
+ {
+ if(ThisTask == 0)
+ {
+ printf("\nNo Ewald tables in file `%s' found.\nRecomputing them...\n", buf);
+ fflush(stdout);
+ }
+
+ /* ok, let's recompute things. Actually, we do that in parallel. */
+
+ size = (EN + 1) * (EN + 1) * (EN + 1) / NTask;
+
+
+ beg = ThisTask * size;
+ len = size;
+ if(ThisTask == (NTask - 1))
+ len = (EN + 1) * (EN + 1) * (EN + 1) - beg;
+
+ for(i = 0, count = 0; i <= EN; i++)
+ for(j = 0; j <= EN; j++)
+ for(k = 0; k <= EN; k++)
+ {
+ n = (i * (EN + 1) + j) * (EN + 1) + k;
+ if(n >= beg && n < (beg + len))
+ {
+ if(ThisTask == 0)
+ {
+ if((count % (len / 20)) == 0)
+ {
+ printf("%4.1f percent done\n", count / (len / 100.0));
+ fflush(stdout);
+ }
+ }
+
+ x[0] = 0.5 * ((double) i) / EN;
+ x[1] = 0.5 * ((double) j) / EN;
+ x[2] = 0.5 * ((double) k) / EN;
+
+ ewald_force(i, j, k, x, force);
+
+ fcorrx[i][j][k] = force[0];
+ fcorry[i][j][k] = force[1];
+ fcorrz[i][j][k] = force[2];
+
+ if(i + j + k == 0)
+ potcorr[i][j][k] = 2.8372975;
+ else
+ potcorr[i][j][k] = ewald_psi(x);
+
+ count++;
+ }
+ }
+
+ for(task = 0; task < NTask; task++)
+ {
+ beg = task * size;
+ len = size;
+ if(task == (NTask - 1))
+ len = (EN + 1) * (EN + 1) * (EN + 1) - beg;
+
+#ifdef DOUBLEPRECISION
+ MPI_Bcast(&fcorrx[0][0][beg], len, MPI_DOUBLE, task, MPI_COMM_WORLD);
+ MPI_Bcast(&fcorry[0][0][beg], len, MPI_DOUBLE, task, MPI_COMM_WORLD);
+ MPI_Bcast(&fcorrz[0][0][beg], len, MPI_DOUBLE, task, MPI_COMM_WORLD);
+ MPI_Bcast(&potcorr[0][0][beg], len, MPI_DOUBLE, task, MPI_COMM_WORLD);
+#else
+ MPI_Bcast(&fcorrx[0][0][beg], len, MPI_FLOAT, task, MPI_COMM_WORLD);
+ MPI_Bcast(&fcorry[0][0][beg], len, MPI_FLOAT, task, MPI_COMM_WORLD);
+ MPI_Bcast(&fcorrz[0][0][beg], len, MPI_FLOAT, task, MPI_COMM_WORLD);
+ MPI_Bcast(&potcorr[0][0][beg], len, MPI_FLOAT, task, MPI_COMM_WORLD);
+#endif
+ }
+
+ if(ThisTask == 0)
+ {
+ printf("\nwriting Ewald tables to file `%s'\n", buf);
+ fflush(stdout);
+
+ if((fd = fopen(buf, "w")))
+ {
+ my_fwrite(&fcorrx[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fwrite(&fcorry[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fwrite(&fcorrz[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ my_fwrite(&potcorr[0][0][0], sizeof(FLOAT), (EN + 1) * (EN + 1) * (EN + 1), fd);
+ fclose(fd);
+ }
+ }
+ }
+
+ fac_intp = 2 * EN / All.BoxSize;
+
+ for(i = 0; i <= EN; i++)
+ for(j = 0; j <= EN; j++)
+ for(k = 0; k <= EN; k++)
+ {
+ potcorr[i][j][k] /= All.BoxSize;
+ fcorrx[i][j][k] /= All.BoxSize * All.BoxSize;
+ fcorry[i][j][k] /= All.BoxSize * All.BoxSize;
+ fcorrz[i][j][k] /= All.BoxSize * All.BoxSize;
+ }
+
+ if(ThisTask == 0)
+ {
+ printf("initialization of periodic boundaries finished.\n");
+ fflush(stdout);
+ }
+}
+
+
+/*! This function looks up the correction force due to the infinite number
+ * of periodic particle/node images. We here use trilinear interpolation
+ * to get it from the precomputed tables, which contain one octant
+ * around the target particle at the origin. The other octants are
+ * obtained from it by exploiting the symmetry properties.
+ */
+#ifdef FORCETEST
+void ewald_corr(double dx, double dy, double dz, double *fper)
+{
+ int signx, signy, signz;
+ int i, j, k;
+ double u, v, w;
+ double f1, f2, f3, f4, f5, f6, f7, f8;
+
+ if(dx < 0)
+ {
+ dx = -dx;
+ signx = +1;
+ }
+ else
+ signx = -1;
+
+ if(dy < 0)
+ {
+ dy = -dy;
+ signy = +1;
+ }
+ else
+ signy = -1;
+
+ if(dz < 0)
+ {
+ dz = -dz;
+ signz = +1;
+ }
+ else
+ signz = -1;
+
+ u = dx * fac_intp;
+ i = (int) u;
+ if(i >= EN)
+ i = EN - 1;
+ u -= i;
+ v = dy * fac_intp;
+ j = (int) v;
+ if(j >= EN)
+ j = EN - 1;
+ v -= j;
+ w = dz * fac_intp;
+ k = (int) w;
+ if(k >= EN)
+ k = EN - 1;
+ w -= k;
+
+ f1 = (1 - u) * (1 - v) * (1 - w);
+ f2 = (1 - u) * (1 - v) * (w);
+ f3 = (1 - u) * (v) * (1 - w);
+ f4 = (1 - u) * (v) * (w);
+ f5 = (u) * (1 - v) * (1 - w);
+ f6 = (u) * (1 - v) * (w);
+ f7 = (u) * (v) * (1 - w);
+ f8 = (u) * (v) * (w);
+
+ fper[0] = signx * (fcorrx[i][j][k] * f1 +
+ fcorrx[i][j][k + 1] * f2 +
+ fcorrx[i][j + 1][k] * f3 +
+ fcorrx[i][j + 1][k + 1] * f4 +
+ fcorrx[i + 1][j][k] * f5 +
+ fcorrx[i + 1][j][k + 1] * f6 +
+ fcorrx[i + 1][j + 1][k] * f7 + fcorrx[i + 1][j + 1][k + 1] * f8);
+
+ fper[1] = signy * (fcorry[i][j][k] * f1 +
+ fcorry[i][j][k + 1] * f2 +
+ fcorry[i][j + 1][k] * f3 +
+ fcorry[i][j + 1][k + 1] * f4 +
+ fcorry[i + 1][j][k] * f5 +
+ fcorry[i + 1][j][k + 1] * f6 +
+ fcorry[i + 1][j + 1][k] * f7 + fcorry[i + 1][j + 1][k + 1] * f8);
+
+ fper[2] = signz * (fcorrz[i][j][k] * f1 +
+ fcorrz[i][j][k + 1] * f2 +
+ fcorrz[i][j + 1][k] * f3 +
+ fcorrz[i][j + 1][k + 1] * f4 +
+ fcorrz[i + 1][j][k] * f5 +
+ fcorrz[i + 1][j][k + 1] * f6 +
+ fcorrz[i + 1][j + 1][k] * f7 + fcorrz[i + 1][j + 1][k + 1] * f8);
+}
+#endif
+
+
+/*! This function looks up the correction potential due to the infinite
+ * number of periodic particle/node images. We here use tri-linear
+ * interpolation to get it from the precomputed table, which contains
+ * one octant around the target particle at the origin. The other
+ * octants are obtained from it by exploiting symmetry properties.
+ */
+double ewald_pot_corr(double dx, double dy, double dz)
+{
+ int i, j, k;
+ double u, v, w;
+ double f1, f2, f3, f4, f5, f6, f7, f8;
+
+ if(dx < 0)
+ dx = -dx;
+
+ if(dy < 0)
+ dy = -dy;
+
+ if(dz < 0)
+ dz = -dz;
+
+ u = dx * fac_intp;
+ i = (int) u;
+ if(i >= EN)
+ i = EN - 1;
+ u -= i;
+ v = dy * fac_intp;
+ j = (int) v;
+ if(j >= EN)
+ j = EN - 1;
+ v -= j;
+ w = dz * fac_intp;
+ k = (int) w;
+ if(k >= EN)
+ k = EN - 1;
+ w -= k;
+
+ f1 = (1 - u) * (1 - v) * (1 - w);
+ f2 = (1 - u) * (1 - v) * (w);
+ f3 = (1 - u) * (v) * (1 - w);
+ f4 = (1 - u) * (v) * (w);
+ f5 = (u) * (1 - v) * (1 - w);
+ f6 = (u) * (1 - v) * (w);
+ f7 = (u) * (v) * (1 - w);
+ f8 = (u) * (v) * (w);
+
+ return potcorr[i][j][k] * f1 +
+ potcorr[i][j][k + 1] * f2 +
+ potcorr[i][j + 1][k] * f3 +
+ potcorr[i][j + 1][k + 1] * f4 +
+ potcorr[i + 1][j][k] * f5 +
+ potcorr[i + 1][j][k + 1] * f6 + potcorr[i + 1][j + 1][k] * f7 + potcorr[i + 1][j + 1][k + 1] * f8;
+}
+
+
+
+/*! This function computes the potential correction term by means of Ewald
+ * summation.
+ */
+double ewald_psi(double x[3])
+{
+ double alpha, psi;
+ double r, sum1, sum2, hdotx;
+ double dx[3];
+ int i, n[3], h[3], h2;
+
+ alpha = 2.0;
+
+ for(n[0] = -4, sum1 = 0; n[0] <= 4; n[0]++)
+ for(n[1] = -4; n[1] <= 4; n[1]++)
+ for(n[2] = -4; n[2] <= 4; n[2]++)
+ {
+ for(i = 0; i < 3; i++)
+ dx[i] = x[i] - n[i];
+
+ r = sqrt(dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]);
+ sum1 += erfc(alpha * r) / r;
+ }
+
+ for(h[0] = -4, sum2 = 0; h[0] <= 4; h[0]++)
+ for(h[1] = -4; h[1] <= 4; h[1]++)
+ for(h[2] = -4; h[2] <= 4; h[2]++)
+ {
+ hdotx = x[0] * h[0] + x[1] * h[1] + x[2] * h[2];
+ h2 = h[0] * h[0] + h[1] * h[1] + h[2] * h[2];
+ if(h2 > 0)
+ sum2 += 1 / (M_PI * h2) * exp(-M_PI * M_PI * h2 / (alpha * alpha)) * cos(2 * M_PI * hdotx);
+ }
+
+ r = sqrt(x[0] * x[0] + x[1] * x[1] + x[2] * x[2]);
+
+ psi = M_PI / (alpha * alpha) - sum1 - sum2 + 1 / r;
+
+ return psi;
+}
+
+
+/*! This function computes the force correction term (difference between full
+ * force of infinite lattice and nearest image) by Ewald summation.
+ */
+void ewald_force(int iii, int jjj, int kkk, double x[3], double force[3])
+{
+ double alpha, r2;
+ double r, val, hdotx, dx[3];
+ int i, h[3], n[3], h2;
+
+ alpha = 2.0;
+
+ for(i = 0; i < 3; i++)
+ force[i] = 0;
+
+ if(iii == 0 && jjj == 0 && kkk == 0)
+ return;
+
+ r2 = x[0] * x[0] + x[1] * x[1] + x[2] * x[2];
+
+ for(i = 0; i < 3; i++)
+ force[i] += x[i] / (r2 * sqrt(r2));
+
+ for(n[0] = -4; n[0] <= 4; n[0]++)
+ for(n[1] = -4; n[1] <= 4; n[1]++)
+ for(n[2] = -4; n[2] <= 4; n[2]++)
+ {
+ for(i = 0; i < 3; i++)
+ dx[i] = x[i] - n[i];
+
+ r = sqrt(dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]);
+
+ val = erfc(alpha * r) + 2 * alpha * r / sqrt(M_PI) * exp(-alpha * alpha * r * r);
+
+ for(i = 0; i < 3; i++)
+ force[i] -= dx[i] / (r * r * r) * val;
+ }
+
+ for(h[0] = -4; h[0] <= 4; h[0]++)
+ for(h[1] = -4; h[1] <= 4; h[1]++)
+ for(h[2] = -4; h[2] <= 4; h[2]++)
+ {
+ hdotx = x[0] * h[0] + x[1] * h[1] + x[2] * h[2];
+ h2 = h[0] * h[0] + h[1] * h[1] + h[2] * h[2];
+
+ if(h2 > 0)
+ {
+ val = 2.0 / ((double) h2) * exp(-M_PI * M_PI * h2 / (alpha * alpha)) * sin(2 * M_PI * hdotx);
+
+ for(i = 0; i < 3; i++)
+ force[i] -= h[i] * val;
+ }
+ }
+}
+
+#endif
diff --git a/global.c b/global.c
new file mode 100644
index 0000000..e79a6b3
--- /dev/null
+++ b/global.c
@@ -0,0 +1,379 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file global.c
+ * \brief Computes global physical properties of the system
+ */
+
+
+/*! This routine computes various global properties of the particle
+ * distribution and stores the result in the struct `SysState'.
+ * Currently, not all the information that's computed here is actually
+ * used (e.g. momentum is not really used anywhere), just the energies are
+ * written to a log-file every once in a while.
+ */
+void compute_global_quantities_of_system(void)
+{
+ int i, j, n;
+ struct state_of_system sys;
+ double a1, a2, a3;
+ double entr = 0, egyspec, vel[3];
+#ifdef AGN_HEATING
+ double especagnheat = 0;
+#endif
+#ifdef SFR
+ /* see starformation.c */
+#endif
+ double dt_entr, dt_gravkick, dt_hydrokick;
+
+
+
+ if(All.ComovingIntegrationOn)
+ {
+ a1 = All.Time;
+ a2 = All.Time * All.Time;
+ a3 = All.Time * All.Time * All.Time;
+ }
+ else
+ {
+ a1 = a2 = a3 = 1;
+ }
+
+
+ for(n = 0; n < 6; n++)
+ {
+ sys.MassComp[n] = sys.EnergyKinComp[n] = sys.EnergyPotComp[n] = sys.EnergyIntComp[n] = 0;
+#ifdef COOLING
+ sys.EnergyRadSphComp[n] = 0;
+#endif
+#ifdef AGN_HEATING
+ sys.EnergyAGNHeatComp[n] = 0;
+#endif
+ for(j = 0; j < 4; j++)
+ sys.CenterOfMassComp[n][j] = sys.MomentumComp[n][j] = sys.AngMomentumComp[n][j] = 0;
+ }
+
+
+
+
+#ifdef SFR
+ rearrange_particle_sequence();
+#endif
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+ sys.MassComp[P[i].Type] += P[i].Mass;
+
+
+ sys.EnergyPotComp[P[i].Type] += 0.5 * P[i].Mass * P[i].Potential / a1;
+
+ if(All.ComovingIntegrationOn)
+ {
+ dt_entr = (All.Ti_Current - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+ dt_gravkick = get_gravkick_factor(P[i].Ti_begstep, All.Ti_Current) -
+ get_gravkick_factor(P[i].Ti_begstep, (P[i].Ti_begstep + P[i].Ti_endstep) / 2);
+ dt_hydrokick = get_hydrokick_factor(P[i].Ti_begstep, All.Ti_Current) -
+ get_hydrokick_factor(P[i].Ti_begstep, (P[i].Ti_begstep + P[i].Ti_endstep) / 2);
+ }
+ else
+ dt_entr = dt_gravkick = dt_hydrokick =
+ (All.Ti_Current - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+
+ for(j = 0; j < 3; j++)
+ {
+ vel[j] = P[i].Vel[j] + P[i].GravAccel[j] * dt_gravkick;
+
+ if(P[i].Type == 0)
+ vel[j] += SphP[i].HydroAccel[j] * dt_hydrokick;
+ }
+ if(P[i].Type == 0)
+ {
+ entr = SphP[i].Entropy + SphP[i].DtEntropy * dt_entr;
+#ifdef AGN_HEATING
+ especagnheat = SphP[i].EgySpecAGNHeat + SphP[i].DtEgySpecAGNHeat * dt_entr;
+#endif
+#ifdef SFR
+ /* see starformation.c */
+#endif
+
+ }
+#ifdef PMGRID
+ if(All.ComovingIntegrationOn)
+ dt_gravkick = get_gravkick_factor(All.PM_Ti_begstep, All.Ti_Current) -
+ get_gravkick_factor(All.PM_Ti_begstep, (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2);
+ else
+ dt_gravkick = (All.Ti_Current - (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2) * All.Timebase_interval;
+
+ for(j = 0; j < 3; j++)
+ vel[j] += P[i].GravPM[j] * dt_gravkick;
+#endif
+
+ sys.EnergyKinComp[P[i].Type] +=
+ 0.5 * P[i].Mass * (vel[0] * vel[0] + vel[1] * vel[1] + vel[2] * vel[2]) / a2;
+
+ if(P[i].Type == 0)
+ {
+#ifdef ISOTHERM_EQS
+ egyspec = entr;
+#else
+#ifdef MULTIPHASE
+ if (SphP[i].Phase== GAS_SPH)
+ egyspec = entr / (GAMMA_MINUS1) * pow(SphP[i].Density / a3, GAMMA_MINUS1);
+ else
+ egyspec = entr;
+#else
+ egyspec = entr / (GAMMA_MINUS1) * pow(SphP[i].Density / a3, GAMMA_MINUS1);
+
+#endif
+ if(All.MinEgySpec)
+ egyspec = dmax(All.MinEgySpec,egyspec);
+#endif
+ sys.EnergyIntComp[0] += P[i].Mass * egyspec;
+#ifdef AGN_HEATING
+ sys.EnergyAGNHeatComp[0] += P[i].Mass * especagnheat;
+#endif
+#ifdef SFR
+ /* see starformation.c */
+#endif
+ }
+
+
+
+ for(j = 0; j < 3; j++)
+ {
+ sys.MomentumComp[P[i].Type][j] += P[i].Mass * vel[j];
+ sys.CenterOfMassComp[P[i].Type][j] += P[i].Mass * P[i].Pos[j];
+ }
+
+ sys.AngMomentumComp[P[i].Type][0] += P[i].Mass * (P[i].Pos[1] * vel[2] - P[i].Pos[2] * vel[1]);
+ sys.AngMomentumComp[P[i].Type][1] += P[i].Mass * (P[i].Pos[2] * vel[0] - P[i].Pos[0] * vel[2]);
+ sys.AngMomentumComp[P[i].Type][2] += P[i].Mass * (P[i].Pos[0] * vel[1] - P[i].Pos[1] * vel[0]);
+ }
+
+/* count energy lost by different processes */
+
+#ifdef COOLING
+ sys.EnergyRadSphComp[0]=LocalSysState.RadiatedEnergy;
+#endif
+
+#ifdef SFR
+ sys.EnergyIntComp[ST]=LocalSysState.StarEnergyInt;
+#endif
+
+
+/* count thermal feedback */
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ sys.EnergyThermalFeedbackComp[0]=LocalSysState.EnergyThermalFeedback;
+ for(i = 1; i < 6; i++)
+ sys.EnergyThermalFeedbackComp[i]=0;
+#endif
+/* count kinetic feedback */
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ sys.EnergyKineticFeedbackComp[0]=LocalSysState.EnergyKineticFeedback;
+ for(i = 1; i < 6; i++)
+ sys.EnergyKineticFeedbackComp[i]=0;
+#endif
+/* count sticky */
+#ifdef MULTIPHASE
+ sys.EnergyRadStickyComp[0]=LocalSysState.EnergyRadSticky;
+ for(i = 1; i < 6; i++)
+ sys.EnergyRadStickyComp[i]=0;
+#endif
+/* count feedback wind */
+#ifdef FEEDBACK_WIND
+ sys.EnergyFeedbackWindComp[0]=LocalSysState.EnergyFeedbackWind;
+ for(i = 1; i < 6; i++)
+ sys.EnergyFeedbackWindComp[i]=0;
+#endif
+
+
+ /* some the stuff over all processors */
+ MPI_Reduce(&sys.MassComp[0], &SysState.MassComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&sys.EnergyPotComp[0], &SysState.EnergyPotComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&sys.EnergyIntComp[0], &SysState.EnergyIntComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#ifdef COOLING
+ MPI_Reduce(&sys.EnergyRadSphComp[0], &SysState.EnergyRadSphComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+#ifdef AGN_HEATING
+ MPI_Reduce(&sys.EnergyAGNHeatComp[0], &SysState.EnergyAGNHeatComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ MPI_Reduce(&sys.EnergyThermalFeedbackComp[0], &SysState.EnergyThermalFeedbackComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ MPI_Reduce(&sys.EnergyKineticFeedbackComp[0], &SysState.EnergyKineticFeedbackComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+#ifdef MULTIPHASE
+ MPI_Reduce(&sys.EnergyRadStickyComp[0], &SysState.EnergyRadStickyComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+#ifdef FEEDBACK_WIND
+ MPI_Reduce(&sys.EnergyFeedbackWindComp[0], &SysState.EnergyFeedbackWindComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+
+
+ MPI_Reduce(&sys.EnergyKinComp[0], &SysState.EnergyKinComp[0], 6, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&sys.MomentumComp[0][0], &SysState.MomentumComp[0][0], 6 * 4, MPI_DOUBLE, MPI_SUM, 0,
+ MPI_COMM_WORLD);
+ MPI_Reduce(&sys.AngMomentumComp[0][0], &SysState.AngMomentumComp[0][0], 6 * 4, MPI_DOUBLE, MPI_SUM, 0,
+ MPI_COMM_WORLD);
+ MPI_Reduce(&sys.CenterOfMassComp[0][0], &SysState.CenterOfMassComp[0][0], 6 * 4, MPI_DOUBLE, MPI_SUM, 0,
+ MPI_COMM_WORLD);
+
+
+#ifdef BUBBLES
+ SysState.EnergyBubblesComp[0]=All.EnergyBubbles;
+ for(i = 1; i < 6; i++)
+ SysState.EnergyBubblesComp[i]=0;
+#endif
+
+
+
+
+
+
+ if(ThisTask == 0)
+ {
+ for(i = 0; i < 6; i++)
+ {
+ SysState.EnergyTotComp[i] = SysState.EnergyKinComp[i]
+ + SysState.EnergyPotComp[i]
+ + SysState.EnergyIntComp[i];
+#ifdef COOLING
+ SysState.EnergyTotComp[i] += SysState.EnergyRadSphComp[i];
+#endif
+#ifdef AGN_HEATING
+ SysState.EnergyTotComp[i] += SysState.EnergyAGNHeatComp[i];
+#endif
+#ifdef MULTIPHASE
+ SysState.EnergyTotComp[i] += SysState.EnergyRadStickyComp[i];
+#endif
+#ifdef FEEDBACK_WIND
+ SysState.EnergyTotComp[i] += SysState.EnergyFeedbackWindComp[i];
+#endif
+#ifdef BUBBLES
+ SysState.EnergyTotComp[i] += SysState.EnergyBubblesComp[i];
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ SysState.EnergyTotComp[i] += SysState.EnergyThermalFeedbackComp[i];
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ SysState.EnergyTotComp[i] += SysState.EnergyKineticFeedbackComp[i];
+#endif
+ }
+
+ SysState.Mass = SysState.EnergyKin = SysState.EnergyPot = SysState.EnergyInt = SysState.EnergyTot = 0;
+#ifdef COOLING
+ SysState.EnergyRadSph = 0;
+#endif
+#ifdef AGN_HEATING
+ SysState.EnergyAGNHeat = 0;
+#endif
+#ifdef MULTIPHASE
+ SysState.EnergyRadSticky = 0;
+#endif
+#ifdef FEEDBACK_WIND
+ SysState.EnergyFeedbackWind = 0;
+#endif
+#ifdef BUBBLES
+ SysState.EnergyBubbles = 0;
+#endif
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ SysState.EnergyThermalFeedback = 0;
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ SysState.EnergyKineticFeedback = 0;
+#endif
+
+
+ for(j = 0; j < 3; j++)
+ SysState.Momentum[j] = SysState.AngMomentum[j] = SysState.CenterOfMass[j] = 0;
+
+ for(i = 0; i < 6; i++)
+ {
+ SysState.Mass += SysState.MassComp[i];
+ SysState.EnergyKin += SysState.EnergyKinComp[i];
+ SysState.EnergyPot += SysState.EnergyPotComp[i];
+ SysState.EnergyInt += SysState.EnergyIntComp[i];
+ SysState.EnergyTot += SysState.EnergyTotComp[i];
+#ifdef COOLING
+ SysState.EnergyRadSph += SysState.EnergyRadSphComp[i];
+#endif
+#ifdef AGN_HEATING
+ SysState.EnergyAGNHeat += SysState.EnergyAGNHeatComp[i];
+#endif
+#ifdef MULTIPHASE
+ SysState.EnergyRadSticky += SysState.EnergyRadStickyComp[i];
+#endif
+#ifdef FEEDBACK_WIND
+ SysState.EnergyFeedbackWind += SysState.EnergyFeedbackWindComp[i];
+#endif
+#ifdef BUBBLES
+ SysState.EnergyBubbles += SysState.EnergyBubblesComp[i];
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ SysState.EnergyThermalFeedback += SysState.EnergyThermalFeedbackComp[i];
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ SysState.EnergyKineticFeedback += SysState.EnergyKineticFeedbackComp[i];
+#endif
+
+ for(j = 0; j < 3; j++)
+ {
+ SysState.Momentum[j] += SysState.MomentumComp[i][j];
+ SysState.AngMomentum[j] += SysState.AngMomentumComp[i][j];
+ SysState.CenterOfMass[j] += SysState.CenterOfMassComp[i][j];
+ }
+ }
+
+ for(i = 0; i < 6; i++)
+ for(j = 0; j < 3; j++)
+ if(SysState.MassComp[i] > 0)
+ SysState.CenterOfMassComp[i][j] /= SysState.MassComp[i];
+
+ for(j = 0; j < 3; j++)
+ if(SysState.Mass > 0)
+ SysState.CenterOfMass[j] /= SysState.Mass;
+
+ for(i = 0; i < 6; i++)
+ {
+ SysState.CenterOfMassComp[i][3] = SysState.MomentumComp[i][3] = SysState.AngMomentumComp[i][3] = 0;
+ for(j = 0; j < 3; j++)
+ {
+ SysState.CenterOfMassComp[i][3] +=
+ SysState.CenterOfMassComp[i][j] * SysState.CenterOfMassComp[i][j];
+ SysState.MomentumComp[i][3] += SysState.MomentumComp[i][j] * SysState.MomentumComp[i][j];
+ SysState.AngMomentumComp[i][3] +=
+ SysState.AngMomentumComp[i][j] * SysState.AngMomentumComp[i][j];
+ }
+ SysState.CenterOfMassComp[i][3] = sqrt(SysState.CenterOfMassComp[i][3]);
+ SysState.MomentumComp[i][3] = sqrt(SysState.MomentumComp[i][3]);
+ SysState.AngMomentumComp[i][3] = sqrt(SysState.AngMomentumComp[i][3]);
+ }
+
+ SysState.CenterOfMass[3] = SysState.Momentum[3] = SysState.AngMomentum[3] = 0;
+
+ for(j = 0; j < 3; j++)
+ {
+ SysState.CenterOfMass[3] += SysState.CenterOfMass[j] * SysState.CenterOfMass[j];
+ SysState.Momentum[3] += SysState.Momentum[j] * SysState.Momentum[j];
+ SysState.AngMomentum[3] += SysState.AngMomentum[j] * SysState.AngMomentum[j];
+ }
+
+ SysState.CenterOfMass[3] = sqrt(SysState.CenterOfMass[3]);
+ SysState.Momentum[3] = sqrt(SysState.Momentum[3]);
+ SysState.AngMomentum[3] = sqrt(SysState.AngMomentum[3]);
+ }
+
+ /* give everyone the result, maybe the want to do something with it */
+ MPI_Bcast(&SysState, sizeof(struct state_of_system), MPI_BYTE, 0, MPI_COMM_WORLD);
+}
diff --git a/gravtree.c b/gravtree.c
new file mode 100644
index 0000000..7b204f7
--- /dev/null
+++ b/gravtree.c
@@ -0,0 +1,651 @@
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <float.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file gravtree.c
+ * \brief main driver routines for gravitational (short-range) force computation
+ *
+ * This file contains the code for the gravitational force computation by
+ * means of the tree algorithm. To this end, a tree force is computed for
+ * all active local particles, and particles are exported to other
+ * processors if needed, where they can receive additional force
+ * contributions. If the TreePM algorithm is enabled, the force computed
+ * will only be the short-range part.
+ */
+
+/*! This function computes the gravitational forces for all active
+ * particles. If needed, a new tree is constructed, otherwise the
+ * dynamically updated tree is used. Particles are only exported to other
+ * processors when really needed, thereby allowing a good use of the
+ * communication buffer.
+ */
+void gravity_tree(void)
+{
+ long long ntot;
+ int numnodes, nexportsum = 0;
+ int i, j, iter = 0;
+ int *numnodeslist, maxnumnodes, nexport, *numlist, *nrecv, *ndonelist;
+ double tstart, tend, timetree = 0, timecommsumm = 0, timeimbalance = 0, sumimbalance;
+ double ewaldcount;
+ double costtotal, ewaldtot, *costtreelist, *ewaldlist;
+ double maxt, sumt, *timetreelist, *timecommlist;
+ double fac, plb, plb_max, sumcomm;
+
+
+#ifdef DETAILED_CPU_GRAVITY
+ double timetree1 = 0, timecommsumm1 = 0, timeimbalance1 = 0;
+ double timetree2 = 0, timecommsumm2 = 0, timeimbalance2 = 0;
+ double sumt1,sumt2;
+ double sumcomm1,sumcomm2;
+ double sumimbalance1,sumimbalance2;
+#endif
+
+#ifdef DETAILED_OUTPUT_IN_GRAVTREE
+ double *timetreelist1, *timecommlist1, *timeimbalancelist1;
+ double *timetreelist2, *timecommlist2, *timeimbalancelist2;
+ double *timeimbalancelist;
+ int *numpartlist;
+#endif
+
+
+#ifndef NOGRAVITY
+ int *noffset, *nbuffer, *nsend, *nsend_local;
+ long long ntotleft;
+ int ndone, maxfill, ngrp;
+ int k, place;
+ int level, sendTask, recvTask;
+ double ax, ay, az;
+ MPI_Status status;
+#endif
+
+ /* set new softening lengths */
+ if(All.ComovingIntegrationOn)
+ set_softenings();
+
+
+ /* contruct tree if needed */
+ tstart = second();
+ if(TreeReconstructFlag)
+ {
+ if(ThisTask == 0)
+ printf("Tree construction.\n");
+
+ force_treebuild(NumPart);
+
+ TreeReconstructFlag = 0;
+
+ if(ThisTask == 0)
+ printf("Tree construction done.\n");
+ }
+ tend = second();
+ All.CPU_TreeConstruction += timediff(tstart, tend);
+
+ costtotal = ewaldcount = 0;
+
+ /* Note: 'NumForceUpdate' has already been determined in find_next_sync_point_and_drift() */
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumForceUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+
+#ifndef NOGRAVITY
+ if(ThisTask == 0)
+ printf("Begin tree force.\n");
+
+
+#ifdef SELECTIVE_NO_GRAVITY
+ for(i = 0; i < NumPart; i++)
+ if(((1 << P[i].Type) & (SELECTIVE_NO_GRAVITY)))
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+#endif
+
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+ i = 0; /* beginn with this index */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+ iter++;
+
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < NumPart && nexport < All.BunchSizeForce - NTask; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+#ifndef PMGRID
+ costtotal += force_treeevaluate(i, 0, &ewaldcount);
+#else
+ costtotal += force_treeevaluate_shortrange(i, 0);
+#endif
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ for(k = 0; k < 3; k++)
+ GravDataGet[nexport].u.Pos[k] = P[i].Pos[k];
+#if defined(UNEQUALSOFTENINGS) || defined(STELLAR_FLUX)
+ GravDataGet[nexport].Type = P[i].Type;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(P[i].Type == 0)
+ GravDataGet[nexport].Soft = SphP[i].Hsml;
+#endif
+#endif
+ GravDataGet[nexport].w.OldAcc = P[i].OldAcc;
+ GravDataIndexTable[nexport].Task = j;
+ GravDataIndexTable[nexport].Index = i;
+ GravDataIndexTable[nexport].SortIndex = nexport;
+ nexport++;
+ nexportsum++;
+ nsend_local[j]++;
+ }
+ }
+ }
+ tend = second();
+ timetree += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timetree1 += timediff(tstart, tend);
+#endif
+
+ qsort(GravDataIndexTable, nexport, sizeof(struct gravdata_index), grav_tree_compare_key);
+
+ for(j = 0; j < nexport; j++)
+ GravDataIn[j] = GravDataGet[GravDataIndexTable[j].SortIndex];
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timeimbalance1 += timediff(tstart, tend);
+#endif
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&GravDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_GRAV_A,
+ &GravDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_GRAV_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timecommsumm1 += timediff(tstart, tend);
+#endif
+
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ {
+#ifndef PMGRID
+ costtotal += force_treeevaluate(j, 1, &ewaldcount);
+#else
+ costtotal += force_treeevaluate_shortrange(j, 1);
+#endif
+ }
+ tend = second();
+ timetree += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timetree2 += timediff(tstart, tend);
+#endif
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timeimbalance2 += timediff(tstart, tend);
+#endif
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&GravDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_GRAV_B,
+ &GravDataOut[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_GRAV_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ place = GravDataIndexTable[noffset[recvTask] + j].Index;
+
+ for(k = 0; k < 3; k++)
+ P[place].GravAccel[k] += GravDataOut[j + noffset[recvTask]].u.Acc[k];
+
+ P[place].GravCost += GravDataOut[j + noffset[recvTask]].w.Ninteractions;
+#ifdef STELLAR_FLUX
+ if (P[place].Type==0) /* only for gas */
+ SphP[place].EnergyFlux += GravDataOut[j + noffset[recvTask]].EnergyFlux;
+#endif
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+#ifdef DETAILED_CPU_GRAVITY
+ timecommsumm2 += timediff(tstart, tend);
+#endif
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+ /* now add things for comoving integration */
+ /* yr: with vacuum boundary conditions */
+
+#ifndef PERIODIC
+#ifndef PMGRID
+ if(All.ComovingIntegrationOn)
+ {
+ fac = 0.5 * All.Hubble * All.Hubble * All.Omega0 / All.G;
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccel[j] += fac * P[i].Pos[j];
+ }
+#endif
+#endif
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+#ifdef PMGRID
+ ax = P[i].GravAccel[0] + P[i].GravPM[0] / All.G;
+ ay = P[i].GravAccel[1] + P[i].GravPM[1] / All.G;
+ az = P[i].GravAccel[2] + P[i].GravPM[2] / All.G;
+#else
+ ax = P[i].GravAccel[0];
+ ay = P[i].GravAccel[1];
+ az = P[i].GravAccel[2];
+#endif
+ P[i].OldAcc = sqrt(ax * ax + ay * ay + az * az);
+ }
+
+
+ if(All.TypeOfOpeningCriterion == 1)
+ All.ErrTolTheta = 0; /* This will switch to the relative opening criterion for the following force computations */
+
+ /* muliply by G */
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccel[j] *= All.G;
+
+
+ /* Finally, the following factor allows a computation of a cosmological simulation
+ with vacuum energy in physical coordinates */
+#ifndef PERIODIC
+#ifndef PMGRID
+ if(All.ComovingIntegrationOn == 0)
+ {
+ fac = All.OmegaLambda * All.Hubble * All.Hubble;
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccel[j] += fac * P[i].Pos[j];
+ }
+#endif
+#endif
+
+#ifdef SELECTIVE_NO_GRAVITY
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+#endif
+
+ if(ThisTask == 0)
+ printf("tree is done.\n");
+
+
+#else /* gravity is switched off */
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccel[j] = 0;
+
+#endif
+
+#ifdef OUTERPOTENTIAL
+ /* Add outer forces from an outer potential */
+ outer_forces();
+#endif
+
+ /* Now the force computation is finished */
+
+ /* gather some diagnostic information */
+
+ timetreelist = malloc(sizeof(double) * NTask);
+ timecommlist = malloc(sizeof(double) * NTask);
+ costtreelist = malloc(sizeof(double) * NTask);
+ numnodeslist = malloc(sizeof(int) * NTask);
+ ewaldlist = malloc(sizeof(double) * NTask);
+ nrecv = malloc(sizeof(int) * NTask);
+
+#ifdef DETAILED_OUTPUT_IN_GRAVTREE
+ timeimbalancelist = malloc(sizeof(double) * NTask);
+ numpartlist = malloc(sizeof(int) * NTask);
+ timetreelist1 = malloc(sizeof(double) * NTask);
+ timecommlist1 = malloc(sizeof(double) * NTask);
+ timeimbalancelist1 = malloc(sizeof(double) * NTask);
+ timetreelist2 = malloc(sizeof(double) * NTask);
+ timecommlist2 = malloc(sizeof(double) * NTask);
+ timeimbalancelist2 = malloc(sizeof(double) * NTask);
+#endif
+
+ numnodes = Numnodestree;
+
+
+ MPI_Gather(&costtotal, 1, MPI_DOUBLE, costtreelist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&numnodes, 1, MPI_INT, numnodeslist, 1, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timetree, 1, MPI_DOUBLE, timetreelist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timecommsumm, 1, MPI_DOUBLE, timecommlist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&NumPart, 1, MPI_INT, nrecv, 1, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(&ewaldcount, 1, MPI_DOUBLE, ewaldlist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&nexportsum, &nexport, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance, &sumimbalance, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#ifdef DETAILED_CPU_GRAVITY
+ MPI_Reduce(&timetree1, &sumt1, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timetree2, &sumt2, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance1, &sumimbalance1, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance2, &sumimbalance2, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm1, &sumcomm1, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm2, &sumcomm2, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+#endif
+
+#ifdef DETAILED_OUTPUT_IN_GRAVTREE
+ MPI_Gather(&timeimbalance, 1, MPI_DOUBLE, timeimbalancelist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&NumPart, 1, MPI_INT, numpartlist, 1, MPI_INT, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timetree1, 1, MPI_DOUBLE, timetreelist1, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timetree2, 1, MPI_DOUBLE, timetreelist2, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timecommsumm1, 1, MPI_DOUBLE, timecommlist1, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timecommsumm2, 1, MPI_DOUBLE, timecommlist2, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timeimbalance1, 1, MPI_DOUBLE, timeimbalancelist1, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timeimbalance2, 1, MPI_DOUBLE, timeimbalancelist2, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+#endif
+
+#ifdef SPLIT_DOMAIN_USING_TIME
+ CPU_Gravity = timetree1;
+#endif
+
+ if(ThisTask == 0)
+ {
+ All.TotNumOfForces += ntot;
+
+ fprintf(FdTimings, "Step= %d t= %g dt= %g \n", All.NumCurrentTiStep, All.Time, All.TimeStep);
+ fprintf(FdTimings, "Nf= %d%09d total-Nf= %d%09d ex-frac= %g iter= %d\n",
+ (int) (ntot / 1000000000), (int) (ntot % 1000000000),
+ (int) (All.TotNumOfForces / 1000000000), (int) (All.TotNumOfForces % 1000000000),
+ nexport / ((double) ntot), iter);
+ /* note: on Linux, the 8-byte integer could be printed with the format identifier "%qd", but doesn't work on AIX */
+
+ fac = NTask / ((double) All.TotNumPart);
+
+
+ for(i = 0, maxt = timetreelist[0], sumt = 0, plb_max = 0,
+ maxnumnodes = 0, costtotal = 0, sumcomm = 0, ewaldtot = 0; i < NTask; i++)
+ {
+ costtotal += costtreelist[i];
+
+ sumcomm += timecommlist[i];
+
+ if(maxt < timetreelist[i])
+ maxt = timetreelist[i];
+ sumt += timetreelist[i];
+
+ plb = nrecv[i] * fac;
+
+ if(plb > plb_max)
+ plb_max = plb;
+
+ if(numnodeslist[i] > maxnumnodes)
+ maxnumnodes = numnodeslist[i];
+
+ ewaldtot += ewaldlist[i];
+ }
+ fprintf(FdTimings, "work-load balance: %g max=%g avg=%g PE0=%g\n",
+ maxt / (sumt / NTask), maxt, sumt / NTask, timetreelist[0]);
+ fprintf(FdTimings, "particle-load balance: %g\n", plb_max);
+ fprintf(FdTimings, "max. nodes: %d, filled: %g\n", maxnumnodes,
+ maxnumnodes / (All.TreeAllocFactor * All.MaxPart));
+ fprintf(FdTimings, "part/sec=%g | %g ia/part=%g (%g)\n", ntot / (sumt + 1.0e-20),
+ ntot / (maxt * NTask), ((double) (costtotal)) / ntot, ((double) ewaldtot) / ntot);
+
+
+#ifdef DETAILED_OUTPUT_IN_GRAVTREE
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12d ",numpartlist[i]); /* nombre de part par proc */
+ fprintf(FdTimings, "\n");
+
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",costtreelist[i]); /* nombre d'interactions */
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timetreelist[i]);
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timeimbalancelist[i]);
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timecommlist[i]);
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timetreelist1[i]);
+ fprintf(FdTimings, "\n");
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timetreelist2[i]);
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timeimbalancelist1[i]);
+ fprintf(FdTimings, "\n");
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timeimbalancelist2[i]);
+ fprintf(FdTimings, "\n");
+
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timecommlist1[i]);
+ fprintf(FdTimings, "\n");
+ for (i=0;i<NTask;i++)
+ fprintf(FdTimings, "%12g ",timecommlist2[i]);
+ fprintf(FdTimings, "\n");
+#endif
+
+
+ fprintf(FdTimings, "\n");
+
+ fflush(FdTimings);
+
+ All.CPU_TreeWalk += sumt / NTask;
+ All.CPU_Imbalance += sumimbalance / NTask;
+ All.CPU_CommSum += sumcomm / NTask;
+
+#ifdef DETAILED_CPU_GRAVITY
+ All.CPU_Gravity_TreeWalk1 += sumt1 / NTask;
+ All.CPU_Gravity_Imbalance1 += sumimbalance1 / NTask;
+ All.CPU_Gravity_CommSum1 += sumcomm1 / NTask;
+
+ All.CPU_Gravity_TreeWalk2 += sumt2 / NTask;
+ All.CPU_Gravity_Imbalance2 += sumimbalance2 / NTask;
+ All.CPU_Gravity_CommSum2 += sumcomm2 / NTask;
+#endif
+ }
+
+ free(nrecv);
+ free(ewaldlist);
+ free(numnodeslist);
+ free(costtreelist);
+ free(timecommlist);
+ free(timetreelist);
+}
+
+
+
+/*! This function sets the (comoving) softening length of all particle
+ * types in the table All.SofteningTable[...]. We check that the physical
+ * softening length is bounded by the Softening-MaxPhys values.
+ */
+void set_softenings(void)
+{
+ int i;
+
+ if(All.ComovingIntegrationOn)
+ {
+ if(All.SofteningGas * All.Time > All.SofteningGasMaxPhys)
+ All.SofteningTable[0] = All.SofteningGasMaxPhys / All.Time;
+ else
+ All.SofteningTable[0] = All.SofteningGas;
+
+ if(All.SofteningHalo * All.Time > All.SofteningHaloMaxPhys)
+ All.SofteningTable[1] = All.SofteningHaloMaxPhys / All.Time;
+ else
+ All.SofteningTable[1] = All.SofteningHalo;
+
+ if(All.SofteningDisk * All.Time > All.SofteningDiskMaxPhys)
+ All.SofteningTable[2] = All.SofteningDiskMaxPhys / All.Time;
+ else
+ All.SofteningTable[2] = All.SofteningDisk;
+
+ if(All.SofteningBulge * All.Time > All.SofteningBulgeMaxPhys)
+ All.SofteningTable[3] = All.SofteningBulgeMaxPhys / All.Time;
+ else
+ All.SofteningTable[3] = All.SofteningBulge;
+
+ if(All.SofteningStars * All.Time > All.SofteningStarsMaxPhys)
+ All.SofteningTable[4] = All.SofteningStarsMaxPhys / All.Time;
+ else
+ All.SofteningTable[4] = All.SofteningStars;
+
+ if(All.SofteningBndry * All.Time > All.SofteningBndryMaxPhys)
+ All.SofteningTable[5] = All.SofteningBndryMaxPhys / All.Time;
+ else
+ All.SofteningTable[5] = All.SofteningBndry;
+ }
+ else
+ {
+ All.SofteningTable[0] = All.SofteningGas;
+ All.SofteningTable[1] = All.SofteningHalo;
+ All.SofteningTable[2] = All.SofteningDisk;
+ All.SofteningTable[3] = All.SofteningBulge;
+ All.SofteningTable[4] = All.SofteningStars;
+ All.SofteningTable[5] = All.SofteningBndry;
+ }
+
+ for(i = 0; i < 6; i++)
+ All.ForceSoftening[i] = 2.8 * All.SofteningTable[i];
+
+ All.MinGasHsml = All.MinGasHsmlFractional * All.ForceSoftening[0];
+}
+
+
+/*! This function is used as a comparison kernel in a sort routine. It is
+ * used to group particles in the communication buffer that are going to
+ * be sent to the same CPU.
+ */
+int grav_tree_compare_key(const void *a, const void *b)
+{
+ if(((struct gravdata_index *) a)->Task < (((struct gravdata_index *) b)->Task))
+ return -1;
+
+ if(((struct gravdata_index *) a)->Task > (((struct gravdata_index *) b)->Task))
+ return +1;
+
+ return 0;
+}
diff --git a/gravtree_forcetest.c b/gravtree_forcetest.c
new file mode 100644
index 0000000..177ac09
--- /dev/null
+++ b/gravtree_forcetest.c
@@ -0,0 +1,354 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <float.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file gravtree_forcetest.c
+ * \brief routines for direct summation forces
+ *
+ * The code in this file allows to compute checks of the force accuracy by
+ * an independent direct summation computation. To this end, one can
+ * instruct GADGET in the Makefile to coompute direct summation forces for
+ * a certain random subfraction of particles.
+ */
+
+
+
+#ifdef FORCETEST
+
+/*! This routine does the test of the gravitational tree force by computing
+ * the force for a random subset of particles with direct summation.
+ */
+void gravity_forcetest(void)
+{
+ int ntot, iter = 0, ntotleft, nthis;
+ double tstart, tend, timetree = 0;
+ int i, j, ndone, ngrp, maxfill, place, ndonetot;
+
+#ifndef NOGRAVITY
+ int *noffset, *nbuffer, *nsend, *nsend_local;
+ int k, nexport;
+ int level, sendTask, recvTask;
+ double fac1;
+ MPI_Status status;
+#endif
+ double costtotal, *costtreelist;
+ double maxt, sumt, *timetreelist;
+ double fac;
+ char buf[200];
+
+#ifdef PMGRID
+ if(All.PM_Ti_endstep != All.Ti_Current)
+ return;
+#endif
+
+ if(All.ComovingIntegrationOn)
+ set_softenings(); /* set new softening lengths */
+
+ for(i = 0, NumForceUpdate = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+ if(get_random_number(P[i].ID) < FORCETEST)
+ {
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+ NumForceUpdate++;
+ }
+ }
+ }
+
+ /* NumForceUpdate is the number of particles on this processor that want a force update */
+
+ MPI_Allreduce(&NumForceUpdate, &ntot, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ costtotal = 0;
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+
+ i = 0; /* beginn with this index */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+ iter++;
+
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < NumPart && nexport < All.BunchSizeForce - NTask; i++)
+ if(P[i].Ti_endstep < 0)
+ {
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 1;
+ Exportflag[ThisTask] = 0;
+
+ costtotal += force_treeevaluate_direct(i, 0);
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ for(k = 0; k < 3; k++)
+ GravDataGet[nexport].u.Pos[k] = P[i].Pos[k];
+
+#ifdef UNEQUALSOFTENINGS
+ GravDataGet[nexport].Type = P[i].Type;
+#endif
+ GravDataGet[nexport].w.OldAcc = P[i].OldAcc;
+
+ GravDataIndexTable[nexport].Task = j;
+ GravDataIndexTable[nexport].Index = i;
+ GravDataIndexTable[nexport].SortIndex = nexport;
+
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+ }
+ tend = second();
+ timetree += timediff(tstart, tend);
+
+ qsort(GravDataIndexTable, nexport, sizeof(struct gravdata_index), grav_tree_compare_key);
+
+ for(j = 0; j < nexport; j++)
+ GravDataIn[j] = GravDataGet[GravDataIndexTable[j].SortIndex];
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&GravDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_DIRECT_A,
+ &GravDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_DIRECT_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ {
+ costtotal += force_treeevaluate_direct(j, 1);
+ }
+ tend = second();
+ timetree += timediff(tstart, tend);
+
+
+ /* get the result */
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&GravDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_DIRECT_B,
+ &GravDataOut[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_DIRECT_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ place = GravDataIndexTable[noffset[recvTask] + j].Index;
+
+ for(k = 0; k < 3; k++)
+ P[place].GravAccelDirect[k] += GravDataOut[j + noffset[recvTask]].u.Acc[k];
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allreduce(&ndone, &ndonetot, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ ntotleft -= ndonetot;
+ }
+
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+ /* now add things for comoving integration */
+
+ if(All.ComovingIntegrationOn)
+ {
+#ifndef PERIODIC
+ fac1 = 0.5 * All.Hubble * All.Hubble * All.Omega0 / All.G;
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccelDirect[j] += fac1 * P[i].Pos[j];
+#endif
+ }
+
+
+
+ /* muliply by G */
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccelDirect[j] *= All.G;
+
+
+
+ /* Finally, the following factor allows a computation of cosmological simulation
+ with vacuum energy in physical coordinates */
+
+ if(All.ComovingIntegrationOn == 0)
+ {
+ fac1 = All.OmegaLambda * All.Hubble * All.Hubble;
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ for(j = 0; j < 3; j++)
+ P[i].GravAccelDirect[j] += fac1 * P[i].Pos[j];
+ }
+
+ /* now output the forces to a file */
+
+ for(nthis = 0; nthis < NTask; nthis++)
+ {
+ if(nthis == ThisTask)
+ {
+ sprintf(buf, "%s%s", All.OutputDir, "forcetest.txt");
+ if(!(FdForceTest = fopen(buf, "a")))
+ {
+ printf("error in opening file '%s'\n", buf);
+ endrun(17);
+ }
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ {
+#ifndef PMGRID
+ fprintf(FdForceTest, "%d %g %g %g %g %g %g %g %g %g %g %g\n",
+ P[i].Type, All.Time, All.Time - TimeOfLastTreeConstruction,
+ P[i].Pos[0], P[i].Pos[1], P[i].Pos[2],
+ P[i].GravAccelDirect[0], P[i].GravAccelDirect[1], P[i].GravAccelDirect[2],
+ P[i].GravAccel[0], P[i].GravAccel[1], P[i].GravAccel[2]);
+#else
+ fprintf(FdForceTest, "%d %g %g %g %g %g %g %g %g %g %g %g %g %g %g\n",
+ P[i].Type, All.Time, All.Time - TimeOfLastTreeConstruction,
+ P[i].Pos[0], P[i].Pos[1], P[i].Pos[2],
+ P[i].GravAccelDirect[0], P[i].GravAccelDirect[1], P[i].GravAccelDirect[2],
+ P[i].GravAccel[0], P[i].GravAccel[1], P[i].GravAccel[2],
+ P[i].GravPM[0] + P[i].GravAccel[0],
+ P[i].GravPM[1] + P[i].GravAccel[1], P[i].GravPM[2] + P[i].GravAccel[2]);
+#endif
+ }
+ fclose(FdForceTest);
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+
+ /* Now the force computation is finished */
+
+
+
+ timetreelist = malloc(sizeof(double) * NTask);
+ costtreelist = malloc(sizeof(double) * NTask);
+
+ MPI_Gather(&costtotal, 1, MPI_DOUBLE, costtreelist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+ MPI_Gather(&timetree, 1, MPI_DOUBLE, timetreelist, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ fac = NTask / ((double) All.TotNumPart);
+
+ for(i = 0, maxt = timetreelist[0], sumt = 0, costtotal = 0; i < NTask; i++)
+ {
+ costtotal += costtreelist[i];
+
+ if(maxt < timetreelist[i])
+ maxt = timetreelist[i];
+ sumt += timetreelist[i];
+ }
+
+ fprintf(FdTimings, "DIRECT Nf= %d part/sec=%g | %g ia/part=%g \n", ntot, ntot / (sumt + 1.0e-20),
+ ntot / (maxt * NTask), ((double) (costtotal)) / ntot);
+ fprintf(FdTimings, "\n");
+
+ fflush(FdTimings);
+ }
+
+ free(costtreelist);
+ free(timetreelist);
+}
+
+#endif
diff --git a/hydra.c b/hydra.c
new file mode 100644
index 0000000..2335cbd
--- /dev/null
+++ b/hydra.c
@@ -0,0 +1,761 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_math.h>
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file hydra.c
+ * \brief Computation of SPH forces and rate of entropy generation
+ *
+ * This file contains the "second SPH loop", where the SPH forces are
+ * computed, and where the rate of change of entropy due to the shock heating
+ * (via artificial viscosity) is computed.
+ */
+
+
+static double hubble_a, atime, hubble_a2, fac_mu, fac_vsic_fix, a3inv, fac_egy;
+#ifdef FEEDBACK
+static double fac_pow;
+#endif
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+
+
+/*! This function is the driver routine for the calculation of hydrodynamical
+ * force and rate of change of entropy due to shock heating for all active
+ * particles .
+ */
+void hydro_force(void)
+{
+ long long ntot, ntotleft;
+ int i, j, k, n, ngrp, maxfill, source, ndone;
+ int *nbuffer, *noffset, *nsend_local, *nsend, *numlist, *ndonelist;
+ int level, sendTask, recvTask, nexport, place;
+ double soundspeed_i;
+ double tstart, tend, sumt, sumcomm;
+ double timecomp = 0, timecommsumm = 0, timeimbalance = 0, sumimbalance;
+ MPI_Status status;
+
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ double v1m,v2m;
+#endif
+
+ if(All.ComovingIntegrationOn)
+ {
+ /* Factors for comoving integration of hydro */
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+ hubble_a2 = All.Time * All.Time * hubble_a;
+
+ fac_mu = pow(All.Time, 3 * (GAMMA - 1) / 2) / All.Time;
+
+ fac_egy = pow(All.Time, 3 * (GAMMA - 1));
+
+ fac_vsic_fix = hubble_a * pow(All.Time, 3 * GAMMA_MINUS1);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ atime = All.Time;
+
+#ifdef FEEDBACK
+ fac_pow = fac_egy*atime*atime;
+#endif
+
+ }
+ else
+ {
+ hubble_a = hubble_a2 = atime = fac_mu = fac_vsic_fix = a3inv = fac_egy = 1.0;
+#ifdef FEEDBACK
+ fac_pow = 1.0;
+#endif
+ }
+
+
+ /* `NumSphUpdate' gives the number of particles on this processor that want a force update */
+ for(n = 0, NumSphUpdate = 0; n < N_gas; n++)
+ {
+#ifdef SFR
+ if((P[n].Ti_endstep == All.Ti_Current) && (P[n].Type == 0))
+#else
+ if(P[n].Ti_endstep == All.Ti_Current)
+#endif
+#ifdef MULTIPHASE
+ if(SphP[n].Phase == GAS_SPH)
+#endif
+ NumSphUpdate++;
+
+
+ }
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumSphUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+ i = 0; /* first particle for this task */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < N_gas && nexport < All.BunchSizeHydro - NTask; i++)
+#ifdef SFR
+ if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+ {
+#ifdef MULTIPHASE
+ if(SphP[i].Phase == GAS_SPH)
+ {
+#endif
+
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ hydro_evaluate(i, 0);
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ for(k = 0; k < 3; k++)
+ {
+ HydroDataIn[nexport].Pos[k] = P[i].Pos[k];
+ HydroDataIn[nexport].Vel[k] = SphP[i].VelPred[k];
+ }
+ HydroDataIn[nexport].Hsml = SphP[i].Hsml;
+#ifdef FEEDBACK
+ HydroDataIn[nexport].EnergySN = SphP[i].EnergySN;
+#endif
+ HydroDataIn[nexport].Mass = P[i].Mass;
+ HydroDataIn[nexport].DhsmlDensityFactor = SphP[i].DhsmlDensityFactor;
+ HydroDataIn[nexport].Density = SphP[i].Density;
+ HydroDataIn[nexport].Pressure = SphP[i].Pressure;
+ HydroDataIn[nexport].Timestep = P[i].Ti_endstep - P[i].Ti_begstep;
+
+#ifdef WITH_ID_IN_HYDRA
+ HydroDataIn[nexport].ID = P[i].ID;
+#endif
+
+ /* calculation of F1 */
+ soundspeed_i = sqrt(GAMMA * SphP[i].Pressure / SphP[i].Density);
+ HydroDataIn[nexport].F1 = fabs(SphP[i].DivVel) /
+ (fabs(SphP[i].DivVel) + SphP[i].CurlVel +
+ 0.0001 * soundspeed_i / SphP[i].Hsml / fac_mu);
+
+ HydroDataIn[nexport].Index = i;
+ HydroDataIn[nexport].Task = j;
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+#ifdef MULTIPHASE
+ }
+#endif
+ }
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ qsort(HydroDataIn, nexport, sizeof(struct hydrodata_in), hydro_compare_key);
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeHydro)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&HydroDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct hydrodata_in), MPI_BYTE,
+ recvTask, TAG_HYDRO_A,
+ &HydroDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct hydrodata_in), MPI_BYTE,
+ recvTask, TAG_HYDRO_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ /* now do the imported particles */
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ hydro_evaluate(j, 1);
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ /* do a block to measure imbalance */
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeHydro)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&HydroDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct hydrodata_out),
+ MPI_BYTE, recvTask, TAG_HYDRO_B,
+ &HydroDataPartialResult[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct hydrodata_out),
+ MPI_BYTE, recvTask, TAG_HYDRO_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ source = j + noffset[recvTask];
+ place = HydroDataIn[source].Index;
+
+ for(k = 0; k < 3; k++)
+ SphP[place].HydroAccel[k] += HydroDataPartialResult[source].Acc[k];
+
+ SphP[place].DtEntropy += HydroDataPartialResult[source].DtEntropy;
+#ifdef FEEDBACK
+ SphP[place].DtEgySpecFeedback += HydroDataPartialResult[source].DtEgySpecFeedback;
+#endif
+ if(SphP[place].MaxSignalVel < HydroDataPartialResult[source].MaxSignalVel)
+ SphP[place].MaxSignalVel = HydroDataPartialResult[source].MaxSignalVel;
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+ SphP[place].VelocityDispersion[k] += HydroDataPartialResult[source].VelocityDispersion[k];
+#endif
+
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+
+ /* do final operations on results */
+ tstart = second();
+
+ for(i = 0; i < N_gas; i++)
+#ifdef SFR
+ if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+ {
+ SphP[i].DtEntropy *= GAMMA_MINUS1 / (hubble_a2 * pow(SphP[i].Density, GAMMA_MINUS1));
+
+#ifdef SPH_BND_PARTICLES
+ if(P[i].ID == 0)
+ {
+ SphP[i].DtEntropy = 0;
+ for(k = 0; k < 3; k++)
+ SphP[i].HydroAccel[k] = 0;
+ }
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ if (SphP[i].VelocityDispersion[0] != 0)
+ {
+ /* compute sigma r */
+
+ v1m = SphP[i].VelocityDispersion[1]/SphP[i].VelocityDispersion[0];
+ v2m = SphP[i].VelocityDispersion[2]/SphP[i].VelocityDispersion[0];
+
+ if (v2m > v1m*v1m)
+ SphP[i].OptVar = sqrt(v2m - v1m*v1m);
+ else
+ SphP[i].OptVar = 0.0;
+
+ }
+ else
+ SphP[i].OptVar = 0.0;
+#endif
+
+ }
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+
+
+
+ /* collect some timing information */
+
+ MPI_Reduce(&timecomp, &sumt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm, &sumcomm, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance, &sumimbalance, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ All.CPU_HydCompWalk += sumt / NTask;
+ All.CPU_HydCommSumm += sumcomm / NTask;
+ All.CPU_HydImbalance += sumimbalance / NTask;
+ }
+}
+
+
+/*! This function is the 'core' of the SPH force computation. A target
+ * particle is specified which may either be local, or reside in the
+ * communication buffer.
+ */
+void hydro_evaluate(int target, int mode)
+{
+ int j, k, n, timestep, startnode, numngb;
+ FLOAT *pos, *vel;
+ FLOAT mass, h_i, dhsmlDensityFactor, rho, pressure, f1, f2;
+ double acc[3], dtEntropy, maxSignalVel;
+ double dx, dy, dz, dvx, dvy, dvz;
+ double h_i2, hinv=1, hinv4;
+ double p_over_rho2_i, p_over_rho2_j, soundspeed_i, soundspeed_j;
+ double hfc, dwk_i, vdotr, vdotr2, visc, mu_ij, rho_ij=0, vsig;
+ double h_j, dwk_j, r, r2, u=0, hfc_visc;
+ int phase=0;
+#ifdef FEEDBACK
+ int EnergySN;
+ double wk,wk_i,wk_j,uintspec,hinv3;
+ double dtEgySpecFeedback=0;
+#endif
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ double VelocityDispersion[VELOCITY_DISPERSION_SIZE];
+ for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+ VelocityDispersion[k]=0.0;
+#endif
+
+#ifndef NOVISCOSITYLIMITER
+ double dt;
+#endif
+
+ if(mode == 0)
+ {
+ pos = P[target].Pos;
+ vel = SphP[target].VelPred;
+ h_i = SphP[target].Hsml;
+#ifdef FEEDBACK
+ EnergySN = SphP[target].EnergySN;
+#endif
+ mass = P[target].Mass;
+ dhsmlDensityFactor = SphP[target].DhsmlDensityFactor;
+ rho = SphP[target].Density;
+ pressure = SphP[target].Pressure;
+ timestep = P[target].Ti_endstep - P[target].Ti_begstep;
+ soundspeed_i = sqrt(GAMMA * pressure / rho);
+ f1 = fabs(SphP[target].DivVel) /
+ (fabs(SphP[target].DivVel) + SphP[target].CurlVel +
+ 0.0001 * soundspeed_i / SphP[target].Hsml / fac_mu);
+ }
+ else
+ {
+ pos = HydroDataGet[target].Pos;
+ vel = HydroDataGet[target].Vel;
+ h_i = HydroDataGet[target].Hsml;
+#ifdef FEEDBACK
+ EnergySN = HydroDataGet[target].EnergySN;
+#endif
+ mass = HydroDataGet[target].Mass;
+ dhsmlDensityFactor = HydroDataGet[target].DhsmlDensityFactor;
+ rho = HydroDataGet[target].Density;
+ pressure = HydroDataGet[target].Pressure;
+ timestep = HydroDataGet[target].Timestep;
+ soundspeed_i = sqrt(GAMMA * pressure / rho);
+ f1 = HydroDataGet[target].F1;
+ }
+
+
+ /* initialize variables before SPH loop is started */
+ acc[0] = acc[1] = acc[2] = dtEntropy = 0;
+ maxSignalVel = 0;
+#ifdef FEEDBACK
+ dtEgySpecFeedback=0;
+#endif
+
+ p_over_rho2_i = pressure / (rho * rho) * dhsmlDensityFactor;
+ h_i2 = h_i * h_i;
+
+ /* Now start the actual SPH computation for this particle */
+ startnode = All.MaxPart;
+ do
+ {
+ numngb = ngb_treefind_pairs(&pos[0], h_i, phase, &startnode);
+
+ for(n = 0; n < numngb; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+#ifdef PERIODIC /* find the closest image in the given box size */
+ if(dx > boxHalf_X)
+ dx -= boxSize_X;
+ if(dx < -boxHalf_X)
+ dx += boxSize_X;
+ if(dy > boxHalf_Y)
+ dy -= boxSize_Y;
+ if(dy < -boxHalf_Y)
+ dy += boxSize_Y;
+ if(dz > boxHalf_Z)
+ dz -= boxSize_Z;
+ if(dz < -boxHalf_Z)
+ dz += boxSize_Z;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+ h_j = SphP[j].Hsml;
+ if(r2 < h_i2 || r2 < h_j * h_j)
+ {
+ r = sqrt(r2);
+ if(r > 0)
+ {
+ p_over_rho2_j = SphP[j].Pressure / (SphP[j].Density * SphP[j].Density);
+ soundspeed_j = sqrt(GAMMA * p_over_rho2_j * SphP[j].Density);
+ dvx = vel[0] - SphP[j].VelPred[0];
+ dvy = vel[1] - SphP[j].VelPred[1];
+ dvz = vel[2] - SphP[j].VelPred[2];
+ vdotr = dx * dvx + dy * dvy + dz * dvz;
+
+ if(All.ComovingIntegrationOn)
+ vdotr2 = vdotr + hubble_a2 * r2;
+ else
+ vdotr2 = vdotr;
+
+ if(r2 < h_i2)
+ {
+ hinv = 1.0 / h_i;
+#ifndef TWODIMS
+ hinv4 = hinv * hinv * hinv * hinv;
+#else
+ hinv4 = hinv * hinv * hinv / boxSize_Z;
+#endif
+ u = r * hinv;
+ if(u < 0.5)
+ dwk_i = hinv4 * u * (KERNEL_COEFF_3 * u - KERNEL_COEFF_4);
+ else
+ dwk_i = hinv4 * KERNEL_COEFF_6 * (1.0 - u) * (1.0 - u);
+ }
+ else
+ {
+ dwk_i = 0;
+ }
+
+ if(r2 < h_j * h_j)
+ {
+ hinv = 1.0 / h_j;
+#ifndef TWODIMS
+ hinv4 = hinv * hinv * hinv * hinv;
+#else
+ hinv4 = hinv * hinv * hinv / boxSize_Z;
+#endif
+ u = r * hinv;
+ if(u < 0.5)
+ dwk_j = hinv4 * u * (KERNEL_COEFF_3 * u - KERNEL_COEFF_4);
+ else
+ dwk_j = hinv4 * KERNEL_COEFF_6 * (1.0 - u) * (1.0 - u);
+ }
+ else
+ {
+ dwk_j = 0;
+ }
+
+ if(soundspeed_i + soundspeed_j > maxSignalVel)
+ maxSignalVel = soundspeed_i + soundspeed_j;
+
+ if(vdotr2 < 0) /* ... artificial viscosity */
+ {
+ mu_ij = fac_mu * vdotr2 / r; /* note: this is negative! */
+
+ vsig = soundspeed_i + soundspeed_j - 3 * mu_ij;
+
+ if(vsig > maxSignalVel)
+ maxSignalVel = vsig;
+
+ rho_ij = 0.5 * (rho + SphP[j].Density);
+ f2 =
+ fabs(SphP[j].DivVel) / (fabs(SphP[j].DivVel) + SphP[j].CurlVel +
+ 0.0001 * soundspeed_j / fac_mu / SphP[j].Hsml);
+
+ visc = 0.25 * All.ArtBulkViscConst * vsig * (-mu_ij) / rho_ij * (f1 + f2);
+
+ /* .... end artificial viscosity evaluation */
+#ifndef NOVISCOSITYLIMITER
+ /* make sure that viscous acceleration is not too large */
+ dt = imax(timestep, (P[j].Ti_endstep - P[j].Ti_begstep)) * All.Timebase_interval;
+ if(dt > 0 && (dwk_i + dwk_j) < 0)
+ {
+ visc = dmin(visc, 0.5 * fac_vsic_fix * vdotr2 /
+ (0.5 * (mass + P[j].Mass) * (dwk_i + dwk_j) * r * dt));
+ }
+#endif
+ }
+ else
+ visc = 0;
+
+ p_over_rho2_j *= SphP[j].DhsmlDensityFactor;
+
+ hfc_visc = 0.5 * P[j].Mass * visc * (dwk_i + dwk_j) / r;
+
+ hfc = hfc_visc + P[j].Mass * (p_over_rho2_i * dwk_i + p_over_rho2_j * dwk_j) / r;
+
+ acc[0] -= hfc * dx;
+ acc[1] -= hfc * dy;
+ acc[2] -= hfc * dz;
+ dtEntropy += 0.5 * hfc_visc * vdotr2;
+
+
+ /*****************************************/
+ /* FEEDBACK INTERACTION */
+ /*****************************************/
+
+#ifdef FEEDBACK
+ rho_ij = 0.5 * (rho + SphP[j].Density);
+
+ if(P[j].Ti_endstep == All.Ti_Current)
+ {
+
+ /* additional feedback entropy */
+ if ((EnergySN > 0)||(SphP[j].EnergySN > 0))
+ {
+
+ /* find the thermal specific energy to release */
+ uintspec = 0.;
+
+ if (EnergySN > 0)
+ {
+ uintspec = 0;
+ }
+
+ if (SphP[j].EnergySN > 0)
+ {
+ uintspec += SphP[j].EnergySN * (1-All.SupernovaFractionInEgyKin)/ mass;
+ }
+
+ if(r2 < h_i2)
+ {
+ hinv = 1.0 / h_i;
+ hinv3 = hinv * hinv * hinv ;
+ u = r * hinv;
+
+ if(u < 0.5)
+ wk_i = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ else
+ wk_i = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ }
+ else
+ wk_i = 0;
+
+ if(r2 < h_j * h_j)
+ {
+ hinv = 1.0 / h_j;
+ hinv3 = hinv * hinv * hinv ;
+ u = r * hinv;
+
+ if(u < 0.5)
+ wk_j = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ else
+ wk_j = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ }
+ else
+ wk_j = 0;
+
+ wk = 0.5*(wk_i+wk_j);
+
+ /* dt in physical units */
+ dt = imax(timestep, (P[j].Ti_endstep - P[j].Ti_begstep)) * All.Timebase_interval/ hubble_a;
+ if(dt <= 0);
+ uintspec = 0;
+
+ /* thermal feedback */
+ uintspec = (uintspec/dt) *wk *0.5*(mass + P[j].Mass) /rho_ij *fac_pow ;
+ dtEntropy += uintspec;
+ dtEgySpecFeedback += uintspec;
+
+ }
+ }
+#endif /* FEEDBACK */
+
+
+
+ }
+ }
+ }
+ }
+ while(startnode >= 0);
+
+ /* Now collect the result at the right place */
+ if(mode == 0)
+ {
+ for(k = 0; k < 3; k++)
+ SphP[target].HydroAccel[k] = acc[k];
+ SphP[target].DtEntropy = dtEntropy;
+#ifdef FEEDBACK
+ SphP[target].DtEgySpecFeedback = dtEgySpecFeedback;
+#endif
+ SphP[target].MaxSignalVel = maxSignalVel;
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+ SphP[target].VelocityDispersion[k] = VelocityDispersion[k];
+#endif
+ }
+ else
+ {
+ for(k = 0; k < 3; k++)
+ HydroDataResult[target].Acc[k] = acc[k];
+ HydroDataResult[target].DtEntropy = dtEntropy;
+#ifdef FEEDBACK
+ HydroDataResult[target].DtEgySpecFeedback = dtEgySpecFeedback;
+#endif
+ HydroDataResult[target].MaxSignalVel = maxSignalVel;
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ for(k = 0; k < VELOCITY_DISPERSION_SIZE; k++)
+ HydroDataResult[target].VelocityDispersion[k] = VelocityDispersion[k];
+#endif
+ }
+}
+
+
+
+
+
+/*! This is a comparison kernel for a sort routine, which is used to group
+ * particles that are going to be exported to the same CPU.
+ */
+int hydro_compare_key(const void *a, const void *b)
+{
+ if(((struct hydrodata_in *) a)->Task < (((struct hydrodata_in *) b)->Task))
+ return -1;
+ if(((struct hydrodata_in *) a)->Task > (((struct hydrodata_in *) b)->Task))
+ return +1;
+ return 0;
+}
diff --git a/init.c b/init.c
new file mode 100644
index 0000000..296567a
--- /dev/null
+++ b/init.c
@@ -0,0 +1,576 @@
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file init.c
+ * \brief Code for initialisation of a simulation from initial conditions
+ */
+
+
+/*! This function reads the initial conditions, and allocates storage for the
+ * tree. Various variables of the particle data are initialised and An intial
+ * domain decomposition is performed. If SPH particles are present, the inial
+ * SPH smoothing lengths are determined.
+ */
+void init(void)
+{
+ int i, j;
+ double a3;
+#ifdef SFR
+ double Mgas=0,sum_Mgas=0;
+ int nstars=0;
+ int *numlist;
+#endif
+
+ All.Time = All.TimeBegin;
+
+ switch (All.ICFormat)
+ {
+ case 1:
+#if (MAKEGLASS > 1)
+ seed_glass();
+#else
+ read_ic(All.InitCondFile);
+#endif
+ break;
+ case 2:
+ case 3:
+ read_ic(All.InitCondFile);
+ break;
+ default:
+ if(ThisTask == 0)
+ printf("ICFormat=%d not supported.\n", All.ICFormat);
+ endrun(0);
+ }
+
+ All.Time = All.TimeBegin;
+ All.Ti_Current = 0;
+
+ if(All.ComovingIntegrationOn)
+ {
+ All.Timebase_interval = (log(All.TimeMax) - log(All.TimeBegin)) / TIMEBASE;
+ a3 = All.Time * All.Time * All.Time;
+ }
+ else
+ {
+ All.Timebase_interval = (All.TimeMax - All.TimeBegin) / TIMEBASE;
+ a3 = 1;
+ }
+
+
+
+ if (ThisTask==0)
+ printf("\nMinimum Time Step (Timebase_interval) = %g \n\n", All.Timebase_interval);
+
+
+
+
+ set_softenings();
+
+ All.NumCurrentTiStep = 0; /* setup some counters */
+ All.SnapshotFileCount = 0;
+ if(RestartFlag == 2)
+ All.SnapshotFileCount = atoi(All.InitCondFile + strlen(All.InitCondFile) - 3) + 1;
+
+ All.TotNumOfForces = 0;
+ All.NumForcesSinceLastDomainDecomp = 0;
+
+ if(All.ComovingIntegrationOn)
+ if(All.PeriodicBoundariesOn == 1)
+ check_omega();
+
+ All.TimeLastStatistics = All.TimeBegin - All.TimeBetStatistics;
+#ifdef AGN_ACCRETION
+ All.TimeLastAccretion = All.TimeBegin - All.TimeBetAccretion;
+ All.LastMTotInRa = 0;
+#endif
+#ifdef BONDI_ACCRETION
+ All.BondiTimeLast = All.TimeBegin - All.BondiTimeBet;
+#endif
+#ifdef BUBBLES
+ All.EnergyBubbles=0;
+#endif
+
+ if(All.ComovingIntegrationOn) /* change to new velocity variable */
+ {
+ for(i = 0; i < NumPart; i++)
+ for(j = 0; j < 3; j++)
+ P[i].Vel[j] *= sqrt(All.Time) * All.Time;
+ }
+
+ for(i = 0; i < NumPart; i++) /* start-up initialization */
+ {
+ for(j = 0; j < 3; j++)
+ P[i].GravAccel[j] = 0;
+#ifdef PMGRID
+ for(j = 0; j < 3; j++)
+ P[i].GravPM[j] = 0;
+#endif
+ P[i].Ti_endstep = 0;
+ P[i].Ti_begstep = 0;
+
+ P[i].OldAcc = 0;
+ P[i].GravCost = 1;
+ P[i].Potential = 0;
+#ifdef PARTICLE_FLAG
+ P[i].Flag = 0;
+#endif
+ }
+
+#ifdef PMGRID
+ All.PM_Ti_endstep = All.PM_Ti_begstep = 0;
+#endif
+
+#ifdef FLEXSTEPS
+ All.PresentMinStep = TIMEBASE;
+ for(i = 0; i < NumPart; i++) /* start-up initialization */
+ {
+ P[i].FlexStepGrp = (int) (TIMEBASE * get_random_number(P[i].ID));
+ }
+#endif
+
+
+ for(i = 0; i < N_gas; i++) /* initialize sph_properties */
+ {
+ for(j = 0; j < 3; j++)
+ {
+ SphP[i].VelPred[j] = P[i].Vel[j];
+ SphP[i].HydroAccel[j] = 0;
+ }
+
+ SphP[i].DtEntropy = 0;
+
+
+#ifdef STELLAR_FLUX
+ SphP[i].EnergyFlux = 0.;
+#endif
+#ifdef AGN_HEATING
+ SphP[i].EgySpecAGNHeat = 0.;
+ SphP[i].DtEgySpecAGNHeat = 0.;
+#endif
+#ifdef MULTIPHASE
+#ifdef COUNT_COLLISIONS
+ SphP[i].StickyCollisionNumber = 0;
+#endif
+#endif
+#ifdef FEEDBACK
+ SphP[i].EgySpecFeedback = 0.;
+ SphP[i].DtEgySpecFeedback = 0.;
+ SphP[i].EnergySN = 0.;
+ SphP[i].EnergySNrem = 0.;
+ SphP[i].TimeSN = 0.;
+ for(j = 0; j < 3; j++)
+ {
+ SphP[i].FeedbackVel[j] = 0;
+ }
+#endif
+#ifdef FEEDBACK_WIND
+ for(j = 0; j < 3; j++)
+ {
+ SphP[i].FeedbackWindVel[j] = 0;
+ }
+#endif
+
+
+#ifdef OUTPUTOPTVAR
+ SphP[i].OptVar = 0.;
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ for(j = 0; j < VELOCITY_DISPERSION_SIZE; j++)
+ SphP[i].VelocityDispersion[j] = 0.;
+#endif
+
+ if(RestartFlag == 0)
+ {
+ SphP[i].Hsml = 0;
+ SphP[i].Density = -1;
+ }
+
+#ifdef MULTIPHASE
+ /* here, we set the Phase, according to the SpecificEnergy and not Entropy */
+ if (SphP[i].Entropy > All.CriticalEgySpec)
+ SphP[i].Phase = GAS_SPH; /* warmer phase */
+ else
+ {
+ if (SphP[i].Entropy >= All.CriticalNonCollisionalEgySpec)
+ SphP[i].Phase = GAS_STICKY;
+ else
+ SphP[i].Phase = GAS_DARK;
+ }
+
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = 0;
+ //SphP[i].StickyTime = All.Time + All.StickyIdleTime*get_random_number(P[i].ID);
+#endif
+
+#ifdef SFR
+ Mgas += P[i].Mass;
+#endif
+
+ }
+
+
+
+
+#ifdef SFR
+ RearrangeParticlesFlag=0;
+
+ if (All.StarFormationStarMass==0)
+ {
+ /* compute the mean gas mass */
+ MPI_Allreduce(&Mgas, &sum_Mgas, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ All.StarFormationStarMass = (sum_Mgas/All.TotN_gas) / All.StarFormationNStarsFromGas;
+ }
+
+
+ for(i = 0; i < NumPart; i++) /* initialize st_properties */
+ {
+ if (P[i].Type==ST)
+ nstars++;
+#ifdef STELLAR_PROP
+ if (P[i].Type==ST)
+ {
+
+ P[i].StPIdx = i;
+
+ if (RestartFlag==0) /* only if starting from scratch */
+ {
+ StP[P[i].StPIdx].FormationTime = 0; /* bad */
+ StP[P[i].StPIdx].InitialMass = 0; /* bad */
+ StP[P[i].StPIdx].IDProj = P[i].ID;
+
+ StP[P[i].StPIdx].Hsml = 0;
+ StP[P[i].StPIdx].Density = -1;
+
+ for(j = 0; j < NELEMENTS; j++)
+ StP[P[i].StPIdx].Metal[j] = 0.;
+
+ StP[P[i].StPIdx].Flag = 0;
+ }
+
+ StP[P[i].StPIdx].PIdx = i;
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[i].StPIdx].ID = P[i].ID;
+#endif
+ }
+ //else
+ // P[i].StPIdx = -1; /* shoud be set, however, may be a problem in domain.c --> must be corrected */
+#endif
+
+
+#ifdef CHIMIE
+ if (P[i].Type==0)
+ {
+ if (RestartFlag==0 && header.flag_metals==0) /* only if starting from scratch and metal block not present */
+ {
+ for(j = 0; j < NELEMENTS; j++)
+ {
+ SphP[i].Metal[j] = (pow(10,All.InitGasMetallicity)-1e-10)*get_SolarAbundance(j);
+ //if (j==FE)
+ // SphP[i].Metal[j] = (pow(10,All.InitGasMetallicity)-1e-10)*All.CoolingParameters_FeHSolar;
+ //else
+ // SphP[i].Metal[j] = 0;
+ }
+ }
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ SphP[i].DeltaEgySpec = 0;
+ SphP[i].ThermalTime = All.TimeBegin-2*All.ChimieThermalTime;
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ SphP[i].WindTime = All.TimeBegin-2*All.ChimieWindTime;
+ SphP[i].WindFlag = 0;
+#endif
+ }
+
+#endif /* chimie */
+
+ }
+
+ /* here, we would like to reduce N_stars to TotN_stars */
+ /* MPI_Allreduce(&N_stars, &All.TotN_stars, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); does not works */
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&N_stars, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_stars = 0; i < NTask; i++)
+ All.TotN_stars += numlist[i];
+ free(numlist);
+
+
+ if(ThisTask == 0)
+ {
+ printf("Total number of star particles : %d%09d\n\n",(int) (All.TotN_stars / 1000000000), (int) (All.TotN_stars % 1000000000));
+ fflush(stdout);
+ }
+
+
+#endif /*SFR*/
+
+
+
+
+
+
+
+
+ ngb_treeallocate(MAX_NGB);
+
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+
+ Flag_FullStep = 1; /* to ensure that Peano-Hilber order is done */
+
+ domain_Decomposition(); /* do initial domain decomposition (gives equal numbers of particles) */
+
+ ngb_treebuild(); /* will build tree */
+
+ setup_smoothinglengths();
+
+#ifdef CHIMIE
+ stars_setup_smoothinglengths();
+#endif
+
+ TreeReconstructFlag = 1;
+
+
+
+ /* at this point, the entropy variable normally contains the
+ * internal energy, read in from the initial conditions file, unless the file
+ * explicitly signals that the initial conditions contain the entropy directly.
+ * Once the density has been computed, we can convert thermal energy to entropy.
+ */
+#ifndef ISOTHERM_EQS
+ if(header.flag_entropy_instead_u == 0)
+ {
+ for(i = 0; i < N_gas; i++)
+#ifdef MULTIPHASE
+ {
+ switch(SphP[i].Phase)
+ {
+ case GAS_SPH:
+ SphP[i].Entropy = GAMMA_MINUS1 * SphP[i].Entropy / pow(SphP[i].Density / a3, GAMMA_MINUS1);
+ break;
+
+ case GAS_STICKY:
+ break;
+
+ case GAS_DARK:
+ SphP[i].Entropy = -SphP[i].Entropy;
+ break;
+ }
+
+ }
+#else
+ SphP[i].Entropy = GAMMA_MINUS1 * SphP[i].Entropy / pow(SphP[i].Density / a3, GAMMA_MINUS1);
+#endif
+ }
+
+
+#endif
+
+
+#ifdef ENTROPYPRED
+ for(i = 0; i < N_gas; i++)
+ SphP[i].EntropyPred = SphP[i].Entropy;
+#endif
+
+
+}
+
+
+/*! This routine computes the mass content of the box and compares it to the
+ * specified value of Omega-matter. If discrepant, the run is terminated.
+ */
+void check_omega(void)
+{
+ double mass = 0, masstot, omega;
+ int i;
+
+ for(i = 0; i < NumPart; i++)
+ mass += P[i].Mass;
+
+ MPI_Allreduce(&mass, &masstot, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+ omega =
+ masstot / (All.BoxSize * All.BoxSize * All.BoxSize) / (3 * All.Hubble * All.Hubble / (8 * M_PI * All.G));
+
+ if(fabs(omega - All.Omega0) > 1.0e-3)
+ {
+ if(ThisTask == 0)
+ {
+ printf("\n\nI've found something odd!\n");
+ printf
+ ("The mass content accounts only for Omega=%g,\nbut you specified Omega=%g in the parameterfile.\n",
+ omega, All.Omega0);
+ printf("\nI better stop.\n");
+
+ fflush(stdout);
+ }
+ endrun(1);
+ }
+}
+
+
+
+/*! This function is used to find an initial smoothing length for each SPH
+ * particle. It guarantees that the number of neighbours will be between
+ * desired_ngb-MAXDEV and desired_ngb+MAXDEV. For simplicity, a first guess
+ * of the smoothing length is provided to the function density(), which will
+ * then iterate if needed to find the right smoothing length.
+ */
+void setup_smoothinglengths(void)
+{
+ int i, no, p;
+
+ if(RestartFlag == 0)
+ {
+
+ for(i = 0; i < N_gas; i++)
+ {
+ no = Father[i];
+
+ while(10 * All.DesNumNgb * P[i].Mass > Nodes[no].u.d.mass)
+ {
+ p = Nodes[no].u.d.father;
+
+ if(p < 0)
+ break;
+
+ no = p;
+ }
+#ifndef TWODIMS
+ SphP[i].Hsml =
+ pow(3.0 / (4 * M_PI) * All.DesNumNgb * P[i].Mass / Nodes[no].u.d.mass, 1.0 / 3) * Nodes[no].len;
+#else
+ SphP[i].Hsml =
+ pow(1.0 / (M_PI) * All.DesNumNgb * P[i].Mass / Nodes[no].u.d.mass, 1.0 / 2) * Nodes[no].len;
+#endif
+ }
+ }
+
+ density(0);
+}
+
+
+#ifdef CHIMIE
+/*! This function is used to find an initial smoothing length for each SPH
+ * particle. It guarantees that the number of neighbours will be between
+ * desired_ngb-MAXDEV and desired_ngb+MAXDEV. For simplicity, a first guess
+ * of the smoothing length is provided to the function density(), which will
+ * then iterate if needed to find the right smoothing length.
+ */
+void stars_setup_smoothinglengths(void)
+{
+ int i, no, p;
+
+ if(RestartFlag == 0)
+ {
+
+ for(i = 0; i < NumPart; i++)
+ {
+
+ if(P[i].Type == ST)
+ {
+
+ no = Father[i];
+
+ while(10 * All.DesNumNgb * P[i].Mass > Nodes[no].u.d.mass)
+ {
+ p = Nodes[no].u.d.father;
+
+ if(p < 0)
+ break;
+
+ no = p;
+ }
+#ifndef TWODIMS
+ StP[P[i].StPIdx].Hsml =
+ pow(3.0 / (4 * M_PI) * All.DesNumNgb * P[i].Mass / Nodes[no].u.d.mass, 1.0 / 3) * Nodes[no].len;
+#else
+ StP[P[i].StPIdx].Hsml =
+ pow(1.0 / (M_PI) * All.DesNumNgb * P[i].Mass / Nodes[no].u.d.mass, 1.0 / 2) * Nodes[no].len;
+#endif
+
+ }
+
+ }
+ }
+
+ stars_density();
+}
+#endif
+
+
+
+/*! If the code is run in glass-making mode, this function populates the
+ * simulation box with a Poisson sample of particles.
+ */
+#if (MAKEGLASS > 1)
+void seed_glass(void)
+{
+ int i, k, n_for_this_task;
+ double Range[3], LowerBound[3];
+ double drandom, partmass;
+ long long IDstart;
+
+ All.TotNumPart = MAKEGLASS;
+ partmass = All.Omega0 * (3 * All.Hubble * All.Hubble / (8 * M_PI * All.G))
+ * (All.BoxSize * All.BoxSize * All.BoxSize) / All.TotNumPart;
+
+ All.MaxPart = All.PartAllocFactor * (All.TotNumPart / NTask); /* sets the maximum number of particles that may */
+
+ allocate_memory();
+
+ header.npartTotal[1] = All.TotNumPart;
+ header.mass[1] = partmass;
+
+ if(ThisTask == 0)
+ {
+ printf("\nGlass initialising\nPartMass= %g\n", partmass);
+ printf("TotNumPart= %d%09d\n\n",
+ (int) (All.TotNumPart / 1000000000), (int) (All.TotNumPart % 1000000000));
+ }
+
+ /* set the number of particles assigned locally to this task */
+ n_for_this_task = All.TotNumPart / NTask;
+
+ if(ThisTask == NTask - 1)
+ n_for_this_task = All.TotNumPart - (NTask - 1) * n_for_this_task;
+
+ NumPart = 0;
+ IDstart = 1 + (All.TotNumPart / NTask) * ThisTask;
+
+ /* split the temporal domain into Ntask slabs in z-direction */
+
+ Range[0] = Range[1] = All.BoxSize;
+ Range[2] = All.BoxSize / NTask;
+ LowerBound[0] = LowerBound[1] = 0;
+ LowerBound[2] = ThisTask * Range[2];
+
+ srand48(ThisTask);
+
+ for(i = 0; i < n_for_this_task; i++)
+ {
+ for(k = 0; k < 3; k++)
+ {
+ drandom = drand48();
+
+ P[i].Pos[k] = LowerBound[k] + Range[k] * drandom;
+ P[i].Vel[k] = 0;
+ }
+
+ P[i].Mass = partmass;
+ P[i].Type = 1;
+ P[i].ID = IDstart + i;
+
+ NumPart++;
+ }
+}
+#endif
diff --git a/io.c b/io.c
new file mode 100644
index 0000000..d6cfd47
--- /dev/null
+++ b/io.c
@@ -0,0 +1,1610 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <errno.h>
+
+#ifdef HAVE_HDF5
+#include <hdf5.h>
+#endif
+
+#include "allvars.h"
+#include "proto.h"
+
+
+
+/*! \file io.c
+ * \brief Routines for producing a snapshot file on disk.
+ */
+
+static int n_type[6];
+static long long ntot_type_all[6];
+
+
+
+
+/*! This function writes a snapshot of the particle distribution to one or
+ * several files using the selected file format. If NumFilesPerSnapshot>1,
+ * the snapshot is distributed onto several files, several of them can be
+ * written simultaneously (up to NumFilesWrittenInParallel). Each file
+ * contains data from a group of processors.
+ */
+void savepositions(int num)
+{
+ double t0, t1;
+ char buf[500];
+ int i, j, *temp, n, filenr, gr, ngroups, masterTask, lastTask;
+
+ t0 = second();
+
+ if(ThisTask == 0)
+ printf("\nwriting snapshot file... \n");
+
+#if defined(SFR) || defined(BLACK_HOLES)
+ rearrange_particle_sequence();
+ /* ensures that new tree will be constructed */
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TreeDomainUpdateFrequency * All.TotNumPart;
+#endif
+
+ if(NTask < All.NumFilesPerSnapshot)
+ {
+ if(ThisTask == 0)
+ printf("Fatal error.\nNumber of processors must be larger or equal than All.NumFilesPerSnapshot.\n");
+ endrun(0);
+ }
+ if(All.SnapFormat < 1 || All.SnapFormat > 3)
+ {
+ if(ThisTask == 0)
+ printf("Unsupported File-Format\n");
+ endrun(0);
+ }
+#ifndef HAVE_HDF5
+ if(All.SnapFormat == 3)
+ {
+ if(ThisTask == 0)
+ printf("Code wasn't compiled with HDF5 support enabled!\n");
+ endrun(0);
+ }
+#endif
+
+
+ /* determine global and local particle numbers */
+ for(n = 0; n < 6; n++)
+ n_type[n] = 0;
+
+ for(n = 0; n < NumPart; n++)
+ n_type[P[n].Type]++;
+
+#ifdef CHECK_TYPE_DURING_IO
+ if (n_type[0]!=N_gas)
+ {
+ printf("(%d) n_type[0]=%d N_gas=%d !!!\n\n",ThisTask,n_type[0],N_gas);
+ endrun(111000);
+ }
+#ifdef SFR
+ if (n_type[ST]!=N_stars)
+ {
+ printf("(%d) n_type[ST]=%d N_gas=%d !!!\n\n",ThisTask,n_type[ST],N_stars);
+ endrun(111001);
+ }
+#endif
+#endif
+
+
+ /* because ntot_type_all[] is of type `long long', we cannot do a simple
+ * MPI_Allreduce() to sum the total particle numbers
+ */
+ temp = malloc(NTask * 6 * sizeof(int));
+ MPI_Allgather(n_type, 6, MPI_INT, temp, 6, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0; i < 6; i++)
+ {
+ ntot_type_all[i] = 0;
+ for(j = 0; j < NTask; j++)
+ ntot_type_all[i] += temp[j * 6 + i];
+ }
+ free(temp);
+
+
+ /* assign processors to output files */
+ distribute_file(All.NumFilesPerSnapshot, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask);
+
+ fill_Tab_IO_Labels();
+
+ if(All.NumFilesPerSnapshot > 1)
+ sprintf(buf, "%s%s_%04d.%d", All.OutputDir, All.SnapshotFileBase, num, filenr);
+ else
+ sprintf(buf, "%s%s_%04d", All.OutputDir, All.SnapshotFileBase, num);
+
+ ngroups = All.NumFilesPerSnapshot / All.NumFilesWrittenInParallel;
+ if((All.NumFilesPerSnapshot % All.NumFilesWrittenInParallel))
+ ngroups++;
+
+ for(gr = 0; gr < ngroups; gr++)
+ {
+ if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */
+ write_file(buf, masterTask, lastTask);
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+
+ if(ThisTask == 0)
+ printf("done with snapshot.\n");
+
+ t1 = second();
+
+ All.CPU_Snapshot += timediff(t0, t1);
+
+}
+
+
+
+/*! This function fills the write buffer with particle data. New output blocks
+ * can in principle be added here.
+ */
+void fill_write_buffer(enum iofields blocknr, int *startindex, int pc, int type)
+{
+ int n, k, pindex;
+ float *fp;
+
+#ifdef LONGIDS
+ long long *ip;
+#else
+ int *ip;
+#endif
+
+#ifdef PERIODIC
+ FLOAT boxSize;
+#endif
+#ifdef PMGRID
+ double dt_gravkick_pm = 0;
+#endif
+ double dt_gravkick, dt_hydrokick, a3inv = 1, fac1, fac2;
+
+
+ if(All.ComovingIntegrationOn)
+ {
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ fac1 = 1 / (All.Time * All.Time);
+ fac2 = 1 / pow(All.Time, 3 * GAMMA - 2);
+ }
+ else
+ a3inv = fac1 = fac2 = 1;
+
+#ifdef PMGRID
+ if(All.ComovingIntegrationOn)
+ dt_gravkick_pm =
+ get_gravkick_factor(All.PM_Ti_begstep,
+ All.Ti_Current) -
+ get_gravkick_factor(All.PM_Ti_begstep, (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2);
+ else
+ dt_gravkick_pm = (All.Ti_Current - (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2) * All.Timebase_interval;
+#endif
+
+
+
+ fp = CommBuffer;
+ ip = CommBuffer;
+
+ pindex = *startindex;
+
+ switch (blocknr)
+ {
+ case IO_POS: /* positions */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ for(k = 0; k < 3; k++)
+ {
+ fp[k] = P[pindex].Pos[k];
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+#ifdef LONG_X
+ if(k == 0)
+ boxSize = All.BoxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ if(k == 1)
+ boxSize = All.BoxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ if(k == 2)
+ boxSize = All.BoxSize * LONG_Z;
+#endif
+ while(fp[k] < 0)
+ fp[k] += boxSize;
+ while(fp[k] >= boxSize)
+ fp[k] -= boxSize;
+#endif
+ }
+ n++;
+ fp += 3;
+ }
+ break;
+
+ case IO_VEL: /* velocities */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ if(All.ComovingIntegrationOn)
+ {
+ dt_gravkick =
+ get_gravkick_factor(P[pindex].Ti_begstep,
+ All.Ti_Current) -
+ get_gravkick_factor(P[pindex].Ti_begstep,
+ (P[pindex].Ti_begstep + P[pindex].Ti_endstep) / 2);
+ dt_hydrokick =
+ get_hydrokick_factor(P[pindex].Ti_begstep,
+ All.Ti_Current) -
+ get_hydrokick_factor(P[pindex].Ti_begstep,
+ (P[pindex].Ti_begstep + P[pindex].Ti_endstep) / 2);
+ }
+ else
+ dt_gravkick = dt_hydrokick =
+ (All.Ti_Current - (P[pindex].Ti_begstep + P[pindex].Ti_endstep) / 2) * All.Timebase_interval;
+
+ for(k = 0; k < 3; k++)
+ {
+ fp[k] = P[pindex].Vel[k] + P[pindex].GravAccel[k] * dt_gravkick;
+ if(P[pindex].Type == 0)
+ {
+ fp[k] += SphP[pindex].HydroAccel[k] * dt_hydrokick;
+ }
+ }
+#ifdef PMGRID
+ for(k = 0; k < 3; k++)
+ fp[k] += P[pindex].GravPM[k] * dt_gravkick_pm;
+#endif
+ for(k = 0; k < 3; k++)
+ fp[k] *= sqrt(a3inv);
+
+ n++;
+ fp += 3;
+ }
+ break;
+
+ case IO_ID: /* particle ID */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *ip++ = P[pindex].ID;
+ n++;
+ }
+ break;
+
+ case IO_MASS: /* particle mass */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = P[pindex].Mass;
+ n++;
+ }
+ break;
+
+ case IO_U: /* internal energy */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+#ifdef ISOTHERM_EQS
+ *fp++ = SphP[pindex].Entropy;
+#else
+#ifdef MULTIPHASE
+ switch(SphP[pindex].Phase)
+ {
+ case GAS_SPH:
+ *fp++ = dmax(All.MinEgySpec,SphP[pindex].Entropy / GAMMA_MINUS1 * pow(SphP[pindex].Density * a3inv, GAMMA_MINUS1));
+ break;
+ case GAS_STICKY:
+ *fp++ = SphP[pindex].Entropy;
+ break;
+ case GAS_DARK:
+ *fp++ = -SphP[pindex].Entropy;
+ break;
+ }
+#else
+ *fp++ =
+ dmax(All.MinEgySpec,
+ SphP[pindex].Entropy / GAMMA_MINUS1 * pow(SphP[pindex].Density * a3inv, GAMMA_MINUS1));
+#endif
+#endif
+ n++;
+ }
+ break;
+
+ case IO_RHO: /* density */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].Density;
+ n++;
+ }
+ break;
+
+ case IO_HSML: /* SPH smoothing length */
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].Hsml;
+ n++;
+ }
+ break;
+
+ /*********************************************/
+ /* here it is the end of the minimal output */
+ /*********************************************/
+
+
+ case IO_STAR_FORMATIONTIME: /* stellar formation time */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = StP[P[pindex].StPIdx].FormationTime;
+ n++;
+ }
+#endif
+ break;
+
+
+ case IO_INITIAL_MASS: /* stellar formation time */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = StP[P[pindex].StPIdx].InitialMass;
+ n++;
+ }
+#endif
+ break;
+
+
+ case IO_STAR_IDPROJ: /* stellar projenitor */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *ip++ = StP[P[pindex].StPIdx].IDProj;
+ n++;
+ }
+#endif
+ break;
+
+
+ case IO_STAR_RHO: /* gas density around a star */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = StP[P[pindex].StPIdx].Density;
+ n++;
+ }
+#endif
+ break;
+
+
+ case IO_STAR_HSML: /* SPH smoothing length of a star */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = StP[P[pindex].StPIdx].Hsml;
+ n++;
+ }
+#endif
+ break;
+
+
+
+ case IO_STAR_METALS: /* stars metal */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ for(k = 0; k < NELEMENTS; k++)
+ {
+ fp[k] = StP[P[pindex].StPIdx].Metal[k];
+ }
+ n++;
+ fp += NELEMENTS;
+ }
+#endif
+ break;
+
+
+
+ case IO_METALS: /* gas metal */
+#ifdef OUTPUTSTELLAR_PROP
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ for(k = 0; k < NELEMENTS; k++)
+ {
+ fp[k] = SphP[pindex].Metal[k];
+ }
+ n++;
+ fp += NELEMENTS;
+ }
+#endif
+ break;
+
+
+
+
+
+
+
+
+
+ case IO_POT: /* gravitational potential */
+#ifdef OUTPUTPOTENTIAL
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = P[pindex].Potential;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_ACCEL: /* acceleration */
+#ifdef OUTPUTACCELERATION
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ for(k = 0; k < 3; k++)
+ fp[k] = fac1 * P[pindex].GravAccel[k];
+#ifdef PMGRID
+ for(k = 0; k < 3; k++)
+ fp[k] += fac1 * P[pindex].GravPM[k];
+#endif
+ if(P[pindex].Type == 0)
+ for(k = 0; k < 3; k++)
+ {
+ fp[k] += fac2 * SphP[pindex].HydroAccel[k];
+ }
+ fp += 3;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_DTENTR: /* rate of change of entropy */
+#ifdef OUTPUTCHANGEOFENTROPY
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].DtEntropy;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_TSTP: /* timestep */
+#ifdef OUTPUTTIMESTEP
+
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = (P[pindex].Ti_endstep - P[pindex].Ti_begstep) * All.Timebase_interval;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_ERADSTICKY: /* sticky radiated energy */
+#ifdef OUTPUTERADSTICKY
+ /* obsolete */
+#endif
+ break;
+
+ case IO_ERADFEEDBACK: /* feedback injected energy */
+#ifdef OUTPUTERADFEEDBACK
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].EgySpecFeedback;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_ENERGYFLUX: /* energyflux */
+#ifdef OUTPUTENERGYFLUX
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].EnergyFlux;
+ n++;
+ }
+#endif
+ break;
+
+ case IO_OPTVAR: /* optional variable */
+#ifdef OUTPUTOPTVAR
+ for(n = 0; n < pc; pindex++)
+ if(P[pindex].Type == type)
+ {
+ *fp++ = SphP[pindex].OptVar;
+ n++;
+ }
+#endif
+ break;
+
+
+ }
+
+ *startindex = pindex;
+}
+
+
+
+
+/*! This function tells the size of one data entry in each of the blocks
+ * defined for the output file. If one wants to add a new output-block, this
+ * function should be augmented accordingly.
+ */
+int get_bytes_per_blockelement(enum iofields blocknr)
+{
+ int bytes_per_blockelement = 0;
+
+ switch (blocknr)
+ {
+ case IO_POS:
+ case IO_VEL:
+ case IO_ACCEL:
+ bytes_per_blockelement = 3 * sizeof(float);
+ break;
+
+ case IO_ID:
+#ifdef LONGIDS
+ bytes_per_blockelement = sizeof(long long);
+#else
+ bytes_per_blockelement = sizeof(int);
+#endif
+ break;
+
+ case IO_MASS:
+ case IO_U:
+ case IO_RHO:
+ case IO_HSML:
+ case IO_POT:
+ case IO_DTENTR:
+ case IO_TSTP:
+ case IO_ERADSPH:
+ case IO_ERADSTICKY:
+ case IO_ERADFEEDBACK:
+ case IO_ENERGYFLUX:
+ case IO_OPTVAR:
+ bytes_per_blockelement = sizeof(float);
+ break;
+
+
+ case IO_STAR_FORMATIONTIME:
+ case IO_INITIAL_MASS:
+ case IO_STAR_RHO:
+ case IO_STAR_HSML:
+ bytes_per_blockelement = sizeof(float);
+ break;
+
+ case IO_METALS:
+#ifdef CHIMIE
+ case IO_STAR_METALS:
+ bytes_per_blockelement = NELEMENTS*sizeof(float);
+#endif
+ break;
+
+ case IO_STAR_IDPROJ:
+#ifdef LONGIDS
+ bytes_per_blockelement = sizeof(long long);
+#else
+ bytes_per_blockelement = sizeof(int);
+#endif
+ break;
+
+
+
+
+ }
+
+ return bytes_per_blockelement;
+}
+
+
+/*! This function returns the type of the data contained in a given block of
+ * the output file. If one wants to add a new output-block, this function
+ * should be augmented accordingly.
+ */
+int get_datatype_in_block(enum iofields blocknr)
+{
+ int typekey;
+
+ switch (blocknr)
+ {
+ case IO_ID:
+ case IO_STAR_IDPROJ:
+#ifdef LONGIDS
+ typekey = 2; /* native long long */
+#else
+ typekey = 0; /* native int */
+#endif
+ break;
+
+ default:
+ typekey = 1; /* native float */
+ break;
+ }
+
+ return typekey;
+}
+
+
+/*! This function informs about the number of elements stored per particle for
+ * the given block of the output file. If one wants to add a new
+ * output-block, this function should be augmented accordingly.
+ */
+int get_values_per_blockelement(enum iofields blocknr)
+{
+ int values = 0;
+
+ switch (blocknr)
+ {
+ case IO_POS:
+ case IO_VEL:
+ case IO_ACCEL:
+ values = 3;
+ break;
+
+ case IO_ID:
+ case IO_MASS:
+ case IO_U:
+ case IO_RHO:
+ case IO_HSML:
+ case IO_POT:
+ case IO_DTENTR:
+ case IO_TSTP:
+ case IO_ERADSPH:
+ case IO_ERADSTICKY:
+ case IO_ERADFEEDBACK:
+ case IO_ENERGYFLUX:
+ case IO_OPTVAR:
+ case IO_STAR_FORMATIONTIME:
+ case IO_INITIAL_MASS:
+ case IO_STAR_IDPROJ:
+ case IO_STAR_RHO:
+ case IO_STAR_HSML:
+ values = 1;
+ break;
+
+ case IO_METALS:
+#ifdef CHIMIE
+ case IO_STAR_METALS:
+ values = NELEMENTS;
+#endif
+ break;
+
+ }
+
+ return values;
+}
+
+
+/*! This function determines how many particles there are in a given block,
+ * based on the information in the header-structure. It also flags particle
+ * types that are present in the block in the typelist array. If one wants to
+ * add a new output-block, this function should be augmented accordingly.
+ */
+int get_particles_in_block(enum iofields blocknr, int *typelist)
+{
+ int i, nall, ntot_withmasses, ngas, nstars;
+
+ nall = 0;
+ ntot_withmasses = 0;
+
+ for(i = 0; i < 6; i++)
+ {
+ typelist[i] = 0;
+
+ if(header.npart[i] > 0)
+ {
+ nall += header.npart[i];
+ typelist[i] = 1;
+ }
+
+ if(All.MassTable[i] == 0)
+ ntot_withmasses += header.npart[i];
+ }
+
+ ngas = header.npart[0];
+#if defined(SFR) || defined(STELLAR_PROP)
+ nstars = header.npart[ST];
+#else
+ nstars = header.npart[4];
+#endif
+
+ switch (blocknr)
+ {
+ case IO_POS:
+ case IO_VEL:
+ case IO_ACCEL:
+ case IO_TSTP:
+ case IO_ID:
+ case IO_POT:
+ return nall;
+ break;
+
+ case IO_MASS:
+ for(i = 0; i < 6; i++)
+ {
+ typelist[i] = 0;
+ if(All.MassTable[i] == 0 && header.npart[i] > 0)
+ typelist[i] = 1;
+ }
+ return ntot_withmasses;
+ break;
+
+ case IO_U:
+ case IO_RHO:
+ case IO_HSML:
+ case IO_DTENTR:
+ case IO_ERADSPH:
+ case IO_ERADSTICKY:
+ case IO_ERADFEEDBACK:
+ case IO_ENERGYFLUX:
+ case IO_OPTVAR:
+ case IO_METALS:
+ for(i = 1; i < 6; i++)
+ typelist[i] = 0;
+ return ngas;
+ break;
+
+
+ case IO_STAR_FORMATIONTIME:
+ case IO_INITIAL_MASS:
+ case IO_STAR_IDPROJ:
+ case IO_STAR_RHO:
+ case IO_STAR_HSML:
+#if defined(STELLAR_PROP)
+ case IO_STAR_METALS:
+ for(i = 0; i < 6; i++)
+ {
+ typelist[i] = 0;
+ if(i == ST)
+ typelist[i] = 1;
+ }
+ return nstars;
+#endif
+ break;
+ }
+
+ endrun(212);
+ return 0;
+}
+
+
+
+/*! This function tells whether or not a given block in the output file is
+ * present, depending on the type of simulation run and the compile-time
+ * options. If one wants to add a new output-block, this function should be
+ * augmented accordingly.
+ */
+int blockpresent(enum iofields blocknr)
+{
+
+#ifndef OUTPUTPOTENTIAL
+ if(blocknr == IO_POT)
+ return 0;
+#endif
+
+#ifndef OUTPUTACCELERATION
+ if(blocknr == IO_ACCEL)
+ return 0;
+#endif
+
+#ifndef OUTPUTCHANGEOFENTROPY
+ if(blocknr == IO_DTENTR)
+ return 0;
+#endif
+
+#ifndef OUTPUTTIMESTEP
+ if(blocknr == IO_TSTP)
+ return 0;
+#endif
+
+#ifndef OUTPUTERADSPH
+ if(blocknr == IO_ERADSPH)
+ return 0;
+#endif
+
+#ifndef OUTPUTERADSTICKY
+ if(blocknr == IO_ERADSTICKY)
+ return 0;
+#endif
+
+#ifndef OUTPUTERADFEEDBACK
+ if(blocknr == IO_ERADFEEDBACK)
+ return 0;
+#endif
+
+#ifndef OUTPUTENERGYFLUX
+ if(blocknr == IO_ENERGYFLUX)
+ return 0;
+#endif
+
+#ifndef OUTPUTOPTVAR
+ if(blocknr == IO_OPTVAR)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_STAR_FORMATIONTIME)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_INITIAL_MASS)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_STAR_IDPROJ)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_STAR_RHO)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_STAR_HSML)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_STAR_METALS)
+ return 0;
+#endif
+
+#ifndef OUTPUTSTELLAR_PROP
+ if(blocknr == IO_METALS)
+ return 0;
+#endif
+
+ return 1; /* default: present */
+}
+
+
+#ifdef BLOCK_SKIPPING
+/*! This function tells whether or not a given block in the output file is
+ * present, depending on the type of simulation run and the compile-time
+ * options. If one wants to add a new output-block, this function should be
+ * augmented accordingly.
+ */
+int blockabsent(enum iofields blocknr)
+{
+
+ /* here we will for exampe read the gas initial metallicity if needed */
+
+#ifdef CHIMIE
+ if(RestartFlag == 0 && header.flag_metals)
+ if(blocknr > IO_METALS) /* read in addition IO_RHO,IO_HSML and IO_METALS */
+ return 1;
+ else
+ return 0;
+#endif
+
+
+ if(RestartFlag == 0 && blocknr > IO_U)
+ return 1; /* ignore all other blocks in initial conditions */
+
+ return 0; /* default: present */
+}
+#endif
+
+
+
+/*! This function associates a short 4-character block name with each block
+ * number. This is stored in front of each block for snapshot
+ * FileFormat=2. If one wants to add a new output-block, this function should
+ * be augmented accordingly.
+ */
+void fill_Tab_IO_Labels(void)
+{
+ enum iofields i;
+
+ for(i = 0; i < IO_NBLOCKS; i++)
+ switch (i)
+ {
+ case IO_POS:
+ strncpy(Tab_IO_Labels[IO_POS], "POS ", 4);
+ break;
+ case IO_VEL:
+ strncpy(Tab_IO_Labels[IO_VEL], "VEL ", 4);
+ break;
+ case IO_ID:
+ strncpy(Tab_IO_Labels[IO_ID], "ID ", 4);
+ break;
+ case IO_MASS:
+ strncpy(Tab_IO_Labels[IO_MASS], "MASS", 4);
+ break;
+ case IO_U:
+ strncpy(Tab_IO_Labels[IO_U], "U ", 4);
+ break;
+ case IO_RHO:
+ strncpy(Tab_IO_Labels[IO_RHO], "RHO ", 4);
+ break;
+ case IO_HSML:
+ strncpy(Tab_IO_Labels[IO_HSML], "HSML", 4);
+ break;
+ case IO_POT:
+ strncpy(Tab_IO_Labels[IO_POT], "POT ", 4);
+ break;
+ case IO_ACCEL:
+ strncpy(Tab_IO_Labels[IO_ACCEL], "ACCE", 4);
+ break;
+ case IO_DTENTR:
+ strncpy(Tab_IO_Labels[IO_DTENTR], "ENDT", 4);
+ break;
+ case IO_TSTP:
+ strncpy(Tab_IO_Labels[IO_TSTP], "TSTP", 4);
+ break;
+ case IO_ERADSPH:
+ strncpy(Tab_IO_Labels[IO_ERADSPH], "ERADSPH", 4);
+ break;
+ case IO_ERADSTICKY:
+ strncpy(Tab_IO_Labels[IO_ERADSTICKY], "ERADSTICKY", 4);
+ break;
+ case IO_ERADFEEDBACK:
+ strncpy(Tab_IO_Labels[IO_ERADFEEDBACK], "ERADFEEDBACK", 4);
+ break;
+ case IO_ENERGYFLUX:
+ strncpy(Tab_IO_Labels[IO_ENERGYFLUX], "ENERGYFLUX", 4);
+ break;
+ case IO_OPTVAR:
+ strncpy(Tab_IO_Labels[IO_OPTVAR], "OPTVAR", 4);
+ break;
+ case IO_STAR_FORMATIONTIME:
+ strncpy(Tab_IO_Labels[IO_STAR_FORMATIONTIME], "STAR_FORMATIONTIME", 4);
+ break;
+ case IO_INITIAL_MASS:
+ strncpy(Tab_IO_Labels[IO_INITIAL_MASS], "INITIAL_MASS", 4);
+ break;
+ case IO_STAR_IDPROJ:
+ strncpy(Tab_IO_Labels[IO_STAR_IDPROJ], "STAR_IDPROJ", 4);
+ break;
+ case IO_STAR_RHO:
+ strncpy(Tab_IO_Labels[IO_STAR_RHO], "STAR_RHO", 4);
+ break;
+ case IO_STAR_HSML:
+ strncpy(Tab_IO_Labels[IO_STAR_HSML], "STAR_HSML", 4);
+ break;
+ case IO_STAR_METALS:
+ strncpy(Tab_IO_Labels[IO_STAR_METALS], "STAR_METALS", 4);
+ break;
+ case IO_METALS:
+ strncpy(Tab_IO_Labels[IO_METALS], "METALS", 4);
+ break;
+
+ }
+}
+
+/*! This function returns a descriptive character string that describes the
+ * name of the block when the HDF5 file format is used. If one wants to add
+ * a new output-block, this function should be augmented accordingly.
+ */
+void get_dataset_name(enum iofields blocknr, char *buf)
+{
+
+ strcpy(buf, "default");
+
+ switch (blocknr)
+ {
+ case IO_POS:
+ strcpy(buf, "Coordinates");
+ break;
+ case IO_VEL:
+ strcpy(buf, "Velocities");
+ break;
+ case IO_ID:
+ strcpy(buf, "ParticleIDs");
+ break;
+ case IO_MASS:
+ strcpy(buf, "Masses");
+ break;
+ case IO_U:
+ strcpy(buf, "InternalEnergy");
+ break;
+ case IO_RHO:
+ strcpy(buf, "Density");
+ break;
+ case IO_HSML:
+ strcpy(buf, "SmoothingLength");
+ break;
+ case IO_POT:
+ strcpy(buf, "Potential");
+ break;
+ case IO_ACCEL:
+ strcpy(buf, "Acceleration");
+ break;
+ case IO_DTENTR:
+ strcpy(buf, "RateOfChangeOfEntropy");
+ break;
+ case IO_TSTP:
+ strcpy(buf, "TimeStep");
+ break;
+ case IO_ERADSPH:
+ strcpy(buf, "EnergyRadiatedSph");
+ break;
+ case IO_ERADSTICKY:
+ strcpy(buf, "EnergyRadiatedSticky");
+ break;
+ case IO_ERADFEEDBACK:
+ strcpy(buf, "EnergyRadiatedFeedback");
+ break;
+ case IO_ENERGYFLUX:
+ strcpy(buf, "EnergyFlux");
+ break;
+ case IO_OPTVAR:
+ strcpy(buf, "OptVar");
+ break;
+ case IO_STAR_FORMATIONTIME:
+ strcpy(buf, "StarFormationTime");
+ break;
+ case IO_INITIAL_MASS:
+ strcpy(buf, "InitialMass");
+ break;
+ case IO_STAR_IDPROJ:
+ strcpy(buf, "StarIDProj");
+ break;
+ case IO_STAR_RHO:
+ strcpy(buf, "StarRho");
+ break;
+ case IO_STAR_HSML:
+ strcpy(buf, "StarHsml");
+ break;
+ case IO_STAR_METALS:
+ strcpy(buf, "StarMetals");
+ break;
+ case IO_METALS:
+ strcpy(buf, "Metals");
+ break;
+
+ }
+}
+
+
+
+/*! This function writes an actual snapshot file containing the data from
+ * processors 'writeTask' to 'lastTask'. 'writeTask' is the one that actually
+ * writes. Each snapshot file contains a header first, then particle
+ * positions, velocities and ID's. Particle masses are written only for
+ * those particle types with zero entry in MassTable. After that, first the
+ * internal energies u, and then the density is written for the SPH
+ * particles. If cooling is enabled, mean molecular weight and neutral
+ * hydrogen abundance are written for the gas particles. This is followed by
+ * the SPH smoothing length and further blocks of information, depending on
+ * included physics and compile-time flags. If HDF5 is used, the header is
+ * stored in a group called "/Header", and the particle data is stored
+ * separately for each particle type in groups calles "/PartType0",
+ * "/PartType1", etc. The sequence of the blocks is unimportant in this case.
+ */
+void write_file(char *fname, int writeTask, int lastTask)
+{
+ int type, bytes_per_blockelement, npart, nextblock, typelist[6];
+ int n_for_this_task, ntask, n, p, pc, offset = 0, task;
+ int blockmaxlen, ntot_type[6], nn[6];
+ enum iofields blocknr;
+ int blksize;
+ int i;
+ MPI_Status status;
+ FILE *fd = 0;
+
+#ifdef HAVE_HDF5
+ hid_t hdf5_file = 0, hdf5_grp[6], hdf5_headergrp = 0, hdf5_dataspace_memory;
+ hid_t hdf5_datatype = 0, hdf5_dataspace_in_file = 0, hdf5_dataset = 0;
+ herr_t hdf5_status;
+ hsize_t dims[2], count[2], start[2];
+ int rank, pcsum = 0;
+ char buf[500];
+#endif
+
+#define SKIP {my_fwrite(&blksize,sizeof(int),1,fd);}
+
+ /* determine particle numbers of each type in file */
+
+ if(ThisTask == writeTask)
+ {
+ for(n = 0; n < 6; n++)
+ ntot_type[n] = n_type[n];
+
+ for(task = writeTask + 1; task <= lastTask; task++)
+ {
+ MPI_Recv(&nn[0], 6, MPI_INT, task, TAG_LOCALN, MPI_COMM_WORLD, &status);
+ for(n = 0; n < 6; n++)
+ ntot_type[n] += nn[n];
+ }
+
+ for(task = writeTask + 1; task <= lastTask; task++)
+ MPI_Send(&ntot_type[0], 6, MPI_INT, task, TAG_N, MPI_COMM_WORLD);
+ }
+ else
+ {
+ MPI_Send(&n_type[0], 6, MPI_INT, writeTask, TAG_LOCALN, MPI_COMM_WORLD);
+ MPI_Recv(&ntot_type[0], 6, MPI_INT, writeTask, TAG_N, MPI_COMM_WORLD, &status);
+ }
+
+
+
+ /* fill file header */
+
+ for(n = 0; n < 6; n++)
+ {
+ header.npart[n] = ntot_type[n];
+ header.npartTotal[n] = (unsigned int) ntot_type_all[n];
+ header.npartTotalHighWord[n] = (unsigned int) (ntot_type_all[n] >> 32);
+ }
+
+ for(n = 0; n < 6; n++)
+ header.mass[n] = All.MassTable[n];
+
+ header.time = All.Time;
+
+ if(All.ComovingIntegrationOn)
+ header.redshift = 1.0 / All.Time - 1;
+ else
+ header.redshift = 0;
+
+ header.flag_sfr = 0;
+ header.flag_feedback = 0;
+ header.flag_cooling = 0;
+ header.flag_stellarage = 0;
+ header.flag_metals = 0;
+
+
+#ifdef MULTIPHASE
+ header.critical_energy_spec = All.CriticalEgySpec;
+#endif
+
+#ifdef COOLING
+ header.flag_cooling = 1;
+#endif
+
+#ifdef SFR
+ header.flag_sfr = 1;
+#ifdef OUTPUTSTELLAR_PROP
+ header.flag_stellarage = 1;
+ header.flag_metals = NELEMENTS;
+#endif
+
+#ifdef FEEDBACK
+ header.flag_feedback = 1;
+#endif
+
+#endif
+
+
+ header.num_files = All.NumFilesPerSnapshot;
+ header.BoxSize = All.BoxSize;
+ header.Omega0 = All.Omega0;
+ header.OmegaLambda = All.OmegaLambda;
+ header.HubbleParam = All.HubbleParam;
+
+ /* set fill to " " : yr Thu Aug 13 17:34:07 CEST 2009*/
+ memset(header.fill,' ',sizeof(header.fill));
+
+
+ /* fill file chimie-header */
+ header.flag_chimie_extraheader = 0;
+
+#ifdef CHIMIE_EXTRAHEADER
+ header.flag_chimie_extraheader = 1;
+ chimie_extraheader.nelts = get_nelts();
+ for (i=0;i<get_nelts();i++)
+ {
+ chimie_extraheader.SolarAbundances[i]=get_SolarAbundance(i);
+ }
+
+ memset(chimie_extraheader.labels,' ',sizeof(chimie_extraheader.labels));
+ for (i=0,n=0;i<get_nelts();i++)
+ {
+ strcpy(&chimie_extraheader.labels[n],get_Element(i));
+ n+= strlen(get_Element(i));
+ strncpy(&chimie_extraheader.labels[n++],",",1);
+ }
+
+#endif
+
+ /* open file and write header */
+
+ if(ThisTask == writeTask)
+ {
+ if(All.SnapFormat == 3)
+ {
+#ifdef HAVE_HDF5
+ sprintf(buf, "%s.hdf5", fname);
+ hdf5_file = H5Fcreate(buf, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+ hdf5_headergrp = H5Gcreate(hdf5_file, "/Header", 0);
+
+ for(type = 0; type < 6; type++)
+ {
+ if(header.npart[type] > 0)
+ {
+ sprintf(buf, "/PartType%d", type);
+ hdf5_grp[type] = H5Gcreate(hdf5_file, buf, 0);
+ }
+ }
+
+ write_header_attributes_in_hdf5(hdf5_headergrp);
+#endif
+ }
+ else
+ {
+ if(!(fd = fopen(fname, "w")))
+ {
+ printf("can't open file `%s' for writing snapshot.\n", fname);
+ endrun(123);
+ }
+
+ if(All.SnapFormat == 2)
+ {
+ blksize = sizeof(int) + 4 * sizeof(char);
+ SKIP;
+ my_fwrite("HEAD", sizeof(char), 4, fd);
+ nextblock = sizeof(header) + 2 * sizeof(int);
+ my_fwrite(&nextblock, sizeof(int), 1, fd);
+ SKIP;
+ }
+
+ blksize = sizeof(header);
+ SKIP;
+ my_fwrite(&header, sizeof(header), 1, fd);
+ SKIP;
+
+#ifdef CHIMIE_EXTRAHEADER
+ blksize = sizeof(chimie_extraheader);
+ SKIP;
+ my_fwrite(&chimie_extraheader, sizeof(chimie_extraheader), 1, fd);
+ SKIP;
+#endif
+ }
+ }
+
+ ntask = lastTask - writeTask + 1;
+
+ for(blocknr = 0; blocknr < IO_NBLOCKS; blocknr++)
+ {
+ if(blockpresent(blocknr))
+ {
+ bytes_per_blockelement = get_bytes_per_blockelement(blocknr);
+
+ blockmaxlen = ((int) (All.BufferSize * 1024 * 1024)) / bytes_per_blockelement;
+
+ npart = get_particles_in_block(blocknr, &typelist[0]);
+
+ if(npart > 0)
+ {
+ if(ThisTask == writeTask)
+ {
+
+ if(All.SnapFormat == 1 || All.SnapFormat == 2)
+ {
+ if(All.SnapFormat == 2)
+ {
+ blksize = sizeof(int) + 4 * sizeof(char);
+ SKIP;
+ my_fwrite(Tab_IO_Labels[blocknr], sizeof(char), 4, fd);
+ nextblock = npart * bytes_per_blockelement + 2 * sizeof(int);
+ my_fwrite(&nextblock, sizeof(int), 1, fd);
+ SKIP;
+ }
+
+ blksize = npart * bytes_per_blockelement;
+ SKIP;
+
+ }
+ }
+
+ for(type = 0; type < 6; type++)
+ {
+ if(typelist[type])
+ {
+#ifdef HAVE_HDF5
+ if(ThisTask == writeTask && All.SnapFormat == 3 && header.npart[type] > 0)
+ {
+ switch (get_datatype_in_block(blocknr))
+ {
+ case 0:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_UINT);
+ break;
+ case 1:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_FLOAT);
+ break;
+ case 2:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_UINT64);
+ break;
+ }
+
+ dims[0] = header.npart[type];
+ dims[1] = get_values_per_blockelement(blocknr);
+ if(dims[1] == 1)
+ rank = 1;
+ else
+ rank = 2;
+
+ get_dataset_name(blocknr, buf);
+
+ hdf5_dataspace_in_file = H5Screate_simple(rank, dims, NULL);
+ hdf5_dataset =
+ H5Dcreate(hdf5_grp[type], buf, hdf5_datatype, hdf5_dataspace_in_file,
+ H5P_DEFAULT);
+ pcsum = 0;
+ }
+#endif
+
+ for(task = writeTask, offset = 0; task <= lastTask; task++)
+ {
+ if(task == ThisTask)
+ {
+ n_for_this_task = n_type[type];
+
+ for(p = writeTask; p <= lastTask; p++)
+ if(p != ThisTask)
+ MPI_Send(&n_for_this_task, 1, MPI_INT, p, TAG_NFORTHISTASK, MPI_COMM_WORLD);
+ }
+ else
+ MPI_Recv(&n_for_this_task, 1, MPI_INT, task, TAG_NFORTHISTASK, MPI_COMM_WORLD,
+ &status);
+
+ while(n_for_this_task > 0)
+ {
+ pc = n_for_this_task;
+
+ if(pc > blockmaxlen)
+ pc = blockmaxlen;
+
+ if(ThisTask == task)
+ fill_write_buffer(blocknr, &offset, pc, type);
+
+ if(ThisTask == writeTask && task != writeTask)
+ MPI_Recv(CommBuffer, bytes_per_blockelement * pc, MPI_BYTE, task,
+ TAG_PDATA, MPI_COMM_WORLD, &status);
+
+ if(ThisTask != writeTask && task == ThisTask)
+ MPI_Ssend(CommBuffer, bytes_per_blockelement * pc, MPI_BYTE, writeTask,
+ TAG_PDATA, MPI_COMM_WORLD);
+
+ if(ThisTask == writeTask)
+ {
+ if(All.SnapFormat == 3)
+ {
+#ifdef HAVE_HDF5
+ start[0] = pcsum;
+ start[1] = 0;
+
+ count[0] = pc;
+ count[1] = get_values_per_blockelement(blocknr);
+ pcsum += pc;
+
+ H5Sselect_hyperslab(hdf5_dataspace_in_file, H5S_SELECT_SET,
+ start, NULL, count, NULL);
+
+ dims[0] = pc;
+ dims[1] = get_values_per_blockelement(blocknr);
+ hdf5_dataspace_memory = H5Screate_simple(rank, dims, NULL);
+
+ hdf5_status =
+ H5Dwrite(hdf5_dataset, hdf5_datatype, hdf5_dataspace_memory,
+ hdf5_dataspace_in_file, H5P_DEFAULT, CommBuffer);
+
+ H5Sclose(hdf5_dataspace_memory);
+#endif
+ }
+ else
+ my_fwrite(CommBuffer, bytes_per_blockelement, pc, fd);
+ }
+
+ n_for_this_task -= pc;
+ }
+ }
+
+#ifdef HAVE_HDF5
+ if(ThisTask == writeTask && All.SnapFormat == 3 && header.npart[type] > 0)
+ {
+ if(All.SnapFormat == 3)
+ {
+ H5Dclose(hdf5_dataset);
+ H5Sclose(hdf5_dataspace_in_file);
+ H5Tclose(hdf5_datatype);
+ }
+ }
+#endif
+ }
+ }
+
+ if(ThisTask == writeTask)
+ {
+ if(All.SnapFormat == 1 || All.SnapFormat == 2)
+ SKIP;
+ }
+ }
+ }
+ }
+
+ if(ThisTask == writeTask)
+ {
+ if(All.SnapFormat == 3)
+ {
+#ifdef HAVE_HDF5
+ for(type = 5; type >= 0; type--)
+ if(header.npart[type] > 0)
+ H5Gclose(hdf5_grp[type]);
+ H5Gclose(hdf5_headergrp);
+ H5Fclose(hdf5_file);
+#endif
+ }
+ else
+ fclose(fd);
+ }
+}
+
+
+
+
+/*! This function writes the header information in case HDF5 is selected as
+ * file format.
+ */
+#ifdef HAVE_HDF5
+void write_header_attributes_in_hdf5(hid_t handle)
+{
+ hsize_t adim[1] = { 6 };
+ hid_t hdf5_dataspace, hdf5_attribute;
+
+ hdf5_dataspace = H5Screate(H5S_SIMPLE);
+ H5Sset_extent_simple(hdf5_dataspace, 1, adim, NULL);
+ hdf5_attribute = H5Acreate(handle, "NumPart_ThisFile", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_UINT, header.npart);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SIMPLE);
+ H5Sset_extent_simple(hdf5_dataspace, 1, adim, NULL);
+ hdf5_attribute = H5Acreate(handle, "NumPart_Total", H5T_NATIVE_UINT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_UINT, header.npartTotal);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SIMPLE);
+ H5Sset_extent_simple(hdf5_dataspace, 1, adim, NULL);
+ hdf5_attribute = H5Acreate(handle, "NumPart_Total_HW", H5T_NATIVE_UINT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_UINT, header.npartTotalHighWord);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+
+ hdf5_dataspace = H5Screate(H5S_SIMPLE);
+ H5Sset_extent_simple(hdf5_dataspace, 1, adim, NULL);
+ hdf5_attribute = H5Acreate(handle, "MassTable", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, header.mass);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Time", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.time);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Redshift", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.redshift);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "BoxSize", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.BoxSize);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "NumFilesPerSnapshot", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.num_files);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Omega0", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.Omega0);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "OmegaLambda", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.OmegaLambda);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "HubbleParam", H5T_NATIVE_DOUBLE, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.HubbleParam);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Flag_Sfr", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.flag_sfr);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Flag_Cooling", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.flag_cooling);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Flag_StellarAge", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.flag_stellarage);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Flag_Metals", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.flag_metals);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ hdf5_dataspace = H5Screate(H5S_SCALAR);
+ hdf5_attribute = H5Acreate(handle, "Flag_Feedback", H5T_NATIVE_INT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_INT, &header.flag_feedback);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+
+ header.flag_entropy_instead_u = 0;
+
+ hdf5_dataspace = H5Screate(H5S_SIMPLE);
+ H5Sset_extent_simple(hdf5_dataspace, 1, adim, NULL);
+ hdf5_attribute = H5Acreate(handle, "Flag_Entropy_ICs", H5T_NATIVE_UINT, hdf5_dataspace, H5P_DEFAULT);
+ H5Awrite(hdf5_attribute, H5T_NATIVE_UINT, header.flag_entropy_instead_u);
+ H5Aclose(hdf5_attribute);
+ H5Sclose(hdf5_dataspace);
+}
+#endif
+
+
+
+
+
+/*! This catches I/O errors occuring for my_fwrite(). In this case we
+ * better stop.
+ */
+size_t my_fwrite(void *ptr, size_t size, size_t nmemb, FILE * stream)
+{
+ size_t nwritten;
+
+ if((nwritten = fwrite(ptr, size, nmemb, stream)) != nmemb)
+ {
+ printf("I/O error (fwrite) on task=%d has occured: %s\n", ThisTask, strerror(errno));
+ fflush(stdout);
+ endrun(777);
+ }
+ return nwritten;
+}
+
+
+/*! This catches I/O errors occuring for fread(). In this case we
+ * better stop.
+ */
+size_t my_fread(void *ptr, size_t size, size_t nmemb, FILE * stream)
+{
+ size_t nread;
+
+ if((nread = fread(ptr, size, nmemb, stream)) != nmemb)
+ {
+ printf("I/O error (fread) on task=%d has occured: %s\n", ThisTask, strerror(errno));
+ fflush(stdout);
+ endrun(778);
+ }
+ return nread;
+}
diff --git a/longrange.c b/longrange.c
new file mode 100644
index 0000000..16322c2
--- /dev/null
+++ b/longrange.c
@@ -0,0 +1,144 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file longrange.c
+ * \brief driver routines for computation of long-range gravitational PM force
+ */
+
+#ifdef PMGRID
+
+/*! Calls initializiation routines of periodic or/and non-periodic FFT
+ * routines.
+ */
+void long_range_init(void)
+{
+#ifdef PERIODIC
+ pm_init_periodic();
+#ifdef PLACEHIGHRESREGION
+ pm_init_nonperiodic();
+#endif
+#else
+ pm_init_nonperiodic();
+#endif
+}
+
+
+/*! This function calls subroutines that determine the spatial region covered
+ * by the PM mesh.
+ */
+void long_range_init_regionsize(void)
+{
+#ifdef PERIODIC
+#ifdef PLACEHIGHRESREGION
+ if(RestartFlag != 1)
+ pm_init_regionsize();
+ pm_setup_nonperiodic_kernel();
+#endif
+#else
+ if(RestartFlag != 1)
+ pm_init_regionsize();
+ pm_setup_nonperiodic_kernel();
+#endif
+}
+
+
+/*! This function is a driver routine for the long-range PM force
+ * computation. It calls periodic and/or non-periodic FFT routines as needed
+ * for the present simulation set-up.
+ */
+void long_range_force(void)
+{
+ int i;
+
+#ifndef PERIODIC
+ int j;
+ double fac;
+#endif
+
+
+ for(i = 0; i < NumPart; i++)
+ P[i].GravPM[0] = P[i].GravPM[1] = P[i].GravPM[2] = 0;
+
+#ifdef NOGRAVITY
+ return;
+#endif
+
+
+#ifdef PERIODIC
+ pmforce_periodic();
+#ifdef PLACEHIGHRESREGION
+ i = pmforce_nonperiodic(1);
+ if(i == 1) /* this is returned if a particle lied outside allowed range */
+ {
+ pm_init_regionsize();
+ pm_setup_nonperiodic_kernel();
+ i = pmforce_nonperiodic(1); /* try again */
+ }
+ if(i == 1)
+ endrun(68686);
+#endif
+#else
+ i = pmforce_nonperiodic(0);
+ if(i == 1) /* this is returned if a particle lied outside allowed range */
+ {
+ pm_init_regionsize();
+ pm_setup_nonperiodic_kernel();
+ i = pmforce_nonperiodic(0); /* try again */
+ }
+ if(i == 1)
+ endrun(68687);
+#ifdef PLACEHIGHRESREGION
+ i = pmforce_nonperiodic(1);
+ if(i == 1) /* this is returned if a particle lied outside allowed range */
+ {
+ pm_init_regionsize();
+ pm_setup_nonperiodic_kernel();
+
+ /* try again */
+
+ for(i = 0; i < NumPart; i++)
+ P[i].GravPM[0] = P[i].GravPM[1] = P[i].GravPM[2] = 0;
+
+ i = pmforce_nonperiodic(0) + pmforce_nonperiodic(1);
+ }
+ if(i != 0)
+ endrun(68688);
+#endif
+#endif
+
+
+#ifndef PERIODIC
+ if(All.ComovingIntegrationOn)
+ {
+ fac = 0.5 * All.Hubble * All.Hubble * All.Omega0;
+
+ for(i = 0; i < NumPart; i++)
+ for(j = 0; j < 3; j++)
+ P[i].GravPM[j] += fac * P[i].Pos[j];
+ }
+
+
+ /* Finally, the following factor allows a computation of cosmological simulation
+ with vacuum energy in physical coordinates */
+
+ if(All.ComovingIntegrationOn == 0)
+ {
+ fac = All.OmegaLambda * All.Hubble * All.Hubble;
+
+ for(i = 0; i < NumPart; i++)
+ for(j = 0; j < 3; j++)
+ P[i].GravPM[j] += fac * P[i].Pos[j];
+ }
+#endif
+
+}
+
+
+#endif
diff --git a/main.c b/main.c
new file mode 100644
index 0000000..2c46daa
--- /dev/null
+++ b/main.c
@@ -0,0 +1,874 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file main.c
+ * \brief start of the program
+ */
+
+/*!
+ * This function initializes the MPI communication packages, and sets
+ * cpu-time counters to 0. Then begrun() is called, which sets up
+ * the simulation either from IC's or from restart files. Finally,
+ * run() is started, the main simulation loop, which iterates over
+ * the timesteps.
+ */
+int main(int argc, char **argv)
+{
+ double t0, t1;
+
+ MPI_Init(&argc, &argv);
+ MPI_Comm_rank(MPI_COMM_WORLD, &ThisTask);
+ MPI_Comm_size(MPI_COMM_WORLD, &NTask);
+
+ if(NTask <= 1)
+ {
+ if(ThisTask == 0)
+ printf
+ ("Note: This is a massively parallel code, but you are running with 1 processor only.\nCompared to an equivalent serial code, there is some unnecessary overhead.\n");
+ }
+
+ for(PTask = 0; NTask > (1 << PTask); PTask++);
+
+ if(argc < 2)
+ {
+ if(ThisTask == 0)
+ {
+ printf("Parameters are missing.\n");
+ printf("Call with <ParameterFile> [<RestartFlag>]\n");
+ }
+ endrun(0);
+ }
+
+ strcpy(ParameterFile, argv[1]);
+
+ if(argc >= 3)
+ RestartFlag = atoi(argv[2]);
+ else
+ RestartFlag = 0;
+
+ All.CPU_TreeConstruction = All.CPU_TreeWalk = All.CPU_Gravity = All.CPU_Potential = All.CPU_Domain =
+ All.CPU_Snapshot = All.CPU_Total = All.CPU_CommSum = All.CPU_Imbalance = All.CPU_Hydro =
+ All.CPU_HydCompWalk = All.CPU_HydCommSumm = All.CPU_HydImbalance =
+ All.CPU_EnsureNgb = All.CPU_Predict = All.CPU_TimeLine = All.CPU_PM = All.CPU_Peano = 0;
+
+#ifdef COOLING
+ All.CPU_Cooling = 0;
+#endif
+#ifdef SFR
+ All.CPU_StarFormation = 0;
+#endif
+#ifdef CHIMIE
+ All.CPU_Chimie = 0;
+#endif
+#ifdef MULTIPHASE
+ All.CPU_Sticky = 0;
+#endif
+
+#ifdef DETAILED_CPU_DOMAIN
+ All.CPU_Domain_findExtend = 0;
+ All.CPU_Domain_determineTopTree = 0;
+ All.CPU_Domain_sumCost = 0;
+ All.CPU_Domain_findSplit = 0;
+ All.CPU_Domain_shiftSplit = 0;
+ All.CPU_Domain_countToGo = 0;
+ All.CPU_Domain_exchange = 0;
+#endif
+
+#ifdef DETAILED_CPU_GRAVITY
+ All.CPU_Gravity_TreeWalk1 = 0;
+ All.CPU_Gravity_TreeWalk2 = 0;
+ All.CPU_Gravity_CommSum1 = 0;
+ All.CPU_Gravity_CommSum2 = 0;
+ All.CPU_Gravity_Imbalance1 = 0;
+ All.CPU_Gravity_Imbalance2 = 0;
+#endif
+
+#ifdef DETAILED_CPU
+ All.CPU_Leapfrog = 0;
+ All.CPU_Physics = 0;
+ All.CPU_Residual = 0;
+ All.CPU_Accel = 0;
+ All.CPU_Begrun = 0;
+#endif
+
+ CPUThisRun = 0;
+
+ t0 = second();
+
+ begrun(); /* set-up run */
+
+ t1 = second();
+ CPUThisRun += timediff(t0, t1);
+ All.CPU_Total += timediff(t0, t1);
+
+ run(); /* main simulation loop */
+
+ MPI_Finalize(); /* clean up & finalize MPI */
+
+ return 0;
+}
+
+
+
+
+/* ----------------------------------------------------------------------
+ The rest of this file contains documentation for compiling and
+ running the code, in a format appropriate for 'doxygen'.
+ ----------------------------------------------------------------------
+ */
+
+/*! \mainpage Reference documentation for GADGET-2
+
+\author Volker Springel \n
+ Max-Planck-Institute for Astrophysics \n
+ Karl-Schwarzschild-Str. 1 \n
+ 85740 Garching \n
+ Germany \n
+ volker@mpa-garching.mpg.de \n
+
+\n
+
+\section prelim Getting started
+
+GADGET-2 is a massively parallel code for hydrodynamical cosmological
+simulations. It is a flexible code that can be applied to a variety of
+different types of simulations, offering a number of sophisticated
+simulation algorithms.
+
+A full account of the numerical algorithms employed by the code is given
+in the accompanying code paper, and detailed instructions for usage of
+the code are given in the included code documentation.
+
+This html-document serves as a cross-referenced documentation of the
+source code itself - in fact, using the doxygen tool, the html-pages
+have been produced from comments inlined in the source code. Apart from
+the source-code documentation, a brief guide to code compilation is
+given below, and under <b>Related Pages (see link on top)</b> you can find an
+explanation of GADGET's parameterfile and a short guide to compile-time
+options of the code.
+
+
+\section install Compilation
+
+GADGET-2 needs the following non-standard libraries for compilation:
+
+- \b MPI - the Message Passing Interface (version 1.0 or higher). Many
+ vendor supplied versions exist, in addition to excellent open source
+ implementations, e.g. MPICH
+ (http://www-unix.mcs.anl.gov/mpi/mpich/) or LAM
+ (http://www.lam-mpi.org/).
+
+- \b GSL - the GNU scientific library. This open-source package can be
+ obtained at http://www.gnu.org/software/gsl , for example. GADGET-2
+ needs this library for a few simple cosmological
+ integrations at start-up, and for random number generation.
+
+- \b HDF5 - the <em>Hierarchical Data Format</em>. This library has been
+ developed by NCSA and can be obtained at http://hdf.ncsa.uiuc.edu/HDF5 .
+ GADGET-2 can be compiled without this library, but then the HDF5 format
+ is not supported.
+
+- \b FFTW - the <em>Fastest Fourier Transform in the West</em>. This
+ open-source package can be obtained at http://www.fftw.org . It is only
+ needed for simulations that use the TreePM algorithm. Note that the
+ MPI-capable version 2.x of FFTW is required, and that FFTW needs to be
+ explicitly compiled with parallel support enabled. This can be achieved
+ by passing the option <b>--enable-mpi</b> to the configure script. When
+ at it, you might as well add <b>--enable-type-prefix</b> to obtain the
+ libraries in both a single and double precision version. If this has not
+ been done, you should set the option <em>NOTYPEPREFIX_FFTW</em> in GADGET's
+ \ref Gadget-Makefile "Makefile".
+
+Note that if any of the above libraries is not installed in standard
+locations on your system, the \ref Gadget-Makefile "Makefile" provided with
+the code may need slight adjustments. Similarly, compiler options,
+particularly with respect to optimisations, may need adjustment to the
+C-compiler that is used. Finally, the \ref Gadget-Makefile "Makefile"
+contains a number of compile-time options that need to be set appropriately
+for the type of simulation that is simulated.
+
+The provided makefile is compatible with GNU-make, i.e. typing \b make or
+\b gmake should then build the executable <b>Gadget2</b>. If your site
+does not have GNU-make, get it, or write your own makefile.
+
+\section howtorun Running the code
+
+In order to start the simulation code, a \ref parameterfile "parameterfile"
+needs to be specified. An additional optional numerical parameter can be
+used to signal whether a continuation from a set of restart files, or from
+a snapshot file, is desired. A typical command to start the code looks like
+the following: \n \n
+
+ <b> mpirun -np 8 ./Gadget2 <parameterfile> [restartflag]</b> \n \n
+
+This would start the code using 8 processors, assuming that the parallel
+environment uses the <em>mpirun</em> command to start MPI
+applications. Depending on the operating system, other commands may be
+required for this task, e.g. <em>poe</em> on IBM/AIX machines. Note that
+the code can in principle be started using an arbitrary number of
+processors, but the communication algorithms will be most efficient for
+powers of 2. It is also possible to use a single processor only, in
+which case the code behaves like a serial code, except that GADGET-2
+will still go through some of the overhead induced by the
+parallelization algorithms, so the code will not quite reach the same
+performance as an optimum serial solution in this case.
+
+
+The optional <em>restartflag</em> can have the values 0, 1, or 2, only. "1"
+signals a continuation from restart files, while "2" can be used to restart
+from a snapshot file produced by the code. If omitted (equivalent to the
+default of "0"), the code starts from initial conditions.
+
+*/
+
+
+
+
+
+
+
+
+
+
+
+/*! \page parameterfile Parameterfile of GADGET-2
+
+The parameterfile for GADGET-2 is a simple text file, consisting of pairs of
+tags and values. For each parameter, a separate line needs to be specified,
+first listing the name (tag) of the parameter, and then the assigned value,
+separated by whitespace. It is allowed to add further text behind the
+assigned parameter value. The order of the parameters is arbitrary, but
+each one needs to occur exactly one time, otherwise an error message will
+be produced. Empty lines, or lines beginning with a \%-sign, are ignored and
+treated as comments.
+
+
+- \b InitCondFile \n The filename of the initial conditions file. If a
+ restart from a snapshot with the "2" option is desired, one
+ needs to specify the snapshot file here.
+
+- \b OutputDir \n Pathname of the output directory of the code.
+
+- \b EnergyFile \n Filename of the log-file that contain the energy
+ statistics.
+
+- \b InfoFile \n Log-file that contains a list of the timesteps taken.
+
+- \b TimingsFile \n Log-file with performance metrics of the gravitational
+ tree computation.
+
+- \b CpuFile \n Log-file with CPU time consumption in various parts of the
+ code.
+
+- \b RestartFile \n Basename of restart-files produced by the code.
+
+- \b SnapshotFileBase \n Basename of snapshot files produced by the code.
+
+- \b OutputListFilename \n File with a list of the desired output times.
+
+- \b TimeLimitCPU \n CPU-time limit for the present submission of the
+ code. If 85 percent of this time have been reached at the end
+ of a timestep, the code terminates itself and produces restart
+ files.
+
+- \b ResubmitOn \n If set to "1", the code will try to resubmit itself to
+ the queuing system when an interruption of the run due to the
+ CPU-time limit occurs. The resubmission itself is done by
+ executing the program/script given with
+ <em>ResubmitCommand</em>.
+
+- \b ResubmitCommand \n The name of a script file or program that is
+ executed for automatic resubmission of the job to the queuing
+ system. Note that the file given here needs to be executable.
+
+- \b ICFormat \n The file format of the initial conditions. Currently,
+ three different formats are supported, selected by one of the
+ choices "1", "2", or "3". Format "1" is the traditional
+ fortran-style unformatted format familiar from
+ GADGET-1. Format "2" is a variant of this format, where each
+ block of data is preceeded by a 4-character
+ block-identifier. Finally, format "3" selects the HDF-5
+ format.
+
+- \b SnapFormat \n Similar as <em>ICFormat</em>, this parameter selects the
+ file-format of snapshot dumps produced by the code.
+
+- \b ComovingIntegrationOn \n If set to "1", the code assumes that a
+ cosmological integration in comoving coordinates is carried
+ out, otherwise ordinary Newtonian dynamics is assumed.
+
+
+- \b TypeOfTimestepCriterion \n This parameter can in principle be used to
+ select different kinds of timestep criteria for gravitational
+ dynamics. However, GADGET-2 presently only supports the
+ standard criterion "0".
+
+- \b OutputListOn \n If set to "1", the code tries to read a list of
+ desired output times from the file given in
+ <em>OutputListFilename</em>. Otherwise, output times are
+ generated equally spaced from the values assigned for
+ <em>TimeOfFirstSnapshot</em> and <em>TimeBetSnapshot</em>.
+
+- \b PeriodicBoundariesOn \n If set to "1", periodic boundary conditions
+ are assumed, with a cubical box-size of side-length
+ <em>BoxSize</em>. Particle coordinates are expected to be in
+ the range [0,<em>BoxSize</em>[.
+
+
+- \b TimeBegin \n This sets the starting time of a simulation when the code
+ is started from initial conditions. For cosmological
+ integrations, the value specified here is taken as the initial
+ scale factor.
+
+- \b TimeMax \n This sets the final time for the simulation. The code
+ normally tries to run until this time is reached. For
+ cosmological integrations, the value given here is the final
+ scale factor.
+
+- \b Omega0 \n Gives the total matter density (in units of the critical
+ density) at z=0 for cosmological simulations.
+
+- \b OmegaLambda \n Gives the vacuum energy density at z=0 for cosmological
+ simulations.
+
+- \b OmegaBaryon \n Gives the baryon density at z=0 for cosmological
+ simulations.
+
+- \b HubbleParam \n This gives the Hubble constant at z=0 in units of 100
+ km/sec/Mpc. Note that this parameter has been basically
+ absorbed into the definition of the internal code units, such
+ that for gravitational dynamics and adiabatic gas dynamics the
+ actual value assigned for <em>HubbleParam</em> is not used by
+ the code.
+
+- \b BoxSize \n The boxsize for simulations with periodic boundary
+ conditions.
+
+- \b TimeOfFirstSnapshot \n The time of the first desired snapshot file in
+ case a file with output times is not specified. For
+ cosmological simulations, the value given here is the scale factor
+ of the first desired output.
+
+- \b TimeBetSnapshot \n The time interval between two subsequent snapshot
+ files in case a file with output times is not specified. For
+ cosmological simulations, this is a multiplicative factor
+ applied to the time of the last snapshot, such that the
+ snapshots will have a constant logarithmic spacing in the
+ scale factor. Otherwise, the parameter is an additive constant
+ that gives the linear spacing between snapshot times.
+
+- \b CpuTimeBetRestartFile \n The value specfied here gives the time in
+ seconds the code will run before it writes regularly produced
+ restart files. This can be useful to protect against
+ unexpected interruptions (for example due to a hardware
+ problem) of a simulation, particularly if it is run for a long
+ time. It is then possible to resume a simulation from the last
+ restart file, reducing the potential loss to the elapsed
+ CPU-time since this was produced.
+
+- \b TimeBetStatistics \n The code can be asked to measure the total
+ kinetic, thermal, and potential energy in regular intervals,
+ and to write the results to the file given in
+ <em>EnergyFile</em>. The time interval between two such
+ measurements is given by the parameter
+ <em>TimeBetStatistics</em>, in an analogous way as with
+ <em>TimeBetSnapshot</em>. Note that the compile time option
+ <em>COMPUTE_POTENTIAL_ENERGY</em> needs to be activated to
+ obtain a measurement of the gravitational potential energy.
+
+- \b NumFilesPerSnapshot \n The number of separate files requested for each
+ snapshot dump. Each file of the snapshot will hold the data of
+ one or several processors, up to all of
+ them. <em>NumFilesPerSnapshot</em> must hence lie between 1
+ and the number of processors used. Distributing a snapshot
+ onto several files can be done in parallel and may lead to
+ much better I/O performance, depending on the hardware
+ configuration. It can also help to avoid problems due to big
+ files (>2GB) for large simulations. Note that initial
+ conditions may also be distributed into several files, the
+ number of which is automatically recognised by the code and
+ does not have to be equal to <em>NumFilesPerSnapshot</em> (it
+ may also be larger than the number of processors).
+
+
+- \b NumFilesWrittenInParallel \n The number of files the code may read or
+ write simultaneously when writing or reading snapshot/restart
+ files. The value of this parameter must be smaller or equal to
+ the number of processors.
+
+
+- \b ErrTolIntAccuracy \n This dimensionless parameter controls the
+ accuracy of the timestep criterion selected by
+ <em>TypeOfTimestepCriterion</em>.
+
+- \b CourantFac \n This sets the value of the Courant parameter used in the
+ determination of the hydrodynamical timestep of SPH particles.
+
+- \b MaxSizeTimestep \n This gives the maximum timestep a particle may
+ take. This should be set to a sensible value in order to
+ protect against too large timesteps for particles with very
+ small acceleration. For cosmological simulations, the
+ parameter given here is the maximum allowed step in the
+ logarithm of the expansion factor. Note that the definition
+ of MaxSizeTimestep has <em>changed</em> compared to Gadget-1.1 for cosmological simulations.
+
+
+- \b MinSizeTimestep \n If a particle requests a timestep smaller than the
+ value specified here, the code will normally terminate with a
+ warning message. If compiled with the
+ <em>NOSTOP_WHEN_BELOW_MINTIMESTEP</em> option, the code will
+ instead force the timesteps to be at least as large as
+ <em>MinSizeTimestep</em>.
+
+- \b TypeOfOpeningCriterion \n This selects the type of cell-opening
+ criterion used in the tree walks. A value of `0' results in
+ standard Barnes & Hut, while `1' selects the relative opening
+ criterion of GADGET-2.
+
+- \b ErrTolTheta \n This gives the maximum opening angle if the BH
+ criterion is used for the tree walk. If the relative opening
+ criterion is used instead, a first force estimate is computed
+ using the BH algorithm, which is then recomputed with the
+ relative opening criterion.
+
+- \b ErrTolForceAcc \n The accuracy parameter for the relative opening
+ criterion for the tree walk.
+
+- \b TreeDomainUpdateFrequency \n The domain decomposition and tree
+ construction need not necessarily be done every single
+ timestep. Instead, tree nodes can be dynamically updated,
+ which is faster. However, the tree walk will become more
+ expensive since the tree nodes have to "grow" to keep
+ accomodating all particles they enclose. The parameter
+ <em>TreeDomainUpdateFrequency</em> controls how often the
+ domain decomposition is carried out and the tree is
+ reconstructed from scratch. For example, a value of 0.1 means
+ that the domain decomposition and the tree are reconstructed
+ whenever there have been more than 0.1*N force computations
+ since the last reconstruction, where N is the total particle
+ number. A value of 0 will reconstruct the tree every timestep.
+
+- \b MaxRMSDisplacementFac \n This parameter is an additional timestep
+ criterion for the long-range integration in case the TreePM
+ algorithm is used. It limits the long-range timestep such that
+ the rms-displacement of particles per step is at most
+ <em>MaxRMSDisplacementFac</em> times the mean
+ particle separation, or the mesh-scale, whichever is smaller.
+
+- \b DesNumNgb \n This sets the desired number of SPH smoothing neighbours.
+
+- \b MaxNumNgbDeviation \n This sets the allowed variation of the number of
+ neighbours around the target value <em>DesNumNgb</em>.
+
+- \b ArtBulkViscConst \n This sets the value of the artificial viscosity
+ parameter used by GADGET-2.
+
+
+- \b InitGasTemp \n This sets the initial gas temperature (assuming either
+ a mean molecular weight corresponding to full ionization or
+ full neutrality, depending on whether the temperature is above
+ or below 10^4 K) in Kelvin when initial conditions are
+ read. However, the gas temperature is only set to a certain
+ temperature if <em>InitGasTemp</em>>0, and if the temperature
+ of the gas particles in the initial conditions file is zero,
+ otherwise the initial gas temperature is left at the value
+ stored in the IC file.
+
+- \b MinGasTemp \n A minimum temperature floor imposed by the code. This
+ may be set to zero.
+
+- \b PartAllocFactor \n Each processor allocates space for
+ <em>PartAllocFactor</em> times the average number of particles
+ per processor. This number needs to be larger than 1 to allow
+ the simulation to achieve a good work-load balancing, which
+ requires to trade particle-load balance for work-load
+ balance. It is good to make <em>PartAllocFactor</em> quite a
+ bit larger than 1, but values in excess of 3 will typically
+ not improve performance any more. For a value that is too
+ small, the code may not be able to succeed in the domain
+ decomposition and terminate.
+
+- \b TreeAllocFactor \n To construct the BH-tree for N particles, somewhat
+ less than N internal tree-nodes are necessary for `normal'
+ particle distributions. <em>TreeAllocFactor</em> sets the
+ number of internal tree-nodes allocated in units of the
+ particle number. By experience, space for 0.65 N internal
+ nodes is usually fully sufficient, so a value of 0.7 should
+ put you on the safe side.
+
+- \b BufferSize \n This specifies the size (in MByte per processor) of a
+ communication buffer used by the code.
+
+- \b UnitLength_in_cm \n This sets the internal length unit in cm/h, where
+ H_0 = 100 h km/sec/Mpc. For example, a choice of 3.085678e21
+ sets the length unit to 1.0 kpc/h.
+
+- \b UnitMass_in_g \n This sets the internal mass unit in g/h, where H_0 =
+ 100 h km/sec/Mpc. For example, a choice of 1.989e43 sets the
+ mass unit to 10^10 M_sun/h.
+
+- \b UnitVelocity_in_cm_per_s \n This sets the internal velocity unit in
+ cm/sec. For example, a choice of 1e5 sets the velocity unit to
+ km/sec. Note that the specification of
+ <em>UnitLength_in_cm</em>, <em>UnitMass_in_g</em>, and
+ <em>UnitVelocity_in_cm_per_s</em> also determines the internal
+ unit of time.
+
+- \b GravityConstantInternal \n The numerical value of the gravitational
+ constant G in internal units depends on the system of units
+ you choose. For example, for the choices above, G=43007.1 in
+ internal units. For <em>GravityConstantInternal</em>=0, the
+ code calculates the value corresponding to the physical value
+ of G automatically. However, you might want to set G
+ yourself. For example, by specifying
+ <em>GravityConstantInternal</em>=1,
+ <em>UnitLength_in_cm</em>=1, <em>UnitMass_in_g</em>=1, and
+ <em>UnitVelocity_in_cm_per_s</em>=1, one obtains a `natural'
+ system of units. Note that the code will nevertheless try to
+ use the `correct' value of the Hubble constant in this case,
+ so you should not set <em>GravityConstantInternal</em> in
+ cosmological integrations.
+
+- \b MinGasHsmlFractional \n This parameter sets the minimum allowed SPH
+ smoothing length in units of the gravitational softening
+ length of the gas particles. The smoothing length will be
+ prevented from falling below this value. When this bound is
+ actually reached, the number of smoothing neighbors will
+ instead be increased above <em>DesNumNgb</em>.
+
+- \b SofteningGas \n The Plummer equivalent gravitational softening length
+ for particle type 0, which are the gas particles. For
+ cosmological simulations in comoving coordinates, this is
+ interpreted as a comoving softening length.
+
+- \b SofteningHalo \n The Plummer equivalent gravitational softening length
+ for particle type 1.
+
+- \b SofteningDisk \n The Plummer equivalent gravitational softening length
+ for particle type 2.
+
+- \b SofteningBulge \n The Plummer equivalent gravitational softening
+ length for particle type 3.
+
+- \b SofteningStars \n The Plummer equivalent gravitational softening
+ length for particle type 4.
+
+- \b SofteningBndry \n The Plummer equivalent gravitational softening
+ length for particle type 5.
+
+
+- \b SofteningGasMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 0. Depening on the relative settings
+ of <em>SofteningGas</em> and <em>SofteningGasMaxPhys</em>, the
+ code will hence switch from a softening constant in comoving
+ units to one constant in physical units.
+
+- \b SofteningHaloMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 1.
+
+- \b SofteningDiskMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 2.
+
+- \b SofteningBulgeMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 3.
+
+- \b SofteningStarsMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 4.
+
+- \b SofteningBndryMaxPhys \n When comoving integration is used, this
+ parameter gives the maximum physical gravitational softening
+ length for particle type 5.
+
+
+*/
+
+
+
+
+
+
+
+
+
+
+/*! \page Gadget-Makefile Makefile of GADGET-2
+
+A number of features of GADGET-2 are controlled with compile-time options
+in the makefile rather than by the parameterfile. This has been done in
+order to allow the generation of highly optimised binaries by the compiler,
+even when the underlying source code allows for many different ways to run the
+code.
+
+The makefile contains a dummy list of all available compile-time options,
+with most of them commented out by default. To activate a certain feature,
+the corresponding parameter should be commented in, and given the desired
+value, where appropriate. Below, a brief guide to these options is
+included.
+
+<b>Important Note:</b> Whenever one of the compile-time options
+described below is modified, a full recompilation of the code may be
+necessary. To guarantee that this is done when a simple <b>make</b> is
+specified, all source files have been specified in the Makefile as being
+dependent on the Makefile itself. Alternatively, one can also issue the
+command <b>make clean</b>, which will erase all object files, followed
+by <b>make</b>.
+
+Note that the above technique has the disadvantage that different
+simulations may require different binaries of GADGET-2. If several
+simulations are run concurrently, there is hence the danger that a
+simulation is started/resumed with the `wrong' binary. Note that while
+GADGET-2 checks the plausibility of some of the most important code
+options, this is not done for all of them. To minimise the risk of using
+the wrong executable for a simulation, it is recommended to produce a
+separate executable for each simulation that is run. For example, a good
+strategy is to make a copy of the whole code together with its makefile in
+the output directory of each simulation run, and then to use this copy to
+compile the code and to run the simulation.
+
+
+\n
+\section secmake1 Basic operation mode of code
+- \b PERIODIC \n Set this if you want to have periodic boundary conditions.
+
+- \b UNEQUALSOFTENINGS \n Set this if you use particles with different
+ gravitational softening lengths.
+
+\n
+\section secmake2 Things that are always recommended
+- \b PEANOHILBERT \n This is a tuning option. When set, the code will bring
+ the particles into Peano-Hilbert order after each domain
+ decomposition. This improves cache utilisation and performance.
+
+- \b WALLCLOCK \n If set, a wallclock timer is used by the code to measure
+ internal time consumption (see cpu-log file). Otherwise, a timer that
+ measures consumed processor ticks is used.
+
+\n
+\section secmake3 TreePM options
+- \b PMGRID=128 \n This enables the TreePM method, i.e. the long-range
+ force is computed with a PM-algorithm, and the short range force with
+ the tree. The parameter has to be set to the size of the mesh that
+ should be used, e.g.~64, 96, 128, etc. The mesh dimensions need not
+ necessarily be a power of two, but the FFT is fastest for such a
+ choice. Note: If the simulation is not in a periodic box, then a FFT
+ method for vacuum boundaries is employed, using a mesh with dimension
+ twice that specified by <b>PMGRID</b>.
+
+- \b PLACEHIGHRESREGION=1+8 \n If this option is set (will only work
+ together with \b PMGRID), then the long range force is computed in two
+ stages: One Fourier-grid is used to cover the whole simulation volume,
+ allowing the computation of the large-scale force. A second Fourier
+ mesh is placed on the region occupied by `high-resolution' particles,
+ allowing the computation of an intermediate-scale force. Finally, the
+ force on very small scales is computed by the tree. This procedure can
+ be useful for `zoom-simulations', where the majority of particles (the
+ high-res particles) are occupying only a small fraction of the
+ volume. To activate this option, the parameter needs to be set to an
+ integer that encodes the particle types that make up the high-res
+ particles in the form of a bit mask. For example, if types 0, 1, and 4
+ are the high-res particles, then the parameter should be set to
+ <b>PLACEHIGHRESREGION=1+2+16</b>, i.e. to the sum
+ \f$2^0+2^1+2^4\f$. The spatial region covered by the high-res grid is
+ determined automatically from the initial conditions. Note: If a
+ periodic box is used, the high-res zone is not allowed to intersect the box
+ boundaries.
+
+- <b> ENLARGEREGION=1.1</b> \n The spatial region covered by the high-res zone
+ normally has a fixed size during the simulation, which initially is
+ set to the smallest region that encompasses all high-res
+ particles. Normally, the simulation will be interrupted if high-res
+ particles leave this region in the course of the run. However, by
+ setting this parameter to a value larger than one, the high-res region
+ can be expanded on the fly. For example, setting it to 1.4 will enlarge its
+ side-length by 40% in such an event (it remains centred on the high-res
+ particles). Hence, with such a setting, the high-res region may expand
+ or move by a limited amount. If in addition \b SYNCHRONIZATION is
+ activated, then the code will be able to continue even if high-res
+ particles leave the initial high-res grid. In this case, the code will
+ update the size and position of the grid that is placed onto the
+ high-resolution region automatically. To prevent that this potentially
+ happens every single PM step, one should nevertheless assign a value
+ slightly larger than 1 to \b ENLARGEREGION.
+
+- <b> ASMTH=1.25</b> \n This can be used to override the value assumed for the
+ scale that defines the long-range/short-range force-split in the
+ TreePM algorithm. The default value is 1.25, in mesh-cells.
+
+- <b> RCUT=4.5</b> \n This can be used to override the maximum radius in which
+ the short-range tree-force is evaluated (in case the TreePM algorithm
+ is used). The default value is 4.5, given in mesh-cells.
+
+\n
+\section secmake4 Single or double precision
+- \b DOUBLEPRECISION \n This makes the code store and compute internal
+ particle data in double precision. Note that output files are
+ nevertheless written by converting the values that are saved to single
+ precision.
+
+- \b DOUBLEPRECISION_FFTW \n If this is set, the code will use the
+ double-precision version of FTTW, provided the latter has been
+ explicitly installed with a "d" prefix, and NOTYPEPREFIX_FFTW is not
+ set. Otherwise the single precision version ("s" prefix) is used.
+
+
+\n
+\section secmake5 Time integration options
+- \b SYNCHRONIZATION \n When this is set, particles may only increase their
+ timestep if the new timestep will put them into synchronisation with
+ the higher time level. This typically means that only on half of the
+ timesteps of a particle an increase of its step may occur. Especially
+ for TreePM runs, it is usually advisable to set this option.
+
+- \b FLEXSTEPS \n This is an alternative to SYNCHRONIZATION. Particle
+ timesteps are here allowed to be integer multiples of the minimum
+ timestep that occurs among the particles, which in turn is rounded
+ down to the nearest power-of-two devision of the total simulated
+ timespan. This option distributes particles more evenly over
+ individual system timesteps, particularly once a simulation has run
+ for a while, and may then result in a reduction of work-load imbalance
+ losses.
+
+- \b PSEUDOSYMMETRIC \n When this option is set, the code will try to
+ `anticipate' timestep changes by extrapolating the change of the
+ acceleration into the future. This in general improves the long-term
+ integration behaviour of periodic orbits, because then the adaptive
+ integration becomes more akin to a strictly time reversible
+ integrator. Note: This option has no effect if FLEXSTEPS is set.
+
+- \b NOSTOP_WHEN_BELOW_MINTIMESTEP \n If this is activated, the code will
+ not terminate when the timestep falls below the value of \b
+ MinSizeTimestep specified in the parameterfile. This is useful for
+ runs where one wants to enforce a constant timestep for all
+ particles. This can be done by activating this option, and by setting
+ \b MinSizeTimestep and \b MaxSizeTimestep to an equal value.
+
+- \b NOPMSTEPADJUSTMENT \n When this is set, the long-range timestep for
+ the PM force computation is always determined by \b MaxSizeTimeStep.
+ Otherwise, it is set to the minimum of \b MaxSizeTimeStep and the
+ timestep obtained for the maximum long-range force with an effective
+ softening scale equal to the PM smoothing-scale.
+
+\n
+\section secmake6 Output options
+- \b HAVE_HDF5 \n If this is set, the code will be compiled with support
+ for input and output in the HDF5 format. You need to have the HDF5
+ libraries and headers installed on your computer for this option to
+ work. The HDF5 format can then be selected as format "3" in Gadget's
+ parameterfile.
+
+- \b OUTPUTPOTENTIAL \n This will force the code to compute gravitational
+ potentials for all particles each time a snapshot file is
+ generated. These values are then included in the snapshot files. Note
+ that the computation of the values of the potential costs additional
+ time.
+
+- \b OUTPUTACCELERATION \n This will include the physical acceleration of
+ each particle in snapshot files.
+
+- \b OUTPUTCHANGEOFENTROPY \n This will include the rate of change of
+ entropy of gas particles in snapshot files.
+
+- \b OUTPUTTIMESTEP \n This will include the timesteps actually taken by
+ each particle in the snapshot files.
+
+\n
+\section secmake7 Things for special behaviour
+- \b NOGRAVITY \n This switches off gravity. Makes only sense for pure SPH
+ simulations in non-expanding space.
+
+- \b NOTREERND \n If this is not set, the tree construction will succeed
+ even when there are a few particles at identical locations. This is
+ done by `rerouting' particles once the node-size has fallen below
+ \f$10^{-3}\f$ of the softening length. When this option is activated,
+ this will be suppressed and the tree construction will always fail if
+ there are particles at extremely close or identical coordinates.
+
+- \b NOTYPEPREFIX_FFTW \n If this is set, the fftw-header/libraries are
+ accessed without type prefix (adopting whatever was chosen as default
+ at compile-time of fftw). Otherwise, the type prefix 'd' for
+ double-precision is used.
+
+- \b LONG_X/Y/Z \n These options can be used together with PERIODIC and
+ NOGRAVITY only. When set, the options define numerical factors that
+ can be used to distort the periodic simulation cube into a
+ parallelepiped of arbitrary aspect ratio. This can be useful for
+ idealized SPH tests.
+
+- \b TWODIMS \n This effectively switches of one dimension in SPH,
+ i.e. the code follows only 2d hydrodynamics in the xy-, yz-, or
+ xz-plane. This only works with NOGRAVITY, and if all coordinates of
+ the third axis are exactly equal. Can be useful for idealized SPH
+ tests.
+
+- \b SPH_BND_PARTICLES \n If this is set, particles with a particle-ID
+ equal to zero do not receive any SPH acceleration. This can be useful
+ for idealized SPH tests, where these particles represent fixed
+ "walls".
+
+- \b NOVISCOSITYLIMITER \n If this is set, there is no explicit upper
+ limit on the viscosity. In the default version, this limiter will
+ try to protect against possible particle `reflections', which could
+ in principle occur if very poor timestepping is used in the
+ presence of strong shocks.
+
+- \b COMPUTE_POTENTIAL_ENERGY \n When this option is set, the code will
+ compute the gravitational potential energy each time a global
+ statistics is computed. This can be useful for testing global energy
+ conservation.
+
+- \b ISOTHERM_EQS \n This special option makes the gas behave like an
+ isothermal gas with equation of state \f$ P = c_s^2 \rho \f$. The
+ sound-speed \f$ c_s \f$ is set by the thermal energy per unit mass
+ in the intial conditions, i.e. \f$ c_s^2=u \f$. If the value for \f$ u \f$
+ is zero, then the initial gas temperature in the parameter file is
+ used to define the sound speed according to \f$ c_s^2= 3/4\,
+ k\,T/m_p \f$ , where \f$ m_p \f$ is the proton mass.
+
+- \b ADAPTIVE_GRAVSOFT_FORGAS \n When this option is set, the gravitational
+ softening lengths used for gas particles is tied to their SPH smoothing
+ length. This can be useful for dissipative collapse simulations. The
+ option requires the setting of UNEQUALSOFTENINGS.
+
+- \b SELECTIVE_NO_GRAVITY \n This can be used for special computations where
+ one wants to exclude certain particle types from receiving gravitational
+ forces. The particle types that are excluded in this fashion are specified
+ by a bit mask, in the same as for the PLACEHIGHRESREGION option.
+
+- \b LONGIDS \n If this is set, the code assumes that particle-IDs are
+ stored as 64-bit long integers. This is only really needed if you want
+ to go beyond ~2 billion particles.
+
+\n
+\section secmake8 Testing and Debugging options
+- \b FORCETEST=0.01 \n This can be set to check the force accuracy of the
+ code, and is only included as a debugging option. The option needs to
+ be set to a number between 0 and 1 (e.g. 0.01), which specifies the
+ fraction of randomly chosen particles for which at each timestep
+ forces by direct summation are computed. The normal tree-forces and
+ the `correct' direct summation forces are then collected in a file \b
+ forcetest.txt for later inspection. Note that the simulation itself is
+ unaffected by this option, but it will of course run much(!) slower,
+ particularly if <b> FORCETEST*NumPart*NumPart>>NumPart</b>
+ Note: Particle IDs must be set to numbers >=1 for this
+ option to work.
+
+\n
+\section secmake9 Glass making
+- \b MAKEGLASS=262144 \n This option can be used to generate a glass-like
+ particle configuration. The value assigned gives the particle load,
+ which is initially generated as a Poisson sample and then evolved
+ towards a glass with the sign of gravity reversed
+
+*/
diff --git a/new.phase.c b/new.phase.c
new file mode 100644
index 0000000..0bb015b
--- /dev/null
+++ b/new.phase.c
@@ -0,0 +1,339 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef MULTIPHASE
+
+
+/*! \file phase.c
+ * \brief Compute phase mixing
+ *
+*/
+
+/*! compute phase mixing
+ *
+ */
+void update_phase()
+{
+ int i;
+ double a3,hubble_a;
+#ifdef PHASE_MIXING
+ FLOAT EgySpec=0;
+#endif
+
+ double dt;
+
+#ifdef COLDGAS_CYCLE
+ double Pwc,Pcw;
+ double flux_in_cgs,X,tau;
+#endif
+
+ double Pcol,ColTime;
+ double DensityNorm;
+
+ int *numlist;
+
+
+ N_sph = 0;
+ N_sticky = 0;
+ N_stickyflaged = 0;
+ N_dark = 0;
+ NumColPotLocal = 0;
+
+
+ if(All.ComovingIntegrationOn)
+ {
+ a3 = All.Time * All.Time * All.Time;
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+
+ }
+ else
+ {
+ a3 = 1;
+ hubble_a = 1;
+ }
+
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+
+ //if(P[i].Ti_endstep == All.Ti_Current) /* if the particle is active */
+ {
+
+
+ SphP[i].StickyFlag = 0;
+
+
+#ifdef PHASE_MIXING
+
+
+
+#ifdef FEEDBACK
+ if((P[i].Type == 0) && (SphP[i].EnergySN==0))
+#else
+ if(P[i].Type == 0)
+#endif
+ {
+
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ /* compute temperature from entropy */
+ EgySpec = 1./GAMMA_MINUS1 * pow(SphP[i].Density / a3,GAMMA_MINUS1)* SphP[i].Entropy;
+
+ if (EgySpec < All.CriticalEgySpec) /* to sticky phase */
+ {
+ SphP[i].Phase = GAS_STICKY;
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time; /* may be elligible for collision */
+ SphP[i].Entropy = EgySpec;
+ }
+
+ break;
+
+ case GAS_STICKY:
+ /* compute temperature from specific energy */
+ EgySpec = SphP[i].Entropy;
+
+ if (EgySpec >= All.CriticalEgySpec) /* to sph phase */
+ {
+ SphP[i].Phase = GAS_SPH;
+ SphP[i].Entropy = GAMMA_MINUS1 * EgySpec / pow(SphP[i].Density / a3, GAMMA_MINUS1);
+ }
+
+ break;
+ }
+
+
+#ifdef COLDGAS_CYCLE
+
+ /*
+
+ Here, we do nothing during the first step, because the UV flux is unknown
+
+ */
+
+
+ if (((SphP[i].Phase==GAS_STICKY)||(SphP[i].Phase==GAS_DARK))&&All.NumCurrentTiStep>0)
+ {
+ /* cold/warm gas cycle */
+
+ /* compute dt */
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+
+ X = 0.;
+
+#ifdef STELLAR_FLUX
+ /* compute stellar flux */
+ flux_in_cgs = SphP[i].EnergyFlux* All.UnitEnergy_in_cgs/All.UnitTime_in_s/pow(All.UnitLength_in_cm, 2);
+ X = X + flux_in_cgs/C / All.HeatingPeSolarEnergyDensity;
+#endif
+
+#ifdef EXTERNAL_FLUX
+ /* compute outer flux */
+ X = X + All.HeatingExternalFLuxEnergyDensity/All.HeatingPeSolarEnergyDensity ;
+#endif
+
+
+
+ if (X>0)
+ {
+ /* warm gas */
+
+ tau = pow(X,+All.ColdGasCycleTransitionParameter) * All.ColdGasCycleTransitionTime;
+ Pwc = 1.0-exp(-dt/tau);
+
+ /* cold gas */
+ tau = pow(X,-All.ColdGasCycleTransitionParameter) * All.ColdGasCycleTransitionTime;
+ Pcw = 1.0-exp(-dt/tau);
+
+ }
+ else
+ {
+ Pwc = 1.0;
+ Pcw = 0.0;
+ }
+
+
+ /* compute transition */
+ switch(SphP[i].Phase)
+ {
+ case GAS_STICKY:
+ if (get_random_number(P[i].ID) < Pwc)
+ {
+ SphP[i].Phase = GAS_DARK;
+ }
+ break;
+
+ case GAS_DARK:
+ if (get_random_number(P[i].ID) < Pcw)
+ {
+ SphP[i].Phase = GAS_STICKY;
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time; /* may be elligible for collision */
+ }
+ break;
+
+ }
+
+ }
+
+#endif /* COLDGAS_CYCLE */
+
+
+#endif /* PHASE_MIXING */
+
+
+ /* determine if a sticky particle may collide or not */
+
+
+ if(SphP[i].Phase==GAS_STICKY)
+ {
+
+
+ if(SphP[i].StickyTime <= All.Time) /* may collide */
+ {
+
+ /* new implementation */
+ SphP[i].StickyFlag = 1;
+
+
+ /* this was the old implementation */
+
+ //dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+ //
+ ///* because mean free path depends on density */
+ //if (All.StickyDensity>0)
+ // {
+ // //ColTime = All.StickyTime * pow(All.StickyDensity/SphP[i].Density,All.StickyDensityPower);
+ // DensityNorm = SphP[i].Density/All.StickyDensity;
+ // ColTime = All.StickyTime / (DensityNorm * pow( (1+pow(DensityNorm/3,2)) ,1/2.));
+ // if (DensityNorm > 10)
+ // {
+ // ColTime = ColTime*1e10; /* cut off */
+ // }
+ // }
+ //else
+ // ColTime = All.StickyTime;
+ //
+ //
+ //Pcol = 1.0-exp(-dt/ColTime);
+ //if (get_random_number(P[i].ID) < Pcol)
+ // {
+ // SphP[i].StickyFlag = 1; /* the particle may collide */
+ // SphP[i].StickyTime = All.Time + All.StickyIdleTime;
+ // NumColPotLocal++;
+ // }
+
+
+
+ }
+ }
+
+
+ } /* end of Ptype==0 */
+
+ } /* end of active particles */
+
+ /* count number of each type */
+ if(P[i].Type == 0)
+ {
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ N_sph++;
+ break;
+
+ case GAS_STICKY:
+ N_sticky++;
+ if (SphP[i].StickyFlag == 1)
+ N_stickyflaged++;
+ break;
+
+ case GAS_DARK:
+ N_dark++;
+ break;
+ }
+ }
+
+
+
+
+ } /* end of main loop */
+
+ /* share results */
+ //MPI_Allreduce(&N_sph, &All.TotN_sph, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_sticky, &All.TotN_sticky, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_stickyflaged, &All.TotN_stickyflaged, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_dark, &All.TotN_dark, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+
+ MPI_Allgather(&N_sph, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_sph = 0; i < NTask; i++)
+ All.TotN_sph += numlist[i];
+
+ MPI_Allgather(&N_sticky, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_sticky = 0; i < NTask; i++)
+ All.TotN_sticky += numlist[i];
+
+ MPI_Allgather(&N_stickyflaged, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_stickyflaged = 0; i < NTask; i++)
+ All.TotN_stickyflaged += numlist[i];
+
+ MPI_Allgather(&N_dark, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_dark = 0; i < NTask; i++)
+ All.TotN_dark += numlist[i];
+
+ free(numlist);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ if (ThisTask==0)
+ {
+ fprintf(FdPhase, "Step %d, Time: %g GasPart: %d SphPart: %d StickyPart: %d DarkPart: %d StickyPartFlaged: %d\n", All.NumCurrentTiStep, All.Time, (int)All.TotN_gas, (int)All.TotN_sph, (int)All.TotN_sticky,(int)All.TotN_dark, (int)All.TotN_stickyflaged);
+ fflush(FdPhase);
+ }
+
+
+}
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/ngb.c b/ngb.c
new file mode 100644
index 0000000..22143f1
--- /dev/null
+++ b/ngb.c
@@ -0,0 +1,880 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file ngb.c
+ * \brief neighbour search by means of the tree
+ *
+ * This file contains routines for neighbour finding. We use the
+ * gravity-tree and a range-searching technique to find neighbours.
+ */
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+
+/*! these macros maps a coordinate difference to the nearest periodic
+ * image
+ */
+
+#define NGB_PERIODIC_X(x) (xtmp=(x),(xtmp>boxHalf_X)?(xtmp-boxSize_X):((xtmp<-boxHalf_X)?(xtmp+boxSize_X):xtmp))
+#define NGB_PERIODIC_Y(x) (xtmp=(x),(xtmp>boxHalf_Y)?(xtmp-boxSize_Y):((xtmp<-boxHalf_Y)?(xtmp+boxSize_Y):xtmp))
+#define NGB_PERIODIC_Z(x) (xtmp=(x),(xtmp>boxHalf_Z)?(xtmp-boxSize_Z):((xtmp<-boxHalf_Z)?(xtmp+boxSize_Z):xtmp))
+
+
+
+/*! This routine finds all neighbours `j' that can interact with the
+ * particle `i' in the communication buffer.
+ *
+ * Note that an interaction can take place if
+ * \f$ r_{ij} < h_i \f$ OR if \f$ r_{ij} < h_j \f$.
+ *
+ * In the range-search this is taken into account, i.e. it is guaranteed that
+ * all particles are found that fulfil this condition, including the (more
+ * difficult) second part of it. For this purpose, each node knows the
+ * maximum h occuring among the particles it represents.
+ */
+int ngb_treefind_pairs(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode)
+{
+ int k, no, p, numngb;
+ FLOAT hdiff;
+ FLOAT searchmin[3], searchmax[3];
+ struct NODE *this;
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(k = 0; k < 3; k++) /* cube-box window */
+ {
+ searchmin[k] = searchcenter[k] - hsml;
+ searchmax[k] = searchcenter[k] + hsml;
+ }
+
+ numngb = 0;
+ no = *startnode;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ p = no;
+ no = Nextnode[no];
+
+ if(P[p].Type > 0)
+ continue;
+
+
+#ifdef MULTIPHASE
+ /* WARNING - WARNING - WARNING - WARNING */
+
+ /* keep only particles of same phase */
+ //if(SphP[p].Phase != phase)
+ // continue;
+
+ /* WARNING - WARNING - WARNING - WARNING */
+ /* this may be problematic for the feedback */
+
+ /* sticky sees only sticky */
+
+ //if ((phase==GAS_STICKY) && SphP[p].Phase!=GAS_STICKY)
+ // continue;
+
+#endif
+
+
+
+ hdiff = SphP[p].Hsml - hsml;
+ if(hdiff < 0)
+ hdiff = 0;
+
+#ifdef PERIODIC
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) > (hsml + hdiff))
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) > (hsml + hdiff))
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) > (hsml + hdiff))
+ continue;
+#else
+ if(P[p].Pos[0] < (searchmin[0] - hdiff))
+ continue;
+ if(P[p].Pos[0] > (searchmax[0] + hdiff))
+ continue;
+ if(P[p].Pos[1] < (searchmin[1] - hdiff))
+ continue;
+ if(P[p].Pos[1] > (searchmax[1] + hdiff))
+ continue;
+ if(P[p].Pos[2] < (searchmin[2] - hdiff))
+ continue;
+ if(P[p].Pos[2] > (searchmax[2] + hdiff))
+ continue;
+#endif
+ Ngblist[numngb++] = p;
+
+ if(numngb == MAX_NGB)
+ {
+ printf
+ ("ThisTask=%d: Need to do a second neighbour loop in hydro-force for (%g|%g|%g) hsml=%g no=%d\n",
+ ThisTask, searchcenter[0], searchcenter[1], searchcenter[2], hsml, no);
+ *startnode = no;
+ return numngb;
+ }
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ this = &Nodes[no];
+ hdiff = Extnodes[no].hmax - hsml;
+ if(hdiff < 0)
+ hdiff = 0;
+
+ no = this->u.d.sibling; /* in case the node can be discarded */
+
+#ifdef PERIODIC
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+#else
+ if((this->center[0] + 0.5 * this->len) < (searchmin[0] - hdiff))
+ continue;
+ if((this->center[0] - 0.5 * this->len) > (searchmax[0] + hdiff))
+ continue;
+ if((this->center[1] + 0.5 * this->len) < (searchmin[1] - hdiff))
+ continue;
+ if((this->center[1] - 0.5 * this->len) > (searchmax[1] + hdiff))
+ continue;
+ if((this->center[2] + 0.5 * this->len) < (searchmin[2] - hdiff))
+ continue;
+ if((this->center[2] - 0.5 * this->len) > (searchmax[2] + hdiff))
+ continue;
+#endif
+ no = this->u.d.nextnode; /* ok, we need to open the node */
+ }
+ }
+
+ *startnode = -1;
+ return numngb;
+}
+
+
+
+
+
+
+
+/*! This routine finds all neighbours `j' that can interact with the
+ * particle `i' in the communication buffer.
+ *
+ * Note that an interaction can take place if
+ * \f$ r_{ij} < h_i \f$ OR if \f$ r_{ij} < h_j \f$.
+ *
+ * In the range-search this is taken into account, i.e. it is guaranteed that
+ * all particles are found that fulfil this condition, including the (more
+ * difficult) second part of it. For this purpose, each node knows the
+ * maximum h occuring among the particles it represents.
+ *
+ * This function is similar to ngb_treefind_pairs but returns only
+ * pairs of same phase
+ */
+#ifdef MULTIPHASE
+int ngb_treefind_phase_pairs(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode)
+{
+ int k, no, p, numngb;
+ FLOAT hdiff;
+ FLOAT searchmin[3], searchmax[3];
+ struct NODE *this;
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(k = 0; k < 3; k++) /* cube-box window */
+ {
+ searchmin[k] = searchcenter[k] - hsml;
+ searchmax[k] = searchcenter[k] + hsml;
+ }
+
+ numngb = 0;
+ no = *startnode;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ p = no;
+ no = Nextnode[no];
+
+ if(P[p].Type > 0)
+ continue;
+
+ /* here, we select all gas particles */
+
+#ifdef MULTIPHASE
+ /* keep only particles of same phase */
+ if(SphP[p].Phase != phase)
+ continue;
+#endif
+
+ hdiff = SphP[p].Hsml - hsml;
+ if(hdiff < 0)
+ hdiff = 0;
+
+#ifdef PERIODIC
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) > (hsml + hdiff))
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) > (hsml + hdiff))
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) < (-hsml - hdiff))
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) > (hsml + hdiff))
+ continue;
+#else
+ if(P[p].Pos[0] < (searchmin[0] - hdiff))
+ continue;
+ if(P[p].Pos[0] > (searchmax[0] + hdiff))
+ continue;
+ if(P[p].Pos[1] < (searchmin[1] - hdiff))
+ continue;
+ if(P[p].Pos[1] > (searchmax[1] + hdiff))
+ continue;
+ if(P[p].Pos[2] < (searchmin[2] - hdiff))
+ continue;
+ if(P[p].Pos[2] > (searchmax[2] + hdiff))
+ continue;
+#endif
+ Ngblist[numngb++] = p;
+
+ if(numngb == MAX_NGB)
+ {
+ printf
+ ("ThisTask=%d: Need to do a second neighbour loop in hydro-force for (%g|%g|%g) hsml=%g no=%d\n",
+ ThisTask, searchcenter[0], searchcenter[1], searchcenter[2], hsml, no);
+ *startnode = no;
+ return numngb;
+ }
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ this = &Nodes[no];
+ hdiff = Extnodes[no].hmax - hsml;
+ if(hdiff < 0)
+ hdiff = 0;
+
+ no = this->u.d.sibling; /* in case the node can be discarded */
+
+#ifdef PERIODIC
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) + 0.5 * this->len) < (-hsml - hdiff))
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) - 0.5 * this->len) > (hsml + hdiff))
+ continue;
+#else
+ if((this->center[0] + 0.5 * this->len) < (searchmin[0] - hdiff))
+ continue;
+ if((this->center[0] - 0.5 * this->len) > (searchmax[0] + hdiff))
+ continue;
+ if((this->center[1] + 0.5 * this->len) < (searchmin[1] - hdiff))
+ continue;
+ if((this->center[1] - 0.5 * this->len) > (searchmax[1] + hdiff))
+ continue;
+ if((this->center[2] + 0.5 * this->len) < (searchmin[2] - hdiff))
+ continue;
+ if((this->center[2] - 0.5 * this->len) > (searchmax[2] + hdiff))
+ continue;
+#endif
+ no = this->u.d.nextnode; /* ok, we need to open the node */
+ }
+ }
+
+ *startnode = -1;
+ return numngb;
+}
+#endif
+
+/*! This function returns neighbours with distance <= hsml and returns them in
+ * Ngblist. Actually, particles in a box of half side length hsml are
+ * returned, i.e. the reduction to a sphere still needs to be done in the
+ * calling routine.
+ */
+int ngb_treefind_variable(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode)
+{
+ int k, numngb;
+ int no, p;
+ struct NODE *this;
+ FLOAT searchmin[3], searchmax[3];
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(k = 0; k < 3; k++) /* cube-box window */
+ {
+ searchmin[k] = searchcenter[k] - hsml;
+ searchmax[k] = searchcenter[k] + hsml;
+ }
+
+ numngb = 0;
+ no = *startnode;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ p = no;
+ no = Nextnode[no];
+
+ if(P[p].Type > 0)
+ continue;
+
+#ifdef MULTIPHASE
+ /* GAS_SPH sees only GAS_SPH */
+ if((phase == GAS_SPH) && (SphP[p].Phase != GAS_SPH))
+ continue;
+
+ /* GAS_SPH sees only GAS_STICKY (better for sticky interaction) */
+ if((phase == GAS_STICKY) && (SphP[p].Phase != GAS_STICKY))
+ continue;
+
+ /* GAS_DARK sees everybody */
+
+
+
+#endif
+
+
+#ifdef PERIODIC
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) > hsml)
+ continue;
+#else
+ if(P[p].Pos[0] < searchmin[0])
+ continue;
+ if(P[p].Pos[0] > searchmax[0])
+ continue;
+ if(P[p].Pos[1] < searchmin[1])
+ continue;
+ if(P[p].Pos[1] > searchmax[1])
+ continue;
+ if(P[p].Pos[2] < searchmin[2])
+ continue;
+ if(P[p].Pos[2] > searchmax[2])
+ continue;
+#endif
+ Ngblist[numngb++] = p;
+
+ if(numngb == MAX_NGB)
+ {
+ numngb = ngb_clear_buf(searchcenter, hsml, numngb);
+ if(numngb == MAX_NGB)
+ {
+ printf("ThisTask=%d: Need to do a second neighbour loop for (%g|%g|%g) hsml=%g no=%d\n",
+ ThisTask, searchcenter[0], searchcenter[1], searchcenter[2], hsml, no);
+ *startnode = no;
+ return numngb;
+ }
+ }
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ this = &Nodes[no];
+
+ no = this->u.d.sibling; /* in case the node can be discarded */
+#ifdef PERIODIC
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) - 0.5 * this->len) > hsml)
+ continue;
+#else
+ if((this->center[0] + 0.5 * this->len) < (searchmin[0]))
+ continue;
+ if((this->center[0] - 0.5 * this->len) > (searchmax[0]))
+ continue;
+ if((this->center[1] + 0.5 * this->len) < (searchmin[1]))
+ continue;
+ if((this->center[1] - 0.5 * this->len) > (searchmax[1]))
+ continue;
+ if((this->center[2] + 0.5 * this->len) < (searchmin[2]))
+ continue;
+ if((this->center[2] - 0.5 * this->len) > (searchmax[2]))
+ continue;
+#endif
+ no = this->u.d.nextnode; /* ok, we need to open the node */
+ }
+ }
+
+ *startnode = -1;
+ return numngb;
+}
+
+
+
+
+
+
+#ifdef CHIMIE
+/*! This function returns neighbours with distance <= hsml and returns them in
+ * Ngblist. Actually, particles in a box of half side length hsml are
+ * returned, i.e. the reduction to a sphere still needs to be done in the
+ * calling routine.
+ */
+int ngb_treefind_variable_for_chimie(FLOAT searchcenter[3], FLOAT hsml, int *startnode)
+{
+ int k, numngb;
+ int no, p;
+ struct NODE *this;
+ FLOAT searchmin[3], searchmax[3];
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(k = 0; k < 3; k++) /* cube-box window */
+ {
+ searchmin[k] = searchcenter[k] - hsml;
+ searchmax[k] = searchcenter[k] + hsml;
+ }
+
+ numngb = 0;
+ no = *startnode;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ p = no;
+ no = Nextnode[no];
+
+ if(P[p].Type > 0)
+ continue;
+
+#ifdef CHIMIE_KINETIC_FEEDBACK /* no, we need to keep all particles, especially when the wind is low */
+// if (SphP[p].WindTime >= (All.Time-All.ChimieWindTime))
+// continue;
+#endif
+
+#ifdef PERIODIC
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) > hsml)
+ continue;
+#else
+ if(P[p].Pos[0] < searchmin[0])
+ continue;
+ if(P[p].Pos[0] > searchmax[0])
+ continue;
+ if(P[p].Pos[1] < searchmin[1])
+ continue;
+ if(P[p].Pos[1] > searchmax[1])
+ continue;
+ if(P[p].Pos[2] < searchmin[2])
+ continue;
+ if(P[p].Pos[2] > searchmax[2])
+ continue;
+#endif
+ Ngblist[numngb++] = p;
+
+ if(numngb == MAX_NGB)
+ {
+ numngb = ngb_clear_buf(searchcenter, hsml, numngb);
+ if(numngb == MAX_NGB)
+ {
+ printf("ThisTask=%d: Need to do a second neighbour loop for (%g|%g|%g) hsml=%g no=%d\n",
+ ThisTask, searchcenter[0], searchcenter[1], searchcenter[2], hsml, no);
+ *startnode = no;
+ return numngb;
+ }
+ }
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ Exportflag[DomainTask[no - (All.MaxPart + MaxNodes)]] = 1;
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ this = &Nodes[no];
+
+ no = this->u.d.sibling; /* in case the node can be discarded */
+#ifdef PERIODIC
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) - 0.5 * this->len) > hsml)
+ continue;
+#else
+ if((this->center[0] + 0.5 * this->len) < (searchmin[0]))
+ continue;
+ if((this->center[0] - 0.5 * this->len) > (searchmax[0]))
+ continue;
+ if((this->center[1] + 0.5 * this->len) < (searchmin[1]))
+ continue;
+ if((this->center[1] - 0.5 * this->len) > (searchmax[1]))
+ continue;
+ if((this->center[2] + 0.5 * this->len) < (searchmin[2]))
+ continue;
+ if((this->center[2] - 0.5 * this->len) > (searchmax[2]))
+ continue;
+#endif
+ no = this->u.d.nextnode; /* ok, we need to open the node */
+ }
+ }
+
+ *startnode = -1;
+ return numngb;
+}
+#endif
+
+
+
+
+
+
+
+
+
+
+
+
+/*! This function returns the first particle that may sticky-interact with
+ * the given particle, in the sph kernel.
+ */
+#ifdef MULTIPHASE
+int ngb_treefind_sticky_collisions(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode)
+{
+ int k, numngb;
+ int no, p;
+ struct NODE *this;
+ FLOAT searchmin[3], searchmax[3];
+
+ double dx,dy,dz,dr2;
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(k = 0; k < 3; k++) /* cube-box window */
+ {
+ searchmin[k] = searchcenter[k] - hsml;
+ searchmax[k] = searchcenter[k] + hsml;
+ }
+
+ numngb = 0;
+ no = *startnode;
+
+ while(no >= 0)
+ {
+ if(no < All.MaxPart) /* single particle */
+ {
+ p = no;
+ no = Nextnode[no];
+
+ if((P[p].Type > 0) || (SphP[p].Phase!=GAS_STICKY)) /* keep only GAS_STICKY */
+ continue;
+
+ if(SphP[p].StickyFlag==0) /* the particle may not collide */
+ continue;
+
+
+#ifdef PERIODIC
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]) > hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) < -hsml)
+ continue;
+ if(NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]) > hsml)
+ continue;
+#else
+ if(P[p].Pos[0] < searchmin[0])
+ continue;
+ if(P[p].Pos[0] > searchmax[0])
+ continue;
+ if(P[p].Pos[1] < searchmin[1])
+ continue;
+ if(P[p].Pos[1] > searchmax[1])
+ continue;
+ if(P[p].Pos[2] < searchmin[2])
+ continue;
+ if(P[p].Pos[2] > searchmax[2])
+ continue;
+#endif
+
+ dx = searchcenter[0] - P[p].Pos[0];
+ dy = searchcenter[1] - P[p].Pos[1];
+ dz = searchcenter[2] - P[p].Pos[2];
+ dr2= dx * dx + dy * dy + dz * dz;
+
+ if (dr2>hsml*hsml)
+ continue;
+
+ if (dr2==0)
+ continue;
+
+ return p;
+
+
+ }
+ else
+ {
+ if(no >= All.MaxPart + MaxNodes) /* pseudo particle */
+ {
+ /* here, we do nothing */
+ no = Nextnode[no - MaxNodes];
+ continue;
+ }
+
+ this = &Nodes[no];
+
+ no = this->u.d.sibling; /* in case the node can be discarded */
+#ifdef PERIODIC
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_X(this->center[0] - searchcenter[0]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Y(this->center[1] - searchcenter[1]) - 0.5 * this->len) > hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) + 0.5 * this->len) < -hsml)
+ continue;
+ if((NGB_PERIODIC_Z(this->center[2] - searchcenter[2]) - 0.5 * this->len) > hsml)
+ continue;
+#else
+ if((this->center[0] + 0.5 * this->len) < (searchmin[0]))
+ continue;
+ if((this->center[0] - 0.5 * this->len) > (searchmax[0]))
+ continue;
+ if((this->center[1] + 0.5 * this->len) < (searchmin[1]))
+ continue;
+ if((this->center[1] - 0.5 * this->len) > (searchmax[1]))
+ continue;
+ if((this->center[2] + 0.5 * this->len) < (searchmin[2]))
+ continue;
+ if((this->center[2] - 0.5 * this->len) > (searchmax[2]))
+ continue;
+#endif
+ no = this->u.d.nextnode; /* ok, we need to open the node */
+ }
+ }
+
+ return -1;
+}
+#endif
+
+
+
+
+
+
+
+
+/*! The buffer for the neighbour list has a finite length MAX_NGB. For a large
+ * search region, this buffer can get full, in which case this routine can be
+ * called to eliminate some of the superfluous particles in the "corners" of
+ * the search box - only the ones in the inscribed sphere need to be kept.
+ */
+int ngb_clear_buf(FLOAT searchcenter[3], FLOAT hsml, int numngb)
+{
+ int i, p;
+ FLOAT dx, dy, dz, r2;
+
+#ifdef PERIODIC
+ double xtmp;
+#endif
+
+ for(i = 0; i < numngb; i++)
+ {
+ p = Ngblist[i];
+#ifdef PERIODIC
+ dx = NGB_PERIODIC_X(P[p].Pos[0] - searchcenter[0]);
+ dy = NGB_PERIODIC_Y(P[p].Pos[1] - searchcenter[1]);
+ dz = NGB_PERIODIC_Z(P[p].Pos[2] - searchcenter[2]);
+#else
+ dx = P[p].Pos[0] - searchcenter[0];
+ dy = P[p].Pos[1] - searchcenter[1];
+ dz = P[p].Pos[2] - searchcenter[2];
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(r2 > hsml * hsml)
+ {
+ Ngblist[i] = Ngblist[numngb - 1];
+ i--;
+ numngb--;
+ }
+ }
+
+ return numngb;
+}
+
+
+
+/*! Allocates memory for the neighbour list buffer.
+ */
+void ngb_treeallocate(int npart)
+{
+ double totbytes = 0;
+ size_t bytes;
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+ if(!(Ngblist = malloc(bytes = npart * (long) sizeof(int))))
+ {
+ printf("Failed to allocate %g MB for ngblist array\n", bytes / (1024.0 * 1024.0));
+ endrun(78);
+ }
+ totbytes += bytes;
+
+ if(ThisTask == 0)
+ printf("allocated %g Mbyte for ngb search.\n", totbytes / (1024.0 * 1024.0));
+}
+
+
+/*! free memory allocated for neighbour list buffer.
+ */
+void ngb_treefree(void)
+{
+ free(Ngblist);
+}
+
+/*! This function constructs the neighbour tree. To this end, we actually need
+ * to construct the gravitational tree, because we use it now for the
+ * neighbour search.
+ */
+void ngb_treebuild(void)
+{
+ if(ThisTask == 0)
+ printf("Begin Ngb-tree construction.\n");
+
+ force_treebuild(N_gas);
+
+ if(ThisTask == 0)
+ printf("Ngb-Tree contruction finished \n");
+}
+
diff --git a/outerpotential.c b/outerpotential.c
new file mode 100644
index 0000000..18c1145
--- /dev/null
+++ b/outerpotential.c
@@ -0,0 +1,471 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+#include <gsl/gsl_integration.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef OUTERPOTENTIAL
+
+
+
+/**************************************
+*
+* * * * driving functions * * *
+*
+***************************************/
+
+
+void init_outer_potential()
+{
+
+#ifdef NFW
+ init_outer_potential_nfw();
+#endif
+
+#ifdef PLUMMER
+ init_outer_potential_plummer();
+#endif
+
+#ifdef PISOTHERM
+ init_outer_potential_pisotherm();
+#endif
+
+#ifdef CORIOLIS
+ init_outer_potential_coriolis();
+#endif
+}
+
+
+void outer_forces()
+{
+
+#ifdef NFW
+ outer_forces_nfw();
+#endif
+
+#ifdef PLUMMER
+ outer_forces_plummer();
+#endif
+
+#ifdef PISOTHERM
+ outer_forces_pisotherm();
+#endif
+
+#ifdef CORIOLIS
+ outer_forces_coriolis();
+#endif
+
+}
+
+void outer_potential()
+{
+
+#ifdef NFW
+ outer_potential_nfw();
+#endif
+
+#ifdef PLUMMER
+ outer_potential_plummer();
+#endif
+
+#ifdef PISOTHERM
+ outer_potential_pisotherm();
+#endif
+
+#ifdef CORIOLIS
+ outer_potential_coriolis();
+#endif
+
+}
+
+
+
+#ifdef NFW
+/****************************
+*
+* * * * NFW * * *
+*
+*****************************/
+
+
+/*! \file phase.c
+ * \brief Compute forces and potential from the outer potential
+ *
+*/
+
+
+/*! init outer potential constantes
+ *
+ */
+void init_outer_potential_nfw()
+{
+ double rhoc;
+
+ All.HaloMass = All.HaloMass / All.UnitMass_in_g;
+ All.HaloMass = All.HaloMass*1.9891e+33; /* from Msolar in user units */
+ All.NFWPotentialCte = (1.-All.GasMassFraction) * All.G*All.HaloMass/(log(1+All.HaloConcentration) - All.HaloConcentration/(1+All.HaloConcentration));
+
+ /* here, convert HubbleParam in 1/(unit_time) */
+ rhoc = pow(All.HubbleParam*1e5/3.085e+22*All.UnitTime_in_s,2)*3/(8*PI*All.G);
+ All.Rs = pow(( All.HaloMass/(4/3.*PI*200*rhoc) ),(1./3.)) / All.HaloConcentration;
+ printf("\nRs (internal units) = %g\n", All.Rs);
+
+
+}
+
+
+
+/*! compute forces du to the outer potential
+ *
+ */
+void outer_forces_nfw()
+{
+ int i;
+ FLOAT accx, accy, accz;
+ FLOAT r,dPhi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ dPhi = - All.NFWPotentialCte * ( log(1+r/All.Rs) - r/(All.Rs+r) ) / (r*r);
+
+ accx = dPhi * P[i].Pos[0]/r;
+ accy = dPhi * P[i].Pos[1]/r;
+ accz = dPhi * P[i].Pos[2]/r;
+
+ /*
+ if (P[i].ID==8)
+ {
+ printf("------> x=%g y=%g z=%g ax=%g ay=%g az=%g r=%g\n",P[i].Pos[0],P[i].Pos[1],P[i].Pos[2],accx,accy,accz,r);
+ printf("------> x=%g y=%g z=%g ax=%g ay=%g az=%g\n",P[i].Pos[0],P[i].Pos[1],P[i].Pos[2],P[i].GravAccel[0],P[i].GravAccel[1],P[i].GravAccel[2]);
+ }
+ */
+
+
+ P[i].GravAccel[0] = P[i].GravAccel[0] + accx;
+ P[i].GravAccel[1] = P[i].GravAccel[1] + accy;
+ P[i].GravAccel[2] = P[i].GravAccel[2] + accz;
+ }
+ }
+}
+
+/*! compute outer potential
+ *
+ */
+void outer_potential_nfw()
+{
+ int i;
+ FLOAT r,Phi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ Phi = - All.NFWPotentialCte * (log(1+r/All.Rs) / r);
+
+ /*
+ if (P[i].ID==8)
+ {
+ printf("------> x=%g y=%g z=%g Phi=%g\n",P[i].Pos[0],P[i].Pos[1],P[i].Pos[2],Phi);
+ printf("------> x=%g y=%g z=%g Phi=%g\n",P[i].Pos[0],P[i].Pos[1],P[i].Pos[2],P[i].Potential);
+ }
+ */
+
+
+ /* !!! do not forget do multiply by 2 (Springel choice ?) !!! */
+ P[i].Potential = P[i].Potential + 2*Phi;
+ }
+}
+
+
+#endif
+
+
+#ifdef PLUMMER
+/****************************
+*
+* * * * PLUMMER * * *
+*
+*****************************/
+
+
+/*! \file phase.c
+ * \brief Compute forces and potential from the outer potential
+ *
+*/
+
+
+/*! init outer potential constantes
+ *
+ */
+void init_outer_potential_plummer()
+{
+ All.PlummerPotentialCte = All.G*All.PlummerMass /All.UnitMass_in_g *1.9891e+33; /* from Msolar in user units */
+}
+
+
+
+/*! compute forces due to the outer potential
+ *
+ */
+void outer_forces_plummer()
+{
+ int i;
+ FLOAT accx, accy, accz;
+ FLOAT r,dPhi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ dPhi = - All.PlummerPotentialCte / pow(r*r+All.PlummerSoftenning*All.PlummerSoftenning,1.5)*r;
+
+ accx = dPhi * P[i].Pos[0]/r;
+ accy = dPhi * P[i].Pos[1]/r;
+ accz = dPhi * P[i].Pos[2]/r;
+
+
+ P[i].GravAccel[0] = P[i].GravAccel[0] + accx;
+ P[i].GravAccel[1] = P[i].GravAccel[1] + accy;
+ P[i].GravAccel[2] = P[i].GravAccel[2] + accz;
+ }
+ }
+}
+
+/*! compute outer potential
+ *
+ */
+void outer_potential_plummer()
+{
+ int i;
+ FLOAT r,Phi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ Phi = - All.PlummerPotentialCte / pow(r*r+All.PlummerSoftenning*All.PlummerSoftenning,0.5);
+
+
+
+ /* !!! do not forget do multiply by 2 (Springel choice ?) !!! */
+ P[i].Potential = P[i].Potential + 2*Phi;
+ }
+}
+
+
+#endif
+
+
+#ifdef PISOTHERM
+/**********************************************
+*
+* * * * PSEUDO ISOTHERM POTENTIAL * * *
+*
+***********************************************/
+
+/*! \file phase.c
+ * \brief Compute forces and potential from the outer potential
+ *
+*/
+
+
+/*! init outer potential constantes
+ *
+ */
+void init_outer_potential_pisotherm()
+{
+
+
+ All.PisothermPotentialCte = 4*PI*All.Rho0*pow(All.Rc,3) *All.G *(1.-All.GasMassFraction);
+ All.Potentialw = gsl_integration_workspace_alloc (1000);
+ All.PotentialF.function = &potential_f;
+
+ /* potential normalisation */
+ /*All.PotentialInf = All.PisothermPotentialCte/All.Rc; viriel Pfen. */
+ All.PotentialInf = 3.3263051716357208; /* in order to have 2Uint = Epot */
+}
+
+
+/*! potential function
+ *
+ */
+double potential_f (double r, void * params)
+ {
+ double f = All.PisothermPotentialCte*(r/All.Rc-atan(r/All.Rc))/(r*r);
+ return f;
+ }
+
+
+/*! potential function
+ *
+ */
+double get_potential(double r)
+ {
+
+ double result, error;
+ gsl_integration_qags (&All.PotentialF, 0, r, 0, 1e-5, 1000, All.Potentialw, &result, &error);
+
+ return result;
+
+ }
+
+/*! compute forces du to the outer potential
+ *
+ */
+void outer_forces_pisotherm()
+{
+ int i;
+ FLOAT accx, accy, accz;
+ FLOAT r,dPhi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ dPhi = - All.PisothermPotentialCte * ( r/All.Rc - atan(r/All.Rc) ) / (r*r);
+
+ accx = dPhi * P[i].Pos[0]/r;
+ accy = dPhi * P[i].Pos[1]/r;
+ accz = dPhi * P[i].Pos[2]/r;
+
+
+ P[i].GravAccel[0] = P[i].GravAccel[0] + accx;
+ P[i].GravAccel[1] = P[i].GravAccel[1] + accy;
+ P[i].GravAccel[2] = P[i].GravAccel[2] + accz;
+ }
+ }
+}
+
+/*! compute outer potential
+ *
+ */
+void outer_potential_pisotherm()
+{
+ int i;
+ FLOAT Phi;
+ FLOAT r;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ Phi = get_potential(r)-All.PotentialInf;
+
+ /* !!! do not forget do multiply by 2 (Springel choice ?) !!! */
+ P[i].Potential = P[i].Potential + 2*Phi;
+ }
+}
+
+
+#endif
+
+
+
+#ifdef CORIOLIS
+/****************************
+*
+* * * * CORIOLIS * * *
+*
+*****************************/
+
+
+/*! \file phase.c
+ * \brief Compute forces and potential from the outer potential
+ *
+*/
+
+
+/*! init outer potential constantes
+ *
+ */
+void init_outer_potential_coriolis()
+{
+ set_outer_potential_coriolis();
+}
+
+void set_outer_potential_coriolis()
+{
+
+ double amp;
+ double CoriolisTime = 1000;
+
+ amp = All.Time/CoriolisTime;
+
+ if (amp>1)
+ amp = 1;
+
+
+ All.CoriolisOmegaX = All.CoriolisOmegaX0*amp;
+ All.CoriolisOmegaY = All.CoriolisOmegaY0*amp;
+ All.CoriolisOmegaZ = All.CoriolisOmegaZ0*amp;
+
+ printf("coriolis CoriolisOmegaZ = %g\n",All.CoriolisOmegaZ);
+}
+
+/*! compute forces due to the outer potential
+ *
+ */
+void outer_forces_coriolis()
+{
+ int i;
+ FLOAT accx, accy, accz;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ accx = pow(All.CoriolisOmegaZ,2)*P[i].Pos[0] - All.CoriolisOmegaZ*P[i].Vel[1];
+ accy = pow(All.CoriolisOmegaZ,2)*P[i].Pos[1] + All.CoriolisOmegaZ*P[i].Vel[0];
+ accz = 0;
+
+
+ P[i].GravAccel[0] = P[i].GravAccel[0] + accx;
+ P[i].GravAccel[1] = P[i].GravAccel[1] + accy;
+ P[i].GravAccel[2] = P[i].GravAccel[2] + accz;
+ }
+ }
+}
+
+/*! compute outer potential
+ *
+ */
+void outer_potential_coriolis()
+{
+ int i;
+ FLOAT r,Phi;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ r = sqrt(P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2]);
+ Phi = -0.5*pow(All.CoriolisOmegaZ*r,2);
+
+
+ /* !!! do not forget do multiply by 2 (Springel choice ?) !!! */
+ P[i].Potential = P[i].Potential + 2*Phi;
+ }
+}
+
+
+#endif
+
+
+#endif
+
+
+
+
+
+
diff --git a/peano.c b/peano.c
new file mode 100644
index 0000000..b373d05
--- /dev/null
+++ b/peano.c
@@ -0,0 +1,579 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file peano.c
+ * \brief Routines to compute a Peano-Hilbert order
+ *
+ * This file contains routines to compute Peano-Hilbert keys, and to put the
+ * particle data into the order of these keys, i.e. into the order of a
+ * space-filling fractal curve.
+ */
+
+
+static struct peano_hilbert_data
+{
+ peanokey key;
+ int index;
+}
+ *mp;
+
+static int *Id;
+
+
+/*! This function puts the particles into Peano-Hilbert order by sorting them
+ * according to their keys. The latter half already been computed in the
+ * domain decomposition. Since gas particles need to stay at the beginning of
+ * the particle list, they are sorted as a separate block.
+ */
+void peano_hilbert_order(void)
+{
+ int i;
+
+ if(ThisTask == 0)
+ printf("begin Peano-Hilbert order...\n");
+
+ if(N_gas)
+ {
+ mp = malloc(sizeof(struct peano_hilbert_data) * N_gas);
+ Id = malloc(sizeof(int) * N_gas);
+
+ for(i = 0; i < N_gas; i++)
+ {
+ mp[i].index = i;
+ mp[i].key = Key[i];
+ }
+
+ qsort(mp, N_gas, sizeof(struct peano_hilbert_data), compare_key);
+
+ for(i = 0; i < N_gas; i++)
+ Id[mp[i].index] = i;
+
+ reorder_gas();
+
+ free(Id);
+ free(mp);
+ }
+
+#ifdef STELLAR_PROP
+ if(N_stars>0)
+ {
+ mp = malloc(sizeof(struct peano_hilbert_data) * (N_stars));
+ mp -= (N_gas);
+
+ Id = malloc(sizeof(int) * (N_stars));
+ Id -= (N_gas);
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ mp[i].index = i;
+ mp[i].key = Key[i];
+ }
+
+ qsort(mp + N_gas, N_stars, sizeof(struct peano_hilbert_data), compare_key);
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ Id[mp[i].index] = i;
+
+ reorder_stars();
+
+ Id += N_gas;
+ free(Id);
+ mp += N_gas;
+ free(mp);
+ }
+
+ if(NumPart - N_gas - N_stars > 0)
+ {
+ mp = malloc(sizeof(struct peano_hilbert_data) * (NumPart - N_gas - N_stars));
+ mp -= (N_gas+N_stars);
+
+ Id = malloc(sizeof(int) * (NumPart - N_gas - N_stars));
+ Id -= (N_gas+N_stars);
+
+ for(i = N_gas+N_stars; i < NumPart; i++)
+ {
+ mp[i].index = i;
+ mp[i].key = Key[i];
+ }
+
+ qsort(mp + N_gas+N_stars, NumPart - N_gas - N_stars, sizeof(struct peano_hilbert_data), compare_key);
+
+ for(i = N_gas+N_stars; i < NumPart; i++)
+ Id[mp[i].index] = i;
+
+ reorder_particles();
+
+ Id += N_gas+N_stars;
+ free(Id);
+ mp += N_gas+N_stars;
+ free(mp);
+ }
+
+
+ /* now, do StP */
+ if(N_stars>0)
+ {
+ Id = malloc(sizeof(int) * (N_stars));
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ Id[P[i].StPIdx] = i - N_gas;
+ }
+
+ reorder_st();
+ free(Id);
+ }
+
+
+
+#else
+ if(NumPart - N_gas > 0)
+ {
+ mp = malloc(sizeof(struct peano_hilbert_data) * (NumPart - N_gas));
+ mp -= (N_gas);
+
+ Id = malloc(sizeof(int) * (NumPart - N_gas));
+ Id -= (N_gas);
+
+ for(i = N_gas; i < NumPart; i++)
+ {
+ mp[i].index = i;
+ mp[i].key = Key[i];
+ }
+
+ qsort(mp + N_gas, NumPart - N_gas, sizeof(struct peano_hilbert_data), compare_key);
+
+ for(i = N_gas; i < NumPart; i++)
+ Id[mp[i].index] = i;
+
+ reorder_particles();
+
+ Id += N_gas;
+ free(Id);
+ mp += N_gas;
+ free(mp);
+ }
+#endif
+
+
+#ifdef CHECK_ID_CORRESPONDENCE
+
+ if (ThisTask==0)
+ printf("Check id correspondence after peano-hilbert order...\n");
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+
+ //printf("-> i=%d ID=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d\n",i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx);
+
+ if(P[i].ID != StP[i-N_gas].ID)
+ {
+ printf("\nP/StP ID correspondance error\n");
+ printf("(%d) (in peano-hilbert) N_stars=%d N_gas=%d i=%d id=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d StP[P[i].StPIdx].ID=%d\n\n",ThisTask,N_stars,N_gas,i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx,StP[P[i].StPIdx].ID);
+ endrun(1212001);
+ }
+
+
+ if( StP[P[i].StPIdx].PIdx != i )
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in peano-hilbert) N_stars=%d N_gas=%d i=%d id=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d\n\n",ThisTask,N_stars,N_gas,i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx);
+ endrun(1212001);
+ }
+
+
+ if(StP[P[i].StPIdx].ID != P[i].ID)
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in peano-hilbert) N_gas=%d N_stars=%d i=%d Type=%d P.Id=%d P[i].StPIdx=%d StP[P[i].StPIdx].ID=%d \n\n",ThisTask,N_gas,N_stars,i,P[i].Type,P[i].ID, P[i].StPIdx, StP[P[i].StPIdx].ID);
+ endrun(1212002);
+ }
+ }
+
+ if (ThisTask==0)
+ printf("Check id correspondence after peano-hilbert order...\n");
+
+#endif
+
+
+
+ if(ThisTask == 0)
+ printf("Peano-Hilbert done.\n");
+}
+
+
+/*! This function is a comparison kernel for sorting the Peano-Hilbert keys.
+ */
+int compare_key(const void *a, const void *b)
+{
+ if(((struct peano_hilbert_data *) a)->key < (((struct peano_hilbert_data *) b)->key))
+ return -1;
+
+ if(((struct peano_hilbert_data *) a)->key > (((struct peano_hilbert_data *) b)->key))
+ return +1;
+
+ return 0;
+}
+
+
+/*! This function brings the gas particles into the same order as the sorted
+ * keys. (The sort is first done only on the keys themselves and done
+ * directly on the gas particles in order to reduce the amount of data that
+ * needs to be moved in memory. Only once the order is established, the gas
+ * particles are rearranged, such that each particle has to be moved at most
+ * once.)
+ */
+void reorder_gas(void)
+{
+ int i;
+ struct particle_data Psave, Psource;
+ struct sph_particle_data SphPsave, SphPsource;
+
+ int idsource, idsave, dest;
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if(Id[i] != i)
+ {
+ Psource = P[i];
+ SphPsource = SphP[i];
+
+ idsource = Id[i];
+ dest = Id[i];
+
+ do
+ {
+ Psave = P[dest];
+ SphPsave = SphP[dest];
+ idsave = Id[dest];
+
+ P[dest] = Psource;
+ SphP[dest] = SphPsource;
+ Id[dest] = idsource;
+
+ if(dest == i)
+ break;
+
+ Psource = Psave;
+ SphPsource = SphPsave;
+ idsource = idsave;
+
+ dest = idsource;
+ }
+ while(1);
+ }
+ }
+}
+
+#ifdef STELLAR_PROP
+/*! This function brings the stars into the same order as
+ * the sorted keys. (The sort is first done only on the keys themselves and
+ * done directly on the particles in order to reduce the amount of data that
+ * needs to be moved in memory. Only once the order is established, the
+ * particles are rearranged, such that each particle has to be moved at most
+ * once.)
+ */
+void reorder_stars(void)
+{
+ int i;
+ struct particle_data Psave, Psource;
+ int idsource, idsave, dest;
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ if(Id[i] != i)
+ {
+ Psource = P[i];
+ idsource = Id[i];
+
+ dest = Id[i];
+
+ do
+ {
+ Psave = P[dest];
+ idsave = Id[dest];
+
+ P[dest] = Psource;
+ Id[dest] = idsource;
+
+ /* restore the link with Stp */
+ StP[ P[dest].StPIdx ].PIdx = dest;
+
+ if(dest == i)
+ break;
+
+ Psource = Psave;
+ idsource = idsave;
+
+ dest = idsource;
+ }
+ while(1);
+ }
+ }
+}
+
+
+void reorder_st(void)
+{
+ int i;
+ struct st_particle_data StPsave, StPsource;
+ int idsource, idsave, dest;
+
+
+ for(i = 0; i < N_stars; i++)
+ {
+ if(Id[i] != i)
+ {
+ StPsource = StP[i];
+ idsource = Id[i];
+
+ dest = Id[i];
+
+ do
+ {
+ StPsave = StP[dest];
+ idsave = Id[dest];
+
+ StP[dest] = StPsource;
+ Id[dest] = idsource;
+
+ /* restore the link with P */
+ P[ StP[dest].PIdx ].StPIdx = dest;
+
+ if(dest == i)
+ break;
+
+ StPsource = StPsave;
+ idsource = idsave;
+
+ dest = idsource;
+ }
+ while(1);
+ }
+ }
+
+
+
+}
+#endif
+
+
+/*! This function brings the collisionless particles into the same order as
+ * the sorted keys. (The sort is first done only on the keys themselves and
+ * done directly on the particles in order to reduce the amount of data that
+ * needs to be moved in memory. Only once the order is established, the
+ * particles are rearranged, such that each particle has to be moved at most
+ * once.)
+ */
+void reorder_particles(void)
+{
+ int i;
+ struct particle_data Psave, Psource;
+ int idsource, idsave, dest;
+
+#ifdef STELLAR_PROP
+ for(i = N_gas+N_stars; i < NumPart; i++)
+#else
+ for(i = N_gas; i < NumPart; i++)
+#endif
+ {
+ if(Id[i] != i)
+ {
+ Psource = P[i];
+ idsource = Id[i];
+
+ dest = Id[i];
+
+ do
+ {
+ Psave = P[dest];
+ idsave = Id[dest];
+
+ P[dest] = Psource;
+ Id[dest] = idsource;
+
+ if(dest == i)
+ break;
+
+ Psource = Psave;
+ idsource = idsave;
+
+ dest = idsource;
+ }
+ while(1);
+ }
+ }
+}
+
+
+
+
+
+
+static int quadrants[24][2][2][2] = {
+ /* rotx=0, roty=0-3 */
+ {{{0, 7}, {1, 6}}, {{3, 4}, {2, 5}}},
+ {{{7, 4}, {6, 5}}, {{0, 3}, {1, 2}}},
+ {{{4, 3}, {5, 2}}, {{7, 0}, {6, 1}}},
+ {{{3, 0}, {2, 1}}, {{4, 7}, {5, 6}}},
+ /* rotx=1, roty=0-3 */
+ {{{1, 0}, {6, 7}}, {{2, 3}, {5, 4}}},
+ {{{0, 3}, {7, 4}}, {{1, 2}, {6, 5}}},
+ {{{3, 2}, {4, 5}}, {{0, 1}, {7, 6}}},
+ {{{2, 1}, {5, 6}}, {{3, 0}, {4, 7}}},
+ /* rotx=2, roty=0-3 */
+ {{{6, 1}, {7, 0}}, {{5, 2}, {4, 3}}},
+ {{{1, 2}, {0, 3}}, {{6, 5}, {7, 4}}},
+ {{{2, 5}, {3, 4}}, {{1, 6}, {0, 7}}},
+ {{{5, 6}, {4, 7}}, {{2, 1}, {3, 0}}},
+ /* rotx=3, roty=0-3 */
+ {{{7, 6}, {0, 1}}, {{4, 5}, {3, 2}}},
+ {{{6, 5}, {1, 2}}, {{7, 4}, {0, 3}}},
+ {{{5, 4}, {2, 3}}, {{6, 7}, {1, 0}}},
+ {{{4, 7}, {3, 0}}, {{5, 6}, {2, 1}}},
+ /* rotx=4, roty=0-3 */
+ {{{6, 7}, {5, 4}}, {{1, 0}, {2, 3}}},
+ {{{7, 0}, {4, 3}}, {{6, 1}, {5, 2}}},
+ {{{0, 1}, {3, 2}}, {{7, 6}, {4, 5}}},
+ {{{1, 6}, {2, 5}}, {{0, 7}, {3, 4}}},
+ /* rotx=5, roty=0-3 */
+ {{{2, 3}, {1, 0}}, {{5, 4}, {6, 7}}},
+ {{{3, 4}, {0, 7}}, {{2, 5}, {1, 6}}},
+ {{{4, 5}, {7, 6}}, {{3, 2}, {0, 1}}},
+ {{{5, 2}, {6, 1}}, {{4, 3}, {7, 0}}}
+};
+
+
+static int rotxmap_table[24] = { 4, 5, 6, 7, 8, 9, 10, 11,
+ 12, 13, 14, 15, 0, 1, 2, 3, 17, 18, 19, 16, 23, 20, 21, 22
+};
+
+static int rotymap_table[24] = { 1, 2, 3, 0, 16, 17, 18, 19,
+ 11, 8, 9, 10, 22, 23, 20, 21, 14, 15, 12, 13, 4, 5, 6, 7
+};
+
+static int rotx_table[8] = { 3, 0, 0, 2, 2, 0, 0, 1 };
+static int roty_table[8] = { 0, 1, 1, 2, 2, 3, 3, 0 };
+
+static int sense_table[8] = { -1, -1, -1, +1, +1, -1, -1, -1 };
+
+static int flag_quadrants_inverse = 1;
+static char quadrants_inverse_x[24][8];
+static char quadrants_inverse_y[24][8];
+static char quadrants_inverse_z[24][8];
+
+
+/*! This function computes a Peano-Hilbert key for an integer triplet (x,y,z),
+ * with x,y,z in the range between 0 and 2^bits-1.
+ */
+peanokey peano_hilbert_key(int x, int y, int z, int bits)
+{
+ int i, quad, bitx, bity, bitz;
+ int mask, rotation, rotx, roty, sense;
+ peanokey key;
+
+
+ mask = 1 << (bits - 1);
+ key = 0;
+ rotation = 0;
+ sense = 1;
+
+
+ for(i = 0; i < bits; i++, mask >>= 1)
+ {
+ bitx = (x & mask) ? 1 : 0;
+ bity = (y & mask) ? 1 : 0;
+ bitz = (z & mask) ? 1 : 0;
+
+ quad = quadrants[rotation][bitx][bity][bitz];
+
+ key <<= 3;
+ key += (sense == 1) ? (quad) : (7 - quad);
+
+ rotx = rotx_table[quad];
+ roty = roty_table[quad];
+ sense *= sense_table[quad];
+
+ while(rotx > 0)
+ {
+ rotation = rotxmap_table[rotation];
+ rotx--;
+ }
+
+ while(roty > 0)
+ {
+ rotation = rotymap_table[rotation];
+ roty--;
+ }
+ }
+
+ return key;
+}
+
+
+/*! This function computes for a given Peano-Hilbert key, the inverse,
+ * i.e. the integer triplet (x,y,z) with a Peano-Hilbert key equal to the
+ * input key. (This functionality is actually not needed in the present
+ * code.)
+ */
+void peano_hilbert_key_inverse(peanokey key, int bits, int *x, int *y, int *z)
+{
+ int i, keypart, bitx, bity, bitz, mask, quad, rotation, shift;
+ char sense, rotx, roty;
+
+ if(flag_quadrants_inverse)
+ {
+ flag_quadrants_inverse = 0;
+ for(rotation = 0; rotation < 24; rotation++)
+ for(bitx = 0; bitx < 2; bitx++)
+ for(bity = 0; bity < 2; bity++)
+ for(bitz = 0; bitz < 2; bitz++)
+ {
+ quad = quadrants[rotation][bitx][bity][bitz];
+ quadrants_inverse_x[rotation][quad] = bitx;
+ quadrants_inverse_y[rotation][quad] = bity;
+ quadrants_inverse_z[rotation][quad] = bitz;
+ }
+ }
+
+ shift = 3 * (bits - 1);
+ mask = 7 << shift;
+
+ rotation = 0;
+ sense = 1;
+
+ *x = *y = *z = 0;
+
+ for(i = 0; i < bits; i++, mask >>= 3, shift -= 3)
+ {
+ keypart = (key & mask) >> shift;
+
+ quad = (sense == 1) ? (keypart) : (7 - keypart);
+
+ *x = (*x << 1) + quadrants_inverse_x[rotation][quad];
+ *y = (*y << 1) + quadrants_inverse_y[rotation][quad];
+ *z = (*z << 1) + quadrants_inverse_z[rotation][quad];
+
+ rotx = rotx_table[quad];
+ roty = roty_table[quad];
+ sense *= sense_table[quad];
+
+ while(rotx > 0)
+ {
+ rotation = rotxmap_table[rotation];
+ rotx--;
+ }
+
+ while(roty > 0)
+ {
+ rotation = rotymap_table[rotation];
+ roty--;
+ }
+ }
+}
diff --git a/phase.c b/phase.c
new file mode 100644
index 0000000..8093837
--- /dev/null
+++ b/phase.c
@@ -0,0 +1,343 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef MULTIPHASE
+
+
+/*! \file phase.c
+ * \brief Compute phase mixing
+ *
+*/
+
+/*! compute phase mixing
+ *
+ */
+void update_phase()
+{
+ int i;
+ double a3,hubble_a;
+#ifdef PHASE_MIXING
+ FLOAT EgySpec=0;
+#endif
+
+ double dt;
+
+#ifdef COLDGAS_CYCLE
+ double Pwc,Pcw;
+ double flux_in_cgs,X,tau;
+#endif
+
+ double Pcol,ColTime;
+ double DensityNorm;
+
+ int *numlist;
+
+
+ N_sph = 0;
+ N_sticky = 0;
+ N_stickyflaged = 0;
+ N_dark = 0;
+ NumColPotLocal = 0;
+
+
+ if(All.ComovingIntegrationOn)
+ {
+ a3 = All.Time * All.Time * All.Time;
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+
+ }
+ else
+ {
+ a3 = 1;
+ hubble_a = 1;
+ }
+
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+
+ //if(P[i].Ti_endstep == All.Ti_Current) /* if the particle is active */
+ if(P[i].Type == 0)
+ {
+
+
+ SphP[i].StickyFlag = 0;
+
+
+#ifdef PHASE_MIXING
+
+
+
+#ifdef FEEDBACK
+ if(SphP[i].EnergySN==0)
+#endif
+ {
+
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ /* compute temperature from entropy */
+ EgySpec = 1./GAMMA_MINUS1 * pow(SphP[i].Density / a3,GAMMA_MINUS1)* SphP[i].Entropy;
+
+ if (EgySpec < All.CriticalEgySpec) /* to sticky phase */
+ {
+ SphP[i].Phase = GAS_STICKY;
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time; /* may be elligible for collision */
+ SphP[i].Entropy = EgySpec;
+ }
+
+ break;
+
+ case GAS_STICKY:
+ /* compute temperature from specific energy */
+ EgySpec = SphP[i].Entropy;
+
+ if (EgySpec >= All.CriticalEgySpec) /* to sph phase */
+ {
+ SphP[i].Phase = GAS_SPH;
+ SphP[i].Entropy = GAMMA_MINUS1 * EgySpec / pow(SphP[i].Density / a3, GAMMA_MINUS1);
+ }
+
+ break;
+ }
+ }
+
+
+#ifdef COLDGAS_CYCLE
+
+ /*
+
+ Here, we do nothing during the first step, because the UV flux is unknown
+
+ */
+
+
+ if (((SphP[i].Phase==GAS_STICKY)||(SphP[i].Phase==GAS_DARK))&&All.NumCurrentTiStep>0)
+ {
+ /* cold/warm gas cycle */
+
+ /* compute dt */
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+
+ X = 0.;
+
+#ifdef STELLAR_FLUX
+ /* compute stellar flux */
+ flux_in_cgs = SphP[i].EnergyFlux* All.UnitEnergy_in_cgs/All.UnitTime_in_s/pow(All.UnitLength_in_cm, 2);
+ X = X + flux_in_cgs/C / All.HeatingPeSolarEnergyDensity;
+#endif
+
+#ifdef EXTERNAL_FLUX
+ /* compute outer flux */
+ X = X + All.HeatingExternalFLuxEnergyDensity/All.HeatingPeSolarEnergyDensity ;
+#endif
+
+
+
+ if (X>0)
+ {
+ /* warm gas */
+
+ tau = pow(X,+All.ColdGasCycleTransitionParameter) * All.ColdGasCycleTransitionTime;
+ Pwc = 1.0-exp(-dt/tau);
+
+ /* cold gas */
+ tau = pow(X,-All.ColdGasCycleTransitionParameter) * All.ColdGasCycleTransitionTime;
+ Pcw = 1.0-exp(-dt/tau);
+
+ }
+ else
+ {
+ Pwc = 1.0;
+ Pcw = 0.0;
+ }
+
+
+ /* compute transition */
+ switch(SphP[i].Phase)
+ {
+ case GAS_STICKY:
+ if (get_random_number(P[i].ID) < Pwc)
+ {
+ SphP[i].Phase = GAS_DARK;
+ }
+ break;
+
+ case GAS_DARK:
+ if (get_random_number(P[i].ID) < Pcw)
+ {
+ SphP[i].Phase = GAS_STICKY;
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time; /* may be elligible for collision */
+ }
+ break;
+
+ }
+
+ }
+
+#endif
+
+
+#endif
+
+
+
+
+
+ /* determine if a sticky particle may collide or not */
+
+
+ if(SphP[i].Phase==GAS_STICKY)
+ {
+
+
+ if(SphP[i].StickyTime <= All.Time) /* may collide */
+ {
+
+ /* new implementation */
+ SphP[i].StickyFlag = 1;
+
+
+ /* this was the old implementation */
+
+ //dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+ //
+ ///* because mean free path depends on density */
+ //if (All.StickyDensity>0)
+ // {
+ // //ColTime = All.StickyTime * pow(All.StickyDensity/SphP[i].Density,All.StickyDensityPower);
+ // DensityNorm = SphP[i].Density/All.StickyDensity;
+ // ColTime = All.StickyTime / (DensityNorm * pow( (1+pow(DensityNorm/3,2)) ,1/2.));
+ // if (DensityNorm > 10)
+ // {
+ // ColTime = ColTime*1e10; /* cut off */
+ // }
+ // }
+ //else
+ // ColTime = All.StickyTime;
+ //
+ //
+ //Pcol = 1.0-exp(-dt/ColTime);
+ //if (get_random_number(P[i].ID) < Pcol)
+ // {
+ // SphP[i].StickyFlag = 1; /* the particle may collide */
+ // SphP[i].StickyTime = All.Time + All.StickyIdleTime;
+ // NumColPotLocal++;
+ // }
+
+
+
+ }
+ }
+
+
+
+
+
+ } /* end of active particles */ /* P[i].Type==0 */
+
+ /* count number of each type */
+ if(P[i].Type == 0)
+ {
+ switch (SphP[i].Phase)
+ {
+ case GAS_SPH:
+ N_sph++;
+ break;
+
+ case GAS_STICKY:
+ N_sticky++;
+ if (SphP[i].StickyFlag == 1)
+ N_stickyflaged++;
+ break;
+
+ case GAS_DARK:
+ N_dark++;
+ break;
+ }
+ }
+
+
+
+
+ } /* end of main loop */
+
+ /* share results */
+ //MPI_Allreduce(&N_sph, &All.TotN_sph, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_sticky, &All.TotN_sticky, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_stickyflaged, &All.TotN_stickyflaged, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+ //MPI_Allreduce(&N_dark, &All.TotN_dark, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
+
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+
+ MPI_Allgather(&N_sph, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_sph = 0; i < NTask; i++)
+ All.TotN_sph += numlist[i];
+
+ MPI_Allgather(&N_sticky, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_sticky = 0; i < NTask; i++)
+ All.TotN_sticky += numlist[i];
+
+ MPI_Allgather(&N_stickyflaged, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_stickyflaged = 0; i < NTask; i++)
+ All.TotN_stickyflaged += numlist[i];
+
+ MPI_Allgather(&N_dark, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_dark = 0; i < NTask; i++)
+ All.TotN_dark += numlist[i];
+
+ free(numlist);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ if (ThisTask==0)
+ {
+ fprintf(FdPhase, "Step %d, Time: %g GasPart: %d SphPart: %d StickyPart: %d DarkPart: %d StickyPartFlaged: %d\n", All.NumCurrentTiStep, All.Time, (int)All.TotN_gas, (int)All.TotN_sph, (int)All.TotN_sticky,(int)All.TotN_dark, (int)All.TotN_stickyflaged);
+ fflush(FdPhase);
+ }
+
+
+}
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/pm_nonperiodic.c b/pm_nonperiodic.c
new file mode 100644
index 0000000..9f0543a
--- /dev/null
+++ b/pm_nonperiodic.c
@@ -0,0 +1,1445 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+
+/*! \file pm_nonperiodic.c
+ * \brief code for non-periodic FFT to compute long-range PM force
+ */
+
+
+#ifdef PMGRID
+#if !defined (PERIODIC) || defined (PLACEHIGHRESREGION)
+
+#ifdef NOTYPEPREFIX_FFTW
+#include <rfftw_mpi.h>
+#else
+#ifdef DOUBLEPRECISION_FFTW
+#include <drfftw_mpi.h> /* double precision FFTW */
+#else
+#include <srfftw_mpi.h>
+#endif
+#endif
+
+#include "allvars.h"
+#include "proto.h"
+
+#define GRID (2*PMGRID)
+#define GRID2 (2*(GRID/2 + 1))
+
+
+
+static rfftwnd_mpi_plan fft_forward_plan, fft_inverse_plan;
+
+static int slab_to_task[GRID];
+static int *slabs_per_task;
+static int *first_slab_of_task;
+
+static int *meshmin_list, *meshmax_list;
+
+static int slabstart_x, nslab_x, slabstart_y, nslab_y;
+
+static int fftsize, maxfftsize;
+
+static fftw_real *kernel[2], *rhogrid, *forcegrid, *workspace;
+static fftw_complex *fft_of_kernel[2], *fft_of_rhogrid;
+
+/*! This function determines the particle extension of all particles, and for
+ * those types selected with PLACEHIGHRESREGION if this is used, and then
+ * determines the boundaries of the non-periodic FFT-mesh that can be placed
+ * on this region. Note that a sufficient buffer region at the rim of the
+ * occupied part of the mesh needs to be reserved in order to allow a correct
+ * finite differencing using a 4-point formula. In addition, to allow
+ * non-periodic boundaries, the actual FFT mesh used is twice as large in
+ * each dimension compared with PMGRID.
+ */
+void pm_init_regionsize(void)
+{
+ double meshinner[2], xmin[2][3], xmax[2][3];
+ int i, j, t;
+
+ /* find enclosing rectangle */
+
+ for(j = 0; j < 3; j++)
+ {
+ xmin[0][j] = xmin[1][j] = 1.0e36;
+ xmax[0][j] = xmax[1][j] = -1.0e36;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ for(j = 0; j < 3; j++)
+ {
+ t = 0;
+#ifdef PLACEHIGHRESREGION
+ if(((1 << P[i].Type) & (PLACEHIGHRESREGION)))
+ t = 1;
+#endif
+ if(P[i].Pos[j] > xmax[t][j])
+ xmax[t][j] = P[i].Pos[j];
+ if(P[i].Pos[j] < xmin[t][j])
+ xmin[t][j] = P[i].Pos[j];
+ }
+
+ MPI_Allreduce(xmin, All.Xmintot, 6, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+ MPI_Allreduce(xmax, All.Xmaxtot, 6, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+
+ for(j = 0; j < 2; j++)
+ {
+ All.TotalMeshSize[j] = All.Xmaxtot[j][0] - All.Xmintot[j][0];
+ All.TotalMeshSize[j] = dmax(All.TotalMeshSize[j], All.Xmaxtot[j][1] - All.Xmintot[j][1]);
+ All.TotalMeshSize[j] = dmax(All.TotalMeshSize[j], All.Xmaxtot[j][2] - All.Xmintot[j][2]);
+#ifdef ENLARGEREGION
+ All.TotalMeshSize[j] *= ENLARGEREGION;
+#endif
+
+ /* symmetrize the box onto the center */
+ for(i = 0; i < 3; i++)
+ {
+ All.Xmintot[j][i] = (All.Xmintot[j][i] + All.Xmaxtot[j][i]) / 2 - All.TotalMeshSize[j] / 2;
+ All.Xmaxtot[j][i] = All.Xmintot[j][i] + All.TotalMeshSize[j];
+ }
+ }
+
+ /* this will produce enough room for zero-padding and buffer region to
+ allow finite differencing of the potential */
+
+ for(j = 0; j < 2; j++)
+ {
+ meshinner[j] = All.TotalMeshSize[j];
+ All.TotalMeshSize[j] *= 2.001 * (GRID) / ((double) (GRID - 2 - 8));
+ }
+
+ /* move lower left corner by two cells to allow finite differencing of the potential by a 4-point function */
+
+ for(j = 0; j < 2; j++)
+ for(i = 0; i < 3; i++)
+ {
+ All.Corner[j][i] = All.Xmintot[j][i] - 2.0005 * All.TotalMeshSize[j] / GRID;
+ All.UpperCorner[j][i] = All.Corner[j][i] + (GRID / 2 - 1) * (All.TotalMeshSize[j] / GRID);
+ }
+
+
+#ifndef PERIODIC
+ All.Asmth[0] = ASMTH * All.TotalMeshSize[0] / GRID;
+ All.Rcut[0] = RCUT * All.Asmth[0];
+#endif
+
+#ifdef PLACEHIGHRESREGION
+ All.Asmth[1] = ASMTH * All.TotalMeshSize[1] / GRID;
+ All.Rcut[1] = RCUT * All.Asmth[1];
+#endif
+
+#ifdef PLACEHIGHRESREGION
+ if(2 * All.TotalMeshSize[1] / GRID < All.Rcut[0])
+ {
+ All.TotalMeshSize[1] = 2 * (meshinner[1] + 2 * All.Rcut[0]) * (GRID) / ((double) (GRID - 2));
+
+ for(i = 0; i < 3; i++)
+ {
+ All.Corner[1][i] = All.Xmintot[1][i] - 1.0001 * All.Rcut[0];
+ All.UpperCorner[1][i] = All.Corner[1][i] + (GRID / 2 - 1) * (All.TotalMeshSize[1] / GRID);
+ }
+
+ if(2 * All.TotalMeshSize[1] / GRID > All.Rcut[0])
+ {
+ All.TotalMeshSize[1] = 2 * (meshinner[1] + 2 * All.Rcut[0]) * (GRID) / ((double) (GRID - 10));
+
+ for(i = 0; i < 3; i++)
+ {
+ All.Corner[1][i] = All.Xmintot[1][i] - 1.0001 * (All.Rcut[0] + 2 * All.TotalMeshSize[j] / GRID);
+ All.UpperCorner[1][i] = All.Corner[1][i] + (GRID / 2 - 1) * (All.TotalMeshSize[1] / GRID);
+ }
+ }
+
+ All.Asmth[1] = ASMTH * All.TotalMeshSize[1] / GRID;
+ All.Rcut[1] = RCUT * All.Asmth[1];
+ }
+#endif
+
+ if(ThisTask == 0)
+ {
+#ifndef PERIODIC
+ printf("\nAllowed region for isolated PM mesh (coarse):\n");
+ printf("(%g|%g|%g) -> (%g|%g|%g) ext=%g totmeshsize=%g meshsize=%g\n\n",
+ All.Xmintot[0][0], All.Xmintot[0][1], All.Xmintot[0][2],
+ All.Xmaxtot[0][0], All.Xmaxtot[0][1], All.Xmaxtot[0][2], meshinner[0], All.TotalMeshSize[0],
+ All.TotalMeshSize[0] / GRID);
+#endif
+#ifdef PLACEHIGHRESREGION
+ printf("\nAllowed region for isolated PM mesh (high-res):\n");
+ printf("(%g|%g|%g) -> (%g|%g|%g) ext=%g totmeshsize=%g meshsize=%g\n\n",
+ All.Xmintot[1][0], All.Xmintot[1][1], All.Xmintot[1][2],
+ All.Xmaxtot[1][0], All.Xmaxtot[1][1], All.Xmaxtot[1][2],
+ meshinner[1], All.TotalMeshSize[1], All.TotalMeshSize[1] / GRID);
+#endif
+ }
+
+}
+
+/*! Initialization of the non-periodic PM routines. The plan-files for FFTW
+ * are created. Finally, the routine to set-up the non-periodic Greens
+ * function is called.
+ */
+void pm_init_nonperiodic(void)
+{
+ int i, slab_to_task_local[GRID];
+ double bytes_tot = 0;
+ size_t bytes;
+
+ /* Set up the FFTW plan files. */
+
+ fft_forward_plan = rfftw3d_mpi_create_plan(MPI_COMM_WORLD, GRID, GRID, GRID,
+ FFTW_REAL_TO_COMPLEX, FFTW_ESTIMATE | FFTW_IN_PLACE);
+ fft_inverse_plan = rfftw3d_mpi_create_plan(MPI_COMM_WORLD, GRID, GRID, GRID,
+ FFTW_COMPLEX_TO_REAL, FFTW_ESTIMATE | FFTW_IN_PLACE);
+
+ /* Workspace out the ranges on each processor. */
+
+ rfftwnd_mpi_local_sizes(fft_forward_plan, &nslab_x, &slabstart_x, &nslab_y, &slabstart_y, &fftsize);
+
+
+ for(i = 0; i < GRID; i++)
+ slab_to_task_local[i] = 0;
+
+ for(i = 0; i < nslab_x; i++)
+ slab_to_task_local[slabstart_x + i] = ThisTask;
+
+ MPI_Allreduce(slab_to_task_local, slab_to_task, GRID, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ slabs_per_task = malloc(NTask * sizeof(int));
+ MPI_Allgather(&nslab_x, 1, MPI_INT, slabs_per_task, 1, MPI_INT, MPI_COMM_WORLD);
+
+#ifndef PERIODIC
+ if(ThisTask == 0)
+ {
+ for(i = 0; i < NTask; i++)
+ printf("Task=%d FFT-Slabs=%d\n", i, slabs_per_task[i]);
+ }
+#endif
+
+ first_slab_of_task = malloc(NTask * sizeof(int));
+ MPI_Allgather(&slabstart_x, 1, MPI_INT, first_slab_of_task, 1, MPI_INT, MPI_COMM_WORLD);
+
+ meshmin_list = malloc(3 * NTask * sizeof(int));
+ meshmax_list = malloc(3 * NTask * sizeof(int));
+
+ MPI_Allreduce(&fftsize, &maxfftsize, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+
+ /* now allocate memory to hold the FFT fields */
+
+#if !defined(PERIODIC)
+ if(!(kernel[0] = (fftw_real *) malloc(bytes = fftsize * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-kernel[0]' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+ fft_of_kernel[0] = (fftw_complex *) kernel[0];
+#endif
+
+#if defined(PLACEHIGHRESREGION)
+ if(!(kernel[1] = (fftw_real *) malloc(bytes = fftsize * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-kernel[1]' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+ fft_of_kernel[1] = (fftw_complex *) kernel[1];
+#endif
+
+ if(ThisTask == 0)
+ printf("\nAllocated %g MByte for FFT kernel(s).\n\n", bytes_tot / (1024.0 * 1024.0));
+
+}
+
+
+/*! This function allocates the workspace needed for the non-periodic FFT
+ * algorithm. Three fields are used, one for the density/potential fields,
+ * one to hold the force field obtained by finite differencing, and finally
+ * an additional workspace which is used both in the parallel FFT itself, and
+ * as a buffer for the communication algorithm.
+ */
+void pm_init_nonperiodic_allocate(int dimprod)
+{
+ static int first_alloc = 1;
+ int dimprodmax;
+ double bytes_tot = 0;
+ size_t bytes;
+
+ MPI_Allreduce(&dimprod, &dimprodmax, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+
+ if(!(rhogrid = (fftw_real *) malloc(bytes = fftsize * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-rhogrid' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ fft_of_rhogrid = (fftw_complex *) rhogrid;
+
+ if(!(forcegrid = (fftw_real *) malloc(bytes = imax(fftsize, dimprodmax) * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-forcegrid' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(!(workspace = (fftw_real *) malloc(bytes = imax(maxfftsize, dimprodmax) * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-workspace' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(first_alloc == 1)
+ {
+ first_alloc = 0;
+ if(ThisTask == 0)
+ printf("\nUsing %g MByte for non-periodic FFT computation.\n\n", bytes_tot / (1024.0 * 1024.0));
+ }
+}
+
+
+/*! This function frees the memory allocated for the non-periodic FFT
+ * computation. (With the exception of the Greens function(s), which are kept
+ * statically in memory for the next force computation.)
+ */
+void pm_init_nonperiodic_free(void)
+{
+ /* deallocate memory */
+ free(workspace);
+ free(forcegrid);
+ free(rhogrid);
+}
+
+
+/*! This function sets-up the Greens function for the non-periodic potential
+ * in real space, and then converts it to Fourier space by means of a FFT.
+ */
+void pm_setup_nonperiodic_kernel(void)
+{
+ int i, j, k;
+ double x, y, z, r, u, fac;
+ double kx, ky, kz, k2, fx, fy, fz, ff;
+ int ip;
+
+ /* now set up kernel and its Fourier transform */
+
+ pm_init_nonperiodic_allocate(0);
+
+#if !defined(PERIODIC)
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ kernel[0][i] = 0;
+
+ for(i = slabstart_x; i < (slabstart_x + nslab_x); i++)
+ for(j = 0; j < GRID; j++)
+ for(k = 0; k < GRID; k++)
+ {
+ x = ((double) i) / GRID;
+ y = ((double) j) / GRID;
+ z = ((double) k) / GRID;
+
+ if(x >= 0.5)
+ x -= 1.0;
+ if(y >= 0.5)
+ y -= 1.0;
+ if(z >= 0.5)
+ z -= 1.0;
+
+ r = sqrt(x * x + y * y + z * z);
+
+ u = 0.5 * r / (((double) ASMTH) / GRID);
+
+ fac = 1 - erfc(u);
+
+ if(r > 0)
+ kernel[0][GRID * GRID2 * (i - slabstart_x) + GRID2 * j + k] = -fac / r;
+ else
+ kernel[0][GRID * GRID2 * (i - slabstart_x) + GRID2 * j + k] =
+ -1 / (sqrt(M_PI) * (((double) ASMTH) / GRID));
+ }
+
+ /* do the forward transform of the kernel */
+
+ rfftwnd_mpi(fft_forward_plan, 1, kernel[0], workspace, FFTW_TRANSPOSED_ORDER);
+#endif
+
+
+#if defined(PLACEHIGHRESREGION)
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ kernel[1][i] = 0;
+
+ for(i = slabstart_x; i < (slabstart_x + nslab_x); i++)
+ for(j = 0; j < GRID; j++)
+ for(k = 0; k < GRID; k++)
+ {
+ x = ((double) i) / GRID;
+ y = ((double) j) / GRID;
+ z = ((double) k) / GRID;
+
+ if(x >= 0.5)
+ x -= 1.0;
+ if(y >= 0.5)
+ y -= 1.0;
+ if(z >= 0.5)
+ z -= 1.0;
+
+ r = sqrt(x * x + y * y + z * z);
+
+ u = 0.5 * r / (((double) ASMTH) / GRID);
+
+ fac = erfc(u * All.Asmth[1] / All.Asmth[0]) - erfc(u);
+
+ if(r > 0)
+ kernel[1][GRID * GRID2 * (i - slabstart_x) + GRID2 * j + k] = -fac / r;
+ else
+ {
+ fac = 1 - All.Asmth[1] / All.Asmth[0];
+ kernel[1][GRID * GRID2 * (i - slabstart_x) + GRID2 * j + k] =
+ -fac / (sqrt(M_PI) * (((double) ASMTH) / GRID));
+ }
+ }
+
+ /* do the forward transform of the kernel */
+
+ rfftwnd_mpi(fft_forward_plan, 1, kernel[1], workspace, FFTW_TRANSPOSED_ORDER);
+#endif
+
+ /* deconvolve the Greens function twice with the CIC kernel */
+
+ for(y = slabstart_y; y < slabstart_y + nslab_y; y++)
+ for(x = 0; x < GRID; x++)
+ for(z = 0; z < GRID / 2 + 1; z++)
+ {
+ if(x > GRID / 2)
+ kx = x - GRID;
+ else
+ kx = x;
+ if(y > GRID / 2)
+ ky = y - GRID;
+ else
+ ky = y;
+ if(z > GRID / 2)
+ kz = z - GRID;
+ else
+ kz = z;
+
+ k2 = kx * kx + ky * ky + kz * kz;
+
+ if(k2 > 0)
+ {
+ fx = fy = fz = 1;
+ if(kx != 0)
+ {
+ fx = (M_PI * kx) / GRID;
+ fx = sin(fx) / fx;
+ }
+ if(ky != 0)
+ {
+ fy = (M_PI * ky) / GRID;
+ fy = sin(fy) / fy;
+ }
+ if(kz != 0)
+ {
+ fz = (M_PI * kz) / GRID;
+ fz = sin(fz) / fz;
+ }
+ ff = 1 / (fx * fy * fz);
+ ff = ff * ff * ff * ff;
+
+ ip = GRID * (GRID / 2 + 1) * (y - slabstart_y) + (GRID / 2 + 1) * x + z;
+#if !defined(PERIODIC)
+ fft_of_kernel[0][ip].re *= ff;
+ fft_of_kernel[0][ip].im *= ff;
+#endif
+#if defined(PLACEHIGHRESREGION)
+ fft_of_kernel[1][ip].re *= ff;
+ fft_of_kernel[1][ip].im *= ff;
+#endif
+ }
+ }
+ /* end deconvolution */
+
+ pm_init_nonperiodic_free();
+}
+
+
+
+/*! Calculates the long-range non-periodic forces using the PM method. The
+ * potential is Gaussian filtered with Asmth, given in mesh-cell units. The
+ * potential is finite differenced using a 4-point finite differencing
+ * formula to obtain the force fields, which are then interpolated to the
+ * particle positions. We carry out a CIC charge assignment, and compute the
+ * potenial by Fourier transform methods. The CIC kernel is deconvolved.
+ */
+int pmforce_nonperiodic(int grnr)
+{
+ double dx, dy, dz;
+ double fac, to_slab_fac;
+ double re, im, acc_dim;
+ int i, j, slab, level, sendTask, recvTask, flag, flagsum;
+ int x, y, z, xl, yl, zl, xr, yr, zr, xll, yll, zll, xrr, yrr, zrr, ip, dim;
+ int slab_x, slab_y, slab_z;
+ int slab_xx, slab_yy, slab_zz;
+ int meshmin[3], meshmax[3], sendmin, sendmax, recvmin, recvmax;
+ int dimx, dimy, dimz, recv_dimx, recv_dimy, recv_dimz;
+ MPI_Status status;
+
+ if(ThisTask == 0)
+ printf("Starting non-periodic PM calculation (grid=%d).\n", grnr);
+
+ fac = All.G / pow(All.TotalMeshSize[grnr], 4) * pow(All.TotalMeshSize[grnr] / GRID, 3); /* to get potential */
+ fac *= 1 / (2 * All.TotalMeshSize[grnr] / GRID); /* for finite differencing */
+
+ to_slab_fac = GRID / All.TotalMeshSize[grnr];
+
+
+ /* first, establish the extension of the local patch in GRID (for binning) */
+
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = GRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0, flag = 0; i < NumPart; i++)
+ {
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 0 || (grnr == 1 && ((1 << P[i].Type) & (PLACEHIGHRESREGION))))
+#endif
+ {
+ for(j = 0; j < 3; j++)
+ {
+ if(P[i].Pos[j] < All.Xmintot[grnr][j] || P[i].Pos[j] > All.Xmaxtot[grnr][j])
+ {
+ if(flag == 0)
+ {
+ printf
+ ("Particle Id=%d on task=%d with coordinates (%g|%g|%g) lies outside PM mesh.\nStopping\n",
+ (int)P[i].ID, ThisTask, P[i].Pos[0], P[i].Pos[1], P[i].Pos[2]);
+ fflush(stdout);
+ }
+ flag++;
+ break;
+ }
+ }
+ }
+
+ if(flag > 0)
+ continue;
+
+ if(P[i].Pos[0] >= All.Corner[grnr][0] && P[i].Pos[0] < All.UpperCorner[grnr][0])
+ if(P[i].Pos[1] >= All.Corner[grnr][1] && P[i].Pos[1] < All.UpperCorner[grnr][1])
+ if(P[i].Pos[2] >= All.Corner[grnr][2] && P[i].Pos[2] < All.UpperCorner[grnr][2])
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * (P[i].Pos[j] - All.Corner[grnr][j]);
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+ }
+
+
+ MPI_Allreduce(&flag, &flagsum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ if(flagsum > 0)
+ {
+ if(ThisTask == 0)
+ {
+ printf("In total %d particles were outside allowed range.\n", flagsum);
+ fflush(stdout);
+ }
+ return 1; /* error - need to return because particle were outside allowed range */
+ }
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+
+ force_treefree();
+
+ pm_init_nonperiodic_allocate((dimx + 4) * (dimy + 4) * (dimz + 4));
+
+ for(i = 0; i < dimx * dimy * dimz; i++)
+ workspace[i] = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Pos[0] < All.Corner[grnr][0] || P[i].Pos[0] >= All.UpperCorner[grnr][0])
+ continue;
+ if(P[i].Pos[1] < All.Corner[grnr][1] || P[i].Pos[1] >= All.UpperCorner[grnr][1])
+ continue;
+ if(P[i].Pos[2] < All.Corner[grnr][2] || P[i].Pos[2] >= All.UpperCorner[grnr][2])
+ continue;
+
+ slab_x = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]);
+ dx = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]) - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]);
+ dy = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]) - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]);
+ dz = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]) - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * dy * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * dz;
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * dy * dz;
+
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (dx) * dy * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (dx) * (1.0 - dy) * dz;
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (dx) * dy * dz;
+ }
+
+
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ rhogrid[i] = 0;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * GRID;
+ sendmax = -1;
+ for(slab_x = meshmin[0]; slab_x < meshmax[0] + 2; slab_x++)
+ if(slab_to_task[slab_x] == recvTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -1)
+ sendmin = 0;
+
+ /* check how much we have to receive */
+ recvmin = 2 * GRID;
+ recvmax = -1;
+ for(slab_x = meshmin_list[3 * recvTask]; slab_x < meshmax_list[3 * recvTask] + 2; slab_x++)
+ if(slab_to_task[slab_x] == sendTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -1)
+ recvmin = 0;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 2;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 2;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 2;
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE, recvTask,
+ TAG_NONPERIOD_A, forcegrid,
+ (recvmax - recvmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_NONPERIOD_A, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(forcegrid, workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ slab_xx = slab_x - first_slab_of_task[ThisTask];
+
+ if(slab_xx >= 0 && slab_xx < slabs_per_task[ThisTask])
+ {
+ for(slab_y = meshmin_list[3 * recvTask + 1];
+ slab_y <= meshmax_list[3 * recvTask + 1] + 1; slab_y++)
+ {
+ slab_yy = slab_y;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2];
+ slab_z <= meshmax_list[3 * recvTask + 2] + 1; slab_z++)
+ {
+ slab_zz = slab_z;
+
+ rhogrid[GRID * GRID2 * slab_xx + GRID2 * slab_yy + slab_zz] +=
+ forcegrid[((slab_x - recvmin) * recv_dimy +
+ (slab_y - meshmin_list[3 * recvTask + 1])) * recv_dimz +
+ (slab_z - meshmin_list[3 * recvTask + 2])];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ /* Do the FFT of the density field */
+
+ rfftwnd_mpi(fft_forward_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+
+ /* multiply with the Fourier transform of the Green's function (kernel) */
+
+ for(y = 0; y < nslab_y; y++)
+ for(x = 0; x < GRID; x++)
+ for(z = 0; z < GRID / 2 + 1; z++)
+ {
+ ip = GRID * (GRID / 2 + 1) * y + (GRID / 2 + 1) * x + z;
+
+ re =
+ fft_of_rhogrid[ip].re * fft_of_kernel[grnr][ip].re -
+ fft_of_rhogrid[ip].im * fft_of_kernel[grnr][ip].im;
+
+ im =
+ fft_of_rhogrid[ip].re * fft_of_kernel[grnr][ip].im +
+ fft_of_rhogrid[ip].im * fft_of_kernel[grnr][ip].re;
+
+ fft_of_rhogrid[ip].re = re;
+ fft_of_rhogrid[ip].im = im;
+ }
+
+ /* get the potential by inverse FFT */
+
+ rfftwnd_mpi(fft_inverse_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* Now rhogrid holds the potential */
+ /* construct the potential for the local patch */
+
+
+ /* if we have a high-res mesh, establish the extension of the local patch in GRID (for reading out the
+ * forces)
+ */
+
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 1)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = GRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(!((1 << P[i].Type) & (PLACEHIGHRESREGION)))
+ continue;
+
+
+ if(P[i].Pos[0] >= All.Corner[grnr][0] && P[i].Pos[0] < All.UpperCorner[grnr][0])
+ if(P[i].Pos[1] >= All.Corner[grnr][1] && P[i].Pos[1] < All.UpperCorner[grnr][1])
+ if(P[i].Pos[2] >= All.Corner[grnr][2] && P[i].Pos[2] < All.UpperCorner[grnr][2])
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * (P[i].Pos[j] - All.Corner[grnr][j]);
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+ }
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+ }
+#endif
+
+ dimx = meshmax[0] - meshmin[0] + 6;
+ dimy = meshmax[1] - meshmin[1] + 6;
+ dimz = meshmax[2] - meshmin[2] + 6;
+
+ for(j = 0; j < 3; j++)
+ {
+ if(meshmin[j] < 2)
+ endrun(131231);
+ if(meshmax[j] > GRID / 2 - 3)
+ endrun(131288);
+ }
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * GRID;
+ sendmax = -GRID;
+ for(slab_x = meshmin_list[3 * recvTask] - 2; slab_x < meshmax_list[3 * recvTask] + 4; slab_x++)
+ if(slab_to_task[slab_x] == sendTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -GRID)
+ sendmin = sendmax + 1;
+
+
+ /* check how much we have to receive */
+ recvmin = 2 * GRID;
+ recvmax = -GRID;
+ for(slab_x = meshmin[0] - 2; slab_x < meshmax[0] + 4; slab_x++)
+ if(slab_to_task[slab_x] == recvTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -GRID)
+ recvmin = recvmax + 1;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 6;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 6;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 6;
+
+ /* prepare what we want to send */
+ if(sendmax - sendmin >= 0)
+ {
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ slab_xx = slab_x - first_slab_of_task[ThisTask];
+
+ for(slab_y = meshmin_list[3 * recvTask + 1] - 2;
+ slab_y < meshmax_list[3 * recvTask + 1] + 4; slab_y++)
+ {
+ slab_yy = slab_y;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2] - 2;
+ slab_z < meshmax_list[3 * recvTask + 2] + 4; slab_z++)
+ {
+ slab_zz = slab_z;
+
+ forcegrid[((slab_x - sendmin) * recv_dimy +
+ (slab_y - (meshmin_list[3 * recvTask + 1] - 2))) * recv_dimz +
+ slab_z - (meshmin_list[3 * recvTask + 2] - 2)] =
+ rhogrid[GRID * GRID2 * slab_xx + GRID2 * slab_yy + slab_zz];
+ }
+ }
+ }
+ }
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(forcegrid,
+ (sendmax - sendmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real),
+ MPI_BYTE, recvTask, TAG_NONPERIOD_B,
+ workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_NONPERIOD_B, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ forcegrid, (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+ }
+ }
+ }
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ recv_dimx = meshmax[0] - meshmin[0] + 6;
+ recv_dimy = meshmax[1] - meshmin[1] + 6;
+ recv_dimz = meshmax[2] - meshmin[2] + 6;
+
+
+ for(dim = 0; dim < 3; dim++) /* Calculate each component of the force. */
+ {
+ /* get the force component by finite differencing the potential */
+ /* note: "workspace" now contains the potential for the local patch, plus a suffiently large buffer region */
+
+ for(x = 0; x < meshmax[0] - meshmin[0] + 2; x++)
+ for(y = 0; y < meshmax[1] - meshmin[1] + 2; y++)
+ for(z = 0; z < meshmax[2] - meshmin[2] + 2; z++)
+ {
+ xrr = xll = xr = xl = x;
+ yrr = yll = yr = yl = y;
+ zrr = zll = zr = zl = z;
+
+ switch (dim)
+ {
+ case 0:
+ xr = x + 1;
+ xrr = x + 2;
+ xl = x - 1;
+ xll = x - 2;
+ break;
+ case 1:
+ yr = y + 1;
+ yl = y - 1;
+ yrr = y + 2;
+ yll = y - 2;
+ break;
+ case 2:
+ zr = z + 1;
+ zl = z - 1;
+ zrr = z + 2;
+ zll = z - 2;
+ break;
+ }
+
+ forcegrid[(x * dimy + y) * dimz + z]
+ =
+ fac * ((4.0 / 3) *
+ (workspace[((xl + 2) * recv_dimy + (yl + 2)) * recv_dimz + (zl + 2)]
+ - workspace[((xr + 2) * recv_dimy + (yr + 2)) * recv_dimz + (zr + 2)]) -
+ (1.0 / 6) *
+ (workspace[((xll + 2) * recv_dimy + (yll + 2)) * recv_dimz + (zll + 2)] -
+ workspace[((xrr + 2) * recv_dimy + (yrr + 2)) * recv_dimz + (zrr + 2)]));
+ }
+
+
+ /* read out the forces */
+
+ for(i = 0; i < NumPart; i++)
+ {
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 1)
+ if(!((1 << P[i].Type) & (PLACEHIGHRESREGION)))
+ continue;
+#endif
+ slab_x = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]);
+ dx = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]) - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]);
+ dy = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]) - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]);
+ dz = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]) - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ acc_dim =
+ forcegrid[(slab_x * dimy + slab_y) * dimz + slab_z] * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ acc_dim += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_z] * (1.0 - dx) * dy * (1.0 - dz);
+ acc_dim += forcegrid[(slab_x * dimy + slab_y) * dimz + slab_zz] * (1.0 - dx) * (1.0 - dy) * dz;
+ acc_dim += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_zz] * (1.0 - dx) * dy * dz;
+
+ acc_dim += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_z] * (dx) * (1.0 - dy) * (1.0 - dz);
+ acc_dim += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_z] * (dx) * dy * (1.0 - dz);
+ acc_dim += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_zz] * (dx) * (1.0 - dy) * dz;
+ acc_dim += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_zz] * (dx) * dy * dz;
+
+ P[i].GravPM[dim] += acc_dim;
+ }
+ }
+
+ pm_init_nonperiodic_free();
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+
+ if(ThisTask == 0)
+ printf("done PM.\n");
+
+ return 0;
+}
+
+
+
+
+/*! Calculates the long-range non-periodic potential using the PM method. The
+ * potential is Gaussian filtered with Asmth, given in mesh-cell units. We
+ * carry out a CIC charge assignment, and compute the potenial by Fourier
+ * transform methods. The CIC kernel is deconvolved.
+ */
+int pmpotential_nonperiodic(int grnr)
+{
+ double dx, dy, dz;
+ double fac, to_slab_fac;
+ double re, im, pot;
+ int i, j, slab, level, sendTask, recvTask, flag, flagsum;
+ int x, y, z, ip;
+ int slab_x, slab_y, slab_z;
+ int slab_xx, slab_yy, slab_zz;
+ int meshmin[3], meshmax[3], sendmin, sendmax, recvmin, recvmax;
+ int dimx, dimy, dimz, recv_dimx, recv_dimy, recv_dimz;
+ MPI_Status status;
+
+
+ if(ThisTask == 0)
+ printf("Starting non-periodic PM-potential calculation.\n");
+
+ fac = All.G / pow(All.TotalMeshSize[grnr], 4) * pow(All.TotalMeshSize[grnr] / GRID, 3); /* to get potential */
+
+ to_slab_fac = GRID / All.TotalMeshSize[grnr];
+
+ /* first, establish the extension of the local patch in GRID (for binning) */
+
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = GRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0, flag = 0; i < NumPart; i++)
+ {
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 0 || (grnr == 1 && ((1 << P[i].Type) & (PLACEHIGHRESREGION))))
+#endif
+ {
+ for(j = 0; j < 3; j++)
+ {
+ if(P[i].Pos[j] < All.Xmintot[grnr][j] || P[i].Pos[j] > All.Xmaxtot[grnr][j])
+ {
+ if(flag == 0)
+ {
+ printf
+ ("Particle Id=%d on task=%d with coordinates (%g|%g|%g) lies outside PM mesh.\nStopping\n",
+ (int)P[i].ID, ThisTask, P[i].Pos[0], P[i].Pos[1], P[i].Pos[2]);
+ fflush(stdout);
+ }
+ flag++;
+ break;
+ }
+ }
+ }
+
+ if(flag > 0)
+ continue;
+
+ if(P[i].Pos[0] >= All.Corner[grnr][0] && P[i].Pos[0] < All.UpperCorner[grnr][0])
+ if(P[i].Pos[1] >= All.Corner[grnr][1] && P[i].Pos[1] < All.UpperCorner[grnr][1])
+ if(P[i].Pos[2] >= All.Corner[grnr][2] && P[i].Pos[2] < All.UpperCorner[grnr][2])
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * (P[i].Pos[j] - All.Corner[grnr][j]);
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+ }
+
+
+ MPI_Allreduce(&flag, &flagsum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ if(flagsum > 0)
+ {
+ if(ThisTask == 0)
+ {
+ printf("In total %d particles were outside allowed range.\n", flagsum);
+ fflush(stdout);
+ }
+ return 1; /* error - need to return because particle were outside allowed range */
+ }
+
+
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+
+ force_treefree();
+
+ pm_init_nonperiodic_allocate((dimx + 4) * (dimy + 4) * (dimz + 4));
+
+ for(i = 0; i < dimx * dimy * dimz; i++)
+ workspace[i] = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Pos[0] < All.Corner[grnr][0] || P[i].Pos[0] >= All.UpperCorner[grnr][0])
+ continue;
+ if(P[i].Pos[1] < All.Corner[grnr][1] || P[i].Pos[1] >= All.UpperCorner[grnr][1])
+ continue;
+ if(P[i].Pos[2] < All.Corner[grnr][2] || P[i].Pos[2] >= All.UpperCorner[grnr][2])
+ continue;
+
+ slab_x = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]);
+ dx = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]) - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]);
+ dy = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]) - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]);
+ dz = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]) - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * dy * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * dz;
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * dy * dz;
+
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (dx) * dy * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (dx) * (1.0 - dy) * dz;
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (dx) * dy * dz;
+ }
+
+
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ rhogrid[i] = 0;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * GRID;
+ sendmax = -1;
+ for(slab_x = meshmin[0]; slab_x < meshmax[0] + 2; slab_x++)
+ if(slab_to_task[slab_x] == recvTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -1)
+ sendmin = 0;
+
+ /* check how much we have to receive */
+ recvmin = 2 * GRID;
+ recvmax = -1;
+ for(slab_x = meshmin_list[3 * recvTask]; slab_x < meshmax_list[3 * recvTask] + 2; slab_x++)
+ if(slab_to_task[slab_x] == sendTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -1)
+ recvmin = 0;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 2;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 2;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 2;
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE, recvTask,
+ TAG_NONPERIOD_C, forcegrid,
+ (recvmax - recvmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_NONPERIOD_C, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(forcegrid, workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ slab_xx = slab_x - first_slab_of_task[ThisTask];
+
+ if(slab_xx >= 0 && slab_xx < slabs_per_task[ThisTask])
+ {
+ for(slab_y = meshmin_list[3 * recvTask + 1];
+ slab_y <= meshmax_list[3 * recvTask + 1] + 1; slab_y++)
+ {
+ slab_yy = slab_y;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2];
+ slab_z <= meshmax_list[3 * recvTask + 2] + 1; slab_z++)
+ {
+ slab_zz = slab_z;
+
+ rhogrid[GRID * GRID2 * slab_xx + GRID2 * slab_yy + slab_zz] +=
+ forcegrid[((slab_x - recvmin) * recv_dimy +
+ (slab_y - meshmin_list[3 * recvTask + 1])) * recv_dimz +
+ (slab_z - meshmin_list[3 * recvTask + 2])];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ /* Do the FFT of the density field */
+
+ rfftwnd_mpi(fft_forward_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+
+ /* multiply with the Fourier transform of the Green's function (kernel) */
+
+ for(y = 0; y < nslab_y; y++)
+ for(x = 0; x < GRID; x++)
+ for(z = 0; z < GRID / 2 + 1; z++)
+ {
+ ip = GRID * (GRID / 2 + 1) * y + (GRID / 2 + 1) * x + z;
+
+ re =
+ fft_of_rhogrid[ip].re * fft_of_kernel[grnr][ip].re -
+ fft_of_rhogrid[ip].im * fft_of_kernel[grnr][ip].im;
+
+ im =
+ fft_of_rhogrid[ip].re * fft_of_kernel[grnr][ip].im +
+ fft_of_rhogrid[ip].im * fft_of_kernel[grnr][ip].re;
+
+ fft_of_rhogrid[ip].re = fac * re;
+ fft_of_rhogrid[ip].im = fac * im;
+ }
+
+ /* get the potential by inverse FFT */
+
+ rfftwnd_mpi(fft_inverse_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* Now rhogrid holds the potential */
+ /* construct the potential for the local patch */
+
+
+ /* if we have a high-res mesh, establish the extension of the local patch in GRID (for reading out the
+ * forces)
+ */
+
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 1)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = GRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(!((1 << P[i].Type) & (PLACEHIGHRESREGION)))
+ continue;
+
+
+ if(P[i].Pos[0] >= All.Corner[grnr][0] && P[i].Pos[0] < All.UpperCorner[grnr][0])
+ if(P[i].Pos[1] >= All.Corner[grnr][1] && P[i].Pos[1] < All.UpperCorner[grnr][1])
+ if(P[i].Pos[2] >= All.Corner[grnr][2] && P[i].Pos[2] < All.UpperCorner[grnr][2])
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * (P[i].Pos[j] - All.Corner[grnr][j]);
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+ }
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+ }
+#endif
+
+ dimx = meshmax[0] - meshmin[0] + 6;
+ dimy = meshmax[1] - meshmin[1] + 6;
+ dimz = meshmax[2] - meshmin[2] + 6;
+
+ for(j = 0; j < 3; j++)
+ {
+ if(meshmin[j] < 2)
+ endrun(131231);
+ if(meshmax[j] > GRID / 2 - 3)
+ endrun(131288);
+ }
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * GRID;
+ sendmax = -GRID;
+ for(slab_x = meshmin_list[3 * recvTask] - 2; slab_x < meshmax_list[3 * recvTask] + 4; slab_x++)
+ if(slab_to_task[slab_x] == sendTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -GRID)
+ sendmin = sendmax + 1;
+
+
+ /* check how much we have to receive */
+ recvmin = 2 * GRID;
+ recvmax = -GRID;
+ for(slab_x = meshmin[0] - 2; slab_x < meshmax[0] + 4; slab_x++)
+ if(slab_to_task[slab_x] == recvTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -GRID)
+ recvmin = recvmax + 1;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 6;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 6;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 6;
+
+ /* prepare what we want to send */
+ if(sendmax - sendmin >= 0)
+ {
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ slab_xx = slab_x - first_slab_of_task[ThisTask];
+
+ for(slab_y = meshmin_list[3 * recvTask + 1] - 2;
+ slab_y < meshmax_list[3 * recvTask + 1] + 4; slab_y++)
+ {
+ slab_yy = slab_y;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2] - 2;
+ slab_z < meshmax_list[3 * recvTask + 2] + 4; slab_z++)
+ {
+ slab_zz = slab_z;
+
+ forcegrid[((slab_x - sendmin) * recv_dimy +
+ (slab_y - (meshmin_list[3 * recvTask + 1] - 2))) * recv_dimz +
+ slab_z - (meshmin_list[3 * recvTask + 2] - 2)] =
+ rhogrid[GRID * GRID2 * slab_xx + GRID2 * slab_yy + slab_zz];
+ }
+ }
+ }
+ }
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(forcegrid,
+ (sendmax - sendmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real),
+ MPI_BYTE, recvTask, TAG_NONPERIOD_D,
+ workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_NONPERIOD_D, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ forcegrid, (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+ }
+ }
+ }
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ recv_dimx = meshmax[0] - meshmin[0] + 6;
+ recv_dimy = meshmax[1] - meshmin[1] + 6;
+ recv_dimz = meshmax[2] - meshmin[2] + 6;
+
+
+ for(x = 0; x < meshmax[0] - meshmin[0] + 2; x++)
+ for(y = 0; y < meshmax[1] - meshmin[1] + 2; y++)
+ for(z = 0; z < meshmax[2] - meshmin[2] + 2; z++)
+ {
+ forcegrid[(x * dimy + y) * dimz + z]
+ = workspace[((x + 2) * recv_dimy + (y + 2)) * recv_dimz + (z + 2)];
+ }
+
+
+ /* read out the potential */
+
+ for(i = 0; i < NumPart; i++)
+ {
+#ifdef PLACEHIGHRESREGION
+ if(grnr == 1)
+ if(!((1 << P[i].Type) & (PLACEHIGHRESREGION)))
+ continue;
+#endif
+ slab_x = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]);
+ dx = to_slab_fac * (P[i].Pos[0] - All.Corner[grnr][0]) - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]);
+ dy = to_slab_fac * (P[i].Pos[1] - All.Corner[grnr][1]) - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]);
+ dz = to_slab_fac * (P[i].Pos[2] - All.Corner[grnr][2]) - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ pot = forcegrid[(slab_x * dimy + slab_y) * dimz + slab_z] * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ pot += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_z] * (1.0 - dx) * dy * (1.0 - dz);
+ pot += forcegrid[(slab_x * dimy + slab_y) * dimz + slab_zz] * (1.0 - dx) * (1.0 - dy) * dz;
+ pot += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_zz] * (1.0 - dx) * dy * dz;
+
+ pot += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_z] * (dx) * (1.0 - dy) * (1.0 - dz);
+ pot += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_z] * (dx) * dy * (1.0 - dz);
+ pot += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_zz] * (dx) * (1.0 - dy) * dz;
+ pot += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_zz] * (dx) * dy * dz;
+
+ P[i].Potential += pot;
+ }
+
+ pm_init_nonperiodic_free();
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+
+ if(ThisTask == 0)
+ printf("done PM-potential.\n");
+
+ return 0;
+}
+
+
+#endif
+#endif
diff --git a/pm_periodic.c b/pm_periodic.c
new file mode 100644
index 0000000..513a847
--- /dev/null
+++ b/pm_periodic.c
@@ -0,0 +1,1158 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <float.h>
+#include <mpi.h>
+
+/*! \file pm_periodic.c
+ * \brief routines for periodic PM-force computation
+ */
+
+#ifdef PMGRID
+#ifdef PERIODIC
+
+#ifdef NOTYPEPREFIXFFTW
+#include <rfftw_mpi.h>
+#else
+#ifdef DOUBLEPRECISION_FFTW
+#include <drfftw_mpi.h> /* double precision FFTW */
+#else
+#include <srfftw_mpi.h>
+#endif
+#endif
+
+
+#include "allvars.h"
+#include "proto.h"
+
+#define PMGRID2 (2*(PMGRID/2 + 1))
+
+
+
+
+static rfftwnd_mpi_plan fft_forward_plan, fft_inverse_plan;
+
+static int slab_to_task[PMGRID];
+static int *slabs_per_task;
+static int *first_slab_of_task;
+static int *meshmin_list, *meshmax_list;
+
+static int slabstart_x, nslab_x, slabstart_y, nslab_y, smallest_slab;
+
+static int fftsize, maxfftsize;
+
+static fftw_real *rhogrid, *forcegrid, *workspace;
+static fftw_complex *fft_of_rhogrid;
+
+
+static FLOAT to_slab_fac;
+
+
+/*! This routines generates the FFTW-plans to carry out the parallel FFTs
+ * later on. Some auxiliary variables are also initialized.
+ */
+void pm_init_periodic(void)
+{
+ int i;
+ int slab_to_task_local[PMGRID];
+
+ All.Asmth[0] = ASMTH * All.BoxSize / PMGRID;
+ All.Rcut[0] = RCUT * All.Asmth[0];
+
+ /* Set up the FFTW plan files. */
+
+ fft_forward_plan = rfftw3d_mpi_create_plan(MPI_COMM_WORLD, PMGRID, PMGRID, PMGRID,
+ FFTW_REAL_TO_COMPLEX, FFTW_ESTIMATE | FFTW_IN_PLACE);
+ fft_inverse_plan = rfftw3d_mpi_create_plan(MPI_COMM_WORLD, PMGRID, PMGRID, PMGRID,
+ FFTW_COMPLEX_TO_REAL, FFTW_ESTIMATE | FFTW_IN_PLACE);
+
+ /* Workspace out the ranges on each processor. */
+
+ rfftwnd_mpi_local_sizes(fft_forward_plan, &nslab_x, &slabstart_x, &nslab_y, &slabstart_y, &fftsize);
+
+ for(i = 0; i < PMGRID; i++)
+ slab_to_task_local[i] = 0;
+
+ for(i = 0; i < nslab_x; i++)
+ slab_to_task_local[slabstart_x + i] = ThisTask;
+
+ MPI_Allreduce(slab_to_task_local, slab_to_task, PMGRID, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ MPI_Allreduce(&nslab_x, &smallest_slab, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+
+ slabs_per_task = malloc(NTask * sizeof(int));
+ MPI_Allgather(&nslab_x, 1, MPI_INT, slabs_per_task, 1, MPI_INT, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ for(i = 0; i < NTask; i++)
+ printf("Task=%d FFT-Slabs=%d\n", i, slabs_per_task[i]);
+ }
+
+ first_slab_of_task = malloc(NTask * sizeof(int));
+ MPI_Allgather(&slabstart_x, 1, MPI_INT, first_slab_of_task, 1, MPI_INT, MPI_COMM_WORLD);
+
+ meshmin_list = malloc(3 * NTask * sizeof(int));
+ meshmax_list = malloc(3 * NTask * sizeof(int));
+
+
+ to_slab_fac = PMGRID / All.BoxSize;
+
+ MPI_Allreduce(&fftsize, &maxfftsize, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+}
+
+
+/*! This function allocates the memory neeed to compute the long-range PM
+ * force. Three fields are used, one to hold the density (and its FFT, and
+ * then the real-space potential), one to hold the force field obtained by
+ * finite differencing, and finally a workspace field, which is used both as
+ * workspace for the parallel FFT, and as buffer for the communication
+ * algorithm used in the force computation.
+ */
+void pm_init_periodic_allocate(int dimprod)
+{
+ static int first_alloc = 1;
+ int dimprodmax;
+ double bytes_tot = 0;
+ size_t bytes;
+
+ MPI_Allreduce(&dimprod, &dimprodmax, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+
+ /* allocate the memory to hold the FFT fields */
+
+ if(!(rhogrid = (fftw_real *) malloc(bytes = fftsize * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-rhogrid' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+
+ if(!(forcegrid = (fftw_real *) malloc(bytes = imax(fftsize, dimprodmax) * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-forcegrid' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(!(workspace = (fftw_real *) malloc(bytes = imax(maxfftsize, dimprodmax) * sizeof(fftw_real))))
+ {
+ printf("failed to allocate memory for `FFT-workspace' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(1);
+ }
+ bytes_tot += bytes;
+
+ if(first_alloc == 1)
+ {
+ first_alloc = 0;
+ if(ThisTask == 0)
+ printf("\nAllocated %g MByte for FFT data.\n\n", bytes_tot / (1024.0 * 1024.0));
+ }
+
+ fft_of_rhogrid = (fftw_complex *) & rhogrid[0];
+}
+
+
+
+/*! This routine frees the space allocated for the parallel FFT algorithm.
+ */
+void pm_init_periodic_free(void)
+{
+ /* allocate the memory to hold the FFT fields */
+ free(workspace);
+ free(forcegrid);
+ free(rhogrid);
+}
+
+
+
+/*! Calculates the long-range periodic force given the particle positions
+ * using the PM method. The force is Gaussian filtered with Asmth, given in
+ * mesh-cell units. We carry out a CIC charge assignment, and compute the
+ * potenial by Fourier transform methods. The potential is finite differenced
+ * using a 4-point finite differencing formula, and the forces are
+ * interpolated tri-linearly to the particle positions. The CIC kernel is
+ * deconvolved. Note that the particle distribution is not in the slab
+ * decomposition that is used for the FFT. Instead, overlapping patches
+ * between local domains and FFT slabs are communicated as needed.
+ */
+void pmforce_periodic(void)
+{
+ double k2, kx, ky, kz, smth;
+ double dx, dy, dz;
+ double fx, fy, fz, ff;
+ double asmth2, fac, acc_dim;
+ int i, j, slab, level, sendTask, recvTask;
+ int x, y, z, xl, yl, zl, xr, yr, zr, xll, yll, zll, xrr, yrr, zrr, ip, dim;
+ int slab_x, slab_y, slab_z;
+ int slab_xx, slab_yy, slab_zz;
+ int meshmin[3], meshmax[3], sendmin, sendmax, recvmin, recvmax;
+ int rep, ncont, cont_sendmin[2], cont_sendmax[2], cont_recvmin[2], cont_recvmax[2];
+ int dimx, dimy, dimz, recv_dimx, recv_dimy, recv_dimz;
+ MPI_Status status;
+
+
+ if(ThisTask == 0)
+ {
+ printf("Starting periodic PM calculation.\n");
+ fflush(stdout);
+ }
+
+
+ force_treefree();
+
+
+ asmth2 = (2 * M_PI) * All.Asmth[0] / All.BoxSize;
+ asmth2 *= asmth2;
+
+ fac = All.G / (M_PI * All.BoxSize); /* to get potential */
+ fac *= 1 / (2 * All.BoxSize / PMGRID); /* for finite differencing */
+
+ /* first, establish the extension of the local patch in the PMGRID */
+
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = PMGRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * P[i].Pos[j];
+ if(slab >= PMGRID)
+ slab = PMGRID - 1;
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ pm_init_periodic_allocate((dimx + 4) * (dimy + 4) * (dimz + 4));
+
+ for(i = 0; i < dimx * dimy * dimz; i++)
+ workspace[i] = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ slab_x = to_slab_fac * P[i].Pos[0];
+ if(slab_x >= PMGRID)
+ slab_x = PMGRID - 1;
+ dx = to_slab_fac * P[i].Pos[0] - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * P[i].Pos[1];
+ if(slab_y >= PMGRID)
+ slab_y = PMGRID - 1;
+ dy = to_slab_fac * P[i].Pos[1] - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * P[i].Pos[2];
+ if(slab_z >= PMGRID)
+ slab_z = PMGRID - 1;
+ dz = to_slab_fac * P[i].Pos[2] - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * dy * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * dz;
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * dy * dz;
+
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (dx) * dy * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (dx) * (1.0 - dy) * dz;
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (dx) * dy * dz;
+ }
+
+
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ rhogrid[i] = 0;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * PMGRID;
+ sendmax = -1;
+ for(slab_x = meshmin[0]; slab_x < meshmax[0] + 2; slab_x++)
+ if(slab_to_task[slab_x % PMGRID] == recvTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -1)
+ sendmin = 0;
+
+ /* check how much we have to receive */
+ recvmin = 2 * PMGRID;
+ recvmax = -1;
+ for(slab_x = meshmin_list[3 * recvTask]; slab_x < meshmax_list[3 * recvTask] + 2; slab_x++)
+ if(slab_to_task[slab_x % PMGRID] == sendTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -1)
+ recvmin = 0;
+
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 2;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 2;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 2;
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE, recvTask,
+ TAG_PERIODIC_A, forcegrid,
+ (recvmax - recvmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_PERIODIC_A, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(forcegrid, workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ slab_xx = (slab_x % PMGRID) - first_slab_of_task[ThisTask];
+
+ if(slab_xx >= 0 && slab_xx < slabs_per_task[ThisTask])
+ {
+ for(slab_y = meshmin_list[3 * recvTask + 1];
+ slab_y <= meshmax_list[3 * recvTask + 1] + 1; slab_y++)
+ {
+ slab_yy = slab_y;
+ if(slab_yy >= PMGRID)
+ slab_yy -= PMGRID;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2];
+ slab_z <= meshmax_list[3 * recvTask + 2] + 1; slab_z++)
+ {
+ slab_zz = slab_z;
+ if(slab_zz >= PMGRID)
+ slab_zz -= PMGRID;
+
+ rhogrid[PMGRID * PMGRID2 * slab_xx + PMGRID2 * slab_yy + slab_zz] +=
+ forcegrid[((slab_x - recvmin) * recv_dimy +
+ (slab_y - meshmin_list[3 * recvTask + 1])) * recv_dimz +
+ (slab_z - meshmin_list[3 * recvTask + 2])];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Do the FFT of the density field */
+
+ rfftwnd_mpi(fft_forward_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* multiply with Green's function for the potential */
+
+ for(y = slabstart_y; y < slabstart_y + nslab_y; y++)
+ for(x = 0; x < PMGRID; x++)
+ for(z = 0; z < PMGRID / 2 + 1; z++)
+ {
+ if(x > PMGRID / 2)
+ kx = x - PMGRID;
+ else
+ kx = x;
+ if(y > PMGRID / 2)
+ ky = y - PMGRID;
+ else
+ ky = y;
+ if(z > PMGRID / 2)
+ kz = z - PMGRID;
+ else
+ kz = z;
+
+ k2 = kx * kx + ky * ky + kz * kz;
+
+ if(k2 > 0)
+ {
+ smth = -exp(-k2 * asmth2) / k2;
+
+ /* do deconvolution */
+
+ fx = fy = fz = 1;
+ if(kx != 0)
+ {
+ fx = (M_PI * kx) / PMGRID;
+ fx = sin(fx) / fx;
+ }
+ if(ky != 0)
+ {
+ fy = (M_PI * ky) / PMGRID;
+ fy = sin(fy) / fy;
+ }
+ if(kz != 0)
+ {
+ fz = (M_PI * kz) / PMGRID;
+ fz = sin(fz) / fz;
+ }
+ ff = 1 / (fx * fy * fz);
+ smth *= ff * ff * ff * ff;
+
+ /* end deconvolution */
+
+ ip = PMGRID * (PMGRID / 2 + 1) * (y - slabstart_y) + (PMGRID / 2 + 1) * x + z;
+ fft_of_rhogrid[ip].re *= smth;
+ fft_of_rhogrid[ip].im *= smth;
+ }
+ }
+
+ if(slabstart_y == 0)
+ fft_of_rhogrid[0].re = fft_of_rhogrid[0].im = 0.0;
+
+ /* Do the FFT to get the potential */
+
+ rfftwnd_mpi(fft_inverse_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* Now rhogrid holds the potential */
+ /* construct the potential for the local patch */
+
+
+ dimx = meshmax[0] - meshmin[0] + 6;
+ dimy = meshmax[1] - meshmin[1] + 6;
+ dimz = meshmax[2] - meshmin[2] + 6;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ {
+
+ /* check how much we have to send */
+ sendmin = 2 * PMGRID;
+ sendmax = -PMGRID;
+ for(slab_x = meshmin_list[3 * recvTask] - 2; slab_x < meshmax_list[3 * recvTask] + 4; slab_x++)
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] == sendTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -PMGRID)
+ sendmin = sendmax + 1;
+
+
+ /* check how much we have to receive */
+ recvmin = 2 * PMGRID;
+ recvmax = -PMGRID;
+ for(slab_x = meshmin[0] - 2; slab_x < meshmax[0] + 4; slab_x++)
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] == recvTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -PMGRID)
+ recvmin = recvmax + 1;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 6;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 6;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 6;
+
+ ncont = 1;
+ cont_sendmin[0] = sendmin;
+ cont_sendmax[0] = sendmax;
+ cont_sendmin[1] = sendmax + 1;
+ cont_sendmax[1] = sendmax;
+
+ cont_recvmin[0] = recvmin;
+ cont_recvmax[0] = recvmax;
+ cont_recvmin[1] = recvmax + 1;
+ cont_recvmax[1] = recvmax;
+
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] != ThisTask)
+ {
+ /* non-contiguous */
+ cont_sendmax[0] = slab_x - 1;
+ while(slab_to_task[(slab_x + PMGRID) % PMGRID] != ThisTask)
+ slab_x++;
+ cont_sendmin[1] = slab_x;
+ ncont++;
+ }
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] != recvTask)
+ {
+ /* non-contiguous */
+ cont_recvmax[0] = slab_x - 1;
+ while(slab_to_task[(slab_x + PMGRID) % PMGRID] != recvTask)
+ slab_x++;
+ cont_recvmin[1] = slab_x;
+ if(ncont == 1)
+ ncont++;
+ }
+ }
+
+
+ for(rep = 0; rep < ncont; rep++)
+ {
+ sendmin = cont_sendmin[rep];
+ sendmax = cont_sendmax[rep];
+ recvmin = cont_recvmin[rep];
+ recvmax = cont_recvmax[rep];
+
+ /* prepare what we want to send */
+ if(sendmax - sendmin >= 0)
+ {
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ slab_xx = ((slab_x + PMGRID) % PMGRID) - first_slab_of_task[ThisTask];
+
+ for(slab_y = meshmin_list[3 * recvTask + 1] - 2;
+ slab_y < meshmax_list[3 * recvTask + 1] + 4; slab_y++)
+ {
+ slab_yy = (slab_y + PMGRID) % PMGRID;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2] - 2;
+ slab_z <= meshmax_list[3 * recvTask + 2] + 4; slab_z++)
+ {
+ slab_zz = (slab_z + PMGRID) % PMGRID;
+
+ forcegrid[((slab_x - sendmin) * recv_dimy +
+ (slab_y - (meshmin_list[3 * recvTask + 1] - 2))) * recv_dimz +
+ slab_z - (meshmin_list[3 * recvTask + 2] - 2)] =
+ rhogrid[PMGRID * PMGRID2 * slab_xx + PMGRID2 * slab_yy + slab_zz];
+ }
+ }
+ }
+ }
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(forcegrid,
+ (sendmax - sendmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real),
+ MPI_BYTE, recvTask, TAG_PERIODIC_B,
+ workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_PERIODIC_B, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ forcegrid, (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+ }
+ }
+ }
+ }
+
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ recv_dimx = meshmax[0] - meshmin[0] + 6;
+ recv_dimy = meshmax[1] - meshmin[1] + 6;
+ recv_dimz = meshmax[2] - meshmin[2] + 6;
+
+
+ for(dim = 0; dim < 3; dim++) /* Calculate each component of the force. */
+ {
+ /* get the force component by finite differencing the potential */
+ /* note: "workspace" now contains the potential for the local patch, plus a suffiently large buffer region */
+
+ for(x = 0; x < meshmax[0] - meshmin[0] + 2; x++)
+ for(y = 0; y < meshmax[1] - meshmin[1] + 2; y++)
+ for(z = 0; z < meshmax[2] - meshmin[2] + 2; z++)
+ {
+ xrr = xll = xr = xl = x;
+ yrr = yll = yr = yl = y;
+ zrr = zll = zr = zl = z;
+
+ switch (dim)
+ {
+ case 0:
+ xr = x + 1;
+ xrr = x + 2;
+ xl = x - 1;
+ xll = x - 2;
+ break;
+ case 1:
+ yr = y + 1;
+ yl = y - 1;
+ yrr = y + 2;
+ yll = y - 2;
+ break;
+ case 2:
+ zr = z + 1;
+ zl = z - 1;
+ zrr = z + 2;
+ zll = z - 2;
+ break;
+ }
+
+ forcegrid[(x * dimy + y) * dimz + z]
+ =
+ fac * ((4.0 / 3) *
+ (workspace[((xl + 2) * recv_dimy + (yl + 2)) * recv_dimz + (zl + 2)]
+ - workspace[((xr + 2) * recv_dimy + (yr + 2)) * recv_dimz + (zr + 2)]) -
+ (1.0 / 6) *
+ (workspace[((xll + 2) * recv_dimy + (yll + 2)) * recv_dimz + (zll + 2)] -
+ workspace[((xrr + 2) * recv_dimy + (yrr + 2)) * recv_dimz + (zrr + 2)]));
+ }
+
+ /* read out the forces */
+
+ for(i = 0; i < NumPart; i++)
+ {
+ slab_x = to_slab_fac * P[i].Pos[0];
+ if(slab_x >= PMGRID)
+ slab_x = PMGRID - 1;
+ dx = to_slab_fac * P[i].Pos[0] - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * P[i].Pos[1];
+ if(slab_y >= PMGRID)
+ slab_y = PMGRID - 1;
+ dy = to_slab_fac * P[i].Pos[1] - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * P[i].Pos[2];
+ if(slab_z >= PMGRID)
+ slab_z = PMGRID - 1;
+ dz = to_slab_fac * P[i].Pos[2] - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ acc_dim =
+ forcegrid[(slab_x * dimy + slab_y) * dimz + slab_z] * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ acc_dim += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_z] * (1.0 - dx) * dy * (1.0 - dz);
+ acc_dim += forcegrid[(slab_x * dimy + slab_y) * dimz + slab_zz] * (1.0 - dx) * (1.0 - dy) * dz;
+ acc_dim += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_zz] * (1.0 - dx) * dy * dz;
+
+ acc_dim += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_z] * (dx) * (1.0 - dy) * (1.0 - dz);
+ acc_dim += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_z] * (dx) * dy * (1.0 - dz);
+ acc_dim += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_zz] * (dx) * (1.0 - dy) * dz;
+ acc_dim += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_zz] * (dx) * dy * dz;
+
+ P[i].GravPM[dim] = acc_dim;
+ }
+ }
+
+ pm_init_periodic_free();
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+
+ if(ThisTask == 0)
+ {
+ printf("done PM.\n");
+ fflush(stdout);
+ }
+}
+
+
+/*! Calculates the long-range potential using the PM method. The potential is
+ * Gaussian filtered with Asmth, given in mesh-cell units. We carry out a CIC
+ * charge assignment, and compute the potenial by Fourier transform
+ * methods. The CIC kernel is deconvolved.
+ */
+void pmpotential_periodic(void)
+{
+ double k2, kx, ky, kz, smth;
+ double dx, dy, dz;
+ double fx, fy, fz, ff;
+ double asmth2, fac;
+ int i, j, slab, level, sendTask, recvTask;
+ int x, y, z, ip;
+ int slab_x, slab_y, slab_z;
+ int slab_xx, slab_yy, slab_zz;
+ int meshmin[3], meshmax[3], sendmin, sendmax, recvmin, recvmax;
+ int rep, ncont, cont_sendmin[2], cont_sendmax[2], cont_recvmin[2], cont_recvmax[2];
+ int dimx, dimy, dimz, recv_dimx, recv_dimy, recv_dimz;
+ MPI_Status status;
+
+ if(ThisTask == 0)
+ {
+ printf("Starting periodic PM calculation.\n");
+ fflush(stdout);
+ }
+
+ asmth2 = (2 * M_PI) * All.Asmth[0] / All.BoxSize;
+ asmth2 *= asmth2;
+
+ fac = All.G / (M_PI * All.BoxSize); /* to get potential */
+
+ force_treefree();
+
+ /* first, establish the extension of the local patch in the PMGRID */
+
+ for(j = 0; j < 3; j++)
+ {
+ meshmin[j] = PMGRID;
+ meshmax[j] = 0;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ slab = to_slab_fac * P[i].Pos[j];
+ if(slab >= PMGRID)
+ slab = PMGRID - 1;
+
+ if(slab < meshmin[j])
+ meshmin[j] = slab;
+
+ if(slab > meshmax[j])
+ meshmax[j] = slab;
+ }
+ }
+
+ MPI_Allgather(meshmin, 3, MPI_INT, meshmin_list, 3, MPI_INT, MPI_COMM_WORLD);
+ MPI_Allgather(meshmax, 3, MPI_INT, meshmax_list, 3, MPI_INT, MPI_COMM_WORLD);
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ pm_init_periodic_allocate((dimx + 4) * (dimy + 4) * (dimz + 4));
+
+ for(i = 0; i < dimx * dimy * dimz; i++)
+ workspace[i] = 0;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ slab_x = to_slab_fac * P[i].Pos[0];
+ if(slab_x >= PMGRID)
+ slab_x = PMGRID - 1;
+ dx = to_slab_fac * P[i].Pos[0] - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * P[i].Pos[1];
+ if(slab_y >= PMGRID)
+ slab_y = PMGRID - 1;
+ dy = to_slab_fac * P[i].Pos[1] - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * P[i].Pos[2];
+ if(slab_z >= PMGRID)
+ slab_z = PMGRID - 1;
+ dz = to_slab_fac * P[i].Pos[2] - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (1.0 - dx) * dy * (1.0 - dz);
+ workspace[(slab_x * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * (1.0 - dy) * dz;
+ workspace[(slab_x * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (1.0 - dx) * dy * dz;
+
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_z] += P[i].Mass * (dx) * (1.0 - dy) * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_z] += P[i].Mass * (dx) * dy * (1.0 - dz);
+ workspace[(slab_xx * dimy + slab_y) * dimz + slab_zz] += P[i].Mass * (dx) * (1.0 - dy) * dz;
+ workspace[(slab_xx * dimy + slab_yy) * dimz + slab_zz] += P[i].Mass * (dx) * dy * dz;
+ }
+
+
+ for(i = 0; i < fftsize; i++) /* clear local density field */
+ rhogrid[i] = 0;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+ if(recvTask < NTask)
+ {
+ /* check how much we have to send */
+ sendmin = 2 * PMGRID;
+ sendmax = -1;
+ for(slab_x = meshmin[0]; slab_x < meshmax[0] + 2; slab_x++)
+ if(slab_to_task[slab_x % PMGRID] == recvTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -1)
+ sendmin = 0;
+
+ /* check how much we have to receive */
+ recvmin = 2 * PMGRID;
+ recvmax = -1;
+ for(slab_x = meshmin_list[3 * recvTask]; slab_x < meshmax_list[3 * recvTask] + 2; slab_x++)
+ if(slab_to_task[slab_x % PMGRID] == sendTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -1)
+ recvmin = 0;
+
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 2;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 2;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 2;
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE, recvTask,
+ TAG_PERIODIC_C, forcegrid,
+ (recvmax - recvmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_PERIODIC_C, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(forcegrid, workspace + (sendmin - meshmin[0]) * dimy * dimz,
+ (sendmax - sendmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ slab_xx = (slab_x % PMGRID) - first_slab_of_task[ThisTask];
+
+ if(slab_xx >= 0 && slab_xx < slabs_per_task[ThisTask])
+ {
+ for(slab_y = meshmin_list[3 * recvTask + 1];
+ slab_y <= meshmax_list[3 * recvTask + 1] + 1; slab_y++)
+ {
+ slab_yy = slab_y;
+ if(slab_yy >= PMGRID)
+ slab_yy -= PMGRID;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2];
+ slab_z <= meshmax_list[3 * recvTask + 2] + 1; slab_z++)
+ {
+ slab_zz = slab_z;
+ if(slab_zz >= PMGRID)
+ slab_zz -= PMGRID;
+
+ rhogrid[PMGRID * PMGRID2 * slab_xx + PMGRID2 * slab_yy + slab_zz] +=
+ forcegrid[((slab_x - recvmin) * recv_dimy +
+ (slab_y - meshmin_list[3 * recvTask + 1])) * recv_dimz +
+ (slab_z - meshmin_list[3 * recvTask + 2])];
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+
+ /* Do the FFT of the density field */
+
+ rfftwnd_mpi(fft_forward_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* multiply with Green's function for the potential */
+
+ for(y = slabstart_y; y < slabstart_y + nslab_y; y++)
+ for(x = 0; x < PMGRID; x++)
+ for(z = 0; z < PMGRID / 2 + 1; z++)
+ {
+ if(x > PMGRID / 2)
+ kx = x - PMGRID;
+ else
+ kx = x;
+ if(y > PMGRID / 2)
+ ky = y - PMGRID;
+ else
+ ky = y;
+ if(z > PMGRID / 2)
+ kz = z - PMGRID;
+ else
+ kz = z;
+
+ k2 = kx * kx + ky * ky + kz * kz;
+
+ if(k2 > 0)
+ {
+ smth = -exp(-k2 * asmth2) / k2 * fac;
+ /* do deconvolution */
+ fx = fy = fz = 1;
+ if(kx != 0)
+ {
+ fx = (M_PI * kx) / PMGRID;
+ fx = sin(fx) / fx;
+ }
+ if(ky != 0)
+ {
+ fy = (M_PI * ky) / PMGRID;
+ fy = sin(fy) / fy;
+ }
+ if(kz != 0)
+ {
+ fz = (M_PI * kz) / PMGRID;
+ fz = sin(fz) / fz;
+ }
+ ff = 1 / (fx * fy * fz);
+ smth *= ff * ff * ff * ff;
+ /* end deconvolution */
+
+ ip = PMGRID * (PMGRID / 2 + 1) * (y - slabstart_y) + (PMGRID / 2 + 1) * x + z;
+ fft_of_rhogrid[ip].re *= smth;
+ fft_of_rhogrid[ip].im *= smth;
+ }
+ }
+
+ if(slabstart_y == 0)
+ fft_of_rhogrid[0].re = fft_of_rhogrid[0].im = 0.0;
+
+ /* Do the FFT to get the potential */
+
+ rfftwnd_mpi(fft_inverse_plan, 1, rhogrid, workspace, FFTW_TRANSPOSED_ORDER);
+
+ /* note: "rhogrid" now contains the potential */
+
+
+
+ dimx = meshmax[0] - meshmin[0] + 6;
+ dimy = meshmax[1] - meshmin[1] + 6;
+ dimz = meshmax[2] - meshmin[2] + 6;
+
+ for(level = 0; level < (1 << PTask); level++) /* note: for level=0, target is the same task */
+ {
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ level;
+
+ if(recvTask < NTask)
+ {
+
+ /* check how much we have to send */
+ sendmin = 2 * PMGRID;
+ sendmax = -PMGRID;
+ for(slab_x = meshmin_list[3 * recvTask] - 2; slab_x < meshmax_list[3 * recvTask] + 4; slab_x++)
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] == sendTask)
+ {
+ if(slab_x < sendmin)
+ sendmin = slab_x;
+ if(slab_x > sendmax)
+ sendmax = slab_x;
+ }
+ if(sendmax == -PMGRID)
+ sendmin = sendmax + 1;
+
+
+ /* check how much we have to receive */
+ recvmin = 2 * PMGRID;
+ recvmax = -PMGRID;
+ for(slab_x = meshmin[0] - 2; slab_x < meshmax[0] + 4; slab_x++)
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] == recvTask)
+ {
+ if(slab_x < recvmin)
+ recvmin = slab_x;
+ if(slab_x > recvmax)
+ recvmax = slab_x;
+ }
+ if(recvmax == -PMGRID)
+ recvmin = recvmax + 1;
+
+ if((recvmax - recvmin) >= 0 || (sendmax - sendmin) >= 0) /* ok, we have a contribution to the slab */
+ {
+ recv_dimx = meshmax_list[3 * recvTask + 0] - meshmin_list[3 * recvTask + 0] + 6;
+ recv_dimy = meshmax_list[3 * recvTask + 1] - meshmin_list[3 * recvTask + 1] + 6;
+ recv_dimz = meshmax_list[3 * recvTask + 2] - meshmin_list[3 * recvTask + 2] + 6;
+
+ ncont = 1;
+ cont_sendmin[0] = sendmin;
+ cont_sendmax[0] = sendmax;
+ cont_sendmin[1] = sendmax + 1;
+ cont_sendmax[1] = sendmax;
+
+ cont_recvmin[0] = recvmin;
+ cont_recvmax[0] = recvmax;
+ cont_recvmin[1] = recvmax + 1;
+ cont_recvmax[1] = recvmax;
+
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] != ThisTask)
+ {
+ /* non-contiguous */
+ cont_sendmax[0] = slab_x - 1;
+ while(slab_to_task[(slab_x + PMGRID) % PMGRID] != ThisTask)
+ slab_x++;
+ cont_sendmin[1] = slab_x;
+ ncont++;
+ }
+ }
+
+ for(slab_x = recvmin; slab_x <= recvmax; slab_x++)
+ {
+ if(slab_to_task[(slab_x + PMGRID) % PMGRID] != recvTask)
+ {
+ /* non-contiguous */
+ cont_recvmax[0] = slab_x - 1;
+ while(slab_to_task[(slab_x + PMGRID) % PMGRID] != recvTask)
+ slab_x++;
+ cont_recvmin[1] = slab_x;
+ if(ncont == 1)
+ ncont++;
+ }
+ }
+
+
+ for(rep = 0; rep < ncont; rep++)
+ {
+ sendmin = cont_sendmin[rep];
+ sendmax = cont_sendmax[rep];
+ recvmin = cont_recvmin[rep];
+ recvmax = cont_recvmax[rep];
+
+ /* prepare what we want to send */
+ if(sendmax - sendmin >= 0)
+ {
+ for(slab_x = sendmin; slab_x <= sendmax; slab_x++)
+ {
+ slab_xx = ((slab_x + PMGRID) % PMGRID) - first_slab_of_task[ThisTask];
+
+ for(slab_y = meshmin_list[3 * recvTask + 1] - 2;
+ slab_y < meshmax_list[3 * recvTask + 1] + 4; slab_y++)
+ {
+ slab_yy = (slab_y + PMGRID) % PMGRID;
+
+ for(slab_z = meshmin_list[3 * recvTask + 2] - 2;
+ slab_z <= meshmax_list[3 * recvTask + 2] + 4; slab_z++)
+ {
+ slab_zz = (slab_z + PMGRID) % PMGRID;
+
+ forcegrid[((slab_x - sendmin) * recv_dimy +
+ (slab_y - (meshmin_list[3 * recvTask + 1] - 2))) * recv_dimz +
+ slab_z - (meshmin_list[3 * recvTask + 2] - 2)] =
+ rhogrid[PMGRID * PMGRID2 * slab_xx + PMGRID2 * slab_yy + slab_zz];
+ }
+ }
+ }
+ }
+
+ if(level > 0)
+ {
+ MPI_Sendrecv(forcegrid,
+ (sendmax - sendmin + 1) * recv_dimy * recv_dimz * sizeof(fftw_real),
+ MPI_BYTE, recvTask, TAG_PERIODIC_D,
+ workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real), MPI_BYTE,
+ recvTask, TAG_PERIODIC_D, MPI_COMM_WORLD, &status);
+ }
+ else
+ {
+ memcpy(workspace + (recvmin - (meshmin[0] - 2)) * dimy * dimz,
+ forcegrid, (recvmax - recvmin + 1) * dimy * dimz * sizeof(fftw_real));
+ }
+ }
+ }
+ }
+ }
+
+
+ dimx = meshmax[0] - meshmin[0] + 2;
+ dimy = meshmax[1] - meshmin[1] + 2;
+ dimz = meshmax[2] - meshmin[2] + 2;
+
+ recv_dimx = meshmax[0] - meshmin[0] + 6;
+ recv_dimy = meshmax[1] - meshmin[1] + 6;
+ recv_dimz = meshmax[2] - meshmin[2] + 6;
+
+
+
+ for(x = 0; x < meshmax[0] - meshmin[0] + 2; x++)
+ for(y = 0; y < meshmax[1] - meshmin[1] + 2; y++)
+ for(z = 0; z < meshmax[2] - meshmin[2] + 2; z++)
+ {
+ forcegrid[(x * dimy + y) * dimz + z] =
+ workspace[((x + 2) * recv_dimy + (y + 2)) * recv_dimz + (z + 2)];
+ }
+
+
+ /* read out the potential */
+
+ for(i = 0; i < NumPart; i++)
+ {
+ slab_x = to_slab_fac * P[i].Pos[0];
+ if(slab_x >= PMGRID)
+ slab_x = PMGRID - 1;
+ dx = to_slab_fac * P[i].Pos[0] - slab_x;
+ slab_x -= meshmin[0];
+ slab_xx = slab_x + 1;
+
+ slab_y = to_slab_fac * P[i].Pos[1];
+ if(slab_y >= PMGRID)
+ slab_y = PMGRID - 1;
+ dy = to_slab_fac * P[i].Pos[1] - slab_y;
+ slab_y -= meshmin[1];
+ slab_yy = slab_y + 1;
+
+ slab_z = to_slab_fac * P[i].Pos[2];
+ if(slab_z >= PMGRID)
+ slab_z = PMGRID - 1;
+ dz = to_slab_fac * P[i].Pos[2] - slab_z;
+ slab_z -= meshmin[2];
+ slab_zz = slab_z + 1;
+
+ P[i].Potential +=
+ forcegrid[(slab_x * dimy + slab_y) * dimz + slab_z] * (1.0 - dx) * (1.0 - dy) * (1.0 - dz);
+ P[i].Potential += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_z] * (1.0 - dx) * dy * (1.0 - dz);
+ P[i].Potential += forcegrid[(slab_x * dimy + slab_y) * dimz + slab_zz] * (1.0 - dx) * (1.0 - dy) * dz;
+ P[i].Potential += forcegrid[(slab_x * dimy + slab_yy) * dimz + slab_zz] * (1.0 - dx) * dy * dz;
+
+ P[i].Potential += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_z] * (dx) * (1.0 - dy) * (1.0 - dz);
+ P[i].Potential += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_z] * (dx) * dy * (1.0 - dz);
+ P[i].Potential += forcegrid[(slab_xx * dimy + slab_y) * dimz + slab_zz] * (dx) * (1.0 - dy) * dz;
+ P[i].Potential += forcegrid[(slab_xx * dimy + slab_yy) * dimz + slab_zz] * (dx) * dy * dz;
+ }
+
+ pm_init_periodic_free();
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+
+ if(ThisTask == 0)
+ {
+ printf("done PM-Potential.\n");
+ fflush(stdout);
+ }
+}
+
+#endif
+#endif
diff --git a/potential.c b/potential.c
new file mode 100644
index 0000000..976c3dd
--- /dev/null
+++ b/potential.c
@@ -0,0 +1,331 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <float.h>
+#include <mpi.h>
+
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file potential.c
+ * \brief Computation of the gravitational potential of particles
+ */
+
+
+/*! This function computes the gravitational potential for ALL the particles.
+ * First, the (short-range) tree potential is computed, and then, if needed,
+ * the long range PM potential is added.
+ */
+void compute_potential(void)
+{
+ int i;
+
+#ifndef NOGRAVITY
+ long long ntot, ntotleft;
+ int j, k, level, sendTask, recvTask;
+ int ndone;
+ int maxfill, ngrp, place, nexport;
+ int *nsend, *noffset, *nsend_local, *nbuffer, *ndonelist, *numlist;
+ double fac;
+ double t0, t1, tstart, tend;
+ MPI_Status status;
+ double r2;
+
+ t0 = second();
+
+ if(All.ComovingIntegrationOn)
+ set_softenings();
+
+ if(ThisTask == 0)
+ {
+ printf("Start computation of potential for all particles...\n");
+ fflush(stdout);
+ }
+
+
+ tstart = second();
+ if(TreeReconstructFlag)
+ {
+ if(ThisTask == 0)
+ printf("Tree construction.\n");
+
+ force_treebuild(NumPart);
+
+ TreeReconstructFlag = 0;
+
+ if(ThisTask == 0)
+ printf("Tree construction done.\n");
+ }
+ tend = second();
+ All.CPU_TreeConstruction += timediff(tstart, tend);
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumPart, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+ i = 0; /* beginn with this index */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ for(nexport = 0, ndone = 0; i < NumPart && nexport < All.BunchSizeForce - NTask; i++)
+ {
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+#ifndef PMGRID
+ force_treeevaluate_potential(i, 0);
+#else
+ force_treeevaluate_potential_shortrange(i, 0);
+#endif
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ for(k = 0; k < 3; k++)
+ GravDataGet[nexport].u.Pos[k] = P[i].Pos[k];
+#ifdef UNEQUALSOFTENINGS
+ GravDataGet[nexport].Type = P[i].Type;
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(P[i].Type == 0)
+ GravDataGet[nexport].Soft = SphP[i].Hsml;
+#endif
+#endif
+ GravDataGet[nexport].w.OldAcc = P[i].OldAcc;
+
+ GravDataIndexTable[nexport].Task = j;
+ GravDataIndexTable[nexport].Index = i;
+ GravDataIndexTable[nexport].SortIndex = nexport;
+
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+ }
+
+ qsort(GravDataIndexTable, nexport, sizeof(struct gravdata_index), grav_tree_compare_key);
+
+ for(j = 0; j < nexport; j++)
+ GravDataIn[j] = GravDataGet[GravDataIndexTable[j].SortIndex];
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&GravDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_POTENTIAL_A,
+ &GravDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in), MPI_BYTE,
+ recvTask, TAG_POTENTIAL_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ {
+#ifndef PMGRID
+ force_treeevaluate_potential(j, 1);
+#else
+ force_treeevaluate_potential_shortrange(j, 1);
+#endif
+ }
+
+
+ /* get the result */
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeForce)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&GravDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_POTENTIAL_B,
+ &GravDataOut[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct gravdata_in),
+ MPI_BYTE, recvTask, TAG_POTENTIAL_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ place = GravDataIndexTable[noffset[recvTask] + j].Index;
+
+ P[place].Potential += GravDataOut[j + noffset[recvTask]].u.Potential;
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+ /* add correction to exclude self-potential */
+
+ for(i = 0; i < NumPart; i++)
+ {
+ /* remove self-potential */
+ P[i].Potential += P[i].Mass / All.SofteningTable[P[i].Type];
+
+ if(All.ComovingIntegrationOn)
+ if(All.PeriodicBoundariesOn)
+ P[i].Potential -= 2.8372975 * pow(P[i].Mass, 2.0 / 3) *
+ pow(All.Omega0 * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G), 1.0 / 3);
+ }
+
+
+ /* multiply with the gravitational constant */
+
+ for(i = 0; i < NumPart; i++)
+ P[i].Potential *= All.G;
+
+
+#ifdef PMGRID
+
+#ifdef PERIODIC
+ pmpotential_periodic();
+#ifdef PLACEHIGHRESREGION
+ pmpotential_nonperiodic(1);
+#endif
+#else
+ pmpotential_nonperiodic(0);
+#ifdef PLACEHIGHRESREGION
+ pmpotential_nonperiodic(1);
+#endif
+#endif
+
+#endif
+
+
+
+ if(All.ComovingIntegrationOn)
+ {
+#ifndef PERIODIC
+ fac = -0.5 * All.Omega0 * All.Hubble * All.Hubble;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(k = 0, r2 = 0; k < 3; k++)
+ r2 += P[i].Pos[k] * P[i].Pos[k];
+
+ P[i].Potential += fac * r2;
+ }
+#endif
+ }
+ else
+ {
+ fac = -0.5 * All.OmegaLambda * All.Hubble * All.Hubble;
+ if(fac != 0)
+ {
+ for(i = 0; i < NumPart; i++)
+ {
+ for(k = 0, r2 = 0; k < 3; k++)
+ r2 += P[i].Pos[k] * P[i].Pos[k];
+
+ P[i].Potential += fac * r2;
+ }
+ }
+ }
+
+
+ if(ThisTask == 0)
+ {
+ printf("potential done.\n");
+ fflush(stdout);
+ }
+
+ t1 = second();
+
+ All.CPU_Potential += timediff(t0, t1);
+
+#else
+ for(i = 0; i < NumPart; i++)
+ P[i].Potential = 0;
+#endif
+
+#ifdef OUTERPOTENTIAL
+ /* Add contribution from an outer potential */
+ outer_potential();
+#endif
+}
diff --git a/predict.c b/predict.c
new file mode 100644
index 0000000..cc4d5cc
--- /dev/null
+++ b/predict.c
@@ -0,0 +1,165 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_math.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file predict.c
+ * \brief drift particles by a small time interval
+ *
+ * This function contains code to implement a drift operation on all the
+ * particles, which represents one part of the leapfrog integration scheme.
+ */
+
+
+/*! This function drifts all particles from the current time to the future:
+ * time0 - > time1
+ *
+ * If there is no explicit tree construction in the following timestep, the
+ * tree nodes are also drifted and updated accordingly. Note: For periodic
+ * boundary conditions, the mapping of coordinates onto the interval
+ * [0,All.BoxSize] is only done before the domain decomposition, or for
+ * outputs to snapshot files. This simplifies dynamic tree updates, and
+ * allows the domain decomposition to be carried out only every once in a
+ * while.
+ */
+void move_particles(int time0, int time1)
+{
+ int i, j;
+ double dt_drift, dt_gravkick, dt_hydrokick, dt_entr;
+ double t0, t1;
+
+
+ t0 = second();
+
+ if(All.ComovingIntegrationOn)
+ {
+ dt_drift = get_drift_factor(time0, time1);
+ dt_gravkick = get_gravkick_factor(time0, time1);
+ dt_hydrokick = get_hydrokick_factor(time0, time1);
+ }
+ else
+ {
+ dt_drift = dt_gravkick = dt_hydrokick = (time1 - time0) * All.Timebase_interval;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ P[i].Pos[j] += P[i].Vel[j] * dt_drift;
+
+ if(P[i].Type == 0)
+ {
+#ifdef PMGRID
+ for(j = 0; j < 3; j++)
+ {
+ SphP[i].VelPred[j] += (P[i].GravAccel[j] + P[i].GravPM[j]) * dt_gravkick + SphP[i].HydroAccel[j] * dt_hydrokick;
+ }
+#else
+ for(j = 0; j < 3; j++)
+ {
+ SphP[i].VelPred[j] += P[i].GravAccel[j] * dt_gravkick + SphP[i].HydroAccel[j] * dt_hydrokick;
+ }
+#endif
+ SphP[i].Density *= exp(-SphP[i].DivVel * dt_drift);
+ SphP[i].Hsml *= exp(0.333333333333 * SphP[i].DivVel * dt_drift);
+
+ if(SphP[i].Hsml < All.MinGasHsml)
+ SphP[i].Hsml = All.MinGasHsml;
+
+ dt_entr = (time1 - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+
+ SphP[i].Pressure = (SphP[i].Entropy + SphP[i].DtEntropy * dt_entr) * pow(SphP[i].Density, GAMMA);
+
+#ifdef ENTROPYPRED
+ SphP[i].EntropyPred = (SphP[i].Entropy + SphP[i].DtEntropy * dt_entr);
+#endif
+
+
+
+
+#ifdef CHECK_ENTROPY_SIGN
+ if ((SphP[i].EntropyPred < 0)||(SphP[i].Entropy < 0))
+ {
+ printf("\ntask=%d: entropy less than zero in move_particles !\n", ThisTask);
+ printf("ID=%d Entropy=%g EntropyPred=%g DtEntropy=%g dt_entr=%g\n",P[i].ID,SphP[i].Entropy,SphP[i].EntropyPred,SphP[i].DtEntropy,dt_entr);
+ fflush(stdout);
+ endrun(333001);
+
+ }
+#endif
+
+
+
+#ifdef NO_NEGATIVE_PRESSURE
+ if (SphP[i].Pressure<0)
+ {
+ printf("\ntask=%d: pressure less than zero in move_particles !\n", ThisTask);
+ printf("ID=%d Entropy=%g DtEntropy*dt=%g Density=%g DtEntropy=%g dt=%g\n",P[i].ID,SphP[i].Entropy,SphP[i].DtEntropy*dt_entr,SphP[i].Density,SphP[i].DtEntropy,dt_entr);
+ fflush(stdout);
+ endrun(333002);
+ }
+#endif
+
+ }
+ }
+
+ /* if domain-decomp and tree are not going to be reconstructed, update dynamically. */
+ if(All.NumForcesSinceLastDomainDecomp < All.TotNumPart * All.TreeDomainUpdateFrequency)
+ {
+ for(i = 0; i < Numnodestree; i++)
+ for(j = 0; j < 3; j++)
+ Nodes[All.MaxPart + i].u.d.s[j] += Extnodes[All.MaxPart + i].vs[j] * dt_drift;
+
+ force_update_len();
+
+ force_update_pseudoparticles();
+ }
+
+ t1 = second();
+
+ All.CPU_Predict += timediff(t0, t1);
+}
+
+
+
+/*! This function makes sure that all particle coordinates (Pos) are
+ * periodically mapped onto the interval [0, BoxSize]. After this function
+ * has been called, a new domain decomposition should be done, which will
+ * also force a new tree construction.
+ */
+#ifdef PERIODIC
+void do_box_wrapping(void)
+{
+ int i, j;
+ double boxsize[3];
+
+ for(j = 0; j < 3; j++)
+ boxsize[j] = All.BoxSize;
+
+#ifdef LONG_X
+ boxsize[0] *= LONG_X;
+#endif
+#ifdef LONG_Y
+ boxsize[1] *= LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxsize[2] *= LONG_Z;
+#endif
+
+ for(i = 0; i < NumPart; i++)
+ for(j = 0; j < 3; j++)
+ {
+ while(P[i].Pos[j] < 0)
+ P[i].Pos[j] += boxsize[j];
+
+ while(P[i].Pos[j] >= boxsize[j])
+ P[i].Pos[j] -= boxsize[j];
+ }
+}
+#endif
diff --git a/proto.h b/proto.h
new file mode 100644
index 0000000..62f4c2d
--- /dev/null
+++ b/proto.h
@@ -0,0 +1,399 @@
+/*! \file proto.h
+ * \brief this file contains all function prototypes of the code
+ */
+
+#ifndef ALLVARS_H
+#include "allvars.h"
+#endif
+
+#ifdef HAVE_HDF5
+#include <hdf5.h>
+#endif
+
+void advance_and_find_timesteps(void);
+void allocate_commbuffers(void);
+void allocate_memory(void);
+void begrun(void);
+int blockpresent(enum iofields blocknr);
+#ifdef BLOCK_SKIPPING
+int blockabsent(enum iofields blocknr);
+#endif
+void catch_abort(int sig);
+void catch_fatal(int sig);
+void check_omega(void);
+void close_outputfiles(void);
+int compare_key(const void *a, const void *b);
+void compute_accelerations(int mode);
+void compute_global_quantities_of_system(void);
+void compute_potential(void);
+int dens_compare_key(const void *a, const void *b);
+void density(int mode);
+void density_decouple(void);
+void density_evaluate(int i, int mode);
+#ifdef CHIMIE
+int stars_dens_compare_key(const void *a, const void *b);
+void stars_density(void);
+void stars_density_evaluate(int i, int mode);
+#endif
+
+void distribute_file(int nfiles, int firstfile, int firsttask, int lasttask, int *filenr, int *master, int *last);
+double dmax(double, double);
+double dmin(double, double);
+void do_box_wrapping(void);
+
+void domain_Decomposition(void);
+int domain_compare_key(const void *a, const void *b);
+int domain_compare_key(const void *a, const void *b);
+int domain_compare_toplist(const void *a, const void *b);
+void domain_countToGo(void);
+void domain_decompose(void);
+void domain_determineTopTree(void);
+void domain_exchangeParticles(int partner, int sphflag, int send_count, int recv_count);
+void domain_findExchangeNumbers(int task, int partner, int sphflag, int *send, int *recv);
+void domain_findExtent(void);
+int domain_findSplit(int cpustart, int ncpu, int first, int last);
+int domain_findSplityr(int cpustart, int ncpu, int first, int last);
+void domain_shiftSplit(void);
+void domain_shiftSplityr(void);
+void domain_sumCost(void);
+void domain_topsplit(int node, peanokey startkey);
+void domain_topsplit_local(int node, peanokey startkey);
+
+double drift_integ(double a, void *param);
+void dump_particles(void);
+void empty_read_buffer(enum iofields blocknr, int offset, int pc, int type);
+void endrun(int);
+void energy_statistics(void);
+#ifdef ADVANCEDSTATISTICS
+void advanced_energy_statistics(void);
+#endif
+void every_timestep_stuff(void);
+
+void ewald_corr(double dx, double dy, double dz, double *fper);
+void ewald_force(int ii, int jj, int kk, double x[3], double force[3]);
+void ewald_init(void);
+double ewald_pot_corr(double dx, double dy, double dz);
+double ewald_psi(double x[3]);
+
+void fill_Tab_IO_Labels(void);
+void fill_write_buffer(enum iofields blocknr, int *pindex, int pc, int type);
+void find_dt_displacement_constraint(double hfac);
+int find_files(char *fname);
+int find_next_outputtime(int time);
+void find_next_sync_point_and_drift(void);
+
+void force_create_empty_nodes(int no, int topnode, int bits, int x, int y, int z, int *nodecount, int *nextfree);
+void force_exchange_pseudodata(void);
+void force_flag_localnodes(void);
+void force_insert_pseudo_particles(void);
+void force_setupnonrecursive(int no);
+void force_treeallocate(int maxnodes, int maxpart);
+int force_treebuild(int npart);
+int force_treebuild_single(int npart);
+int force_treeevaluate(int target, int mode, double *ewaldcountsum);
+int force_treeevaluate_direct(int target, int mode);
+int force_treeevaluate_ewald_correction(int target, int mode, double pos_x, double pos_y, double pos_z, double aold);
+void force_treeevaluate_potential(int target, int type);
+void force_treeevaluate_potential_shortrange(int target, int mode);
+int force_treeevaluate_shortrange(int target, int mode);
+void force_treefree(void);
+void force_treeupdate_pseudos(void);
+void force_update_hmax(void);
+void force_update_len(void);
+void force_update_node(int no, int flag);
+void force_update_node_hmax_local(void);
+void force_update_node_hmax_toptree(void);
+void force_update_node_len_local(void);
+void force_update_node_len_toptree(void);
+void force_update_node_recursive(int no, int sib, int father);
+void force_update_pseudoparticles(void);
+void force_update_size_of_parent_node(int no);
+
+void free_memory(void);
+
+int get_bytes_per_blockelement(enum iofields blocknr);
+void get_dataset_name(enum iofields blocknr, char *buf);
+int get_datatype_in_block(enum iofields blocknr);
+double get_drift_factor(int time0, int time1);
+double get_gravkick_factor(int time0, int time1);
+double get_hydrokick_factor(int time0, int time1);
+int get_particles_in_block(enum iofields blocknr, int *typelist);
+double get_random_number(int id);
+#ifdef SFR
+double get_StarFormation_random_number(int id);
+#endif
+#ifdef FEEDBACK_WIND
+double get_FeedbackWind_random_number(int id);
+#endif
+#ifdef CHIMIE
+double get_Chimie_random_number(int id);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+double get_ChimieKineticFeedback_random_number(int id);
+#endif
+int get_timestep(int p, double *a, int flag);
+int get_values_per_blockelement(enum iofields blocknr);
+
+int grav_tree_compare_key(const void *a, const void *b);
+void gravity_forcetest(void);
+void gravity_tree(void);
+void gravity_tree_shortrange(void);
+double gravkick_integ(double a, void *param);
+
+int hydro_compare_key(const void *a, const void *b);
+void hydro_evaluate(int target, int mode);
+void hydro_force(void);
+double hydrokick_integ(double a, void *param);
+
+int imax(int, int);
+int imin(int, int);
+
+void init(void);
+void init_drift_table(void);
+void init_peano_map(void);
+
+void long_range_force(void);
+void long_range_init(void);
+void long_range_init_regionsize(void);
+void move_particles(int time0, int time1);
+size_t my_fread(void *ptr, size_t size, size_t nmemb, FILE * stream);
+size_t my_fwrite(void *ptr, size_t size, size_t nmemb, FILE * stream);
+
+int ngb_clear_buf(FLOAT searchcenter[3], FLOAT hguess, int numngb);
+void ngb_treeallocate(int npart);
+void ngb_treebuild(void);
+int ngb_treefind_pairs(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode);
+#ifdef MULTIPHASE
+int ngb_treefind_phase_pairs(FLOAT searchcenter[3], FLOAT hsml, int phase, int *startnode);
+int ngb_treefind_sticky_collisions(FLOAT searchcenter[3], FLOAT hguess, int phase, int *startnode);
+#endif
+int ngb_treefind_variable(FLOAT searchcenter[3], FLOAT hguess, int phase, int *startnode);
+#ifdef CHIMIE
+int ngb_treefind_variable_for_chimie(FLOAT searchcenter[3], FLOAT hguess, int *startnode);
+#endif
+void ngb_treefree(void);
+void ngb_treesearch(int);
+void ngb_treesearch_pairs(int);
+void ngb_update_nodes(void);
+
+void open_outputfiles(void);
+
+peanokey peano_hilbert_key(int x, int y, int z, int bits);
+void peano_hilbert_order(void);
+
+void pm_init_nonperiodic(void);
+void pm_init_nonperiodic_allocate(int dimprod);
+void pm_init_nonperiodic_free(void);
+void pm_init_periodic(void);
+void pm_init_periodic_allocate(int dimprod);
+void pm_init_periodic_free(void);
+void pm_init_regionsize(void);
+void pm_setup_nonperiodic_kernel(void);
+int pmforce_nonperiodic(int grnr);
+void pmforce_periodic(void);
+int pmpotential_nonperiodic(int grnr);
+void pmpotential_periodic(void);
+
+double pow(double, double); /* on some old DEC Alphas, the correct prototype for pow() is missing, even when math.h is included */
+
+void read_file(char *fname, int readTask, int lastTask);
+void read_header_attributes_in_hdf5(char *fname);
+void read_ic(char *fname);
+int read_outputlist(char *fname);
+void read_parameter_file(char *fname);
+void readjust_timebase(double TimeMax_old, double TimeMax_new);
+
+void reorder_gas(void);
+void reorder_particles(void);
+#ifdef STELLAR_PROP
+void reorder_stars(void);
+void reorder_st(void);
+#endif
+void restart(int mod);
+void run(void);
+void savepositions(int num);
+
+double second(void);
+
+void seed_glass(void);
+void set_random_numbers(void);
+void set_softenings(void);
+void set_units(void);
+void init_local_sys_state(void);
+
+void setup_smoothinglengths(void);
+#ifdef CHIMIE
+void stars_setup_smoothinglengths(void);
+#endif
+void statistics(void);
+void terminate_processes(void);
+double timediff(double t0, double t1);
+
+#ifdef HAVE_HDF5
+void write_header_attributes_in_hdf5(hid_t handle);
+#endif
+void write_file(char *fname, int readTask, int lastTask);
+void write_pid_file(void);
+
+#ifdef COOLING
+int init_cooling(FLOAT metallicity);
+int init_cooling_with_metals();
+double cooling_function(double temperature);
+double cooling_function_with_metals(double temperature,double metal);
+void init_from_new_redshift(double Redshift);
+double J_0();
+double J_nu(double e);
+double sigma_rad_HI(double e);
+double sigma_rad_HeI(double e);
+double sigma_rad_HeII(double e);
+double cooling_bremstrahlung_HI(double T);
+double cooling_bremstrahlung_HeI(double T);
+double cooling_bremstrahlung_HeII(double T);
+double cooling_ionization_HI(double T);
+double cooling_ionization_HeI(double T);
+double cooling_ionization_HeII(double T);
+double cooling_recombination_HI(double T);
+double cooling_recombination_HeI(double T);
+double cooling_recombination_HeII(double T);
+double cooling_dielectric_recombination(double T);
+double cooling_excitation_HI(double T);
+double cooling_excitation_HII(double T);
+double cooling_compton(double T);
+double A_HII(double T);
+double A_HeIId(double T);
+double A_HeII(double T);
+double A_HeIII(double T);
+double G_HI(double T);
+double G_HeI(double T);
+double G_HeII(double T);
+double G_gHI();
+double G_gHeI();
+double G_gHeII();
+double G_gHI_t(double J0);
+double G_gHeI_t(double J0);
+double G_gHeII_t(double J0);
+double G_gHI_w();
+double G_gHeI_w();
+double G_gHeII_w();
+double heating_radiative_HI();
+double heating_radiative_HeI();
+double heating_radiative_HeII();
+double heating_radiative_HI_t(double J0);
+double heating_radiative_HeI_t(double J0);
+double heating_radiative_HeII_t(double J0);
+double heating_radiative_HI_w();
+double heating_radiative_HeI_w();
+double heating_radiative_HeII_w();
+double heating_compton();
+void print_cooling(double T,double c1,double c2,double c3,double c4,double c5,double c6,double c7,double c8,double c9,double c10,double c11,double c12,double c13,double h1, double h2, double h3, double h4);
+void compute_densities(double T,double X,double* n_H, double* n_HI,double* n_HII,double* n_HEI,double* n_HEII,double* n_HEIII,double* n_E,double* mu);
+void compute_cooling_from_T_and_Nh(double T,double X,double n_H,double *c1,double *c2,double *c3,double *c4,double *c5,double *c6,double *c7,double *c8,double *c9,double *c10,double *c11,double *c12,double *c13,double *h1, double *h2, double *h3, double *h4);
+double compute_cooling_from_Egyspec_and_Density(double Egyspec,double Density, double *MeanWeight);
+double DoCooling(FLOAT Density,FLOAT Entropy,int Phase,int i,FLOAT DtEntropyVisc, double dt, double hubble_a);
+void CoolingForOne(int i,int t0,int t1,double a3inv,double hubble_a);
+void cooling();
+double lambda(FLOAT density,FLOAT egyspec, int phase, int i);
+#endif
+
+#ifdef HEATING
+void heating();
+double gamma_fct(FLOAT Density,FLOAT Entropy,int i);
+#endif
+
+#ifdef AGN_HEATING
+void agn_heating();
+double gamma_fct(FLOAT density,double r, double SpecPower);
+double HeatingRadialDependency(double r);
+#endif
+
+#ifdef MULTIPHASE
+void update_phase(void);
+void init_sticky(void);
+void sticky(void);
+void sticky_compute_energy_kin(int mode);
+void sticky_collisions(void);
+void sticky_collisions2(int loop);
+void sticky_evaluate(int target, int mode, int loop);
+int sticky_compare_key(const void *a, const void *b);
+#endif
+
+
+#ifdef FEEDBACK_WIND
+void feedbackwind_compute_energy_kin(int mode);
+#endif
+
+
+#ifdef CHIMIE
+void init_chimie(void);
+void chimie(void);
+void do_chimie(void);
+void chimie_evaluate(int target, int mode);
+int chimie_compare_key(const void *a, const void *b);
+int get_nelts();
+char* get_Element(i);
+float get_SolarAbundance(i);
+#if defined(CHIMIE_THERMAL_FEEDBACK) && defined(CHIMIE_COMPUTE_THERMAL_FEEDBACK_ENERGY)
+void chimie_compute_energy_int(int mode);
+#endif
+#if defined(CHIMIE_KINETIC_FEEDBACK) && defined(CHIMIE_COMPUTE_KINETIC_FEEDBACK_ENERGY)
+void chimie_compute_energy_kin(int mode);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+void chimie_apply_wind(void);
+#endif
+#endif
+
+
+#ifdef OUTERPOTENTIAL
+void init_outer_potential(void);
+void outer_forces(void);
+void outer_potential(void);
+
+#ifdef NFW
+void init_outer_potential_nfw(void);
+void outer_forces_nfw(void);
+void outer_potential_nfw(void);
+#endif
+
+#ifdef PLUMMER
+void init_outer_potential_plummer(void);
+void outer_forces_plummer(void);
+void outer_potential_plummer(void);
+#endif
+
+#ifdef PISOTHERM
+void init_outer_potential_pisotherm(void);
+void outer_forces_pisotherm(void);
+void outer_potential_pisotherm(void);
+double potential_f(double r, void * params);
+double get_potential(double r);
+#endif
+
+#ifdef CORIOLIS
+void init_outer_potential_coriolis(void);
+void set_outer_potential_coriolis(void);
+void outer_forces_coriolis(void);
+void outer_potential_coriolis(void);
+#endif
+#endif
+
+#ifdef SFR
+void star_formation(void);
+void rearrange_particle_sequence(void);
+void sfr_compute_energy_int(int mode);
+void sfr_check_number_of_stars(int mode);
+#endif
+
+#ifdef AGN_ACCRETION
+void compute_agn_accretion(void);
+#endif
+
+#ifdef BUBBLES
+void init_bubble(void);
+void make_bubble(void);
+void create_bubble(int sign);
+#endif
+
+#ifdef BONDI_ACCRETION
+void bondi_accretion(void);
+#endif
diff --git a/read_ic.c b/read_ic.c
new file mode 100644
index 0000000..be80560
--- /dev/null
+++ b/read_ic.c
@@ -0,0 +1,933 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <string.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file read_ic.c
+ * \brief Read initial conditions in one of Gadget's file formats
+ */
+
+/*! This function reads initial conditions, in one of the three possible file
+ * formats currently supported by Gadget. Note: When a snapshot file is
+ * started from initial conditions (start-option 0), not all the information
+ * in the header is used, in particular, the STARTING TIME needs to be set in
+ * the parameterfile. Also, for gas particles, only the internal energy is
+ * read, the density and mean molecular weight will be recomputed by the
+ * code. When InitGasTemp>0 is given, the gas temperature will be initialzed
+ * to this value assuming a mean colecular weight either corresponding to
+ * complete neutrality, or full ionization.
+ *
+ * However, when the code is started with start-option 2, then all the this
+ * data in the snapshot files is preserved, i.e. this is also the way to
+ * resume a simulation from a snapshot file in case a regular restart file is
+ * not available.
+ */
+void read_ic(char *fname)
+{
+ int i, num_files, rest_files, ngroups, gr, filenr, masterTask, lastTask, groupMaster;
+ double u_init;
+ char buf[500];
+#ifndef ISOTHERM_EQS
+ double molecular_weight;
+#endif
+
+
+ NumPart = 0;
+ N_gas = 0;
+#ifdef SFR
+ N_stars = 0;
+#endif
+ All.TotNumPart = 0;
+
+ num_files = find_files(fname);
+
+ rest_files = num_files;
+
+ fill_Tab_IO_Labels();
+
+ while(rest_files > NTask)
+ {
+ sprintf(buf, "%s.%d", fname, ThisTask + (rest_files - NTask));
+ if(All.ICFormat == 3)
+ sprintf(buf, "%s.%d.hdf5", fname, ThisTask + (rest_files - NTask));
+
+ ngroups = NTask / All.NumFilesWrittenInParallel;
+ if((NTask % All.NumFilesWrittenInParallel))
+ ngroups++;
+ groupMaster = (ThisTask / ngroups) * ngroups;
+
+ for(gr = 0; gr < ngroups; gr++)
+ {
+ if(ThisTask == (groupMaster + gr)) /* ok, it's this processor's turn */
+ read_file(buf, ThisTask, ThisTask);
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ rest_files -= NTask;
+ }
+
+
+ if(rest_files > 0)
+ {
+ distribute_file(rest_files, 0, 0, NTask - 1, &filenr, &masterTask, &lastTask);
+
+ if(num_files > 1)
+ {
+ sprintf(buf, "%s.%d", fname, filenr);
+ if(All.ICFormat == 3)
+ sprintf(buf, "%s.%d.hdf5", fname, filenr);
+ }
+ else
+ {
+ sprintf(buf, "%s", fname);
+ if(All.ICFormat == 3)
+ sprintf(buf, "%s.hdf5", fname);
+ }
+
+ ngroups = rest_files / All.NumFilesWrittenInParallel;
+ if((rest_files % All.NumFilesWrittenInParallel))
+ ngroups++;
+
+ for(gr = 0; gr < ngroups; gr++)
+ {
+ if((filenr / All.NumFilesWrittenInParallel) == gr) /* ok, it's this processor's turn */
+ read_file(buf, masterTask, lastTask);
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+ }
+
+
+ /* this makes sure that masses are initialized in the case that the mass-block
+ is completely empty */
+ for(i = 0; i < NumPart; i++)
+ {
+ if(All.MassTable[P[i].Type] != 0)
+ P[i].Mass = All.MassTable[P[i].Type];
+ }
+
+ if(RestartFlag == 0)
+ {
+ if(All.InitGasTemp > 0)
+ {
+ u_init = (BOLTZMANN / PROTONMASS) * All.InitGasTemp;
+ u_init *= All.UnitMass_in_g / All.UnitEnergy_in_cgs; /* unit conversion */
+
+#ifdef ISOTHERM_EQS
+ u_init *= (3.0/2);
+#else
+ u_init *= (1.0 / GAMMA_MINUS1);
+
+ if(All.InitGasTemp > 1.0e4) /* assuming FULL ionization */
+ molecular_weight = 4 / (8 - 5 * (1 - HYDROGEN_MASSFRAC));
+ else /* assuming NEUTRAL GAS */
+ molecular_weight = 4 / (1 + 3 * HYDROGEN_MASSFRAC);
+
+ u_init /= molecular_weight;
+#endif
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if(SphP[i].Entropy == 0)
+ SphP[i].Entropy = u_init;
+
+ /* Note: the conversion to entropy will be done in the function init(),
+ after the densities have been computed */
+ }
+ }
+ }
+
+
+#ifdef WRITE_ALL_MASSES
+ for(i = 0; i < 6; i++)
+ All.MassTable[i] = 0;
+#endif
+
+#ifdef MULTIPHASE
+ for(i = 0; i < N_gas; i++)
+ {
+ if(SphP[i].Entropy>0)
+ SphP[i].Entropy = dmax(All.MinEgySpec, SphP[i].Entropy);
+ else
+ SphP[i].Entropy = -dmax(All.MinEgySpec, -SphP[i].Entropy);
+ }
+#else
+ for(i = 0; i < N_gas; i++)
+ SphP[i].Entropy = dmax(All.MinEgySpec, SphP[i].Entropy);
+#endif
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ printf("reading done.\n");
+ fflush(stdout);
+ }
+
+ if(ThisTask == 0)
+ {
+ printf("Total number of particles : %d%09d\n\n",
+ (int) (All.TotNumPart / 1000000000), (int) (All.TotNumPart % 1000000000));
+ fflush(stdout);
+ }
+}
+
+
+/*! This function reads out the buffer that was filled with particle data, and
+ * stores it at the appropriate place in the particle structures.
+ */
+void empty_read_buffer(enum iofields blocknr, int offset, int pc, int type)
+{
+ int n, k;
+ float *fp;
+
+#ifdef LONGIDS
+ long long *ip;
+#else
+ int *ip;
+#endif
+
+ fp = CommBuffer;
+ ip = CommBuffer;
+
+ switch (blocknr)
+ {
+ case IO_POS: /* positions */
+ for(n = 0; n < pc; n++)
+ for(k = 0; k < 3; k++)
+ P[offset + n].Pos[k] = *fp++;
+
+ for(n = 0; n < pc; n++)
+ P[offset + n].Type = type; /* initialize type here as well */
+ break;
+
+ case IO_VEL: /* velocities */
+ for(n = 0; n < pc; n++)
+ for(k = 0; k < 3; k++)
+ P[offset + n].Vel[k] = *fp++;
+ break;
+
+ case IO_ID: /* particle ID */
+ for(n = 0; n < pc; n++)
+ P[offset + n].ID = *ip++;
+ break;
+
+ case IO_MASS: /* particle mass */
+ for(n = 0; n < pc; n++)
+ P[offset + n].Mass = *fp++;
+ break;
+
+ case IO_U: /* temperature */
+ for(n = 0; n < pc; n++)
+ SphP[offset + n].Entropy = *fp++;
+ break;
+
+ case IO_RHO: /* density */
+ for(n = 0; n < pc; n++)
+ SphP[offset + n].Density = *fp++;
+ break;
+
+
+ case IO_HSML: /* SPH smoothing length */
+ for(n = 0; n < pc; n++)
+ SphP[offset + n].Hsml = *fp++;
+ break;
+
+
+
+ /* the other input fields (if present) are not needed to define the
+ initial conditions of the code */
+
+ case IO_POT:
+ case IO_ACCEL:
+ case IO_DTENTR:
+ case IO_TSTP:
+ case IO_ERADSPH:
+ case IO_ERADSTICKY:
+ case IO_ERADFEEDBACK:
+ case IO_ENERGYFLUX:
+ case IO_OPTVAR:
+ break;
+
+
+ case IO_METALS: /* gas metallicity */
+#ifdef CHIMIE
+ for(n = 0; n < pc; n++)
+ {
+ for(k = 0; k < NELEMENTS; k++)
+ {
+ SphP[offset + n].Metal[k] = *fp++;
+ }
+ }
+#endif
+ break;
+
+
+ case IO_STAR_FORMATIONTIME:
+#ifdef STELLAR_PROP
+ for(n = 0; n < pc; n++)
+ StP[offset + n].FormationTime = *fp++;
+#endif
+ break;
+
+ case IO_INITIAL_MASS:
+#ifdef STELLAR_PROP
+ for(n = 0; n < pc; n++)
+ StP[offset + n].InitialMass = *fp++;
+#endif
+ break;
+
+ case IO_STAR_IDPROJ:
+#ifdef STELLAR_PROP
+ for(n = 0; n < pc; n++)
+ StP[offset + n].IDProj = *ip++;
+#endif
+ break;
+
+ case IO_STAR_RHO:
+#ifdef STELLAR_PROP
+ for(n = 0; n < pc; n++)
+ StP[offset + n].Density = *fp++;
+#endif
+ break;
+
+ case IO_STAR_HSML:
+#ifdef STELLAR_PROP
+ for(n = 0; n < pc; n++)
+ StP[offset + n].Hsml = *fp++;
+#endif
+ break;
+
+ case IO_STAR_METALS:
+#ifdef CHIMIE
+ for(n = 0; n < pc; n++)
+ {
+ for(k = 0; k < NELEMENTS; k++)
+ {
+ StP[offset + n].Metal[k] = *fp++;
+ }
+ }
+#endif
+ break;
+
+ }
+}
+
+
+
+/*! This function reads a snapshot file and distributes the data it contains
+ * to tasks 'readTask' to 'lastTask'.
+ */
+void read_file(char *fname, int readTask, int lastTask)
+{
+ int blockmaxlen;
+ int i, n_in_file, n_for_this_task, ntask, pc, offset = 0, task;
+ int blksize1, blksize2;
+ MPI_Status status;
+ FILE *fd = 0;
+ int nall;
+ int type;
+ char label[4];
+ int nstart, bytes_per_blockelement, npart, nextblock, typelist[6];
+ enum iofields blocknr;
+
+#ifdef HAVE_HDF5
+ char buf[500];
+ int rank, pcsum;
+ hid_t hdf5_file, hdf5_grp[6], hdf5_dataspace_in_file;
+ hid_t hdf5_datatype, hdf5_dataspace_in_memory, hdf5_dataset;
+ hsize_t dims[2], count[2], start[2];
+#endif
+
+#define SKIP {my_fread(&blksize1,sizeof(int),1,fd);}
+#define SKIP2 {my_fread(&blksize2,sizeof(int),1,fd);}
+
+ if(ThisTask == readTask)
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ {
+ if(!(fd = fopen(fname, "r")))
+ {
+ printf("can't open file `%s' for reading initial conditions.\n", fname);
+ endrun(123);
+ }
+
+ if(All.ICFormat == 2)
+ {
+ SKIP;
+ my_fread(&label, sizeof(char), 4, fd);
+ my_fread(&nextblock, sizeof(int), 1, fd);
+ printf("Reading header => '%c%c%c%c' (%d byte)\n", label[0], label[1], label[2], label[3],
+ nextblock);
+ SKIP2;
+ }
+
+ SKIP;
+ my_fread(&header, sizeof(header), 1, fd);
+ SKIP2;
+
+ if(blksize1 != 256 || blksize2 != 256)
+ {
+ printf("incorrect header format\n");
+ fflush(stdout);
+ endrun(890);
+ }
+
+#ifdef CHIMIE_EXTRAHEADER
+ if (header.flag_chimie_extraheader)
+ {
+ SKIP;
+ my_fread(&chimie_extraheader, sizeof(chimie_extraheader), 1, fd);
+ SKIP2;
+
+ if(blksize1 != 256 || blksize2 != 256)
+ {
+ printf("incorrect chimie_extraheader format\n");
+ fflush(stdout);
+ endrun(891);
+ }
+ }
+#endif
+
+ }
+
+
+#ifdef HAVE_HDF5
+ if(All.ICFormat == 3)
+ {
+ read_header_attributes_in_hdf5(fname);
+
+ hdf5_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
+
+ for(type = 0; type < 6; type++)
+ {
+ if(header.npart[type] > 0)
+ {
+ sprintf(buf, "/PartType%d", type);
+ hdf5_grp[type] = H5Gopen(hdf5_file, buf);
+ }
+ }
+ }
+#endif
+
+ for(task = readTask + 1; task <= lastTask; task++)
+ {
+ MPI_Ssend(&header, sizeof(header), MPI_BYTE, task, TAG_HEADER, MPI_COMM_WORLD);
+#ifdef CHIMIE_EXTRAHEADER
+ if (header.flag_chimie_extraheader)
+ MPI_Ssend(&header, sizeof(chimie_extraheader), MPI_BYTE, task, TAG_CHIMIE_EXTRAHEADER, MPI_COMM_WORLD);
+#endif
+ }
+ }
+ else
+ {
+ MPI_Recv(&header, sizeof(header), MPI_BYTE, readTask, TAG_HEADER, MPI_COMM_WORLD, &status);
+#ifdef CHIMIE_EXTRAHEADER
+ if (header.flag_chimie_extraheader)
+ MPI_Recv(&chimie_extraheader, sizeof(chimie_extraheader), MPI_BYTE, readTask, TAG_CHIMIE_EXTRAHEADER, MPI_COMM_WORLD, &status);
+#endif
+ }
+
+ if(All.TotNumPart == 0)
+ {
+ if(header.num_files <= 1)
+ for(i = 0; i < 6; i++)
+ header.npartTotal[i] = header.npart[i];
+
+ All.TotN_gas = header.npartTotal[0] + (((long long) header.npartTotalHighWord[0]) << 32);
+#if defined(SFR) || defined(STELLAR_PROP)
+ All.TotN_stars = header.npartTotal[ST] + (((long long) header.npartTotalHighWord[ST]) << 32);
+#endif
+ for(i = 0, All.TotNumPart = 0; i < 6; i++)
+ {
+ All.TotNumPart += header.npartTotal[i];
+ All.TotNumPart += (((long long) header.npartTotalHighWord[i]) << 32);
+ }
+
+
+ for(i = 0; i < 6; i++)
+ All.MassTable[i] = header.mass[i];
+
+ All.MaxPart = All.PartAllocFactor * (All.TotNumPart / NTask); /* sets the maximum number of particles that may */
+ All.MaxPartSph = All.PartAllocFactor * (All.TotN_gas / NTask); /* sets the maximum number of particles that may
+ reside on a processor */
+#ifdef STELLAR_PROP
+ All.MaxPartStars = All.PartAllocFactor * (All.TotN_stars / NTask); /* existing star particles */
+#ifdef SFR
+ All.MaxPartStars = All.MaxPartStars + All.StarFormationNStarsFromGas*All.MaxPartSph; /* potental star particles */
+#endif
+#endif
+ allocate_memory();
+
+ if(RestartFlag == 2)
+ All.Time = All.TimeBegin = header.time;
+ }
+
+ if(ThisTask == readTask)
+ {
+ for(i = 0, n_in_file = 0; i < 6; i++)
+ n_in_file += header.npart[i];
+
+ printf("\nreading file `%s' on task=%d (contains %d particles.)\n"
+ "distributing this file to tasks %d-%d\n"
+ "Type 0 (gas): %8d (tot=%6d%09d) masstab=%g\n"
+ "Type 1 (halo): %8d (tot=%6d%09d) masstab=%g\n"
+ "Type 2 (disk): %8d (tot=%6d%09d) masstab=%g\n"
+ "Type 3 (bulge): %8d (tot=%6d%09d) masstab=%g\n"
+ "Type 4 (stars): %8d (tot=%6d%09d) masstab=%g\n"
+ "Type 5 (bndry): %8d (tot=%6d%09d) masstab=%g\n\n", fname, ThisTask, n_in_file, readTask,
+ lastTask, header.npart[0], (int) (header.npartTotal[0] / 1000000000),
+ (int) (header.npartTotal[0] % 1000000000), All.MassTable[0], header.npart[1],
+ (int) (header.npartTotal[1] / 1000000000), (int) (header.npartTotal[1] % 1000000000),
+ All.MassTable[1], header.npart[2], (int) (header.npartTotal[2] / 1000000000),
+ (int) (header.npartTotal[2] % 1000000000), All.MassTable[2], header.npart[3],
+ (int) (header.npartTotal[3] / 1000000000), (int) (header.npartTotal[3] % 1000000000),
+ All.MassTable[3], header.npart[4], (int) (header.npartTotal[4] / 1000000000),
+ (int) (header.npartTotal[4] % 1000000000), All.MassTable[4], header.npart[5],
+ (int) (header.npartTotal[5] / 1000000000), (int) (header.npartTotal[5] % 1000000000),
+ All.MassTable[5]);
+ fflush(stdout);
+ }
+
+
+ ntask = lastTask - readTask + 1;
+
+
+ /* to collect the gas particles all at the beginning (in case several
+ snapshot files are read on the current CPU) we move the collisionless
+ particles such that a gap of the right size is created */
+
+ for(type = 0, nall = 0; type < 6; type++)
+ {
+ n_in_file = header.npart[type];
+
+ n_for_this_task = n_in_file / ntask;
+ if((ThisTask - readTask) < (n_in_file % ntask))
+ n_for_this_task++;
+
+ nall += n_for_this_task;
+ }
+
+ memmove(&P[N_gas + nall], &P[N_gas], (NumPart - N_gas) * sizeof(struct particle_data));
+ nstart = N_gas;
+
+
+
+ for(blocknr = 0; blocknr < IO_NBLOCKS; blocknr++)
+ {
+ if(blockpresent(blocknr))
+ {
+
+#ifndef BLOCK_SKIPPING
+ if(RestartFlag == 0 && blocknr > IO_U)
+ continue; /* ignore all other blocks in initial conditions */
+#else
+ if(blockabsent(blocknr))
+ continue;
+#endif
+
+ bytes_per_blockelement = get_bytes_per_blockelement(blocknr);
+
+ blockmaxlen = ((int) (All.BufferSize * 1024 * 1024)) / bytes_per_blockelement;
+
+ npart = get_particles_in_block(blocknr, &typelist[0]);
+
+ if(npart > 0)
+ {
+ if(ThisTask == readTask)
+ {
+ if(All.ICFormat == 2)
+ {
+ SKIP;
+ my_fread(&label, sizeof(char), 4, fd);
+ my_fread(&nextblock, sizeof(int), 1, fd);
+ printf("Reading header => '%c%c%c%c' (%d byte)\n", label[0], label[1], label[2],
+ label[3], nextblock);
+ SKIP2;
+
+ if(strncmp(label, Tab_IO_Labels[blocknr], 4) != 0)
+ {
+ printf("incorrect block-structure!\n");
+ printf("expected '%c%c%c%c' but found '%c%c%c%c'\n",
+ label[0], label[1], label[2], label[3],
+ Tab_IO_Labels[blocknr][0], Tab_IO_Labels[blocknr][1],
+ Tab_IO_Labels[blocknr][2], Tab_IO_Labels[blocknr][3]);
+ fflush(stdout);
+ endrun(1890);
+ }
+ }
+
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ SKIP;
+ }
+
+ for(type = 0, offset = 0; type < 6; type++)
+ {
+ n_in_file = header.npart[type];
+#ifdef HAVE_HDF5
+ pcsum = 0;
+#endif
+ if(typelist[type] == 0)
+ {
+ n_for_this_task = n_in_file / ntask;
+ if((ThisTask - readTask) < (n_in_file % ntask))
+ n_for_this_task++;
+
+ offset += n_for_this_task;
+ }
+ else
+ {
+ for(task = readTask; task <= lastTask; task++)
+ {
+ n_for_this_task = n_in_file / ntask;
+ if((task - readTask) < (n_in_file % ntask))
+ n_for_this_task++;
+
+ if(task == ThisTask)
+ if(NumPart + n_for_this_task > All.MaxPart)
+ {
+ printf("too many particles\n");
+ endrun(1313);
+ }
+
+
+ do
+ {
+ pc = n_for_this_task;
+
+ if(pc > blockmaxlen)
+ pc = blockmaxlen;
+
+ if(ThisTask == readTask)
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ my_fread(CommBuffer, bytes_per_blockelement, pc, fd);
+#ifdef HAVE_HDF5
+ if(All.ICFormat == 3)
+ {
+ get_dataset_name(blocknr, buf);
+ hdf5_dataset = H5Dopen(hdf5_grp[type], buf);
+
+ dims[0] = header.npart[type];
+ dims[1] = get_values_per_blockelement(blocknr);
+ if(dims[1] == 1)
+ rank = 1;
+ else
+ rank = 2;
+
+ hdf5_dataspace_in_file = H5Screate_simple(rank, dims, NULL);
+
+ dims[0] = pc;
+ hdf5_dataspace_in_memory = H5Screate_simple(rank, dims, NULL);
+
+ start[0] = pcsum;
+ start[1] = 0;
+
+ count[0] = pc;
+ count[1] = get_values_per_blockelement(blocknr);
+ pcsum += pc;
+
+ H5Sselect_hyperslab(hdf5_dataspace_in_file, H5S_SELECT_SET,
+ start, NULL, count, NULL);
+
+ switch (get_datatype_in_block(blocknr))
+ {
+ case 0:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_UINT);
+ break;
+ case 1:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_FLOAT);
+ break;
+ case 2:
+ hdf5_datatype = H5Tcopy(H5T_NATIVE_UINT64);
+ break;
+ }
+
+ H5Dread(hdf5_dataset, hdf5_datatype, hdf5_dataspace_in_memory,
+ hdf5_dataspace_in_file, H5P_DEFAULT, CommBuffer);
+
+ H5Tclose(hdf5_datatype);
+ H5Sclose(hdf5_dataspace_in_memory);
+ H5Sclose(hdf5_dataspace_in_file);
+ H5Dclose(hdf5_dataset);
+ }
+#endif
+ }
+
+ if(ThisTask == readTask && task != readTask)
+ MPI_Ssend(CommBuffer, bytes_per_blockelement * pc, MPI_BYTE, task, TAG_PDATA,
+ MPI_COMM_WORLD);
+
+ if(ThisTask != readTask && task == ThisTask)
+ MPI_Recv(CommBuffer, bytes_per_blockelement * pc, MPI_BYTE, readTask,
+ TAG_PDATA, MPI_COMM_WORLD, &status);
+
+ if(ThisTask == task)
+ {
+ empty_read_buffer(blocknr, nstart + offset, pc, type);
+
+ offset += pc;
+ }
+
+ n_for_this_task -= pc;
+ }
+ while(n_for_this_task > 0);
+ }
+ }
+ }
+ if(ThisTask == readTask)
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ {
+ SKIP2;
+ if(blksize1 != blksize2)
+ {
+ printf("incorrect block-sizes detected!\n");
+ printf("Task=%d blocknr=%d blksize1=%d blksize2=%d\n", ThisTask, blocknr,
+ blksize1, blksize2);
+ fflush(stdout);
+ endrun(1889);
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ for(type = 0; type < 6; type++)
+ {
+ n_in_file = header.npart[type];
+
+ n_for_this_task = n_in_file / ntask;
+ if((ThisTask - readTask) < (n_in_file % ntask))
+ n_for_this_task++;
+
+ NumPart += n_for_this_task;
+
+ if(type == 0)
+ N_gas += n_for_this_task;
+
+#ifdef SFR
+ if(type == ST)
+ N_stars += n_for_this_task;
+#endif
+ }
+
+ if(ThisTask == readTask)
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ fclose(fd);
+#ifdef HAVE_HDF5
+ if(All.ICFormat == 3)
+ {
+ for(type = 5; type >= 0; type--)
+ if(header.npart[type] > 0)
+ H5Gclose(hdf5_grp[type]);
+ H5Fclose(hdf5_file);
+ }
+#endif
+ }
+}
+
+
+
+
+/*! This function determines onto how many files a given snapshot is
+ * distributed.
+ */
+int find_files(char *fname)
+{
+ FILE *fd;
+ char buf[200], buf1[200];
+ int dummy;
+
+ sprintf(buf, "%s.%d", fname, 0);
+ sprintf(buf1, "%s", fname);
+
+ if(All.SnapFormat == 3)
+ {
+ sprintf(buf, "%s.%d.hdf5", fname, 0);
+ sprintf(buf1, "%s.hdf5", fname);
+ }
+
+#ifndef HAVE_HDF5
+ if(All.SnapFormat == 3)
+ {
+ if(ThisTask == 0)
+ printf("Code wasn't compiled with HDF5 support enabled!\n");
+ endrun(0);
+ }
+#endif
+
+ header.num_files = 0;
+
+ if(ThisTask == 0)
+ {
+ if((fd = fopen(buf, "r")))
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ {
+ if(All.ICFormat == 2)
+ {
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ }
+
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&header, sizeof(header), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ }
+ fclose(fd);
+
+#ifdef HAVE_HDF5
+ if(All.ICFormat == 3)
+ read_header_attributes_in_hdf5(buf);
+#endif
+ }
+ }
+
+ MPI_Bcast(&header, sizeof(header), MPI_BYTE, 0, MPI_COMM_WORLD);
+
+ if(header.num_files > 0)
+ return header.num_files;
+
+ if(ThisTask == 0)
+ {
+ if((fd = fopen(buf1, "r")))
+ {
+ if(All.ICFormat == 1 || All.ICFormat == 2)
+ {
+ if(All.ICFormat == 2)
+ {
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ }
+
+ fread(&dummy, sizeof(dummy), 1, fd);
+ fread(&header, sizeof(header), 1, fd);
+ fread(&dummy, sizeof(dummy), 1, fd);
+ }
+ fclose(fd);
+
+#ifdef HAVE_HDF5
+ if(All.ICFormat == 3)
+ read_header_attributes_in_hdf5(buf1);
+#endif
+ header.num_files = 1;
+ }
+ }
+
+ MPI_Bcast(&header, sizeof(header), MPI_BYTE, 0, MPI_COMM_WORLD);
+
+ if(header.num_files > 0)
+ return header.num_files;
+
+ if(ThisTask == 0)
+ {
+ printf("\nCan't find initial conditions file.");
+ printf("neither as '%s'\nnor as '%s'\n", buf, buf1);
+ fflush(stdout);
+ }
+
+ endrun(0);
+ return 0;
+}
+
+
+
+/*! This function assigns a certain number of files to processors, such that
+ * each processor is exactly assigned to one file, and the number of cpus per
+ * file is as homogenous as possible. The number of files may at most be
+ * equal to the number of processors.
+ */
+void distribute_file(int nfiles, int firstfile, int firsttask, int lasttask, int *filenr, int *master,
+ int *last)
+{
+ int ntask, filesleft, filesright, tasksleft, tasksright;
+
+ if(nfiles > 1)
+ {
+ ntask = lasttask - firsttask + 1;
+
+ filesleft = (((double) (ntask / 2)) / ntask) * nfiles;
+ if(filesleft <= 0)
+ filesleft = 1;
+ if(filesleft >= nfiles)
+ filesleft = nfiles - 1;
+
+ filesright = nfiles - filesleft;
+
+ tasksleft = ntask / 2;
+ tasksright = ntask - tasksleft;
+
+ distribute_file(filesleft, firstfile, firsttask, firsttask + tasksleft - 1, filenr, master, last);
+ distribute_file(filesright, firstfile + filesleft, firsttask + tasksleft, lasttask, filenr, master,
+ last);
+ }
+ else
+ {
+ if(ThisTask >= firsttask && ThisTask <= lasttask)
+ {
+ *filenr = firstfile;
+ *master = firsttask;
+ *last = lasttask;
+ }
+ }
+}
+
+
+/*! This function reads the header information in case the HDF5 file format is
+ * used.
+ */
+#ifdef HAVE_HDF5
+void read_header_attributes_in_hdf5(char *fname)
+{
+ hid_t hdf5_file, hdf5_headergrp, hdf5_attribute;
+
+
+ hdf5_file = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
+ hdf5_headergrp = H5Gopen(hdf5_file, "/Header");
+
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "NumPart_ThisFile");
+ H5Aread(hdf5_attribute, H5T_NATIVE_INT, header.npart);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "NumPart_Total");
+ H5Aread(hdf5_attribute, H5T_NATIVE_UINT, header.npartTotal);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "NumPart_Total_HighWord");
+ H5Aread(hdf5_attribute, H5T_NATIVE_UINT, header.npartTotalHighWord);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "MassTable");
+ H5Aread(hdf5_attribute, H5T_NATIVE_DOUBLE, header.mass);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "Time");
+ H5Aread(hdf5_attribute, H5T_NATIVE_DOUBLE, &header.time);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "NumFilesPerSnapshot");
+ H5Aread(hdf5_attribute, H5T_NATIVE_INT, &header.num_files);
+ H5Aclose(hdf5_attribute);
+
+ hdf5_attribute = H5Aopen_name(hdf5_headergrp, "Flag_Entropy_ICs");
+ H5Aread(hdf5_attribute, H5T_NATIVE_INT, &header.flag_entropy_instead_u);
+ H5Aclose(hdf5_attribute);
+
+ H5Gclose(hdf5_headergrp);
+ H5Fclose(hdf5_file);
+}
+#endif
diff --git a/restart.c b/restart.c
new file mode 100644
index 0000000..a944b51
--- /dev/null
+++ b/restart.c
@@ -0,0 +1,386 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/file.h>
+#include <unistd.h>
+#include <gsl/gsl_rng.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file restart.c
+ * \brief Code for reading and writing restart files
+ */
+
+static FILE *fd,*fd_bak;
+
+static void in(int *x, int modus);
+static void byten(void *x, size_t n, int modus);
+
+
+/*! This function reads or writes the restart files. Each processor writes
+ * its own restart file, with the I/O being done in parallel. To avoid
+ * congestion of the disks you can tell the program to restrict the number of
+ * files that are simultaneously written to NumFilesWrittenInParallel.
+ *
+ * If modus>0 the restart()-routine reads, if modus==0 it writes a restart
+ * file.
+ */
+void restart(int modus)
+{
+ char buf[200], buf_bak[200], buf_mv[500];
+ double save_PartAllocFactor, save_TreeAllocFactor;
+ int i, nprocgroup, masterTask, groupTask, old_MaxPart, old_MaxNodes;
+ struct global_data_all_processes all_task0;
+
+
+ sprintf(buf, "%s%s.%d", All.OutputDir, All.RestartFile, ThisTask);
+ sprintf(buf_bak, "%s%s.%d.bak", All.OutputDir, All.RestartFile, ThisTask);
+ sprintf(buf_mv, "mv %s %s", buf, buf_bak);
+
+
+ if((NTask < All.NumFilesWrittenInParallel))
+ {
+ printf
+ ("Fatal error.\nNumber of processors must be a smaller or equal than `NumFilesWrittenInParallel'.\n");
+ endrun(2131);
+ }
+
+ nprocgroup = NTask / All.NumFilesWrittenInParallel;
+
+ if((NTask % All.NumFilesWrittenInParallel))
+ {
+ nprocgroup++;
+ }
+
+ masterTask = (ThisTask / nprocgroup) * nprocgroup;
+
+ for(groupTask = 0; groupTask < nprocgroup; groupTask++)
+ {
+ if(ThisTask == (masterTask + groupTask)) /* ok, it's this processor's turn */
+ {
+ if(modus)
+ {
+ if(!(fd = fopen(buf, "r")))
+ {
+ printf("Restart file '%s' not found.\n", buf);
+ endrun(7870);
+ }
+ }
+ else
+ {
+
+ system(buf_mv); /* move old restart files to .bak files */
+
+
+/*
+
+ /* move old restart files to .bak files */
+/*
+ system(buf_mv);
+
+ This does not work !!!
+
+ fd = fopen(buf,"r");
+ fd_bak = fopen(buf_bak,"w");
+
+ while(1)
+ {
+ fgets(buf, 200, fd);
+ if (feof(fd)) break;
+ fprintf(fd_bak, buf, 200);
+ }
+
+ fclose(fd);
+ fclose(fd_bak);
+*/
+
+
+
+ if(!(fd = fopen(buf, "w")))
+ {
+ printf("Restart file '%s' cannot be opened.\n", buf);
+ endrun(7878);
+ }
+ }
+
+
+ save_PartAllocFactor = All.PartAllocFactor;
+ save_TreeAllocFactor = All.TreeAllocFactor;
+
+ /* common data */
+ byten(&All, sizeof(struct global_data_all_processes), modus);
+
+ if(ThisTask == 0 && modus > 0)
+ all_task0 = All;
+
+ if(modus > 0 && groupTask == 0) /* read */
+ {
+ MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, MPI_COMM_WORLD);
+ }
+
+ old_MaxPart = All.MaxPart;
+ old_MaxNodes = All.TreeAllocFactor * All.MaxPart;
+
+ if(modus) /* read */
+ {
+ if(All.PartAllocFactor != save_PartAllocFactor)
+ {
+ All.PartAllocFactor = save_PartAllocFactor;
+ All.MaxPart = All.PartAllocFactor * (All.TotNumPart / NTask);
+ All.MaxPartSph = All.PartAllocFactor * (All.TotN_gas / NTask);
+#ifdef STELLAR_PROP
+ All.MaxPartStars = All.PartAllocFactor * (All.TotN_stars / NTask); /* existing star particles */
+#ifdef SFR
+ All.MaxPartStars = All.MaxPartStars = + All.StarFormationNStarsFromGas*All.MaxPartSph; /* potental star particles */
+#endif
+#endif
+
+ save_PartAllocFactor = -1;
+ }
+
+ if(All.TreeAllocFactor != save_TreeAllocFactor)
+ {
+ All.TreeAllocFactor = save_TreeAllocFactor;
+ save_TreeAllocFactor = -1;
+ }
+
+ if(all_task0.Time != All.Time)
+ {
+ printf("The restart file on task=%d is not consistent with the one on task=0\n", ThisTask);
+ fflush(stdout);
+ endrun(16);
+ }
+
+ allocate_memory();
+ }
+
+ in(&NumPart, modus);
+
+ if(NumPart > All.MaxPart)
+ {
+ printf
+ ("it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n",
+ NumPart / (((double) All.TotNumPart) / NTask));
+ printf("fatal error\n");
+ endrun(22);
+ }
+
+ /* Particle data */
+ byten(&P[0], NumPart * sizeof(struct particle_data), modus);
+
+ in(&N_gas, modus);
+
+ if(N_gas > 0)
+ {
+ if(N_gas > All.MaxPartSph)
+ {
+ printf
+ ("SPH: it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n",
+ N_gas / (((double) All.TotN_gas) / NTask));
+ printf("fatal error\n");
+ endrun(222);
+ }
+ /* Sph-Particle data */
+ byten(&SphP[0], N_gas * sizeof(struct sph_particle_data), modus);
+ }
+
+
+#ifdef SFR
+ in(&N_stars, modus);
+
+#ifdef STELLAR_PROP
+ if(N_stars > 0)
+ {
+ if(N_stars > All.MaxPartStars)
+ {
+ printf
+ ("StP : it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n",
+ N_stars / (((double) All.TotN_stars) / NTask));
+ printf("fatal error\n");
+ endrun(2222);
+ }
+
+ /* Particle data */
+ byten(&StP[0], N_stars * sizeof(struct st_particle_data), modus);
+ }
+
+
+ /* this is a test, in case the restart fails */
+ //for(i=0;i<N_stars;i++)
+ // {
+ // printf("(%d) %d %15g %15g \n",ThisTask,N_stars,StP[i].FormationTime,StP[i].Hsml);
+ // }
+
+
+#endif
+#endif
+
+
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ in(&SetMinTimeStepForActives, modus);
+#endif
+
+ /* write state of random number generator */
+ byten(gsl_rng_state(random_generator), gsl_rng_size(random_generator), modus);
+
+
+ /* now store relevant data for tree */
+
+ if(modus) /* read */
+ {
+ ngb_treeallocate(MAX_NGB);
+
+ force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
+ }
+
+
+ in(&Numnodestree, modus);
+
+ if(Numnodestree > MaxNodes)
+ {
+ printf
+ ("Tree storage: it seems you have reduced(!) 'PartAllocFactor' below the value needed to load the restart file (task=%d). "
+ "Numnodestree=%d MaxNodes=%d\n", ThisTask, Numnodestree, MaxNodes);
+ endrun(221);
+ }
+
+ byten(Nodes_base, Numnodestree * sizeof(struct NODE), modus);
+ byten(Extnodes_base, Numnodestree * sizeof(struct extNODE), modus);
+
+ byten(Father, NumPart * sizeof(int), modus);
+
+ byten(Nextnode, NumPart * sizeof(int), modus);
+ byten(Nextnode + All.MaxPart, MAXTOPNODES * sizeof(int), modus);
+
+ byten(DomainStartList, NTask * sizeof(int), modus);
+ byten(DomainEndList, NTask * sizeof(int), modus);
+ byten(DomainTask, MAXTOPNODES * sizeof(int), modus);
+ byten(DomainNodeIndex, MAXTOPNODES * sizeof(int), modus);
+ byten(DomainTreeNodeLen, MAXTOPNODES * sizeof(FLOAT), modus);
+ byten(DomainHmax, MAXTOPNODES * sizeof(FLOAT), modus);
+ byten(DomainMoment, MAXTOPNODES * sizeof(struct DomainNODE), modus);
+
+ byten(DomainCorner, 3 * sizeof(double), modus);
+ byten(DomainCenter, 3 * sizeof(double), modus);
+ byten(&DomainLen, sizeof(double), modus);
+ byten(&DomainFac, sizeof(double), modus);
+ byten(&DomainMyStart, sizeof(int), modus);
+ byten(&DomainMyLast, sizeof(int), modus);
+
+ if(modus) /* read */
+ if(All.PartAllocFactor != save_PartAllocFactor || All.TreeAllocFactor != save_TreeAllocFactor)
+ {
+ for(i = 0; i < NumPart; i++)
+ Father[i] += (All.MaxPart - old_MaxPart);
+
+ for(i = 0; i < NumPart; i++)
+ if(Nextnode[i] >= old_MaxPart)
+ {
+ if(Nextnode[i] >= old_MaxPart + old_MaxNodes)
+ Nextnode[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxPart);
+ else
+ Nextnode[i] += (All.MaxPart - old_MaxPart);
+ }
+
+ for(i = 0; i < Numnodestree; i++)
+ {
+ if(Nodes_base[i].u.d.sibling >= old_MaxPart)
+ {
+ if(Nodes_base[i].u.d.sibling >= old_MaxPart + old_MaxNodes)
+ Nodes_base[i].u.d.sibling +=
+ (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
+ else
+ Nodes_base[i].u.d.sibling += (All.MaxPart - old_MaxPart);
+ }
+
+ if(Nodes_base[i].u.d.father >= old_MaxPart)
+ {
+ if(Nodes_base[i].u.d.father >= old_MaxPart + old_MaxNodes)
+ Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
+ else
+ Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart);
+ }
+
+ if(Nodes_base[i].u.d.nextnode >= old_MaxPart)
+ {
+ if(Nodes_base[i].u.d.nextnode >= old_MaxPart + old_MaxNodes)
+ Nodes_base[i].u.d.nextnode +=
+ (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
+ else
+ Nodes_base[i].u.d.nextnode += (All.MaxPart - old_MaxPart);
+ }
+ }
+
+ for(i = 0; i < MAXTOPNODES; i++)
+ if(Nextnode[i + All.MaxPart] >= old_MaxPart)
+ {
+ if(Nextnode[i + All.MaxPart] >= old_MaxPart + old_MaxNodes)
+ Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
+ else
+ Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart);
+ }
+
+ for(i = 0; i < MAXTOPNODES; i++)
+ if(DomainNodeIndex[i] >= old_MaxPart)
+ {
+ if(DomainNodeIndex[i] >= old_MaxPart + old_MaxNodes)
+ DomainNodeIndex[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
+ else
+ DomainNodeIndex[i] += (All.MaxPart - old_MaxPart);
+ }
+ }
+
+
+
+ /* local system state (yr) */
+ if (sizeof(struct local_state_of_system))
+ byten(&LocalSysState, sizeof(struct local_state_of_system), modus);
+
+
+
+
+ fclose(fd);
+ }
+ else /* wait inside the group */
+ {
+ if(modus > 0 && groupTask == 0) /* read */
+ {
+ MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, MPI_COMM_WORLD);
+ }
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+}
+
+
+
+/*! reads/writes n bytes in restart routine
+ */
+void byten(void *x, size_t n, int modus)
+{
+ if(modus)
+ my_fread(x, n, 1, fd);
+ else
+ my_fwrite(x, n, 1, fd);
+}
+
+
+/*! reads/writes one `int' variable in restart routine
+ */
+void in(int *x, int modus)
+{
+ if(modus)
+ my_fread(x, 1, sizeof(int), fd);
+ else
+ my_fwrite(x, 1, sizeof(int), fd);
+}
diff --git a/run.c b/run.c
new file mode 100644
index 0000000..1acba49
--- /dev/null
+++ b/run.c
@@ -0,0 +1,762 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <unistd.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file run.c
+ * \brief iterates over timesteps, main loop
+ */
+
+/*! This routine contains the main simulation loop that iterates over single
+ * timesteps. The loop terminates when the cpu-time limit is reached, when a
+ * `stop' file is found in the output directory, or when the simulation ends
+ * because we arrived at TimeMax.
+ */
+void run(void)
+{
+ FILE *fd;
+ int stopflag = 0;
+ char stopfname[200], contfname[200];
+ double t0, t1;
+
+#ifdef DETAILED_CPU
+ double tstart,tend;
+#endif
+
+
+
+ sprintf(stopfname, "%sstop", All.OutputDir);
+ sprintf(contfname, "%scont", All.OutputDir);
+ unlink(contfname);
+
+
+ do /* main loop */
+ {
+ t0 = second();
+
+ find_next_sync_point_and_drift(); /* find next synchronization point and drift particles to this time.
+ * If needed, this function will also write an output file
+ * at the desired time.
+ */
+
+ every_timestep_stuff(); /* write some info to log-files */
+
+
+
+#ifdef OUTPUT_EVERY_TIMESTEP
+ savepositions(All.SnapshotFileCount++); /* write snapshot file */
+#endif
+
+
+
+#ifdef DETAILED_CPU
+ tstart = second();
+#endif
+
+
+
+
+#ifdef AGN_ACCRETION
+ compute_agn_accretion(); /* compute accretion */
+#endif
+
+#ifdef BONDI_ACCRETION
+ compute_bondi_accretion(); /* compute bondi accretion */
+#endif
+
+#ifdef BUBBLES
+ make_bubble(); /* create a bubble */
+#endif
+
+#ifdef MULTIPHASE
+ update_phase(); /* allow particles to change their phase */
+#endif
+
+
+#ifdef CORIOLIS
+ set_outer_potential_coriolis(); /* coriolis */
+#endif
+
+
+
+#ifdef CHIMIE
+ chimie();
+#endif
+
+
+
+
+
+#ifdef SFR
+#ifdef COMPUTE_SFR_ENERGY
+ density(1);
+ force_update_hmax();
+ sfr_compute_energy_int(1);
+#endif
+ star_formation(); /* starformation */
+#endif
+
+
+#ifdef MULTIPHASE
+ sticky();
+#endif
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Physics += timediff(tstart, tend);
+#endif
+
+
+
+ domain_Decomposition(); /* do domain decomposition if needed */
+
+
+ compute_accelerations(0); /* compute accelerations for
+ * the particles that are to be advanced
+ */
+
+
+#if defined(SFR) && defined(COMPUTE_SFR_ENERGY)
+#ifdef DETAILED_CPU
+ tstart = second();
+#endif
+ //sfr_compute_energy_int(2);
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Physics += timediff(tstart, tend);
+#endif
+#endif
+
+
+
+
+
+
+
+
+
+ /* check whether we want a full energy statistics */
+ if((All.Time - All.TimeLastStatistics) >= All.TimeBetStatistics)
+ {
+#ifdef COMPUTE_POTENTIAL_ENERGY
+ compute_potential();
+#endif
+
+#ifndef ADVANCEDSTATISTICS
+ energy_statistics(); /* compute and output energy statistics */
+#else
+ advanced_energy_statistics(); /* compute and output energy statistics */
+#endif
+
+
+ All.TimeLastStatistics += All.TimeBetStatistics;
+ }
+
+ advance_and_find_timesteps(); /* 'kick' active particles in
+ * momentum space and compute new
+ * timesteps for them
+ */
+ All.NumCurrentTiStep++;
+
+ /* Check whether we need to interrupt the run */
+ if(ThisTask == 0)
+ {
+ /* Is the stop-file present? If yes, interrupt the run. */
+ if((fd = fopen(stopfname, "r")))
+ {
+ fclose(fd);
+ stopflag = 1;
+ unlink(stopfname);
+ }
+
+ /* are we running out of CPU-time ? If yes, interrupt run. */
+ if(CPUThisRun > 0.85 * All.TimeLimitCPU)
+ {
+ printf("reaching time-limit. stopping.\n");
+ stopflag = 2;
+ }
+ }
+
+ MPI_Bcast(&stopflag, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if(stopflag)
+ {
+ restart(0); /* write restart file */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(stopflag == 2 && ThisTask == 0)
+ {
+ if((fd = fopen(contfname, "w")))
+ fclose(fd);
+ }
+
+ if(stopflag == 2 && All.ResubmitOn && ThisTask == 0)
+ {
+ close_outputfiles();
+ system(All.ResubmitCommand);
+ }
+ return;
+ }
+
+ /* is it time to write a regular restart-file? (for security) */
+ if(ThisTask == 0)
+ {
+ if((CPUThisRun - All.TimeLastRestartFile) >= All.CpuTimeBetRestartFile)
+ {
+ All.TimeLastRestartFile = CPUThisRun;
+ stopflag = 3;
+ }
+ else
+ stopflag = 0;
+ }
+
+ MPI_Bcast(&stopflag, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if(stopflag == 3)
+ {
+ restart(0); /* write an occasional restart file */
+ stopflag = 0;
+ }
+
+ t1 = second();
+
+ All.CPU_Total += timediff(t0, t1);
+ CPUThisRun += timediff(t0, t1);
+ }
+ while(All.Ti_Current < TIMEBASE && All.Time <= All.TimeMax);
+
+ restart(0);
+
+ savepositions(All.SnapshotFileCount++); /* write a last snapshot
+ * file at final time (will
+ * be overwritten if
+ * All.TimeMax is increased
+ * and the run is continued)
+ */
+}
+
+
+/*! This function finds the next synchronization point of the system (i.e. the
+ * earliest point of time any of the particles needs a force computation),
+ * and drifts the system to this point of time. If the system drifts over
+ * the desired time of a snapshot file, the function will drift to this
+ * moment, generate an output, and then resume the drift.
+ */
+void find_next_sync_point_and_drift(void)
+{
+ int n, min, min_glob, flag, *temp;
+ double timeold;
+ double t0, t1;
+#ifdef DETAILED_CPU
+ double tstart,tend;
+#endif
+
+ t0 = second();
+#ifdef DETAILED_CPU
+ tstart = t0;
+#endif
+
+ timeold = All.Time;
+
+ for(n = 1, min = P[0].Ti_endstep; n < NumPart; n++)
+ if(min > P[n].Ti_endstep)
+ min = P[n].Ti_endstep;
+
+ MPI_Allreduce(&min, &min_glob, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+
+ /* We check whether this is a full step where all particles are synchronized */
+ flag = 1;
+ for(n = 0; n < NumPart; n++)
+ if(P[n].Ti_endstep > min_glob)
+ flag = 0;
+
+ MPI_Allreduce(&flag, &Flag_FullStep, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+
+#ifdef PMGRID
+ if(min_glob >= All.PM_Ti_endstep)
+ {
+ min_glob = All.PM_Ti_endstep;
+ Flag_FullStep = 1;
+ }
+#endif
+
+ /* Determine 'NumForceUpdate', i.e. the number of particles on this processor that are going to be active */
+ for(n = 0, NumForceUpdate = 0; n < NumPart; n++)
+ {
+ if(P[n].Ti_endstep == min_glob)
+#ifdef SELECTIVE_NO_GRAVITY
+ if(!((1 << P[n].Type) & (SELECTIVE_NO_GRAVITY)))
+#endif
+ NumForceUpdate++;
+ }
+
+ /* note: NumForcesSinceLastDomainDecomp has type "long long" */
+ temp = malloc(NTask * sizeof(int));
+ MPI_Allgather(&NumForceUpdate, 1, MPI_INT, temp, 1, MPI_INT, MPI_COMM_WORLD);
+ for(n = 0; n < NTask; n++)
+ All.NumForcesSinceLastDomainDecomp += temp[n];
+
+#ifdef COUNT_ACTIVE_PARTICLES
+ long long NumActivePatricles;
+ NumActivePatricles=0;
+ for(n = 0; n < NTask; n++)
+ NumActivePatricles+=temp[n];
+#endif
+
+ free(temp);
+
+
+
+ t1 = second();
+ All.CPU_Predict += timediff(t0, t1);
+
+ while(min_glob >= All.Ti_nextoutput && All.Ti_nextoutput >= 0)
+ {
+ move_particles(All.Ti_Current, All.Ti_nextoutput);
+
+ All.Ti_Current = All.Ti_nextoutput;
+
+ if(All.ComovingIntegrationOn)
+ All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
+ else
+ All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
+
+#ifdef OUTPUTPOTENTIAL
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+ domain_Decomposition();
+ compute_potential();
+#endif
+#ifndef OUTPUT_EVERY_TIMESTEP
+ savepositions(All.SnapshotFileCount++); /* write snapshot file */
+#endif
+
+ All.Ti_nextoutput = find_next_outputtime(All.Ti_nextoutput + 1);
+ }
+
+ move_particles(All.Ti_Current, min_glob);
+
+ All.Ti_Current = min_glob;
+
+ if(All.ComovingIntegrationOn)
+ All.Time = All.TimeBegin * exp(All.Ti_Current * All.Timebase_interval);
+ else
+ All.Time = All.TimeBegin + All.Ti_Current * All.Timebase_interval;
+
+ All.TimeStep = All.Time - timeold;
+
+#ifdef COUNT_ACTIVE_PARTICLES
+ if (ThisTask==0)
+ fprintf(FdLog,"%g %g : Total number of active particles : %d%09d\n\n",All.Time,All.TimeStep,(int) (NumActivePatricles / 1000000000), (int) (NumActivePatricles % 1000000000));
+#endif
+
+
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Leapfrog += timediff(tstart, tend);
+#endif
+
+
+}
+
+
+
+/*! this function returns the next output time that is equal or larger to
+ * ti_curr
+ */
+int find_next_outputtime(int ti_curr)
+{
+
+ int i, ti, ti_next, iter = 0;
+ double next, time;
+
+ ti_next = -1;
+
+ if(All.OutputListOn)
+ {
+ for(i = 0; i < All.OutputListLength; i++)
+ {
+ time = All.OutputListTimes[i];
+
+ if(time >= All.TimeBegin && time <= All.TimeMax)
+ {
+ if(All.ComovingIntegrationOn)
+ ti = log(time / All.TimeBegin) / All.Timebase_interval;
+ else
+ ti = (time - All.TimeBegin) / All.Timebase_interval;
+
+ if(ti >= ti_curr)
+ {
+ if(ti_next == -1)
+ ti_next = ti;
+
+ if(ti_next > ti)
+ ti_next = ti;
+ }
+ }
+ }
+ }
+ else
+ {
+ if(All.ComovingIntegrationOn)
+ {
+ if(All.TimeBetSnapshot <= 1.0)
+ {
+ printf("TimeBetSnapshot > 1.0 required for your simulation.\n");
+ endrun(13123);
+ }
+ }
+ else
+ {
+ if(All.TimeBetSnapshot <= 0.0)
+ {
+ printf("TimeBetSnapshot > 0.0 required for your simulation.\n");
+ endrun(13123);
+ }
+ }
+
+ time = All.TimeOfFirstSnapshot;
+
+ iter = 0;
+
+ while(time < All.TimeBegin)
+ {
+ if(All.ComovingIntegrationOn)
+ time *= All.TimeBetSnapshot;
+ else
+ time += All.TimeBetSnapshot;
+
+ iter++;
+
+ if(iter > 1000000)
+ {
+ printf("Can't determine next output time.\n");
+ endrun(110);
+ }
+ }
+
+ while(time <= All.TimeMax)
+ {
+ if(All.ComovingIntegrationOn)
+ ti = log(time / All.TimeBegin) / All.Timebase_interval;
+ else
+ ti = (time - All.TimeBegin) / All.Timebase_interval;
+
+ if(ti >= ti_curr)
+ {
+ ti_next = ti;
+ break;
+ }
+
+ if(All.ComovingIntegrationOn)
+ time *= All.TimeBetSnapshot;
+ else
+ time += All.TimeBetSnapshot;
+
+ iter++;
+
+ if(iter > 1000000)
+ {
+ printf("Can't determine next output time.\n");
+ endrun(111);
+ }
+ }
+ }
+
+ if(ti_next == -1)
+ {
+ ti_next = 2 * TIMEBASE; /* this will prevent any further output */
+
+ if(ThisTask == 0)
+ printf("\nThere is no valid time for a further snapshot file.\n");
+ }
+ else
+ {
+ if(All.ComovingIntegrationOn)
+ next = All.TimeBegin * exp(ti_next * All.Timebase_interval);
+ else
+ next = All.TimeBegin + ti_next * All.Timebase_interval;
+
+ if(ThisTask == 0)
+ printf("\nSetting next time for snapshot file to Time_next= %g\n\n", next);
+ }
+
+ return ti_next;
+}
+
+
+
+
+/*! This routine writes one line for every timestep to two log-files. In
+ * FdInfo, we just list the timesteps that have been done, while in FdCPU the
+ * cumulative cpu-time consumption in various parts of the code is stored.
+ */
+void every_timestep_stuff(void)
+{
+ double z;
+
+#ifdef DETAILED_CPU
+ double tstart,tend;
+ tstart = second();
+#endif
+
+
+ if(ThisTask == 0)
+ {
+ if(All.ComovingIntegrationOn)
+ {
+ z = 1.0 / (All.Time) - 1;
+ fprintf(FdInfo, "\nBegin Step %d, Time: %g, Redshift: %g, Systemstep: %g, Dloga: %g\n",
+ All.NumCurrentTiStep, All.Time, z, All.TimeStep,
+ log(All.Time) - log(All.Time - All.TimeStep));
+ printf("\nBegin Step %d, Time: %g, Redshift: %g, Systemstep: %g, Dloga: %g\n", All.NumCurrentTiStep,
+ All.Time, z, All.TimeStep, log(All.Time) - log(All.Time - All.TimeStep));
+ fflush(FdInfo);
+ }
+ else
+ {
+ fprintf(FdInfo, "\nBegin Step %d, Time: %g, Systemstep: %g\n", All.NumCurrentTiStep, All.Time,
+ All.TimeStep);
+ printf("\nBegin Step %d, Time: %g, Systemstep: %g\n", All.NumCurrentTiStep, All.Time, All.TimeStep);
+ fflush(FdInfo);
+ }
+
+ printf("-------------------------------------------------------------\n");
+ fflush(stdout);
+
+#ifdef ADVANCEDCPUSTATISTICS
+ fprintf(FdCPU, "%d ", All.NumCurrentTiStep);
+ fprintf(FdCPU, "%g ", All.Time);
+ fprintf(FdCPU, "%d ", NTask);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Total);
+#ifdef DETAILED_CPU
+ fprintf(FdCPU,"%10.2f ",All.CPU_Leapfrog);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Physics);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Residual);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Accel);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Begrun);
+#endif
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Hydro);
+#ifdef COOLING
+ fprintf(FdCPU,"%10.2f ",All.CPU_Cooling);
+#endif
+#ifdef SFR
+ fprintf(FdCPU,"%10.2f ",All.CPU_StarFormation);
+#endif
+#ifdef CHIMIE
+ fprintf(FdCPU,"%10.2f ",All.CPU_Chimie);
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdCPU,"%10.2f ",All.CPU_Sticky);
+#endif
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Potential);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Predict);
+ fprintf(FdCPU,"%10.2f ",All.CPU_TimeLine);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Snapshot);
+ fprintf(FdCPU,"%10.2f ",All.CPU_TreeWalk);
+ fprintf(FdCPU,"%10.2f ",All.CPU_TreeConstruction);
+ fprintf(FdCPU,"%10.2f ",All.CPU_CommSum);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Imbalance);
+ fprintf(FdCPU,"%10.2f ",All.CPU_HydCompWalk);
+ fprintf(FdCPU,"%10.2f ",All.CPU_HydCommSumm);
+ fprintf(FdCPU,"%10.2f ",All.CPU_HydImbalance);
+ fprintf(FdCPU,"%10.2f ",All.CPU_EnsureNgb);
+ fprintf(FdCPU,"%10.2f ",All.CPU_PM);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Peano);
+#ifdef DETAILED_CPU_DOMAIN
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_findExtend);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_determineTopTree);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_sumCost);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_findSplit);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_shiftSplit);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_countToGo);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Domain_exchange);
+#endif
+#ifdef DETAILED_CPU_GRAVITY
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_TreeWalk1);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_TreeWalk2);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_CommSum1);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_CommSum2);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_Imbalance1);
+ fprintf(FdCPU,"%10.2f ",All.CPU_Gravity_Imbalance2);
+#endif
+ fprintf(FdCPU,"\n");
+ fflush(FdCPU);
+#else
+ fprintf(FdCPU, "Step %d, Time: %g, CPUs: %d\n", All.NumCurrentTiStep, All.Time, NTask);
+
+ fprintf(FdCPU,
+ "%10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f %10.2f\n",
+ All.CPU_Total, All.CPU_Gravity, All.CPU_Hydro, All.CPU_Domain, All.CPU_Potential,
+ All.CPU_Predict, All.CPU_TimeLine, All.CPU_Snapshot, All.CPU_TreeWalk, All.CPU_TreeConstruction,
+ All.CPU_CommSum, All.CPU_Imbalance, All.CPU_HydCompWalk, All.CPU_HydCommSumm,
+ All.CPU_HydImbalance, All.CPU_EnsureNgb, All.CPU_PM, All.CPU_Peano);
+ fflush(FdCPU);
+#endif
+
+ }
+
+
+ set_random_numbers();
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Residual += timediff(tstart, tend);
+#endif
+
+
+}
+
+/*! This routine first calls a computation of various global quantities of the
+ * particle distribution, and then writes some statistics about the energies
+ * in the various particle components to the file FdEnergy.
+ */
+void energy_statistics(void)
+{
+ compute_global_quantities_of_system();
+
+ if(ThisTask == 0)
+ {
+ fprintf(FdEnergy,
+ "%g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g %g\n",
+ All.Time, SysState.EnergyInt, SysState.EnergyPot, SysState.EnergyKin, SysState.EnergyIntComp[0],
+ SysState.EnergyPotComp[0], SysState.EnergyKinComp[0], SysState.EnergyIntComp[1],
+ SysState.EnergyPotComp[1], SysState.EnergyKinComp[1], SysState.EnergyIntComp[2],
+ SysState.EnergyPotComp[2], SysState.EnergyKinComp[2], SysState.EnergyIntComp[3],
+ SysState.EnergyPotComp[3], SysState.EnergyKinComp[3], SysState.EnergyIntComp[4],
+ SysState.EnergyPotComp[4], SysState.EnergyKinComp[4], SysState.EnergyIntComp[5],
+ SysState.EnergyPotComp[5], SysState.EnergyKinComp[5], SysState.MassComp[0],
+ SysState.MassComp[1], SysState.MassComp[2], SysState.MassComp[3], SysState.MassComp[4],
+ SysState.MassComp[5]);
+ fflush(FdEnergy);
+ }
+}
+
+
+/*! This routine first calls a computation of various global quantities of the
+ * particle distribution, and then writes some statistics about the energies
+ * in the various particle components to the file FdEnergy.
+ */
+#ifdef ADVANCEDSTATISTICS
+void advanced_energy_statistics(void)
+{
+ int i;
+
+#ifdef DETAILED_CPU
+ double tstart,tend;
+ tstart = second();
+#endif
+
+
+ compute_global_quantities_of_system();
+
+ if(ThisTask == 0)
+ {
+
+ /**************/
+ /* energy */
+ /**************/
+
+ /* time */
+ fprintf(FdEnergy,"%g ",All.Time);
+
+ /* total */
+ fprintf(FdEnergy,"%g %g %g ",SysState.EnergyInt, SysState.EnergyPot, SysState.EnergyKin);
+#ifdef COOLING
+ fprintf(FdEnergy,"%g ",SysState.EnergyRadSph);
+#endif
+#ifdef AGN_HEATING
+ fprintf(FdEnergy,"%g ",SysState.EnergyAGNHeat);
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdEnergy,"%g ",SysState.EnergyRadSticky);
+#endif
+#ifdef FEEDBACK_WIND
+ fprintf(FdEnergy,"%g ",SysState.EnergyFeedbackWind);
+#endif
+#ifdef BUBBLES
+ fprintf(FdEnergy,"%g ",SysState.EnergyBubbles);
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ fprintf(FdEnergy,"%g ",SysState.EnergyThermalFeedback);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ fprintf(FdEnergy,"%g ",SysState.EnergyKineticFeedback);
+#endif
+
+ /* comp */
+ for (i=0;i<6;i++)
+ {
+ fprintf(FdEnergy,"%g %g %g ",SysState.EnergyIntComp[i],SysState.EnergyPotComp[i], SysState.EnergyKinComp[i]);
+#ifdef COOLING
+ fprintf(FdEnergy,"%g ",SysState.EnergyRadSphComp[i]);
+#endif
+#ifdef AGN_HEATING
+ fprintf(FdEnergy,"%g ",SysState.EnergyAGNHeatComp[i]);
+#endif
+#ifdef MULTIPHASE
+ fprintf(FdEnergy,"%g ",SysState.EnergyRadStickyComp[i]);
+#endif
+#ifdef FEEDBACK_WIND
+ fprintf(FdEnergy,"%g ",SysState.EnergyFeedbackWindComp[i]);
+#endif
+#ifdef BUBBLES
+ fprintf(FdEnergy,"%g ",SysState.EnergyBubblesComp[i]);
+#endif
+#ifdef CHIMIE_THERMAL_FEEDBACK
+ fprintf(FdEnergy,"%g ",SysState.EnergyThermalFeedbackComp[i]);
+#endif
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ fprintf(FdEnergy,"%g ",SysState.EnergyKineticFeedbackComp[i]);
+#endif
+
+ }
+
+ /* mass */
+ for (i=0;i<6;i++)
+ {
+ fprintf(FdEnergy,"%g ",SysState.MassComp[i]);
+ }
+
+ /* return */
+ fprintf(FdEnergy,"\n");
+
+ fflush(FdEnergy);
+
+#ifdef SYSTEMSTATISTICS
+
+ /**************/
+ /* system */
+ /**************/
+
+
+ fprintf(FdSystem,"%g %g %g %g %g %g %g %g %g %g %g %g %g\n",
+ All.Time,
+ SysState.Momentum[0], SysState.Momentum[1], SysState.Momentum[2], SysState.Momentum[3],
+ SysState.AngMomentum[0], SysState.AngMomentum[1], SysState.AngMomentum[2], SysState.AngMomentum[3],
+ SysState.CenterOfMass[0], SysState.CenterOfMass[1], SysState.CenterOfMass[2], SysState.CenterOfMass[3]);
+
+ fflush(FdSystem);
+
+
+#endif
+
+ }
+
+#ifdef DETAILED_CPU
+ tend = second();
+ All.CPU_Residual += timediff(tstart, tend);
+#endif
+
+
+}
+
+#endif
+
diff --git a/starformation.c b/starformation.c
new file mode 100644
index 0000000..0840b24
--- /dev/null
+++ b/starformation.c
@@ -0,0 +1,960 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+#ifdef SFR
+#ifdef FEEDBACK_WIND
+
+void feedbackwind_compute_energy_kin(int mode)
+{
+ int i;
+ double DeltaEgyKin;
+ double Tot_DeltaEgyKin;
+
+ DeltaEgyKin = 0;
+ Tot_DeltaEgyKin = 0;
+
+ if (mode==1)
+ {
+ LocalSysState.EnergyKin1 = 0;
+ LocalSysState.EnergyKin2 = 0;
+ }
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+
+ if (mode==1)
+ LocalSysState.EnergyKin1 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ else
+ LocalSysState.EnergyKin2 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ }
+ }
+
+ if (mode==2)
+ {
+ DeltaEgyKin = LocalSysState.EnergyKin2 - LocalSysState.EnergyKin1;
+ MPI_Reduce(&DeltaEgyKin, &Tot_DeltaEgyKin, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ LocalSysState.EnergyFeedbackWind -= DeltaEgyKin;
+ }
+
+
+}
+
+#endif
+
+
+
+
+
+
+/*! \file starformation.c
+ * \brief Compute jet
+ *
+*/
+
+static double hubble_a,a3inv=1.;
+
+/*! Compute starformation
+ *
+ */
+void star_formation(void)
+{
+ int i,j;
+ int NumNewStars=0,firststar;
+ int *AllNumNewStars;
+ double T;
+ double p;
+ double dt=0;
+ double tstar=1.;
+ double MeanBaryonicDensity;
+ int flagST=1,flagToStar=0;
+ double mstar;
+ double mstars=0,Tot_mstars=0;
+ double mnewstars,Tot_mnewstars=0;
+ int Nnewstars,Tot_Nnewstars=0;
+ double star_formation_rate;
+ int *numlist;
+ double t0,t1;
+
+ struct particle_data Psave;
+
+ int k;
+
+ t0 = second();
+
+ if(ThisTask == 0)
+ {
+ printf("start star formation... \n");
+ fflush(stdout);
+ }
+
+ AllNumNewStars = malloc(sizeof(int) * NTask);
+
+
+#ifdef FEEDBACK_WIND
+ double phi,costh,sinth;
+#endif
+
+ if(All.ComovingIntegrationOn)
+ {
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ }
+ else
+ {
+ hubble_a = 1.;
+ a3inv = 1.;
+ }
+
+ switch (All.StarFormationType)
+ {
+ case 0:
+ All.ThresholdDensity = All.StarFormationDensity;
+ break;
+
+ case 1:
+ MeanBaryonicDensity = All.OmegaBaryon * (3 * All.Hubble * All.Hubble / (8 * M_PI * All.G));
+ All.ThresholdDensity = dmax(All.StarFormationDensity/a3inv,200*MeanBaryonicDensity);
+ break;
+
+ case 2:
+ All.ThresholdDensity = All.StarFormationDensity;
+ All.StarFormationTime = 1./All.StarFormationCstar / pow(4*PI*All.G*All.StarFormationDensity,0.5);
+ break;
+
+ }
+
+ flagST=1;
+ mnewstars=0;
+ Nnewstars=0;
+#ifdef FEEDBACK_WIND
+ feedbackwind_compute_energy_kin(1);
+#endif
+
+
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+
+ /* only active particles */
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ if (SphP[i].WindTime < (All.Time-All.ChimieWindTime)) /* not a wind particle */
+ {
+#endif
+
+ /* if the particle is a false gas (new star) */
+ if(P[i].Type != 0)
+ continue;
+
+#ifdef FEEDBACK
+ /* if the particle is a SN */
+ if(SphP[i].EnergySN)
+ {
+
+ /* the particle is not allowed to becomes a stars, as it is already a SN */
+ flagST = 0;
+
+ SphP[i].EnergySNrem = SphP[i].EnergySNrem - SphP[i].EnergySN;
+
+ /* dt in time unit here, even for ComovingIntegrationOn */
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+
+ if (All.SupernovaTime==0)
+ SphP[i].EnergySN = SphP[i].EnergySNrem;
+ else
+ SphP[i].EnergySN = (P[i].Mass*All.SupernovaEgySpecPerMassUnit)/All.SupernovaTime*dt;
+
+
+ /* limit the feedback energy to the remaining energy */
+ if (SphP[i].EnergySN>SphP[i].EnergySNrem)
+ SphP[i].EnergySN = SphP[i].EnergySNrem;
+
+
+ if (SphP[i].EnergySNrem == 0) /* not enough energy, make a star */
+ {
+ //printf("Particle %d becomes a star\n",P[i].ID);
+ P[i].Type = ST;
+ RearrangeParticlesFlag=1;
+ }
+
+
+ }
+#endif
+
+ /* check if the particle may become a star */
+#ifdef SFR_NEG_DIV_ONLY
+ if ((SphP[i].DivVel < 0) && flagST)
+#endif
+ {
+
+
+
+
+
+#ifdef MULTIPHASE
+ if (SphP[i].Phase == GAS_STICKY)
+ {
+#endif
+
+ if (SphP[i].Density > All.ThresholdDensity)
+ {
+
+ /* check temperature */
+#ifdef ISOTHERM_EQS
+ T = All.InitGasTemp;
+#else
+#ifdef MULTIPHASE
+ T = GAMMA_MINUS1*All.mumh/All.Boltzmann * SphP[i].EntropyPred;
+#else
+ T = All.mumh/All.Boltzmann * SphP[i].EntropyPred * pow(SphP[i].Density*a3inv,GAMMA_MINUS1);
+#endif
+#endif
+
+ if (T < All.StarFormationTemperature)
+ {
+ /* dt in time unit here, even for ComovingIntegrationOn */
+ /* dt has units of Gyr/h also tstar */
+ dt = (All.Ti_Current - P[i].Ti_begstep) * All.Timebase_interval / hubble_a;
+ tstar = All.StarFormationTime / pow(SphP[i].Density*a3inv/All.StarFormationDensity,0.5);
+
+
+
+ /* check the mass of the particle */
+
+ flagToStar = 0;
+
+ if (All.StarFormationNStarsFromGas==1)
+ {
+ /* transform the gas particle into a star particle */
+ mstar = P[i].Mass;
+ flagToStar=1;
+ }
+ else
+ {
+ if (P[i].Mass<=(1.0+All.StarFormationMgMsFraction)*All.StarFormationStarMass)
+ /* transform the gas particle into a star particle */
+ {
+ mstar = P[i].Mass;
+ flagToStar=1;
+ }
+ else
+ /* extract a star particle from the gas particle */
+ {
+ mstar = All.StarFormationStarMass;
+ flagToStar=0;
+ }
+ }
+
+
+
+
+ /* compute probability of forming star */
+ p = (P[i].Mass/mstar)*(1. - exp(-dt/tstar) );
+
+ if (get_StarFormation_random_number(P[i].ID) < p)
+ {
+
+#ifdef FEEDBACK
+ SphP[i].EnergySNrem = P[i].Mass*All.SupernovaEgySpecPerMassUnit;
+
+ if (All.SupernovaTime==0)
+ SphP[i].EnergySN = SphP[i].EnergySNrem;
+ else
+ SphP[i].EnergySN = (P[i].Mass*All.SupernovaEgySpecPerMassUnit)/All.SupernovaTime*dt;
+
+
+ /* limit the feedback energy to the remaining energy */
+ if (SphP[i].EnergySN>SphP[i].EnergySNrem)
+ SphP[i].EnergySN = SphP[i].EnergySNrem;
+
+ if (SphP[i].EnergySNrem==0)
+ {
+ P[i].Type = ST;
+ RearrangeParticlesFlag=1;
+ }
+#else
+
+ if (flagToStar) /* turns the stellar particule into star particle */
+ {
+#ifdef STELLAR_PROP
+ if (N_stars+1 > All.MaxPartStars)
+ {
+ printf("No memory left for new star particle : N_stars+1=%d MaxPartStars=%d\n\n",N_stars+1,All.MaxPartStars);
+ endrun(999008);
+ }
+#else
+ if (N_stars+1 > All.MaxPart-N_gas)
+ {
+ printf("No memory left for new star particle : N_stars+1=%d All.MaxPart-N_gas=%d\n\n",N_stars+1,All.MaxPart-N_gas);
+ endrun(999009);
+ }
+
+#endif
+
+ P[i].Type = ST;
+ RearrangeParticlesFlag=1;
+
+ //printf("(%d) move gas particle to star (i=%d,id=%d)\n",ThisTask,i,P[i].ID);
+
+ /* count mass */
+ mnewstars+=P[i].Mass;
+ Nnewstars+=1;
+
+
+ /* gather energy */
+#ifdef FEEDBACK
+ LocalSysState.StarEnergyFeedback += SphP[i].EgySpecFeedback * P[i].Mass;
+#endif
+
+
+
+
+ }
+ else /* create a new star particle */
+ {
+
+#ifdef STELLAR_PROP
+ if (N_stars+1 > All.MaxPartStars)
+ {
+ printf("No memory left for new star particle : N_stars+1=%d MaxPartStars=%d\n\n",N_stars+1,All.MaxPartStars);
+ endrun(999000);
+ }
+#else
+ if (N_stars+1 > All.MaxPart-N_gas)
+ {
+ printf("No memory left for new star particle : N_stars+1=%d All.MaxPart-N_gas=%d\n\n",N_stars+1,All.MaxPart-N_gas);
+ endrun(999001);
+ }
+#endif
+
+ if (NumPart+1 > All.MaxPart)
+ {
+ printf("No memory left for new particle : NumPart=%d MaxPart=%d",NumPart,All.MaxPart);
+ endrun(999002);
+ }
+
+ if (P[i].Mass <= mstar)
+ {
+ printf("Mass too small %g %g !",P[i].Mass,mstar);
+ endrun(999003);
+ }
+
+
+ /* the new particle get all properties of the SPH particle, except the mass */
+ /* the type is defined further */
+
+ j = NumPart+NumNewStars;
+
+ P[j] = P[i];
+ P[j].Mass = mstar;
+ P[j].Type = ST;
+
+ /* count mass */
+ mnewstars+=mstar;
+ Nnewstars+=1;
+
+
+ /* this is needed to kick parent nodes dynamically in timestep.c */
+ /* could be avoided if All.NumForcesSinceLastDomainDecomp > All.TotNumPart * All.TreeDomainUpdateFrequency*/
+ Father[j] = Father[i];
+
+#ifdef STELLAR_PROP
+ /* index to Stp */
+ P[j].StPIdx = N_stars+NumNewStars;
+
+ /* record info at the end of StP */
+
+ /* record time */
+ StP[P[j].StPIdx].FormationTime = All.Time;
+ /* record time */
+ StP[P[j].StPIdx].IDProj = P[i].ID;
+ /* record hsml */
+ StP[P[j].StPIdx].Hsml = SphP[i].Hsml;
+
+#ifdef CHIMIE
+ /* record initial mass */
+ StP[P[j].StPIdx].InitialMass = P[j].Mass;
+ /* record density */
+ StP[P[j].StPIdx].Density = SphP[i].Density;
+ /* record metalicity */
+ for(k = 0; k < NELEMENTS; k++)
+ StP[P[j].StPIdx].Metal[k] = SphP[i].Metal[k];
+#endif
+ /* index to P */
+ StP[P[j].StPIdx].PIdx = j;
+
+
+#endif
+
+
+ /* change proj properties */
+ P[i].Mass = P[i].Mass-mstar;
+
+
+
+
+ /* gather energy */
+#ifdef FEEDBACK
+ LocalSysState.StarEnergyFeedback += SphP[i].EgySpecFeedback * P[j].Mass;
+#endif
+
+ /* here, we should add the self force, but it is negligible */
+
+
+ NumNewStars++;
+
+
+ }
+
+#endif
+ }
+#ifdef FEEDBACK_WIND
+ else
+ {
+ tstar = tstar/All.SupernovaWindParameter;
+ /* here, we devide by p, because there is (1-p) prob. to be here */
+ p = (P[i].Mass/mstar)*(1. - exp(-dt/tstar))/(1-p);
+
+ if (get_FeedbackWind_random_number(P[i].ID) < p)
+ {
+ phi = PI*2.*get_FeedbackWind_random_number(P[i].ID+1);
+ costh = 1.-2.*get_FeedbackWind_random_number(P[i].ID+2);
+ sinth = sqrt(1.-costh*costh);
+
+ SphP[i].FeedbackWindVel[0] = All.SupernovaWindSpeed *sinth*cos(phi);
+ SphP[i].FeedbackWindVel[1] = All.SupernovaWindSpeed *sinth*sin(phi);
+ SphP[i].FeedbackWindVel[2] = All.SupernovaWindSpeed *costh;
+
+
+ //printf("(%d) %d vx vy vz %g %g %g %g\n",ThisTask,P[i].ID,P[i].Vel[0],P[i].Vel[1],P[i].Vel[2],sqrt( P[i].Vel[0]*P[i].Vel[0] + P[i].Vel[1]*P[i].Vel[1] + P[i].Vel[2]*P[i].Vel[2] ));
+ //printf("(%d) SupernovaWindSpeed %g dv=%g\n",ThisTask,All.SupernovaWindSpeed,sqrt( SphP[i].FeedbackWindVel[0]*SphP[i].FeedbackWindVel[0] + SphP[i].FeedbackWindVel[1]*SphP[i].FeedbackWindVel[1] + SphP[i].FeedbackWindVel[2]*SphP[i].FeedbackWindVel[2] ));
+
+ /* new velocities */
+ for(k = 0; k < 3; k++)
+ {
+ SphP[i].VelPred[k] += SphP[i].FeedbackWindVel[k];
+ P[i].Vel[k] += SphP[i].FeedbackWindVel[k];
+ }
+
+ //printf("(%d) %d vx vy vz %g %g %g %g\n",ThisTask,P[i].ID,P[i].Vel[0],P[i].Vel[1],P[i].Vel[2],sqrt( P[i].Vel[0]*P[i].Vel[0] + P[i].Vel[1]*P[i].Vel[1] + P[i].Vel[2]*P[i].Vel[2] ));
+
+
+ }
+
+
+
+ }
+#endif
+
+
+
+ }
+ }
+
+#ifdef MULTIPHASE
+ }
+#endif
+
+
+
+
+ }
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ }
+#endif
+
+
+ }
+
+ }
+
+
+
+ if (All.StarFormationNStarsFromGas != 1)
+ {
+
+
+ /* get NumNewStars from all other procs */
+ MPI_Allgather(&NumNewStars, 1, MPI_INT, AllNumNewStars, 1, MPI_INT, MPI_COMM_WORLD);
+
+ /* find the id of the fist local star */
+ firststar = All.TotNumPart;
+
+ for (i=0;i<ThisTask;i++)
+ firststar += AllNumNewStars[i];
+
+ /* now, set ids */
+ for (i=0;i< NumNewStars;i++)
+ P[NumPart+i].ID = firststar + i;
+
+
+ /* move the new stars to the right place
+ in both cases, the relative position of the new stars is preseved,
+ so, there is no need to rearrange StP.
+ */
+ if (NumNewStars < NumPart-N_gas-N_stars) /* move stars inwards */
+ {
+ for(i=0;i<NumNewStars;i++)
+ {
+ Psave = P[N_gas+N_stars+i];
+ P[N_gas+N_stars+i] = P[NumPart+i];
+ P[NumPart+i]=Psave;
+#ifdef STELLAR_PROP
+ /* set index for StP */
+ StP[P[N_gas+N_stars+i].StPIdx].PIdx = N_gas+N_stars+i;
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[N_gas+N_stars+i].StPIdx].ID = P[N_gas+N_stars+i].ID;
+#endif
+ //printf("(%d) new star (a) i=%d id=%d (StPi=%d PIdx=%d)\n",ThisTask,N_gas+N_stars+i,P[N_gas+N_stars+i].ID,P[N_gas+N_stars+i].StPIdx,StP[P[N_gas+N_stars+i].StPIdx].PIdx);
+#endif
+
+ }
+ }
+ else /* move other particles outwards */
+ {
+
+ for(i=N_gas+N_stars;i<NumPart;i++)
+ {
+ Psave = P[i];
+ P[i] = P[i+NumNewStars];
+ P[i+NumNewStars] = Psave;
+#ifdef STELLAR_PROP
+ /* set new index for StP */
+ StP[P[i].StPIdx].PIdx = i;
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[i].StPIdx].ID = P[i].ID;
+#endif
+
+#endif
+ //printf("(%d) new star (b) i=%d id=%d\n",ThisTask,N_gas+N_stars+i,P[N_gas+N_stars+i].ID);
+ }
+
+ }
+
+
+ /* set new NumPart */
+ NumPart = NumPart+NumNewStars;
+ N_stars = N_stars+NumNewStars;
+
+
+
+ /*MPI_Allreduce(&NumPart, &All.TotNumPart, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);*/
+ numlist = malloc(NTask * sizeof(int) * NTask);
+
+ MPI_Allgather(&NumPart, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotNumPart = 0; i < NTask; i++)
+ All.TotNumPart += numlist[i];
+
+ MPI_Allgather(&N_stars, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_stars = 0; i < NTask; i++)
+ All.TotN_stars += numlist[i];
+
+ free(numlist);
+
+
+ }
+
+
+
+ /* compute star mass */
+ mstars=0;
+ for (i=0;i< NumPart;i++)
+ {
+ if (P[i].Type==ST)
+ mstars+= P[i].Mass;
+ }
+
+#ifdef FEEDBACK_WIND
+ feedbackwind_compute_energy_kin(2);
+#endif
+
+
+
+ /* share results */
+ MPI_Allreduce(&mstars, &Tot_mstars, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&mnewstars, &Tot_mnewstars, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&Nnewstars, &Tot_Nnewstars, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ if (All.TimeStep>0)
+ star_formation_rate = Tot_mnewstars/All.TimeStep;
+ else
+ star_formation_rate = 0.;
+
+
+ if (ThisTask==0)
+ {
+ //fprintf(FdSfr, "Step %d, Time: %g, NewStarsPart: %g \n", All.NumCurrentTiStep, All.Time, Tot_mstars);
+ fprintf(FdSfr, "%15g %15g %15g %15g %15g\n", All.Time,All.TimeStep,Tot_mstars,Tot_mnewstars,star_formation_rate);
+ fflush(FdSfr);
+ }
+
+
+
+ //printf("(%d) N_gas = %d N_stars = %d NumPart = %d\n",ThisTask,N_gas,N_stars,NumPart);
+ //fflush(stdout);
+
+ if(ThisTask == 0)
+ {
+ printf("Total number of new star particles : %d \n",(int) Tot_Nnewstars);
+ printf("Total number of star particles : %d%09d\n",(int) (All.TotN_stars / 1000000000), (int) (All.TotN_stars % 1000000000));
+ fflush(stdout);
+ }
+
+
+
+#ifdef CHECK_BLOCK_ORDER
+ int typenow;
+
+ rearrange_particle_sequence();
+
+ typenow = 0;
+ for(i = 0; i < NumPart; i++)
+ {
+
+ if ((P[i].Type<typenow)&&(typenow<=ST))
+ {
+ printf("\nBlock order error\n");
+ printf("(%d) i=%d id=%d Type=%d typenow=%d\n\n",ThisTask,i,P[i].ID,P[i].Type,typenow);
+ endrun(999004);
+ }
+
+ typenow = P[i].Type;
+ }
+#endif
+
+
+
+#ifdef CHECK_ID_CORRESPONDENCE
+
+ rearrange_particle_sequence();
+
+ for(i = N_gas; i < N_gas+N_stars; i++)
+ {
+ if( StP[P[i].StPIdx].PIdx != i )
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in starformation) N_stars=%d N_gas=%d i=%d id=%d P[i].StPIdx=%d StP[P[i].StPIdx].PIdx=%d\n\n",ThisTask,N_stars,N_gas,i,P[i].ID,P[i].StPIdx,StP[P[i].StPIdx].PIdx);
+ endrun(999005);
+ }
+
+
+ if(StP[P[i].StPIdx].ID != P[i].ID)
+ {
+ printf("\nP/StP correspondance error\n");
+ printf("(%d) (in starformation) N_gas=%d N_stars=%d i=%d Type=%d P.Id=%d P[i].StPIdx=%d StP[P[i].StPIdx].ID=%d \n\n",ThisTask,N_gas,N_stars,i,P[i].Type,P[i].ID, P[i].StPIdx, StP[P[i].StPIdx].ID);
+ endrun(999006);
+ }
+ }
+
+#endif
+
+
+
+ if(ThisTask == 0)
+ {
+ printf("star formation done. \n");
+ fflush(stdout);
+ }
+
+
+ t1 = second();
+ All.CPU_StarFormation += timediff(t0, t1);
+
+}
+
+
+/*
+ * This routine rearrange particle sequence in order to group particles of same type
+ */
+void rearrange_particle_sequence(void)
+{
+ int i,j;
+ struct particle_data Psave;
+ struct sph_particle_data Psave_sph;
+ int nstars=0;
+ int *numlist;
+ int sumflag;
+#ifdef CHIMIE
+ int k;
+#endif
+
+
+ MPI_Allreduce(&RearrangeParticlesFlag, &sumflag, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+
+
+ if(RearrangeParticlesFlag)
+ {
+
+
+ if(ThisTask == 0)
+ {
+ printf("Start rearrange particle sequence...\n");
+ fflush(stdout);
+ }
+
+
+ /* loop over all gasous particles */
+ for(i = 0; i < N_gas; i++)
+ {
+
+ if (P[i].Type == ST)
+ {
+
+
+ /* find the first non star (flag) particle */
+ for(j=N_gas-1;j>0;j--)
+ {
+ if (P[j].Type==0) /* we have a gasous particle */
+ break;
+
+ if (j==i) /* the particule is the last gaseous particle */
+ break; /* the particle will be inverted with itself */
+
+
+ if (P[j].Type==ST) /* the particule is selected to becomes a star, turn it to star */
+ {
+#ifdef STELLAR_PROP
+ P[j].StPIdx = N_stars;
+
+ StP[P[j].StPIdx].FormationTime = All.Time;
+ StP[P[j].StPIdx].IDProj = P[j].ID;
+ StP[P[j].StPIdx].Hsml = SphP[j].Hsml;
+#ifdef CHIMIE
+ /* record initial mass */
+ StP[P[j].StPIdx].InitialMass = P[j].Mass;
+ /* record density */
+ StP[P[j].StPIdx].Density = SphP[j].Density;
+ /* record metalicity */
+ for(k = 0; k < NELEMENTS; k++)
+ StP[P[j].StPIdx].Metal[k] = SphP[j].Metal[k];
+#endif
+
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[j].StPIdx].ID = P[j].ID;
+#endif
+
+ StP[P[j].StPIdx].PIdx = j;
+ //printf("(%d) move to stars (b) i=%d -> j=%d id=%d Stpi=%d (N_gas=%d N_stars=%d)\n",ThisTask,j,j,P[j].ID,P[j].StPIdx,N_gas,N_stars);
+#endif
+
+ nstars++;
+ N_gas--;
+ N_stars++;
+ continue;
+ }
+
+
+ }
+
+ //printf("(%d) --> i=%d id=%d Type=%d (N_gas=%d)\n",ThisTask,i,P[i].ID,P[i].Type,N_gas);
+ //printf("(%d) <-- i=%d id=%d Type=%d\n",ThisTask,j,P[j].ID,P[j].Type);
+
+ /* move the particle */
+ Psave = P[i]; /* copy the particle */
+ Psave_sph = SphP[i]; /* copy the particle */
+
+ P[i] = P[j]; /* replace by the last gaseous one */
+ SphP[i] = SphP[j]; /* replace by the last gaseous one */
+ P[j] = Psave;
+
+#ifdef STELLAR_PROP
+ P[j].StPIdx = N_stars;
+ StP[P[j].StPIdx].FormationTime = All.Time;
+ StP[P[j].StPIdx].IDProj = P[j].ID;
+ StP[P[j].StPIdx].Hsml = Psave_sph.Hsml;
+#ifdef CHIMIE
+ /* record initial mass */
+ StP[P[j].StPIdx].InitialMass = P[j].Mass;
+ /* record density */
+ StP[P[j].StPIdx].Density = Psave_sph.Density;
+ /* record metalicity */
+ for(k = 0; k < NELEMENTS; k++)
+ StP[P[j].StPIdx].Metal[k] = Psave_sph.Metal[k];
+#endif
+
+
+#ifdef CHECK_ID_CORRESPONDENCE
+ StP[P[j].StPIdx].ID = P[j].ID;
+#endif
+ StP[P[j].StPIdx].PIdx = j;
+ //printf("(%d) move to stars (a) i=%d -> j=%d id=%d Stpi=%d (N_gas=%d N_stars=%d)\n",ThisTask,i,j,P[j].ID,P[j].StPIdx,N_gas,N_stars);
+#endif
+
+
+
+ nstars++;
+ N_gas--;
+ N_stars++;
+ }
+ }
+ RearrangeParticlesFlag=0;
+
+ //if(ThisTask == 0)
+ // {
+ // printf("%d new star(s)\n",nstars);
+ // printf("Start rearrange particle sequence done.\n");
+ // fflush(stdout);
+ // }
+
+
+ }
+
+
+
+ if (sumflag) /* do this only if at least one Task has rearrange particles */
+ {
+
+
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+
+ /* update all.TotN_gas */
+ MPI_Allgather(&N_gas, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_gas = 0; i < NTask; i++)
+ All.TotN_gas += numlist[i];
+
+ /* update all.TotN_stars */
+ MPI_Allgather(&N_stars, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, All.TotN_stars = 0; i < NTask; i++)
+ All.TotN_stars += numlist[i];
+
+ free(numlist);
+
+
+ }
+
+
+
+
+
+}
+
+
+
+
+
+
+void sfr_compute_energy_int(int mode)
+{
+ int i;
+ double DeltaEgyInt;
+ double Tot_DeltaEgyInt;
+ double egyspec;
+
+
+ DeltaEgyInt = 0;
+ Tot_DeltaEgyInt = 0;
+
+ if (mode==1)
+ {
+ LocalSysState.EnergyInt1 = 0;
+ LocalSysState.EnergyInt2 = 0;
+ }
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+
+#ifdef MULTIPHASE
+ if (SphP[i].Phase== GAS_SPH)
+ egyspec = SphP[i].Entropy / (GAMMA_MINUS1) * pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+ else
+ egyspec = SphP[i].Entropy;
+#else
+ egyspec = SphP[i].Entropy / (GAMMA_MINUS1) * pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+
+#endif
+
+ if (mode==1)
+ LocalSysState.EnergyInt1 += P[i].Mass * egyspec;
+ else
+ LocalSysState.EnergyInt2 += P[i].Mass * egyspec;
+ }
+ }
+
+ if (mode==2)
+ {
+ DeltaEgyInt = LocalSysState.EnergyInt2 - LocalSysState.EnergyInt1;
+ MPI_Reduce(&DeltaEgyInt, &Tot_DeltaEgyInt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ LocalSysState.StarEnergyInt -= DeltaEgyInt;
+ }
+
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+void sfr_check_number_of_stars(int mode)
+{
+
+ int i;
+ int nstars;
+ int npotstars;
+
+
+ for(i = 0, nstars = 0, npotstars = 0; i < NumPart; i++)
+ {
+
+ if (P[i].Type==ST)
+ if (i<N_gas)
+ {
+ npotstars++;
+ }
+ else
+ {
+ nstars++;
+ }
+
+ }
+
+
+ printf("(%d) N_stars=%d npotstars+nstars=%d npotstars=%d nstars=%d\n",ThisTask,N_stars,npotstars+nstars,npotstars,nstars);
+ if (nstars != N_stars)
+ endrun(987987);
+
+}
+
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/stars_density.c b/stars_density.c
new file mode 100644
index 0000000..c7cd06d
--- /dev/null
+++ b/stars_density.c
@@ -0,0 +1,654 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file density.c
+ * \brief SPH density computation and smoothing length determination
+ *
+ * This file contains the "first SPH loop", where the SPH densities and
+ * some auxiliary quantities are computed. If the number of neighbours
+ * obtained falls outside the target range, the correct smoothing
+ * length is determined iteratively, if needed.
+ */
+
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+#ifdef CHIMIE
+/*! This function computes the local density for each active SPH particle,
+ * the number of neighbours in the current smoothing radius, and the
+ * divergence and curl of the velocity field. The pressure is updated as
+ * well. If a particle with its smoothing region is fully inside the
+ * local domain, it is not exported to the other processors. The function
+ * also detects particles that have a number of neighbours outside the
+ * allowed tolerance range. For these particles, the smoothing length is
+ * adjusted accordingly, and the density computation is executed again.
+ * Note that the smoothing length is not allowed to fall below the lower
+ * bound set by MinGasHsml.
+ */
+void stars_density(void)
+{
+ long long ntot, ntotleft;
+ int *noffset, *nbuffer, *nsend, *nsend_local, *numlist, *ndonelist;
+ int i, j, n,m, ndone, npleft, maxfill, source, iter = 0;
+ int level, ngrp, sendTask, recvTask, place, nexport;
+ double tstart, tend, tstart_ngb = 0, tend_ngb = 0;
+ double sumt, sumcomm, timengb, sumtimengb;
+ double timecomp = 0, timeimbalance = 0, timecommsumm = 0, sumimbalance;
+ MPI_Status status;
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+ if (ThisTask==0)
+ printf("Start star density computation.\n");
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+
+ /* `NumStUpdate' gives the number of particles on this processor that want a chimie computation */
+ for(n = N_gas, NumStUpdate = 0; n < N_gas+N_stars; n++)
+ {
+ m = P[n].StPIdx;
+ StP[m].Left = StP[m].Right = 0;
+#ifdef AVOIDNUMNGBPROBLEM
+ StP[m].OldNumNgb = -1;
+#endif
+ if(P[n].Ti_endstep == All.Ti_Current)
+ NumStUpdate++;
+ }
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumStUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+ /* we will repeat the whole thing for those particles where we didn't
+ * find enough neighbours
+ */
+ do
+ {
+ i = N_gas; /* beginn with the first star */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+ while(ntotleft > 0)
+ {
+
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < N_gas+N_stars && nexport < All.BunchSizeDensity - NTask; i++)
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+
+ if(P[i].Type != ST)
+ {
+ printf("P[i].Type != ST, we better stop.\n");
+ printf("N_gas=%d (type=%d) i=%d (type=%d)\n",N_gas,P[N_gas].Type,i,P[i].Type);
+ printf("Please, check that you do not use PEANOHILBERT\n");
+ endrun(666001);
+ }
+
+ ndone++;
+
+ m = P[i].StPIdx;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ stars_density_evaluate(i, 0);
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+ StarsDensDataIn[nexport].Pos[0] = P[i].Pos[0];
+ StarsDensDataIn[nexport].Pos[1] = P[i].Pos[1];
+ StarsDensDataIn[nexport].Pos[2] = P[i].Pos[2];
+ StarsDensDataIn[nexport].Hsml = StP[m].Hsml;
+ StarsDensDataIn[nexport].Index = i;
+ StarsDensDataIn[nexport].Task = j;
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+
+ }
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ qsort(StarsDensDataIn, nexport, sizeof(struct starsdensdata_in), stars_dens_compare_key);
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+
+ /* now do the particles that need to be exported */
+
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeDensity)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&StarsDensDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct starsdensdata_in), MPI_BYTE,
+ recvTask, TAG_DENS_A,
+ &StarsDensDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct starsdensdata_in),
+ MPI_BYTE, recvTask, TAG_DENS_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ stars_density_evaluate(j, 1);
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ /* do a block to explicitly measure imbalance */
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeDensity)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&StarsDensDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct starsdensdata_out),
+ MPI_BYTE, recvTask, TAG_DENS_B,
+ &StarsDensDataPartialResult[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct starsdensdata_out),
+ MPI_BYTE, recvTask, TAG_DENS_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ source = j + noffset[recvTask];
+ place = P[StarsDensDataIn[source].Index].StPIdx;
+
+ StP[place].NumNgb += StarsDensDataPartialResult[source].Ngb;
+ StP[place].Density += StarsDensDataPartialResult[source].Rho;
+ StP[place].Volume += StarsDensDataPartialResult[source].Volume;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ StP[place].NgbMass += StarsDensDataPartialResult[source].NgbMass;
+#endif
+ StP[place].DhsmlDensityFactor += StarsDensDataPartialResult[source].DhsmlDensity;
+ }
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+
+
+ /* do final operations on results */
+ tstart = second();
+ for(i = N_gas, npleft = 0; i < N_gas+N_stars; i++)
+ {
+ if((P[i].Ti_endstep == All.Ti_Current))
+ {
+
+
+ m = P[i].StPIdx;
+ StP[m].DhsmlDensityFactor =
+ 1 / (1 + StP[m].Hsml * StP[m].DhsmlDensityFactor / (NUMDIMS * StP[m].Density));
+
+
+
+ /* now check whether we had enough neighbours */
+
+
+ if(StP[m].NumNgb < (All.DesNumNgb - All.MaxNumNgbDeviation) ||
+ (StP[m].NumNgb > (All.DesNumNgb + All.MaxNumNgbDeviation)
+ && StP[m].Hsml > (1.01 * All.MinGasHsml)))
+ {
+
+#ifdef AVOIDNUMNGBPROBLEM
+// if((StP[m].NumNgb>StP[m].OldNumNgb-All.MaxNumNgbDeviation/10000.)
+// &&(StP[m].NumNgb<StP[m].OldNumNgb+All.MaxNumNgbDeviation/10000.))
+ if(StP[m].NumNgb==StP[m].OldNumNgb)
+ {
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+ printf("ID=%d NumNgb=%g OldNumNgb=%g\n",P[i].ID,StP[m].NumNgb,StP[m].OldNumNgb);
+ continue;
+ }
+
+ StP[m].OldNumNgb = StP[m].NumNgb;
+#endif
+
+
+ /* need to redo this particle */
+ npleft++;
+
+ if(StP[m].Left > 0 && StP[m].Right > 0)
+ if((StP[m].Right - StP[m].Left) < 1.0e-3 * StP[m].Left)
+ {
+ /* this one should be ok */
+ npleft--;
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1; /* Mark as inactive */
+ continue;
+ }
+
+ if(StP[m].NumNgb < (All.DesNumNgb - All.MaxNumNgbDeviation))
+ StP[m].Left = dmax(StP[m].Hsml, StP[m].Left);
+ else
+ {
+ if(StP[m].Right != 0)
+ {
+ if(StP[m].Hsml < StP[m].Right)
+ StP[m].Right = StP[m].Hsml;
+ }
+ else
+ StP[m].Right = StP[m].Hsml;
+ }
+
+ if(iter >= MAXITER - 10)
+ {
+ printf
+ ("i=%d task=%d ID=%d Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
+ i, ThisTask, (int) P[i].ID, StP[m].Hsml, StP[m].Left, StP[m].Right,
+ (float) StP[m].NumNgb, StP[m].Right - StP[m].Left, P[i].Pos[0], P[i].Pos[1],
+ P[i].Pos[2]);
+ fflush(stdout);
+ }
+
+ if(StP[m].Right > 0 && StP[m].Left > 0)
+ StP[m].Hsml = pow(0.5 * (pow(StP[m].Left, 3) + pow(StP[m].Right, 3)), 1.0 / 3);
+ else
+ {
+ if(StP[m].Right == 0 && StP[m].Left == 0)
+ {
+ printf
+ ("i=%d task=%d ID=%d Hsml=%g Left=%g Right=%g Ngbs=%g Right-Left=%g\n pos=(%g|%g|%g)\n",
+ i, ThisTask, (int) P[i].ID, StP[m].Hsml, StP[m].Left, StP[m].Right,
+ (float) StP[m].NumNgb, StP[m].Right - StP[m].Left, P[i].Pos[0], P[i].Pos[1],
+ P[i].Pos[2]);
+ fflush(stdout);
+ endrun(8188); /* can't occur */
+ }
+
+ if(StP[m].Right == 0 && StP[m].Left > 0)
+ {
+ if(P[i].Type == ST && fabs(StP[m].NumNgb - All.DesNumNgb) < 0.5 * All.DesNumNgb)
+ {
+ StP[m].Hsml *=
+ 1 - (StP[m].NumNgb -
+ All.DesNumNgb) / (NUMDIMS * StP[m].NumNgb) * StP[m].DhsmlDensityFactor;
+ }
+ else
+ StP[m].Hsml *= 1.26;
+ }
+
+ if(StP[m].Right > 0 && StP[m].Left == 0)
+ {
+ if(P[i].Type == ST && fabs(StP[m].NumNgb - All.DesNumNgb) < 0.5 * All.DesNumNgb)
+ {
+ StP[m].Hsml *=
+ 1 - (StP[m].NumNgb -
+ All.DesNumNgb) / (NUMDIMS * StP[m].NumNgb) * StP[m].DhsmlDensityFactor;
+ }
+ else
+ StP[m].Hsml /= 1.26;
+ }
+ }
+
+ if(StP[m].Hsml < All.MinGasHsml)
+ StP[m].Hsml = All.MinGasHsml;
+ }
+ else
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1; /* Mark as inactive */
+ }
+ }
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&npleft, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+ if(ntot > 0)
+ {
+ if(iter == 0)
+ tstart_ngb = second();
+
+ iter++;
+
+ if(iter > 0 && ThisTask == 0)
+ {
+ printf("ngb iteration %d: need to repeat for %d%09d particles.\n", iter,
+ (int) (ntot / 1000000000), (int) (ntot % 1000000000));
+ fflush(stdout);
+ }
+
+ if(iter > MAXITER)
+ {
+ printf("failed to converge in neighbour iteration in density()\n");
+ fflush(stdout);
+ endrun(1155);
+ }
+ }
+ else
+ tend_ngb = second();
+ }
+ while(ntot > 0);
+
+
+ /* mark as active again */
+ for(i = 0; i < NumPart; i++)
+ if(P[i].Ti_endstep < 0)
+ P[i].Ti_endstep = -P[i].Ti_endstep - 1;
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+ /* collect some timing information */
+ if(iter > 0)
+ timengb = timediff(tstart_ngb, tend_ngb);
+ else
+ timengb = 0;
+
+ MPI_Reduce(&timengb, &sumtimengb, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecomp, &sumt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timecommsumm, &sumcomm, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ MPI_Reduce(&timeimbalance, &sumimbalance, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ All.CPU_HydCompWalk += sumt / NTask;
+ All.CPU_HydCommSumm += sumcomm / NTask;
+ All.CPU_HydImbalance += sumimbalance / NTask;
+ All.CPU_EnsureNgb += sumtimengb / NTask;
+ }
+
+ if (ThisTask==0)
+ printf("Stars density computation done.\n");
+
+
+}
+
+
+
+/*! This function represents the core of the SPH density computation. The
+ * target particle may either be local, or reside in the communication
+ * buffer.
+ */
+void stars_density_evaluate(int target, int mode)
+{
+ int j, n, startnode, numngb, numngb_inbox;
+ double h, h2, hinv, hinv3, hinv4;
+ double rho, volume, wk, dwk;
+ double dx, dy, dz, r, r2, u, mass_j,rho_j;
+ double weighted_numngb, dhsmlrho;
+ int target_stp;
+ FLOAT *pos;
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ double ngbmass;
+#endif
+
+ if(mode == 0)
+ {
+ pos = P[target].Pos;
+ target_stp = P[target].StPIdx;
+ h = StP[target_stp].Hsml;
+ }
+ else
+ {
+ pos = StarsDensDataGet[target].Pos;
+ h = StarsDensDataGet[target].Hsml;
+ }
+
+ h2 = h * h;
+ hinv = 1.0 / h;
+#ifndef TWODIMS
+ hinv3 = hinv * hinv * hinv;
+#else
+ hinv3 = hinv * hinv / boxSize_Z;
+#endif
+ hinv4 = hinv3 * hinv;
+
+ rho = 0;
+ volume = 0;
+ weighted_numngb = 0;
+ dhsmlrho = 0;
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ ngbmass=0;
+#endif
+
+ startnode = All.MaxPart;
+ numngb = 0;
+ do
+ {
+ numngb_inbox = ngb_treefind_variable_for_chimie(&pos[0], h, &startnode);
+
+ for(n = 0; n < numngb_inbox; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+#ifdef PERIODIC /* now find the closest image in the given box size */
+ if(dx > boxHalf_X)
+ dx -= boxSize_X;
+ if(dx < -boxHalf_X)
+ dx += boxSize_X;
+ if(dy > boxHalf_Y)
+ dy -= boxSize_Y;
+ if(dy < -boxHalf_Y)
+ dy += boxSize_Y;
+ if(dz > boxHalf_Z)
+ dz -= boxSize_Z;
+ if(dz < -boxHalf_Z)
+ dz += boxSize_Z;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+
+ if(r2 < h2)
+ {
+ numngb++;
+
+ r = sqrt(r2);
+
+ u = r * hinv;
+
+ if(u < 0.5)
+ {
+ wk = hinv3 * (KERNEL_COEFF_1 + KERNEL_COEFF_2 * (u - 1) * u * u);
+ dwk = hinv4 * u * (KERNEL_COEFF_3 * u - KERNEL_COEFF_4);
+ }
+ else
+ {
+ wk = hinv3 * KERNEL_COEFF_5 * (1.0 - u) * (1.0 - u) * (1.0 - u);
+ dwk = hinv4 * KERNEL_COEFF_6 * (1.0 - u) * (1.0 - u);
+ }
+
+ mass_j = P[j].Mass;
+ rho_j = SphP[j].Density;
+
+ rho += mass_j * wk;
+ volume += mass_j/rho_j * wk;
+
+ weighted_numngb += NORM_COEFF * wk / hinv3;
+
+ dhsmlrho += -mass_j * (NUMDIMS * hinv * wk + u * dwk);
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ ngbmass += mass_j;
+#endif
+
+
+ }
+ }
+ }
+ while(startnode >= 0);
+
+ if(mode == 0)
+ {
+ StP[target_stp].NumNgb = weighted_numngb;
+ StP[target_stp].Density = rho;
+ StP[target_stp].Volume = volume;
+ StP[target_stp].DhsmlDensityFactor = dhsmlrho;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ StP[target_stp].NgbMass = ngbmass;
+#endif
+ }
+ else
+ {
+ StarsDensDataResult[target].Rho = rho;
+ StarsDensDataResult[target].Volume = volume;
+ StarsDensDataResult[target].Ngb = weighted_numngb;
+ StarsDensDataResult[target].DhsmlDensity = dhsmlrho;
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ StarsDensDataResult[target].NgbMass = ngbmass;
+#endif
+ }
+}
+
+
+
+
+/*! This routine is a comparison kernel used in a sort routine to group
+ * particles that are exported to the same processor.
+ */
+int stars_dens_compare_key(const void *a, const void *b)
+{
+ if(((struct starsdensdata_in *) a)->Task < (((struct starsdensdata_in *) b)->Task))
+ return -1;
+
+ if(((struct starsdensdata_in *) a)->Task > (((struct starsdensdata_in *) b)->Task))
+ return +1;
+
+ return 0;
+}
+
+
+#endif
diff --git a/sticky.c b/sticky.c
new file mode 100644
index 0000000..b4faab6
--- /dev/null
+++ b/sticky.c
@@ -0,0 +1,1307 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include <gsl/gsl_errno.h>
+#include <gsl/gsl_spline.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+
+
+
+#ifdef PERIODIC
+static double boxSize, boxHalf;
+
+#ifdef LONG_X
+static double boxSize_X, boxHalf_X;
+#else
+#define boxSize_X boxSize
+#define boxHalf_X boxHalf
+#endif
+#ifdef LONG_Y
+static double boxSize_Y, boxHalf_Y;
+#else
+#define boxSize_Y boxSize
+#define boxHalf_Y boxHalf
+#endif
+#ifdef LONG_Z
+static double boxSize_Z, boxHalf_Z;
+#else
+#define boxSize_Z boxSize
+#define boxHalf_Z boxHalf
+#endif
+#endif
+
+
+
+
+
+
+
+#ifdef MULTIPHASE
+
+static int Ncz;
+static int Ncxy;
+static int Ncxyz;
+
+
+static double hubble_a, atime, hubble_a2, fac_mu, fac_vsic_fix, a3inv, fac_egy;
+
+
+
+/*! \file sticky.c
+ * \brief Compute sticky collisions
+ *
+*/
+
+
+
+/*! init sticky
+ *
+ */
+void init_sticky()
+{
+
+ Ncz = All.StickyGridNz;
+ Ncxy = All.StickyGridNx*All.StickyGridNy;
+ Ncxyz = All.StickyGridNx*All.StickyGridNy*All.StickyGridNz;
+
+
+}
+
+
+
+
+/*! This function is the driver routine for the calculation of sticky collisions
+ */
+void sticky()
+{
+
+ double t0, t1;
+
+ t0 = second(); /* measure the time for the full chimie computation */
+
+
+ if((All.Time - All.StickyLastCollisionTime) >= All.StickyCollisionTime)
+ {
+
+ sticky_compute_energy_kin(1);
+ //sticky_collisions(); /* old implementation */
+
+ if(ThisTask==0)
+ printf("sticky first loop\n");
+ sticky_collisions2(1);
+ if(ThisTask==0)
+ printf("sticky second loop\n");
+ sticky_collisions2(2);
+
+ sticky_compute_energy_kin(2);
+
+ All.StickyLastCollisionTime += All.StickyCollisionTime;
+
+ }
+
+
+ t1 = second();
+ All.CPU_Sticky += timediff(t0, t1);
+
+
+}
+
+
+
+
+
+
+/*! This function is used as a comparison kernel in a sort routine.
+ */
+int compare_sticky_index(const void *a, const void *b)
+{
+ if(((struct Sticky_index *) a)->CellIndex < (((struct Sticky_index *) b)->CellIndex))
+ return -1;
+
+ if(((struct Sticky_index *) a)->CellIndex > (((struct Sticky_index *) b)->CellIndex))
+ return +1;
+
+ return 0;
+}
+
+
+
+/*! This function returns the first particle that may sticky-interact with
+ * the given particle. This routine use a pseudo-grid to find pairs, based
+ * on the Bournaud technique
+ */
+#ifdef MULTIPHASE
+int find_sticky_pairs(int ii)
+{
+
+ int jj;
+ int ci,cj;
+
+ int i,j;
+ double dx,dy,dz,r2,r;
+ double dvx,dvy,dvz,vdotr,vdotr2,dv;
+
+ if (ii>=N_gas)
+ return -11;
+
+ ci = StickyIndex[ii].CellIndex;
+
+ if (ci>=Ncxyz)
+ return -12;
+
+ jj = ii;
+
+
+ do
+ {
+
+ jj++;
+
+ if (jj>=N_gas)
+ return -1;
+
+ cj = StickyIndex[jj].CellIndex;
+
+ if (cj>=Ncxyz)
+ return -2;
+
+ if (ci!=cj)
+ return -3;
+
+ if (SphP[StickyIndex[jj].Index].StickyFlag) /* if its free, return it, else, go to the next one */
+ {
+ /* here we check that we can interact */
+
+ i = StickyIndex[ii].Index;
+ j = StickyIndex[jj].Index;
+
+
+ dx = P[i].Pos[0] - P[j].Pos[0];
+ dy = P[i].Pos[1] - P[j].Pos[1];
+ dz = P[i].Pos[2] - P[j].Pos[2];
+ r2 = dx * dx + dy * dy + dz * dz;
+ r = sqrt(r2);
+
+ dvx = P[i].Vel[0] - P[j].Vel[0];
+ dvy = P[i].Vel[1] - P[j].Vel[1];
+ dvz = P[i].Vel[2] - P[j].Vel[2];
+ vdotr = dx * dvx + dy * dvy + dz * dvz;
+ vdotr2 = vdotr;
+ dv = vdotr2/r;
+
+ if ( (vdotr2<0) && (-vdotr2>All.StickyMinVelocity))
+ {
+ printf("(1)%d %d\n",i,j);
+ return StickyIndex[jj].Index; /* ok, we accept the collision */
+
+ }
+ }
+
+
+ }
+ while(1);
+
+
+}
+#endif
+
+
+
+
+
+
+
+void sticky_compute_energy_kin(int mode)
+{
+ int i;
+ double DeltaEgyKin;
+ double Tot_DeltaEgyKin;
+
+ DeltaEgyKin = 0;
+ Tot_DeltaEgyKin = 0;
+
+ if (mode==1)
+ {
+ LocalSysState.EnergyKin1 = 0;
+ LocalSysState.EnergyKin2 = 0;
+ }
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+ if (P[i].Type==0)
+ {
+
+ if (mode==1)
+ LocalSysState.EnergyKin1 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ else
+ LocalSysState.EnergyKin2 += 0.5 * P[i].Mass * (P[i].Vel[0]*P[i].Vel[0]+P[i].Vel[1]*P[i].Vel[1]+P[i].Vel[2]*P[i].Vel[2]);
+ }
+ }
+
+ if (mode==2)
+ {
+ DeltaEgyKin = LocalSysState.EnergyKin2 - LocalSysState.EnergyKin1;
+ MPI_Reduce(&DeltaEgyKin, &Tot_DeltaEgyKin, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+ LocalSysState.EnergyRadSticky -= DeltaEgyKin;
+ }
+
+
+}
+
+
+
+
+
+
+
+
+
+
+/*! compute sticky collisions
+ *
+ */
+void sticky_collisions()
+{
+
+ int i,j,k;
+ int startnode;
+ FLOAT *pos, *vel;
+ FLOAT mass, h_i;
+ int phase;
+
+ double vcm[3],newv1[3],newv2[3];
+ double dv1[3],dv2[3];
+ double dx, dy, dz, dvx, dvy, dvz;
+ double r,r2;
+ double vdotr, vdotr2;
+
+ double dv;
+ double beta_r,beta_t;
+
+ double M1M,M2M,M12;
+ double dbeta_dv,dbeta_dv_er_x,dbeta_dv_er_y,dbeta_dv_er_z;
+ double dv_beta_t_x,dv_beta_t_y,dv_beta_t_z;
+
+
+ NumColLocal = 0;
+ NumNoColLocal = 0;
+
+ beta_r = All.StickyBetaR;
+ beta_t = All.StickyBetaT;
+
+
+
+ double xi,yi,zi;
+ int ix,iy,iz;
+ size_t bytes;
+
+ if (ThisTask==0)
+ printf("Start sticky collisions...\n");
+
+
+ /*****************************/
+ /* first, update StickyIndex */
+ /*****************************/
+
+ if (All.StickyUseGridForCollisions)
+ {
+
+ /* allocate memory */
+ if(!(StickyIndex = malloc(bytes = N_gas * sizeof(struct Sticky_index))))
+ {
+ printf("failed to allocate memory for `StickyIndex' (%g MB).\n", bytes / (1024.0 * 1024.0));
+ endrun(2233);
+ }
+
+
+ /* loop gas particles */
+ for(i = 0; i < N_gas; i++)
+ {
+
+ //if (P[i].Type==0) /* we also take also into account the stars not transfered in stars now */
+ {
+
+
+ StickyIndex[i].Index = i;
+ StickyIndex[i].CellIndex = Ncxyz;
+ StickyIndex[i].Flag = 1;
+
+
+ if (SphP[i].StickyFlag) /* particle that may collide (set in phase.c) */
+ {
+
+
+ /* scale between 0 and nx,ny,nz */
+ xi = All.StickyGridNx*(P[i].Pos[0]-All.StickyGridXmin)/(All.StickyGridXmax-All.StickyGridXmin);
+ yi = All.StickyGridNy*(P[i].Pos[1]-All.StickyGridYmin)/(All.StickyGridYmax-All.StickyGridYmin);
+ zi = All.StickyGridNz*(P[i].Pos[2]-All.StickyGridZmin)/(All.StickyGridZmax-All.StickyGridZmin);
+
+ ix = (int)(xi);
+ iy = (int)(yi);
+ iz = (int)(zi);
+
+
+ if (ix >= 0 && ix < All.StickyGridNx)
+ if (iy >= 0 && iy < All.StickyGridNy)
+ if (iz >= 0 && iz < All.StickyGridNz)
+ {
+ j = (ix)*(Ncxy) + (iy)*(Ncz) + (iz);
+ StickyIndex[i].Index = i;
+ StickyIndex[i].CellIndex = j;
+ }
+ }
+
+ }
+
+
+ }
+
+
+
+ /* sort particles */
+ qsort(StickyIndex, N_gas, sizeof(struct Sticky_index), compare_sticky_index);
+
+
+
+
+
+ /* record index in SphP (not necessary, but simplifies) */
+
+ /* loop over cells */
+ for(i = 0; i < N_gas; i++)
+ {
+ //printf("%d %d\n",StickyIndex[i].CellIndex,StickyIndex[i].Index);
+ j = StickyIndex[i].Index;
+ SphP[j].StickyIndex = i;
+ }
+
+
+ }
+
+
+ /*****************************/
+ /* loop over all particles */
+ /*****************************/
+
+
+ for(i = 0; i < N_gas; i++)
+ {
+
+//#ifdef SFR
+// if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+//#else
+// if(P[i].Ti_endstep == All.Ti_Current)
+//#endif
+
+#ifdef SFR
+ if(P[i].Type == 0)
+#endif
+ {
+
+ if(SphP[i].Phase == GAS_STICKY)
+ {
+ if(SphP[i].StickyFlag)
+ {
+
+ pos = P[i].Pos;
+ vel = SphP[i].VelPred;
+ mass = P[i].Mass;
+ h_i = SphP[i].Hsml;
+ phase = SphP[i].Phase;
+
+ startnode = All.MaxPart;
+
+ if (All.StickyUseGridForCollisions)
+ j = find_sticky_pairs(SphP[i].StickyIndex);
+ else
+ j = ngb_treefind_sticky_collisions(&pos[0], h_i, phase, &startnode);
+
+ /* check that particles are in the same cell */
+ /*
+ if (j>=0 && j!=i)
+ {
+ int i1,i2;
+
+ xi = All.StickyGridNx*(P[i].Pos[0]-All.StickyGridXmin)/(All.StickyGridXmax-All.StickyGridXmin);
+ yi = All.StickyGridNy*(P[i].Pos[1]-All.StickyGridYmin)/(All.StickyGridYmax-All.StickyGridYmin);
+ zi = All.StickyGridNz*(P[i].Pos[2]-All.StickyGridZmin)/(All.StickyGridZmax-All.StickyGridZmin);
+
+ ix = (int)(xi);
+ iy = (int)(yi);
+ iz = (int)(zi);
+
+ i1 = (ix)*(Ncxy) + (iy)*(Ncz) + (iz);
+
+
+ xi = All.StickyGridNx*(P[i].Pos[0]-All.StickyGridXmin)/(All.StickyGridXmax-All.StickyGridXmin);
+ yi = All.StickyGridNy*(P[i].Pos[1]-All.StickyGridYmin)/(All.StickyGridYmax-All.StickyGridYmin);
+ zi = All.StickyGridNz*(P[i].Pos[2]-All.StickyGridZmin)/(All.StickyGridZmax-All.StickyGridZmin);
+
+ ix = (int)(xi);
+ iy = (int)(yi);
+ iz = (int)(zi);
+
+ i2 = (ix)*(Ncxy) + (iy)*(Ncz) + (iz);
+
+ if (i1!=i2)
+ {
+ printf("\n\nSomething odd here!\n");
+ endrun("999888777");
+ }
+
+ }
+ */
+
+
+
+
+ if (j>=0 && j!=i) /* particles may collide */
+ {
+
+ if (j>=N_gas)
+ {
+ printf("sticky_collisions : j=%d,N_gas=%d j>N_gas !!!",j,N_gas);
+ exit(-1);
+ }
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+ r2 = dx * dx + dy * dy + dz * dz;
+ r = sqrt(r2);
+
+ dvx = vel[0] - SphP[j].VelPred[0];
+ dvy = vel[1] - SphP[j].VelPred[1];
+ dvz = vel[2] - SphP[j].VelPred[2];
+ vdotr = dx * dvx + dy * dvy + dz * dvz;
+ vdotr2 = vdotr;
+ dv = vdotr2/r;
+
+
+
+ if (vdotr2<0) /* particles approaches !!! */
+ {
+
+ /* compute new velocities */
+
+ M12 = mass+P[j].Mass;
+ M1M = mass/M12;
+ M2M = P[j].Mass/M12;
+
+
+ /* compute velocity of the mass center */
+ for(k = 0; k < 3; k++)
+ vcm[k] = (mass*vel[k]+P[j].Mass*SphP[j].VelPred[k])/M12;
+
+
+ dbeta_dv = (beta_r-beta_t)*dv;
+ dbeta_dv_er_x = -dbeta_dv *dx/r;
+ dbeta_dv_er_y = -dbeta_dv *dy/r;
+ dbeta_dv_er_z = -dbeta_dv *dz/r;
+
+ dv_beta_t_x = dvx*beta_t;
+ dv_beta_t_y = dvy*beta_t;
+ dv_beta_t_z = dvz*beta_t;
+
+
+ newv1[0] = M2M * ( +dv_beta_t_x - dbeta_dv_er_x ) + vcm[0];
+ newv1[1] = M2M * ( +dv_beta_t_y - dbeta_dv_er_y ) + vcm[1];
+ newv1[2] = M2M * ( +dv_beta_t_z - dbeta_dv_er_z ) + vcm[2];
+
+ newv2[0] = M1M * ( -dv_beta_t_x + dbeta_dv_er_x ) + vcm[0];
+ newv2[1] = M1M * ( -dv_beta_t_y + dbeta_dv_er_y ) + vcm[1];
+ newv2[2] = M1M * ( -dv_beta_t_z + dbeta_dv_er_z ) + vcm[2];
+
+
+ /* new velocities */
+ for(k = 0; k < 3; k++)
+ {
+
+ dv1[k] = newv1[k] - SphP[i].VelPred[k];
+ dv2[k] = newv2[k] - SphP[j].VelPred[k];
+
+ SphP[i].VelPred[k] = newv1[k];
+ SphP[j].VelPred[k] = newv2[k];
+
+ P[i].Vel[k] += dv1[k];
+ P[j].Vel[k] += dv2[k];
+
+ }
+
+
+
+ /* set particles as non sticky-active */
+ SphP[i].StickyFlag = 0;
+ SphP[j].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time + All.StickyIdleTime;
+ SphP[j].StickyTime = All.Time + All.StickyIdleTime;
+
+
+ /* record collision */
+ NumColLocal+=2;
+
+ }
+
+ }
+
+
+
+ }
+ }
+
+
+ }
+ }
+
+
+
+
+ /* write some statistics */
+
+ MPI_Allreduce(&NumColLocal, &NumCol, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&NumColPotLocal,&NumColPot, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&NumNoColLocal, &NumNoCol, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ fprintf(FdSticky, "Step %d, Time: %g NumColPot: %d NumCol: %d NumNoCol: %d\n", All.NumCurrentTiStep, All.Time,(int)NumColPot,(int)NumCol,(int)NumNoCol);
+ fflush(FdSticky);
+ }
+
+
+
+ if (All.StickyUseGridForCollisions)
+ free(StickyIndex);
+
+ if (ThisTask==0)
+ printf("sticky collisions done.\n");
+
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+/*! This function is the driver routine for the calculation of hydrodynamical
+ * force and rate of change of entropy due to shock heating for all active
+ * particles .
+ *
+ * During the first loop, each particle compute j (a potentially colliding particle)
+ * During the second loop, it check if it can collide and collide if it can.
+ *
+ */
+void sticky_collisions2(int loop)
+{
+ long long ntot, ntotleft;
+ int i, j, k, n, ngrp, maxfill, source, ndone;
+ int *nbuffer, *noffset, *nsend_local, *nsend, *numlist, *ndonelist;
+ int level, sendTask, recvTask, nexport, place;
+ double tstart, tend, sumt, sumcomm;
+ double timecomp = 0, timecommsumm = 0, timeimbalance = 0, sumimbalance;
+ MPI_Status status;
+
+ double dv[3];
+
+
+#ifdef PERIODIC
+ boxSize = All.BoxSize;
+ boxHalf = 0.5 * All.BoxSize;
+#ifdef LONG_X
+ boxHalf_X = boxHalf * LONG_X;
+ boxSize_X = boxSize * LONG_X;
+#endif
+#ifdef LONG_Y
+ boxHalf_Y = boxHalf * LONG_Y;
+ boxSize_Y = boxSize * LONG_Y;
+#endif
+#ifdef LONG_Z
+ boxHalf_Z = boxHalf * LONG_Z;
+ boxSize_Z = boxSize * LONG_Z;
+#endif
+#endif
+
+#ifdef COMPUTE_VELOCITY_DISPERSION
+ double v1m,v2m;
+#endif
+
+ if(All.ComovingIntegrationOn)
+ {
+ /* Factors for comoving integration of hydro */
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+ hubble_a2 = All.Time * All.Time * hubble_a;
+
+ fac_mu = pow(All.Time, 3 * (GAMMA - 1) / 2) / All.Time;
+
+ fac_egy = pow(All.Time, 3 * (GAMMA - 1));
+
+ fac_vsic_fix = hubble_a * pow(All.Time, 3 * GAMMA_MINUS1);
+
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ atime = All.Time;
+
+#ifdef FEEDBACK
+ fac_pow = fac_egy*atime*atime;
+#endif
+
+ }
+ else
+ {
+ hubble_a = hubble_a2 = atime = fac_mu = fac_vsic_fix = a3inv = fac_egy = 1.0;
+#ifdef FEEDBACK
+ fac_pow = 1.0;
+#endif
+ }
+
+
+ /* `NumSphUpdate' gives the number of particles on this processor that want a force update */
+ for(n = 0, NumSphUpdate = 0; n < N_gas; n++)
+ {
+#ifdef SFR
+ //if((P[n].Ti_endstep == All.Ti_Current) && (P[n].Type == 0))
+#else
+ //if(P[n].Ti_endstep == All.Ti_Current)
+#endif
+ if((SphP[n].Phase == GAS_STICKY) && (SphP[n].StickyFlag) )
+ {
+
+ if (loop==1)
+ {
+ SphP[n].StickyMaxFs = 0;
+ SphP[n].StickyMaxID =-1;
+ }
+ SphP[n].StickyNgb =-1;
+
+
+ NumSphUpdate++;
+ }
+ }
+
+ numlist = malloc(NTask * sizeof(int) * NTask);
+ MPI_Allgather(&NumSphUpdate, 1, MPI_INT, numlist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0, ntot = 0; i < NTask; i++)
+ ntot += numlist[i];
+ free(numlist);
+
+
+ noffset = malloc(sizeof(int) * NTask); /* offsets of bunches in common list */
+ nbuffer = malloc(sizeof(int) * NTask);
+ nsend_local = malloc(sizeof(int) * NTask);
+ nsend = malloc(sizeof(int) * NTask * NTask);
+ ndonelist = malloc(sizeof(int) * NTask);
+
+ i = 0; /* first particle for this task */
+ ntotleft = ntot; /* particles left for all tasks together */
+
+
+ NumColLocal = 0;
+ NumNoColLocal = 0;
+
+ while(ntotleft > 0)
+ {
+ for(j = 0; j < NTask; j++)
+ nsend_local[j] = 0;
+
+ /* do local particles and prepare export list */
+ tstart = second();
+ for(nexport = 0, ndone = 0; i < N_gas && nexport < All.BunchSizeSticky - NTask; i++)
+ {
+#ifdef SFR
+ //if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ //if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+
+ if((SphP[i].Phase == GAS_STICKY)&&(SphP[i].StickyFlag))
+ {
+
+ ndone++;
+
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ sticky_evaluate(i, 0,loop);
+
+ /* the particle is not exported if an ngb has been found locally */
+ if (SphP[i].StickyNgb!=-1)
+ for(j = 0; j < NTask; j++)
+ Exportflag[j] = 0;
+
+ for(j = 0; j < NTask; j++)
+ {
+ if(Exportflag[j])
+ {
+
+ for(k = 0; k < 3; k++)
+ {
+ StickyDataIn[nexport].Pos[k] = P[i].Pos[k];
+ StickyDataIn[nexport].Vel[k] = SphP[i].VelPred[k];
+ }
+ StickyDataIn[nexport].Mass = P[i].Mass;
+ StickyDataIn[nexport].Hsml = SphP[i].Hsml;
+ StickyDataIn[nexport].StickyMaxID = SphP[i].StickyMaxID;
+ StickyDataIn[nexport].StickyMaxFs = SphP[i].StickyMaxFs;
+ StickyDataIn[nexport].StickyNgb = SphP[i].StickyNgb;
+ StickyDataIn[nexport].ID = P[i].ID;
+
+
+ StickyDataIn[nexport].Index = i;
+ StickyDataIn[nexport].Task = j;
+ nexport++;
+ nsend_local[j]++;
+ }
+ }
+
+ }
+ }
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ qsort(StickyDataIn, nexport, sizeof(struct stickydata_in), sticky_compare_key);
+
+ for(j = 1, noffset[0] = 0; j < NTask; j++)
+ noffset[j] = noffset[j - 1] + nsend_local[j - 1];
+
+ tstart = second();
+
+ MPI_Allgather(nsend_local, NTask, MPI_INT, nsend, NTask, MPI_INT, MPI_COMM_WORLD);
+
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* now do the particles that need to be exported */
+ for(level = 1; level < (1 << PTask); level++)
+ {
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeSticky)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* get the particles */
+ MPI_Sendrecv(&StickyDataIn[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct stickydata_in), MPI_BYTE,
+ recvTask, TAG_HYDRO_A,
+ &StickyDataGet[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct stickydata_in), MPI_BYTE,
+ recvTask, TAG_HYDRO_A, MPI_COMM_WORLD, &status);
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ /* now do the imported particles */
+ tstart = second();
+ for(j = 0; j < nbuffer[ThisTask]; j++)
+ sticky_evaluate(j, 1,loop);
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+ /* do a block to measure imbalance */
+ tstart = second();
+ MPI_Barrier(MPI_COMM_WORLD);
+ tend = second();
+ timeimbalance += timediff(tstart, tend);
+
+ /* get the result */
+ tstart = second();
+ for(j = 0; j < NTask; j++)
+ nbuffer[j] = 0;
+ for(ngrp = level; ngrp < (1 << PTask); ngrp++)
+ {
+ maxfill = 0;
+ for(j = 0; j < NTask; j++)
+ {
+ if((j ^ ngrp) < NTask)
+ if(maxfill < nbuffer[j] + nsend[(j ^ ngrp) * NTask + j])
+ maxfill = nbuffer[j] + nsend[(j ^ ngrp) * NTask + j];
+ }
+ if(maxfill >= All.BunchSizeSticky)
+ break;
+
+ sendTask = ThisTask;
+ recvTask = ThisTask ^ ngrp;
+
+ if(recvTask < NTask)
+ {
+ if(nsend[ThisTask * NTask + recvTask] > 0 || nsend[recvTask * NTask + ThisTask] > 0)
+ {
+ /* send the results */
+ MPI_Sendrecv(&StickyDataResult[nbuffer[ThisTask]],
+ nsend[recvTask * NTask + ThisTask] * sizeof(struct stickydata_out),
+ MPI_BYTE, recvTask, TAG_HYDRO_B,
+ &StickyDataPartialResult[noffset[recvTask]],
+ nsend_local[recvTask] * sizeof(struct stickydata_out),
+ MPI_BYTE, recvTask, TAG_HYDRO_B, MPI_COMM_WORLD, &status);
+
+ /* add the result to the particles */
+ for(j = 0; j < nsend_local[recvTask]; j++)
+ {
+ source = j + noffset[recvTask];
+ place = StickyDataIn[source].Index;
+
+ if (StickyDataPartialResult[source].StickyMaxFs>SphP[place].StickyMaxFs)
+ {
+ SphP[place].StickyMaxID = StickyDataPartialResult[source].StickyMaxID;
+ SphP[place].StickyMaxFs = StickyDataPartialResult[source].StickyMaxFs;
+ }
+
+ if (StickyDataPartialResult[source].StickyNgb!=-1) /* copy only if met a neighbor */
+ {
+ SphP[place].StickyNgb = StickyDataPartialResult[source].StickyNgb;
+ for(k = 0; k < 3; k++)
+ SphP[place].StickyNewVel[k]= StickyDataPartialResult[source].StickyNewVel[k];
+ }
+ }
+
+
+
+ }
+ }
+
+ for(j = 0; j < NTask; j++)
+ if((j ^ ngrp) < NTask)
+ nbuffer[j] += nsend[(j ^ ngrp) * NTask + j];
+ }
+ tend = second();
+ timecommsumm += timediff(tstart, tend);
+
+ level = ngrp - 1;
+ }
+
+ MPI_Allgather(&ndone, 1, MPI_INT, ndonelist, 1, MPI_INT, MPI_COMM_WORLD);
+ for(j = 0; j < NTask; j++)
+ ntotleft -= ndonelist[j];
+ }
+
+ free(ndonelist);
+ free(nsend);
+ free(nsend_local);
+ free(nbuffer);
+ free(noffset);
+
+
+ if (loop==2)
+ {
+
+ /* do final operations on results */
+ double norm_dv;
+ double max_dv;
+ int amax_dv;
+
+ max_dv = 0;
+
+
+ for(i = 0; i < N_gas; i++)
+#ifdef SFR
+ //if((P[i].Ti_endstep == All.Ti_Current) && (P[i].Type == 0))
+#else
+ //if(P[i].Ti_endstep == All.Ti_Current)
+#endif
+ if((SphP[i].Phase == GAS_STICKY)&&(SphP[i].StickyFlag))
+ {
+
+ if (loop==1)
+ {
+ if (SphP[i].StickyMaxID!=-1)
+ {
+ NumColLocal+=1;
+ }
+ }
+
+
+
+
+ if (SphP[i].StickyNgb!=-1)
+ {
+
+ //printf("(%d) COLLISION : %d -> %d\n",ThisTask,P[i].ID,SphP[i].StickyNgb);
+ //printf(" : OldV %g NewV %g\n",SphP[i].VelPred[0],SphP[i].StickyNewVel[0]);
+ //printf(" : OldV %g NewV %g\n",SphP[i].VelPred[1],SphP[i].StickyNewVel[1]);
+ //printf(" : OldV %g NewV %g\n",SphP[i].VelPred[2],SphP[i].StickyNewVel[2]);
+
+ /* new velocities */
+ for(k = 0; k < 3; k++)
+ {
+ dv[k] = SphP[i].StickyNewVel[k] - SphP[i].VelPred[k];
+ SphP[i].VelPred[k] = SphP[i].StickyNewVel[k];
+ P[i].Vel[k] += dv[k];
+ }
+
+ norm_dv = sqrt( dv[0]*dv[0] + dv[1]*dv[1] + dv[2]*dv[2]);
+
+ if (norm_dv>max_dv)
+ {
+ max_dv = norm_dv;
+ amax_dv= P[i].ID;
+ }
+
+
+ /* set particles as non sticky-active */
+ SphP[i].StickyFlag = 0;
+ SphP[i].StickyTime = All.Time + All.StickyIdleTime;
+
+
+ /* record collision */
+ NumColLocal+=1;
+ }
+
+
+
+ }
+
+
+
+ printf("(%d) max_dv=%g amax_dv=%d\n",ThisTask,max_dv,amax_dv);
+
+ }
+
+ tend = second();
+ timecomp += timediff(tstart, tend);
+
+
+
+
+ /* write some statistics */
+ if(loop==2)
+ {
+
+ MPI_Allreduce(&NumColLocal, &NumCol, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&NumColPotLocal,&NumColPot, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(&NumNoColLocal, &NumNoCol, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+
+ if(ThisTask == 0)
+ {
+ fprintf(FdSticky, "Step %d, Time: %g NumColPot: %d NumCol: %d NumNoCol: %d\n", All.NumCurrentTiStep, All.Time,(int)NumColPot,(int)NumCol,(int)NumNoCol);
+ fflush(FdSticky);
+ }
+
+
+ }
+
+ if (ThisTask==0)
+ printf("sticky collisions done.\n");
+
+}
+
+
+
+
+
+/*!
+ *
+ * first sticky
+ *
+ */
+void sticky_evaluate(int target, int mode, int loop)
+{
+ int j,k, n, startnode, numngb;
+ FLOAT *pos, *vel;
+ FLOAT mass;
+ FLOAT h_i;
+ double dx, dy, dz;
+ double dvx,dvy,dvz;
+ double vdotr, vdotr2,dv,dv2;
+ double vcm[3],newvel[3];
+
+ double h_i2;
+ double h_j, r, r2;
+ int phase=0;
+ int StickyMaxID;
+ int id;
+
+ double M1M,M2M,M12;
+ double dbeta_dv,dbeta_dv_er_x,dbeta_dv_er_y,dbeta_dv_er_z;
+ double dv_beta_t_x,dv_beta_t_y,dv_beta_t_z;
+ double beta_r,beta_t;
+
+
+ float fs,maxfs;
+ int id_max;
+ int id_ngb;
+
+
+ beta_r = All.StickyBetaR;
+ beta_t = All.StickyBetaT;
+
+ if(mode == 0)
+ {
+ pos = P[target].Pos;
+ vel = SphP[target].VelPred;
+ mass= P[target].Mass;
+ h_i = SphP[target].Hsml;
+ id = P[target].ID;
+ StickyMaxID = SphP[target].StickyMaxID;
+ maxfs = SphP[target].StickyMaxFs;
+ id_max= SphP[target].StickyMaxID;
+ id_ngb= SphP[target].StickyNgb;
+ }
+ else
+ {
+ pos = StickyDataGet[target].Pos;
+ vel = StickyDataGet[target].Vel;
+ mass= StickyDataGet[target].Mass;
+ h_i = StickyDataGet[target].Hsml;
+ id = StickyDataGet[target].ID;
+
+ StickyMaxID = StickyDataGet[target].StickyMaxID;
+ maxfs = StickyDataGet[target].StickyMaxFs;
+ id_max= StickyDataGet[target].StickyMaxID;
+ id_ngb=StickyDataGet[target].StickyNgb;
+ }
+
+
+ h_i2 = h_i * h_i;
+
+ for(k = 0; k < 3; k++)
+ newvel[k] = vel[k];
+
+ /* Now start sticky 1 for this particle */
+ startnode = All.MaxPart;
+ do
+ {
+ numngb = ngb_treefind_pairs(&pos[0], h_i, phase, &startnode);
+
+
+
+ for(n = 0; n < numngb; n++)
+ {
+ j = Ngblist[n];
+
+ dx = pos[0] - P[j].Pos[0];
+ dy = pos[1] - P[j].Pos[1];
+ dz = pos[2] - P[j].Pos[2];
+
+#ifdef PERIODIC /* find the closest image in the given box size */
+ if(dx > boxHalf_X)
+ dx -= boxSize_X;
+ if(dx < -boxHalf_X)
+ dx += boxSize_X;
+ if(dy > boxHalf_Y)
+ dy -= boxSize_Y;
+ if(dy < -boxHalf_Y)
+ dy += boxSize_Y;
+ if(dz > boxHalf_Z)
+ dz -= boxSize_Z;
+ if(dz < -boxHalf_Z)
+ dz += boxSize_Z;
+#endif
+ r2 = dx * dx + dy * dy + dz * dz;
+ h_j = SphP[j].Hsml;
+
+ if(r2 < h_i2 || r2 < h_j * h_j)
+ {
+ r = sqrt(r2);
+ if(r > 0)
+ {
+
+ dvx = vel[0] - SphP[j].VelPred[0];
+ dvy = vel[1] - SphP[j].VelPred[1];
+ dvz = vel[2] - SphP[j].VelPred[2];
+ vdotr = dx * dvx + dy * dvy + dz * dvz;
+ vdotr2 = vdotr;
+ dv = vdotr2/r;
+
+
+ if (loop==1)
+ {
+
+ if (vdotr2<0) /* particles approaches !!! */
+ {
+
+
+
+ /* compute new velocities */
+
+ M12 = mass+P[j].Mass;
+ M1M = mass/M12;
+ M2M = P[j].Mass/M12;
+
+
+ /* compute velocity of the mass center */
+ for(k = 0; k < 3; k++)
+ vcm[k] = (mass*vel[k]+P[j].Mass*SphP[j].VelPred[k])/M12;
+
+
+ dbeta_dv = (beta_r-beta_t)*dv;
+ dbeta_dv_er_x = -dbeta_dv *dx/r;
+ dbeta_dv_er_y = -dbeta_dv *dy/r;
+ dbeta_dv_er_z = -dbeta_dv *dz/r;
+
+ dv_beta_t_x = dvx*beta_t;
+ dv_beta_t_y = dvy*beta_t;
+ dv_beta_t_z = dvz*beta_t;
+
+
+ newvel[0] = M2M * ( +dv_beta_t_x - dbeta_dv_er_x ) + vcm[0];
+ newvel[1] = M2M * ( +dv_beta_t_y - dbeta_dv_er_y ) + vcm[1];
+ newvel[2] = M2M * ( +dv_beta_t_z - dbeta_dv_er_z ) + vcm[2];
+
+
+ /* check that dv is not too large */
+ dv2 = (newvel[0]-vel[0])*(newvel[0]-vel[0])
+ + (newvel[1]-vel[1])*(newvel[1]-vel[1])
+ + (newvel[2]-vel[2])*(newvel[2]-vel[2]);
+
+
+ if (dv2<All.StickyMaxVelocity*All.StickyMaxVelocity)
+ {
+ fs = r/( (0.5*(h_i+h_j)+r) * (0.5*(h_i+h_j)+r));
+
+ if (fs>maxfs)
+ {
+ id_max = P[j].ID;
+ maxfs = fs;
+ }
+ }
+ }
+
+ }
+ else
+ {
+
+ if ((SphP[j].StickyMaxID == id) && (StickyMaxID == P[j].ID))
+ {
+ id_ngb = P[j].ID;
+
+
+ /* compute new velocities */
+
+ M12 = mass+P[j].Mass;
+ M1M = mass/M12;
+ M2M = P[j].Mass/M12;
+
+
+ /* compute velocity of the mass center */
+ for(k = 0; k < 3; k++)
+ vcm[k] = (mass*vel[k]+P[j].Mass*SphP[j].VelPred[k])/M12;
+
+
+ dbeta_dv = (beta_r-beta_t)*dv;
+ dbeta_dv_er_x = -dbeta_dv *dx/r;
+ dbeta_dv_er_y = -dbeta_dv *dy/r;
+ dbeta_dv_er_z = -dbeta_dv *dz/r;
+
+ dv_beta_t_x = dvx*beta_t;
+ dv_beta_t_y = dvy*beta_t;
+ dv_beta_t_z = dvz*beta_t;
+
+
+ newvel[0] = M2M * ( +dv_beta_t_x - dbeta_dv_er_x ) + vcm[0];
+ newvel[1] = M2M * ( +dv_beta_t_y - dbeta_dv_er_y ) + vcm[1];
+ newvel[2] = M2M * ( +dv_beta_t_z - dbeta_dv_er_z ) + vcm[2];
+
+
+ }
+ }
+
+ }
+ }
+ }
+ }
+ while(startnode >= 0);
+
+ /* Now collect the result at the right place */
+ if(mode == 0)
+ {
+ if(loop == 1)
+ {
+ SphP[target].StickyMaxID = id_max;
+ SphP[target].StickyMaxFs = maxfs;
+ }
+ SphP[target].StickyNgb = id_ngb;
+ for(k = 0; k < 3; k++)
+ SphP[target].StickyNewVel[k] = newvel[k];
+
+ }
+ else
+ {
+ if(loop == 1)
+ {
+ StickyDataResult[target].StickyMaxID = id_max;
+ StickyDataResult[target].StickyMaxFs= maxfs;
+ }
+ StickyDataResult[target].StickyNgb= id_ngb;
+ for(k = 0; k < 3; k++)
+ StickyDataResult[target].StickyNewVel[k] = newvel[k];
+
+ }
+
+
+}
+
+
+
+
+
+/*! This is a comparison kernel for a sort routine, which is used to group
+ * particles that are going to be exported to the same CPU.
+ */
+int sticky_compare_key(const void *a, const void *b)
+{
+ if(((struct stickydata_in *) a)->Task < (((struct stickydata_in *) b)->Task))
+ return -1;
+ if(((struct stickydata_in *) a)->Task > (((struct stickydata_in *) b)->Task))
+ return +1;
+ return 0;
+}
+
+
+
+
+
+
+
+
+
+
+#endif
+
+
+
+
+
+
diff --git a/system.c b/system.c
new file mode 100644
index 0000000..1c9d257
--- /dev/null
+++ b/system.c
@@ -0,0 +1,178 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+#include <signal.h>
+#include <gsl/gsl_rng.h>
+#include <mpi.h>
+
+#include "allvars.h"
+#include "proto.h"
+
+
+/*! \file system.c
+ * \brief contains miscellaneous routines, e.g. elapsed time measurements
+ */
+
+
+/*! This routine returns a random number taken from a table of random numbers,
+ * which is refilled every timestep. This method is used to allow random
+ * number application to particles independent of the number of processors
+ * used, and independent of the particular order the particles have. In order
+ * to work properly, the particle IDs should be set properly to unique
+ * integer values.
+ */
+double get_random_number(int id)
+{
+ return RndTable[(id % RNDTABLE)];
+}
+
+
+#ifdef SFR
+double get_StarFormation_random_number(int id)
+{
+ return StarFormationRndTable[(id % RNDTABLE)];
+}
+#endif
+
+#ifdef FEEDBACK_WIND
+double get_FeedbackWind_random_number(int id)
+{
+ return FeedbackWindRndTable[(id % RNDTABLE)];
+}
+#endif
+
+
+#ifdef CHIMIE
+double get_Chimie_random_number(int id)
+{
+ return ChimieRndTable[(id % RNDTABLE)];
+}
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+double get_ChimieKineticFeedback_random_number(int id)
+{
+ return ChimieKineticFeedbackRndTable[(id % RNDTABLE)];
+}
+#endif
+
+
+
+/*! This routine fills the random number table.
+ */
+void set_random_numbers(void)
+{
+ int i;
+
+ for(i = 0; i < RNDTABLE; i++)
+ RndTable[i] = gsl_rng_uniform(random_generator);
+
+#ifdef SFR
+ for(i = 0; i < RNDTABLE; i++)
+ StarFormationRndTable[i] = gsl_rng_uniform(random_generator);
+#endif
+
+#ifdef FEEDBACK_WIND
+ for(i = 0; i < RNDTABLE; i++)
+ FeedbackWindRndTable[i] = gsl_rng_uniform(random_generator);
+#endif
+
+#ifdef CHIMIE
+ for(i = 0; i < RNDTABLE; i++)
+ ChimieRndTable[i] = gsl_rng_uniform(random_generator);
+#endif
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ for(i = 0; i < RNDTABLE; i++)
+ ChimieKineticFeedbackRndTable[i] = gsl_rng_uniform(random_generator);
+#endif
+}
+
+
+/*! returns the number of cpu-ticks in seconds that have elapsed, or the
+ * wall-clock time obtained with MPI_Wtime().
+ */
+double second(void)
+{
+#ifdef WALLCLOCK
+ return MPI_Wtime();
+#else
+ return ((double) clock()) / CLOCKS_PER_SEC;
+#endif
+
+ /* note: on AIX and presumably many other 32bit systems,
+ * clock() has only a resolution of 10ms=0.01sec
+ */
+}
+
+
+/*! returns the time difference between two measurements obtained with
+ * second(). The routine takes care of the possible overflow of the tick
+ * counter on 32bit systems, but depending on the system, this may not always
+ * work properly. Similarly, in some MPI implementations, the MPI_Wtime()
+ * function may also overflow, in which case a negative time difference would
+ * be returned. The routine returns instead a time difference equal to 0.
+ */
+double timediff(double t0, double t1)
+{
+ double dt;
+
+ dt = t1 - t0;
+
+ if(dt < 0) /* overflow has occured (for systems with 32bit tick counter) */
+ {
+#ifdef WALLCLOCK
+ dt = 0;
+#else
+ dt = t1 + pow(2, 32) / CLOCKS_PER_SEC - t0;
+#endif
+ }
+
+ return dt;
+}
+
+
+/*! returns the maximum of two double
+ */
+double dmax(double x, double y)
+{
+ if(x > y)
+ return x;
+ else
+ return y;
+}
+
+/*! returns the minimum of two double
+ */
+double dmin(double x, double y)
+{
+ if(x < y)
+ return x;
+ else
+ return y;
+}
+
+/*! returns the maximum of two integers
+ */
+int imax(int x, int y)
+{
+ if(x > y)
+ return x;
+ else
+ return y;
+}
+
+/*! returns the minimum of two integers
+ */
+int imin(int x, int y)
+{
+ if(x < y)
+ return x;
+ else
+ return y;
+}
diff --git a/tags.h b/tags.h
new file mode 100644
index 0000000..f0225d3
--- /dev/null
+++ b/tags.h
@@ -0,0 +1,36 @@
+/*! \file tags.h
+ * \brief declares various tags for labelling MPI messages.
+ */
+
+#define TAG_N 10 /*!< Various tags used for labelling MPI messages */
+#define TAG_HEADER 11
+#define TAG_PDATA 12
+#define TAG_SPHDATA 13
+#define TAG_KEY 14
+#define TAG_DMOM 15
+#define TAG_NODELEN 16
+#define TAG_HMAX 17
+#define TAG_GRAV_A 18
+#define TAG_GRAV_B 19
+#define TAG_DIRECT_A 20
+#define TAG_DIRECT_B 21
+#define TAG_HYDRO_A 22
+#define TAG_HYDRO_B 23
+#define TAG_NFORTHISTASK 24
+#define TAG_PERIODIC_A 25
+#define TAG_PERIODIC_B 26
+#define TAG_PERIODIC_C 27
+#define TAG_PERIODIC_D 28
+#define TAG_NONPERIOD_A 29
+#define TAG_NONPERIOD_B 30
+#define TAG_NONPERIOD_C 31
+#define TAG_NONPERIOD_D 32
+#define TAG_POTENTIAL_A 33
+#define TAG_POTENTIAL_B 34
+#define TAG_DENS_A 35
+#define TAG_DENS_B 36
+#define TAG_LOCALN 37
+#define TAG_STDATA 38
+#define TAG_CHIMIE_A 39
+#define TAG_CHIMIE_B 40
+#define TAG_CHIMIE_EXTRAHEADER 41
\ No newline at end of file
diff --git a/timestep.c b/timestep.c
new file mode 100644
index 0000000..2590f2a
--- /dev/null
+++ b/timestep.c
@@ -0,0 +1,870 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <mpi.h>
+#include "allvars.h"
+#include "proto.h"
+
+/*! \file timestep.c
+ * \brief routines for 'kicking' particles in momentum space and assigning new timesteps
+ */
+
+static double fac1, fac2, fac3, hubble_a, atime, a3inv;
+static double dt_displacement = 0;
+
+
+/*! This function advances the system in momentum space, i.e. it does apply
+ * the 'kick' operation after the forces have been computed. Additionally, it
+ * assigns new timesteps to particles. At start-up, a half-timestep is
+ * carried out, as well as at the end of the simulation. In between, the
+ * half-step kick that ends the previous timestep and the half-step kick for
+ * the new timestep are combined into one operation.
+ */
+void advance_and_find_timesteps(void)
+{
+ int i, j, no, ti_step, ti_min, tend, tstart;
+ double dt_entr, dt_entr2, dt_gravkick, dt_hydrokick, dt_gravkick2, dt_hydrokick2, t0, t1;
+ double minentropy, aphys;
+ FLOAT dv[3];
+
+#ifdef COOLING
+ double t2,t3;
+#endif
+
+#ifdef FLEXSTEPS
+ int ti_grp;
+#endif
+#if defined(PSEUDOSYMMETRIC) && !defined(FLEXSTEPS)
+ double apred, prob;
+ int ti_step2;
+#endif
+#ifdef PMGRID
+ double dt_gravkickA, dt_gravkickB;
+#endif
+#ifdef MAKEGLASS
+ double disp, dispmax, globmax, dmean, fac, disp2sum, globdisp2sum;
+#endif
+
+
+ t0 = second();
+
+ if(All.ComovingIntegrationOn)
+ {
+ fac1 = 1 / (All.Time * All.Time);
+ fac2 = 1 / pow(All.Time, 3 * GAMMA - 2);
+ fac3 = pow(All.Time, 3 * (1 - GAMMA) / 2.0);
+ hubble_a = All.Omega0 / (All.Time * All.Time * All.Time)
+ + (1 - All.Omega0 - All.OmegaLambda) / (All.Time * All.Time) + All.OmegaLambda;
+
+ hubble_a = All.Hubble * sqrt(hubble_a);
+ a3inv = 1 / (All.Time * All.Time * All.Time);
+ atime = All.Time;
+ }
+ else
+ fac1 = fac2 = fac3 = hubble_a = a3inv = atime = 1;
+
+#ifdef NOPMSTEPADJUSTMENT
+ dt_displacement = All.MaxSizeTimestep;
+#else
+ if(Flag_FullStep || dt_displacement == 0)
+ find_dt_displacement_constraint(hubble_a * atime * atime);
+#endif
+
+#ifdef PMGRID
+ if(All.ComovingIntegrationOn)
+ dt_gravkickB = get_gravkick_factor(All.PM_Ti_begstep, All.Ti_Current) -
+ get_gravkick_factor(All.PM_Ti_begstep, (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2);
+ else
+ dt_gravkickB = (All.Ti_Current - (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2) * All.Timebase_interval;
+
+ if(All.PM_Ti_endstep == All.Ti_Current) /* need to do long-range kick */
+ {
+ /* make sure that we reconstruct the domain/tree next time because we don't kick the tree nodes in this case */
+ All.NumForcesSinceLastDomainDecomp = 1 + All.TotNumPart * All.TreeDomainUpdateFrequency;
+ }
+#endif
+
+
+#ifdef MAKEGLASS
+ for(i = 0, dispmax = 0, disp2sum = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ P[i].GravPM[j] *= -1;
+ P[i].GravAccel[j] *= -1;
+ P[i].GravAccel[j] += P[i].GravPM[j];
+ P[i].GravPM[j] = 0;
+ }
+
+ disp = sqrt(P[i].GravAccel[0] * P[i].GravAccel[0] +
+ P[i].GravAccel[1] * P[i].GravAccel[1] + P[i].GravAccel[2] * P[i].GravAccel[2]);
+
+ disp *= 2.0 / (3 * All.Hubble * All.Hubble);
+
+ disp2sum += disp * disp;
+
+ if(disp > dispmax)
+ dispmax = disp;
+ }
+
+ MPI_Allreduce(&dispmax, &globmax, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
+ MPI_Allreduce(&disp2sum, &globdisp2sum, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+ dmean = pow(P[0].Mass / (All.Omega0 * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G)), 1.0 / 3);
+
+ if(globmax > dmean)
+ fac = dmean / globmax;
+ else
+ fac = 1.0;
+
+ if(ThisTask == 0)
+ {
+ printf("\nglass-making: dmean= %g global disp-maximum= %g rms= %g\n\n",
+ dmean, globmax, sqrt(globdisp2sum / All.TotNumPart));
+ fflush(stdout);
+ }
+
+ for(i = 0, dispmax = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++)
+ {
+ P[i].Vel[j] = 0;
+ P[i].Pos[j] += fac * P[i].GravAccel[j] * 2.0 / (3 * All.Hubble * All.Hubble);
+ P[i].GravAccel[j] = 0;
+ }
+ }
+#endif
+
+
+ /* Now assign new timesteps and kick */
+
+#ifdef FLEXSTEPS
+ if((All.Ti_Current % (4 * All.PresentMinStep)) == 0)
+ if(All.PresentMinStep < TIMEBASE)
+ All.PresentMinStep *= 2;
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+ ti_step = get_timestep(i, &aphys, 0);
+
+ /* make it a power 2 subdivision */
+ ti_min = TIMEBASE;
+ while(ti_min > ti_step)
+ ti_min >>= 1;
+ ti_step = ti_min;
+
+ if(ti_step < All.PresentMinStep)
+ All.PresentMinStep = ti_step;
+ }
+ }
+
+ ti_step = All.PresentMinStep;
+ MPI_Allreduce(&ti_step, &All.PresentMinStep, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+
+ if(dt_displacement < All.MaxSizeTimestep)
+ ti_step = (int) (dt_displacement / All.Timebase_interval);
+ else
+ ti_step = (int) (All.MaxSizeTimestep / All.Timebase_interval);
+
+ /* make it a power 2 subdivision */
+ ti_min = TIMEBASE;
+ while(ti_min > ti_step)
+ ti_min >>= 1;
+ All.PresentMaxStep = ti_min;
+
+
+ if(ThisTask == 0)
+ printf("Syn Range = %g PresentMinStep = %d PresentMaxStep = %d \n",
+ (double) All.PresentMaxStep / All.PresentMinStep, All.PresentMinStep, All.PresentMaxStep);
+
+
+#endif
+
+
+ for(i = 0; i < NumPart; i++)
+ {
+ if(P[i].Ti_endstep == All.Ti_Current)
+ {
+ ti_step = get_timestep(i, &aphys, 0);
+
+ /* make it a power 2 subdivision */
+ ti_min = TIMEBASE;
+ while(ti_min > ti_step)
+ ti_min >>= 1;
+ ti_step = ti_min;
+
+#ifdef FLEXSTEPS
+ ti_grp = P[i].FlexStepGrp % All.PresentMaxStep;
+ ti_grp = (ti_grp / All.PresentMinStep) * All.PresentMinStep;
+ ti_step = ((P[i].Ti_endstep + ti_grp + ti_step) / ti_step) * ti_step - (P[i].Ti_endstep + ti_grp);
+#else
+
+#ifdef PSEUDOSYMMETRIC
+ if(P[i].Type != 0)
+ {
+ if(P[i].Ti_endstep > P[i].Ti_begstep)
+ {
+ apred = aphys + ((aphys - P[i].AphysOld) / (P[i].Ti_endstep - P[i].Ti_begstep)) * ti_step;
+ if(fabs(apred - aphys) < 0.5 * aphys)
+ {
+ ti_step2 = get_timestep(i, &apred, -1);
+ ti_min = TIMEBASE;
+ while(ti_min > ti_step2)
+ ti_min >>= 1;
+ ti_step2 = ti_min;
+
+ if(ti_step2 < ti_step)
+ {
+ get_timestep(i, &apred, ti_step);
+ prob =
+ ((apred - aphys) / (aphys - P[i].AphysOld) * (P[i].Ti_endstep -
+ P[i].Ti_begstep)) / ti_step;
+ if(prob < get_random_number(P[i].ID))
+ ti_step /= 2;
+ }
+ else if(ti_step2 > ti_step)
+ {
+ get_timestep(i, &apred, 2 * ti_step);
+ prob =
+ ((apred - aphys) / (aphys - P[i].AphysOld) * (P[i].Ti_endstep -
+ P[i].Ti_begstep)) / ti_step;
+ if(prob < get_random_number(P[i].ID + 1))
+ ti_step *= 2;
+ }
+ }
+ }
+ P[i].AphysOld = aphys;
+ }
+#endif
+
+#ifdef SYNCHRONIZATION
+ if(ti_step > (P[i].Ti_endstep - P[i].Ti_begstep)) /* timestep wants to increase */
+ {
+
+
+ //if(((TIMEBASE - P[i].Ti_endstep) % ti_step) > 0)
+ // ti_step = P[i].Ti_endstep - P[i].Ti_begstep; /* leave at old step */
+
+ while(((TIMEBASE - P[i].Ti_endstep) % ti_step) > 0) /* yr : allow to increase */
+ ti_step = ti_step/2;
+
+
+
+ }
+#endif
+#endif /* end of FLEXSTEPS */
+
+ if(All.Ti_Current == TIMEBASE) /* we here finish the last timestep. */
+ ti_step = 0;
+
+ if((TIMEBASE - All.Ti_Current) < ti_step) /* check that we don't run beyond the end */
+ ti_step = TIMEBASE - All.Ti_Current;
+
+ tstart = (P[i].Ti_begstep + P[i].Ti_endstep) / 2; /* midpoint of old step */
+ tend = P[i].Ti_endstep + ti_step / 2; /* midpoint of new step */
+
+ if(All.ComovingIntegrationOn)
+ {
+ dt_entr = (tend - tstart) * All.Timebase_interval;
+ dt_entr2 = (tend - P[i].Ti_endstep) * All.Timebase_interval;
+ dt_gravkick = get_gravkick_factor(tstart, tend);
+ dt_hydrokick = get_hydrokick_factor(tstart, tend);
+ dt_gravkick2 = get_gravkick_factor(P[i].Ti_endstep, tend);
+ dt_hydrokick2 = get_hydrokick_factor(P[i].Ti_endstep, tend);
+ }
+ else
+ {
+ dt_entr = dt_gravkick = dt_hydrokick = (tend - tstart) * All.Timebase_interval;
+ dt_gravkick2 = dt_hydrokick2 = dt_entr2 = (tend - P[i].Ti_endstep) * All.Timebase_interval;
+ }
+
+ P[i].Ti_begstep = P[i].Ti_endstep;
+ P[i].Ti_endstep = P[i].Ti_begstep + ti_step;
+
+
+
+
+
+#ifdef CYLINDRICAL_SYMMETRY
+
+ double r,factor;
+
+ r = sqrt( P[i].Pos[0]*P[i].Pos[0] + P[i].Pos[1]*P[i].Pos[1] + P[i].Pos[2]*P[i].Pos[2] );
+
+ factor = 1/(r*r) * (P[i].Pos[0]*P[i].GravAccel[0] + P[i].Pos[1]*P[i].GravAccel[1]);
+
+ P[i].GravAccel[0] = factor * P[i].Pos[0];
+ P[i].GravAccel[1] = factor * P[i].Pos[1];
+
+
+#endif
+
+ /* do the kick */
+
+ for(j = 0; j < 3; j++)
+ {
+
+ dv[j] = 0.0;
+
+#ifdef LIMIT_DVEL
+ if (fabs(P[i].GravAccel[j] * dt_gravkick)>LIMIT_DVEL)
+ {
+#ifdef MULTIPHASE
+ printf("Warning(LIMIT_DVEL): ID=%d j=%d dv[j]=%g Phase=%d(setting GravAccel[j] to 0.0)\n",P[i].ID,j,P[i].GravAccel[j]*dt_hydrokick,SphP[i].Phase);
+#else
+ printf("Warning(LIMIT_DVEL): ID=%d j=%d dv[j]=%g Phase=-(setting GravAccel[j] to 0.0)\n",P[i].ID,j,P[i].GravAccel[j]*dt_hydrokick);
+#endif
+ P[i].GravAccel[j] = 0.0;
+ }
+#endif
+
+ dv[j] += P[i].GravAccel[j] * dt_gravkick;
+ P[i].Vel[j] += P[i].GravAccel[j] * dt_gravkick;
+ }
+
+ if(P[i].Type == 0) /* SPH stuff */
+ {
+ for(j = 0; j < 3; j++)
+ {
+
+#ifdef LIMIT_DVEL /* begin LIMIT_DVEL */
+ if (fabs(SphP[i].HydroAccel[j] * dt_hydrokick)>LIMIT_DVEL)
+ {
+#ifdef MULTIPHASE
+ printf("Warning(LIMIT_DVEL): ID=%d j=%d dv[j]=%g Phase=%d(setting HydroAccel[j] to 0.0)\n",P[i].ID,j,SphP[i].HydroAccel[j] *dt_hydrokick,SphP[i].Phase);
+#else
+ printf("Warning(LIMIT_DVEL): ID=%d j=%d dv[j]=%g Phase=-(setting HydroAccel[j] to 0.0)\n",P[i].ID,j,SphP[i].HydroAccel[j] *dt_hydrokick);
+#endif
+ SphP[i].HydroAccel[j] = 0.0;
+ }
+
+#endif /* end LIMIT_DVEL */
+
+
+ dv[j] += SphP[i].HydroAccel[j] * dt_hydrokick;
+ P[i].Vel[j] += SphP[i].HydroAccel[j] * dt_hydrokick;
+
+
+ SphP[i].VelPred[j] =
+ P[i].Vel[j] - dt_gravkick2 * P[i].GravAccel[j] - dt_hydrokick2 * SphP[i].HydroAccel[j];
+#ifdef PMGRID
+ SphP[i].VelPred[j] += P[i].GravPM[j] * dt_gravkickB;
+#endif
+ }
+
+
+ /***********************************************************/
+ /* compute spec energy lost/win by different other process */
+ /***********************************************************/
+
+
+
+ /***********************************************************/
+ /* compute entropy variation */
+ /***********************************************************/
+
+
+
+ /*******************************/
+ /* compute cooling */
+ /*******************************/
+#ifdef COOLING
+ t2 = second();
+ CoolingForOne(i,tstart,tend,a3inv,hubble_a);
+ t3 = second();
+ All.CPU_Cooling += timediff(t2, t3);
+#else
+
+
+ /* In case of cooling, we prevent that the entropy (and
+ hence temperature decreases by more than a factor 0.5 */
+
+ if(SphP[i].DtEntropy * dt_entr > -0.5 * SphP[i].Entropy)
+ SphP[i].Entropy += SphP[i].DtEntropy * dt_entr;
+ else
+ SphP[i].Entropy *= 0.5;
+
+
+#ifdef MULTIPHASE
+ if (SphP[i].Phase==GAS_SPH)
+ {
+#endif
+
+ if(All.MinEgySpec)
+ {
+ minentropy = All.MinEgySpec * GAMMA_MINUS1 / pow(SphP[i].Density * a3inv, GAMMA_MINUS1);
+ if(SphP[i].Entropy < minentropy)
+ {
+ SphP[i].Entropy = minentropy;
+ SphP[i].DtEntropy = 0;
+ }
+ }
+
+#ifdef MULTIPHASE
+ }
+#endif
+
+
+
+#endif /* COOLING */
+
+
+
+
+
+ /* In case the timestep increases in the new step, we
+ make sure that we do not 'overcool' when deriving
+ predicted temperatures. The maximum timespan over
+ which prediction can occur is ti_step/2, i.e. from
+ the middle to the end of the current step */
+
+ //dt_entr = ti_step / 2 * All.Timebase_interval;
+ dt_entr = imax(ti_step / 2,1) * All.Timebase_interval; /* yr : prevent dt_entr to be zero if ti_step=1 */
+
+ if(SphP[i].Entropy + SphP[i].DtEntropy * dt_entr < 0.5 * SphP[i].Entropy)
+ SphP[i].DtEntropy = -0.5 * SphP[i].Entropy / dt_entr;
+
+
+#ifdef ENTROPYPRED
+ /* now, we correct the predicted Entropy */
+ SphP[i].EntropyPred = SphP[i].Entropy - dt_entr2 * SphP[i].DtEntropy ;
+#ifdef CHECK_ENTROPY_SIGN
+ if ((SphP[i].EntropyPred < 0))
+ {
+ printf("\ntask=%d: EntropyPred less than zero in advance_and_find_timesteps !\n", ThisTask);
+ printf("ID=%d Entropy=%g EntropyPred=%g DtEntropy=%g dt_entr=%g\n",P[i].ID,SphP[i].Entropy,SphP[i].EntropyPred,SphP[i].DtEntropy,dt_entr);
+ fflush(stdout);
+ endrun(1010101000);
+
+ }
+#endif
+
+#endif
+
+ }
+
+ /* if tree is not going to be reconstructed, kick parent nodes dynamically.
+ */
+ if(All.NumForcesSinceLastDomainDecomp < All.TotNumPart * All.TreeDomainUpdateFrequency)
+ {
+ no = Father[i];
+ while(no >= 0)
+ {
+ for(j = 0; j < 3; j++)
+ Extnodes[no].vs[j] += dv[j] * P[i].Mass / Nodes[no].u.d.mass;
+
+ no = Nodes[no].u.d.father;
+ }
+ }
+ }
+ }
+
+
+
+#ifdef PMGRID
+ if(All.PM_Ti_endstep == All.Ti_Current) /* need to do long-range kick */
+ {
+ ti_step = TIMEBASE;
+ while(ti_step > (dt_displacement / All.Timebase_interval))
+ ti_step >>= 1;
+
+ if(ti_step > (All.PM_Ti_endstep - All.PM_Ti_begstep)) /* PM-timestep wants to increase */
+ {
+ /* we only increase if an integer number of steps will bring us to the end */
+ if(((TIMEBASE - All.PM_Ti_endstep) % ti_step) > 0)
+ ti_step = All.PM_Ti_endstep - All.PM_Ti_begstep; /* leave at old step */
+ }
+
+ if(All.Ti_Current == TIMEBASE) /* we here finish the last timestep. */
+ ti_step = 0;
+
+ tstart = (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2;
+ tend = All.PM_Ti_endstep + ti_step / 2;
+
+ if(All.ComovingIntegrationOn)
+ dt_gravkick = get_gravkick_factor(tstart, tend);
+ else
+ dt_gravkick = (tend - tstart) * All.Timebase_interval;
+
+ All.PM_Ti_begstep = All.PM_Ti_endstep;
+ All.PM_Ti_endstep = All.PM_Ti_begstep + ti_step;
+
+ if(All.ComovingIntegrationOn)
+ dt_gravkickB = -get_gravkick_factor(All.PM_Ti_begstep, (All.PM_Ti_begstep + All.PM_Ti_endstep) / 2);
+ else
+ dt_gravkickB =
+ -((All.PM_Ti_begstep + All.PM_Ti_endstep) / 2 - All.PM_Ti_begstep) * All.Timebase_interval;
+
+ for(i = 0; i < NumPart; i++)
+ {
+ for(j = 0; j < 3; j++) /* do the kick */
+ P[i].Vel[j] += P[i].GravPM[j] * dt_gravkick;
+
+ if(P[i].Type == 0)
+ {
+ if(All.ComovingIntegrationOn)
+ {
+ dt_gravkickA = get_gravkick_factor(P[i].Ti_begstep, All.Ti_Current) -
+ get_gravkick_factor(P[i].Ti_begstep, (P[i].Ti_begstep + P[i].Ti_endstep) / 2);
+ dt_hydrokick = get_hydrokick_factor(P[i].Ti_begstep, All.Ti_Current) -
+ get_hydrokick_factor(P[i].Ti_begstep, (P[i].Ti_begstep + P[i].Ti_endstep) / 2);
+ }
+ else
+ dt_gravkickA = dt_hydrokick =
+ (All.Ti_Current - (P[i].Ti_begstep + P[i].Ti_endstep) / 2) * All.Timebase_interval;
+
+ for(j = 0; j < 3; j++)
+ SphP[i].VelPred[j] = P[i].Vel[j]
+ + P[i].GravAccel[j] * dt_gravkickA
+ + SphP[i].HydroAccel[j] * dt_hydrokick
+ + P[i].GravPM[j] * dt_gravkickB;
+ }
+ }
+ }
+#endif
+
+ t1 = second();
+ All.CPU_TimeLine += timediff(t0, t1);
+
+#ifdef DETAILED_CPU
+ All.CPU_Leapfrog += timediff(t0, t1);
+#endif
+
+
+#ifdef COOLING
+ //All.CPU_TimeLine -= All.CPU_Cooling;
+#endif
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ if(SetMinTimeStepForActives)
+ SetMinTimeStepForActives=0;
+#endif
+
+
+}
+
+
+
+
+/*! This function normally (for flag==0) returns the maximum allowed timestep
+ * of a particle, expressed in terms of the integer mapping that is used to
+ * represent the total simulated timespan. The physical acceleration is
+ * returned in `aphys'. The latter is used in conjunction with the
+ * PSEUDOSYMMETRIC integration option, which also makes of the second
+ * function of get_timestep. When it is called with a finite timestep for
+ * flag, it returns the physical acceleration that would lead to this
+ * timestep, assuming timestep criterion 0.
+ */
+int get_timestep(int p, /*!< particle index */
+ double *aphys, /*!< acceleration (physical units) */
+ int flag /*!< either 0 for normal operation, or finite timestep to get corresponding
+ aphys */ )
+{
+ double ax, ay, az, ac, csnd;
+ double dt = 0, dt_courant = 0, dt_accel;
+ int ti_step;
+
+#ifdef CONDUCTION
+ double dt_cond;
+#endif
+
+ if(flag == 0)
+ {
+ ax = fac1 * P[p].GravAccel[0];
+ ay = fac1 * P[p].GravAccel[1];
+ az = fac1 * P[p].GravAccel[2];
+
+#ifdef PMGRID
+ ax += fac1 * P[p].GravPM[0];
+ ay += fac1 * P[p].GravPM[1];
+ az += fac1 * P[p].GravPM[2];
+#endif
+
+ if(P[p].Type == 0)
+ {
+ ax += fac2 * SphP[p].HydroAccel[0];
+ ay += fac2 * SphP[p].HydroAccel[1];
+ az += fac2 * SphP[p].HydroAccel[2];
+ }
+
+ ac = sqrt(ax * ax + ay * ay + az * az); /* this is now the physical acceleration */
+ *aphys = ac;
+ }
+ else
+ ac = *aphys;
+
+ if(ac == 0)
+ ac = 1.0e-30;
+
+ switch (All.TypeOfTimestepCriterion)
+ {
+ case 0:
+ if(flag > 0)
+ {
+ dt = flag * All.Timebase_interval;
+ dt /= hubble_a; /* convert dloga to physical timestep */
+ ac = 2 * All.ErrTolIntAccuracy * atime * All.SofteningTable[P[p].Type] / (dt * dt);
+ *aphys = ac;
+ return flag;
+ }
+ dt = dt_accel = sqrt(2 * All.ErrTolIntAccuracy * atime * All.SofteningTable[P[p].Type] / ac);
+
+
+
+
+#ifdef ADAPTIVE_GRAVSOFT_FORGAS
+ if(P[p].Type == 0)
+ dt = dt_accel = sqrt(2 * All.ErrTolIntAccuracy * atime * SphP[p].Hsml / 2.8 / ac);
+#endif
+ break;
+ default:
+ endrun(888);
+ break;
+ }
+
+ if(P[p].Type == 0)
+ {
+ csnd = sqrt(GAMMA * SphP[p].Pressure / SphP[p].Density);
+
+ if(All.ComovingIntegrationOn)
+ dt_courant = 2 * All.CourantFac * All.Time * SphP[p].Hsml / (fac3 * SphP[p].MaxSignalVel);
+ else
+ dt_courant = 2 * All.CourantFac * SphP[p].Hsml / SphP[p].MaxSignalVel;
+
+ if(dt_courant < dt)
+#ifndef MULTIPHASE
+ dt = dt_courant;
+#else
+ {
+ if (SphP[p].MaxSignalVel != 0);
+ dt = dt_courant;
+ }
+#endif
+
+
+
+#ifdef CHIMIE_THERMAL_FEEDBACK
+
+ float f;
+ double EgySpec,NewEgySpec;
+
+ if (SphP[p].DeltaEgySpec > 0)
+ {
+
+ /* spec energy at current step */
+ EgySpec = SphP[p].EntropyPred / GAMMA_MINUS1 * pow(SphP[p].Density, GAMMA_MINUS1);
+
+ /* new egyspec */
+ NewEgySpec = EgySpec + SphP[p].DeltaEgySpec;
+
+ f = NewEgySpec/EgySpec;
+
+ //if (f>1)
+ // dt = dt / f;
+ }
+
+#endif
+
+
+
+#ifdef FEEDBACK_WIND
+ double dt_feedback_wind;
+ double vwind;
+
+ vwind = sqrt( SphP[p].FeedbackWindVel[0]*SphP[p].FeedbackWindVel[0] + SphP[p].FeedbackWindVel[1]*SphP[p].FeedbackWindVel[1] + SphP[p].FeedbackWindVel[2]*SphP[p].FeedbackWindVel[2] );
+
+ if (vwind > 0)
+ {
+ dt_feedback_wind = All.SupernovaWindIntAccuracy * All.SofteningTable[P[p].Type] / vwind;
+
+ SphP[p].FeedbackWindVel[0]=0;
+ SphP[p].FeedbackWindVel[1]=0;
+ SphP[p].FeedbackWindVel[2]=0;
+
+ if(dt_feedback_wind < dt)
+ dt = dt_feedback_wind;
+ }
+#endif
+
+ }
+
+
+#ifdef CHIMIE
+ int m;
+ double dt_chimie;
+
+ if(P[p].Type == ST)
+ {
+ //m = P[p].StPIdx;
+ //if (StP[m].Flag)
+ {
+ dt_chimie = All.ChimieMaxSizeTimestep;
+ }
+ if(dt_chimie < dt)
+ dt = dt_chimie;
+ }
+#endif
+
+
+ /* convert the physical timestep to dloga if needed. Note: If comoving integration has not been selected,
+ hubble_a=1.
+ */
+ dt *= hubble_a;
+
+ if(dt >= All.MaxSizeTimestep)
+ dt = All.MaxSizeTimestep;
+
+ if(dt >= dt_displacement)
+ dt = dt_displacement;
+
+ if(dt < All.MinSizeTimestep)
+ {
+#ifndef NOSTOP_WHEN_BELOW_MINTIMESTEP
+ printf("warning: Timestep wants to be below the limit `MinSizeTimestep'\n");
+
+ if(P[p].Type == 0)
+ {
+ printf
+ ("Part-ID=%d dt=%g dtc=%g ac=%g xyz=(%g|%g|%g) hsml=%g maxsignalvel=%g dt0=%g eps=%g\n",
+ (int) P[p].ID, dt, dt_courant * hubble_a, ac, P[p].Pos[0], P[p].Pos[1], P[p].Pos[2],
+ SphP[p].Hsml, SphP[p].MaxSignalVel,
+ sqrt(2 * All.ErrTolIntAccuracy * atime * All.SofteningTable[P[p].Type] / ac) * hubble_a,
+ All.SofteningTable[P[p].Type]);
+ }
+ else
+ {
+ printf("Part-ID=%d dt=%g ac=%g xyz=(%g|%g|%g)\n", (int) P[p].ID, dt, ac, P[p].Pos[0], P[p].Pos[1],
+ P[p].Pos[2]);
+ }
+ fflush(stdout);
+ endrun(888);
+#endif
+ dt = All.MinSizeTimestep;
+ }
+
+ ti_step = dt / All.Timebase_interval;
+
+
+#ifdef CHIMIE_KINETIC_FEEDBACK
+ //if(SetMinTimeStepForActives)
+ // ti_step=1;
+#endif
+
+ if(!(ti_step > 0 && ti_step < TIMEBASE))
+ {
+ printf("\nError: A timestep of size zero was assigned on the integer timeline!\n"
+ "We better stop.\n"
+ "Task=%d Part-ID=%d dt=%g tibase=%g ti_step=%d ac=%g xyz=(%g|%g|%g) tree=(%g|%g%g)\n\n",
+ ThisTask, (int) P[p].ID, dt, All.Timebase_interval, ti_step, ac,
+ P[p].Pos[0], P[p].Pos[1], P[p].Pos[2], P[p].GravAccel[0], P[p].GravAccel[1], P[p].GravAccel[2]);
+#ifdef PMGRID
+ printf("pm_force=(%g|%g|%g)\n", P[p].GravPM[0], P[p].GravPM[1], P[p].GravPM[2]);
+#endif
+ if(P[p].Type == 0)
+ printf("hydro-frc=(%g|%g|%g)\n", SphP[p].HydroAccel[0], SphP[p].HydroAccel[1], SphP[p].HydroAccel[2]);
+#ifdef FEEDBACK_WIND
+ if(P[p].Type == 0)
+ printf("feedback-vel=(%g|%g|%g)\n", SphP[p].FeedbackWindVel[0], SphP[p].FeedbackWindVel[1], SphP[p].FeedbackWindVel[2]);
+#endif
+
+
+ fflush(stdout);
+ endrun(818);
+ }
+
+ return ti_step;
+}
+
+
+/*! This function computes an upper limit ('dt_displacement') to the global
+ * timestep of the system based on the rms velocities of particles. For
+ * cosmological simulations, the criterion used is that the rms displacement
+ * should be at most a fraction MaxRMSDisplacementFac of the mean particle
+ * separation. Note that the latter is estimated using the assigned particle
+ * masses, separately for each particle type. If comoving integration is not
+ * used, the function imposes no constraint on the timestep.
+ */
+void find_dt_displacement_constraint(double hfac /*!< should be a^2*H(a) */ )
+{
+ int i, j, type, *temp;
+ int count[6];
+ long long count_sum[6];
+ double v[6], v_sum[6], mim[6], min_mass[6];
+ double dt, dmean, asmth = 0;
+
+ dt_displacement = All.MaxSizeTimestep;
+
+ if(All.ComovingIntegrationOn)
+ {
+ for(type = 0; type < 6; type++)
+ {
+ count[type] = 0;
+ v[type] = 0;
+ mim[type] = 1.0e30;
+ }
+
+ for(i = 0; i < NumPart; i++)
+ {
+ v[P[i].Type] += P[i].Vel[0] * P[i].Vel[0] + P[i].Vel[1] * P[i].Vel[1] + P[i].Vel[2] * P[i].Vel[2];
+ if(mim[P[i].Type] > P[i].Mass)
+ mim[P[i].Type] = P[i].Mass;
+ count[P[i].Type]++;
+ }
+
+ MPI_Allreduce(v, v_sum, 6, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+ MPI_Allreduce(mim, min_mass, 6, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
+
+ temp = malloc(NTask * 6 * sizeof(int));
+ MPI_Allgather(count, 6, MPI_INT, temp, 6, MPI_INT, MPI_COMM_WORLD);
+ for(i = 0; i < 6; i++)
+ {
+ count_sum[i] = 0;
+ for(j = 0; j < NTask; j++)
+ count_sum[i] += temp[j * 6 + i];
+ }
+ free(temp);
+
+ for(type = 0; type < 6; type++)
+ {
+ if(count_sum[type] > 0)
+ {
+ if(type == 0)
+ dmean =
+ pow(min_mass[type] / (All.OmegaBaryon * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G)),
+ 1.0 / 3);
+ else
+ dmean =
+ pow(min_mass[type] /
+ ((All.Omega0 - All.OmegaBaryon) * 3 * All.Hubble * All.Hubble / (8 * M_PI * All.G)),
+ 1.0 / 3);
+
+ dt = All.MaxRMSDisplacementFac * hfac * dmean / sqrt(v_sum[type] / count_sum[type]);
+
+#ifdef PMGRID
+ asmth = All.Asmth[0];
+#ifdef PLACEHIGHRESREGION
+ if(((1 << type) & (PLACEHIGHRESREGION)))
+ asmth = All.Asmth[1];
+#endif
+ if(asmth < dmean)
+ dt = All.MaxRMSDisplacementFac * hfac * asmth / sqrt(v_sum[type] / count_sum[type]);
+#endif
+
+ if(ThisTask == 0)
+ printf("type=%d dmean=%g asmth=%g minmass=%g a=%g sqrt(<p^2>)=%g dlogmax=%g\n",
+ type, dmean, asmth, min_mass[type], All.Time, sqrt(v_sum[type] / count_sum[type]), dt);
+
+ if(dt < dt_displacement)
+ dt_displacement = dt;
+ }
+ }
+
+ if(ThisTask == 0)
+ printf("displacement time constraint: %g (%g)\n", dt_displacement, All.MaxSizeTimestep);
+ }
+}

Event Timeline

c4science · Help