diff --git a/Makefile b/Makefile index 8e45789..916a24b 100644 --- a/Makefile +++ b/Makefile @@ -1,138 +1,138 @@ include local/dirs.inc include local/make.inc EXEC = $(BINDIR)/helaz -F90 = mpif90 +F90 = mpiifort # Add Multiple-Precision Library EXTLIBS += -L$(FMDIR)/lib EXTINC += -I$(FMDIR)/mod EXTLIBS += -L$(FFTWDIR)/lib64 EXTINC += -I$(FFTWDIR)/include all: dirs src/srcinfo.h $(EXEC) run: all (cd wk; $(EXEC);) dirs: mkdir -p $(BINDIR) mkdir -p $(OBJDIR) mkdir -p $(MODDIR) src/srcinfo.h: ( cd src/srcinfo; $(MAKE);) clean: cleanobj cleanmod @rm -f src/srcinfo.h @rm -f src/srcinfo/srcinfo.h cleanobj: @rm -f $(OBJDIR)/*o cleanmod: @rm -f $(MODDIR)/*mod @rm -f *.mod cleanbin: @rm -f $(EXEC) $(OBJDIR)/diagnose.o : src/srcinfo.h FOBJ=$(OBJDIR)/advance_field.o $(OBJDIR)/array_mod.o $(OBJDIR)/auxval.o $(OBJDIR)/basic_mod.o \ $(OBJDIR)/coeff_mod.o $(OBJDIR)/compute_Sapj.o $(OBJDIR)/control.o $(OBJDIR)/fourier_mod.o \ $(OBJDIR)/diagnose.o $(OBJDIR)/diagnostics_par_mod.o $(OBJDIR)/endrun.o $(OBJDIR)/fields_mod.o \ $(OBJDIR)/inital.o $(OBJDIR)/initial_par_mod.o $(OBJDIR)/main.o $(OBJDIR)/memory.o \ $(OBJDIR)/model_mod.o $(OBJDIR)/moments_eq_rhs.o $(OBJDIR)/poisson.o \ $(OBJDIR)/ppexit.o $(OBJDIR)/ppinit.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/readinputs.o \ $(OBJDIR)/grid_mod.o $(OBJDIR)/stepon.o $(OBJDIR)/tesend.o $(OBJDIR)/time_integration_mod.o \ $(OBJDIR)/utility_mod.o $(EXEC): $(FOBJ) $(F90) $(LDFLAGS) $(OBJDIR)/*.o $(EXTMOD) $(EXTINC) $(EXTLIBS) -o $@ $(OBJDIR)/advance_field.o : src/advance_field.F90 $(OBJDIR)/grid_mod.o $(OBJDIR)/array_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/advance_field.F90 -o $@ $(OBJDIR)/array_mod.o : src/array_mod.F90 $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/array_mod.F90 -o $@ $(OBJDIR)/auxval.o : src/auxval.F90 $(OBJDIR)/fourier_mod.o $(OBJDIR)/memory.o $(OBJDIR)/model_mod.o $(OBJDIR)/grid_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/auxval.F90 -o $@ $(OBJDIR)/basic_mod.o : src/basic_mod.F90 $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/basic_mod.F90 -o $@ $(OBJDIR)/coeff_mod.o : src/coeff_mod.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/coeff_mod.F90 -o $@ $(OBJDIR)/compute_Sapj.o : src/compute_Sapj.F90 $(OBJDIR)/array_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/fourier_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/time_integration_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/compute_Sapj.F90 -o $@ $(OBJDIR)/control.o : src/control.F90 $(OBJDIR)/auxval.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/ppexit.o $(OBJDIR)/ppinit.o $(OBJDIR)/readinputs.o $(OBJDIR)/tesend.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/control.F90 -o $@ $(OBJDIR)/fourier_mod.o : src/fourier_mod.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/fourier_mod.F90 -o $@ $(OBJDIR)/diagnose.o : src/diagnose.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/array_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/diagnostics_par_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/initial_par_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/time_integration_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/diagnose.F90 -o $@ $(OBJDIR)/diagnostics_par_mod.o : src/diagnostics_par_mod.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/diagnostics_par_mod.F90 -o $@ $(OBJDIR)/endrun.o : src/endrun.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/endrun.F90 -o $@ $(OBJDIR)/fields_mod.o : src/fields_mod.F90 $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/fields_mod.F90 -o $@ $(OBJDIR)/grid_mod.o : src/grid_mod.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/grid_mod.F90 -o $@ $(OBJDIR)/inital.o : src/inital.F90 $(OBJDIR)/array_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/initial_par_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/poisson.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/utility_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/inital.F90 -o $@ $(OBJDIR)/initial_par_mod.o : src/initial_par_mod.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/initial_par_mod.F90 -o $@ $(OBJDIR)/main.o : src/main.F90 $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/main.F90 -o $@ $(OBJDIR)/memory.o : src/memory.F90 $ $(OBJDIR)/array_mod.o $(OBJDIR)/basic_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/grid_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/memory.F90 -o $@ $(OBJDIR)/model_mod.o : src/model_mod.F90 $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/model_mod.F90 -o $@ $(OBJDIR)/moments_eq_rhs.o : src/moments_eq_rhs.F90 $(OBJDIR)/array_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/time_integration_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/moments_eq_rhs.F90 -o $@ $(OBJDIR)/poisson.o : src/poisson.F90 $(OBJDIR)/array_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/array_mod.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/poisson.F90 -o $@ $(OBJDIR)/ppexit.o : src/ppexit.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/ppexit.F90 -o $@ $(OBJDIR)/ppinit.o : src/ppinit.F90 $(OBJDIR)/array_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/array_mod.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/basic_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/ppinit.F90 -o $@ $(OBJDIR)/prec_const_mod.o : src/prec_const_mod.F90 $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/prec_const_mod.F90 -o $@ $(OBJDIR)/readinputs.o : src/readinputs.F90 $(OBJDIR)/diagnostics_par_mod.o $(OBJDIR)/initial_par_mod.o $(OBJDIR)/model_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/time_integration_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/readinputs.F90 -o $@ $(OBJDIR)/stepon.o : src/stepon.F90 $(OBJDIR)/prec_const_mod.o $(OBJDIR)/advance_field.o $(OBJDIR)/basic_mod.o $(OBJDIR)/grid_mod.o $(OBJDIR)/array_mod.o $(OBJDIR)/fields_mod.o $(OBJDIR)/moments_eq_rhs.o $(OBJDIR)/poisson.o $(OBJDIR)/time_integration_mod.o $(OBJDIR)/utility_mod.o $(OBJDIR)/model_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/stepon.F90 -o $@ $(OBJDIR)/tesend.o : src/tesend.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/tesend.F90 -o $@ $(OBJDIR)/time_integration_mod.o : src/time_integration_mod.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/prec_const_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/time_integration_mod.F90 -o $@ $(OBJDIR)/utility_mod.o : src/utility_mod.F90 $(OBJDIR)/basic_mod.o $(OBJDIR)/prec_const_mod.o $(OBJDIR)/grid_mod.o $(F90) -c $(F90FLAGS) $(FPPFLAGS) $(EXTMOD) $(EXTINC) src/utility_mod.F90 -o $@ diff --git a/local/dirs.inc b/local/dirs.inc index ab2256e..2850da9 100644 --- a/local/dirs.inc +++ b/local/dirs.inc @@ -1,13 +1,17 @@ #Local code, binaries, pputils library PREFIX = $(HOME)/Documents/HeLaZ#write the path to the root of your HeLaZ distrib here SRCDIR = $(PREFIX)/src BINDIR = $(PREFIX)/bin OBJDIR = $(PREFIX)/obj LIBDIR = $(PREFIX)/lib MODDIR = $(PREFIX)/mod + FMDIR = $(PREFIX)/FM FFTW3DIR = /home/ahoffman/lib/fftw-3.3.8 -INTEL_LOCAL = /home/ahoffman/local/spcpc/intel -FUTILS = /home/ahoffman/lib/futils_1.4/src +FUTILS_DIR = /home/ahoffman/local/spcpc/intel + +#FMDIR = (HOME)/lib/FM +#FFTW3DIR = $(HOME)/lib/fftw-3.3.8 +#FUTILS_DIR = $(HOME)/lib/futils # Naming ideas : HeLaZ, MoNoLiT (Moment Non Linear Torroidal) diff --git a/local/make.inc b/local/make.inc index 0aec7e2..db25768 100644 --- a/local/make.inc +++ b/local/make.inc @@ -1,95 +1,95 @@ ################################################################ # # Section I: Compiler options # ################################################################ #Default optimization level (O=optimized, g=debug) OPTLEVEL = O F90FLAGS = CFLAGS = ifeq ($(OPTLEVEL), O) #optimized ifeq ($(COMPTYPE), i) #intel F90FLAGS += -O3 -xHOST endif ifeq ($(COMPTYPE), g) #gnu F90FLAGS += -ffree-line-length-0 -O3 endif ifeq ($(COMPTYPE), c) #cray F90FLAGS += endif endif ifeq ($(OPTLEVEL), g) #debug ifeq ($(COMPTYPE), i) #intel F90FLAGS += -g -traceback -CB endif ifeq ($(COMPTYPE), g) #gnu F90FLAGS += -ffree-line-length-0 -g -fbacktrace -fcheck=all -pedantic -Wall endif ifeq ($(COMPTYPE), c) #cray F90FLAGS += -g -O0 endif endif ifeq ($(USEOPENMP), 1) ifeq ($(COMPTYPE), i) #intel F90FLAGS += -qopenmp CFLAGS += -qopenmp endif ifeq ($(COMPTYPE), g) #gnu F90FLAGS += -fopenmp CFLAGS += -fopenmp endif endif ################################################################ # # Section II: Libraries and where to find them # ################################################################ -IDIRS := -I$(INTEL_LOCAL)/include/$(OPTLEVEL) +IDIRS := -I$(FUTILS_DIR)/include/$(OPTLEVEL) LIBS := -lfutils -lhdf5_fortran -lhdf5 -lz -ldl -lpthread -LDIRS := -L$(INTEL_LOCAL)/lib/$(OPTLEVEL) -L$(HDF5)/lib +LDIRS := -L$(FUTILS_DIR)/lib/$(OPTLEVEL) -L$(HDF5)/lib # Add Multiple-Precision Library LIBS += -lfm ifdef FFTW3DIR LIBS += -lfftw3 -lfftw3_mpi LDIRS += -L$(FFTW3DIR)/lib64 IDIRS += -I$(FFTW3DIR)/include endif ################################################################ # # Section V: Set up inclusion of modules and libraries during # compiling / linking phase # ################################################################ #Flag for finding external modules in MODDIR ifeq ($(COMPTYPE), i) #intel EXTMOD = -module $(MODDIR) endif ifeq ($(COMPTYPE), g) #gnu EXTMOD = -J $(MODDIR) endif ifeq ($(COMPTYPE), c) #cray EXTMOD = -em -J $(MODDIR) endif #Flag for finding external libraries in LDIR EXTLIBS = $(LDIRS) -Wl,--start-group $(LIBS) -Wl,--end-group #Flag for finding external include files EXTINC = $(IDIRS) diff --git a/src/control.F90 b/src/control.F90 index b2c710a..704fa61 100644 --- a/src/control.F90 +++ b/src/control.F90 @@ -1,94 +1,92 @@ SUBROUTINE control ! Control the run use basic use prec_const IMPLICIT NONE CALL cpu_time(start) !________________________________________________________________________________ ! 1. Prologue ! 1.1 Initialize the parallel environment IF (my_id .EQ. 1) WRITE(*,*) 'Initialize MPI...' CALL ppinit IF (my_id .EQ. 1) WRITE(*,'(a/)') '...MPI initialized' CALL daytim('Start at ') ! 1.2 Define data specific to run IF (my_id .EQ. 1) WRITE(*,*) 'Load basic data...' CALL basic_data IF (my_id .EQ. 1) WRITE(*,'(a/)') '...basic data loaded.' ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! 1.3 Read input parameters from input file IF (my_id .EQ. 1) WRITE(*,*) 'Read input parameters...' CALL readinputs IF (my_id .EQ. 1) WRITE(*,'(a/)') '...input parameters read' ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! 1.4 Set auxiliary values (allocate arrays, set grid, ...) IF (my_id .EQ. 1) WRITE(*,*) 'Calculate auxval...' CALL auxval IF (my_id .EQ. 1) WRITE(*,'(a/)') '...auxval calculated' ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! 1.5 Initial conditions IF (my_id .EQ. 1) WRITE(*,*) 'Create initial state...' CALL inital IF (my_id .EQ. 1) WRITE(*,'(a/)') '...initial state created' ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! 1.6 Initial diagnostics IF (my_id .EQ. 1) WRITE(*,*) 'Initial diagnostics...' CALL diagnose(0) IF (my_id .EQ. 1) WRITE(*,'(a/)') '...initial diagnostics done' ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) CALL FLUSH(stdout) !________________________________________________________________________________ IF (my_id .EQ. 1) WRITE(*,*) 'Time integration loop..' !________________________________________________________________________________ ! 2. Main loop DO CALL cpu_time(t0_step) ! Measuring time step = step + 1 cstep = cstep + 1 - ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) CALL stepon - CALL mpi_barrier(MPI_COMM_WORLD, ierr) + CALL mpi_barrier(MPI_COMM_WORLD, ierr) time = time + dt CALL tesend IF( nlend ) EXIT ! exit do loop -CALL cpu_time(t0_diag) ! Measuring time + CALL cpu_time(t0_diag) ! Measuring time CALL diagnose(step) -CALL cpu_time(t1_diag); tc_diag = tc_diag + (t1_diag - t0_diag) + CALL cpu_time(t1_diag); tc_diag = tc_diag + (t1_diag - t0_diag) - ! CALL mpi_barrier(MPI_COMM_WORLD, ierr) -CALL cpu_time(t1_step); tc_step = tc_step + (t1_step - t0_step) + CALL cpu_time(t1_step); tc_step = tc_step + (t1_step - t0_step) END DO IF (my_id .EQ. 1) WRITE(*,'(a/)') '...time integration done' !________________________________________________________________________________ ! 9. Epilogue CALL diagnose(-1) CALL endrun IF (my_id .EQ. 1) CALL daytim('Done at ') CALL ppexit END SUBROUTINE control diff --git a/src/diagnose.F90 b/src/diagnose.F90 index b0885f3..9a7bf99 100644 --- a/src/diagnose.F90 +++ b/src/diagnose.F90 @@ -1,480 +1,478 @@ SUBROUTINE diagnose(kstep) ! Diagnostics, writing simulation state to disk USE basic USE grid USE diagnostics_par USE futils, ONLY: creatf, creatg, creatd, closef, putarr, putfile, attach, openf USE model USE initial_par USE fields USE time_integration USE prec_const IMPLICIT NONE INCLUDE 'srcinfo.h' INTEGER, INTENT(in) :: kstep INTEGER, parameter :: BUFSIZE = 2 INTEGER :: rank, dims(1) = (/0/) INTEGER :: cp_counter = 0 CHARACTER(len=256) :: str, fname,test_ !_____________________________________________________________________________ ! 1. Initial diagnostics IF ((kstep .EQ. 0)) THEN ! Writing output filename WRITE(resfile,'(a,a1,i2.2,a3)') TRIM(resfile0),'_',jobnum,'.h5' ! 1.1 Initial run ! Main output file creation IF (write_doubleprecision) THEN CALL creatf(resfile, fidres, real_prec='d', mpicomm=MPI_COMM_WORLD) ELSE CALL creatf(resfile, fidres, mpicomm=MPI_COMM_WORLD) END IF IF (my_id .EQ. 0) WRITE(*,'(3x,a,a)') TRIM(resfile), ' created' ! Checkpoint file creation WRITE(rstfile,'(a,a1,i2.2,a3)') TRIM(rstfile0),'_',jobnum,'.h5' CALL creatf(rstfile, fidrst, real_prec='d', mpicomm=MPI_COMM_WORLD) CALL creatg(fidrst, '/Basic', 'Basic data') CALL creatg(fidrst, '/Basic/moments_e', 'electron moments') CALL creatg(fidrst, '/Basic/moments_i', 'ion moments') IF (my_id .EQ. 0) WRITE(*,'(3x,a,a)') TRIM(rstfile), ' created' CALL flush(6) ! Data group CALL creatg(fidres, "/data", "data") CALL creatg(fidres, "/data/var2d", "2d profiles") CALL creatg(fidres, "/data/var5d", "5d profiles") ! Initialize counter of number of saves for each category IF (cstep==0) THEN iframe2d=0 ENDIF CALL attach(fidres,"/data/var2d/" , "frames", iframe2d) IF (cstep==0) THEN iframe5d=0 END IF CALL attach(fidres,"/data/var5d/" , "frames", iframe5d) ! File group CALL creatg(fidres, "/files", "files") CALL attach(fidres, "/files", "jobnum", jobnum) ! Profiler time measurement CALL creatg(fidres, "/profiler", "performance analysis") CALL creatd(fidres, 0, dims, "/profiler/Tc_rhs", "cumulative rhs computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_adv_field", "cumulative adv. fields computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_poisson", "cumulative poisson computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_Sapj", "cumulative Sapj computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_diag", "cumulative sym computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_checkfield", "cumulative checkfield computation time") CALL creatd(fidres, 0, dims, "/profiler/Tc_step", "cumulative total step computation time") CALL creatd(fidres, 0, dims, "/profiler/time", "current simulation time") ! var2d group (electro. pot., Ni00 moment) rank = 0 CALL creatd(fidres, rank, dims, "/data/var2d/time", "Time t*c_s/R") CALL creatd(fidres, rank, dims, "/data/var2d/cstep", "iteration number") IF (write_Na00) THEN CALL creatg(fidres, "/data/var2d/Ne00", "Ne00") IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var2d/Ne00/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var2d/Ne00/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var2d/Ne00/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) CALL creatg(fidres, "/data/var2d/Ni00", "Ni00") IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var2d/Ni00/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var2d/Ni00/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var2d/Ni00/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) END IF IF (write_phi) THEN CALL creatg(fidres, "/data/var2d/phi", "phi") IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var2d/phi/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var2d/phi/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var2d/phi/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) END IF ! var5d group (moments) rank = 0 CALL creatd(fidres, rank, dims, "/data/var5d/time", "Time t*c_s/R") CALL creatd(fidres, rank, dims, "/data/var5d/cstep", "iteration number") IF (write_moments) THEN CALL creatg(fidres, "/data/var5d/moments_e", "moments_e") CALL putarr(fidres, "/data/var5d/moments_e/coordp", parray_e(ips_e:ipe_e), "p_e", ionode=0) CALL putarr(fidres, "/data/var5d/moments_e/coordj", jarray_e(ijs_e:ije_e), "j_e", ionode=0) IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var5d/moments_e/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var5d/moments_e/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var5d/moments_e/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) CALL creatg(fidres, "/data/var5d/moments_i", "moments_i") CALL putarr(fidres, "/data/var5d/moments_i/coordp", parray_i(ips_i:ipe_i), "p_i", ionode=0) CALL putarr(fidres, "/data/var5d/moments_i/coordj", jarray_i(ijs_i:ije_i), "j_i", ionode=0) IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var5d/moments_i/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var5d/moments_i/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var5d/moments_i/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) END IF IF (write_non_lin) THEN CALL creatg(fidres, "/data/var5d/Sepj", "Sepj") CALL putarr(fidres, "/data/var5d/Sepj/coordp", parray_e(ips_e:ipe_e), "p_e", ionode=0) CALL putarr(fidres, "/data/var5d/Sepj/coordj", jarray_e(ijs_e:ije_e), "j_e", ionode=0) IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var5d/Sepj/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var5d/Sepj/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var5d/Sepj/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) CALL creatg(fidres, "/data/var5d/Sipj", "Sipj") CALL putarr(fidres, "/data/var5d/Sipj/coordp", parray_i(ips_i:ipe_i), "p_i", ionode=0) CALL putarr(fidres, "/data/var5d/Sipj/coordj", jarray_i(ijs_i:ije_i), "j_i", ionode=0) IF (num_procs .EQ. 1) THEN CALL putarr(fidres, "/data/var5d/Sipj/coordkr", krarray(ikrs:ikre), "kr*rho_s0", ionode=0) ELSE CALL putarr(fidres, "/data/var5d/Sipj/coordkr", krarray(ikrs:ikre), "kr*rho_s0", pardim=1) ENDIF CALL putarr(fidres, "/data/var5d/Sipj/coordkz", kzarray(ikzs:ikze), "kz*rho_s0", ionode=0) END IF ! Add input namelist variables as attributes of /data/input, defined in srcinfo.h IF (my_id .EQ. 0) WRITE(*,*) 'VERSION=', VERSION IF (my_id .EQ. 0) WRITE(*,*) 'BRANCH=', BRANCH IF (my_id .EQ. 0) WRITE(*,*) 'AUTHOR=', AUTHOR IF (my_id .EQ. 0) WRITE(*,*) 'HOST=', HOST WRITE(str,'(a,i2.2)') "/data/input" rank=0 CALL creatd(fidres, rank,dims,TRIM(str),'Input parameters') CALL attach(fidres, TRIM(str), "version", VERSION) !defined in srcinfo.h CALL attach(fidres, TRIM(str), "branch", BRANCH) !defined in srcinfo.h CALL attach(fidres, TRIM(str), "author", AUTHOR) !defined in srcinfo.h CALL attach(fidres, TRIM(str), "execdate", EXECDATE) !defined in srcinfo.h CALL attach(fidres, TRIM(str), "host", HOST) !defined in srcinfo.h CALL attach(fidres, TRIM(str), "start_time", time) CALL attach(fidres, TRIM(str), "start_cstep", cstep-1) CALL attach(fidres, TRIM(str), "start_iframe2d", iframe2d) CALL attach(fidres, TRIM(str), "start_iframe5d", iframe5d) CALL attach(fidres, TRIM(str), "dt", dt) CALL attach(fidres, TRIM(str), "tmax", tmax) CALL attach(fidres, TRIM(str), "nrun", nrun) CALL attach(fidres, TRIM(str), "cpu_time", -1) CALL grid_outputinputs(fidres, str) CALL output_par_outputinputs(fidres, str) CALL model_outputinputs(fidres, str) CALL initial_outputinputs(fidres, str) CALL time_integration_outputinputs(fidres, str) ! Save STDIN (input file) of this run IF(jobnum .LE. 99) THEN WRITE(str,'(a,i2.2)') "/files/STDIN.",jobnum ELSE WRITE(str,'(a,i3.2)') "/files/STDIN.",jobnum END IF INQUIRE(unit=lu_in, name=fname) CLOSE(lu_in) CALL putfile(fidres, TRIM(str), TRIM(fname),ionode=0) ELSEIF((kstep .EQ. 0)) THEN IF(jobnum .LE. 99) THEN WRITE(resfile,'(a,a1,i2.2,a3)') TRIM(resfile0),'_',jobnum,'.h5' ELSE WRITE(resfile,'(a,a1,i3.2,a3)') TRIM(resfile0),'_',jobnum,'.h5' END IF CALL openf(resfile,fidres, 'D'); ENDIF !_____________________________________________________________________________ ! 2. Periodic diagnostics ! IF (kstep .GE. 0) THEN ! Terminal info IF (MOD(cstep, INT(1.0/dt)) == 0 .AND. (my_id .EQ. 0)) THEN WRITE(*,"(F5.0,A,F5.0)") time,"/",tmax ENDIF ! 2.1 0d history arrays IF (nsave_0d .NE. 0) THEN IF ( MOD(cstep, nsave_0d) == 0 ) THEN CALL diagnose_0d END IF END IF ! 2.2 1d profiles ! empty in our case ! 2.3 2d profiles IF (nsave_2d .NE. 0) THEN IF (MOD(cstep, nsave_2d) == 0) THEN CALL diagnose_2d END IF END IF ! 2.4 3d profiles IF (nsave_5d .NE. 0) THEN IF (MOD(cstep, nsave_5d) == 0) THEN CALL diagnose_5d END IF END IF ! 2.5 Backups nsave_cp = INT(5/dt) IF (MOD(cstep, nsave_cp) == 0) THEN CALL checkpoint_save(cp_counter) cp_counter = cp_counter + 1 ENDIF !_____________________________________________________________________________ ! 3. Final diagnostics ELSEIF (kstep .EQ. -1) THEN CALL cpu_time(finish) CALL attach(fidres, "/data/input","cpu_time",finish-start) ! Display computational time cost IF (my_id .EQ. 0) CALL display_h_min_s(finish-start) ! Close all diagnostic files - CALL mpi_barrier(MPI_COMM_WORLD, ierr) CALL closef(fidres) - CALL mpi_barrier(MPI_COMM_WORLD, ierr) CALL closef(fidrst) CALL mpi_barrier(MPI_COMM_WORLD, ierr) END IF END SUBROUTINE diagnose SUBROUTINE diagnose_0d USE basic USE futils, ONLY: append USE diagnostics_par USE prec_const IMPLICIT NONE CALL append(fidres, "/profiler/Tc_rhs", tc_rhs,ionode=0) CALL append(fidres, "/profiler/Tc_adv_field", tc_adv_field,ionode=0) CALL append(fidres, "/profiler/Tc_poisson", tc_poisson,ionode=0) CALL append(fidres, "/profiler/Tc_Sapj", tc_Sapj,ionode=0) CALL append(fidres, "/profiler/Tc_diag", tc_diag,ionode=0) CALL append(fidres, "/profiler/Tc_checkfield",tc_checkfield,ionode=0) CALL append(fidres, "/profiler/Tc_step", tc_step,ionode=0) CALL append(fidres, "/profiler/time", time,ionode=0) END SUBROUTINE diagnose_0d SUBROUTINE diagnose_2d USE basic USE futils, ONLY: append, getatt, attach, putarrnd USE fields USE time_integration USE diagnostics_par USE prec_const IMPLICIT NONE CALL append(fidres, "/data/var2d/time", time,ionode=0) CALL append(fidres, "/data/var2d/cstep", real(cstep,dp),ionode=0) CALL getatt(fidres, "/data/var2d/", "frames",iframe2d) iframe2d=iframe2d+1 CALL attach(fidres,"/data/var2d/" , "frames", iframe2d) IF (write_phi) THEN CALL write_field2d(phi(:,:), 'phi') END IF IF (write_Na00) THEN CALL write_field2d(moments_e(1,1,:,:,updatetlevel), 'Ne00') CALL write_field2d(moments_i(1,1,:,:,updatetlevel), 'Ni00') END IF CONTAINS SUBROUTINE write_field2d(field, text) USE futils, ONLY: attach, putarr USE grid, ONLY: ikrs,ikre, ikzs,ikze USE prec_const IMPLICIT NONE COMPLEX(dp), DIMENSION(ikrs:ikre, ikzs:ikze), INTENT(IN) :: field CHARACTER(*), INTENT(IN) :: text CHARACTER(LEN=50) :: dset_name WRITE(dset_name, "(A, '/', A, '/', i6.6)") "/data/var2d", TRIM(text), iframe2d IF (num_procs .EQ. 1) THEN CALL putarr(fidres, dset_name, field(ikrs:ikre, ikzs:ikze), ionode=0) ELSE CALL putarr(fidres, dset_name, field(ikrs:ikre, ikzs:ikze), pardim=1) ENDIF CALL attach(fidres, dset_name, "time", time) END SUBROUTINE write_field2d END SUBROUTINE diagnose_2d SUBROUTINE diagnose_5d USE basic USE futils, ONLY: append, getatt, attach, putarrnd USE fields USE array!, ONLY: Sepj, Sipj USE grid, ONLY: ips_e,ipe_e, ips_i, ipe_i, & ijs_e,ije_e, ijs_i, ije_i USE time_integration USE diagnostics_par USE prec_const IMPLICIT NONE CALL append(fidres, "/data/var5d/time", time,ionode=0) CALL append(fidres, "/data/var5d/cstep", real(cstep,dp),ionode=0) CALL getatt(fidres, "/data/var5d/", "frames",iframe5d) iframe5d=iframe5d+1 CALL attach(fidres,"/data/var5d/" , "frames", iframe5d) IF (write_moments) THEN CALL write_field5d_e(moments_e(:,:,:,:,updatetlevel), 'moments_e') CALL write_field5d_i(moments_i(:,:,:,:,updatetlevel), 'moments_i') END IF IF (write_non_lin) THEN CALL write_field5d_e(Sepj, 'Sepj') CALL write_field5d_i(Sipj, 'Sipj') END IF CONTAINS SUBROUTINE write_field5d_e(field, text) USE futils, ONLY: attach, putarr USE grid, ONLY: ips_e,ipe_e, ijs_e,ije_e, ikrs,ikre, ikzs,ikze USE prec_const IMPLICIT NONE COMPLEX(dp), DIMENSION(ips_e:ipe_e,ijs_e:ije_e,ikrs:ikre,ikzs:ikze), INTENT(IN) :: field CHARACTER(*), INTENT(IN) :: text CHARACTER(LEN=50) :: dset_name WRITE(dset_name, "(A, '/', A, '/', i6.6)") "/data/var5d", TRIM(text), iframe5d IF (num_procs .EQ. 1) THEN CALL putarr(fidres, dset_name, field(ips_e:ipe_e,ijs_e:ije_e,ikrs:ikre,ikzs:ikze), ionode=0) ELSE CALL putarr(fidres, dset_name, field(ips_e:ipe_e,ijs_e:ije_e,ikrs:ikre,ikzs:ikze), pardim=3) ENDIF CALL attach(fidres, dset_name, "time", time) END SUBROUTINE write_field5d_e SUBROUTINE write_field5d_i(field, text) USE futils, ONLY: attach, putarr USE grid, ONLY: ips_i,ipe_i, ijs_i,ije_i, ikrs,ikre, ikzs,ikze USE prec_const IMPLICIT NONE COMPLEX(dp), DIMENSION(ips_i:ipe_i,ijs_i:ije_i,ikrs:ikre,ikzs:ikze), INTENT(IN) :: field CHARACTER(*), INTENT(IN) :: text CHARACTER(LEN=50) :: dset_name WRITE(dset_name, "(A, '/', A, '/', i6.6)") "/data/var5d", TRIM(text), iframe5d IF (num_procs .EQ. 1) THEN CALL putarr(fidres, dset_name, field(ips_i:ipe_i,ijs_i:ije_i,ikrs:ikre,ikzs:ikze), ionode=0) ELSE CALL putarr(fidres, dset_name, field(ips_i:ipe_i,ijs_i:ije_i,ikrs:ikre,ikzs:ikze), pardim=3) ENDIF CALL attach(fidres, dset_name, "time", time) END SUBROUTINE write_field5d_i END SUBROUTINE diagnose_5d SUBROUTINE checkpoint_save(cp_step) USE basic USE grid, ONLY: ips_i,ipe_i, ijs_i,ije_i, ips_e,ipe_e, ijs_e,ije_e, ikrs,ikre, ikzs,ikze USE diagnostics_par USE futils, ONLY: putarr,attach USE model USE initial_par USE fields USE time_integration IMPLICIT NONE INTEGER, INTENT(IN) :: cp_step CHARACTER(LEN=50) :: dset_name ! Write state of system to restart file WRITE(dset_name, "(A, '/', i6.6)") "/Basic/moments_e", cp_step IF (num_procs .EQ. 1) THEN CALL putarr(fidrst, dset_name, moments_e(ips_e:ipe_e,ijs_e:ije_e,& ikrs:ikre,ikzs:ikze,1), ionode=0) ELSE CALL putarr(fidrst, dset_name, moments_e(ips_e:ipe_e,ijs_e:ije_e,& ikrs:ikre,ikzs:ikze,1), pardim=3) ENDIF CALL attach(fidrst, dset_name, 'cstep', cstep) CALL attach(fidrst, dset_name, 'time', time) CALL attach(fidrst, dset_name, 'jobnum', jobnum) CALL attach(fidrst, dset_name, 'dt', dt) CALL attach(fidrst, dset_name, 'iframe2d', iframe2d) CALL attach(fidrst, dset_name, 'iframe5d', iframe5d) WRITE(dset_name, "(A, '/', i6.6)") "/Basic/moments_i", cp_step IF (num_procs .EQ. 1) THEN CALL putarr(fidrst, dset_name, moments_i(ips_i:ipe_i,ijs_i:ije_i,& ikrs:ikre,ikzs:ikze,1), ionode=0) ELSE CALL putarr(fidrst, dset_name, moments_i(ips_i:ipe_i,ijs_i:ije_i,& ikrs:ikre,ikzs:ikze,1), pardim=3) ENDIF CALL attach(fidrst, dset_name, 'cstep', cstep) CALL attach(fidrst, dset_name, 'time', time) CALL attach(fidrst, dset_name, 'jobnum', jobnum) CALL attach(fidrst, dset_name, 'dt', dt) CALL attach(fidrst, dset_name, 'iframe2d', iframe2d) CALL attach(fidrst, dset_name, 'iframe5d', iframe5d) IF (my_id .EQ. 0) THEN WRITE(*,'(3x,a)') "Checkpoint file "//TRIM(rstfile)//" updated" ENDIF END SUBROUTINE checkpoint_save diff --git a/src/srcinfo.h b/src/srcinfo.h index 0cc4fa8..be4f109 100644 --- a/src/srcinfo.h +++ b/src/srcinfo.h @@ -1,10 +1,10 @@ character(len=40) VERSION character(len=40) BRANCH character(len=20) AUTHOR character(len=40) EXECDATE character(len=40) HOST -parameter (VERSION='fde3f88-dirty') +parameter (VERSION='8dcebe0-dirty') parameter (BRANCH='MPI') parameter (AUTHOR='ahoffman') -parameter (EXECDATE='Fri Nov 27 15:29:47 CET 2020') +parameter (EXECDATE='Fri Nov 27 17:13:58 CET 2020') parameter (HOST ='spcpc606') diff --git a/src/srcinfo/srcinfo.h b/src/srcinfo/srcinfo.h index 0cc4fa8..be4f109 100644 --- a/src/srcinfo/srcinfo.h +++ b/src/srcinfo/srcinfo.h @@ -1,10 +1,10 @@ character(len=40) VERSION character(len=40) BRANCH character(len=20) AUTHOR character(len=40) EXECDATE character(len=40) HOST -parameter (VERSION='fde3f88-dirty') +parameter (VERSION='8dcebe0-dirty') parameter (BRANCH='MPI') parameter (AUTHOR='ahoffman') -parameter (EXECDATE='Fri Nov 27 15:29:47 CET 2020') +parameter (EXECDATE='Fri Nov 27 17:13:58 CET 2020') parameter (HOST ='spcpc606') diff --git a/wk/fort.90 b/wk/fort.90 index 34f66b4..b2f3f7a 100644 --- a/wk/fort.90 +++ b/wk/fort.90 @@ -1,64 +1,64 @@ &BASIC nrun = 100000000 - dt = 0.01 - tmax = 100 - RESTART = .false. + dt = 0.005 + tmax = 200 + RESTART = .true. / &GRID pmaxe =2 jmaxe = 1 pmaxi = 2 jmaxi = 1 Nr = 400 Lr = 100 Nz = 400 Lz = 100 kpar = 0 CANCEL_ODD_P = .false. / &OUTPUT_PAR - nsave_0d = 100 + nsave_0d = 200 nsave_1d = 0 - nsave_2d = 100 - nsave_5d = 1000 + nsave_2d = 200 + nsave_5d = 2000 write_Na00 = .true. write_moments = .true. write_phi = .true. write_non_lin = .true. write_doubleprecision = .true. resfile0 = '../results/test_parallel/400x200_L_100_Pe_2_Je_1_Pi_2_Ji_1_nB_0.6_nN_1_nu_1e-01_FC_mu_5e-04/outputs' rstfile0 = '../results/test_parallel/400x200_L_100_Pe_2_Je_1_Pi_2_Ji_1_nB_0.6_nN_1_nu_1e-01_FC_mu_5e-04/checkpoint' job2load = 0 / &MODEL_PAR ! Collisionality CO = -1 DK = .false. NON_LIN = .true. mu = 0.0005 nu = 0.1 tau_e = 1 tau_i = 1 sigma_e = 0.023338 sigma_i = 1 q_e = -1 q_i = 1 eta_n = 1 eta_T = 0 eta_B = 0.6 lambdaD = 0 kr0KH = 0 A0KH = 0 / &INITIAL_CON only_Na00 =.false. initback_moments =0 initnoise_moments =1e-05 iseed =42 selfmat_file ='../iCa/self_Coll_GKE_0_GKI_0_ESELF_1_ISELF_1_Pmaxe_2_Jmaxe_1_Pmaxi_2_Jmaxi_1_pamaxx_10.h5' eimat_file ='../iCa/ei_Coll_GKE_0_GKI_0_ETEST_1_EBACK_1_Pmaxe_2_Jmaxe_1_Pmaxi_2_Jmaxi_1_pamaxx_10_tau_1.0000_mu_0.0233.h5' iemat_file ='../iCa/ie_Coll_GKE_0_GKI_0_ITEST_1_IBACK_1_Pmaxe_2_Jmaxe_1_Pmaxi_2_Jmaxi_1_pamaxx_10_tau_1.0000_mu_0.0233.h5' / &TIME_INTEGRATION_PAR numerical_scheme='RK4' / \ No newline at end of file diff --git a/wk/test_parallel.m b/wk/test_parallel.m index 923583a..12a28d4 100644 --- a/wk/test_parallel.m +++ b/wk/test_parallel.m @@ -1,44 +1,44 @@ clear all; addpath(genpath('../matlab')) % ... add %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Set Up parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% PHYSICAL PARAMETERS NU = 1e-1; % Collision frequency TAU = 1.0; % e/i temperature ratio ETAB = 0.6; % Magnetic gradient ETAN = 1.0; % Density gradient ETAT = 0.0; % Temperature gradient MU = 5e-4; % Hyper diffusivity coefficient NOISE0 = 1.0e-5; %% GRID PARAMETERS N = 400; % Frequency gridpoints (Nkr = N/2) L = 100; % Size of the squared frequency domain PMAXE = 2; % Highest electron Hermite polynomial degree JMAXE = 1; % Highest '' Laguerre '' PMAXI = 2; % Highest ion Hermite polynomial degree JMAXI = 1; % Highest '' Laguerre '' %% TIME PARAMETERS -TMAX = 100; % Maximal time unit -DT = 1e-2; % Time step +TMAX = 200; % Maximal time unit +DT = 5e-3; % Time step SPS0D = 1; % Sampling per time unit for profiler SPS2D = 1; % Sampling per time unit for 2D arrays SPS5D = 0.1; % Sampling per time unit for 5D arrays -RESTART = 0; % To restart from last checkpoint +RESTART = 1; % To restart from last checkpoint JOB2LOAD= 0; %% OPTIONS SIMID = 'test_parallel'; % Name of the simulation CO = -1; % Collision operator (0 : L.Bernstein, -1 : Full Coulomb, -2 : Dougherty) %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% unused KR0KH = 0; A0KH = 0; % Background phi mode to drive Ray-Tay inst. NO_E = 0; % Remove electrons dynamic % DK = 0; % Drift kinetic model (put every kernel_n to 0 except n=0 to 1) KREQ0 = 0; % put kr = 0 KPAR = 0.0; % Parellel wave vector component LAMBDAD = 0.0; NON_LIN = 1 *(1-KREQ0); % activate non-linearity (is cancelled if KREQ0 = 1) CANCEL_ODD_P = 0;% Cancels the odd polynomials degree setup