[FLASH-USERS] Sedov scaling test

[Clint Young]

On Mon, 11 Feb 2008 mateuszr at umich.edu wrote:

> 
> Hi all,
> 
> I am testing a very small Dell cluster (8 cpus total). The machine is a 
> dual quad core Xeon with 2 GB per core. The OS is Rocks 4.3, and the 
> compilers used is the stock gcc 3.4.6 with mpich 1.2.7, mpif90 
> (Makefile.h enclosed below). FLASH version is 2.5 and I am using a 
> Sedov test setup.
> 
> The setup does not seem to scale with the number of processors at all.
> If anything, the execution time increases very slightly as I double the 
> number of processors. I also noticed that all timesteps are printed out 
> to the screen n times, where n is the number of processors. It seems to 
> me that for some reason, the code just runs the same calculation n 
> times instead of distributing the load
> evenly among all processors.
> 
> It seems to me that this problem has nothing to do with the processor 
> communication and, quite likely, is (embarrasingly?) trivial to solve.
> 
> Has anybody seen this behavior before ?
> 
> thanks,
> Mateusz
> 
> 
> P.S. I am enclosing Makefile.h below:
> 
> # FLASH makefile definitions for ix86 Linux (Portland Group compiler)
> 
> # note, in order to get pgprof to work properly, it was necessary to # 
> download a new
> version from ftp://ftp.pgroup.com/ that plays nicely
> # with GNOME.
> 
> #----------------------------------------------------------------------------
> # Set the HDF/HDF5 library paths -- these need to be updated for your system
> #----------------------------------------------------------------------------
> #HDF4_PATH = /usr/local/hdf4
> #HDF5_PATH = /opt/HDF5-1.4.2-patch1-serial/
> HDF5_PATH = /opt/packages/hdf5-1.6.6/
> 
> ZLIB_PATH  =
> 
> PAPI_PATH  =
> PAPI_FLAGS =
> 
> NCMPI_PATH =
> MPE_PATH   =
> 
> #----------------------------------------------------------------------------
> # Compiler and linker commands
> #
> #   Use the MPICH wrappers around the compilers -- these will automatically
> #   load the proper libraries and include files.  Version of MPICH prior
> #   to 1.2.2 (?) do not recognize .F90 as a valid Fortran file extension.
> #   You need to edit mpif90 and add .F90 to the test of filename extensions,
> #   or upgrade your MPICH.
> #----------------------------------------------------------------------------
> FCOMP   = /opt/mpich/intel/bin/mpif90
> CCOMP   = /opt/mpich/intel/bin/mpicc
> CPPCOMP = /opt/mpich/intel/bin/mpicc
> LINK    = /opt/mpich/intel/bin/mpif90
> 
> # pre-processor flag
> PP      = -D
> 
> #----------------------------------------------------------------------------
> # Compilation flags
> #
> #  Three sets of compilation/linking flags are defined: one for optimized
> #  code, one for testing, and one for debugging.  The default is to use 
> the #  _OPT
> version.  Specifying -debug to setup will pick the _DEBUG version,
> #  these should enable bounds checking.  Specifying _TEST is used for # 
>   flash_test, and
> is set for quick code generation, and (sometimes) #  profiling.  The 
> Makefile generated
> by setup will assign the generic token #  (ex. FFLAGS) to the proper 
> set of flags (ex.
> FFLAGS_OPT).
> #----------------------------------------------------------------------------
> #FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/packages/hdf5-1.6.6/include
> -I/opt/mpich/intel/include/ -L/opt/mpich/intel/lib 
> -L/opt/packages/hdf5-1.6.6/lib
> -I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
> -lhdf5 -lrfftw_mpi
> -lfftw_mpi -lrfftw
> FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/packages/hdf5-1.6.6/include
> -I/opt/mpich/intel/include/ -L/opt/mpich/intel/lib 
> -L/opt/packages/hdf5-1.6.6/lib
> -I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib -ldrfftw_mpi
> -ldfftw_mpi -ldrfftw -ldfftw
> # FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/usr/include/openmpi
> -I/usr/include/openmpi/64/ -L/usr/lib/openmpi
> #FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/mpich/gnu/include 
> -L/opt/mpich/gnu/lib
> FFLAGS_DEBUG = -g -c -Mbounds -r8 -i4 FFLAGS_TEST = -c -r8 -i4 -Mprof=lines
> 
> F90FLAGS =
> 
> #CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
> -I/usr/include/openmpi
> -I/usr/include/openmpi/64/ -L/usr/lib/openmpi
> #CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
> -I/opt/mpich/gnu/include
> -L/opt/mpich/gnu/lib
> #CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
> -I/opt/intel/impi/3.1/include64/
> -L/opt/intel/impi/3.1/lib64/ -L/opt/packages/hdf5-1.6.6/lib
> -I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
> -lhdf5 -lrfftw_mpi
> -lfftw_mpi -lrttfw
> CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
> -I/opt/intel/impi/3.1/include64/
> -L/opt/intel/impi/3.1/lib64/ -L/opt/packages/hdf5-1.6.6/lib
> -I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
> -lhdf5 -ldrfftw_mpi
> -ldfftw_mpi -ldrfftw -ldfftw
> 
> CFLAGS_DEBUG = -g -c
> CFLAGS_TEST = -c
> 
> # if we are using HDF5, we need to specify the path to the include files
> CFLAGS_HDF5 = -I $(HDF5_PATH)/include
> 
> CFLAGS_NCMPI =
> 
> #----------------------------------------------------------------------------
> # Linker flags
> #
> #  There is a seperate version of the linker flags for each of the 
> _OPT, #  _DEBUG, and
> _TEST cases.
> #----------------------------------------------------------------------------
> 
> LFLAGS_OPT   = -o
> LFLAGS_DEBUG = -g -o
> LFLAGS_TEST  = -Mprof=lines -o
> 
> 
> #----------------------------------------------------------------------------
> # Library specific linking
> #
> #  If a FLASH module has a 'LIBRARY xxx' line in its Config file, we need to
> #  create a macro in this Makefile.h for LIB_xxx, which will be added to the
> #  link line when FLASH is built.  This allows us to switch between different
> #  (incompatible) libraries.  We also create a _OPT, _DEBUG, and _TEST
> #  library macro to add any performance-minded libraries (like fast math),
> #  depending on how FLASH was setup.
> #----------------------------------------------------------------------------
> 
> LIB_OPT   = LIB_DEBUG =
> LIB_TEST  =
> 
> 
> #LIB_MATH  = -L/u/mateusz/fftw-2.1.5/lib -ldrfftw_mpi -ldfftw_mpi 
> -ldrfftw -ldfftw
> #LIB_MATH = -L/opt/packages/fftw-2.1.5/lib -ldrfftw_mpi -ldfftw_mpi 
> -ldrfftw -ldfftw
> LIB_MATH = -L/opt/packages/fftw-2.1.5/lib  
> -L/opt/packages/hdf5-1.6.6/lib -lrfftw_mpi
> -lfftw_mpi -lrfftw -lfftw -lhdf5 #LIB_HDF4  = -L $(HDF4_PATH)/lib 
> -lmfhdf -ldf -ljpeg -lz
> LIB_HDF5 = -L $(HDF5_PATH)/lib $(LIB_MATH) -lmpi -lz -lsz
> #LIB_HDF5  = -lhdf5 $(LIB_MATH) -lmpi
> 
> LIB_PAPI  =
> 
> LIB_NCMPI =
> LIB_MPE   =
> 
> #----------------------------------------------------------------------------
> # Additional machine-dependent object files
> #
> #  Add any machine specific files here -- they will be compiled and linked
> #  when FLASH is built.
> #----------------------------------------------------------------------------
> 
> MACHOBJ =
> 
> 
> #----------------------------------------------------------------------------
> # Additional commands
> #----------------------------------------------------------------------------
> 
> MV = mv -f
> AR = ar -r
> RM = rm -f
> CD = cd
> RL = ranlib
> ECHO = echo
> 
>