[FLASH-USERS] Sedov scaling test

[mateuszr at umich.edu]

Hi all,

I am testing a very small Dell cluster (8 cpus total). The machine is a 
dual quad core Xeon with 2 GB per core. The OS is Rocks 4.3, and the 
compilers used is the stock gcc 3.4.6 with mpich 1.2.7, mpif90 
(Makefile.h enclosed below). FLASH version is 2.5 and I am using a 
Sedov test setup.

The setup does not seem to scale with the number of processors at all.
If anything, the execution time increases very slightly as I double the 
number of processors. I also noticed that all timesteps are printed out 
to the screen n times, where n is the number of processors. It seems to 
me that for some reason, the code just runs the same calculation n 
times instead of distributing the load
evenly among all processors.

It seems to me that this problem has nothing to do with the processor 
communication and, quite likely, is (embarrasingly?) trivial to solve.

Has anybody seen this behavior before ?

thanks,
Mateusz


P.S. I am enclosing Makefile.h below:

# FLASH makefile definitions for ix86 Linux (Portland Group compiler)

# note, in order to get pgprof to work properly, it was necessary to # 
download a new
version from ftp://ftp.pgroup.com/ that plays nicely
# with GNOME.

#----------------------------------------------------------------------------
# Set the HDF/HDF5 library paths -- these need to be updated for your system
#----------------------------------------------------------------------------
#HDF4_PATH = /usr/local/hdf4
#HDF5_PATH = /opt/HDF5-1.4.2-patch1-serial/
HDF5_PATH = /opt/packages/hdf5-1.6.6/

ZLIB_PATH  =

PAPI_PATH  =
PAPI_FLAGS =

NCMPI_PATH =
MPE_PATH   =

#----------------------------------------------------------------------------
# Compiler and linker commands
#
#   Use the MPICH wrappers around the compilers -- these will automatically
#   load the proper libraries and include files.  Version of MPICH prior
#   to 1.2.2 (?) do not recognize .F90 as a valid Fortran file extension.
#   You need to edit mpif90 and add .F90 to the test of filename extensions,
#   or upgrade your MPICH.
#----------------------------------------------------------------------------
FCOMP   = /opt/mpich/intel/bin/mpif90
CCOMP   = /opt/mpich/intel/bin/mpicc
CPPCOMP = /opt/mpich/intel/bin/mpicc
LINK    = /opt/mpich/intel/bin/mpif90

# pre-processor flag
PP      = -D

#----------------------------------------------------------------------------
# Compilation flags
#
#  Three sets of compilation/linking flags are defined: one for optimized
#  code, one for testing, and one for debugging.  The default is to use 
the #  _OPT
version.  Specifying -debug to setup will pick the _DEBUG version,
#  these should enable bounds checking.  Specifying _TEST is used for # 
  flash_test, and
is set for quick code generation, and (sometimes) #  profiling.  The 
Makefile generated
by setup will assign the generic token #  (ex. FFLAGS) to the proper 
set of flags (ex.
FFLAGS_OPT).
#----------------------------------------------------------------------------
#FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/packages/hdf5-1.6.6/include
-I/opt/mpich/intel/include/ -L/opt/mpich/intel/lib 
-L/opt/packages/hdf5-1.6.6/lib
-I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
-lhdf5 -lrfftw_mpi
-lfftw_mpi -lrfftw
FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/packages/hdf5-1.6.6/include
-I/opt/mpich/intel/include/ -L/opt/mpich/intel/lib 
-L/opt/packages/hdf5-1.6.6/lib
-I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib -ldrfftw_mpi
-ldfftw_mpi -ldrfftw -ldfftw
# FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/usr/include/openmpi
-I/usr/include/openmpi/64/ -L/usr/lib/openmpi
#FFLAGS_OPT = -c -r8 -i4 -CB -traceback -I/opt/mpich/gnu/include 
-L/opt/mpich/gnu/lib
FFLAGS_DEBUG = -g -c -Mbounds -r8 -i4 FFLAGS_TEST = -c -r8 -i4 -Mprof=lines

F90FLAGS =

#CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
-I/usr/include/openmpi
-I/usr/include/openmpi/64/ -L/usr/lib/openmpi
#CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
-I/opt/mpich/gnu/include
-L/opt/mpich/gnu/lib
#CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
-I/opt/intel/impi/3.1/include64/
-L/opt/intel/impi/3.1/lib64/ -L/opt/packages/hdf5-1.6.6/lib
-I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
-lhdf5 -lrfftw_mpi
-lfftw_mpi -lrttfw
CFLAGS_OPT = -O2  -c -I/opt/packages/hdf5-1.6.6/include 
-I/opt/intel/impi/3.1/include64/
-L/opt/intel/impi/3.1/lib64/ -L/opt/packages/hdf5-1.6.6/lib
-I/opt/packages/fftw-2.1.5/include -L/opt/packages/fftw-2.1.5/lib 
-lhdf5 -ldrfftw_mpi
-ldfftw_mpi -ldrfftw -ldfftw

CFLAGS_DEBUG = -g -c
CFLAGS_TEST = -c

# if we are using HDF5, we need to specify the path to the include files
CFLAGS_HDF5 = -I $(HDF5_PATH)/include

CFLAGS_NCMPI =

#----------------------------------------------------------------------------
# Linker flags
#
#  There is a seperate version of the linker flags for each of the 
_OPT, #  _DEBUG, and
_TEST cases.
#----------------------------------------------------------------------------

LFLAGS_OPT   = -o
LFLAGS_DEBUG = -g -o
LFLAGS_TEST  = -Mprof=lines -o


#----------------------------------------------------------------------------
# Library specific linking
#
#  If a FLASH module has a 'LIBRARY xxx' line in its Config file, we need to
#  create a macro in this Makefile.h for LIB_xxx, which will be added to the
#  link line when FLASH is built.  This allows us to switch between different
#  (incompatible) libraries.  We also create a _OPT, _DEBUG, and _TEST
#  library macro to add any performance-minded libraries (like fast math),
#  depending on how FLASH was setup.
#----------------------------------------------------------------------------

LIB_OPT   = LIB_DEBUG =
LIB_TEST  =


#LIB_MATH  = -L/u/mateusz/fftw-2.1.5/lib -ldrfftw_mpi -ldfftw_mpi 
-ldrfftw -ldfftw
#LIB_MATH = -L/opt/packages/fftw-2.1.5/lib -ldrfftw_mpi -ldfftw_mpi 
-ldrfftw -ldfftw
LIB_MATH = -L/opt/packages/fftw-2.1.5/lib  
-L/opt/packages/hdf5-1.6.6/lib -lrfftw_mpi
-lfftw_mpi -lrfftw -lfftw -lhdf5 #LIB_HDF4  = -L $(HDF4_PATH)/lib 
-lmfhdf -ldf -ljpeg -lz
LIB_HDF5 = -L $(HDF5_PATH)/lib $(LIB_MATH) -lmpi -lz -lsz
#LIB_HDF5  = -lhdf5 $(LIB_MATH) -lmpi

LIB_PAPI  =

LIB_NCMPI =
LIB_MPE   =

#----------------------------------------------------------------------------
# Additional machine-dependent object files
#
#  Add any machine specific files here -- they will be compiled and linked
#  when FLASH is built.
#----------------------------------------------------------------------------

MACHOBJ =


#----------------------------------------------------------------------------
# Additional commands
#----------------------------------------------------------------------------

MV = mv -f
AR = ar -r
RM = rm -f
CD = cd
RL = ranlib
ECHO = echo