[FLASH-USERS] Problems with parallelisation
Dr. Arthur Hirsch-Passicos
a.hirsch at hzdr.de
Fri Feb 7 03:32:36 EST 2025
Dear FLASH Users,
I am in the process of trying to parallelise my simulations so they run
faster but I'm facing some issues in the calculations when I'm running
the calculations with more than one task. Indeed, I see calculation
artifacts at the junction of the tasks for all values saved. Here is an
example with my initial electronic temperature and the same after some
time step when I use two tasks:
Here is my setup line:
./setup $currentSim -auto -2d +cartesian -maxblocks=300000 -nxb=16
-nyb=16 +hdf5typeio species=cham,targ +mtmmmt +uhd3t +mgd
mgd_meshgroups=1 threadBlockList=True
My submission to the cluster, the par file for the boundaries and the
Makefile.h I'm using are joined in the message.
Has anyone met similar issues and if so, how did you fix that issue?
Thank you in advance for your help.
Best regards,
--
Dr. Arthur Hirsch-Passicos
Post-doctoral Researcher
Laser Particle Acceleration Division
Institute of Radiation Physics
Helmholtz-Zentrum Dresden-Rossendorf e.V.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://flash.rochester.edu/pipermail/flash-users/attachments/20250207/64d8051a/attachment-0001.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: visit0001.png
Type: image/png
Size: 34950 bytes
Desc: not available
URL: <http://flash.rochester.edu/pipermail/flash-users/attachments/20250207/64d8051a/attachment-0002.png>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: visit0002.png
Type: image/png
Size: 45160 bytes
Desc: not available
URL: <http://flash.rochester.edu/pipermail/flash-users/attachments/20250207/64d8051a/attachment-0003.png>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: Makefile.h
Type: text/x-chdr
Size: 6416 bytes
Desc: not available
URL: <http://flash.rochester.edu/pipermail/flash-users/attachments/20250207/64d8051a/attachment-0002.bin>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: submit_slurm.sh
Type: application/x-shellscript
Size: 519 bytes
Desc: not available
URL: <http://flash.rochester.edu/pipermail/flash-users/attachments/20250207/64d8051a/attachment-0003.bin>
-------------- next part --------------
run_comment = "Prepulse for Stefan and Constantin"
log_file = "slac_test.log"
basenm = "slac_test_"
# This particular parfile is used as an example that is described in
# detail in the users guide.
##########################
# #
# I/O PARAMETERS #
# #
##########################
### Checkpoint Options ###
checkpointFileIntervalTime = 1.0
checkpointFileIntervalStep = 100000
### Plot Options ###
plotFileNumber = 0
plotFileIntervalStep = 100
plotFileIntervalTime = 25e-15
plot_var_1 = "dens"
plot_var_2 = "depo"
plot_var_3 = "tele"
plot_var_4 = "tion"
plot_var_5 = "trad"
plot_var_6 = "ye "
plot_var_7 = "sumy"
plot_var_8 = "cham"
plot_var_9 = "targ"
plot_var_10 = "pres"
plot_var_11 = "bdry"
# plot_var_12 = "vlcx"
# plot_var_13 = "vlcy"
### Restart Options ###
restart = .false.
checkpointFileNumber = 0
plotFileNumber = 0
########################################
# #
# RADIATION/OPACITY PARAMETERS #
# #
########################################
rt_useMGD = .false.
rt_mgdNumGroups = 1
rt_mgdBounds_1 = 1.0e-02
rt_mgdBounds_2 = 1.0e+10
rt_mgdFlMode = "fl_harmonic"
rt_mgdFlCoef = 1.0
rt_mgdXlBoundaryType = "vacuum"
rt_mgdXrBoundaryType = "vacuum"
rt_mgdYlBoundaryType = "vacuum"
rt_mgdYrBoundaryType = "vacuum"
rt_mgdZlBoundaryType = "reflecting"
rt_mgdZrBoundaryType = "reflecting"
useOpacity = .false.
### SET CHAMBER (HELIUM) OPACITY OPTIONS ###
op_chamAbsorb = "op_tabpa"
op_chamEmiss = "op_tabpe"
op_chamTrans = "op_tabro"
op_chamFileType = "ionmix4"
op_chamFileName = "Copper.cn4"
### SET TARGET (ALUMINUM) OPACITY OPTIONS ###
op_targAbsorb = "op_tabpa"
op_targEmiss = "op_tabpe"
op_targTrans = "op_tabro"
op_targFileType = "ionmix4"
op_targFileName = "Copper.cn4"
#################################
# #
# CONDUCTION PARAMETERS #
# #
#################################
useDiffuse = .true.
useConductivity = .true.
diff_useEleCond = .true.
diff_eleFlMode = "fl_larsen"
diff_eleFlCoef = 0.06
diff_thetaImplct = 1.0
diff_eleXlBoundaryType = "outflow"
diff_eleXrBoundaryType = "outflow"
diff_eleYlBoundaryType = "outflow"
diff_eleYrBoundaryType = "outflow"
diff_eleZlBoundaryType = "outflow"
diff_eleZrBoundaryType = "outflow"
####################################
# #
# HEAT EXCHANGE PARAMETERS #
# #
####################################
useHeatexchange = .true.
##########################
# #
# EOS PARAMETERS #
# #
##########################
eosModeInit = "dens_temp_gather"
smallt = 1.0
smallx = 1.0e-99
eos_useLogTables = .false.
############################
# #
# HYDRO PARAMETERS #
# #
############################
useHydro = .true.
order = 3 # Interpolation order (first/second/third/fifth order)
slopeLimiter = "minmod" # Slope limiters (minmod, mc, vanLeer, hybrid, limited)
LimitedSlopeBeta = 1. # Slope parameter for the "limited" slope by Toro
charLimiting = .true. # Characteristic limiting vs. Primitive limiting
use_avisc = .true. # use artificial viscosity (originally for PPM)
cvisc = 0.1 # coefficient for artificial viscosity
use_flattening = .false. # use flattening (dissipative) (originally for PPM)
use_steepening = .false. # use contact steepening (originally for PPM)
use_upwindTVD = .false. # use upwind biased TVD slope for PPM (need nguard=6)
RiemannSolver = "hllc" # Roe, HLL, HLLC, LLF, Marquina, hybrid
entropy = .false. # Entropy fix for the Roe solver
shockDetect = .false. # Shock Detect for numerical stability
use_hybridOrder = .true. # Enforce Riemann density jump
# Hydro boundary conditions:
xl_boundary_type = "outflow"
xr_boundary_type = "outflow"
yl_boundary_type = "outflow"
yr_boundary_type = "outflow"
zl_boundary_type = "reflect"
zr_boundary_type = "reflect"
##############################
# #
# INITIAL CONDITIONS #
# #
##############################
sim_targetRadius = 12.5e-04
# Target material defaults set for H at Cryo temperature:
sim_rhoTarg = 8.935
sim_teleTarg = 290.11375
sim_tionTarg = 290.11375
sim_tradTarg = 290.11375
ms_targA = 63.546
ms_targZ = 29.0
ms_targZMin = 0.02
eos_targEosType = "eos_tab"
eos_targSubType = "ionmix4"
eos_targTableFile = "Copper.cn4"
# Chamber material defaults set for Helium at pressure 1.6 mbar:
sim_rhoCham = 1.0e-06
sim_teleCham = 290.11375
sim_tionCham = 290.11375
sim_tradCham = 290.11375
ms_chamA = 63.546
ms_chamZ = 29.0
eos_chamEosType = "eos_tab"
eos_chamSubType = "ionmix4"
eos_chamTableFile = "Copper.cn4"
# Target heated up in skin layer
sim_heatDepth = 1.0e-04
sim_heatRadius = 5.0e-04
#sim_heatTemp = 300.0*11604.53
sim_heatTemp = 3481355.43
#sim_heatTemp = 290.11375
###########################
# #
# TIME PARAMETERS #
# #
###########################
tstep_change_factor = 1.10
cfl = 0.1
dt_diff_factor = 1.0e+100 # Disable diffusion dt
rt_dtFactor = 0.2
hx_dtFactor = 1.0e+100
tmax = 250e-15
#tmax = 500.0e-12
dtmin = 1.0e-17
dtinit = 5e-16
dtmax = 1.0e-14
nend = 10000000
irenorm = 1
###########################
# #
# MESH PARAMETERS #
# #
###########################
geometry = "cartesian"
# Domain size:
xmin = -13.5e-04
xmax = -9.5e-04
ymin = -1.0e-04
ymax = 1.0e-04
# Total number of blocks:
nblockx = 16
nblocky = 16
lrefine_max = 5
lrefine_min = 1
refine_var_1 = "dens"
refine_var_2 = "tele"
# Advanced Parameters
hy_eosModeAfter = "dens_ie_gather"
hy_3TMode = "ragelike"
dr_usePosdefComputeDt = .true.
dr_posdefVar_1 = "eion"
dr_posdefVar_2 = "eele"
dr_posdefVar_3 = "tele"
dr_posdefVar_4 = "tion"
More information about the flash-users
mailing list