Running on EuroHPC systems
This document provides instructions on how execute FALL3D on the Leonardo, Mare Nostrum 5, and Lumi clusters.
Leonardo@CINECA
Expand section for Leonardo CINECA installation instructions
Slurm script GPU
#!/bin/bash
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --partition=standard-g # Partiton
#SBATCH --nodes=4 # Nodes to use
#SBATCH --ntasks=16 # Total execution tasks
#SBATCH --ntasks-per-node=4 # Number of MPI ranks per node
#SBATCH --gres=gpu:4 # Allocate one gpu per MPI rank
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Load modules
module load netcdf-fortran/4.6.0--openmpi--4.1.4--nvhpc--23.1
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NP=16 # Total processor number
NX=4 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
Slurm script CPU (NVHPC)
#!/bin/bash -l
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --partition=standard # Partiton
#SBATCH --nodes=1 # Nodes to use
#SBATCH --ntasks-per-node=32 # Number of MPI ranks per node
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Load modules
module load netcdf-fortran/4.6.0--openmpi--4.1.4--nvhpc--23.1
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NP=16 # Total processor number
NX=8 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
MN5-ACC@BSC
Expand section for Mare Nostrum 5 installation instructions
Slurm script GPU
#!/bin/bash
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --nodes=8 # Nodes to use
#SBATCH --exclusive # Use the entire nodes
#SBATCH --ntasks=32 # Total execution tasks
#SBATCH --ntasks-per-node=4 # Number of MPI ranks per node
#SBATCH --gres=gpu:4 # GPUs used for each node
#SBATCH --cpus-per-task=1 # CPUs assigned to each task
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Load modules
module load nvidia-hpc-sdk/24.3
module load hdf5/1.14.1-2-nvidia-nvhpcx
module load pnetcdf/1.12.3-nvidia-nvhpcx
module load netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-nvidia-nvhpcx
export SLURM_CPU_BIND=none
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NP=32 # Total processor number
NX=8 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
Slurm script CPU (NVHPC)
#!/bin/bash
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --nodes=4 # Nodes to use
#SBATCH --exclusive # Use the entire nodes
#SBATCH --ntasks=32 # Total execution tasks
#SBATCH --ntasks-per-node=32 # Number of MPI ranks per node
#SBATCH --cpus-per-task=1 # CPUs assigned to each task
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Load modules
module load nvidia-hpc-sdk/24.3
module load hdf5/1.14.1-2-nvidia-nvhpcx
module load pnetcdf/1.12.3-nvidia-nvhpcx
module load netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-nvidia-nvhpcx
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NP=32 # Total processor number
NX=8 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
LUMI@CSC
Expand section for LUMI installation instructions
Slurm script GPU
#!/bin/bash -l
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --partition=standard-g # Partiton
#SBATCH --nodes=4 # Nodes to use
#SBATCH --ntasks-per-node=8 # Number of MPI ranks per node
#SBATCH --gpus-per-node=8 # Allocate one gpu per MPI rank
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Create a script to place the jobs in the right numa node
# Each job is placed in the best communicated numa node with the respective GPU
cat << EOF > select_gpu
#!/bin/bash
export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID
exec \$*
EOF
chmod +x ./select_gpu
CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
# Allow device-aware network
export MPICH_GPU_SUPPORT_ENABLED=1
# Load modules
module load LUMI/24.03
module load partition/G
module load rocm
module load cray-hdf5
module load cray-netcdf
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NX=8 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
srun --cpu-bind=${CPU_BIND} ./select_gpu ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
rm ./select_gpu
Slurm script CPU
#!/bin/bash -l
#SBATCH --account=<Account> # Project for billing
#SBATCH --job-name=FALL3D # Job name
#SBATCH --output=FALL3D.out # Name of stdout output file
#SBATCH --error=FALL3D.err # Name of stderr error file
#SBATCH --partition=standard # Partiton
#SBATCH --nodes=1 # Nodes to use
#SBATCH --ntasks-per-node=32 # Number of MPI ranks per node
#SBATCH --time=00:10:00 # Max wall time (d-hh:mm:ss)
# Load modules
module load cray-hdf5
module load cray-netcdf
# FALL3D section
BIN="./Fall3d.CPU.r8.x" # Binary path
INPFILE="test_SC51.inp" # Fall3d input file
FALLTASK="Fall3d" # Task to execute
NX=8 # Divisions along X axis
NY=4 # Divisions along Y axis
NZ=1 # Divisions along Z axis
NENS=1 # Ensemble number
srun ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}