Running on EuroHPC systems

This document provides instructions on how execute FALL3D on the Leonardo, Mare Nostrum 5, and Lumi clusters.

Leonardo@CINECA

Expand section for Leonardo CINECA installation instructions

Slurm script GPU

#!/bin/bash

#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --partition=standard-g  # Partiton
#SBATCH --nodes=4               # Nodes to use
#SBATCH --ntasks=16             # Total execution tasks
#SBATCH --ntasks-per-node=4     # Number of MPI ranks per node
#SBATCH --gres=gpu:4            # Allocate one gpu per MPI rank
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)

# Load modules
module load netcdf-fortran/4.6.0--openmpi--4.1.4--nvhpc--23.1

# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NP=16                           # Total processor number
NX=4                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}

Slurm script CPU (NVHPC)

#!/bin/bash -l
#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --partition=standard    # Partiton
#SBATCH --nodes=1               # Nodes to use
#SBATCH --ntasks-per-node=32    # Number of MPI ranks per node
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)
# Load modules
module load netcdf-fortran/4.6.0--openmpi--4.1.4--nvhpc--23.1
# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NP=16                           # Total processor number
NX=8                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}

MN5-ACC@BSC

Expand section for Mare Nostrum 5 installation instructions

Slurm script GPU

#!/bin/bash
#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --nodes=8               # Nodes to use
#SBATCH --exclusive             # Use the entire nodes
#SBATCH --ntasks=32             # Total execution tasks
#SBATCH --ntasks-per-node=4     # Number of MPI ranks per node
#SBATCH --gres=gpu:4            # GPUs used for each node
#SBATCH --cpus-per-task=1       # CPUs assigned to each task
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)
# Load modules
module load nvidia-hpc-sdk/24.3
module load hdf5/1.14.1-2-nvidia-nvhpcx
module load pnetcdf/1.12.3-nvidia-nvhpcx
module load netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-nvidia-nvhpcx
export SLURM_CPU_BIND=none
# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NP=32                           # Total processor number
NX=8                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}

Slurm script CPU (NVHPC)

#!/bin/bash
#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --nodes=4               # Nodes to use
#SBATCH --exclusive             # Use the entire nodes
#SBATCH --ntasks=32             # Total execution tasks
#SBATCH --ntasks-per-node=32    # Number of MPI ranks per node
#SBATCH --cpus-per-task=1       # CPUs assigned to each task
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)
# Load modules
module load nvidia-hpc-sdk/24.3
module load hdf5/1.14.1-2-nvidia-nvhpcx
module load pnetcdf/1.12.3-nvidia-nvhpcx
module load netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-nvidia-nvhpcx
# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NP=32                           # Total processor number
NX=8                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number
# Run it!
mpirun -np ${NP} ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}

LUMI@CSC

Expand section for LUMI installation instructions

Slurm script GPU

#!/bin/bash -l

#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --partition=standard-g  # Partiton
#SBATCH --nodes=4               # Nodes to use
#SBATCH --ntasks-per-node=8     # Number of MPI ranks per node
#SBATCH --gpus-per-node=8       # Allocate one gpu per MPI rank
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)

# Create a script to place the jobs in the right numa node
# Each job is placed in the best communicated numa node with the respective GPU
cat << EOF > select_gpu
#!/bin/bash

export ROCR_VISIBLE_DEVICES=\$SLURM_LOCALID
exec \$*
EOF

chmod +x ./select_gpu

CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
# Allow device-aware network
export MPICH_GPU_SUPPORT_ENABLED=1
# Load modules
module load LUMI/24.03 
module load partition/G
module load rocm
module load cray-hdf5
module load cray-netcdf
# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NX=8                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number

srun --cpu-bind=${CPU_BIND} ./select_gpu ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}
rm ./select_gpu

Slurm script CPU

#!/bin/bash -l
#SBATCH --account=<Account>     # Project for billing
#SBATCH --job-name=FALL3D       # Job name
#SBATCH --output=FALL3D.out     # Name of stdout output file
#SBATCH --error=FALL3D.err      # Name of stderr error file
#SBATCH --partition=standard    # Partiton
#SBATCH --nodes=1               # Nodes to use
#SBATCH --ntasks-per-node=32    # Number of MPI ranks per node
#SBATCH --time=00:10:00         # Max wall time (d-hh:mm:ss)
# Load modules
module load cray-hdf5
module load cray-netcdf
# FALL3D section
BIN="./Fall3d.CPU.r8.x"         # Binary path
INPFILE="test_SC51.inp"         # Fall3d input file
FALLTASK="Fall3d"               # Task to execute

NX=8                            # Divisions along X axis
NY=4                            # Divisions along Y axis
NZ=1                            # Divisions along Z axis
NENS=1                          # Ensemble number

srun ${BIN} ${FALLTASK} ${INPFILE} ${NX} ${NY} ${NZ} -nens ${NENS}