OpenMPI

Description

According to the documentation of OpenMPI, OpenMPI is an open-source implementation of the MPI (Message Passing Interface) standard, widely used in high-performance computing for parallel programming on distributed-memory systems.

Available Versions

  • openmpi/2.1.1 (default)

  • openmpi/3.1.4

  • openmpi/4.0.1

  • openmpi/4.1.1

  • openmpi/4.1.1.v2

  • openmpi/4.1.5

  • openmpi/4.1.6

  • openmpi/4.1.6.v2

  • openmpi/4.1.8

Loading the Module

# Load specific version
module load openmpi/4.1.8

# Verify installation
mpirun --version
ompi_info

Compilation with OpenMPI

C

compile_mpi_c.sh
#!/bin/bash
#SBATCH -J compile_mpi_c
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:10:00
#SBATCH --mem=2G

export INPUT="mpi_hello.c"
export OUTPUT="mpi_hello"

module load openmpi/4.1.8

job-nanny mpicc -O3 -o mpi_hello mpi_hello.c

C++

compile_mpi_cpp.sh
#!/bin/bash
#SBATCH -J compile_mpi_cpp
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:10:00
#SBATCH --mem=2G

export INPUT="mpi_hello.cpp"
export OUTPUT="mpi_hello"

module load openmpi/4.1.8

job-nanny mpicxx -O3 -o mpi_hello mpi_hello.cpp

Fortran

compile_mpi_fortran.sh
#!/bin/bash
#SBATCH -J compile_mpi_fortran
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:10:00
#SBATCH --mem=2G

export INPUT="mpi_hello.f90"
export OUTPUT="mpi_hello"

module load openmpi/4.1.8

job-nanny mpifort -O3 -o mpi_hello mpi_hello.f90

MPI Code Example (C)

mpi_hello.c
#include <mpi.h>
#include <stdio.h>
#include <unistd.h>

int main(int argc, char** argv) {
    int rank, size;
    char hostname[256];

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    gethostname(hostname, sizeof(hostname));

    printf("Process %d of %d running on %s\n",
           rank, size, hostname);

    MPI_Finalize();
    return 0;
}

MPI Code Example (Fortran)

mpi_hello.f90
program mpi_hello
  use mpi
  implicit none
  integer :: rank, size, ierr
  character(len=256) :: hostname

  call MPI_INIT(ierr)
  call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr)
  call MPI_COMM_SIZE(MPI_COMM_WORLD, size, ierr)
  call hostnm(hostname)

  print *, "Process ", rank, " of ", size, " on ", trim(hostname)

  call MPI_FINALIZE(ierr)
end program mpi_hello

MPI Job Submission

submit_mpi.sh
#!/bin/bash
#SBATCH -J mpi_job
#SBATCH -N 2
#SBATCH --ntasks-per-node=28
#SBATCH -t 01:00:00
#SBATCH --mem-per-cpu=2G

export INPUT="mpi_hello"
export OUTPUT="mpi_output/"

module load openmpi/4.1.8

job-nanny mpirun -np $SLURM_NTASKS ./mpi_hello

Collective Communication (Example)

mpi_collective.c
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>

int main(int argc, char** argv) {
    int rank, size;
    int *sendbuf, *recvbuf;
    int i;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    // Allocate buffers
    sendbuf = (int*)malloc(size * sizeof(int));
    recvbuf = (int*)malloc(size * sizeof(int));

    // Each process prepares its data
    for (i = 0; i < size; i++) {
        sendbuf[i] = rank * size + i;
    }

    // MPI_Alltoall: each process sends to all
    MPI_Alltoall(sendbuf, 1, MPI_INT,
                 recvbuf, 1, MPI_INT,
                 MPI_COMM_WORLD);

    printf("Process %d received: ", rank);
    for (i = 0; i < size; i++) {
        printf("%d ", recvbuf[i]);
    }
    printf("\n");

    free(sendbuf);
    free(recvbuf);
    MPI_Finalize();
    return 0;
}

MPI + OpenMP (Hybrid)

submit_hybrid.sh
#!/bin/bash
#SBATCH -J hybrid_job
#SBATCH -N 2
#SBATCH --ntasks-per-node=4
#SBATCH -c 7
#SBATCH -t 02:00:00
#SBATCH --mem-per-cpu=2G

export INPUT="mpi_omp"
export OUTPUT="hybrid_output/"
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK

module load openmpi/4.1.8

job-nanny mpirun -np $SLURM_NTASKS ./mpi_omp
mpi_omp.c
#include <mpi.h>
#include <omp.h>
#include <stdio.h>

int main(int argc, char** argv) {
    int mpi_rank, mpi_size, omp_thread;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);

    #pragma omp parallel private(omp_thread)
    {
        omp_thread = omp_get_thread_num();
        printf("MPI rank %d, OpenMP thread %d\n",
               mpi_rank, omp_thread);
    }

    MPI_Finalize();
    return 0;
}

Job Array for Scalability Tests

test_scaling.sh
#!/bin/bash
#SBATCH -J scaling_test
#SBATCH --array=1-5
#SBATCH -N $((2**SLURM_ARRAY_TASK_ID))
#SBATCH --ntasks-per-node=28
#SBATCH -t 01:00:00
#SBATCH --mem-per-cpu=2G

export INPUT="mpi_benchmark"
export OUTPUT="scaling_${SLURM_ARRAY_TASK_ID}/"

module load openmpi/4.1.8

mkdir -p scaling_${SLURM_ARRAY_TASK_ID}
cd scaling_${SLURM_ARRAY_TASK_ID}

# Run benchmark
job-nanny mpirun -np $SLURM_NTASKS ../mpi_benchmark

# Record time
echo "Nodes: $SLURM_NNODES, Processes: $SLURM_NTASKS" > scaling.txt

References

See also