NetCDF

Description

According to the page of NetCDF, NetCDF (Network Common Data Form) is a set of machine-independent data formats and libraries that support the creation, access, and sharing of array-oriented scientific data. It is also a community standard for sharing scientific data.

Available Versions

  • netcdf/c/4.6.3 (default)

  • netcdf/fortran/4.4.5 (default)

Loading Modules

# For C
module load netcdf/c/4.6.3

# For Fortran
module load netcdf/fortran/4.4.5

# For both
module load netcdf/c/4.6.3
module load netcdf/fortran/4.4.5

Compilation with NetCDF

C

compile_netcdf_c.sh
#!/bin/bash
#SBATCH -J compile_netcdf_c
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:10:00
#SBATCH --mem=2G

export INPUT="read_write.c"
export OUTPUT="read_write"

module load netcdf/c/4.6.3

job-nanny gcc -o read_write read_write.c -lnetcdf

Fortran

compile_netcdf_fortran.sh
#!/bin/bash
#SBATCH -J compile_netcdf_fortran
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:10:00
#SBATCH --mem=2G

export INPUT="read_write.f90"
export OUTPUT="read_write"

module load netcdf/fortran/4.4.5

job-nanny gfortran -o read_write read_write.f90 -lnetcdff

C Code Example

write_netcdf.c
#include <netcdf.h>
#include <stdio.h>
#include <stdlib.h>

#define NDIMS 2
#define NX 6
#define NY 12

int main() {
    int ncid, x_dimid, y_dimid, varid;
    int dimids[NDIMS];
    int data[NX][NY];
    int retval;

    // Create data
    for (int i = 0; i < NX; i++)
        for (int j = 0; j < NY; j++)
            data[i][j] = i * NY + j;

    // Create file
    if ((retval = nc_create("simple.nc", NC_CLOBBER, &ncid)))
        return retval;

    // Define dimensions
    if ((retval = nc_def_dim(ncid, "x", NX, &x_dimid)))
        return retval;
    if ((retval = nc_def_dim(ncid, "y", NY, &y_dimid)))
        return retval;

    // Define variable
    dimids[0] = x_dimid;
    dimids[1] = y_dimid;
    if ((retval = nc_def_var(ncid, "data", NC_INT, NDIMS,
                             dimids, &varid)))
        return retval;

    // Exit define mode
    if ((retval = nc_enddef(ncid)))
        return retval;

    // Write data
    if ((retval = nc_put_var_int(ncid, varid, &data[0][0])))
        return retval;

    // Close file
    if ((retval = nc_close(ncid)))
        return retval;

    printf("File simple.nc created successfully!\n");
    return 0;
}

Fortran Code Example

write_netcdf.f90
program write_netcdf
  use netcdf
  implicit none

  integer :: ncid, x_dimid, y_dimid, varid
  integer, parameter :: NX = 6, NY = 12
  integer :: data(NX, NY)
  integer :: i, j

  ! Create data
  do i = 1, NX
    do j = 1, NY
      data(i,j) = (i-1)*NY + (j-1)
    end do
  end do

  ! Create file
  call check( nf90_create("simple_f.nc", NF90_CLOBBER, ncid) )

  ! Define dimensions
  call check( nf90_def_dim(ncid, "x", NX, x_dimid) )
  call check( nf90_def_dim(ncid, "y", NY, y_dimid) )

  ! Define variable
  call check( nf90_def_var(ncid, "data", NF90_INT, &
                           (/ x_dimid, y_dimid /), varid) )

  ! Exit define mode
  call check( nf90_enddef(ncid) )

  ! Write data
  call check( nf90_put_var(ncid, varid, data) )

  ! Close file
  call check( nf90_close(ncid) )

  print *, "File simple_f.nc created successfully!"

contains
  subroutine check(status)
    integer, intent(in) :: status
    if (status /= nf90_noerr) then
      print *, trim(nf90_strerror(status))
      stop
    end if
  end subroutine check
end program write_netcdf

Batch Processing Script

submit_netcdf_process.sh
#!/bin/bash
#SBATCH -J netcdf_process
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 04:00:00
#SBATCH --mem=8G

export INPUT="data/*.nc"
export OUTPUT="processed/"

module load netcdf/c/4.6.3
module load netcdf4-python

mkdir -p processed

cat > process_netcdf.py << 'EOF'
import netCDF4 as nc
import numpy as np
import glob
import os

# Process all .nc files
for file in glob.glob('data/*.nc'):
    print(f"Processing {file}...")

    # Open file
    ds = nc.Dataset(file, 'r')

    # Read variables
    for var_name in ds.variables:
        var = ds.variables[var_name]

        # Calculate statistics
        data = var[:]
        if data.size > 0:
            mean = np.mean(data)
            std = np.std(data)
            min_val = np.min(data)
            max_val = np.max(data)

            print(f"  {var_name}: mean={mean:.3f}, "
                  f"std={std:.3f}, min={min_val:.3f}, max={max_val:.3f}")

    # Save results
    base_name = os.path.basename(file)
    with open(f"processed/{base_name}.txt", 'w') as f:
        f.write(f"Analysis of {file}\n")
        for var_name in ds.variables:
            var = ds.variables[var_name]
            data = var[:]
            if data.size > 0:
                f.write(f"{var_name}: {np.mean(data):.6f} "
                       f"{np.std(data):.6f}\n")

    ds.close()
EOF

python3 process_netcdf.py

Command Line Tools

NetCDF includes useful command-line utilities:

# File information
ncdump -h file.nc

# Extract data in text format
ncdump file.nc > file.cdl

# Compare files
ncdiff -v var file1.nc file2.nc diff.nc

# Join files in time
ncrcat file1.nc file2.nc file3.nc output.nc

Job Array for Parallel Processing

submit_netcdf_array.sh
#!/bin/bash
#SBATCH -J netcdf_array
#SBATCH --array=1-20
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 02:00:00
#SBATCH --mem=4G

FILES=($(ls data/*.nc))
FILE=${FILES[$SLURM_ARRAY_TASK_ID-1]}

export INPUT="$FILE"
export OUTPUT="output_${SLURM_ARRAY_TASK_ID}/"

module load netcdf/c/4.6.3

mkdir -p output_${SLURM_ARRAY_TASK_ID}
cd output_${SLURM_ARRAY_TASK_ID}

# Extract metadata
ncdump -h ../$FILE > metadata.txt

# Extract specific variables
ncdump -v temperature ../$FILE > temperature.txt

References

See also