[PETSc] [package list]

PETSc/3.19.5-cpeCray-23.09-rocm-5.6.1 (PETSc-3.19.5-cpeCray-23.09-rocm-5.6.1.eb)

Install with the EasyBuild-user module:

eb PETSc-3.19.5-cpeCray-23.09-rocm-5.6.1.eb -r

To access module help after installation and get reminded for which stacks and partitions the module is installed, use module spider PETSc/3.19.5-cpeCray-23.09-rocm-5.6.1.

EasyConfig:

# Based on EPCC recipe from https://github.com/ARCHER2-HPC/pe-scripts/blob/modules-2022-12/sh/petsc.sh
# and https://gitlab.com/petsc/petsc/-/blob/main/config/examples/arch-olcf-frontier-opt.py
#DOC Due to linking problems for which we have no solution, fewer external libraries are included
#DOC than in the cpeGNU version.
easyblock = 'ConfigureMake'

name = 'PETSc'
version = '3.19.5'
versionsuffix = '-rocm-5.6.1'

homepage = 'https://www.mcs.anl.gov/petsc'
description = """PETSc, pronounced PET-see (the S is silent), is a suite of data structures and routines for the
 scalable (parallel) solution of scientific applications modeled by partial differential equations."""

toolchain = {'name': 'cpeCray', 'version': '23.09'}
toolchainopts = {'openmp': True, 'usempi': True, 'pic': True}

source_urls = [
    'https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/',
]

sources = [SOURCELOWER_TAR_GZ]


dependencies = [
    ('rocm', '5.6.1', '', SYSTEM),
    ('gcc-mixed/12.2.0', EXTERNAL_MODULE),
    ('cray-python/3.10.10', EXTERNAL_MODULE),
    ('cray-hdf5-parallel/1.12.2.7', EXTERNAL_MODULE),
    ('buildtools', '%(toolchain_version)s', '', True),
    ('Boost', '1.82.0'),
#    ('METIS', '5.1.0'),
#    ('ParMETIS','4.0.3'),
    ('SCOTCH', '6.1.3'),
#    ('MUMPS', '5.5.1',),
#    ('Hypre', '2.26.0'),
#    ('SuperLU', '6.0.0', '-OpenMP'),
#    ('SuperLU_DIST', '8.1.2', '-OpenMP'),
]

# cc wrapper needs clean LD_LIBRARY_PATH to work with rocm/5.6.1
preconfigopts =  'export LD_LIBRARY_PATH=$ROCM_PATH/lib && '
preconfigopts += 'export LD_LIBRARY_PATH=$GCC_PATH/snos/lib64:$LD_LIBRARY_PATH && '

configopts = '--PETSC_ARCH="$CRAY_CPU_TARGET-$CRAY_ACCEL_TARGET" '
configopts += '--known-has-attribute-aligned=1 '
configopts += '--known-mpi-int64_t=0 '
configopts += '--known-bits-per-byte=8 '
configopts += '--known-64-bit-blas-indices=0 '
configopts += '--known-sdot-returns-double=0 '
configopts += '--known-snrm2-returns-double=0 '
configopts += '--known-memcmp-ok=1 '
configopts += '--known-mpi-c-double-complex=1 '
configopts += '--known-mpi-long-double=1 '
configopts += '--known-mpi-shared-libraries=0 '
configopts += '--known-sizeof-MPI_Comm=4 '
configopts += '--known-sizeof-MPI_Fint=4 '
configopts += '--known-sizeof-char=1 '
configopts += '--known-sizeof-double=8 '
configopts += '--known-sizeof-float=4 '
configopts += '--known-sizeof-int=4 '
configopts += '--known-sizeof-long-long=8 '
configopts += '--known-sizeof-long=8 '
configopts += '--known-sizeof-short=2 '
configopts += '--known-sizeof-size_t=8 '
configopts += '--known-sizeof-void-p=8 '
configopts += '--with-ar=ar '
configopts += '--with-batch=0 '
configopts += '--with-cc=cc '
configopts += '--with-clib-autodetect=0 '
configopts += '--with-cxx=CC '
configopts += '--with-cxxlib-autodetect=0 '
configopts += '--with-fc=ftn '
configopts += '--with-fortran-datatypes=1 '
configopts += '--with-fortran-interfaces=1 '
configopts += '--with-fortran-bindings=1 '
configopts += '--with-fortranlib-autodetect=0 '
configopts += '--with-ranlib=ranlib '
configopts += '--with-scalar-type=real '
configopts += '--with-shared-ld=ar '
configopts += '--with-etags=0 '
configopts += '--with-x=0 '
configopts += '--with-ssl=0 '
configopts += '--with-shared-libraries=1 ' 
configopts += '--with-mpi-lib=[] '
configopts += '--with-mpi-include=[] '
configopts += '--with-mpiexec=srun '
configopts += '--with-blas-lapack=1 '
#configopts += '--with-superlu=1 '
#configopts += '--with-superlu-dir="$EBROOTSUPERLU" '
#configopts += '--with-superlu_dist=1 '
#configopts += '--with-superlu_dist-dir="$EBROOTSUPERLU_DIST" '
#configopts += '--with-parmetis=1 '
#configopts += '--with-parmetis-dir="$EBROOTPARMETIS" '
#configopts += '--with-metis=1 '
#configopts += '--with-metis-dir="$EBROOTMETIS" '
#configopts += '--with-hypre=1 '
configopts += '--with-scalapack=1 '
configopts += '--with-ptscotch=1 '
configopts += '--with-ptscotch-dir="$EBROOTSCOTCH" '
#configopts += '--with-mumps=1 '
#configopts += '--with-mumps-include="$EBROOTMUMPS/include" '
#configopts += '--with-mumps-lib="-L$EBROOTMUMPS/lib -lcmumps -ldmumps -lesmumps -lsmumps -lzmumps -lmumps_common -lptesmumps -lesmumps -lpord -lmpifort" '
configopts += '--with-hdf5=1 '
configopts += '--with-cxx-dialect=C++17 '
configopts += '--LDFLAGS="-fopenmp" '
configopts += '--with-hip '
configopts += '--with-hipc=hipcc '
configopts += '--with-hip-arch=gfx90a '
configopts += '--with-hip-include="${MPICH_DIR}/include" '
configopts += '--with-hip-lib="-L${MPICH_DIR}/lib -lmpi ${PE_MPICH_GTL_DIR_amd_gfx90a} ${PE_MPICH_GTL_LIBS_amd_gfx90a}" '
configopts += '--download-kokkos '
configopts += '--download-kokkos-kernels '
configopts += '--download-kokkos-kernels-cmake-arguments="-DKokkosKernels_ROCBLAS_ROOT=${ROCM_PATH}/rocblas -DKokkosKernels_ROCSPARSE_ROOT=${ROCM_PATH}/rocsparse" '
configopts += '--LIBS="-lgfortran -lgcc -lstdc++ $PE_MPICH_GTL_DIR_amd_gfx90a $PE_MPICH_GTL_LIBS_amd_gfx90a" '


# CC wrapper needs clean LD_LIBRARY_PATH to work with rocm/5.6.1
prebuildopts =  'export LD_LIBRARY_PATH=$ROCM_PATH/lib && '
prebuildopts += 'export LD_LIBRARY_PATH=$GCC_PATH/snos/lib64:$LD_LIBRARY_PATH && '

# This is for checking if linking against the library is correct
runtest = 'check_build MPIEXEC=exec V=1'

sanity_check_paths = {
    'files': ['lib/libpetsc.%s' % SHLIB_EXT],
    'dirs': ['lib/pkgconfig', 'lib/petsc/bin'],
}

moduleclass = 'numlib'

[PETSc] [package list]