From c9cc97a7a94b8d6082ccfe3fafbae09bf59031d3 Mon Sep 17 00:00:00 2001 From: Gaetan Fuss Date: Thu, 11 Jul 2024 14:06:45 +0200 Subject: [PATCH 01/28] add p4 docker files --- .github/workflows/docker_build_tpls.yml | 18 +++++ cmake/blt | 2 +- docker/Stanford/Sherlock.Dockerfile | 3 +- docker/TotalEnergies/pangea4.Dockerfile | 86 +++++++++++++++++++++ docker/TotalEnergies/tpl-pangea4.Dockerfile | 39 ++++++++++ 5 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 docker/TotalEnergies/pangea4.Dockerfile create mode 100644 docker/TotalEnergies/tpl-pangea4.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 175a7aa0..792cbb60 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -20,6 +20,24 @@ jobs: matrix: include: + - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.17.1, mkl 2023.2.0) + DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0 + DOCKERFILE: docker/TotalEnergies/pangea4.Dockerfile + DOCKER_ROOT_IMAGE: bubusch/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0:1.0.0 + HOST_CONFIG: host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake + INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl + - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) + DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake + INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU + - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) + DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake + INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 DOCKERFILE: docker/Stanford/Dockerfile diff --git a/cmake/blt b/cmake/blt index 9ff77344..5a792c17 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit 9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35 +Subproject commit 5a792c1775e7a7628d84dcde31652a689f1df7b5 diff --git a/docker/Stanford/Sherlock.Dockerfile b/docker/Stanford/Sherlock.Dockerfile index ab3992f6..9eb0c2f5 100644 --- a/docker/Stanford/Sherlock.Dockerfile +++ b/docker/Stanford/Sherlock.Dockerfile @@ -1,6 +1,5 @@ # This Dockerfile aims at reproducing (some part of) the SHERLOCK environment. -# Please see TotalEnergies/Pangea2.Dockerfile for fully commented version - +# # While loading module ompi4.1.2 on Sherlock, the following are aut-loaded: # UCX/1.12.1 # LIBFABRIC/1.14.0 diff --git a/docker/TotalEnergies/pangea4.Dockerfile b/docker/TotalEnergies/pangea4.Dockerfile new file mode 100644 index 00000000..b4d6c25e --- /dev/null +++ b/docker/TotalEnergies/pangea4.Dockerfile @@ -0,0 +1,86 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +# +# Uses : +# - fake cray wrappers for gcc (cc, CC, ftn), they just point to the real gcc (real install requires licences) +# - flags for craype-x86-milan CPU target +# - oneAPI MKL for BLAS and LAPACK +# - HPC-X OpenMPI for MPI +# +####################################### +# +# Installs : +# - gcc = 12.1 +# - hpcx = 2.17.1 +# - intel-oneapi-mkl = 2023.2.0 +# - cmake = 3.27.2 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) +# - gcc, hpcx, oneAPI MKL, cmake and python are installed via spack (as done for Pangea 4) +# - hpcx is installed by extracting the tarball (as done for Pangea 4) +# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) +# +####################################### + +# ------------------------------------- +# PANGEA4 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/x86_64 architecture and redhat8 platform +FROM --platform=linux/x86_64 spack/centos-stream:latest AS pangea4-base +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=12.1 +ARG CMAKE_VERSION=3.27.2 +ARG PYTHON_VERSION=3.11 +# ------ +# INSTALL +# gcc 12.1 +RUN spack install gcc@$GCC_VERSION +# python 3.11 +RUN spack load gcc@$GCC_VERSION && spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake 3.27.2 +RUN spack load gcc@$GCC_VERSION && spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION +# intel-oneapi-mkl 2023.2.0 +RUN spack load gcc@$GCC_VERSION && spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION + +# ------------------------------------- +# PANGEA4 - GCC-HPCX-MKL [GCC - CMAKE - PYTHON - HPCX - ONEAPI-MKL] +FROM pangea4-base AS pangea4-gcc-hpcx-mkl +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG ONEAPI_MKL_VERSION=2023.2.0 +ARG HPCX_TARBALL="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz" +ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" +# ------ +# INSTALL +# intel-oneapi-mkl 2023.2.0 +RUN spack load gcc@$GCC_VERSION && spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx 2.17.1 (untar in /sw directory) +# not available in spack -> download and untar +RUN mkdir -p /sw +RUN wget $HPCX_URL -O /tmp/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz +RUN tar -xvf /tmp/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz -C /sw +ENV HPCX_HOME=/sw/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64 +# ------ +# ENV +# - create spack env +RUN spack env activate --create -p prgenv && \ + spack add gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION +# - wrappers for gcc +RUN spack env activate prgenv && export GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) +ENV CC=${GCC_INSTALL_DIR}/bin/gcc \ + CXX=${GCC_INSTALL_DIR}/bin/g++ \ + FC=${GCC_INSTALL_DIR}/bin/gfortran diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile new file mode 100644 index 00000000..a3c79393 --- /dev/null +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -0,0 +1,39 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +# +# Description : +# - generic image for building geos tpls on Pangea 4 environments +# - the docker base image can be any pangea4 docker file +# - tools of the base image are expected to be available in prgenv spack environment +####################################### + +# ------------------------------------- +# PANGEA4 - TPL TOOLCHAIN +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE as pangea4_tpl_toolchain +# ------ +# LABELS +LABEL description="Pangea 4 image : geos_tpl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build +ARG INSTALL_DIR +ARG HOST_CONFIG +# ------ +# ENV +ENV GEOSX_TPL_DIR=$INSTALL_DIR +# ------ +# INSTALL +RUN spack env activate prgenv && --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh +WORKDIR $BLD_DIR +RUN spack env activate prgenv && --mount=src=.,dst=$SRC_DIR make +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR +# ------ +# CACHE +# install `sccache` binaries to speed up the build of `geos` +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache From 42b012ecfcba77a53ceefd5d44ef2e4e31421ca6 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Thu, 18 Jul 2024 14:54:29 +0200 Subject: [PATCH 02/28] fix spack --- docker/TotalEnergies/pangea4.Dockerfile | 44 ++++++++++----------- docker/TotalEnergies/tpl-pangea4.Dockerfile | 6 ++- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/docker/TotalEnergies/pangea4.Dockerfile b/docker/TotalEnergies/pangea4.Dockerfile index b4d6c25e..1d8e180e 100644 --- a/docker/TotalEnergies/pangea4.Dockerfile +++ b/docker/TotalEnergies/pangea4.Dockerfile @@ -1,19 +1,12 @@ ####################################### # Pangea 4 image : gcc - hpcxompi - onemkl -# -# Uses : -# - fake cray wrappers for gcc (cc, CC, ftn), they just point to the real gcc (real install requires licences) -# - flags for craype-x86-milan CPU target -# - oneAPI MKL for BLAS and LAPACK -# - HPC-X OpenMPI for MPI -# ####################################### # # Installs : # - gcc = 12.1 # - hpcx = 2.17.1 # - intel-oneapi-mkl = 2023.2.0 -# - cmake = 3.27.2 +# - cmake = 3.27.9 # - python = 3.11 # ####################################### @@ -38,18 +31,21 @@ LABEL maintainer="TotalEnergies HPC Team" # ------ # ARGS ARG GCC_VERSION=12.1 -ARG CMAKE_VERSION=3.27.2 +ARG CMAKE_VERSION=3.27.9 ARG PYTHON_VERSION=3.11 # ------ # INSTALL -# gcc 12.1 +# gcc RUN spack install gcc@$GCC_VERSION -# python 3.11 -RUN spack load gcc@$GCC_VERSION && spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION -# cmake 3.27.2 -RUN spack load gcc@$GCC_VERSION && spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION -# intel-oneapi-mkl 2023.2.0 -RUN spack load gcc@$GCC_VERSION && spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# show compilers +RUN spack compilers +# cmake +RUN spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION # ------------------------------------- # PANGEA4 - GCC-HPCX-MKL [GCC - CMAKE - PYTHON - HPCX - ONEAPI-MKL] @@ -62,18 +58,18 @@ LABEL maintainer="TotalEnergies HPC Team" # ------ # ARGS ARG ONEAPI_MKL_VERSION=2023.2.0 -ARG HPCX_TARBALL="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz" +ARG HPCX_VERSION="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64" +ARG HPCX_TARBALL="$HPCX_VERSION.tbz" ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" # ------ # INSTALL -# intel-oneapi-mkl 2023.2.0 -RUN spack load gcc@$GCC_VERSION && spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION -# hpcx 2.17.1 (untar in /sw directory) -# not available in spack -> download and untar +# intel-oneapi-mkl +RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx not available in spack -> download and untar (untar in /sw directory) RUN mkdir -p /sw -RUN wget $HPCX_URL -O /tmp/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz -RUN tar -xvf /tmp/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64.tbz -C /sw -ENV HPCX_HOME=/sw/hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64 +RUN wget $HPCX_URL -O /tmp/$HPCX_TARBALL +RUN tar -xvf /tmp/$HPCX_TARBALL -C /sw +ENV HPCX_HOME=/sw/$HPCX_VERSION # ------ # ENV # - create spack env diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index a3c79393..a5568c04 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -28,9 +28,11 @@ ARG HOST_CONFIG ENV GEOSX_TPL_DIR=$INSTALL_DIR # ------ # INSTALL -RUN spack env activate prgenv && --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh +RUN spack env activate prgenv && \ + --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh WORKDIR $BLD_DIR -RUN spack env activate prgenv && --mount=src=.,dst=$SRC_DIR make +RUN spack env activate prgenv && \ + --mount=src=.,dst=$SRC_DIR make COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR # ------ # CACHE From ac4671e25d874e34fb83657327f5f91ca9c24fe3 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Thu, 25 Jul 2024 10:19:11 +0200 Subject: [PATCH 03/28] use onetech images --- .github/workflows/docker_build_tpls.yml | 4 +- .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 59 +++++++++++++ docker/TotalEnergies/pangea4-gcc.Dockerfile | 50 +++++++++++ docker/TotalEnergies/pangea4.Dockerfile | 82 ------------------- docker/TotalEnergies/tpl-pangea4.Dockerfile | 15 ++-- 5 files changed, 118 insertions(+), 92 deletions(-) create mode 100644 docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile create mode 100644 docker/TotalEnergies/pangea4-gcc.Dockerfile delete mode 100644 docker/TotalEnergies/pangea4.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 792cbb60..435d7038 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -22,8 +22,8 @@ jobs: include: - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.17.1, mkl 2023.2.0) DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0 - DOCKERFILE: docker/TotalEnergies/pangea4.Dockerfile - DOCKER_ROOT_IMAGE: bubusch/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0:1.0.0 + DOCKERFILE: docker/TotalEnergies/tpl-pangea4.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.17.1-onemkl2023.2.0_v1.0 HOST_CONFIG: host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile new file mode 100644 index 00000000..a8e3ebce --- /dev/null +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -0,0 +1,59 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +####################################### +# +# Installs : +# - gcc = 12.1 +# - hpcx = 2.17.1 +# - intel-oneapi-mkl = 2023.2.0 +# - cmake = 3.27.9 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile +# - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.17.1-onemkl2023.2.0_v1.0 +# - oneAPI MKL is installed via spack (as done for Pangea 4) +# - hpcx is installed by extracting the tarball (as done for Pangea 4) +# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) +# +####################################### + +# ------------------------------------- +# PANGEA4 - GCC-HPCX-MKL +FROM onetechssc/pangea4:gcc12.1_v1.0 AS pangea4 +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG ONEAPI_MKL_VERSION=2023.2.0 +ARG HPCX_VERSION="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64" +ARG HPCX_TARBALL="$HPCX_VERSION.tbz" +ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" +# ------ +# INSTALL +# intel-oneapi-mkl +RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx not available in spack -> download and untar (untar in /sw directory) +RUN mkdir -p /sw +RUN spack load wget && \ + wget $HPCX_URL -O /tmp/$HPCX_TARBALL +RUN tar -xvf /tmp/$HPCX_TARBALL -C /sw +ENV HPCX_HOME=/sw/$HPCX_VERSION +# ------ +# ENV +# - wrappers for gcc +RUN < /root/set_env.sh +#!/bin/bash +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION +GCC_INSTALL_DIR=\$(spack location -i gcc@$GCC_VERSION) +export GCC_INSTALL_DIR=\$GCC_INSTALL_DIR +export CC=\${GCC_INSTALL_DIR}/bin/gcc +export CXX=\${GCC_INSTALL_DIR}/bin/g++ +export FC=\${GCC_INSTALL_DIR}/bin/gfortran +EOF +RUN chmod +x /root/set_env.sh \ No newline at end of file diff --git a/docker/TotalEnergies/pangea4-gcc.Dockerfile b/docker/TotalEnergies/pangea4-gcc.Dockerfile new file mode 100644 index 00000000..126f4250 --- /dev/null +++ b/docker/TotalEnergies/pangea4-gcc.Dockerfile @@ -0,0 +1,50 @@ +####################################### +# Pangea 4 image : gcc - cmake - python +####################################### +# +# Installs : +# - gcc = 12.1 +# - cmake = 3.27.9 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) +# - this image is deployed as onetechssc/pangea4:gcc12.1_v1.0 +# - gcc, cmake, python and wget are installed via spack (as done for Pangea 4) +# +####################################### + +# ------------------------------------- +# PANGEA4 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/x86_64 architecture and redhat8 platform +FROM --platform=linux/x86_64 spack/centos-stream:latest AS pangea4-base +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=12.1 +ARG CMAKE_VERSION=3.27.9 +ARG PYTHON_VERSION=3.11 +# ------ +# INSTALL +# gcc +RUN spack install gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake +RUN spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION +# wget +RUN spack install wget %gcc@$GCC_VERSION +# ------ +# ENV +ENV GCC_VERSION=${GCC_VERSION} +ENV CMAKE_VERSION=${CMAKE_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pangea4.Dockerfile b/docker/TotalEnergies/pangea4.Dockerfile deleted file mode 100644 index 1d8e180e..00000000 --- a/docker/TotalEnergies/pangea4.Dockerfile +++ /dev/null @@ -1,82 +0,0 @@ -####################################### -# Pangea 4 image : gcc - hpcxompi - onemkl -####################################### -# -# Installs : -# - gcc = 12.1 -# - hpcx = 2.17.1 -# - intel-oneapi-mkl = 2023.2.0 -# - cmake = 3.27.9 -# - python = 3.11 -# -####################################### -# -# Description : -# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) -# - gcc, hpcx, oneAPI MKL, cmake and python are installed via spack (as done for Pangea 4) -# - hpcx is installed by extracting the tarball (as done for Pangea 4) -# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) -# -####################################### - -# ------------------------------------- -# PANGEA4 - BASE [GCC - CMAKE - PYTHON] -# install tools via spack for linux/x86_64 architecture and redhat8 platform -FROM --platform=linux/x86_64 spack/centos-stream:latest AS pangea4-base -# ------ -# LABELS -LABEL description="Pangea 4 image : gcc - cmake - python" -LABEL version="1.0" -LABEL maintainer="TotalEnergies HPC Team" -# ------ -# ARGS -ARG GCC_VERSION=12.1 -ARG CMAKE_VERSION=3.27.9 -ARG PYTHON_VERSION=3.11 -# ------ -# INSTALL -# gcc -RUN spack install gcc@$GCC_VERSION -RUN . /opt/spack/share/spack/setup-env.sh &&\ - spack load gcc@$GCC_VERSION &&\ - spack compiler find -# python -RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION -# show compilers -RUN spack compilers -# cmake -RUN spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION - -# ------------------------------------- -# PANGEA4 - GCC-HPCX-MKL [GCC - CMAKE - PYTHON - HPCX - ONEAPI-MKL] -FROM pangea4-base AS pangea4-gcc-hpcx-mkl -# ------ -# LABELS -LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" -LABEL version="1.0" -LABEL maintainer="TotalEnergies HPC Team" -# ------ -# ARGS -ARG ONEAPI_MKL_VERSION=2023.2.0 -ARG HPCX_VERSION="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64" -ARG HPCX_TARBALL="$HPCX_VERSION.tbz" -ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" -# ------ -# INSTALL -# intel-oneapi-mkl -RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION -# hpcx not available in spack -> download and untar (untar in /sw directory) -RUN mkdir -p /sw -RUN wget $HPCX_URL -O /tmp/$HPCX_TARBALL -RUN tar -xvf /tmp/$HPCX_TARBALL -C /sw -ENV HPCX_HOME=/sw/$HPCX_VERSION -# ------ -# ENV -# - create spack env -RUN spack env activate --create -p prgenv && \ - spack add gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION -# - wrappers for gcc -RUN spack env activate prgenv && export GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) -ENV CC=${GCC_INSTALL_DIR}/bin/gcc \ - CXX=${GCC_INSTALL_DIR}/bin/g++ \ - FC=${GCC_INSTALL_DIR}/bin/gfortran diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index a5568c04..4caff8cf 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -1,16 +1,16 @@ ####################################### -# Pangea 4 image : gcc - hpcxompi - onemkl +# Pangea 4 tpl image # # Description : # - generic image for building geos tpls on Pangea 4 environments # - the docker base image can be any pangea4 docker file -# - tools of the base image are expected to be available in prgenv spack environment +# - tools of the base image are expected to be sourced in the set_env.sh script ####################################### # ------------------------------------- # PANGEA4 - TPL TOOLCHAIN ARG DOCKER_ROOT_IMAGE -FROM $DOCKER_ROOT_IMAGE as pangea4_tpl_toolchain +FROM $DOCKER_ROOT_IMAGE AS pangea4_tpl_toolchain # ------ # LABELS LABEL description="Pangea 4 image : geos_tpl" @@ -28,12 +28,11 @@ ARG HOST_CONFIG ENV GEOSX_TPL_DIR=$INSTALL_DIR # ------ # INSTALL -RUN spack env activate prgenv && \ - --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh +RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ + $SRC_DIR/docker/configure-tpl.sh WORKDIR $BLD_DIR -RUN spack env activate prgenv && \ - --mount=src=.,dst=$SRC_DIR make -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR +RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ + make # ------ # CACHE # install `sccache` binaries to speed up the build of `geos` From bb23d51529c56766928ecb86423d1296135f2b70 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Thu, 25 Jul 2024 10:52:50 +0200 Subject: [PATCH 04/28] rename docker files and add doc --- .github/workflows/docker_build_tpls.yml | 4 ++-- docker/TotalEnergies/README.md | 19 +++++++++++++++++++ .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 2 +- .../{Dockerfile => tpl-pecan.Dockerfile} | 0 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 docker/TotalEnergies/README.md rename docker/TotalEnergies/{Dockerfile => tpl-pecan.Dockerfile} (100%) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 435d7038..e9f5cefe 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -28,13 +28,13 @@ jobs: INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKERFILE: docker/TotalEnergies/tpl-pecan.Dockerfile DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKERFILE: docker/TotalEnergies/tpl-pecan.Dockerfile DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU diff --git a/docker/TotalEnergies/README.md b/docker/TotalEnergies/README.md new file mode 100644 index 00000000..6a915b43 --- /dev/null +++ b/docker/TotalEnergies/README.md @@ -0,0 +1,19 @@ +# TotalEnergies Dockerfiles + +## Overview + +This directory contains several Dockerfiles used for building and deploying the TPLs. +Below is a table summarizing the purpose and key characteristics of each Dockerfile. + +| Dockerfile | Description | Base Image | Key Components Installed | +|------------------------------------------|------------------------------------------------------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| +| `pecan.Dockerfile` | Builds a toolchain with GCC, OpenMPI, CUDA, and Intel MKL for GEOS on Pecan. | `centos:7.7.1908` | GCC, OpenMPI, CUDA, Intel MKL, numactl-devel, ca-certificates, curl, Python3, zlib-devel | +| `tpl-pecan.Dockerfile` | Builds third-party libraries for GEOS on Pecan environment. | `pecan` | CMake, make, bc, file, bison, flex, patch, openssh-clients, texlive, graphviz, libxml2, git, sccache | +| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python, Wget | +| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL | +| `tpl-pangea4.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4_tpl_toolchain` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL, sccache | + +## DockerHub + +The Docker images built from the ${cluster}.Dockerfile are available on DockerHub under the `onetechssc` organization for: + - pangea4 cluster : [onetechssc/pangea4](https://hub.docker.com/r/onetechssc/pangea4) diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile index a8e3ebce..9a298360 100644 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -12,7 +12,7 @@ ####################################### # # Description : -# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.tpl-pecan.Dockerfile # - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.17.1-onemkl2023.2.0_v1.0 # - oneAPI MKL is installed via spack (as done for Pangea 4) # - hpcx is installed by extracting the tarball (as done for Pangea 4) diff --git a/docker/TotalEnergies/Dockerfile b/docker/TotalEnergies/tpl-pecan.Dockerfile similarity index 100% rename from docker/TotalEnergies/Dockerfile rename to docker/TotalEnergies/tpl-pecan.Dockerfile From 11cc69f4f2101ac9ebc05c1806b63c3b7c2583ad Mon Sep 17 00:00:00 2001 From: Gaetan Date: Thu, 25 Jul 2024 10:55:27 +0200 Subject: [PATCH 05/28] rename pecan base docker file --- docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile | 2 +- .../{pecan.Dockerfile => pecan-gcc-openmpi-mkl-cuda.Dockerfile} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docker/TotalEnergies/{pecan.Dockerfile => pecan-gcc-openmpi-mkl-cuda.Dockerfile} (100%) diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile index 9a298360..a8e3ebce 100644 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -12,7 +12,7 @@ ####################################### # # Description : -# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.tpl-pecan.Dockerfile +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile # - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.17.1-onemkl2023.2.0_v1.0 # - oneAPI MKL is installed via spack (as done for Pangea 4) # - hpcx is installed by extracting the tarball (as done for Pangea 4) diff --git a/docker/TotalEnergies/pecan.Dockerfile b/docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile similarity index 100% rename from docker/TotalEnergies/pecan.Dockerfile rename to docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile From 4c809ba194210a47bcab78e93d2e7d8e3d8949d7 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Thu, 25 Jul 2024 10:57:14 +0200 Subject: [PATCH 06/28] update doc --- docker/TotalEnergies/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/TotalEnergies/README.md b/docker/TotalEnergies/README.md index 6a915b43..b9a85a1a 100644 --- a/docker/TotalEnergies/README.md +++ b/docker/TotalEnergies/README.md @@ -5,13 +5,13 @@ This directory contains several Dockerfiles used for building and deploying the TPLs. Below is a table summarizing the purpose and key characteristics of each Dockerfile. -| Dockerfile | Description | Base Image | Key Components Installed | -|------------------------------------------|------------------------------------------------------------------------------|-------------------------|------------------------------------------------------------------------------------------------------| -| `pecan.Dockerfile` | Builds a toolchain with GCC, OpenMPI, CUDA, and Intel MKL for GEOS on Pecan. | `centos:7.7.1908` | GCC, OpenMPI, CUDA, Intel MKL, numactl-devel, ca-certificates, curl, Python3, zlib-devel | -| `tpl-pecan.Dockerfile` | Builds third-party libraries for GEOS on Pecan environment. | `pecan` | CMake, make, bc, file, bison, flex, patch, openssh-clients, texlive, graphviz, libxml2, git, sccache | -| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python, Wget | -| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL | -| `tpl-pangea4.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4_tpl_toolchain` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL, sccache | +| Dockerfile | Description | Base Image | Key Components Installed | +|------------------------------------------|------------------------------------------------------------------------------|-------------------------------|------------------------------------------------------------------------------------------------------| +| `pecan-gcc-openmpi-mkl-cuda.Dockerfile` | Builds a toolchain with GCC, OpenMPI, CUDA, and Intel MKL for GEOS on Pecan. | `centos:7.7.1908` | GCC, OpenMPI, CUDA, Intel MKL, numactl-devel, ca-certificates, curl, Python3, zlib-devel | +| `tpl-pecan.Dockerfile` | Builds third-party libraries for GEOS on Pecan environment. | `pecan-gcc-openmpi-mkl-cuda` | CMake, make, bc, file, bison, flex, patch, openssh-clients, texlive, graphviz, libxml2, git, sccache | +| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python, Wget | +| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL | +| `tpl-pangea4.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4-gcc-hpcxompi-onemkl` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL, sccache | ## DockerHub From 37ebcafd96e2dff88a8aff8c12e13b45a2c343d7 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 12:22:20 +0200 Subject: [PATCH 07/28] fix cray wrappers --- docker/TotalEnergies/README.md | 3 ++- .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 27 ++++++++++--------- docker/TotalEnergies/tpl-pangea4.Dockerfile | 4 ++- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docker/TotalEnergies/README.md b/docker/TotalEnergies/README.md index b9a85a1a..a3066afb 100644 --- a/docker/TotalEnergies/README.md +++ b/docker/TotalEnergies/README.md @@ -15,5 +15,6 @@ Below is a table summarizing the purpose and key characteristics of each Dockerf ## DockerHub -The Docker images built from the ${cluster}.Dockerfile are available on DockerHub under the `onetechssc` organization for: +The Docker images built from the ${cluster}.Dockerfile are available on DockerHub under for: - pangea4 cluster : [onetechssc/pangea4](https://hub.docker.com/r/onetechssc/pangea4) + - pecan cluster : [totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx](https://hub.docker.com/r/totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx) \ No newline at end of file diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile index a8e3ebce..5264bef6 100644 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -38,22 +38,25 @@ ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" # INSTALL # intel-oneapi-mkl RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION -# hpcx not available in spack -> download and untar (untar in /sw directory) -RUN mkdir -p /sw -RUN spack load wget && \ - wget $HPCX_URL -O /tmp/$HPCX_TARBALL -RUN tar -xvf /tmp/$HPCX_TARBALL -C /sw +# hpcx not available in spack -> download and untar in /sw directory +RUN mkdir -p /sw && \ + spack load wget && \ + wget $HPCX_URL -O /tmp/$HPCX_TARBALL && \ + tar -xvf /tmp/$HPCX_TARBALL -C /sw ENV HPCX_HOME=/sw/$HPCX_VERSION # ------ # ENV -# - wrappers for gcc +# - create wrappers for gcc +RUN mkdir -p /sw/cray-wrappers && \ + spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ + GCC_INSTALL_DIR=\$(spack location -i gcc@$GCC_VERSION) && \ + ln -s ${GCC_INSTALL_DIR}/bin/gcc cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran ftn +# create env script RUN < /root/set_env.sh #!/bin/bash -spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION -GCC_INSTALL_DIR=\$(spack location -i gcc@$GCC_VERSION) -export GCC_INSTALL_DIR=\$GCC_INSTALL_DIR -export CC=\${GCC_INSTALL_DIR}/bin/gcc -export CXX=\${GCC_INSTALL_DIR}/bin/g++ -export FC=\${GCC_INSTALL_DIR}/bin/gfortran +source ${HPCX_HOME}/hpcx-init.sh +PATH=/sw/cray-wrappers:\$PATH EOF RUN chmod +x /root/set_env.sh \ No newline at end of file diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index 4caff8cf..ac0e79b5 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -19,13 +19,15 @@ LABEL maintainer="TotalEnergies HPC Team" # ------ # ARGS ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG SRC_DIR=$TMP_DIR/src ARG BLD_DIR=$TMP_DIR/build ARG INSTALL_DIR ARG HOST_CONFIG # ------ # ENV ENV GEOSX_TPL_DIR=$INSTALL_DIR +ENV GCC_PATH=\${GCC_INSTALL_DIR} +ENV HPCX_MPI_DIR=\${HPCX_HOME} # ------ # INSTALL RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ From f7ce25ae04f1cb9002747cd0ade3f15cf0fe6eeb Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 14:54:59 +0200 Subject: [PATCH 08/28] fix hpcx and mkl --- .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 10 ++++++---- docker/TotalEnergies/tpl-pangea4.Dockerfile | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile index 5264bef6..d6db7110 100644 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -49,14 +49,16 @@ ENV HPCX_HOME=/sw/$HPCX_VERSION # - create wrappers for gcc RUN mkdir -p /sw/cray-wrappers && \ spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ - GCC_INSTALL_DIR=\$(spack location -i gcc@$GCC_VERSION) && \ - ln -s ${GCC_INSTALL_DIR}/bin/gcc cc && \ - ln -s ${GCC_INSTALL_DIR}/bin/g++ CC && \ - ln -s ${GCC_INSTALL_DIR}/bin/gfortran ftn + GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ + ln -s ${GCC_INSTALL_DIR}/bin/gcc /sw/cray-wrappers/cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ /sw/cray-wrappers/CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran /sw/cray-wrappers/ftn # create env script RUN < /root/set_env.sh #!/bin/bash +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ source ${HPCX_HOME}/hpcx-init.sh +hpcx_load PATH=/sw/cray-wrappers:\$PATH EOF RUN chmod +x /root/set_env.sh \ No newline at end of file diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index ac0e79b5..acbd78e8 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -31,7 +31,7 @@ ENV HPCX_MPI_DIR=\${HPCX_HOME} # ------ # INSTALL RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ - $SRC_DIR/docker/configure-tpl.sh + $SRC_DIR/docker/configure-tpl.sh -DMKL_LIBRARIES=\${MKLROOT}/lib/intel64/libmkl_rt.so WORKDIR $BLD_DIR RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ make From 048aa9caa023b7069f0ad4ee8922f07d08272b01 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 15:45:49 +0200 Subject: [PATCH 09/28] remove extra hpcx env --- docker/TotalEnergies/tpl-pangea4.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index acbd78e8..f8e5b2bb 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -27,7 +27,6 @@ ARG HOST_CONFIG # ENV ENV GEOSX_TPL_DIR=$INSTALL_DIR ENV GCC_PATH=\${GCC_INSTALL_DIR} -ENV HPCX_MPI_DIR=\${HPCX_HOME} # ------ # INSTALL RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ From cac538585cdd5a84e62745e777ffd1c4df23e322 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 15:46:26 +0200 Subject: [PATCH 10/28] use cray wrapper dir as ARG --- .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile index d6db7110..7b441e0c 100644 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -34,6 +34,7 @@ ARG ONEAPI_MKL_VERSION=2023.2.0 ARG HPCX_VERSION="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64" ARG HPCX_TARBALL="$HPCX_VERSION.tbz" ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" +ARG CRAY_WRAPPERS_DIR=/sw/cray-wrappers # ------ # INSTALL # intel-oneapi-mkl @@ -47,18 +48,18 @@ ENV HPCX_HOME=/sw/$HPCX_VERSION # ------ # ENV # - create wrappers for gcc -RUN mkdir -p /sw/cray-wrappers && \ +RUN mkdir -p $CRAY_WRAPPERS_DIR && \ spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ - ln -s ${GCC_INSTALL_DIR}/bin/gcc /sw/cray-wrappers/cc && \ - ln -s ${GCC_INSTALL_DIR}/bin/g++ /sw/cray-wrappers/CC && \ - ln -s ${GCC_INSTALL_DIR}/bin/gfortran /sw/cray-wrappers/ftn + ln -s ${GCC_INSTALL_DIR}/bin/gcc $CRAY_WRAPPERS_DIR/cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ $CRAY_WRAPPERS_DIR/CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran $CRAY_WRAPPERS_DIR/ftn # create env script RUN < /root/set_env.sh #!/bin/bash spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ source ${HPCX_HOME}/hpcx-init.sh hpcx_load -PATH=/sw/cray-wrappers:\$PATH +PATH=$CRAY_WRAPPERS_DIR:\$PATH EOF RUN chmod +x /root/set_env.sh \ No newline at end of file From 037a9c5645944754b7c9cbd0a68df20a7cbffb84 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 15:57:10 +0200 Subject: [PATCH 11/28] fix merge conflict --- .github/workflows/docker_build_tpls.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index e9f5cefe..63387f20 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -26,18 +26,6 @@ jobs: DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.17.1-onemkl2023.2.0_v1.0 HOST_CONFIG: host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl - - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) - DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - DOCKERFILE: docker/TotalEnergies/tpl-pecan.Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU - - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) - DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - DOCKERFILE: docker/TotalEnergies/tpl-pecan.Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 DOCKERFILE: docker/Stanford/Dockerfile From a56fced821c3e0213636495460fd4ad258afe9bc Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 16:23:23 +0200 Subject: [PATCH 12/28] fix blt merge conflict --- cmake/blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/blt b/cmake/blt index 5a792c17..9ff77344 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit 5a792c1775e7a7628d84dcde31652a689f1df7b5 +Subproject commit 9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35 From 5995cdc460ba3f5d1c93177c608a65dc624d908f Mon Sep 17 00:00:00 2001 From: Gaetan Date: Fri, 26 Jul 2024 16:28:35 +0200 Subject: [PATCH 13/28] fix missing runs on in workflow --- .github/workflows/docker_build_tpls.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 63387f20..b7c578e5 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -26,6 +26,7 @@ jobs: DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.17.1-onemkl2023.2.0_v1.0 HOST_CONFIG: host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl + RUNS_ON: ubuntu-latest - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 DOCKERFILE: docker/Stanford/Dockerfile From e684d42d5bd25521e67994c9699bad3af1833eb2 Mon Sep 17 00:00:00 2001 From: Gaetan Date: Mon, 29 Jul 2024 13:51:15 +0200 Subject: [PATCH 14/28] add ninja install for geos --- docker/TotalEnergies/tpl-pangea4.Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile index f8e5b2bb..d8af562d 100644 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ b/docker/TotalEnergies/tpl-pangea4.Dockerfile @@ -29,13 +29,15 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR ENV GCC_PATH=\${GCC_INSTALL_DIR} # ------ # INSTALL +# - configure TPLs RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ $SRC_DIR/docker/configure-tpl.sh -DMKL_LIBRARIES=\${MKLROOT}/lib/intel64/libmkl_rt.so +# - build TPLs WORKDIR $BLD_DIR RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ - make -# ------ -# CACHE -# install `sccache` binaries to speed up the build of `geos` + make \ +# - install ninja for geos build +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh +# - install `sccache` binaries to speed up the build of `geos` RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh ENV SCCACHE=/opt/sccache/bin/sccache From 999568a86eca62142e6678689c3de8cd2460175f Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Thu, 14 Nov 2024 16:48:14 +0100 Subject: [PATCH 15/28] split p3 vs p4 --- .github/workflows/docker_build_tpls.yml | 14 +- docker/TotalEnergies/README.md | 16 +- .../pangea3-gcc-openmpi-openblas.Dockerfile} | 0 .../pangea3-tpl.Dockerfile} | 0 .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 81 ++++++++ .../pangea-4/pangea4-gcc.Dockerfile | 67 ++++++ .../pangea-4/pangea4-tpl.Dockerfile | 92 +++++++++ .../pangea4-gcc-hpcxompi-onemkl.Dockerfile | 65 ------ docker/TotalEnergies/pangea4-gcc.Dockerfile | 50 ----- docker/TotalEnergies/pecan-CPU.cmake | 31 --- docker/TotalEnergies/pecan-GPU.cmake | 24 --- .../pecan-gcc-openmpi-mkl-cuda.Dockerfile | 193 ------------------ docker/TotalEnergies/tpl-pangea4.Dockerfile | 43 ---- docker/TotalEnergies/tpl-pecan.Dockerfile | 58 ------ .../pangea3-gcc8.4.1-openmpi-4.1.2.cmake | 0 .../pangea-4}/pangea4-base.cmake | 0 ...c12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake} | 10 +- ...c12.1-hpcxompi2.20.0-openblas0.3.23.cmake} | 4 +- ...-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake | 6 +- ...-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake | 0 20 files changed, 269 insertions(+), 485 deletions(-) rename docker/TotalEnergies/{Pangea3-base.Dockerfile => pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile} (100%) rename docker/TotalEnergies/{Pangea3.Dockerfile => pangea-3/pangea3-tpl.Dockerfile} (100%) create mode 100644 docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile create mode 100644 docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile create mode 100644 docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile delete mode 100644 docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile delete mode 100644 docker/TotalEnergies/pangea4-gcc.Dockerfile delete mode 100644 docker/TotalEnergies/pecan-CPU.cmake delete mode 100644 docker/TotalEnergies/pecan-GPU.cmake delete mode 100644 docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile delete mode 100644 docker/TotalEnergies/tpl-pangea4.Dockerfile delete mode 100644 docker/TotalEnergies/tpl-pecan.Dockerfile rename host-configs/{TOTAL => TotalEnergies/pangea-3}/pangea3-gcc8.4.1-openmpi-4.1.2.cmake (100%) rename host-configs/{TOTAL => TotalEnergies/pangea-4}/pangea4-base.cmake (100%) rename host-configs/{TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake => TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake} (92%) rename host-configs/{TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake => TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake} (97%) rename host-configs/{TOTAL => TotalEnergies/pangea-4}/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake (95%) rename host-configs/{TOTAL => TotalEnergies/pangea-4}/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake (100%) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index d644d00e..d8a787b2 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -63,16 +63,16 @@ jobs: RUNS_ON: Runner_4core_16GB - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake + TPL_DOCKERFILE: docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile + HOST_CONFIG: host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 HOST_ARCH: ppc64le RUNS_ON: streak2 - - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.17.1, mkl 2023.2.0) - DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0 - DOCKERFILE: docker/TotalEnergies/tpl-pangea4.Dockerfile - DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.17.1-onemkl2023.2.0_v1.0 - HOST_CONFIG: host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake + - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.20.0, mkl 2023.2.0) + DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0 + DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 + HOST_CONFIG: host-configs/TotalEnergies/pangea4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl steps: diff --git a/docker/TotalEnergies/README.md b/docker/TotalEnergies/README.md index a3066afb..30eba182 100644 --- a/docker/TotalEnergies/README.md +++ b/docker/TotalEnergies/README.md @@ -5,16 +5,16 @@ This directory contains several Dockerfiles used for building and deploying the TPLs. Below is a table summarizing the purpose and key characteristics of each Dockerfile. -| Dockerfile | Description | Base Image | Key Components Installed | -|------------------------------------------|------------------------------------------------------------------------------|-------------------------------|------------------------------------------------------------------------------------------------------| -| `pecan-gcc-openmpi-mkl-cuda.Dockerfile` | Builds a toolchain with GCC, OpenMPI, CUDA, and Intel MKL for GEOS on Pecan. | `centos:7.7.1908` | GCC, OpenMPI, CUDA, Intel MKL, numactl-devel, ca-certificates, curl, Python3, zlib-devel | -| `tpl-pecan.Dockerfile` | Builds third-party libraries for GEOS on Pecan environment. | `pecan-gcc-openmpi-mkl-cuda` | CMake, make, bc, file, bison, flex, patch, openssh-clients, texlive, graphviz, libxml2, git, sccache | -| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python, Wget | -| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL | -| `tpl-pangea4.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4-gcc-hpcxompi-onemkl` | GCC, CMake, Python, Wget, HPCXOMPI, ONEMKL, sccache | +| Dockerfile | Description | Base Image | Key Components Installed | +|------------------------------------------|-------------------------------------------------------------------------------|-------------------------------|----------------------------------------------------------------------| +| `pangea3-gcc-openmpi-openblas.Dockerfile`| Builds a toolchain with GCC, OpenMPI, CUDA, and OpenBLAS for GEOS on Pangea 3.| `ppc64le/almalinux:8` | GCC, OpenMPI, CUDA, OpenBLAS, CMake, Python3.8 | +| `pangea3-tpl.Dockerfile` | Builds third-party libraries for GEOS on Pangea 3 environment. | `pangea3-gcc-openmpi-openblas`| GCC, OpenMPI, CUDA, OpenBLAS, CMake, Python3.8, TPLs | +| `pangea4-gcc.Dockerfile` | Builds a toolchain with GCC, CMake and Python on Pangea 4. | `spack/centos-stream` | GCC, CMake, Python3.11, Wget | +| `pangea4-gcc-hpcxompi-onemkl.Dockerfile` | Builds a toolchain with GCC, HPCXOMPI and ONEMKL on Pangea 4. | `pangea4-gcc` | GCC, CMake, Python3.11, Wget, HPCXOMPI, ONEMKL | +| `pangea4-tpl.Dockerfile` | Builds third-party libraries for GEOS on Pangea 4 environment. | `pangea4-gcc-hpcxompi-onemkl` | GCC, CMake, Python3.11, Wget, HPCXOMPI, ONEMKL, ninja, sccache, TPLs | ## DockerHub The Docker images built from the ${cluster}.Dockerfile are available on DockerHub under for: + - pangea3 cluster : [7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18](https://hub.docker.com/r/7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18), will move to onetechssc/pangea3 once newer version of gcc is available. - pangea4 cluster : [onetechssc/pangea4](https://hub.docker.com/r/onetechssc/pangea4) - - pecan cluster : [totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx](https://hub.docker.com/r/totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx) \ No newline at end of file diff --git a/docker/TotalEnergies/Pangea3-base.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile similarity index 100% rename from docker/TotalEnergies/Pangea3-base.Dockerfile rename to docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile similarity index 100% rename from docker/TotalEnergies/Pangea3.Dockerfile rename to docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile new file mode 100644 index 00000000..5085401a --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -0,0 +1,81 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +####################################### +# +# Installs : +# - hpcx = 2.20.0 +# - intel-oneapi-mkl = 2023.2.0 +# +####################################### +# +# Description : +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile +# - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.20.0-onemkl2023.2.0_v1.0 +# - oneAPI MKL is installed via spack (as done for Pangea 4) +# - hpcx is installed by extracting the tarball (as done for Pangea 4) +# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 -f pangea4-gcc-hpcxompi-onemkl.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc_hpcxompi_onemkl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_gcc_hpcxompi_onemkl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - GCC-HPCX-MKL +FROM docker.io/onetechssc/pangea4:gcc12.1_v1.0 AS pangea4 +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG ONEAPI_MKL_VERSION=2023.2.0 +ARG HPCX_VERSION=2.20 +ARG HPCX_FULL_NAME="hpcx-v$HPCX_VERSION-gcc-mlnx_ofed-redhat8-cuda12-x86_64" +ARG HPCX_TARBALL="$HPCX_FULL_NAME.tbz" +ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v$HPCX_VERSION/$HPCX_TARBALL" +ARG CRAY_WRAPPERS_DIR=/sw/cray-wrappers +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# intel-oneapi-mkl +RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx not available in spack -> download and untar in /sw directory +RUN mkdir -p /sw && \ + spack load wget && \ + wget --ca-certificate=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem $HPCX_URL -O /tmp/$HPCX_TARBALL && \ + tar -xvf /tmp/$HPCX_TARBALL -C /sw && \ + rm -f /tmp/$HPCX_TARBALL +ENV HPCX_HOME=/sw/$HPCX_FULL_NAME +# numpy with mkl +RUN spack install py-numpy %gcc@$GCC_VERSION ^python@$PYTHON_VERSION ^intel-oneapi-mkl@$ONEAPI_MKL_VERSION +# ------ +# ENV +# create wrappers for gcc +RUN mkdir -p $CRAY_WRAPPERS_DIR && \ + spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ + GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ + ln -s ${GCC_INSTALL_DIR}/bin/gcc $CRAY_WRAPPERS_DIR/cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ $CRAY_WRAPPERS_DIR/CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran $CRAY_WRAPPERS_DIR/ftn +# create env script +RUN < /root/.setup_env.sh +. /opt/spack/share/spack/setup-env.sh +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION py-numpy +source ${HPCX_HOME}/hpcx-init.sh +hpcx_load +export PATH=$CRAY_WRAPPERS_DIR:\$PATH +export GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) +export GOMP_ROOT=\${GCC_INSTALL_DIR}/lib64 +EOF +RUN chmod +x /root/.setup_env.sh diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile new file mode 100644 index 00000000..920162fd --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.4 + +####################################### +# Pangea 4 image : gcc - cmake - python +####################################### +# +# Installs : +# - gcc = 12.1 +# - cmake = 3.27.9 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) +# - this image is deployed as onetechssc/pangea4:gcc12.1_v1.0 +# - gcc, cmake, python and wget are installed via spack (as done for Pangea 4) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1_v1.0 -f pangea4-gcc.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1_v1.0 +# - podman exec -it pangea4_gcc /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/x86_64 architecture and redhat8 platform +FROM --platform=linux/x86_64 docker.io/spack/centos-stream:latest AS pangea4-base +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=12.1 +ARG CMAKE_VERSION=3.27.9 +ARG PYTHON_VERSION=3.11 +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# gcc +RUN spack install gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake (openssl error when using curl) +RUN spack config --scope defaults add config:url_fetch_method:urllib &&\ + spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION &&\ + spack config --scope defaults add config:url_fetch_method:curl +# wget +RUN spack install wget %gcc@$GCC_VERSION +# ------ +# ENV +ENV GCC_VERSION=${GCC_VERSION} +ENV CMAKE_VERSION=${CMAKE_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile new file mode 100644 index 00000000..332cc1d1 --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile @@ -0,0 +1,92 @@ +####################################### +# Pangea 4 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 4 environments +# - the docker base image can be any pangea4 docker file +# - tools of the base image are expected to be sourced in the set_env.sh script +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg HOST_CONFIG=host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p4/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# -f docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea4_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea4_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install --build-exe-only +####################################### + +# ------------------------------------- +# PANGEA4 - TPL BASE +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE AS pangea4_tpl_base +# ------ +# LABELS +LABEL description="Pangea 4 image : geos_tpl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG INSTALL_DIR +# ------ +# ENV +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR +ENV GCC_PATH=\${GCC_INSTALL_DIR} + +# ------------------------------------- +# PANGEA4 - TPL BUILDER +FROM pangea4_tpl_base AS pangea4_tpl_builder +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +ARG BLD_DIR=$TMP_DIR/build +ARG HOST_CONFIG +# ------ +# INSTALL +# - configure TPLs +RUN --mount=src=.,dst=$SRC_DIR source /root/.setup_env.sh && \ + $SRC_DIR/docker/configure-tpl.sh +# - build TPLs +WORKDIR $BLD_DIR +RUN --mount=src=.,dst=$SRC_DIR source /root/.setup_env.sh && \ + make + +# ------------------------------------- +# PANGEA4 - TPL TOOLCHAIN +FROM pangea4_tpl_base AS pangea4_tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# - copy TPLs install directory +COPY --from=pangea4_tpl_builder $GEOS_TPL_DIR $GEOS_TPL_DIR +# - install ninja for geos build +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh +# - install `sccache` binaries to speed up the build of `geos` +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +# ------ +# ENV +ENV SCCACHE=/opt/sccache/bin/sccache +# ------ +# ENTRYPOINT +# set entry point for geos ci build script +ENTRYPOINT ["/bin/bash", "-c", "source /root/.setup_env.sh && exec \"$@\"", "--"] \ No newline at end of file diff --git a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile deleted file mode 100644 index 7b441e0c..00000000 --- a/docker/TotalEnergies/pangea4-gcc-hpcxompi-onemkl.Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -####################################### -# Pangea 4 image : gcc - hpcxompi - onemkl -####################################### -# -# Installs : -# - gcc = 12.1 -# - hpcx = 2.17.1 -# - intel-oneapi-mkl = 2023.2.0 -# - cmake = 3.27.9 -# - python = 3.11 -# -####################################### -# -# Description : -# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile -# - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.17.1-onemkl2023.2.0_v1.0 -# - oneAPI MKL is installed via spack (as done for Pangea 4) -# - hpcx is installed by extracting the tarball (as done for Pangea 4) -# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) -# -####################################### - -# ------------------------------------- -# PANGEA4 - GCC-HPCX-MKL -FROM onetechssc/pangea4:gcc12.1_v1.0 AS pangea4 -# ------ -# LABELS -LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" -LABEL version="1.0" -LABEL maintainer="TotalEnergies HPC Team" -# ------ -# ARGS -ARG ONEAPI_MKL_VERSION=2023.2.0 -ARG HPCX_VERSION="hpcx-v2.17.1-gcc-mlnx_ofed-redhat8-cuda12-x86_64" -ARG HPCX_TARBALL="$HPCX_VERSION.tbz" -ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v2.17.1/$HPCX_TARBALL" -ARG CRAY_WRAPPERS_DIR=/sw/cray-wrappers -# ------ -# INSTALL -# intel-oneapi-mkl -RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION -# hpcx not available in spack -> download and untar in /sw directory -RUN mkdir -p /sw && \ - spack load wget && \ - wget $HPCX_URL -O /tmp/$HPCX_TARBALL && \ - tar -xvf /tmp/$HPCX_TARBALL -C /sw -ENV HPCX_HOME=/sw/$HPCX_VERSION -# ------ -# ENV -# - create wrappers for gcc -RUN mkdir -p $CRAY_WRAPPERS_DIR && \ - spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ - GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ - ln -s ${GCC_INSTALL_DIR}/bin/gcc $CRAY_WRAPPERS_DIR/cc && \ - ln -s ${GCC_INSTALL_DIR}/bin/g++ $CRAY_WRAPPERS_DIR/CC && \ - ln -s ${GCC_INSTALL_DIR}/bin/gfortran $CRAY_WRAPPERS_DIR/ftn -# create env script -RUN < /root/set_env.sh -#!/bin/bash -spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ -source ${HPCX_HOME}/hpcx-init.sh -hpcx_load -PATH=$CRAY_WRAPPERS_DIR:\$PATH -EOF -RUN chmod +x /root/set_env.sh \ No newline at end of file diff --git a/docker/TotalEnergies/pangea4-gcc.Dockerfile b/docker/TotalEnergies/pangea4-gcc.Dockerfile deleted file mode 100644 index 126f4250..00000000 --- a/docker/TotalEnergies/pangea4-gcc.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -####################################### -# Pangea 4 image : gcc - cmake - python -####################################### -# -# Installs : -# - gcc = 12.1 -# - cmake = 3.27.9 -# - python = 3.11 -# -####################################### -# -# Description : -# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) -# - this image is deployed as onetechssc/pangea4:gcc12.1_v1.0 -# - gcc, cmake, python and wget are installed via spack (as done for Pangea 4) -# -####################################### - -# ------------------------------------- -# PANGEA4 - BASE [GCC - CMAKE - PYTHON] -# install tools via spack for linux/x86_64 architecture and redhat8 platform -FROM --platform=linux/x86_64 spack/centos-stream:latest AS pangea4-base -# ------ -# LABELS -LABEL description="Pangea 4 image : gcc - cmake - python" -LABEL version="1.0" -LABEL maintainer="TotalEnergies HPC Team" -# ------ -# ARGS -ARG GCC_VERSION=12.1 -ARG CMAKE_VERSION=3.27.9 -ARG PYTHON_VERSION=3.11 -# ------ -# INSTALL -# gcc -RUN spack install gcc@$GCC_VERSION -RUN . /opt/spack/share/spack/setup-env.sh &&\ - spack load gcc@$GCC_VERSION &&\ - spack compiler find -# python -RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION -# cmake -RUN spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION -# wget -RUN spack install wget %gcc@$GCC_VERSION -# ------ -# ENV -ENV GCC_VERSION=${GCC_VERSION} -ENV CMAKE_VERSION=${CMAKE_VERSION} -ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pecan-CPU.cmake b/docker/TotalEnergies/pecan-CPU.cmake deleted file mode 100644 index 722ea2b4..00000000 --- a/docker/TotalEnergies/pecan-CPU.cmake +++ /dev/null @@ -1,31 +0,0 @@ -set(COMPILER_HOME "/apps/gcc/8.2.0/x86_64") -set(MPI_HOME "/hrtc/apps/mpi/openmpi/4.0.1/RDHPC/gcc/8.2.0") - -set(CMAKE_C_COMPILER ${COMPILER_HOME}/bin/gcc CACHE PATH "" FORCE) -set(CMAKE_CXX_COMPILER ${COMPILER_HOME}/bin/g++ CACHE PATH "" FORCE) -set(CMAKE_Fortran_COMPILER ${COMPILER_HOME}/bin/gfortran CACHE PATH "" FORCE) -set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE) - -set(ENABLE_MPI ON CACHE PATH "" FORCE) -set(ENABLE_OPENMP ON CACHE PATH "" FORCE) - -set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "" FORCE) -set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "" FORCE) -set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpifort" CACHE PATH "" FORCE) -set(MPIEXEC_EXECUTABLE "${MPI_HOME}/bin/mpirun" CACHE PATH "" FORCE) -#set(MPIEXEC_EXECUTABLE /apps/slurm/x86/20.02.0/bin/srun CACHE PATH "" FORCE) -#set(MPIEXEC_NUMPROC_FLAG "-p pecan -n" CACHE STRING "") -set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") - -set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "" FORCE) -set(ENABLE_CALIPER ON CACHE BOOL "") - -set(ENABLE_MKL ON CACHE BOOL "") -set(INTEL_ROOT "/apps/intel/2019/u5/compilers_and_libraries_2019.5.281/linux" ) -set(MKL_ROOT "${INTEL_ROOT}/mkl" ) -set(MKL_INCLUDE_DIRS ${MKL_ROOT}/include CACHE STRING "") -set(MKL_LIBRARIES ${MKL_ROOT}/lib/intel64/libmkl_intel_lp64.so - ${MKL_ROOT}/lib/intel64/libmkl_intel_thread.so - ${MKL_ROOT}/lib/intel64/libmkl_core.so - ${INTEL_ROOT}/compiler/lib/intel64_lin/libiomp5.so - CACHE STRING "") diff --git a/docker/TotalEnergies/pecan-GPU.cmake b/docker/TotalEnergies/pecan-GPU.cmake deleted file mode 100644 index ef6e4457..00000000 --- a/docker/TotalEnergies/pecan-GPU.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Retrieve the compilers, standard libraries... from the CPU configuration -include(${CMAKE_CURRENT_LIST_DIR}/pecan-CPU.cmake) - -# Now let's add what's dedicated to GPU. -set(ENABLE_CUDA ON CACHE PATH "" FORCE) -set(CUDA_TOOLKIT_ROOT_DIR /hrtc/apps/cuda/11.5.119/x86_64/centos7 CACHE PATH "") -set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "") -set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "") - -set(CUDA_ARCH sm_75 CACHE STRING "") -set(CMAKE_CUDA_ARCHITECTURES 75 CACHE STRING "") -set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-relaxed-constexpr --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "") - -# Current version of hypre does not build with GPU support inside of docker. -# Hypre's build system awaits to be embedded into a CUDA environment. -# This is a bit tedious to reproduce in docker environment. -# And since most recent version of hypre do build without this constraint. -# Let's wait for an upgrade on our side. -# In the mean time, if you need the GPU support for hypre, -# simply install the classical way, with some `module load cuda`. -#set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE) diff --git a/docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile b/docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile deleted file mode 100644 index 3e19fbef..00000000 --- a/docker/TotalEnergies/pecan-gcc-openmpi-mkl-cuda.Dockerfile +++ /dev/null @@ -1,193 +0,0 @@ -ARG GCC_VERSION=8.2.0 -ARG OPENMPI_VERSION=4.0.1 -ARG UCX_VERSION=1.3.0 -# Note that you need to define both SLURM_TARBALL and SLURM_HOME because one cannot be deduced from the other smoothly. -# It would require (heavy?) text processing for little benefit. -ARG SLURM_TARBALL=slurm-20-02-0-1.tar.gz -ARG SLURM_HOME=/apps/slurm/x86/20.02.0 - -ARG GCC_HOME=/apps/gcc/${GCC_VERSION}/x86_64 -ARG UCX_HOME=/hrtc/apps/devtools/ucx/${UCX_VERSION}/x86_64_nocuda/gcc/${GCC_VERSION} -ARG OPENMPI_HOME=/hrtc/apps/mpi/openmpi/${OPENMPI_VERSION}/RDHPC/gcc/${GCC_VERSION} -# While the installation directory is defined here, the patches and exact versions are still defined in the CUDA stage. -ARG CUDA_HOME=/hrtc/apps/cuda/11.5.119/x86_64/centos7 - -FROM centos:7.7.1908 AS shared_components - -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo - -RUN yum update -y && \ - yum install -y \ - glibc-devel - -FROM shared_components AS gcc_stage - -ARG GCC_VERSION -ARG GCC_HOME - -# FIXME wget could be replaced by curl in the contrib/download_prerequisites script. To be challenged. -RUN yum install -y \ - make \ - gcc \ - gcc-c++ \ - wget \ - bzip2 \ - zlib-devel - -WORKDIR /tmp/src -RUN curl -s https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./contrib/download_prerequisites -RUN ./configure \ - --prefix=${GCC_HOME} \ - --disable-multilib \ - --with-system-zlib \ - --enable-threads=posix \ - --enable-languages=c,c++,fortran -RUN make -j $(nproc) && make install-strip - -FROM shared_components AS ucx_stage - -ARG UCX_VERSION -ARG UCX_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - make \ - numactl-devel - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/ucx-${UCX_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure \ - --prefix=${UCX_HOME} \ - --enable-mt \ - --disable-optimizations \ - --disable-logging \ - --disable-debug \ - --disable-assertions -RUN make -j $(nproc) && make install - -FROM shared_components AS slurm_stage - -ARG SLURM_TARBALL -ARG SLURM_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y perl python3 file make - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/SchedMD/slurm/archive/${SLURM_TARBALL} | tar --strip-components=1 -xzf - -RUN ./configure --prefix=${SLURM_HOME} -RUN make -j $(nproc) && make install - -FROM shared_components AS openmpi_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -# FIXME so we do not use SLURM? -ARG OPENMPI_VERSION -ARG OPENMPI_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - perl \ - make \ - zlib-devel \ - numactl-devel -# FIXME deal with the devel probably too much - -WORKDIR /tmp/src -RUN curl -fsSL https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION%.[0-9]*}/openmpi-${OPENMPI_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure CC=$CC FC=$FC CXX=$CXX \ - --prefix=${OPENMPI_HOME} \ - --enable-static \ - --enable-smp-locks \ - --enable-mpi-thread-multiple \ - --with-slurm \ - --with-ucx=${UCX_HOME} \ - --with-ucx-libdir=${UCX_HOME}/lib \ - --with-io-romio-flags=--with-file-system=testfs+ufs+lustre -RUN make -j $(nproc) && make install - -FROM shared_components AS cuda_stage - -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ARG CUDA_HOME - -# FIXME Why glibc-devel?!?!? -RUN yum install -y which glibc-devel - -ENV PATH=${GCC_HOME}/bin:${PATH} \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -WORKDIR /tmp/src -ADD https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run . -RUN mkdir -p ${CUDA_HOME} -RUN sh cuda_11.5.0_495.29.05_linux.run --silent --toolkit --no-man-page --installpath=${CUDA_HOME} - -FROM shared_components AS intel_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -ARG OPENMPI_HOME -ARG CUDA_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} -COPY --from=openmpi_stage ${OPENMPI_HOME} ${OPENMPI_HOME} -COPY --from=cuda_stage ${CUDA_HOME} ${CUDA_HOME} - -RUN yum install -y \ - numactl-devel -# FIXME maybe only numactl-libs? - -RUN rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB -RUN yum-config-manager \ - --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -RUN yum install -y \ - intel-mkl-2019.5-075.x86_64 && \ - yum clean all - -ARG PECAN_INTEL_HOME=/apps/intel/2019/u5 -ARG INTEL_DIR_NAME=compilers_and_libraries_2019.5.281 -RUN mkdir -p /apps/intel/2019/u5 && \ - ln -s /opt/intel/${INTEL_DIR_NAME} ${PECAN_INTEL_HOME}/${INTEL_DIR_NAME} - -# Exposing quite everything, making future modularization more complicated. -# Most likely there will be no future modularization! -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - MPICC=${OPENMPI_HOME}/bin/mpicc \ - MPICXX=${OPENMPI_HOME}/bin/mpicxx \ - MPIFC=${OPENMPI_HOME}/bin/mpifort \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/linux/mkl/lib/intel64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/compiler/lib/intel64:${OPENMPI_HOME}/lib diff --git a/docker/TotalEnergies/tpl-pangea4.Dockerfile b/docker/TotalEnergies/tpl-pangea4.Dockerfile deleted file mode 100644 index d8af562d..00000000 --- a/docker/TotalEnergies/tpl-pangea4.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -####################################### -# Pangea 4 tpl image -# -# Description : -# - generic image for building geos tpls on Pangea 4 environments -# - the docker base image can be any pangea4 docker file -# - tools of the base image are expected to be sourced in the set_env.sh script -####################################### - -# ------------------------------------- -# PANGEA4 - TPL TOOLCHAIN -ARG DOCKER_ROOT_IMAGE -FROM $DOCKER_ROOT_IMAGE AS pangea4_tpl_toolchain -# ------ -# LABELS -LABEL description="Pangea 4 image : geos_tpl" -LABEL version="1.0" -LABEL maintainer="TotalEnergies HPC Team" -# ------ -# ARGS -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/src -ARG BLD_DIR=$TMP_DIR/build -ARG INSTALL_DIR -ARG HOST_CONFIG -# ------ -# ENV -ENV GEOSX_TPL_DIR=$INSTALL_DIR -ENV GCC_PATH=\${GCC_INSTALL_DIR} -# ------ -# INSTALL -# - configure TPLs -RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ - $SRC_DIR/docker/configure-tpl.sh -DMKL_LIBRARIES=\${MKLROOT}/lib/intel64/libmkl_rt.so -# - build TPLs -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR source /root/set_env.sh && \ - make \ -# - install ninja for geos build -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh -# - install `sccache` binaries to speed up the build of `geos` -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/tpl-pecan.Dockerfile b/docker/TotalEnergies/tpl-pecan.Dockerfile deleted file mode 100644 index 929cd081..00000000 --- a/docker/TotalEnergies/tpl-pecan.Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build - -# The docker base image can be pecan or pangea. -ARG DOCKER_ROOT_IMAGE -FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR - -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR - -RUN yum install -y \ - ca-certificates \ - curl \ - python3 \ - zlib-devel - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain -ARG SRC_DIR -ARG BLD_DIR - -RUN yum install -y \ - make \ - bc \ - file \ - bison \ - flex \ - patch - -ARG HOST_CONFIG - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR make - -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain -ARG SRC_DIR - -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR - -RUN yum -y install \ - openssh-clients \ - ca-certificates \ - curl \ - python3 \ - texlive \ - graphviz \ - libxml2 \ - git - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake b/host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake similarity index 100% rename from host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake rename to host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake diff --git a/host-configs/TOTAL/pangea4-base.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-base.cmake similarity index 100% rename from host-configs/TOTAL/pangea4-base.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-base.cmake diff --git a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake similarity index 92% rename from host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake index 488ca0cc..5b3402c2 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake @@ -20,7 +20,7 @@ # - cray-python = 3.10.10 # - craype-x86-milan = 1.0 # PrgEnv-gnu loads gcc 12 that does not support craype-x86-genoa -# - hpcx = 2.17.1 +# - hpcx = 2.20.0 # - intel-oneapi-mkl = 2023.2.0 # # Load modules this way : @@ -31,7 +31,7 @@ # ######################################## -set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.17.1-onemkl2023.2.0" CACHE PATH "" ) +set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0" CACHE PATH "" ) include(${CMAKE_CURRENT_LIST_DIR}/pangea4-base.cmake) @@ -91,7 +91,11 @@ if( NOT DEFINED ENV{MKLROOT} ) message( FATAL_ERROR "MKL is not loaded. Please load the intel-oneapi-mkl/2023.2.0 module." ) endif() +if ( NOT DEFINED ENV{GOMP_ROOT} ) + set( GOMP_ROOT $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so ) +endif() + set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) set( MKL_LIBRARIES $ENV{MKLROOT}/lib/intel64/libmkl_rt.so - $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so + $ENV{GOMP_ROOT}/libgomp.so CACHE STRING "" ) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake similarity index 97% rename from host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake index 42d05975..f7c09f2e 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23.cmake @@ -20,7 +20,7 @@ # - cray-python = 3.10.10 # - craype-x86-milan = 1.0 # PrgEnv-gnu loads gcc 12 that does not support craype-x86-genoa -# - hpcx = 2.17.1 +# - hpcx = 2.20.0 # - openblas = 0.3.23 # # Load modules this way : @@ -31,7 +31,7 @@ # ######################################## -set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.17.1-openblas0.3.23" CACHE PATH "" ) +set( CONFIG_NAME "pangea4-gcc12.1-hpcxompi2.20.0-openblas0.3.23" CACHE PATH "" ) include(${CMAKE_CURRENT_LIST_DIR}/pangea4-base.cmake) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake similarity index 95% rename from host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake index 5ede5b87..0cd79ad6 100644 --- a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake @@ -91,7 +91,11 @@ if( NOT DEFINED ENV{MKLROOT} ) message( FATAL_ERROR "MKL is not loaded. Please load the intel-oneapi-mkl/2023.2.0 module." ) endif() +if ( NOT DEFINED ENV{GOMP_ROOT} ) + set( GOMP_ROOT $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so ) +endif() + set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) set( MKL_LIBRARIES $ENV{MKLROOT}/lib/intel64/libmkl_rt.so - $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so + $ENV{GOMP_ROOT}/libgomp.so CACHE STRING "" ) diff --git a/host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake similarity index 100% rename from host-configs/TOTAL/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake rename to host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-openblas0.3.23.cmake From fd35c42315496b9de9750b7f5b4e1c2758a63c2a Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Thu, 14 Nov 2024 17:21:58 +0100 Subject: [PATCH 16/28] run P4 build on ubuntu latest --- .github/workflows/docker_build_tpls.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index d8a787b2..9d844f2d 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -74,6 +74,7 @@ jobs: DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 HOST_CONFIG: host-configs/TotalEnergies/pangea4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl + RUNS_ON: ubuntu-latest steps: - name: Checkout From daf5cd55ca8bdaf12af567c5106c4907351a7645 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Thu, 14 Nov 2024 17:25:19 +0100 Subject: [PATCH 17/28] fix tpl docker file tag --- .github/workflows/docker_build_tpls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 9d844f2d..8817d06a 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -70,7 +70,7 @@ jobs: RUNS_ON: streak2 - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.20.0, mkl 2023.2.0) DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0 - DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile + TPL_DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 HOST_CONFIG: host-configs/TotalEnergies/pangea4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl From 2e9e5c16a90f3d2363823080ac9f7adc7bb61a57 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Thu, 14 Nov 2024 17:29:58 +0100 Subject: [PATCH 18/28] fix typo in p4 host config --- .github/workflows/docker_build_tpls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 8817d06a..3b5888f0 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -72,7 +72,7 @@ jobs: DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0 TPL_DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 - HOST_CONFIG: host-configs/TotalEnergies/pangea4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake + HOST_CONFIG: host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl RUNS_ON: ubuntu-latest From b154d2bad96682b79dcfe61c8b039c45a7e4e809 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 27 Nov 2024 13:51:39 +0100 Subject: [PATCH 19/28] set clang version to 17.0.6 --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index ff5752a7..86f115f8 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -14,7 +14,7 @@ RUN dnf clean all && \ dnf -y update && \ dnf -y install \ which \ - clang \ + clang-17.0.6 \ gcc-gfortran \ python3 \ zlib-devel \ From 4046a5173937f2befb1dd927b742749ea1238063 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 27 Nov 2024 15:19:41 +0100 Subject: [PATCH 20/28] no update for clang --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 86f115f8..14bd5d84 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -66,7 +66,7 @@ ENV OMPI_FC=$FC # Install required packages using dnf RUN dnf clean all && \ - dnf -y update && \ + dnf -y --exclude=clang update && \ dnf -y install \ tbb-devel \ bc \ @@ -112,7 +112,7 @@ RUN dnf clean all && \ rm -rf /var/cache/dnf && \ dnf -y install dnf-plugins-core && \ dnf config-manager --set-enabled devel && \ - dnf -y update && \ + dnf -y --exclude=clang update && \ dnf -y install \ openssh-clients \ ca-certificates \ From 4904e2cf1b023d12901fd266ef3ff39e273a603d Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 27 Nov 2024 16:08:58 +0100 Subject: [PATCH 21/28] dnf update only once --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 14bd5d84..9025377c 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -66,7 +66,6 @@ ENV OMPI_FC=$FC # Install required packages using dnf RUN dnf clean all && \ - dnf -y --exclude=clang update && \ dnf -y install \ tbb-devel \ bc \ @@ -112,7 +111,6 @@ RUN dnf clean all && \ rm -rf /var/cache/dnf && \ dnf -y install dnf-plugins-core && \ dnf config-manager --set-enabled devel && \ - dnf -y --exclude=clang update && \ dnf -y install \ openssh-clients \ ca-certificates \ From adb35853120703cc1775a255f98e165b3d38ec92 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 27 Nov 2024 17:54:48 +0100 Subject: [PATCH 22/28] try to fix caliper build --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 9025377c..19a21b42 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -14,7 +14,7 @@ RUN dnf clean all && \ dnf -y update && \ dnf -y install \ which \ - clang-17.0.6 \ + clang-17.0.6 \ gcc-gfortran \ python3 \ zlib-devel \ @@ -47,7 +47,8 @@ ENV CC=/usr/bin/clang \ MPICXX=/usr/lib64/openmpi/bin/mpicxx \ MPIEXEC=/usr/lib64/openmpi/bin/mpirun \ BLAS_LIBRARIES="/usr/lib64/libblas.so.3.8.0" \ - LAPACK_LIBRARIES="/usr/lib64/liblapack.so.3.8.0" + LAPACK_LIBRARIES="/usr/lib64/liblapack.so.3.8.0" \ + LDFLAGS="-lstdc++fs" ENV OMPI_CC=$CC \ OMPI_CXX=$CXX From e7cd018470a8173cf0f31904323d91f1094c8829 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 4 Dec 2024 09:54:32 +0100 Subject: [PATCH 23/28] copy fix from trilinos PR --- docker/tpl-rockylinux-clang-cuda-12.Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 19a21b42..9cc77304 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -3,7 +3,7 @@ ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build -FROM nvidia/cuda:12.5.0-devel-rockylinux8 AS tpl_toolchain_intersect_geosx_toolchain +FROM nvidia/cuda:12.6.3-devel-rockylinux8 AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR ARG INSTALL_DIR @@ -14,7 +14,7 @@ RUN dnf clean all && \ dnf -y update && \ dnf -y install \ which \ - clang-17.0.6 \ + clang-17.0.6 \ gcc-gfortran \ python3 \ zlib-devel \ @@ -47,8 +47,7 @@ ENV CC=/usr/bin/clang \ MPICXX=/usr/lib64/openmpi/bin/mpicxx \ MPIEXEC=/usr/lib64/openmpi/bin/mpirun \ BLAS_LIBRARIES="/usr/lib64/libblas.so.3.8.0" \ - LAPACK_LIBRARIES="/usr/lib64/liblapack.so.3.8.0" \ - LDFLAGS="-lstdc++fs" + LAPACK_LIBRARIES="/usr/lib64/liblapack.so.3.8.0" ENV OMPI_CC=$CC \ OMPI_CXX=$CXX @@ -67,6 +66,7 @@ ENV OMPI_FC=$FC # Install required packages using dnf RUN dnf clean all && \ + dnf -y update --exclude=clang*,compiler-rt,libomp*,llvm* && \ dnf -y install \ tbb-devel \ bc \ @@ -112,6 +112,7 @@ RUN dnf clean all && \ rm -rf /var/cache/dnf && \ dnf -y install dnf-plugins-core && \ dnf config-manager --set-enabled devel && \ + dnf -y update --exclude=clang*,compiler-rt,libomp*,llvm* && \ dnf -y install \ openssh-clients \ ca-certificates \ From 220a28804011953774612aac80afd629990523c9 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Fri, 6 Dec 2024 14:08:37 +0100 Subject: [PATCH 24/28] fix local p4 build --- .../pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake | 3 ++- .../pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake index 5b3402c2..3ab0d9c8 100644 --- a/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0.cmake @@ -92,7 +92,8 @@ if( NOT DEFINED ENV{MKLROOT} ) endif() if ( NOT DEFINED ENV{GOMP_ROOT} ) - set( GOMP_ROOT $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so ) + message( STATUS "GOMP_ROOT is not set. Setting it to $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12" ) + set( ENV{GOMP_ROOT} $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12 ) endif() set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) diff --git a/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake index 0cd79ad6..3a5d3a6c 100644 --- a/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake +++ b/host-configs/TotalEnergies/pangea-4/pangea4-gcc12.1-openmpi4.1.6-onemkl2023.2.0.cmake @@ -92,7 +92,8 @@ if( NOT DEFINED ENV{MKLROOT} ) endif() if ( NOT DEFINED ENV{GOMP_ROOT} ) - set( GOMP_ROOT $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12/libgomp.so ) + message( STATUS "GOMP_ROOT is not set. Setting it to $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12" ) + set( ENV{GOMP_ROOT} $ENV{GCC_PATH}/lib/gcc/x86_64-redhat-linux/12 ) endif() set( MKL_INCLUDE_DIRS $ENV{MKLROOT}/include CACHE STRING "" ) From 15cc460d4ad12ae045e8a96323312751fbf2d2ea Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 9 Jan 2025 09:23:20 +0100 Subject: [PATCH 25/28] changed modules for pangea-3 configuration --- ...-4.1.2.cmake => pangea3-gcc11.4.0-openmpi-4.1.6.cmake} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename host-configs/TotalEnergies/pangea-3/{pangea3-gcc8.4.1-openmpi-4.1.2.cmake => pangea3-gcc11.4.0-openmpi-4.1.6.cmake} (95%) diff --git a/host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake b/host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake similarity index 95% rename from host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake rename to host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake index 04d0b43d..20ba2e84 100644 --- a/host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake +++ b/host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake @@ -1,7 +1,7 @@ # hostconfig for pangea3 # # -set(CONFIG_NAME "pangea3-gcc8.4.1-ompi-4.1.2" CACHE PATH "") +set(CONFIG_NAME "pangea3-gcc11.4.0-ompi-4.1.6" CACHE PATH "") # C options set(CMAKE_C_COMPILER gcc CACHE PATH "") @@ -36,7 +36,7 @@ if (DEFINED ENV{MPI_ROOT}) set(MPIEXEC $ENV{MPI_ROOT}/bin/mpirun CACHE STRING "") set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") else() - message(FATAL_ERROR "You must have MPI_ROOT variable set, we advise loading module openmpi/p3/4.1.2") + message(FATAL_ERROR "You must have MPI_ROOT variable set, we advise loading module ompi/4.1.6") endif() # Cuda options @@ -56,7 +56,7 @@ if (DEFINED ENV{CUDA_ROOT}) # Uncomment this line to make nvcc output register usage for each kernel. # set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE) else() - message(FATAL_ERROR "You must have CUDA_ROOT environment variable set, we advise loading module cuda/11.0.3") + message(FATAL_ERROR "You must have CUDA_ROOT environment variable set, we advise loading module cuda/11.8.0") endif() # GTEST options @@ -90,7 +90,7 @@ if (DEFINED ENV{OPENBLAS_ROOT}) set(BLAS_LIBRARIES $ENV{OPENBLAS_ROOT}/lib/libopenblas.a) set(LAPACK_LIBRARIES $ENV{OPENBLAS_ROOT}/lib/libopenblas.a) else() - message(FATAL_ERROR "You must have OPENBLAS_ROOT environment variable set, we advise loading module openblas/0.3.18") + message(FATAL_ERROR "You must have OPENBLAS_ROOT environment variable set, we advise loading module openblas/0.3.18+openmp") endif() set(ENABLE_DOXYGEN OFF CACHE PATH "") From b5163c8049a5161186d74530d26c85ccf82dfa8c Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 16 Jan 2025 14:07:14 +0100 Subject: [PATCH 26/28] changed P3 dockerfile --- .../pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile index d3510d82..73696841 100644 --- a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile @@ -31,9 +31,9 @@ ARG MODULE_PATH="/data_local/sw" ARG SPACK_PATH="spack/0.17.0/opt/spack/linux-rhel8-power9le" ARG COMPILER="gcc" -ARG DEFAULT_COMPILER_VER="8.4.1" +ARG DEFAULT_COMPILER_VER="11.4.0" ARG SPACK_COMPILER_VER=$DEFAULT_COMPILER_VER -ARG MODULE_COMPILER_VER="9.4.0" +ARG MODULE_COMPILER_VER="11.4.0" ARG SPACK_COMPILER=$COMPILER-$SPACK_COMPILER_VER @@ -72,7 +72,7 @@ $MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_RUN_PATH}" \ ADD ./tarball/ompi-*.tgz / ### Temporary local variables -ARG MPI_VER="4.1.2" +ARG MPI_VER="4.1.6" ARG MPI_DIR="openmpi/$MPI_VER" ARG MPI_PREFIX="OMPI" @@ -99,7 +99,7 @@ ADD ./tarball/cuda-*.tgz / ARG LIBCUDA_VER=450.156.00 ARG CUDA_VER_MAJ="11" -ARG CUDA_VER_MIN="5" +ARG CUDA_VER_MIN="8" ARG CUDA_VER_PATCH="0" ARG CUDA_VER="$CUDA_VER_MAJ.$CUDA_VER_MIN.$CUDA_VER_PATCH" ARG CUDA_DIR="cuda/$CUDA_VER" From 3a5c1a1acf40637aa0b1e74d24edd7db277ff959 Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Wed, 22 Jan 2025 16:48:43 +0100 Subject: [PATCH 27/28] update docker pangea3 with gcc11 --- .github/workflows/docker_build_tpls.yml | 4 ++-- .../pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 3b5888f0..ea972406 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -64,8 +64,8 @@ jobs: - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 TPL_DOCKERFILE: docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile - HOST_CONFIG: host-configs/TotalEnergies/pangea-3/pangea3-gcc8.4.1-openmpi-4.1.2.cmake - DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + DOCKER_ROOT_IMAGE: onetechssc/pangea3:pangea-almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2 + HOST_CONFIG: host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake HOST_ARCH: ppc64le RUNS_ON: streak2 - name: Pangea 4 (redhat 8, gcc 12.1.0, hpcx 2.20.0, mkl 2023.2.0) diff --git a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile index 73696841..321c647f 100644 --- a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile @@ -27,11 +27,10 @@ RUN dnf install -y \ ## Temporary local variables needed buy several modules ARG MODULE_PATH="/data_local/sw" - -ARG SPACK_PATH="spack/0.17.0/opt/spack/linux-rhel8-power9le" +ARG SPACK_PATH="spack/0.22.1/opt/spack/linux-rhel8-power9le" ARG COMPILER="gcc" -ARG DEFAULT_COMPILER_VER="11.4.0" +ARG DEFAULT_COMPILER_VER="8.4.1" ARG SPACK_COMPILER_VER=$DEFAULT_COMPILER_VER ARG MODULE_COMPILER_VER="11.4.0" @@ -51,7 +50,7 @@ ADD ./tarball/gcc-*.tgz / ### Temporary local variables ARG GCC_VER=$COMPILER-$MODULE_COMPILER_VER -ARG GCC_DIR="$GCC_VER-xe5cqnyajaqz75up3gflln5zlj2rue5v" +ARG GCC_DIR="$GCC_VER-bbeypfg5smd3pgbsdswprcja7cxdxyqn" ### Environment variables to export ENV CPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/include:${CPATH}" \ From f47b085fad93a57cd073f130c9b6ee438453db1d Mon Sep 17 00:00:00 2001 From: Gaetan FUSS Date: Thu, 23 Jan 2025 15:44:26 +0100 Subject: [PATCH 28/28] fix cmake version --- .github/workflows/docker_build_tpls.yml | 6 +- .../pangea3-gcc-openmpi-openblas.Dockerfile | 75 ++++++++++++++---- .../pangea-3/pangea3-tpl.Dockerfile | 79 ++++++++++++++----- 3 files changed, 124 insertions(+), 36 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index ea972406..6f4561f0 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -61,10 +61,10 @@ jobs: DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile RUNS_ON: Runner_4core_16GB - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 + - name: Pangea 3 (AlmaLinux 8.8, gcc 11.4.0, open-mpi 4.1.6, cuda 11.8.0, openblas 0.3.10) + DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18 TPL_DOCKERFILE: docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile - DOCKER_ROOT_IMAGE: onetechssc/pangea3:pangea-almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2 + DOCKER_ROOT_IMAGE: onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 HOST_CONFIG: host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake HOST_ARCH: ppc64le RUNS_ON: streak2 diff --git a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile index 321c647f..fdb1cb61 100644 --- a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile @@ -1,10 +1,48 @@ -# This Dockerfile is used to build a docker image reproducing the Pangea installation over a ppc64le architecture: -# It is not directly callable by the TPL ci but the built image is. - # syntax=docker/dockerfile:1 -FROM ppc64le/almalinux:8 -# Install other needed packages +####################################### +# Pangea 3 image : gcc - openmpi - openblas +####################################### +# +# Installs : +# - gcc = 11.4.0 +# - cmake = 3.27.9 +# - openmpi = 4.1.6 +# - openblas = 0.3.18 +# - cuda = 11.8.0 +# +####################################### +# +# Description : +# - this Dockerfile is used to build a docker image reproducing the Pangea-3 installation over a ppc64le architecture: +# - it is not directly callable by the TPL ci but the built image is. +# - the image is based on ppc64le/almalinux:8 +# - this image is deployed as onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - gcc, cmake, openmpi, openblas and cuda are copied from the tarball directory of Pangea 3 +# +# Usage : +# build the image: +# - copy the tarball directory from the Pangea 3 repository to the current directory +# - podman build -f pangea3-gcc-openmpi-openblas.Dockerfile -t onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem/ +# run the image: +# - podman run -it --detach --privileged --name pangea3_gcc_ompi_oblas -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - podman exec -it pangea3_gcc_ompi_oblas /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 docker://docker.io/onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# +####################################### + +FROM ppc64le/almalinux:8 AS pangea3 + +# ------ +# LABELS +LABEL description="Pangea 3 image : gcc - cmake - openmpi - openblas - cuda" +LABEL version="2.0" +LABEL maintainer="TotalEnergies HPC Team" + +# ------ +# INSTALL BASE PACKAGES RUN dnf install -y \ # gcc deps \ libmpc-devel.ppc64le glibc-devel \ @@ -39,13 +77,15 @@ ARG SPACK_COMPILER=$COMPILER-$SPACK_COMPILER_VER ## liblustre COPY ./tarball/liblustreapi.so.1 /lib64/ -## CMake +# ------ +# CMAKE ADD ./tarball/cmake-*.tgz / ### Environment variables to export -ENV PATH="/data_local/appli_local/MTS/GEOSX/cmake/3.26.4/bin:${PATH}" +ENV PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/cmake-3.27.9-yfuovjb3tx73ymsxuw5hoxv3eqdchned/bin:${PATH}" -## gcc +# ------ +# GCC ADD ./tarball/gcc-*.tgz / ### Temporary local variables @@ -67,7 +107,8 @@ $MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_RUN_PATH}" \ FC=gfortran \ GCC_ROOT=$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR -## ompi +# ------ +# OMPI ADD ./tarball/ompi-*.tgz / ### Temporary local variables @@ -91,7 +132,8 @@ ENV CPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/include:${CPATH}" \ OMPI_MCA_btl_openib_allow_ib="true" \ OMPI_MCA_btl_openib_warn_default_gid_prefix="0" -## Cuda +# ------ +# CUDA ADD ./tarball/cuda-*.tgz / ### Temporary local variables @@ -127,7 +169,8 @@ $MODULE_PATH/$CUDA_DIR/samples/bin/ppc64le/linux/release:${PATH}" \ CUDA_VERSION="$CUDA_VER_MAJ.$CUDA_VER_MIN" \ NVHPC_CUDA_HOME="$MODULE_PATH/$CUDA_DIR" -## Openblas +# ------ +# OPENBLAS ADD ./tarball/openblas-*.tgz / ### Temporary local variables @@ -138,7 +181,7 @@ ARG BLAS_VER_MIN="3" ARG BLAS_VER_PATCH="18" ARG BLAS_DISTRIB="openblas" ARG BLAS_VER="$BLAS_VER_MAJ.$BLAS_VER_MIN.$BLAS_VER_PATCH" -ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-vk36pzksytuhylqesg4cca7667np5sjp" +ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-cing5yuan7hsn23qmeemon4zwih3k2hd" ### Environment variables to export ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD_LIBRARY_PATH}" \ @@ -146,7 +189,8 @@ ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/bin:${PATH}" \ OPENBLAS_ROOT="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR" -## lsf +# ------ +# LSF ADD ./tarball/lsf-*.tgz / ### Temporary local variables @@ -173,7 +217,8 @@ ENV LD_LIBRARY_PATH="$MODULE_PATH/$LSF_DIR/lib:${LD_LIBRARY_PATH}" \ # make -C build-pangea3-gcc8.4.1-openmpi-4.1.2-release -j && \ # cd .. && rm -rf thirdPartyLibs -# Install tools needed by geos ci +# ------ +# CI TOOLS RUN dnf -y --enablerepo=powertools install \ ninja-build \ openssh-clients \ @@ -182,7 +227,7 @@ RUN dnf -y --enablerepo=powertools install \ libxml2 # build sccache from source as prebuilt binary is not available for current archi / OS couple -RUN dnf makecache --refresh && dnf -y install cargo openssl-devel +RUN dnf clean all && dnf makecache --refresh && dnf -y install cargo openssl-devel RUN cargo install sccache --locked && mkdir -p /opt/sccache/ && cp -r /root/.cargo/bin /opt/sccache/ RUN dnf remove -y cargo openssl-devel diff --git a/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile index 974f79dd..7ee88321 100644 --- a/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile +++ b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile @@ -1,35 +1,78 @@ -# Temporary local variables dedicated to the TPL build -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build +# syntax=docker/dockerfile:1 -# The docker base image has to be pangea3-almalinux8-* -ARG DOCKER_ROOT_IMAGE +####################################### +# Pangea 3 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 3 environments +# - the docker base image can be any pangea3 docker file +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg HOST_CONFIG=host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea3:almalinux8-gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p3/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# -f docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea3_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# - podman exec -it pangea3_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 docker://docker.io/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea4_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-cuda11.8.0-openblas0.3.18_v2.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config host-configs/TotalEnergies/pangea-3/pangea3-gcc11.4.0-openmpi-4.1.6.cmake \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install --build-exe-only +####################################### +# ------------------------------------- +# PANGEA3 - TPL BASE +ARG DOCKER_ROOT_IMAGE FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR -ARG BLD_DIR - +# ------ +# LABELS +LABEL description="Pangea 3 image : geos_tpl" +LABEL version="2.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +# The installation directory is provided as a docker build argument +ARG INSTALL_DIR +# ------ +# ENV # All the environment variables defined in this Dockerfile # (GEOSX_TPL_DIR but also compiler information like CC, CXX...) # are part of the image contract (otherwise ARG is used). # GEOSX use them so consider modifying their names with care. -# -# The installation directory is provided as a docker build argument. -# We forward it using an environment variable. -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR +# ------------------------------------- +# PANGEA3 - TPL TOOLCHAIN +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build # Get host config file from docker build arguments ARG HOST_CONFIG - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain -# We now configure the build... +# ------ +# INSTALL RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh -# ... before we compile the TPLs! +# - build TPLs WORKDIR $BLD_DIR RUN --mount=src=.,dst=$SRC_DIR make +# ------------------------------------- +# PANGEA3 - GEOS TOOLCHAIN # Extract only TPL's from previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain