diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index 2ccdfca87a..0000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: 'Benchmark' - -on: - pull_request: - pull_request_review: - types: [submitted] - workflow_dispatch: - -jobs: - file-changes: - name: Detect File Changes - runs-on: 'ubuntu-latest' - outputs: - checkall: ${{ steps.changes.outputs.checkall }} - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Detect Changes - uses: dorny/paths-filter@v3 - id: changes - with: - filters: ".github/file-filter.yml" - - self: - name: "${{ matrix.name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" - if: ${{ github.repository=='MFlowCode/MFC' && needs.file-changes.outputs.checkall=='true' && ((github.event_name=='pull_request_review' && github.event.review.state=='approved') || (github.event_name=='pull_request' && (github.event.pull_request.user.login=='sbryngelson' || github.event.pull_request.user.login=='wilfonba'))) }} - needs: file-changes - strategy: - fail-fast: false - matrix: - include: - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: cpu - interface: none - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: acc - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: omp - build_script: "" - - cluster: frontier - name: Oak Ridge | Frontier (CCE) - group: phoenix - labels: frontier - flag: f - device: gpu - interface: acc - build_script: "bash .github/workflows/frontier/build.sh gpu acc bench" - - cluster: frontier - name: Oak Ridge | Frontier (CCE) - group: phoenix - labels: frontier - flag: f - device: gpu - interface: omp - build_script: "bash .github/workflows/frontier/build.sh gpu omp bench" - runs-on: - group: ${{ matrix.group }} - labels: ${{ matrix.labels }} - timeout-minutes: 480 - env: - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Clone - PR - uses: actions/checkout@v4 - with: - path: pr - - - name: Clone - Master - uses: actions/checkout@v4 - with: - repository: MFlowCode/MFC - ref: master - path: master - - - name: Setup & Build - if: matrix.build_script != '' - run: | - (cd pr && ${{ matrix.build_script }}) & - (cd master && ${{ matrix.build_script }}) & - wait %1 && wait %2 - - - name: Bench (Master v. PR) - run: bash pr/.github/scripts/run_parallel_benchmarks.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} - - - name: Generate & Post Comment - run: | - (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g) - (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml) - - - name: Print Logs - if: always() - run: | - cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - - # All other runners (non-Phoenix) just run without special env - - name: Archive Logs (Frontier) - if: always() && matrix.cluster != 'phoenix' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }} - path: | - pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* - pr/build/benchmarks/* - master/bench-${{ matrix.device }}-${{ matrix.interface }}.* - master/build/benchmarks/* diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh new file mode 100755 index 0000000000..6721202130 --- /dev/null +++ b/.github/workflows/delta/submit.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -e + +usage() { + echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]" +} + +if [ ! -z "$1" ]; then + sbatch_script_contents=`cat $1` +else + usage + exit 1 +fi + +sbatch_cpu_opts="\ +#SBATCH -p cpu # partition +#SBATCH --ntasks-per-node=24 # Number of cores per node required +#SBATCH --account=bdiy-delta-cpu # charge account +" + +sbatch_gpu_opts="\ +#SBATCH --account=bdiy-delta-gpu +#SBATCH -p gpu +#SBATCH --ntasks-per-node=4 # Number of cores per node required +#SBATCH -G2\ +" + +if [ "$2" = "cpu" ]; then + sbatch_device_opts="$sbatch_cpu_opts" +elif [ "$2" = "gpu" ]; then + sbatch_device_opts="$sbatch_gpu_opts" +else + usage + exit 1 +fi + +job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3" + +sbatch <> $GITHUB_ENV - echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV - - - name: Setup Ubuntu - if: matrix.os == 'ubuntu' && matrix.intel == false - run: | - sudo apt update -y - sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ - libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ - libblas-dev liblapack-dev - - - name: Setup Ubuntu (Intel) - if: matrix.os == 'ubuntu' && matrix.intel == true - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-mpi intel-oneapi-mpi-devel - source /opt/intel/oneapi/setvars.sh - printenv >> $GITHUB_ENV - - - name: Set up Python 3.14 - uses: actions/setup-python@v5 - with: - python-version: '3.14' - - - name: Build - run: | - /bin/bash mfc.sh test --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} - - - name: Test - run: | - /bin/bash mfc.sh test --max-attempts 3 -j $(nproc) $OPT1 $OPT2 - env: - OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} - OPT2: ${{ matrix.debug == 'debug' && '-% 20' || '' }} - self: name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' @@ -101,30 +27,8 @@ jobs: strategy: matrix: include: - # Phoenix (GT) - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'gpu' - interface: 'acc' - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'gpu' - interface: 'omp' - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'cpu' - interface: 'none' - # Frontier (ORNL) - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' - device: 'gpu' - interface: 'acc' - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' - device: 'gpu' - interface: 'omp' - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' + - lbl: 'delta' + cluster_name: 'NCSA | Delta' device: 'cpu' interface: 'none' runs-on: @@ -142,6 +46,10 @@ jobs: if: matrix.lbl == 'gt' run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }} + - name: Build & Test + if: matrix.lbl == 'delta' + run: bash .github/workflows/delta/submit.sh .github/workflows/delta/test.sh ${{ matrix.device }} ${{ matrix.interface }} + - name: Build if: matrix.lbl == 'frontier' run: bash .github/workflows/frontier/build.sh ${{ matrix.device }} ${{ matrix.interface }} diff --git a/src/simulation/m_muscl.fpp b/src/simulation/m_muscl.fpp index c9d5b4bff0..1b83398373 100644 --- a/src/simulation/m_muscl.fpp +++ b/src/simulation/m_muscl.fpp @@ -348,7 +348,7 @@ contains $:END_GPU_PARALLEL_LOOP() end if - ! Reshaping/Projecting onto Characteristic Fields in z-direction + ! Reshaping onto Characteristic Fields in z-direction if (p == 0) return if (muscl_dir == 3) then $:GPU_PARALLEL_LOOP(private='[j,k,l,q]', collapse=4) diff --git a/toolchain/modules b/toolchain/modules index 69757c3c03..dcab984d94 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -46,8 +46,7 @@ f-all cray-fftw cray-hdf5 python cmake f-gpu python craype-accel-amd-gfx90a rocprofiler-compute/3.0.0 d NCSA Delta -d-all python/3.11.6 -d-cpu gcc/11.4.0 openmpi +d-all python d-gpu nvhpc/24.1 cuda/12.3.0 openmpi/4.1.5+cuda cmake d-gpu CC=nvc CXX=nvc++ FC=nvfortran d-gpu MFC_CUDA_CC=80,86