From 774939aeafb6dbd45804d2b595d8b470a3ac33ea Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 15:09:33 -0600 Subject: [PATCH 01/10] run --- src/simulation/m_muscl.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/m_muscl.fpp b/src/simulation/m_muscl.fpp index c9d5b4bff0..1b83398373 100644 --- a/src/simulation/m_muscl.fpp +++ b/src/simulation/m_muscl.fpp @@ -348,7 +348,7 @@ contains $:END_GPU_PARALLEL_LOOP() end if - ! Reshaping/Projecting onto Characteristic Fields in z-direction + ! Reshaping onto Characteristic Fields in z-direction if (p == 0) return if (muscl_dir == 3) then $:GPU_PARALLEL_LOOP(private='[j,k,l,q]', collapse=4) From b53003b508ff06eb9c787ea203aea2414af22cc8 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 15:12:11 -0600 Subject: [PATCH 02/10] runner --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 251b276ace..2daad7776b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -127,6 +127,10 @@ jobs: cluster_name: 'Oak Ridge | Frontier' device: 'cpu' interface: 'none' + - lbl: 'delta' + cluster_name: 'NCSA | Delta' + device: 'cpu' + interface: 'none' runs-on: group: phoenix labels: ${{ matrix.lbl }} @@ -142,6 +146,10 @@ jobs: if: matrix.lbl == 'gt' run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }} + - name: Build & Test + if: matrix.lbl == 'delta' + run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }} + - name: Build if: matrix.lbl == 'frontier' run: bash .github/workflows/frontier/build.sh ${{ matrix.device }} ${{ matrix.interface }} From 932dac13e89135ed5909130c992c6b987eccf5df Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 15:13:27 -0600 Subject: [PATCH 03/10] cleanup --- .github/workflows/bench.yml | 124 ------------------------------------ .github/workflows/test.yml | 26 -------- 2 files changed, 150 deletions(-) delete mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index 2ccdfca87a..0000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: 'Benchmark' - -on: - pull_request: - pull_request_review: - types: [submitted] - workflow_dispatch: - -jobs: - file-changes: - name: Detect File Changes - runs-on: 'ubuntu-latest' - outputs: - checkall: ${{ steps.changes.outputs.checkall }} - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Detect Changes - uses: dorny/paths-filter@v3 - id: changes - with: - filters: ".github/file-filter.yml" - - self: - name: "${{ matrix.name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" - if: ${{ github.repository=='MFlowCode/MFC' && needs.file-changes.outputs.checkall=='true' && ((github.event_name=='pull_request_review' && github.event.review.state=='approved') || (github.event_name=='pull_request' && (github.event.pull_request.user.login=='sbryngelson' || github.event.pull_request.user.login=='wilfonba'))) }} - needs: file-changes - strategy: - fail-fast: false - matrix: - include: - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: cpu - interface: none - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: acc - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: omp - build_script: "" - - cluster: frontier - name: Oak Ridge | Frontier (CCE) - group: phoenix - labels: frontier - flag: f - device: gpu - interface: acc - build_script: "bash .github/workflows/frontier/build.sh gpu acc bench" - - cluster: frontier - name: Oak Ridge | Frontier (CCE) - group: phoenix - labels: frontier - flag: f - device: gpu - interface: omp - build_script: "bash .github/workflows/frontier/build.sh gpu omp bench" - runs-on: - group: ${{ matrix.group }} - labels: ${{ matrix.labels }} - timeout-minutes: 480 - env: - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Clone - PR - uses: actions/checkout@v4 - with: - path: pr - - - name: Clone - Master - uses: actions/checkout@v4 - with: - repository: MFlowCode/MFC - ref: master - path: master - - - name: Setup & Build - if: matrix.build_script != '' - run: | - (cd pr && ${{ matrix.build_script }}) & - (cd master && ${{ matrix.build_script }}) & - wait %1 && wait %2 - - - name: Bench (Master v. PR) - run: bash pr/.github/scripts/run_parallel_benchmarks.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }} - - - name: Generate & Post Comment - run: | - (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g) - (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml) - - - name: Print Logs - if: always() - run: | - cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - - # All other runners (non-Phoenix) just run without special env - - name: Archive Logs (Frontier) - if: always() && matrix.cluster != 'phoenix' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }} - path: | - pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* - pr/build/benchmarks/* - master/bench-${{ matrix.device }}-${{ matrix.interface }}.* - master/build/benchmarks/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2daad7776b..19c41f90b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,32 +101,6 @@ jobs: strategy: matrix: include: - # Phoenix (GT) - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'gpu' - interface: 'acc' - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'gpu' - interface: 'omp' - - lbl: 'gt' - cluster_name: 'Georgia Tech | Phoenix' - device: 'cpu' - interface: 'none' - # Frontier (ORNL) - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' - device: 'gpu' - interface: 'acc' - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' - device: 'gpu' - interface: 'omp' - - lbl: 'frontier' - cluster_name: 'Oak Ridge | Frontier' - device: 'cpu' - interface: 'none' - lbl: 'delta' cluster_name: 'NCSA | Delta' device: 'cpu' From 95e2e2bf8cd69786fa0ec019b87d948d3c8a40c2 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 15:31:34 -0600 Subject: [PATCH 04/10] test --- .github/workflows/delta/submit.sh | 63 +++++++++++++++++++++++++++++++ .github/workflows/delta/test.sh | 25 ++++++++++++ .github/workflows/test.yml | 2 +- 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/delta/submit.sh create mode 100644 .github/workflows/delta/test.sh diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh new file mode 100755 index 0000000000..d0a2e46871 --- /dev/null +++ b/.github/workflows/delta/submit.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -e + +usage() { + echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]" +} + +if [ ! -z "$1" ]; then + sbatch_script_contents=`cat $1` +else + usage + exit 1 +fi + +sbatch_cpu_opts="\ +#SBATCH -p cpu # partition +#SBATCH --ntasks-per-node=24 # Number of cores per node required +#SBATCH --account=bdiy-delta-cpu # charge account +" + +sbatch_gpu_opts="\ +#SBATCH --account=bdiy-delta-gpu +#SBATCH -p gpu +#SBATCH --ntasks-per-node=4 # Number of cores per node required +#SBATCH -G2\ +" + +if [ "$2" = "cpu" ]; then + sbatch_device_opts="$sbatch_cpu_opts" +elif [ "$2" = "gpu" ]; then + sbatch_device_opts="$sbatch_gpu_opts" +else + usage + exit 1 +fi + +job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3" + +sbatch < Date: Sat, 27 Dec 2025 15:40:48 -0600 Subject: [PATCH 05/10] fix --- .github/workflows/delta/submit.sh | 2 +- .github/workflows/formatting.yml | 19 ------- .github/workflows/lint-source.yml | 53 -------------------- .github/workflows/lint-toolchain.yml | 22 --------- .github/workflows/test.yml | 74 ---------------------------- 5 files changed, 1 insertion(+), 169 deletions(-) delete mode 100644 .github/workflows/formatting.yml delete mode 100644 .github/workflows/lint-source.yml delete mode 100644 .github/workflows/lint-toolchain.yml diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh index d0a2e46871..5683a3fb6d 100755 --- a/.github/workflows/delta/submit.sh +++ b/.github/workflows/delta/submit.sh @@ -56,7 +56,7 @@ job_slug="$job_slug" job_device="$2" job_interface="$3" -. ./mfc.sh load -c p -m $2 +. ./mfc.sh load -c delta -m $2 $sbatch_script_contents diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml deleted file mode 100644 index 16043daa95..0000000000 --- a/.github/workflows/formatting.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Pretty - -on: [push, pull_request, workflow_dispatch] - -jobs: - docs: - name: Formatting - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: MFC Python setup - run: ./mfc.sh init - - - name: Check formatting - run: | - ./mfc.sh format -j $(nproc) - git diff --exit-code diff --git a/.github/workflows/lint-source.yml b/.github/workflows/lint-source.yml deleted file mode 100644 index bc35b887ea..0000000000 --- a/.github/workflows/lint-source.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Lint Source - -on: [push, pull_request, workflow_dispatch] - -jobs: - file-changes: - name: Detect File Changes - runs-on: 'ubuntu-latest' - outputs: - checkall: ${{ steps.changes.outputs.checkall }} - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Detect Changes - uses: dorny/paths-filter@v3 - id: changes - with: - filters: ".github/file-filter.yml" - - lint-source: - name: Lint Source - runs-on: 'ubuntu-latest' - - steps: - - uses: actions/checkout@v4 - - - name: Initialize MFC - run: ./mfc.sh init - - - name: Looking for raw directives - run: | - ! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/* - - - name: No double precision intrinsics - run: | - ! grep -iR 'double_precision\|dsqrt\|dexp\|dlog\|dble\|dabs\|double\ precision\|real(8)\|real(4)\|dprod\|dmin\|dmax\|dfloat\|dreal\|dcos\|dsin\|dtan\|dsign\|dtanh\|dsinh\|dcosh\|d0' --exclude-dir=syscheck --exclude="*nvtx*" --exclude="*precision_select*" ./src/* - - - name: Looking for junk code - run: | - ! grep -iR -e '\.\.\.' -e '\-\-\-' -e '===' ./src/* - - - name: Looking for false integers - run: | - ! grep -onRP '(?> $GITHUB_ENV - echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV - - - name: Setup Ubuntu - if: matrix.os == 'ubuntu' && matrix.intel == false - run: | - sudo apt update -y - sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ - libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ - libblas-dev liblapack-dev - - - name: Setup Ubuntu (Intel) - if: matrix.os == 'ubuntu' && matrix.intel == true - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-mpi intel-oneapi-mpi-devel - source /opt/intel/oneapi/setvars.sh - printenv >> $GITHUB_ENV - - - name: Set up Python 3.14 - uses: actions/setup-python@v5 - with: - python-version: '3.14' - - - name: Build - run: | - /bin/bash mfc.sh test --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} - - - name: Test - run: | - /bin/bash mfc.sh test --max-attempts 3 -j $(nproc) $OPT1 $OPT2 - env: - OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} - OPT2: ${{ matrix.debug == 'debug' && '-% 20' || '' }} - self: name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }})" if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' From e561bf5c3bf2682e0ab4b1ed2d43180d1dc8cbe9 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 15:58:14 -0600 Subject: [PATCH 06/10] fix 2 --- .github/workflows/delta/submit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh index 5683a3fb6d..18625d3a0b 100755 --- a/.github/workflows/delta/submit.sh +++ b/.github/workflows/delta/submit.sh @@ -56,7 +56,7 @@ job_slug="$job_slug" job_device="$2" job_interface="$3" -. ./mfc.sh load -c delta -m $2 +. ./mfc.sh load -c d -m $2 $sbatch_script_contents From a4e1416756da83334e3550cd5d52f13892224e85 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 16:05:26 -0600 Subject: [PATCH 07/10] modules --- toolchain/modules | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolchain/modules b/toolchain/modules index 69757c3c03..dcab984d94 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -46,8 +46,7 @@ f-all cray-fftw cray-hdf5 python cmake f-gpu python craype-accel-amd-gfx90a rocprofiler-compute/3.0.0 d NCSA Delta -d-all python/3.11.6 -d-cpu gcc/11.4.0 openmpi +d-all python d-gpu nvhpc/24.1 cuda/12.3.0 openmpi/4.1.5+cuda cmake d-gpu CC=nvc CXX=nvc++ FC=nvfortran d-gpu MFC_CUDA_CC=80,86 From 05c200fc0e495f2975309c6d1a3f467536b52236 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 19:23:07 -0600 Subject: [PATCH 08/10] more threads --- .github/workflows/delta/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/delta/test.sh b/.github/workflows/delta/test.sh index a08f39ed11..4a1e7d1655 100644 --- a/.github/workflows/delta/test.sh +++ b/.github/workflows/delta/test.sh @@ -10,9 +10,9 @@ if [ "$job_device" = "gpu" ]; then fi fi -./mfc.sh test --dry-run -j 8 $build_opts +./mfc.sh test --dry-run -j 16 $build_opts -n_test_threads=8 +n_test_threads=16 if [ "$job_device" = "gpu" ]; then gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node From acfca93cf22ace15c15c77a8148763ca1ee42845 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Sat, 27 Dec 2025 22:36:23 -0600 Subject: [PATCH 09/10] try something new --- .github/workflows/delta/submit.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh index 18625d3a0b..ea4a037fd7 100755 --- a/.github/workflows/delta/submit.sh +++ b/.github/workflows/delta/submit.sh @@ -42,9 +42,10 @@ sbatch < Date: Sun, 28 Dec 2025 08:21:50 -0600 Subject: [PATCH 10/10] fix --- .github/workflows/delta/submit.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/delta/submit.sh b/.github/workflows/delta/submit.sh index ea4a037fd7..6721202130 100755 --- a/.github/workflows/delta/submit.sh +++ b/.github/workflows/delta/submit.sh @@ -42,7 +42,7 @@ sbatch <