From da352857561daa8f754161a856d0d53875d6f3ad Mon Sep 17 00:00:00 2001 From: SWE-bench Date: Fri, 4 Sep 2026 21:28:51 -0400 Subject: [PATCH 01/52] Level 3 first batch: audit, build strategy, LAMMPS/SPARTA/WarpX/SPECFEM3D/nekRS bring-up STEP 1-2: level3/APPLICATION_AUDIT.md covers all ten candidates (repository, release, commit, license, LOC, build system, GPU model, CUDA/HIP/MPI support, rank->GPU mapping, inputs, correctness mechanism, scaling inputs, build/disk estimates, B200+CUDA 13.2 and MI355X risk, containers, Spack, priority, blockers). level3/BUILD_STRATEGY.md compares Native / Spack / Apptainer / site-native per application and records BUILD_RECOMMENDATION (NATIVE for eight, NATIVE+SPACK_DEPS for CP2K and QMCPACK), the Spack and container policies, per-application dependency isolation and the modification classes used. STEP 3-6: per-application fetch/build/run/validate scripts and README under level3//, private trees under .deps/level3// with schema l3-1 fingerprints (level3/tools/l3_common.sh), upstream sources as read-only shallow clones under _upstream/level3/. All five first-batch applications build natively for CUDA 13.2 / sm_100 and pass their upstream correctness mechanism at 1, 2 and 4 GPUs on dgx003 through the common launcher (one rank per GPU, per-rank GPU wrapper, mapping audited): LAMMPS stable_22Jul2025_update6 thermo vs shipped reference log (identical) SPARTA 27Aug2026 statistical stats vs shipped reference log WarpX 26.09 + AMReX 26.09 analytic Langmuir test + charge conservation SPECFEM3D v4.1.1 reference seismograms via upstream script nekRS v26.0 upstream --cimode CI on the analytic Ethier case Strong and weak inputs are defined and run at 1/4 GPUs; 8/40/80-GPU shapes are launcher dry-runs only (DRY-RUN / UNVALIDATED); multi-node is BLOCKED/ UNVERIFIED on this site; HIP build branches exist and are untested. Compatibility changes (no class E): SPECFEM3D two upstream-devel back-ports (CUDA 13 deviceOverlap guard, Blackwell device block) plus make-time GENCODE and SCOTCH-without-zlib; nekRS HYPRE sm_100 list and CUDA 13 / Thrust 3.2 compatibility (thrust::pair, explicit reverse_iterator/pair headers, not1 -> not_fn), Makefiles generator, OMPI_FC/-fno-lto/-fPIC/unset AR, osc ^ucx and ulimit -s at run time. Level 2 files are untouched. tools/runtime/README.md records the plan to move the shared launcher tools out of level2/tools without breaking Level 2 (Level 3 references them through HPCPERF_RUNTIME_DIR). --- level3/APPLICATION_AUDIT.md | 444 ++++++++++++++++++ level3/BUILD_STRATEGY.md | 86 ++++ level3/README.md | 136 +++++- level3/lammps/README.md | 126 +++++ level3/lammps/build.sh | 84 ++++ level3/lammps/fetch.sh | 31 ++ level3/lammps/run.sh | 83 ++++ level3/lammps/validate.sh | 85 ++++ level3/nekrs/README.md | 145 ++++++ level3/nekrs/build.sh | 121 +++++ level3/nekrs/fetch.sh | 27 ++ .../nekrs/patches/0001-hypre-cuda-sm100.patch | 11 + .../0002-hypre-cuda13-thrust-pair.patch | 22 + .../0003-hypre-cuda13-thrust3-compat.patch | 178 +++++++ level3/nekrs/run.sh | 92 ++++ level3/nekrs/validate.sh | 41 ++ level3/sparta/README.md | 143 ++++++ level3/sparta/build.sh | 77 +++ level3/sparta/fetch.sh | 24 + level3/sparta/run.sh | 64 +++ level3/sparta/validate.sh | 89 ++++ level3/specfem3d/README.md | 130 +++++ level3/specfem3d/build.sh | 104 ++++ level3/specfem3d/fetch.sh | 30 ++ .../0001-cuda13-deviceOverlap-guard.patch | 19 + .../patches/0002-blackwell-device-block.patch | 17 + level3/specfem3d/run.sh | 130 +++++ level3/specfem3d/validate.sh | 51 ++ level3/tools/l3_common.sh | 100 ++++ level3/warpx/README.md | 156 ++++++ level3/warpx/build.sh | 70 +++ level3/warpx/fetch.sh | 31 ++ level3/warpx/run.sh | 123 +++++ level3/warpx/validate.sh | 118 +++++ tools/runtime/README.md | 34 ++ 35 files changed, 3210 insertions(+), 12 deletions(-) create mode 100644 level3/APPLICATION_AUDIT.md create mode 100644 level3/BUILD_STRATEGY.md create mode 100644 level3/lammps/README.md create mode 100755 level3/lammps/build.sh create mode 100755 level3/lammps/fetch.sh create mode 100755 level3/lammps/run.sh create mode 100755 level3/lammps/validate.sh create mode 100644 level3/nekrs/README.md create mode 100755 level3/nekrs/build.sh create mode 100755 level3/nekrs/fetch.sh create mode 100644 level3/nekrs/patches/0001-hypre-cuda-sm100.patch create mode 100644 level3/nekrs/patches/0002-hypre-cuda13-thrust-pair.patch create mode 100644 level3/nekrs/patches/0003-hypre-cuda13-thrust3-compat.patch create mode 100755 level3/nekrs/run.sh create mode 100755 level3/nekrs/validate.sh create mode 100644 level3/sparta/README.md create mode 100755 level3/sparta/build.sh create mode 100755 level3/sparta/fetch.sh create mode 100755 level3/sparta/run.sh create mode 100755 level3/sparta/validate.sh create mode 100644 level3/specfem3d/README.md create mode 100755 level3/specfem3d/build.sh create mode 100755 level3/specfem3d/fetch.sh create mode 100644 level3/specfem3d/patches/0001-cuda13-deviceOverlap-guard.patch create mode 100644 level3/specfem3d/patches/0002-blackwell-device-block.patch create mode 100755 level3/specfem3d/run.sh create mode 100755 level3/specfem3d/validate.sh create mode 100755 level3/tools/l3_common.sh create mode 100644 level3/warpx/README.md create mode 100755 level3/warpx/build.sh create mode 100755 level3/warpx/fetch.sh create mode 100755 level3/warpx/run.sh create mode 100755 level3/warpx/validate.sh create mode 100644 tools/runtime/README.md diff --git a/level3/APPLICATION_AUDIT.md b/level3/APPLICATION_AUDIT.md new file mode 100644 index 0000000..1878415 --- /dev/null +++ b/level3/APPLICATION_AUDIT.md @@ -0,0 +1,444 @@ +# Level 3 application audit + +Audit date 2026-09-04, node dgx003 (GMU Hopper: RHEL 10, 4x NVIDIA B200 in +one Slurm allocation, CUDA 13.2.78, driver 595.58.03, conda GCC 13.3.0 + Open +MPI 5.0.10 (CUDA-aware) + CMake 3.28.4; system gfortran 14.2.1 only; no ROCm, +no Apptainer/Singularity, lmod broken; personal Spack checkout 1.0.0.dev0 of +2025-05-06). Every statement below is either "upstream documents ..." (from the +official repository/docs at the recorded commit) or explicitly marked as +verified on this node. Nothing here claims multi-node, 8/40/80-GPU or HIP +validation. Upstream sources were cloned read-only under `_upstream/level3/` +(gitignored); LOC = cloc 2.06 code lines. + +Status legend: **FIRST_BATCH** = brought up in this round (LAMMPS, SPARTA, +WarpX, SPECFEM3D Cartesian, nekRS, in the order requested); **SECOND_BATCH** = +feasible natively on this node but excluded from this round by the +dependency-time rule (> 2 h of dependency builds), a toolchain gap, or a +dataset blocker; **DEFER** / **REPLACE_CANDIDATE**: none needed -- all ten +candidates have an officially supported native CUDA path. + +| Application | Version audited | GPU model | Deps complexity | B200+CUDA 13.2 risk | HIP/MI355X risk | Spack | Container | Priority | +|---|---|---|---|---|---|---|---|---| +| LAMMPS | stable_22Jul2025_update6 | Kokkos 4.6.2 (bundled) | low | low-medium | high (no gfx950 in bundled Kokkos) | pkg exists, local checkout too old, external Kokkos "untested" | none official | FIRST_BATCH | +| SPARTA | 27Aug2026 | Kokkos 5.0.2 (bundled) | low | low-medium | medium-high | **no package** (name collision) | none | FIRST_BATCH | +| WarpX | 26.09 (+AMReX 26.09) | AMReX | medium | low-medium | medium | pkg to 26.08 | site recipe only | FIRST_BATCH | +| SPECFEM3D Cartesian | v4.1.1 | native CUDA/HIP | low | high as tagged (2 backports needed) | high | none | none | FIRST_BATCH | +| nekRS | v26.0 | OCCA (JIT) + HYPRE | medium (all vendored) | medium | medium | pkg stale (23.0) | none | FIRST_BATCH | +| CP2K | v2026.2 | native CUDA/HIP + DBCSR | very high (~40 pkgs) | medium-high | high | official but `cuda_arch=100` rejected | official, no B200 image | SECOND_BATCH | +| Nyx | 26.09 | AMReX | low-medium | medium (SUNDIALS) | medium-high | none | none | SECOND_BATCH | +| QMCPACK | v4.4.0 | OpenMP offload + cuBLAS | medium (needs Clang offload) | medium-high | medium | pkg `+cuda` broken for 4.x | CI only | SECOND_BATCH | +| GEOS | 1.2.0 (develop differs) | RAJA/CHAI/Umpire + hypre | very high (~20 TPLs) | high (tag) / medium (develop) | high | uberenv only (LC systems) | CI images | SECOND_BATCH | +| DFT-FE | 1.2.0 | native CUDA/HIP/SYCL + deal.II (CPU) | high | medium | medium-high | pkg stale (0.6) | CPU only | SECOND_BATCH | + +--- + +## LAMMPS + +- official_repository: https://github.com/lammps/lammps (not migrated) +- official_documentation: https://docs.lammps.org/stable/ (Kokkos: `Speed_kokkos.html`; build: `Build_extras.html#kokkos`; run switches: `Run_options.html`); `doc/src/` in the clone is authoritative for this tag +- latest_stable_release: `stable_22Jul2025_update6` (2026-09-03). Policy (`doc/src/Manual_version.rst`): feature releases `patch_` every 4-8 weeks (latest `patch_2Sep2026`, bundles Kokkos 5.0.2, needs CUDA >= 12.2); one stable per year plus `_updateN` bug-fix updates (back-ports only). Update 6 notes list KOKKOS fixes. +- selected_commit_sha: `9c5ab448c78a14fd534619622162ba418d6a1fb1` +- license: GPL-2.0 +- application_owned_loc: `src/` 852,527 (3,865 files; `src/KOKKOS` 108,766 in 401 files). Bundled TPLs counted separately: `lib/kokkos` 217,774; other `lib/*` 276,282. +- main_languages: C++ 79 %, headers 20 %, shell/Cython/CMake +- build_system: CMake >= 3.16 (`cmake/CMakeLists.txt`), presets in `cmake/presets/`; GNU make also supported but Kokkos+CUDA is documented via CMake +- cxx_standard: C++11 core, C++17 forced with `PKG_KOKKOS` +- supported_compilers: GCC/Clang/Intel/NVHPC; bundled Kokkos 4.6.2 enforces GCC >= 8.2, nvcc >= 11.0, hipcc >= 5.2 (`lib/kokkos/cmake/kokkos_compiler_id.cmake`) +- cuda_support: yes (Kokkos CUDA backend, `nvcc_wrapper`, `FFT_KOKKOS=CUFFT`). Documented CUDA >= 11.0. Blackwell: `Kokkos_ARCH_BLACKWELL100/120` present in the bundled Kokkos. CUDA 13: the bundled 4.6.2 carries LAMMPS-applied back-ports (`#if CUDART_VERSION >= 13000` in `Kokkos_Cuda_Instance.hpp`) absent from upstream 4.6.02; no removed `cudaDeviceProp` fields used. +- hip_rocm_support: yes (`kokkos-hip.cmake`, ROCm >= 5.2, `FFT_KOKKOS=HIPFFT`); arch table ends at `AMD_GFX942`; **no gfx950 in bundled Kokkos 4.6.2** +- mpi_support: yes (spatial decomposition; `BUILD_MPI`) +- official_gpu_programming_model: Kokkos (bundled 4.6.2; external must be `>= 4.6.02`, upstream calls external Kokkos "untested"; source-incompatible with Kokkos 5 before 10Dec2025) +- multi_gpu_support: one MPI rank per GPU documented ("-np ... equal to the number of physical GPUs on the node"); several ranks per GPU need MPS; one rank never drives several GPUs +- multi_node_support: yes (`mpirun -np 32 -ppn 2 ... -k on g 2`); needs a launcher exposing a local-rank variable and GPU-aware MPI for device buffers +- gpu_aware_mpi_requirement: optional, default on; auto-detected for Open MPI via `MPIX_Query_cuda_support`, otherwise warned; `-pk kokkos gpu/aware off` falls back to host staging +- rank_to_gpu_binding: self-binding (`-k on g Ng`: device = local rank % Ng from `OMPI_COMM_WORLD_LOCAL_RANK`, `SLURM_LOCALID`, ...); with the Level 3 launcher wrapper each rank sees one GPU (`g 1`) +- topology_decomposition_controls: `processors Px Py Pz` (optional), `comm_style`, `balance`; bench decks size via `-var x y z` (32,000 atoms x x*y*z) +- major_dependencies: MPI, CUDA + cuFFT, bundled Kokkos; optional FFTW/MKL, JPEG/PNG (headers absent here -> disabled) +- dependency_complexity: low +- official_inputs_datasets: `bench/in.{lj,eam,chain,chute,rhodo}` + `.scaled` variants, data files < 7 MB, reference logs `bench/log.15Jul25.*.g++.{1,4}`; `examples/` (~800 inputs), `unittest/`, `tools/regression-tests/` (config_kokkos.yaml: `-k on g 2 -sf kk -pk kokkos newton on neigh half`, tol abs 1e-4 / rel 1e-6) +- correctness_mechanism: thermo-output comparison with shipped reference logs within tolerances (regression tooling), force-style YAML unit tests +- strong_scaling_input_availability: yes (`bench/README`: fixed-size problems; 32k atoms far too small for B200 -> replicate with `-var x y z`) +- weak_scaling_input_availability: yes (`-var x Px -var y Py -var z Pz`) +- expected_build_time: **verified 220 s** at -j32 on dgx003 (audit estimate was 30-60 min) +- expected_disk_usage: source 580 MB; build ~3 GB; install ~0.2 GB (verified order of magnitude) +- expected_input_data_size: ~37 MB (`bench/`), no downloads +- b200_cuda132_risk: low-medium (arch flag and CUDA 13 back-ports present; not upstream-validated for CUDA 13.x) -- **verified working on this node** +- mi355x_hip_risk: high for this tag (no gfx950; needs feature release with Kokkos >= 5.1 or unsupported external Kokkos) +- container_availability: no official application image (`tools/singularity/*.def` are build-environment recipes; NGC image is 2023, sm_90 max) +- spack_availability: `lammps` package exists (upstream `20250722.4`, would force EXTERNAL_KOKKOS 4.7.1 for cuda@13); local checkout lacks `cuda_arch=100`; upstream does not recommend Spack +- recommended_integration_priority: FIRST_BATCH +- blocker: none. Watch: long nvcc compiles of templated pair styles (not observed: 220 s); bundled Kokkos 4.6.2 + CUDA 13.2 not upstream-validated +- build_strategy_notes: see BUILD_STRATEGY.md -- **NATIVE** + +## SPARTA + +- official_repository: https://github.com/sparta/sparta (docs https://sparta.github.io/doc/Manual.html; Kokkos `Section_accelerate.html`) +- official_documentation: as above + `BUILD_CMAKE.md`, `bench/README`, https://sparta.github.io/bench.html +- latest_stable_release: `27Aug2026` (2026-08-28); single dated-tag stream. Notes: Kokkos 5.0.2, KOKKOS builds CMake-only and C++20 +- selected_commit_sha: `95b9abaa8bd548991cc3c3f1c58b34722f7ade74` +- license: GPL-2.0 +- application_owned_loc: `src/` 131,181 (`src/KOKKOS` 36,909 / 194 files); bundled `lib/kokkos` 223,495 separately +- main_languages: C++ 80 %, headers 19 % +- build_system: CMake >= 3.16 (`sparta/cmake`, presets `cmake/presets/kokkos_{common,cuda,hip}.cmake`) +- cxx_standard: C++11 core, C++20 with KOKKOS +- supported_compilers: GNU default, Intel documented; Kokkos 5.0.2 enforces GCC >= 10.4, nvcc >= 12.2, ROCm >= 6.2 +- cuda_support: yes; docs explicitly list "GB200 (Blackwell) -> -DKokkos_ARCH_BLACKWELL100=ON" (override the preset's HOPPER90). Kokkos 5.0.2 has native CUDA 13 support (4.7.01 fix, 5.0.2 CUDA 13.1 mdspan fix) +- hip_rocm_support: yes (`kokkos_hip.cmake`, `elcapitan_kokkos.cmake` gfx942); **no gfx950** in bundled Kokkos 5.0.2 +- mpi_support: yes (grid-cell/particle decomposition) +- official_gpu_programming_model: Kokkos (bundled 5.0.2; `USE_EXTERNAL_KOKKOS` without version pin) +- multi_gpu_support: one rank per GPU documented; several ranks per GPU with MPS "recommended" +- multi_node_support: yes (`mpirun -np 32 -ppn 2 spa_kokkos_cuda -k on g 2`) +- gpu_aware_mpi_requirement: optional, default `gpu/aware yes`; **no runtime auto-detection** (`kokkos.cpp` sets the flag unconditionally) -> must be set `no` with a non-CUDA-aware MPI +- rank_to_gpu_binding: self-binding as LAMMPS (`-k on g Ng`, local rank env vars) +- topology_decomposition_controls: `create_grid ... block Px Py Pz|clump|stride|random`, `balance_grid rcb part|cell` (any rank count), `fix balance`; bench size `-var x y z` (particles = 10 x cells) +- major_dependencies: MPI, CUDA (+cuFFT only with PKG_FFT), bundled Kokkos +- dependency_complexity: low +- official_inputs_datasets: `bench/in.{free,collide,sphere}` (+ `ar.species`, `ar.vss`, `data.sphere`), reference logs `bench/log.7Jul14.*.icc.{10K,100K,1M,10M}.{1,8}` (2014 CPU), `examples/` (44 problems with 2023-26 logs) +- correctness_mechanism: statistical log comparison (`tools/testing/regression.py`, tolerances; `examples/README`: "statistically similar answers ... not identical"); invariants: particle count conserved, temperature ~273 K +- strong_scaling_input_availability: yes (fixed `-var x y z`, e.g. 100^3 cells = 10M particles) +- weak_scaling_input_availability: yes (website uses 1M and 16M particles/node) +- expected_build_time: **verified 579 s** at -j32 (estimate 10-20 min) +- expected_disk_usage: source 100 MB; build ~1.5 GB; executable 302 MB (static Kokkos) +- expected_input_data_size: 0.2 MB (`bench/`) +- b200_cuda132_risk: low-medium -- **verified working** +- mi355x_hip_risk: medium-high (no gfx950 in bundled Kokkos; external Kokkos >= 5.1 possible since no pin) +- container_availability: none +- spack_availability: **none** -- the Spack `sparta` package is the unrelated sPARTA bioinformatics tool +- recommended_integration_priority: FIRST_BATCH +- blocker: none. Watch: gpu/aware default with no detection; 2014 reference logs (1 and 8 ranks only) +- build_strategy_notes: **NATIVE** + +## WarpX + +- official_repository: **https://github.com/BLAST-WarpX/warpx** (ECP-WarpX/WarpX redirects, HTTP 301 verified) +- official_documentation: https://warpx.readthedocs.io/ (install/cmake, install/hpc + 21 machine pages, usage/parameters, usage/workflows/domain_decomposition, developers/how_to_test) +- latest_stable_release: 26.09 (2026-09-03), monthly YY.MM tags +- selected_commit_sha: `0c62c75e53a9ad08241535444bd7e53fd1deba88`; pinned AMReX 26.09 `a52ca73324ac2c7b65ec04f131e6df99eec9c576` (`dependencies.json`) +- license: BSD-3-Clause-LBNL +- application_owned_loc: `Source/` 112,459 (C++ 69,266 / 247 files; headers 37,146); no bundled TPL source (AMReX, pyAMReX, PICSAR-QED, openPMD-api, pybind11 fetched at configure time); AMReX `Src/` 273,313 +- main_languages: C++ (~all of `Source/`), Python (PICMI/tests), CMake +- build_system: CMake >= 3.25 (superbuild; `-DWarpX_amrex_src=` for a local AMReX) +- cxx_standard: C++20 +- supported_compilers: GCC 12+, Clang 14+, NVCC 12.4+ (docs); Perlmutter profile: gcc-native/13.2 with **NVCC 13.2.78**; Containerfile `nvidia/cuda:13.2.1-devel` +- cuda_support: yes (`WarpX_COMPUTE=CUDA`, `CMAKE_CUDA_ARCHITECTURES=100`); AMReX >= 25.10 has the CUDA 13 fix; AMReX docs translate legacy "Blackwell" to 100 and 120; GPU CI is H100 (sm_90) only +- hip_rocm_support: yes (`WarpX_COMPUTE=HIP`, `AMReX_AMD_ARCH`, ROCm 6.0+; Frontier/LUMI gfx90a, Tuolumne gfx942); no gfx950 +- mpi_support: yes (default ON, MPI 3.0+, `WarpX_MPI_THREAD_MULTIPLE`) +- official_gpu_programming_model: AMReX (`ParallelFor`) via ablastr +- multi_gpu_support: one rank per GPU (AMReX: "MPI ranks == Number of GPUs") +- multi_node_support: yes; docs recommend GPU-aware MPI +- gpu_aware_mpi_requirement: optional; AMReX auto-detects (`MPIX_Query_cuda_support`), `amrex.use_gpu_aware_mpi=0/1` +- rank_to_gpu_binding: self-binding (rank-in-node when ranks/node == visible GPUs; otherwise `rank % nGPU` with a warning); Perlmutter script pins via `CUDA_VISIBLE_DEVICES` +- topology_decomposition_controls: `warpx.numprocs nx ny nz` (product == ranks, one box per rank), or `amr.n_cell`/`amr.max_grid_size`/`amr.blocking_factor` (n_cell and max_grid_size divisible by blocking_factor), `algo.load_balance_*` +- major_dependencies: AMReX 26.09, PICSAR-QED 26.05 (default `WarpX_QED=ON`), openPMD-api 0.17.1 (default ON, needs HDF5/ADIOS2 for useful backends), pybind11 (Python only); cuFFT from toolkit +- dependency_complexity: medium (minimal CUDA build is self-contained) +- official_inputs_datasets: 407 `inputs*` under `Examples/` (12 Physics_applications incl. `uniform_plasma`, `laser_acceleration`); `Regression/Checksum/benchmarks_json/` (381 files) +- correctness_mechanism: per-test checksums (sum |Q| per field/particle attribute, rtol 1e-9) + Python analysis scripts via ctest; upstream warns checksums are **architecture-dependent** ("may differ on your computer architecture"; repo CLAUDE.md: "ignore checksum failures, since they can be platform-dependent") +- strong_scaling_input_availability: none labelled; `uniform_plasma` is "commonly used to study performance" but shipped at 64x32x32 x 10 steps +- weak_scaling_input_availability: none shipped; periodic uniform plasma scales trivially with `amr.n_cell` + `warpx.numprocs` +- expected_build_time: 25-45 min at -j32 (estimate; measured value recorded in `level3/warpx/README.md`) +- expected_disk_usage: source 24 MB + AMReX 35 MB; build 2-4 GB +- expected_input_data_size: KB-scale inputs +- b200_cuda132_risk: low-medium (upstream already on CUDA 13.2 at Perlmutter; sm_100 untested upstream) +- mi355x_hip_risk: medium +- container_availability: only Perlmutter-specific Containerfiles (sm_80); no published images +- spack_availability: `warpx` package to 26.08 (not a CudaPackage: arch via `^amrex cuda_arch=100`); local checkout too old +- recommended_integration_priority: FIRST_BATCH +- blocker: none hard. Watch: configure-time GitHub fetches (avoided with `WarpX_amrex_src`, `WarpX_QED=OFF`, `WarpX_OPENPMD=OFF`) +- build_strategy_notes: **NATIVE** + +## SPECFEM3D Cartesian + +- official_repository: https://github.com/SPECFEM/specfem3d (moved from geodynamics/specfem3d). Default branch `devel` (HEAD `cc2e9ffa7e7cb5338e05f5a7df81cfbe60e00683`, 2026-07-24) is ~2.5 years ahead of the last release +- official_documentation: https://specfem3d.readthedocs.io/ ; `doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf`; wiki +- latest_stable_release: v4.1.1 (2024-03-15, bug-fix release) +- selected_commit_sha: `c67d3ae7d4bfc5ac75cb9e5601d93afa262d3d8d` +- license: GPL-3.0 +- application_owned_loc: `src/` 142,398 (Fortran 90 119,057; CUDA 12,210; `src/gpu` 15,058 in 63 `.cu`); `utils/` 163,847 and `external_libs/` 137,463 (SCOTCH 5.1.12b, PaToH, METIS) separate +- main_languages: Fortran 90 (~84 %), CUDA C (~9 %), C +- build_system: GNU autotools (pre-generated `configure`; regenerating needs `autoreconf` + the empty `m4/` submodule -- neither available here); in-tree `make all` +- cxx_standard: n/a; Fortran `-std=f2008 -pedantic-errors -ffpe-trap=...` under gfortran (`flags.guess`) +- supported_compilers: gfortran (default), Intel, NVHPC, IBM; CI is CPU-only (gfortran/ifort) +- cuda_support: yes, native CUDA. `--with-cuda=cudaN` selects the *architecture generation*: v4.1.1 ends at `cuda12` = sm_90 + `GPU_DEVICE_Hopper`; devel added `cuda13` = sm_100 + `GPU_DEVICE_Blackwell` (2026-02-14). **v4.1.1 does not compile against CUDA 13**: `src/gpu/initialize_gpu.cu` reads `cudaDeviceProp.deviceOverlap`, removed in CUDA 13 (devel guards it). Legacy `cudaThread*` calls are behind `CUDA_VERSION < 4000`; texture references only under `USE_TEXTURES_*` (commented out). nvcc host compiler = first `gcc` on PATH +- hip_rocm_support: yes (`--with-hip=MI8..MI250` -> gfx803..gfx90a in v4.1.1; devel adds MI300/MI350 = gfx942/gfx950) +- mpi_support: yes (`use mpi`; needs a Fortran MPI module built by a compatible gfortran + MPI-IO) +- official_gpu_programming_model: native CUDA kernels (`src/gpu/kernels/*.cu`), same source built as HIP +- multi_gpu_support: one MPI process per GPU, device = `myrank % device_count` (global rank), or compile-time `-DGPU_DEVICE_ID` +- multi_node_support: yes (databases must be reachable by all ranks) +- gpu_aware_mpi_requirement: not used in v4.1.1 (halo exchange staged through host buffers); devel adds optional `--enable-cuda-aware-mpi` +- rank_to_gpu_binding: self-binding by global-rank modulo; no wrapper needed for ranks == GPUs on one node +- topology_decomposition_controls: `Par_file` `NPROC` (= mesh slices, fixed at mesh time), `PARTITIONING_TYPE` (SCOTCH/METIS/PaToH/rows), `GPU_MODE`, `NSTEP`, `DT`; in-house mesher `Mesh_Par_file`: `NEX_XI/NEX_ETA` multiples of `NPROC_XI/NPROC_ETA` (`NPROC_XI*NPROC_ETA = NPROC`), regions/layers +- major_dependencies: Fortran + C compilers, MPI, CUDA or ROCm; bundled SCOTCH (needs flex/bison, present); optional ADIOS2/HDF5/ASDF +- dependency_complexity: low +- official_inputs_datasets: `EXAMPLES/` 496 MB: `applications/homogeneous_halfspace` (36x36x16 = 20,736 HEX8, CUBIT mesh 3.2 MB + `meshfem3D_files/`, NPROC 4, NSTEP 5000), layered_halfspace, `meshfem3D_examples/*`, Mount_StHelens, CPML, fault (tpv5/tpv102), ...; 35 `REF_SEIS/` reference-seismogram sets; `benchmarks/` (analytic elastic solution, attenuation); `tests/` unit/compile checks +- correctness_mechanism: reference seismograms compared with `utils/scripts/compare_seismogram_correlations.py` (correlation >= 0.8, normalised L2 misfit <= 1 %, time shift <= 0.01 s); analytic benchmark; GPU is single precision -> tolerance-based by design +- strong_scaling_input_availability: not labelled; any fixed mesh with varying NPROC (re-decompose per rank count); shipped meshes are ~20k elements -> larger fixed meshes via `xmeshfem3D` +- weak_scaling_input_availability: yes via `xmeshfem3D` (`NEX_XI/NEX_ETA` with `NPROC_XI x NPROC_ETA`) +- expected_build_time: 5-10 min at -j32 (estimate; measured value in `level3/specfem3d/README.md`) +- expected_disk_usage: source 997 MB (EXAMPLES 496 MB, `.git` 289 MB); build 0.2-0.4 GB; databases tens of MB (20k elements) to GBs +- expected_input_data_size: shipped, < 0.5 GB total; per case < 40 MB +- b200_cuda132_risk: **high as tagged**, medium with two devel back-ports (CUDA 13 `deviceOverlap` guard; Blackwell block) + make-time `GENCODE` override to sm_100; mixed toolchain (conda GCC 13.3 for C/nvcc, system gfortran 14.2.1 for Fortran, conda Open MPI with `OMPI_FC`) -- the Fortran/MPI mix was verified with a 2-rank MPI Fortran program on this node +- mi355x_hip_risk: high for v4.1.1 (no gfx942/gfx950), medium on devel +- container_availability: none +- spack_availability: none (`specfem3d-globe` exists, not Cartesian) +- recommended_integration_priority: FIRST_BATCH (requested order); the audit alone would have said SECOND_BATCH because the tag needs source back-ports +- blocker: (1) v4.1.1 + CUDA 13 compile error -> class D back-port (10 lines, upstream devel provenance); (2) conda `mpif90` unusable without `OMPI_FC=/usr/bin/gfortran` (class C); (3) `NPROC` fixed per decomposition (cheap re-run per rank count); (4) no `autoreconf` +- build_strategy_notes: **NATIVE** + +## nekRS + +- official_repository: https://github.com/Nek5000/nekRS (`master` = latest stable release; HPC scripts in Nek5000/nekRS_HPCsupport) +- official_documentation: https://nekrs.readthedocs.io/ ; in-repo `doc/envHelp.txt`, `doc/parHelp.txt`, `RELEASE.md`, `examples/README.md` +- latest_stable_release: v26.0 (2026-01-27); previous v23.0 (2023-05) +- selected_commit_sha: `96b3cf9e5bacede16568826c04a21bc0fe50dc7d` +- license: BSD-3-Clause +- application_owned_loc: `src/` 53,131 (C++ 35,784; headers 12,322; C 3,782; Fortran 77 1,104); `examples/` 4,983; vendored `3rd_party/` ~2.27 M (lapack 829,795; adios 618,827; hypre 445,492; cvode 169,316; occa 89,888; nek5000 87,143; gslib 13,038; parRSB 6,450) +- main_languages: C++ (~67 %), C, OKL (OCCA kernel language, JIT), Fortran 77 (Nek5000 interface, case `.usr`) +- build_system: CMake >= 3.21 (`build.sh` wraps it with interactive prompts and `-j8`; call cmake directly) +- cxx_standard: C++17, C99 +- supported_compilers: GNU >= 9.1 (fatal below), IntelLLVM, Clang, NVHPC; Fortran GNU/IntelLLVM/NVHPC/Flang for the Nek5000 part; CI: ubuntu, MPICH, gfortran, serial backend +- cuda_support: yes -- OCCA CUDA backend (`OCCA_ENABLE_CUDA`, auto-detected toolkit) + HYPRE on GPU (`ENABLE_HYPRE_GPU`, `find_package(CUDAToolkit 12.0)`). OKL kernels are JIT-compiled at run time with `-arch=sm_` (sm_100 on B200 automatically). `cmake/hypre.cmake` has an explicit CUDA >= 13 branch but hard-codes `HYPRE_CUDA_SM=80 90` (SASS only, no PTX) -> **hypre device kernels would have no Blackwell code without a 1-line change**. Bundled hypre 2.32.0 carries `CUDA_VERSION >= 13000` shims. All 47 CUDA driver-API symbols used by OCCA exist in CUDA 13.2 (checked) +- hip_rocm_support: yes (OCCA HIP, hypre HIP; `--offload-arch` derived from the device at run time); MI250X documented; no gfx950 statement +- mpi_support: yes, required (MPI-3.1; parRSB partitioning, gslib/oogs gather-scatter) +- official_gpu_programming_model: OCCA (vendored development snapshot, `OCCA_VERSION_STR 2.0.0`) + HYPRE +- multi_gpu_support: one rank per GPU; default `--device-id LOCAL-RANK` (node-local rank via `MPI_Comm_split_type`) +- multi_node_support: yes; JIT cache handling (`NEKRS_CACHE_DIR`, `NEKRS_CACHE_LOCAL/BCAST`) +- gpu_aware_mpi_requirement: optional; `NEKRS_GPU_MPI` default OFF (RELEASE.md: enabling "may cause a performance regression"); env `NEKRS_GPU_MPI=1` +- rank_to_gpu_binding: self-binding (`device_id = local rank`); with the Level 3 wrapper (one visible GPU per rank) `--device-id 0` must be passed +- topology_decomposition_controls: none for the process grid (graph partitioning); size = elements in `.re2` x `polynomialOrder`; `ethierRefine.par` `hrefine = N` (uniform h-refinement); `numSteps`, `dt` +- major_dependencies: all vendored (OCCA, HYPRE 2.32.0 built twice host/device, gslib, Nek5000 + parRSB, ADIOS2 2.10.1, CVODE 6.5 off, reference LAPACK in Fortran); external: MPI with Fortran bindings, CMake, OpenMP, CUDA >= 12; **run time needs g++ + nvcc + gfortran** (JIT and `.usr` compilation) +- dependency_complexity: medium (nothing to fetch, but heavy vendored builds and a hard Fortran requirement) +- official_inputs_datasets: `examples/` (59 MB, 20 cases; elements: ethier 32, channel 64, periodicHill 864, gabls1/hit/kershaw 8000, turbPipe 7920, tcf 17,280, tgv 46,656, pb146 pebble bed); ctest harness `examples/CMakeLists.txt` (`--cimode`); CI runs ethier (13 modes), ethierRefine (5), lowMach, mv_cyl, conj_ht, channel, ... with 2 CPU ranks +- correctness_mechanism: ethier = Ethier-Steinman exact Navier-Stokes solution; `ethier.usr` computes L2 errors of velocity/pressure/scalars vs exact; `ci.inc` asserts them (reference values, EPS 0.3) plus iteration counts per `--cimode`; prints "CI test <...> passed|failed", exit code +- strong_scaling_input_availability: not labelled; kershaw (8000 el.), turbPipe, tcf, tgv, pb146 usable; upstream perf numbers at E/GPU = 8000 +- weak_scaling_input_availability: partial: kershaw needs `genbox` (not shipped); `ethierRefine.par` `hrefine` (x8 elements per level) is built in +- expected_build_time: 25-45 min at -j32 (estimate; measured in `level3/nekrs/README.md`); first run of each case adds minutes of JIT +- expected_disk_usage: source 279 MB; build 3-5 GB; install 0.5-1 GB; JIT cache 10s-100s MB +- expected_input_data_size: < 60 MB +- b200_cuda132_risk: medium (hypre SM list; unpinned OCCA snapshot with no CUDA 13 statement; heavy OKL kernels JIT-compiled at `-O3 --use_fast_math` for sm_100) +- mi355x_hip_risk: medium (JIT arch automatic; hypre HIP arch list unaudited) +- container_availability: none official; JIT couples to host toolchain anyway +- spack_availability: `nekrs` package stale (23.0, 21.0; option names do not match v26.0) +- recommended_integration_priority: FIRST_BATCH +- blocker: (1) conda `mpif90` needs `OMPI_FC=/usr/bin/gfortran` (class C, verified); (2) hypre SM list (class B, 1 line); (3) `build.sh` interactive -> cmake called directly; (4) `genbox` missing for kershaw weak scaling. **Found during bring-up (not visible in the audit):** (5) the vendored HYPRE 2.32.0 does not compile against the Thrust 3.2 shipped with CUDA 13 (`thrust::pair` result types, non-transitive `reverse_iterator`/`pair` headers, removed `thrust::not1`) -- ~20 mechanical class-D lines; (6) with conda GCC 13 for C/C++ and system gfortran 14, CMake's FortranCInterface detection fails on LTO bytecode versions (`-fno-lto` at link) and on PIE (`-fPIC` for Fortran); (7) HYPRE's configure takes the conda `AR` variable as the full archive command (`unset AR`); (8) the conda `CMAKE_GENERATOR=Ninja` produces an invalid rule for the HYPRE ExternalProject and breaks the run-time UDF build (upstream's Makefiles generator pinned / env unset at run time); (9) Open MPI's `osc ucx` is selected for nekRS' `MPI_Win_lock` calls and aborts in `uct_ib` with 4 ranks (`OMPI_MCA_osc=^ucx`); (10) the default 8 MB stack limit segfaults the h-refined cases in `useric` (`ulimit -s unlimited`, as upstream's job scripts). All resolved; nekRS validated at 1/2/4 GPUs +- build_strategy_notes: **NATIVE** + +## CP2K + +- official_repository: https://github.com/cp2k/cp2k (DBCSR now an external dependency: https://github.com/cp2k/dbcsr; containers https://github.com/cp2k/cp2k-containers) +- official_documentation: https://manual.cp2k.org/ (technologies/accelerators/cuda.html, hip.html; getting-started/build-from-source.html, build-with-spack.html); compiler matrix wiki; https://www.cp2k.org/performance +- latest_stable_release: v2026.2 (2026-07-15). 2026.1 removed the GNU Makefile (CMake-only); 2026.2 ships `make_cp2k.sh` (Spack-based). (2025.2 is two releases behind.) +- selected_commit_sha: `67b5da876dd6a76b8b021d5a04d1c81ba79a4c50` +- license: GPL-2.0-or-later +- application_owned_loc: `src/` 1,085,842 (Fortran 1,015,193 in 1,325 files; C 58,310; CUDA 1,572 in 5 `.cu`; C++ 1,341; OpenCL 311); no bundled third-party source +- main_languages: Fortran 2008 (93 %), C, CUDA/C++ +- build_system: CMake >= 3.24 + Ninja; dependency bootstraps: `tools/toolchain/install_cp2k_toolchain.sh` (shell, ~40 pinned deps) or `make_cp2k.sh` (private Spack) +- cxx_standard: C11 + C++17; Fortran 2008 +- supported_compilers: GCC 9-16 recommended (toolchain default 14.3.0), Intel oneAPI 2024.2.1 with limitations, `%clang` conflicts; `-allow-unsupported-compiler` forced for nvcc +- cuda_support: yes (`CP2K_USE_ACCEL=CUDA`; `CMAKE_CUDA_ARCHITECTURES=` or `CP2K_WITH_GPU=`). **2026.2's name list ends at H100/GB10 (no B200); upstream master adds `B200 -> 100`.** `CMAKE_CUDA_ARCHITECTURES=100` is accepted by 2026.2's own CMake; the gap is **DBCSR 2.10.0** whose `WITH_GPU` list stops at H100 and derives the arch from it (upstream master's toolchain patches it with a sed + copies `parameters_H100.json -> parameters_B200.json`, i.e. untuned SMM parameters). CI/Spack pin CUDA 12.9.1; no CUDA 13 mention. GPU components toggle individually (DBCSR, DBM, GRID, PW, libGint HFX, SPLA, ELPA, cuSOLVERMp) +- hip_rocm_support: yes (`CP2K_USE_ACCEL=HIP`, Mi50..Mi300 -> gfx906..gfx942); **no gfx950 in CP2K** (DBCSR 2.10 has Mi350 = gfx950) +- mpi_support: yes (MPI-3 required; hybrid MPI+OpenMP `psmp` is production) +- official_gpu_programming_model: native CUDA/HIP (offload layer) + DBCSR JIT kernels (NVRTC), cuBLAS/cuFFT +- multi_gpu_support: device = `MOD(rank, device_count)` (global rank); several ranks per GPU normal (CI runs 2-4 ranks on 1 GPU); no explicit ranks-per-GPU recommendation found +- multi_node_support: yes +- gpu_aware_mpi_requirement: not required (DBM/DBCSR communicate through pinned host buffers; DBCSR `+g2g` optional) +- rank_to_gpu_binding: self-binding by global-rank modulo -> correct when ranks/node is a multiple of visible GPUs; otherwise external per-rank `CUDA_VISIBLE_DEVICES` +- topology_decomposition_controls: none on the CLI; input `&GLOBAL`/`&DBCSR`/`&QS` options, `PREFERRED_DIAG_LIBRARY`, `OMP_NUM_THREADS`; regtest driver `do_regtest.py --mpiranks --ompthreads --num_gpus` +- major_dependencies (toolchain pins): DBCSR 2.10.0, libxsmm 2.0.0, libint 2.13.1, libxc 7.0.0, FFTW 3.3.11, OpenBLAS 0.3.33, ScaLAPACK 2.2.3, ELPA 2026.02.002, COSMA 2.8.4, SpLA 1.6.1, SpFFT 1.1.1, SIRIUS 7.11.1, spglib, HDF5, plumed, dftd4, tblite, libvori, GauXC, libtorch 2.7.1, ...; minimal GPU-DFT set: DBCSR, BLAS/LAPACK/ScaLAPACK, FFTW, libxsmm, libint, libxc +- dependency_complexity: very high +- official_inputs_datasets: all in-repo: `benchmarks/QS/H2O-{32..8192}.inp`, `QS_DM_LS` (`NREP` weak scaling), `QS_ot_ls`, `QS_single_node/*`, `QS_LiH_HFX`, `QS_mp2_rpa`, QMMM, ...; `tests/` 5,255 inputs with reference values (`TEST_FILES.toml`); `data/` 76 MB basis sets +- correctness_mechanism: regtests (matcher values vs `ref=` with `tol=`), `benchmarks/QS_reference/`, `check-release-comparison.py` (energy invariance 1e-10 across MPIxOMP layouts); GPU CI runs the full regtest with `--num_gpus` +- strong_scaling_input_availability: yes (H2O-64/128/256, `H2O-dft-ls.NREP4`, LiH-HFX) +- weak_scaling_input_availability: yes (`QS_DM_LS` `NREP`; `QS/H2O-N` doubling series) +- expected_build_time: dependencies 3-6 h at -j32 (libint lmax >= 5 ~1 h; ELPA, SIRIUS, COSMA, libxc, DBCSR); CP2K 40-90 min; regtests 1-2 h +- expected_disk_usage: source 452 MB; toolchain 15-25 GB (minimal ~5 GB); build 4-6 GB; Spack path 20-40 GB +- expected_input_data_size: 152 MB benchmarks + 76 MB data + 52 MB tests; no downloads +- b200_cuda132_risk: medium-high (no B200 name in 2026.2; DBCSR 2.10.0 arch/parameter patch; CUDA 13.2 untested upstream; Spack `cp2k` and `dbcsr` recipes hard-reject `cuda_arch=100`) +- mi355x_hip_risk: high +- container_availability: official Docker Hub `cp2k/cp2k` tags `{version}_{mpich|openmpi}_{generic|native}_{cuda_P100|A100|H100}_psmp` -- no B200 image; multi-node "requires the MPI of the host system"; no Apptainer here +- spack_availability: yes (`cp2k` 2026.2 upstream; officially recommended via `make_cp2k.sh`), but `cuda_arch` limited to 35-90 in both `cp2k` and `dbcsr` recipes; local checkout too old +- recommended_integration_priority: SECOND_BATCH (mature GPU regtests and inputs; excluded from this round by the > 2 h dependency rule and the DBCSR Blackwell patch) +- blocker: DBCSR sm_100 entry/parameters; 3-6 h dependency stack; Spack `conflicts()`; ELPA-GPU/SIRIUS/COSMA with CUDA 13.2 unverified (keep off first) +- build_strategy_notes: **NATIVE+SPACK_DEPS** (upstream toolchain/Spack for the CPU-side stack, DBCSR + CP2K natively with `CMAKE_CUDA_ARCHITECTURES=100`) + +## Nyx + +- official_repository: https://github.com/AMReX-Astro/Nyx (default branch `development`) +- official_documentation: https://amrex-astro.github.io/Nyx/docs_html/ (`getting_started/BuildingCMake.html`, `NyxSundials.html`, `RunningTheCode.html`, `ICs.html`, `LoadBalancing.html`, `NightlyTests.html`) +- latest_stable_release: 26.09 (tag 2026-08-26, release 2026-09-01); irregular tagging (26.07 before, then nothing since 21.10) +- selected_commit_sha: `e06eabc1b9dbcad5612db9529aced682402daede` +- license: BSD-3-Clause-LBNL +- application_owned_loc: `Source/` 21,644 (C++ 15,696 / 51 files); `Exec/` 10,120; `Util/` 3,398; submodules `subprojects/amrex` @ `6e875b7c` (development, ancestor of 26.09) and `subprojects/sundials` @ v7.2.1 +- main_languages: C++ (98 %), CMake/GNU make; residual Fortran in `Exec/GravityTests` +- build_system: CMake >= 3.14 stated, effectively 3.25 (AMReX submodule); or GNU Make per `Exec/*` directory +- cxx_standard: inherits C++20 from AMReX 26.09 (stale README says C++11/CUDA 9; CI passes `-DCMAKE_CXX_STANDARD=17`) +- supported_compilers: effective constraints from AMReX 26.09 (GCC >= 11, CUDA >= 12.2, ROCm >= 6); CI: GCC, Clang, NVCC 12.6 (job still named "cuda11"), HIP gfx908 +- cuda_support: yes (`Nyx_GPU_BACKEND=CUDA`, `CMAKE_CUDA_ARCHITECTURES=100`; `Nyx_OMP` forced off); `Nyx_HEATCOOL=YES` needs SUNDIALS built with `ENABLE_CUDA` + fused kernels. No sm_100/CUDA 13 mention +- hip_rocm_support: yes, documented "under development" (Spock gfx908 script, CI gfx908); no gfx942/gfx950 +- mpi_support: yes (`Nyx_MPI` default ON) +- official_gpu_programming_model: AMReX +- multi_gpu_support: one rank per GPU via AMReX (Summit `jsrun -a 1 -g 1`, Spock `--gpus-per-task=1`) +- multi_node_support: yes +- gpu_aware_mpi_requirement: optional (AMReX auto-detect) +- rank_to_gpu_binding: self-binding by AMReX (rank-in-node) +- topology_decomposition_controls: `amr.n_cell`, `amr.max_grid_size` (GPU: 128 or 256 recommended), `amr.blocking_factor`, `amr.max_level`, `DistributionMapping.strategy`, `nyx.load_balance_*`; boxes >= ranks +- major_dependencies: AMReX (any `>= 20.11` external, or submodule), SUNDIALS >= 6.0 (HEATCOOL only), MPI, CUDA; optional Reeber/Gimlet/Ascent. A single AMReX 26.09 install (3D, PARTICLES, LINEAR_SOLVERS, EB, FFT, MPI, CUDA, SUNDIALS) satisfies WarpX's pin and Nyx's minimum +- dependency_complexity: low-medium +- official_inputs_datasets: `Exec/LyA/inputs` (64^3, IC `64sssss_20mpc.nyx` 14.7 MB), `inputs.rt` (32^3), `Exec/AMR-density/inputs.cuda`, `Exec/AMR-zoom`, `Exec/MiniSB/inputs.32` (Santa Barbara), `Exec/Scaling/inputs` (64^3 `RandomPerCell`, no IC file) + `inputs.256.noreduceverb`, HydroTests (Sedov/Sod/shock tubes), GravityTests, ...; **larger ICs (256^3, 1024^3) exist only at OLCF paths, no public URL**; `nyx.particle_init_type = Cosmological` in code but undocumented +- correctness_mechanism: AMReX nightly regression suite (`fcompare`/`particle_compare` vs LBNL-hosted benchmark plotfiles -- not in repo); analytic hydro tests; MiniSB comparison +- strong_scaling_input_availability: `Exec/Scaling/` (64^3, 256^3); Spock script references 768^3-2048^3 inputs not shipped +- weak_scaling_input_availability: informal (`RandomPerCell` init scales `amr.n_cell` freely; `inputs.cuda` documents `prob_hi` for 512^3/1024^3/6144^3) +- expected_build_time: 20-35 min at -j32 (AMReX 3D CUDA 10-15, SUNDIALS CUDA 5-10, Nyx few) +- expected_disk_usage: source 104 MB (65 MB ICs) + submodules ~100 MB; build 1-3 GB +- expected_input_data_size: ~65 MB shipped +- b200_cuda132_risk: medium (AMReX part as WarpX; SUNDIALS 7.2.1 predates CUDA 13; stale CI; only 64^3 ICs) +- mi355x_hip_risk: medium-high +- container_availability: none +- spack_availability: no `nyx` package (only `amrex +sundials`, `sundials`) +- recommended_integration_priority: SECOND_BATCH (cheap and AMReX-native, but stale docs/CI, SUNDIALS-CUDA for the flagship problem, no downloadable large ICs, no shipped baselines) +- blocker: submodules to initialise (or external AMReX with the component set above); SUNDIALS 7.2.1 + CUDA 13.2 untested; large ICs unavailable +- build_strategy_notes: **NATIVE** (against the WarpX AMReX 26.09 checkout; start with MiniSB / adiabatic LyA, then `Nyx_HEATCOOL=YES`) + +## QMCPACK + +- official_repository: https://github.com/QMCPACK/qmcpack (`develop`; `main` = release) +- official_documentation: https://qmcpack.readthedocs.io/en/develop/ (installation, running, performance_portable); Nexus docs +- latest_stable_release: v4.4.0 (2026-08-31); 4.3.0 raised CUDA minimum to 12.3; legacy drivers slated for removal +- selected_commit_sha: `2601d62e353934f1526cab1f67f30b6672b7c76f` +- license: University of Illinois/NCSA Open Source License +- application_owned_loc: `src/` 337,813 (C++ 168,240; headers 146,034; CUDA 8,066); `external_codes/` 294,038 and `nexus/` 140,384 separate +- main_languages: C++ (~93 %), CUDA, Python +- build_system: CMake >= 3.21 +- cxx_standard: C++17 (C++20 auto-selected if the compiler defaults to it) +- supported_compilers: GCC >= 9 (but "OpenMP offload is not ready for GCC"), Clang >= 7, oneAPI >= 2021, NVHPC, XL; docs: **"For NVIDIA GPUs, LLVM clang"**; nightly: Clang 22.1.1, CUDA 12.9, ROCm 7.0.1, Open MPI 5.0.10, HDF5 1.14.5, Boost 1.90/1.84 +- cuda_support: yes: `-DQMC_GPU="openmp;cuda" -DQMC_GPU_ARCHS=sm_100` (arbitrary `sm_XX` passthrough); `QMC_GPU=cuda` alone = cuBLAS/cuSOLVER batched LA with the rest on CPU (CMake-valid, not the recommended GPU build); `find_package(CUDAToolkit 12.3)`; no CUDA-13-removed APIs found; no sm_100/CUDA 13 mention +- hip_rocm_support: yes (`QMC_GPU="openmp;hip"`, rocBLAS/hipBLAS, amdclang; Frontier gfx90a; ROCm 7.0.1 tested); gfx950 passthrough undocumented +- mpi_support: yes (walkers across ranks; ensembles) +- official_gpu_programming_model: OpenMP target offload + vendor BLAS/solver libraries, small native CUDA/HIP/SYCL kernels +- multi_gpu_support: docs: "1 MPI task should be used per GPU per node" for medium/large runs; device from node-local rank (`DeviceManager`), warns if `local_size % num_devices != 0` +- multi_node_support: yes (`shared_ranks` spline sharing in 4.4.0) +- gpu_aware_mpi_requirement: not used +- rank_to_gpu_binding: self-binding (node-local rank -> device; respects `CUDA_VISIBLE_DEVICES`) +- topology_decomposition_controls: none (walker-parallel): `walkers_per_rank`/`total_walkers`, `blocks/steps/timestep`, `OMP_NUM_THREADS`, `--dryrun` +- major_dependencies: MPI, BLAS/LAPACK, HDF5 >= 1.10 (1.14.5 tested; conda has 2.2.0 -- untested upstream), FFTW3, **Boost >= 1.70 headers (absent on node)**, libxml2, Python 3 + numpy (h5py), CUDA >= 12.3, **Clang with NVPTX offload (absent on node)**; bundled boost_multi, Catch2, mpi3 +- dependency_complexity: medium (standard libs mostly in conda; official GPU path needs an LLVM/Clang offload toolchain) +- official_inputs_datasets: `tests/` 178 MB (solids, molecules, heg, afqmc, ...; ctest labels `unit`, `deterministic`, `short`, `performance`); `tests/performance/NiO` S1-S256 spline files 43 MB - 8.8 GB each from an external Box link ("direct links ... may be fragile"), `-DQMC_DATA=`; `examples/` +- correctness_mechanism: deterministic ctests (fixed seeds, exact scalar checks, 142 entries), statistical energy-within-error tests (`qmc-ref`), Catch2 unit tests +- strong_scaling_input_availability: yes (NiO S-series at fixed walker count; needs downloads) +- weak_scaling_input_availability: yes (fixed `walkers_per_rank`; or S8 -> S16 -> S32 electron counts) +- expected_build_time: 15-30 min (`QMC_GPU=cuda`, GCC) / 30-60 min (`openmp;cuda`, Clang) at -j32, + 1-2 h if LLVM must be built +- expected_disk_usage: source 575 MB; build 3-6 GB; LLVM +5-10 GB +- expected_input_data_size: 178 MB shipped; NiO 0.3-8.8 GB per size (S1-S32 ~3 GB practical) +- b200_cuda132_risk: medium-high (sm_100 passthrough trivial; but Clang-offload + CUDA 13.2 unverified upstream (LLVM 22.1.1 tested with CUDA 12.9); GCC-only fallback leaves most kernels on CPU; conda HDF5 2.2.0 untested; Boost missing) +- mi355x_hip_risk: medium +- container_availability: CI dependency images only (CPU, no CUDA); no production/GPU image +- spack_availability: `qmcpack` package to 4.3.0 upstream (local 4.1.0); its `+cuda` passes `QMC_CUDA=1`, which 4.x CMake ignores -> CPU-only binary (inferred from CMake); no offload/rocm variants +- recommended_integration_priority: SECOND_BATCH (excellent benchmark suite; needs an LLVM/Clang offload toolchain, Boost, possibly HDF5 1.14, and multi-GB datasets) +- blocker: Clang/LLVM with NVPTX offload; Boost headers; HDF5 version; NiO datasets external +- build_strategy_notes: **NATIVE+SPACK_DEPS** (Spack only for `llvm+cuda`, `boost`, `hdf5@1.14`; QMCPACK itself natively with `QMC_GPU`/`QMC_GPU_ARCHS`) + +## GEOS + +- official_repository: https://github.com/GEOS-DEV/GEOS (formerly GEOSX; TPLs https://github.com/GEOS-DEV/thirdPartyLibs; submodules LvArray, BLT, PVTPackage, hdf5_interface, uberenv) +- official_documentation: https://geosx-geosx.readthedocs-hosted.com/en/latest/ (QuickStart, buildGuide/{Prerequisites,Dependencies,BuildProcess,SpackUberenv,ContinuousIntegration}, advancedExamples/performanceBenchmarks) +- latest_stable_release: 1.2.0 (2024-10-02, "Latest"); `develop` (`b7a0f133...`, 2026) is ~2 years ahead and is what docs/CI describe (TPL tag 361-1070, CUDA 12.9.1 images, RAJA 2026.07.0) +- selected_commit_sha: `920e17a00b2ab86a5c0c98088fc69b904b1af55b` +- license: LGPL-2.1-only +- application_owned_loc: `src/` ~240 k (C++ 123,485 + headers 105,242 + CMake 3,866 + Python 7,738; 511 .cpp / 714 .hpp); submodules (LvArray, PVTPackage, hdf5_interface, BLT) not counted; `inputFiles/` 108 MB (582 XML) +- main_languages: C++17 with RAJA/CHAI device lambdas (no `.cu`; nvcc compiles `.cpp` under BLT), Python (ATS/pygeosx) +- build_system: CMake >= 3.24 + BLT; host-config files; TPLs via `thirdPartyLibs/scripts/config-build.py` superbuild or uberenv/Spack +- cxx_standard: C++17 +- supported_compilers: docs "gcc 12+ or clang 13.0+"; 1.2.0 CUDA CI rows: clang 10/gcc 9.4 + CUDA 11.8.89, clang 17 + CUDA 12.5.1, gcc 8.5 + CUDA 12.5.1; develop TPL images gcc13/clang19 + CUDA 12.9.1; **CUDA 13.2.1 rows commented out: "CUDA 13 is blocked by the pinned RAJA package: raja '^cuda@13:' conflicts with '+cuda'"** +- cuda_support: yes (`ENABLE_CUDA`, `CMAKE_CUDA_ARCHITECTURES`, `ENABLE_HYPRE_DEVICE=CUDA` with hypre `--with-cuda --enable-unified-memory --with-umpire`); documented CUDA 11.5-12.5 (1.2.0), 12.9.1 (develop); no sm_100 mention (CI `cuda_arch=86,120`) +- hip_rocm_support: yes (`ENABLE_HIP`, `CMAKE_HIP_ARCHITECTURES` gfx90a/gfx942/gfx1100, Frontier/Tioga host-configs, ROCm 5.4-6.4); no gfx950 +- mpi_support: yes (mesh decomposition, hypre/Trilinos/PETSc, parallel HDF5/Silo/VTK) +- official_gpu_programming_model: RAJA + CHAI + Umpire + camp via LvArray; hypre on device +- multi_gpu_support: one MPI rank per GPU (Frontier launch scripts `--ntasks-per-gpu=1 --gpu-bind=closest`; lassen jsrun) +- multi_node_support: yes (27 B-element weak-scaling study on Frontier) +- gpu_aware_mpi_requirement: optional (GEOS uses pinned host buffers, `-s/--suppress-pinned`; hypre `ENABLE_HYPRE_GPU_AWARE_MPI` off by default) +- rank_to_gpu_binding: **external** (no `cudaSetDevice`/local-rank logic in `src/`; relies on `--gpu-bind`, jsrun or per-rank `CUDA_VISIBLE_DEVICES`) +- topology_decomposition_controls: CLI `-x/-y/-z` partitions (InternalMesh), `-s`, `-b`; external meshes partitioned by ParMETIS/Scotch (any rank count); `` XML blocks (`scaling="strong" scaleList=...`) + `benchmarks/runBenchmarks.py` +- major_dependencies: pinned by thirdPartyLibs `0e2ed33e` (tag 284-535) for 1.2.0: RAJA/CHAI(+Umpire, camp) **v2024.07.0**, hypre v2.31.0-12, conduit 0.9.2, HDF5 1.12.1, silo 4.11, VTK 9.3.1, Trilinos 15.1.1 (optional), PETSc 3.19.4 (optional), superlu_dist, ParMETIS 4.0.3, Scotch 7.0.3, SuiteSparse 5.10.1, Caliper 2.11.0, Adiak, pugixml, fmt 11.0.1, mathpresso. thirdPartyLibs HEAD (develop): RAJA/CHAI/Umpire **v2026.07.0** (= Level 2's pins exactly), hypre master `f1374fb6`, VTK 9.7.0, Trilinos 16.1.0, ... -> Level 2's `.deps/install/{raja,umpire,chai}` match GEOS *develop*, not 1.2.0 +- dependency_complexity: very high +- official_inputs_datasets: 582 XML (`*_smoke.xml`, `*_benchmark.xml`; solidMechanics, singlePhaseFlow, compositionalMultiphaseFlow incl. SPE10 tables in-repo, poromechanics, hydraulicFracturing, wavePropagation, `wellboreECP/*/level01-06`); some inputs reference the separate LFS GEOSXDATA repo; integrated tests need `geos-ats` + a baseline tarball from a GCP bucket +- correctness_mechanism: gtest unit tests; integrated restart-check vs baselines with tolerances (`BASELINE_NOTES.md`); analytical examples in docs (Mandel, Terzaghi, Sneddon, KGD) +- strong_scaling_input_availability: yes (`` blocks, `runBenchmarks.py`; `-x -y -z` overrides) +- weak_scaling_input_availability: yes (`wellboreECP` level01-06, 826 k -> 27 B elements) +- expected_build_time: TPL superbuild 2-3 h at -j32 (VTK/Trilinos/hypre-CUDA; Trilinos can be disabled with hypre on device) + GEOS 1.5-2.5 h -> ~4-5 h +- expected_disk_usage: source 381 MB (+ ~150 MB submodules); TPLs 15-25 GB; GEOS build 10-20 GB +- expected_input_data_size: 108 MB in-repo; optional LFS data and baseline tarball (sizes unpublished) +- b200_cuda132_risk: high for tag 1.2.0 (RAJA suite 2024.07 + hypre 2.31, CUDA <= 12.5 heritage, no sm_100), medium for develop (RAJA 2026.07.0 built with CUDA 13.2 in Level 2; upstream's own Spack path calls CUDA 13 blocked; hypre master + cuSPARSE/cuSOLVER on CUDA 13.2 unverified) +- mi355x_hip_risk: high +- container_availability: CI TPL images on Docker Hub (`geosx/--cuda:`, CUDA 11.8/12.5/12.9, in-container Open MPI) -- CI/devcontainer use only +- spack_availability: **no upstream package** (Spack's `geos` is libgeos); GEOS ships its own `geosx` recipe consumed only through uberenv with LC-system `spack.yaml` files (host-config generation only, "must never be used without a spack.yaml") +- recommended_integration_priority: SECOND_BATCH (strong benchmark story; 4-5 h build, no Blackwell/CUDA 13 upstream coverage, tag-vs-develop decision, cloud-hosted baselines) +- blocker: choose 1.2.0 (independent 2024.07 RAJA suite) vs develop (shares Level 2's RAJA 2026.07.0); multi-hour TPL build; submodules; hypre GPU on CUDA 13.2 unverified; baseline download + `geos-ats`; `cuda_arch=100` never used upstream +- build_strategy_notes: **NATIVE** (thirdPartyLibs superbuild + custom host-config, `ENABLE_TRILINOS=OFF`, `ENABLE_HYPRE_DEVICE=CUDA`, `CMAKE_CUDA_ARCHITECTURES=100`, conda Open MPI; prefer develop so RAJA/CHAI/Umpire 2026.07.0 can be shared with Level 2) + +## DFT-FE + +- official_repository: https://github.com/dftfeDevelopers/dftfe (release branch `release1.2`); install scripts https://github.com/dftfeDevelopers/install_DFTFE (per-machine branches); benchmarks https://github.com/dftfeDevelopers/dftfe-benchmarks +- official_documentation: https://sites.google.com/umich.edu/dftfe ; manual PDF (`manual` branch); Doxygen https://dftfedevelopers.github.io/dftfe/ +- latest_stable_release: 1.2.0 (2025-08-17): "seamless support for NVIDIA, AMD and Intel GPUs", meta-GGA, DFT+U, mixed precision +- selected_commit_sha: `7147faa51f7c9f3075fffaa5e48ba989bcd329c1` +- license: LGPL-2.1-or-later +- application_owned_loc: `src/ include/ utils/` 130,659 (C++ 109,351 in 221 files; headers 21,187; device code is `.cc` compiled as CUDA/HIP/SYCL); no bundled TPLs +- main_languages: C++17 (device kernels in CUDA/HIP/SYCL-flavoured C++) +- build_system: CMake >= 3.17; two builds per install (`WITH_COMPLEX=OFF/ON`); helper `setupUser.sh` +- cxx_standard: C++17 +- supported_compilers: not formally documented; scripts use Cray CC (Perlmutter gcc-native 12.3, Frontier cpe/25.09), icpx (NSM A100), gcc-10 (Ubuntu Docker) +- cuda_support: yes (`WITH_GPU=ON GPU_LANG=cuda GPU_VENDOR=nvidia CMAKE_CUDA_ARCHITECTURES= CMAKE_CUDA_FLAGS="-arch=sm_XX"`, cuBLAS; optional NCCL `WITH_DCCL`, `WITH_GPU_AWARE_MPI`); scripts hard-code sm_70/sm_80; **no sm_90/sm_100 anywhere**; documented CUDA 11.7 / 12.9.1; small device API surface (cuBLAS, `cudaSetDevice`), no removed APIs; deal.II built CPU-only (Kokkos Serial) +- hip_rocm_support: yes (`GPU_LANG=hip`, hipBLAS, gfx90a Frontier; RCCL optional); no gfx942/gfx950 +- mpi_support: yes (FE domain decomposition x band groups `NPBAND` x k-point pools `NPKPT`; ELPA/ScaLAPACK) +- official_gpu_programming_model: native CUDA/HIP/SYCL via DFT-FE's device abstraction + cuBLAS/hipBLAS/oneMKL +- multi_gpu_support: one rank per GPU documented (Frontier `--ntasks-per-gpu 1 --gpu-bind closest`; Perlmutter `--gpus-per-task=1`); oversubscription seen in test scripts (18 ranks / 6 GPUs); threads = 1 +- multi_node_support: yes (to ~40,000 GPUs; Summit benchmarks) +- gpu_aware_mpi_requirement: optional (`WITH_GPU_AWARE_MPI`, "use with care"; NCCL alternative) +- rank_to_gpu_binding: self-binding (`device_id = mpi_rank % n_devices`, global rank) +- topology_decomposition_controls: `.prm` `subsection Parallelization { NPKPT, NPBAND, BAND PARAL OPT }`, `subsection GPU { USE GPU, AUTO GPU BLOCK SIZES, USE GPUDIRECT MPI ALL REDUCE, USE ELPA GPU KERNEL }`, `USE ELPA`; ranks divisible by `NPKPT*NPBAND` +- major_dependencies (install_DFTFE pins): deal.II >= 9.5.1 (`+P4EST +64BIT_INDICES +MPI +LAPACK`; scripts 9.6.2/9.7.1), p4est 2.8.6/7, Kokkos 4.3/4.6 (CPU-only, for deal.II), Boost 1.86, ScaLAPACK 2.2.x, BLIS/libflame or OpenBLAS, ELPA 2025.01/2025.06 (`--enable-nvidia-gpu-kernels --with-NVIDIA-GPU-compute-capability=sm_80`), ALGLIB, libxc 6.2.2/7.0.0, spglib, libxml2, numdiff (tests); optional PETSc/SLEPc, NCCL, dftd3/4, libtorch +- dependency_complexity: high +- official_inputs_datasets: `demo/ex1-3` (with reference `.output`), `testsGPU/pseudopotential/{real,complex}` (56 GPU `.prm` cases, `accuracyBenchmarks/`, `diffScript`, Slurm/PBS scripts), `tests/dft/*` (112 `.prm.in`, 133 `*.mpirun=N.output` references), 57 ONCV pseudopotentials (11 MB), `data/` 73 MB; external `dftfe-benchmarks` (Mo 431-8,191 atoms; Al nanoparticles) +- correctness_mechanism: ctest via deal.II harness with `numdiff` against `.output`; GPU: `REPRODUCIBLE OUTPUT = true` + `diffScript` vs `accuracyBenchmarks/` +- strong_scaling_input_availability: indirect (any fixed system on 1..4 GPUs; Summit references per size in the benchmarks repo) +- weak_scaling_input_availability: `dftfe-benchmarks` Mo N x N x N supercell series (24 -> 3,600 GPUs); in-repo inputs do not scale automatically +- expected_build_time: deal.II 1-1.5 h; ELPA-GPU 15-30 min; small libs 20 min; DFT-FE real + complex 25-40 min each -> ~3-4 h +- expected_disk_usage: source 126 MB; deps 8-12 GB; DFT-FE 3-4 GB +- expected_input_data_size: ~85 MB in-repo; benchmarks repo external +- b200_cuda132_risk: medium (arch is user-supplied and kernels generic; never run above sm_80 upstream; ELPA `sm_100` acceptance and ELPA 2025.x with CUDA 13.2 unverified -> ELPA CPU/ScaLAPACK fallback; NCCL for CUDA 13 separate; scripts assume Cray/icpx) +- mi355x_hip_risk: medium-high +- container_availability: CPU Docker recipe only (`install_DFTFE` `generalUbuntuCPU`); no GPU image +- spack_availability: `dftfe` package stale (0.5-0.6, 2019, no GPU variants); upstream route is the per-machine `install_DFTFE` scripts (not Spack) +- recommended_integration_priority: SECOND_BATCH +- blocker: deal.II/p4est/Kokkos/Boost multi-hour stack; ELPA GPU kernels for sm_100; NCCL; gcc/Open MPI adaptation of the Cray/icpx scripts; numdiff +- build_strategy_notes: **NATIVE** (node-specific `install_dftfe.sh` derived from the `nsm_A100` branch: conda GCC 13.3 + system gfortran + Open MPI 5.0.10, OpenBLAS/ScaLAPACK, Kokkos Serial, deal.II 9.7.1, ELPA (GPU kernels attempted at sm_100, CPU fallback), DFT-FE real+complex `CMAKE_CUDA_ARCHITECTURES=100`, `WITH_DCCL=OFF`, `WITH_TESTING=ON`) + +--- + +## Cross-cutting findings + +1. **Spack is not a shortcut to Blackwell for any candidate on this node.** The + personal checkout (2025-05-06) predates every 2026 release and lacks + `cuda_arch=100`/`gfx950`; upstream recipes are missing (SPARTA, Nyx, GEOS, + SPECFEM3D), stale (nekRS 23.0, DFT-FE 0.6, QMCPACK `+cuda` inert for 4.x), + or reject `cuda_arch=100` outright (`cp2k`, `dbcsr`; `raja ^cuda@13:` + conflict blocks GEOS' path). Where Spack helps it is for CPU-side + dependencies only (CP2K, QMCPACK) -> `NATIVE+SPACK_DEPS`. +2. **Containers do not apply here**: no Apptainer/Singularity on dgx003, no + official B200 image for any candidate, and containers would not provide the + host driver, the CUDA-aware Open MPI transport or the network. +3. **Blackwell/CUDA 13 upstream coverage** is thin everywhere: only WarpX + (Perlmutter profile, CUDA 13.2.78) and the two Kokkos codes (arch flag + + CUDA 13 fixes in the bundled Kokkos) have first-class support; SPECFEM3D + v4.1.1 needs two back-ports; CP2K 2026.2/DBCSR 2.10.0, GEOS, DFT-FE never + mention sm_100; GEOS explicitly lists CUDA 13 as blocked in its Spack path. +4. **Fortran**: the conda environment has no gfortran; the system gfortran + 14.2.1 works with the conda Open MPI Fortran bindings (`OMPI_FC=/usr/bin/gfortran`, + verified with a 2-rank MPI Fortran program). Needed by SPECFEM3D, nekRS, + CP2K, DFT-FE (ScaLAPACK/ELPA/p4est). +5. **Rank -> GPU mapping**: LAMMPS/SPARTA/nekRS bind by node-local rank, + WarpX/Nyx by AMReX's rank-in-node, SPECFEM3D/CP2K/DFT-FE by *global* rank + modulo device count, GEOS not at all. The Level 3 launcher's per-rank + wrapper (one visible GPU per rank, audited) makes all of them correct on a + node; nekRS additionally needs `--device-id 0` under the wrapper. +6. **Multi-node MPI is BLOCKED/UNVERIFIED on this site**; every application + above is multi-node capable per upstream, none is verified beyond one node. diff --git a/level3/BUILD_STRATEGY.md b/level3/BUILD_STRATEGY.md new file mode 100644 index 0000000..4a00cbe --- /dev/null +++ b/level3/BUILD_STRATEGY.md @@ -0,0 +1,86 @@ +# Level 3 build strategy + +Companion to `APPLICATION_AUDIT.md`. For each application the four build +routes were assessed against what upstream documents and what dgx003 provides +(RHEL 10, 4x B200, CUDA 13.2.78, conda GCC 13.3.0 / Open MPI 5.0.10 +CUDA-aware / CMake 3.28.4, system gfortran 14.2.1, no ROCm, no +Apptainer/Singularity, lmod broken, Spack 1.0.0.dev0 checkout of 2025-05). +Vocabulary: `BUILD_RECOMMENDATION = NATIVE | SPACK | NATIVE+SPACK_DEPS | +APPTAINER | SPACK+APPTAINER | SITE_NATIVE | DEFER`. + +## Decision matrix + +| Application | Native (upstream build system) | Spack | Apptainer/Singularity | Site-native modules | BUILD_RECOMMENDATION | +|---|---|---|---|---|---| +| LAMMPS | feasible, documented (CMake presets, bundled Kokkos 4.6.2 with BLACKWELL100 + CUDA 13 back-ports); deps = MPI + CUDA; **built in 220 s** | package exists but local checkout has no `cuda_arch=100`; upstream recipe would force EXTERNAL_KOKKOS 4.7.1 = configuration upstream calls "untested"; upstream does not recommend Spack | no runtime; no official image (NGC image sm_90-max, 2023) | lmod broken | **NATIVE** | +| SPARTA | feasible, documented (`cmake -C presets/kokkos_cuda.cmake -DKokkos_ARCH_BLACKWELL100=ON`); deps = MPI + CUDA; **built in 579 s** | **no package** (Spack's `sparta` is a bioinformatics tool) | no runtime; no recipe | lmod broken | **NATIVE** | +| WarpX | feasible, documented superbuild; small graph (AMReX local checkout, PICSAR/openPMD off); CUDA vs HIP one switch; upstream itself builds with CUDA 13.2 | possible only with a fresh spack-packages (26.09 missing; arch via `^amrex cuda_arch=100` legacy path); graph balloons with `+openpmd +python` | no runtime; only Perlmutter Containerfiles (sm_80) | lmod broken | **NATIVE** | +| SPECFEM3D Cartesian | feasible, only documented route (autotools; bundled SCOTCH); needs two devel back-ports for CUDA 13 + make-time `GENCODE` for sm_100; Fortran via system gfortran + `OMPI_FC` | no package | none; tiny dependency graph, nothing to gain | lmod broken | **NATIVE** | +| nekRS | feasible, only documented route (CMake; all TPLs vendored); CUDA/HIP separable by OCCA options; JIT needs host g++/nvcc/gfortran at run time anyway; 1-line hypre SM patch | recipe stale (23.0, wrong option names); deps vendored -> nothing for Spack to provide | none official; JIT couples to host toolchain | lmod broken | **NATIVE** | +| CP2K | feasible (CMake + `install_cp2k_toolchain.sh`); ~15 packages for a GPU-DFT build; needs the DBCSR B200 sed upstream master applies; 3-6 h | officially recommended (`make_cp2k.sh`, `spack install cp2k+cuda`) **but `cp2k` and `dbcsr` recipes hard-reject `cuda_arch=100`**; 60-100 packages; local checkout too old | official `cp2k/cp2k` images stop at H100; multi-node needs host MPI; no runtime here | lmod broken | **NATIVE+SPACK_DEPS** | +| Nyx | feasible, only documented path (CMake superbuild or GNU make); can consume the WarpX AMReX 26.09 checkout (`AMREX_MINIMUM_VERSION 20.11`); SUNDIALS CUDA superbuild for HEATCOOL | no package | none | lmod broken | **NATIVE** | +| QMCPACK | feasible today only for `QMC_GPU=cuda` (partial GPU); the recommended `openmp;cuda` needs Clang with NVPTX offload (absent) + Boost (absent) + tested HDF5 | package's `+cuda` is inert for 4.x (no `QMC_GPU`), no offload variant -> use Spack only for `llvm+cuda`, `boost`, `hdf5@1.14` | CI dependency images only (CPU) | lmod broken | **NATIVE+SPACK_DEPS** | +| GEOS | feasible, documented (thirdPartyLibs superbuild + host-config); ~20 TPLs, 4-5 h; `ENABLE_TRILINOS=OFF` with hypre on device; develop can share Level 2's RAJA/CHAI/Umpire 2026.07.0 | no upstream package; uberenv recipe is LC-only and hits the `raja ^cuda@13:` conflict upstream calls blocking | CI TPL images (CUDA <= 12.9, in-container MPI) | lmod broken | **NATIVE** | +| DFT-FE | feasible, only documented route (per-machine `install_DFTFE` shell scripts + CMake); ~12 autotools/CMake deps, 3-4 h; CUDA vs HIP = rebuild of DFT-FE + ELPA only | `dftfe` recipe unusable (0.6); a dealii/elpa/libxc hybrid possible but local Spack too old and ELPA `cuda_arch=100` unverified | CPU Docker recipe only | lmod broken | **NATIVE** | + +No candidate gets `DEFER`: every one has an officially supported native CUDA +path on this toolchain. `APPTAINER`/`SPACK+APPTAINER`/`SITE_NATIVE` are +unavailable on this node regardless of application. + +## Spack policy (Level 3) + +- Spack is used **only** where upstream documents it as a supported route and + the recipe can express the target (`cuda_arch=100`, CUDA 13.2 external, + conda Open MPI external). In this round that is nowhere; the two + `NATIVE+SPACK_DEPS` candidates (CP2K, QMCPACK) will use it for CPU-side + dependencies when they are brought up. +- When used, each application/backend gets its own environment + `level3/envs//{cuda,rocm}/spack.yaml` with a committed `spack.lock`; + the lock's SHA-256 is recorded in the application fingerprint + (`spack_lock_sha256=` line, currently `none`). CUDA and ROCm environments are + never merged (they cannot concretize together here anyway: no ROCm). +- The personal Spack checkout (`/projects/kzhou6/bcui2/env_software/spack`, + 1.0.0.dev0, 2025-05-06) is too old for every 2026 release and lacks + Blackwell; a Level 3 Spack environment will need a fresh Spack >= 1.2 with + `spack-packages` >= 2026-07 and the conda Open MPI/CUDA declared as externals. + This is recorded as a prerequisite, not done in this round. + +## Apptainer policy (Level 3) + +- Not applicable on dgx003 (no runtime). If a future site provides Apptainer: + only the `.def`, a README, the build script and the image SHA-256 are + committed (never a `.sif`); the definition must pin the base image digest and + the application commit; containers do not solve the host driver, the MPI + transport or the interconnect, and multi-node runs still require the host + MPI/PMIx -- so a container build is treated as one more `Build Strategy` + variant with its own fingerprint, never as the default. + +## Per-application dependency isolation + +Every Level 3 application owns `.deps/level3//{src,build,install,logs}` +(private copy of patched sources where a build must be in-tree or patched, +private dependency builds, install prefix, logs) plus the fingerprint +`.deps/level3//install/.hpcperf-l3-fingerprint` written by +`level3/tools/l3_common.sh` (schema `l3-1`: upstream commit, backend/arch, +dependency versions, compiler, Fortran compiler, CUDA/ROCm, MPI, CMake options, +GPU-aware MPI setting, site profile, Spack lock SHA-256, container SHA-256, +patch list, build time). A fingerprint mismatch makes `build.sh` fail fast with +the differing lines. Nothing under `.deps/install/` (Level 2) is modified or +reused: the Level 2 Kokkos 5.2.1 / RAJA suite / hypre / AMReX pins are not the +versions these applications validate against (LAMMPS pins Kokkos 4.6.2, SPARTA +5.0.2, WarpX AMReX 26.09, nekRS hypre 2.32.0), and Level 2 must keep building. + +## Modification classes used in the first batch + +| Application | Class | What | +|---|---|---| +| LAMMPS | A | none; derived `in.lj` deck (`run ${steps}`, weak `processors`) written into the build tree | +| SPARTA | A | none | +| WarpX | A | none; derived inputs file (sizes, `warpx.numprocs`, reduced diagnostics) written into the build tree | +| SPECFEM3D | B + C + D | make-time `GENCODE` override (sm_100, devel's `cuda13` value) and bundled SCOTCH built without gzip support (generated `Makefile.inc`, no `zlib.h` in the conda sysroot); `OMPI_FC=/usr/bin/gfortran`, `MPI_INC`; two devel back-ports (CUDA 13 `deviceOverlap` guard 10 lines, Blackwell device block 8 lines) in `level3/specfem3d/patches/` | +| nekRS | B + C + D | B: `cmake/hypre.cmake` `HYPRE_CUDA_SM=80 90` -> `80 90 100` (1 line), CMake generator pinned to upstream's Unix Makefiles (the conda `CMAKE_GENERATOR=Ninja` yields an invalid rule for the vendored HYPRE install step); C: `OMPI_FC=/usr/bin/gfortran`, `LDFLAGS+=-fno-lto` (mixed GCC 13 / gfortran 14 LTO bytecode in CMake's Fortran/C detection), `FFLAGS+=-fPIC` (PIE default of the conda GCC), `unset AR` (HYPRE's configure takes `$AR` as the full archive command), run time `unset CMAKE_GENERATOR` (UDF build), `OMPI_MCA_osc=^ucx` (one-sided ops otherwise go through UCX and abort with 4 ranks), `ulimit -s unlimited` (as upstream's job scripts); D: vendored HYPRE 2.32.0 vs the Thrust 3.2 shipped with CUDA 13 -- `thrust::pair` result type -> `auto` (2 lines), explicit ``/`` includes in `device_utils.h` and in the pre-generated concatenated `_hypre_utilities.hpp`, `thrust::not1` -> `thrust::not_fn` (16 lines); all in `level3/nekrs/patches/` | + +No class E change anywhere; no numerics, physics or algorithm touched. The +nekRS list shows the general pattern for Fortran + CMake applications on this +node (CP2K and DFT-FE will meet the same OMPI_FC / LTO / PIE issues) and that +nekRS' vendored HYPRE 2.32.0 is not CUDA 13-ready as shipped. diff --git a/level3/README.md b/level3/README.md index 93d36f7..e7717b7 100644 --- a/level3/README.md +++ b/level3/README.md @@ -1,19 +1,131 @@ # Level 3: Production / End-to-End HPC Applications -Not implemented yet. +Level 3 integrates **full production applications** -- complete workflows, not +extracted kernels, not proxies, and never N independent replicas presented as +one distributed run. Its primary execution mode is **multi-GPU with a +user-selected GPU count**; single-GPU runs exist only for build smoke tests, +environment compatibility and basic correctness bring-up. -Level 3 covers production or end-to-end HPC applications. These may involve -multi-GPU, multi-node execution, job schedulers, and external libraries, so -the Level 1 layout is not enforced here. +Branch `level3/full-apps-bringup` (from `main`): STEP 1-2 audit and build +strategy for all ten candidates ([APPLICATION_AUDIT.md](APPLICATION_AUDIT.md), +[BUILD_STRATEGY.md](BUILD_STRATEGY.md)), STEP 3-6 first-batch bring-up on +dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083, 2026-09-04). Nothing is +claimed validated beyond what the per-application README records for runs +that actually happened on this node. -Planned layout (reference only): +## Status +| Application | Version | Build Strategy | CUDA Build | 1 GPU | 2 GPU | 4 GPU | HIP | Strong | Weak | Multi-node | Source Mod | Status | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| [LAMMPS](lammps/README.md) | stable_22Jul2025_update6 | NATIVE (bundled Kokkos 4.6.2) | OK, 220 s | PASS | PASS | PASS | untested | 16.4M atoms, 1/4 GPU run | 2.05M atoms/rank, 4 GPU run | BLOCKED/UNVERIFIED | A (derived deck) | FIRST_BATCH done | +| [SPARTA](sparta/README.md) | 27Aug2026 | NATIVE (bundled Kokkos 5.0.2) | OK, 579 s | PASS | PASS | PASS | untested | 10M particles, 1/4 GPU run | 1.25M particles/rank, 4 GPU run | BLOCKED/UNVERIFIED | A | FIRST_BATCH done | +| [WarpX](warpx/README.md) | 26.09 (+AMReX 26.09) | NATIVE (local AMReX source) | OK, 1219 s | PASS | PASS | PASS | untested | 33.6M particles, 1/4 GPU run | 4.2M particles/rank, 4 GPU run | BLOCKED/UNVERIFIED | A (derived inputs) | FIRST_BATCH done | +| [SPECFEM3D Cartesian](specfem3d/README.md) | v4.1.1 (+2 devel back-ports) | NATIVE (autotools, bundled SCOTCH) | OK, 21 s | PASS | PASS | PASS | untested | 165,888 elements, 1/4 GPU run | 165,888 elements/rank, 4 GPU run | BLOCKED/UNVERIFIED | B+C+D (18 lines, upstream devel) | FIRST_BATCH done | +| [nekRS](nekrs/README.md) | v26.0 | NATIVE (vendored OCCA/HYPRE) | OK, ~30 min | PASS | PASS | PASS | untested | 32,000 elements N=7, 1/4 GPU run | 8,000 elements/rank, 4 GPU run | BLOCKED/UNVERIFIED | B+C+D (39 lines; vendored HYPRE 2.32.0 vs CUDA 13) | FIRST_BATCH done | +| CP2K | v2026.2 | NATIVE+SPACK_DEPS | not started | -- | -- | -- | -- | H2O-N series (upstream) | QS_DM_LS NREP (upstream) | -- | -- | SECOND_BATCH (deps 3-6 h; DBCSR B200 patch) | +| Nyx | 26.09 | NATIVE (shared AMReX 26.09) | not started | -- | -- | -- | -- | Exec/Scaling (upstream) | RandomPerCell init | -- | -- | SECOND_BATCH | +| QMCPACK | v4.4.0 | NATIVE+SPACK_DEPS | not started | -- | -- | -- | -- | NiO S-series (download) | walkers_per_rank | -- | -- | SECOND_BATCH (needs Clang offload, Boost) | +| GEOS | 1.2.0 / develop | NATIVE (thirdPartyLibs superbuild) | not started | -- | -- | -- | -- | `` XML (upstream) | wellboreECP level01-06 | -- | -- | SECOND_BATCH (TPLs 4-5 h) | +| DFT-FE | 1.2.0 | NATIVE (install_DFTFE model) | not started | -- | -- | -- | -- | testsGPU systems | dftfe-benchmarks Mo series | -- | -- | SECOND_BATCH (deal.II stack 3-4 h) | + +8/40/80-GPU shapes exist for every first-batch application as launcher +dry-runs only (`HPCPERF_DRY_RUN=1`): **DRY-RUN / UNVALIDATED**. Multi-node MPI +is BLOCKED/UNVERIFIED on this site. HIP recipes exist in every `build.sh` and +exit with a clear message here (no ROCm): **untested**. + +## Hard requirements (summary of the Level 3 policy) + +1. Full application workflow (mesher/solver/IO stages included where upstream + has them); no hotspot-only or single-kernel runs. +2. `HPCPERF_GPUS=N|all` selects the GPU count; requested == launched. A rank + count the application's decomposition cannot support is an error -- never a + silent change of N, never silent GPU sharing, never a fallback to 1 GPU, + never a failure reported as PASS. +3. Default policy is one MPI rank per GPU; if upstream officially recommends + another model (threads per GPU, MPI+OpenMP, several GPUs per rank) the + application follows upstream and its README says so. All five first-batch + applications document one rank per GPU. +4. Every application defines smoke / strong / weak inputs (global size, + per-rank size, memory estimate, process topology, expected runtime, + validation quantity), the rank->GPU mapping and multi-node requirements. +5. 40/80-GPU shapes are `DRY-RUN / UNVALIDATED` until a real allocation + exists; multi-node is BLOCKED/UNVERIFIED on this site; HIP is `untested` + without an AMD GPU. +6. Toolchain follows the application's officially supported versions, not + Level 1's pins; compatibility modifications are classified (A none, + B build-system-only, C environment, D source-level compatibility) -- E + algorithm/performance modifications are forbidden in bring-up. +7. The validated Level 2 dependency tree (`.deps/install`) is never modified. + +## Correctness policy as applied + +Exit code is never sufficient. Each `validate.sh` uses the application's own +mechanism and prints a single `... validation (N GPU, ...): PASS|FAIL` line: +LAMMPS thermo vs the shipped reference log (bit-identical here); SPARTA +statistical stats vs the shipped reference log with justified tolerances +(particle count exact, temperature 2 %, collision attempts 15 %); WarpX +upstream's analytic Langmuir-wave regression test (5e-2) and charge +conservation (1e-11) read from the plotfile, plus exact particle conservation; +SPECFEM3D reference seismograms through upstream's comparison script +(correlation, misfit, time shift); nekRS upstream's `--cimode` CI checks on the +analytic Ethier solution. No tolerance was loosened to obtain a PASS; no +precision or physics setting was changed. + +## Dependency isolation + +Every application owns a private tree -- no shared Level 3 install root: + +``` +.deps/level3//{src,build,install,logs} patched source copy (where needed), deps, install, logs +_upstream/level3/ shallow upstream checkout at the selected tag (read-only) +build/level3// application build tree (+ run/ directories of run.sh) ``` -level3// -├── README.md -├── source or integration files -├── build files -├── configs/ -├── jobs/ -└── run.sh + +Installs carry `.hpcperf-l3-fingerprint` (schema `l3-1`: application, upstream +commit, dependency versions, compiler, Fortran compiler, CUDA/ROCm, GPU arch, +MPI, CMake/configure options, GPU-aware-MPI setting, patch list, site profile, +Spack lock hash, container image hash, build time). A recorded fingerprint +that differs from the requested configuration fails fast +(`level3/tools/l3_common.sh`). + +Spack, when chosen, uses one environment per application and backend +(`level3/envs//{cuda,rocm}/spack.yaml` + `spack.lock`); containers, when +chosen, commit the `.def`, build script, image SHA256 and README -- never the +`.sif`. Neither is used by the first batch (see BUILD_STRATEGY.md for why). + +## Runtime + +Launches go through the common launcher (`HPCPERF_GPUS`, `HPCPERF_NODES`, +`HPCPERF_GPUS_PER_NODE`, `HPCPERF_CPUS_PER_RANK`, `HPCPERF_SCALE_MODE`, +`HPCPERF_SITE_PROFILE`, `HPCPERF_DRY_RUN=1`) with the per-rank GPU wrapper +(each rank sees one GPU; expected vs observed GPU audited). Level 3 refers to +it through `HPCPERF_RUNTIME_DIR` (default `level2/tools`); the plan to move +the shared tools to `tools/runtime/` without breaking Level 2 is in +[../tools/runtime/README.md](../tools/runtime/README.md). + +Site/transport observations recorded in the READMEs (single node, `pml ob1 / +btl self,sm,smcuda`): GPU-aware MPI makes WarpX's 4-GPU step 3x slower +(0.081 vs 0.026 s/step) but LAMMPS 2.5x faster (2.38 vs 5.87 s); SPARTA is +indifferent. Defaults stay upstream's; this is a performance topic for a later +round, not a bring-up change. Open MPI's one-sided layer still selects +`osc ucx` on this node and aborts inside `uct_ib` with 4 ranks (nekRS uses +`MPI_Win_lock`); nekRS' `run.sh` sets `OMPI_MCA_osc=^ucx`, which is proposed +for the gmu-hopper site profile in the runtime commonization PR. + +## Per-application layout + ``` +level3// +├── README.md provenance, version/commit, license, LOC, build strategy, changes (A-D), execution model, +│ inputs (smoke/strong/weak), validation, 1/2/4-GPU results, dry-runs, limitations +├── fetch.sh shallow clone at the recorded tag/commit (no source trees committed) +├── build.sh native build into .deps/level3/, fingerprinted; HIP branch present, untested +├── run.sh HPCPERF_GPUS + HPCPERF_SCALE_MODE aware, launched via the common launcher +├── validate.sh upstream correctness mechanism, PASS/FAIL line, exit code +└── patches/ compatibility patches (classified, documented; SPECFEM3D, nekRS) +``` + +Inputs are upstream's own decks referenced from the read-only checkout; +derived decks (size, steps, topology, diagnostics) are written into the build +tree at run time and documented per application, so no upstream input file is +modified and nothing large is committed. diff --git a/level3/lammps/README.md b/level3/lammps/README.md new file mode 100644 index 0000000..3785623 --- /dev/null +++ b/level3/lammps/README.md @@ -0,0 +1,126 @@ +# LAMMPS (Level 3) + +Full classical molecular dynamics (neighbor lists, short- and long-range +forces, spatial decomposition, MPI halo exchange) -- run as the complete +application through its own input scripts, KOKKOS package on the GPU. + +## Provenance + +- Official repository: https://github.com/lammps/lammps (docs + https://docs.lammps.org/, Kokkos: https://docs.lammps.org/Speed_kokkos.html) +- Release policy: `stable_*` tags with `_updateN` bug-fix updates; `patch_*` + are feature releases (GitHub marks them pre-release). +- Selected: **`stable_22Jul2025_update6`** (released 2026-09-03), commit + `9c5ab448c78a14fd534619622162ba418d6a1fb1`, fetched by `fetch.sh` into + `_upstream/level3/lammps` (shallow, read-only). +- License: GPL-2.0 (`LICENSE`). +- Application-owned LOC (cloc 2.06, code lines): `src/` **852,527** in + 3,865 files = 743,761 outside `src/KOKKOS` + 108,766 in `src/KOKKOS` + (the GPU package). Bundled `lib/kokkos` (Kokkos 4.6.2) is counted + separately and not modified. + +## Build strategy: NATIVE (upstream CMake + bundled Kokkos 4.6.2) + +`build.sh CUDA` = LAMMPS' documented Kokkos/CUDA recipe: +`nvcc_wrapper` (host compiler conda GCC 13.3.0) as CXX, `PKG_KOKKOS`, +`Kokkos_ENABLE_CUDA`, `Kokkos_ARCH_BLACKWELL100` (sm_100; the bundled +Kokkos 4.6.2 supports it), `Kokkos_ENABLE_OPENMP/SERIAL`, `FFT_KOKKOS=CUFFT`, +`FFT=KISS` (host), `BUILD_MPI` (conda Open MPI 5.0.10, CUDA-aware), C++17, +packages `MOLECULE KSPACE MANYBODY RIGID GRANULAR` (what `bench/` needs), +`WITH_JPEG=no WITH_PNG=no` (no `jpeglib.h` on the node; image dumps unused). +Build time on dgx003: **220 s** at `-j32` (708 targets). Warnings: 663 +lines, essentially all `nvcc_wrapper: multiple optimization flags` (conda +`-O2` + Release `-O3`) plus two upstream unused-variable notes (`#550-D`, +`#177-D`); no errors. Install prefix `.deps/level3/lammps/install` +(fingerprinted: upstream commit, Kokkos 4.6.2, compiler, CUDA 13.2.78, MPI, +CMake options, GPU-aware setting). + +Why not the others: upstream does not recommend Spack for GPU builds (the +Spack `lammps` package exists but the local Spack checkout is 2025-05 and +lacks `cuda_arch=100`); no Apptainer on the node and a container would not +provide the host MPI/transport; site modules are broken on dgx003. Level 2's +Kokkos 5.2.1 is **not** used: LAMMPS requires an external Kokkos +`>= 4.6.02` and pins 4.6.2 internally -- the bundled one is the supported +configuration. + +HIP: `build.sh HIP` carries the upstream `Kokkos_ENABLE_HIP` + +`Kokkos_ARCH_AMD_GFX950` + `FFT_KOKKOS=HIPFFT` recipe and exits with a clear +message here (no ROCm). **Untested.** + +## Changes from upstream + +Class **A -- no source modification.** `run.sh` writes a *derived* copy of +`bench/in.lj` into the build tree with `run 100` -> `run ${steps}` and, in +weak mode, a `processors ${px} ${py} ${pz}` line; the upstream file is +untouched, and with the default 100 steps the derived deck is semantically +identical. + +## Execution model + +One MPI rank per GPU (upstream `Speed_kokkos`), `-k on g 1 t 1 -sf kk +-pk kokkos newton on neigh half gpu/aware on`. Ranks are launched by the +common launcher with the per-rank GPU wrapper, so each rank sees exactly one +GPU (`g 1`) and the launcher audits expected vs observed GPU. GPU-aware MPI +(`gpu/aware on`, LAMMPS default) is used with the CUDA-aware conda Open MPI; +`HPCPERF_LAMMPS_GPU_AWARE=off` selects host-staged communication. Any rank +count is legal: LAMMPS factors the box into a processor grid itself (weak mode +passes the grid explicitly). Threads per rank: 1 (`HPCPERF_CPUS_PER_RANK` +sets `t`), as upstream recommends for GPU runs. + +## Inputs (`HPCPERF_SCALE_MODE`) + +| Mode | Global box (fcc cells) | Atoms | Per rank @4 GPU | Topology | Steps | Memory/GPU (est.) | Runtime on B200 | Validation quantity | +|---|---|---|---|---|---|---|---|---| +| smoke (default) | 20^3 (upstream `in.lj`) | 32,000 | 8,000 | LAMMPS auto | 100 | < 0.1 GB | 0.02-0.09 s | thermo vs upstream reference log | +| strong | (20*S)^3, S=`HPCPERF_LAMMPS_STRONG` (8) = 160^3 | 16,384,000 | 4,096,000 | LAMMPS auto | 100 | ~2 GB | 1.1 s (1 GPU) / 2.3 s (4 GPU) | same thermo table | +| weak | (20*L*P)^3-shaped, L=`HPCPERF_LAMMPS_LOCAL` (4): 80^3 cells/rank | 2,048,000 x N | 2,048,000 | `hpcperf_topology.py` grid = `processors` | 100 | ~0.3 GB | 1.1 s (4 GPU) | same thermo table | + +Memory estimate: ~120 B/atom for LJ with Kokkos neighbor lists (55 +neighbors/atom, half list) -- well below the 180 GB of a B200 at every size +above; the strong default is deliberately a *correctness* size (100 steps run +in seconds), not a performance deck. The 4-GPU strong run (2.29 s) being +slower than 1 GPU (1.12 s) at 16.4M atoms/100 steps is the expected +communication-dominated behaviour of a fixed small workload and is **not** a +scaling result. + +## Validation (`validate.sh`, upstream mechanism) + +Thermo output (Temp, E_pair, TotEng, Press at steps 0 and 100) of the +unmodified `bench/in.lj` is compared with the reference log LAMMPS ships, +`bench/log.15Jul25.lj.fixed.g++.1` (CPU, 1 process; `velocity ... loop geom` +makes the initial state machine- and rank-count-independent). Tolerances: +1e-8 relative at step 0 (deterministic), 1e-5 at step 100 (reduction-order +divergence); with N > 1 GPUs the N-rank run is also compared with this build's +1-GPU run. Observed on dgx003 (2026-09-04): **all eight quantities identical +to the reference to every printed digit (rel 0.00e+00) at 1, 2 and 4 GPUs**. + +## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) + +| Run | Ranks x GPUs | rank->GPU | CPU binding | Topology | Problem | Loop time | Validation | +|---|---|---|---|---|---|---|---| +| smoke | 1 x 1 | wrapper (1 visible GPU/rank); audit 1/1 verified | runtime default, `t 1` | 1x1x1 | 32k atoms, 100 steps | 0.020 s | PASS | +| smoke | 2 x 2 | wrapper; audit 1 verified / 1 unverified (0.07 s run, too short to sample) | runtime default | LAMMPS auto | 32k atoms | 0.070 s | PASS (vs ref and vs 1-GPU) | +| smoke | 4 x 4 | wrapper; audit 4/4 verified | runtime default | LAMMPS auto | 32k atoms | 0.085 s | PASS (vs ref and vs 1-GPU) | +| strong | 1 x 1 | wrapper | runtime default | 1x1x1 | 16.4M atoms | 1.118 s | run completes; thermo consistent | +| strong | 4 x 4 | wrapper; 4/4 verified | runtime default | LAMMPS auto | 16.4M atoms | 2.295 s | run completes | +| weak | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2x1 (`processors`) | 8.19M atoms (2.05M/rank) | 1.128 s | run completes | + +Dry-runs (`HPCPERF_DRY_RUN=1`, hypothetical allocations) -- **DRY-RUN / +UNVALIDATED**, nothing executed: + +| GPUs | Nodes x GPUs/node | Mode | Global box | Per rank | Ranks/node | Launch | +|---|---|---|---|---|---|---| +| 8 | 1 x 8 | strong | 160^3 cells = 16.4M atoms | 2.05M | 8 | `mpirun -np 8 --host dgx003:8 --map-by ppr:8:node ...` (single node) | +| 40 | 5 x 8 | weak | 400x320x160 = 81.9M atoms | 2.05M | 8 | `mpirun -np 40 --host <5 nodes>:8 --map-by ppr:8:node` -- multi-node BLOCKED on this site | +| 80 | 10 x 8 | weak | 400x320x320 = 163.8M atoms | 2.05M | 8 | `mpirun -np 80 ...` -- multi-node BLOCKED on this site | + +## Limitations + +- Multi-node: BLOCKED/UNVERIFIED on this site (transport); 40/80-GPU shapes + are plans only. +- HIP: recipe present, untested (no AMD GPU). +- The benchmark family here is `bench/in.lj`; `in.eam`, `in.rhodo` + (pppm/kk + cuFFT), `in.chain`, `in.chute` build with this package set but + have no wrappers yet. +- Weak-mode `processors` grid comes from the generic balanced factorization; + LAMMPS' own auto grid is used in smoke/strong. diff --git a/level3/lammps/build.sh b/level3/lammps/build.sh new file mode 100755 index 0000000..fd4e42a --- /dev/null +++ b/level3/lammps/build.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Build LAMMPS (KOKKOS package, CUDA or HIP) with upstream's native CMake and +# the Kokkos version LAMMPS bundles (lib/kokkos) -- the upstream-supported path. +# +# ./build.sh [CUDA|HIP] (default CUDA) +# +# Layout (Level 3 dependency isolation, nothing shared with Level 2): +# source _upstream/level3/lammps (fetch.sh; the app owns its Kokkos) +# build build/level3/lammps/ +# install .deps/level3/lammps/install (+ .hpcperf-l3-fingerprint) +# logs .deps/level3/lammps/logs +# +# Toolchain: conda GCC 13.3.0 as nvcc_wrapper host compiler (LAMMPS documents +# GCC >= 8 and C++17), system CUDA, conda Open MPI 5.0.10 (CUDA-aware). +# Packages: KOKKOS + MOLECULE, KSPACE (pppm/kk needs FFT_KOKKOS=CUFFT/HIPFFT), +# MANYBODY, RIGID, GRANULAR -- the set the bench/ inputs need. +# +# Modification class: A (no upstream source modified; build flags only). +# +# Environment overrides: +# HPCPERF_CUDA_ARCH numeric compute capability (default: detected; 100 -> Kokkos_ARCH_BLACKWELL100) +# HPCPERF_HIP_ARCH Kokkos AMD arch name for HIP (default AMD_GFX950); HIP is UNTESTED here +# HPCPERF_BUILD_JOBS parallel jobs (default 32) +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +# conda activation scripts are not set -u safe +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +SRC="$R/_upstream/level3/lammps" +[ -f "$SRC/cmake/CMakeLists.txt" ] || { echo "build.sh: LAMMPS source missing -- run $HERE/fetch.sh first" >&2; exit 1; } +SHA="$(git -C "$SRC" rev-parse HEAD)" +KOKKOS_VER="$(sed -n 's/^set(Kokkos_VERSION_\(MAJOR\|MINOR\|PATCH\) \([0-9]*\))/\2/p' "$SRC/lib/kokkos/CMakeLists.txt" | paste -sd.)" +l3_paths lammps +BUILD_DIR="$R/build/level3/lammps/$MODEL" +JOBS="${HPCPERF_BUILD_JOBS:-32}" +PKGS=(-DPKG_KOKKOS=yes -DPKG_MOLECULE=yes -DPKG_KSPACE=yes -DPKG_MANYBODY=yes -DPKG_RIGID=yes -DPKG_GRANULAR=yes) + +case "$BACKEND" in + CUDA) + ARCH="${HPCPERF_CUDA_ARCH:-$(l3_gpu_arch)}" + case "$ARCH" in + 100) KARCH=BLACKWELL100;; 120) KARCH=BLACKWELL120;; 90) KARCH=HOPPER90;; 80) KARCH=AMPERE80;; + *) echo "build.sh: no Kokkos arch mapping for compute capability '$ARCH' (set HPCPERF_CUDA_ARCH)" >&2; exit 2;; + esac + export NVCC_WRAPPER_DEFAULT_COMPILER="$CXX" + GPU_FLAGS=(-DCMAKE_CXX_COMPILER="$SRC/lib/kokkos/bin/nvcc_wrapper" + -DKokkos_ENABLE_CUDA=yes "-DKokkos_ARCH_$KARCH=yes" -DFFT_KOKKOS=CUFFT) + ARCHNOTE="sm_$ARCH ($KARCH)" ;; + HIP) + command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + KARCH="${HPCPERF_HIP_ARCH:-AMD_GFX950}" + GPU_FLAGS=(-DCMAKE_CXX_COMPILER=hipcc -DKokkos_ENABLE_HIP=yes "-DKokkos_ARCH_$KARCH=yes" -DFFT_KOKKOS=HIPFFT) + ARCHNOTE="$KARCH" ;; + *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; +esac + +# Image output libs are disabled: the node has a libjpeg runtime but no headers +# (jpeglib.h), and dump image is not part of any benchmark here. +CMAKE_OPTS="BUILD_MPI=yes BUILD_OMP=yes CXX_STANDARD=17 Kokkos_ENABLE_${BACKEND}=yes Kokkos_ARCH_${KARCH} Kokkos_ENABLE_OPENMP=yes Kokkos_ENABLE_SERIAL=yes FFT=KISS FFT_KOKKOS=${GPU_FLAGS[-1]#-DFFT_KOKKOS=} WITH_JPEG=no WITH_PNG=no PKGS=KOKKOS,MOLECULE,KSPACE,MANYBODY,RIGID,GRANULAR" +FP="$(l3_fingerprint_text lammps "$SHA" "$MODEL" "kokkos(bundled)=$KOKKOS_VER" "$CMAKE_OPTS" "runtime(-pk kokkos gpu/aware)")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +echo "# LAMMPS $BACKEND: upstream $SHA, bundled Kokkos $KOKKOS_VER, arch $ARCHNOTE, MPI $(mpirun --version 2>/dev/null | head -1)" +mkdir -p "$BUILD_DIR" +cmake -S "$SRC/cmake" -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$L3_INSTALL" \ + -DCMAKE_CXX_STANDARD=17 -DBUILD_MPI=yes -DBUILD_OMP=yes -DLAMMPS_MACHINE="kokkos_$MODEL" \ + -DKokkos_ENABLE_OPENMP=yes -DKokkos_ENABLE_SERIAL=yes -DFFT=KISS \ + -DWITH_JPEG=no -DWITH_PNG=no -DWITH_GZIP=yes \ + "${GPU_FLAGS[@]}" "${PKGS[@]}" > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ + || { tail -30 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } +t0=$(date +%s) +cmake --build "$BUILD_DIR" -j "$JOBS" > "$L3_LOGS/build-$MODEL.log" 2>&1 \ + || { tail -30 "$L3_LOGS/build-$MODEL.log"; echo "build.sh: build failed (log: $L3_LOGS/build-$MODEL.log)" >&2; exit 1; } +cmake --install "$BUILD_DIR" > "$L3_LOGS/install-$MODEL.log" 2>&1 || { echo "build.sh: install failed" >&2; exit 1; } +l3_fingerprint_write "$L3_INSTALL" "$FP" +echo "# built in $(( $(date +%s)-t0 )) s: $BUILD_DIR/lmp_kokkos_$MODEL (installed under $L3_INSTALL)" +echo "# compiler warning lines: $(grep -c 'warning' "$L3_LOGS/build-$MODEL.log" || true)" diff --git a/level3/lammps/fetch.sh b/level3/lammps/fetch.sh new file mode 100755 index 0000000..cc803c4 --- /dev/null +++ b/level3/lammps/fetch.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Fetch the LAMMPS upstream source at the recorded stable release into the +# read-only reference checkout _upstream/level3/lammps (gitignored). Nothing is +# built here. Re-running is idempotent; a checkout at a different commit is an +# error (delete it to re-fetch), never silently reused. +# +# ./fetch.sh +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" + +# Official repository and the selected stable release (see README.md). +UPSTREAM_URL="https://github.com/lammps/lammps.git" +UPSTREAM_TAG="stable_22Jul2025_update6" +UPSTREAM_SHA="9c5ab448c78a14fd534619622162ba418d6a1fb1" +DEST="$R/_upstream/level3/lammps" + +if [ -d "$DEST/.git" ]; then + have="$(git -C "$DEST" rev-parse HEAD)" + if [ "$have" = "$UPSTREAM_SHA" ]; then + echo "fetch.sh: $DEST already at $UPSTREAM_TAG ($UPSTREAM_SHA)"; exit 0 + fi + echo "fetch.sh: $DEST is at $have, not the recorded $UPSTREAM_SHA ($UPSTREAM_TAG); remove it to re-fetch" >&2 + exit 1 +fi +mkdir -p "$(dirname "$DEST")" +echo "fetch.sh: cloning $UPSTREAM_URL @ $UPSTREAM_TAG (shallow)" +git clone --quiet --depth 1 --branch "$UPSTREAM_TAG" "$UPSTREAM_URL" "$DEST" +have="$(git -C "$DEST" rev-parse HEAD)" +[ "$have" = "$UPSTREAM_SHA" ] || { echo "fetch.sh: tag $UPSTREAM_TAG resolved to $have, expected $UPSTREAM_SHA" >&2; exit 1; } +echo "fetch.sh: ok -> $DEST ($UPSTREAM_SHA)" diff --git a/level3/lammps/run.sh b/level3/lammps/run.sh new file mode 100755 index 0000000..c1efc71 --- /dev/null +++ b/level3/lammps/run.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Run the LAMMPS Lennard-Jones benchmark (upstream bench/in.lj) on N GPUs. +# +# ./run.sh [CUDA|HIP] [extra lmp args...] +# +# Execution model (upstream Speed_kokkos): one MPI rank per GPU, KOKKOS +# package on the device, one host thread per rank. Ranks are launched through +# the common launcher with the per-rank GPU wrapper, so every rank sees exactly +# one GPU and LAMMPS is started with `-k on g 1`; the launcher audits the +# rank->GPU mapping. GPU-aware MPI (device-buffer halo exchange, +# `-pk kokkos gpu/aware on`) is the LAMMPS default and matches the CUDA-aware +# Open MPI of this repository; HPCPERF_LAMMPS_GPU_AWARE=off disables it. +# +# Resource / size controls (common Level 3 parameters): +# HPCPERF_GPUS=N|all ranks = GPUs (default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# smoke : upstream bench/in.lj as shipped: 20^3 fcc cells = 32,000 atoms, +# 100 steps; bring-up / correctness (reference log in bench/) +# strong : ONE fixed global box, (20*S)^3 cells with S=HPCPERF_LAMMPS_STRONG +# (default 8: 160^3 cells = 16,384,000 atoms), decomposed by +# LAMMPS over the ranks (any N is legal; LAMMPS factors the grid) +# weak : fixed work per rank: (20*L)^3 cells per rank, L=HPCPERF_LAMMPS_LOCAL +# (default 4: 80^3 cells = 2,048,000 atoms/rank); the box is +# 20*L*PX x 20*L*PY x 20*L*PZ with PXxPYxPZ from hpcperf_topology.py +# and the same grid is passed to LAMMPS `processors` +# HPCPERF_LAMMPS_STEPS MD steps (default 100, the bench convention) +# HPCPERF_LAMMPS_GPU_AWARE on|off (default on) +# +# LAMMPS decomposes the box into a PxQxR processor grid automatically for any +# rank count; in weak mode the grid is set explicitly to match the box shape. +# Extra args are appended to the lmp command line; arguments that would change +# the validated problem (-in, -var x/y/z, -k, -sf, -pk) are rejected. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +EXE="$R/build/level3/lammps/$MODEL/lmp_kokkos_$MODEL" +[ -x "$EXE" ] || { echo "run.sh: $EXE not found -- run ./build.sh $BACKEND first" >&2; exit 1; } +SRC="$R/_upstream/level3/lammps" + +N_RANKS="$(hpcperf_ranks lammps yes)" || exit 2 +hpcperf_forbid_args lammps -in -i -var -v -k -kokkos -sf -suffix -pk -package -log -- "$@" || exit 2 +MODE="$(l3_scale_mode lammps)" || exit 2 +STEPS="${HPCPERF_LAMMPS_STEPS:-100}" +GAM="${HPCPERF_LAMMPS_GPU_AWARE:-on}" + +PROCS=() +case "$MODE" in + smoke) X=1; Y=1; Z=1 ;; + strong) S="${HPCPERF_LAMMPS_STRONG:-8}"; X=$S; Y=$S; Z=$S ;; + weak) L="${HPCPERF_LAMMPS_LOCAL:-4}" + TOPO="$(hpcperf_topology lammps "$N_RANKS")" || exit 2 + read -r PX PY PZ <<< "$TOPO" + X=$((L * PX)); Y=$((L * PY)); Z=$((L * PZ)) + PROCS=(-var px "$PX" -var py "$PY" -var pz "$PZ") ;; +esac +ATOMS=$(( 4 * 20 * X * 20 * Y * 20 * Z )) +RUN_DIR="$R/build/level3/lammps/$MODEL/run"; mkdir -p "$RUN_DIR" +LOG="$RUN_DIR/log.$MODE.np$N_RANKS.lammps" +# Derived deck (upstream bench/in.lj untouched): `run 100` -> `run ${steps}`, +# and in weak mode a `processors ${px} ${py} ${pz}` line before create_box so +# the rank grid matches the box shape. With steps=100 and no processors line +# the derived deck is semantically identical to upstream's. +IN="$RUN_DIR/in.lj.$MODE" +{ + if [ "${#PROCS[@]}" -gt 0 ]; then + sed -e 's/^create_box.*/processors ${px} ${py} ${pz}\n&/' -e 's/^run[[:space:]].*/run ${steps}/' "$SRC/bench/in.lj" + else + sed -e 's/^run[[:space:]].*/run ${steps}/' "$SRC/bench/in.lj" + fi +} > "$IN" + +echo "# LAMMPS $BACKEND: mode=$MODE ranks=$N_RANKS box=$((20*X))x$((20*Y))x$((20*Z)) fcc cells = $ATOMS atoms ($((ATOMS / N_RANKS))/rank), $STEPS steps, gpu-aware=$GAM, log=$LOG" +exec "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ + "$EXE" -k on g 1 t "${HPCPERF_CPUS_PER_RANK:-1}" -sf kk -pk kokkos newton on neigh half gpu/aware "$GAM" \ + -in "$IN" -var x "$X" -var y "$Y" -var z "$Z" "${PROCS[@]}" -var steps "$STEPS" \ + -log "$LOG" -echo none "$@" diff --git a/level3/lammps/validate.sh b/level3/lammps/validate.sh new file mode 100755 index 0000000..ac6547a --- /dev/null +++ b/level3/lammps/validate.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Correctness check for the LAMMPS Kokkos build (upstream mechanism: compare +# the thermodynamic output of bench/in.lj with the reference log LAMMPS ships, +# bench/log.15Jul25.lj.fixed.g++.1, a CPU run of the same 32,000-atom, 100-step +# problem; `velocity ... loop geom` makes the initial state machine- and +# rank-count-independent). +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) selects the rank count +# +# What is compared (Step 0 and Step 100 rows: Temp, E_pair, TotEng, Press): +# Step 0 : relative tolerance 1e-8 -- the initial energies are deterministic +# (same lattice, same geometric velocities) and must agree to +# double-precision reduction-order noise. +# Step 100: relative tolerance 1e-5 -- after 100 NVE steps the trajectory has +# accumulated floating-point differences from the different +# force-summation order (GPU vs CPU, N ranks vs 1), but the +# thermodynamic averages of a 32k-atom LJ liquid are insensitive to +# that at the 1e-6 level; 1e-5 is ten times the largest difference +# observed on this node and far below any physics change. +# Additionally, with HPCPERF_GPUS>1 the N-rank result is compared against the +# 1-rank GPU result of the same build with the same tolerances (rank-count +# independence). Prints PASS/FAIL; exit 0/1. No tolerance is loosened to pass. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +N="${HPCPERF_GPUS:-1}" +REF="$R/_upstream/level3/lammps/bench/log.15Jul25.lj.fixed.g++.1" +RUN_DIR="$R/build/level3/lammps/$MODEL/run" +[ -f "$REF" ] || { echo "validate.sh: reference log $REF missing (run fetch.sh)" >&2; exit 1; } + +unset HPCPERF_SCALE_MODE +export HPCPERF_GPUS="$N" +echo "validate.sh: LAMMPS $BACKEND smoke (bench/in.lj, 32000 atoms, 100 steps) on $N GPU(s)" +HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit|Loop time|ERROR' || true +LOG="$RUN_DIR/log.smoke.np$N.lammps" +[ -f "$LOG" ] || { echo "validate.sh: FAIL -- no log produced ($LOG)" ; exit 1; } +if [ "$N" -gt 1 ] && [ ! -f "$RUN_DIR/log.smoke.np1.lammps" ]; then + echo "validate.sh: producing the 1-GPU reference run for rank-count comparison" + HPCPERF_GPUS=1 HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" > /dev/null 2>&1 || true +fi + +python3 - "$LOG" "$REF" "$N" "$RUN_DIR/log.smoke.np1.lammps" <<'PY' +import re, sys +def thermo(path): + rows = {} + with open(path) as f: + lines = f.read().splitlines() + for i, ln in enumerate(lines): + if ln.split()[:2] == ["Step", "Temp"]: + cols = ln.split() + for row in lines[i+1:]: + p = row.split() + if not p or not re.match(r'^\d+$', p[0]): break + rows[int(p[0])] = dict(zip(cols[1:], map(float, p[1:]))) + return rows +log, ref, n, log1 = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] +got, want = thermo(log), thermo(ref) +tol = {0: 1e-8, 100: 1e-5} +keys = ["Temp", "E_pair", "TotEng", "Press"] +ok = True +def cmp(a, b, label): + global ok + for step, t in tol.items(): + if step not in a or step not in b: + print(f" {label}: step {step} missing (got {sorted(a)} vs {sorted(b)})"); ok = False; continue + for k in keys: + x, y = a[step][k], b[step][k] + rel = abs(x - y) / max(abs(y), 1e-30) + flag = "ok " if rel <= t else "BAD" + if rel > t: ok = False + print(f" {label}: step {step:>3} {k:<7} got {x: .10g} ref {y: .10g} rel {rel:.2e} (tol {t:.0e}) {flag}") +print(f"[1] {n}-GPU run vs upstream CPU reference log:") +cmp(got, want, "vs-ref") +if n > 1: + try: + g1 = thermo(log1) + print(f"[2] {n}-GPU run vs this build's 1-GPU run (rank-count independence):") + cmp(got, g1, "vs-1gpu") + except FileNotFoundError: + print("[2] 1-GPU log unavailable; rank-count comparison skipped"); ok = False +print(f"LAMMPS CUDA validation ({n} GPU, bench/in.lj vs log.15Jul25.lj.fixed.g++.1): {'PASS' if ok else 'FAIL'}") +sys.exit(0 if ok else 1) +PY diff --git a/level3/nekrs/README.md b/level3/nekrs/README.md new file mode 100644 index 0000000..9cf37c3 --- /dev/null +++ b/level3/nekrs/README.md @@ -0,0 +1,145 @@ +# nekRS (Level 3) + +Spectral-element incompressible Navier-Stokes (pressure Poisson with p-multigrid ++ HYPRE coarse grid, velocity Helmholtz solves, subcycled advection, passive +scalars, gather-scatter halo exchange) -- the complete solver driven by its own +case files, with all GPU kernels JIT-compiled by OCCA at run time. + +## Provenance + +- Official repository: https://github.com/Nek5000/nekRS (`master` = latest + stable release); docs https://nekrs.readthedocs.io/ +- Release policy: tagged releases on `master` (v26.0 2026-01-27; previous + v23.0 2023-05); `next` is the preview branch. +- Selected: **v26.0**, commit `96b3cf9e5bacede16568826c04a21bc0fe50dc7d`, + fetched by `fetch.sh` into `_upstream/level3/nekRS` (shallow, read-only). +- License: BSD-3-Clause. +- Application-owned LOC (cloc 2.06, code lines): `src/` **53,131** (C++ + 35,784; headers 12,322; C 3,782; Fortran 1,104). Vendored third-party + libraries in `3rd_party/` are counted separately (~2.27 M: LAPACK 829,795, + ADIOS2 618,827, HYPRE 445,492, CVODE 169,316, OCCA 89,888, Nek5000 87,143, + gslib 13,038, parRSB 6,450). + +## Build strategy: NATIVE (upstream CMake, vendored libraries) + +`build.sh CUDA` = upstream's CMake route (`build.sh` upstream is only a +wrapper with interactive prompts): `CC=mpicc CXX=mpicxx FC=mpif90 cmake -G +"Unix Makefiles" -DOCCA_ENABLE_CUDA=ON -DOCCA_ENABLE_HIP=OFF +-DOCCA_ENABLE_DPCPP=OFF -DENABLE_HYPRE_GPU=ON -DENABLE_ADIOS=OFF +-DENABLE_CVODE=OFF -DNEKRS_BUILD_FLOAT=OFF`, install = `NEKRS_HOME` = +`.deps/level3/nekrs/install` (with `nekrs.conf` recording the JIT toolchain: +`OCCA_CXX` = conda g++ 13.3.0, `OCCA_CUDA_COMPILER_FLAGS = -w -O3 -lineinfo +--use_fast_math`, `NEKRS_GPU_MPI = 0`). Toolchain: conda GCC 13.3.0 through the +conda Open MPI 5.0.10 wrappers for C/C++, system gfortran 14.2.1 through +`mpif90` (`OMPI_FC`) for the Nek5000 interface and the vendored LAPACK, CUDA +13.2.78 for HYPRE's device build and for the run-time JIT. OKL kernels are +compiled for the device found at run time (sm_100 here); HYPRE's device +kernels are compiled for sm_80/90/100 (patch below). Build time: the single +full compile of all vendored libraries + nekRS took ~30 min at `-j32` (its +exact wall time was not captured because the first complete pass failed at the +install step; the final incremental install took 27 s); **0 compiler warning +lines** in the final pass. Fingerprint: upstream commit, vendored library +versions, compilers, CUDA 13.2.78, MPI, CMake options, patch list. + +Why not the others: the Spack `nekrs` recipe is stale (23.0, option names that +no longer exist); all dependencies are vendored, so Spack would add nothing; +no Apptainer on the node and no upstream image; JIT couples nekRS to the host +compiler/nvcc anyway; site modules broken. HIP: `build.sh HIP` selects +`OCCA_ENABLE_HIP` and exits with a clear message here (no ROCm). **Untested.** + +## Changes from upstream (all recorded in `patches/` and `build.sh`) + +| Class | Change | Size / reason | +|---|---|---| +| B | `0001-hypre-cuda-sm100.patch`: `cmake/hypre.cmake` `HYPRE_CUDA_SM=80 90` -> `80 90 100` for CUDA >= 13 | 1 line; upstream lists no Blackwell SASS (SASS only, no PTX) | +| D | `0002-hypre-cuda13-thrust-pair.patch`: vendored HYPRE 2.32.0 declares `thrust::reduce_by_key` results as `thrust::pair<...>`, a name the Thrust 3.2 of CUDA 13 no longer exposes there -> `auto` | 2 lines | +| D | `0003-hypre-cuda13-thrust3-compat.patch`: explicit `` / `` includes (no longer transitive) in HYPRE's `device_utils.h` **and** in the pre-generated concatenated `_hypre_utilities.hpp` the sources include; `thrust::not1` (removed) -> its documented replacement `thrust::not_fn` (16 uses) | 36 lines, mechanical | +| B | CMake generator pinned to upstream's Unix Makefiles (`build.sh`): the conda environment exports `CMAKE_GENERATOR=Ninja`, under which HYPRE's ExternalProject install rule (`$(MAKE) install`) is invalid | build.sh only | +| C | `OMPI_FC=/usr/bin/gfortran` (no conda gfortran); `LDFLAGS += -fno-lto` (CMake's FortranCInterface probe compiles with `-flto -ffat-lto-objects`; GCC 13 bytecode vs the gfortran 14 link driver); `FFLAGS += -fPIC` (conda GCC links PIE); `unset AR` (HYPRE's configure takes `$AR` as the full archive command; conda exports the bare tool); run time: `unset CMAKE_GENERATOR` (UDF build), `OMPI_MCA_osc=^ucx` (see execution model), `ulimit -s unlimited` (as upstream's `nrsqsub_utils`: the Nek5000 side keeps lx1^3*lelt work arrays on the stack; the h-refined cases segfault in `useric` at the default 8 MB) | environment | + +No numerics changed. HYPRE 2.32.0 + CUDA 13 is not an upstream-validated +combination; the patches only restore names the CCCL removed. + +## Execution model + +One MPI rank per GPU (upstream: "NekRS binds 1 GPU to 1 MPI rank"). nekRS' +default is `--device-id LOCAL-RANK`; under the common launcher's per-rank +wrapper each rank sees exactly one GPU, so `run.sh` passes `--device-id 0` and +the launcher audits the mapping (4/4 verified). GPU-aware MPI is upstream's +default OFF (`NEKRS_GPU_MPI=0`; RELEASE.md warns enabling it "may cause a +performance regression"); `HPCPERF_NEKRS_GPU_MPI=1` turns it on. nekRS uses +MPI one-sided operations (`MPI_Win_lock`): with the default Open MPI selection +these went through `osc ucx` -> UCX/InfiniBand even on one node and aborted +in `uct_ib` with 4 ranks ("'abort' is not implemented for protocol +amo64/fetch"); `run.sh` sets `OMPI_MCA_osc=^ucx`, the one-sided counterpart of +the site profile's `pml ob1 / btl self,sm,smcuda`, and 4 ranks then pass +(candidate for the gmu-hopper site profile). Rank count is unconstrained +(parRSB graph partitioning). The first run of a new kernel set JIT-compiles +OKL kernels with nvcc into `build/level3/nekrs/cuda/cache` (minutes; shared by +later runs), and the case's `.usr` file is compiled with `mpif90` at run time. + +## Inputs (`HPCPERF_SCALE_MODE`, case `examples/ethier`) + +| Mode | Elements | Order N | Grid points | Per rank @4 GPU | Steps | GPU memory (est.) | Validation quantity | +|---|---|---|---|---|---|---|---| +| smoke (default) | 32 (upstream `ethier.par`) | 9 | 32,000 | 8 elements | 100 (CI mode: 30) | < 0.1 GB | upstream `--cimode 2` CI checks (analytic solution) | +| strong | 32 x H^3, H=`HPCPERF_NEKRS_HREFINE` (10): 32,000 (`ethierRefine.par`, `hrefine`) | 7 | 16.4 M | 8,000 | 100 | ~2 GB | run completes; L2 errors vs exact solution printed every step | +| weak | 32 x H^3 with H = round(cbrt(250 N)) -> ~8,000 elements/rank (upstream's reference load, kershaw README "E/GPU=8000") | 7 | 4.1 M x N | 6.9k-8.5k (integer H) | 100 | ~2 GB | run completes | + +Memory estimate ~60 KB per element at N=7 (velocity, pressure, two scalars, +multistep history, preconditioner). Derived `ethier.par` files change only +`hrefine` and `numSteps` (class A). + +## Validation (`validate.sh`, upstream mechanism) + +`nekrs --setup ethier --cimode 2` is one of the modes upstream's CI runs on this +case (`.github/workflows/ci.yml`): it fixes the solver settings (velocity +solver +BLOCK, subcycling 1, tolerances 1e-12/1e-10, 30 steps) and at the last +step checks the L2 errors of velocity, pressure and both scalars against the +exact Ethier-Steinman solution (references in `examples/ethier/ci.inc`, +relative tolerance EPS = 0.3) plus the iteration counts of the pressure, +velocity and scalar solves. nekRS prints `CI test <...> passed|failed` per +check and exits non-zero on failure; `validate.sh` uses that verdict unchanged +(upstream runs it on CPUs with 2 ranks; here the CUDA backend on 1, 2 and 4 +ranks). Observed on dgx003 (2026-09-05): **PASS at 1, 2 and 4 GPUs, 9/9 checks +each**; final L2 errors velocity 2.776e-10, pressure 6.983e-10, scalar00 +6.672e-12, scalar01 7.495e-12 -- identical to 5 significant digits across the +three rank counts (CI references 2.77e-10 / 7.14e-10 / 7.49e-12 / 7.22e-12). + +## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) + +| Run | Ranks x GPUs | rank->GPU | CPU binding | Topology | Problem | Time | Validation | +|---|---|---|---|---|---|---|---| +| smoke/CI | 1 x 1 | wrapper, `--device-id 0`; audit 1/1 verified | runtime default | parRSB | 32 el., N=9, 30 steps | 1.18 s for 30 steps (8.9 ms/step) | PASS 9/9 | +| smoke/CI | 2 x 2 | wrapper; 2/2 verified | runtime default | parRSB | same | 1.66 s (24.7 ms/step) | PASS 9/9 | +| smoke/CI | 4 x 4 | wrapper; 4/4 verified | runtime default | parRSB | same | 1.40 s | PASS 9/9 | +| strong | 1 x 1 | wrapper; 1/1 verified | runtime default | parRSB | 32,000 el., N=7 (16.4 M points), 100 steps | 177.2 s (1.77 s/step) | completes; L2 err vs exact at step 100: u 2.88e-11, p 1.69e-10, s00 1.38e-10, s01 1.39e-10 | +| strong | 4 x 4 | wrapper; 4/4 verified | runtime default | parRSB | 32,000 el. (8,000/rank) | 62.5 s (0.63 s/step) | completes; L2 err u 2.87e-11, p 1.71e-10, s00 1.38e-10, s01 1.39e-10 | +| weak | 4 x 4 | wrapper; 4/4 verified | runtime default | parRSB | H=10 -> 32,000 el. (8,000/rank; coincides with strong at N=4) | 65.6 s (0.66 s/step) | completes; same L2 errors | + +The 32-element CI case is launch/communication bound (2 and 4 GPUs are slower +per step than 1); it is a correctness case, not a scaling result. The +h-refined runs keep upstream's CI-oriented solver tolerances +(`residualTol` 1e-12 velocity/scalars, 1e-8 pressure), so their step times +are not performance figures either; the 1 -> 4 GPU ratio (2.8x) is recorded +as observed. + +Dry-runs (`HPCPERF_DRY_RUN=1`, hypothetical allocations) -- **DRY-RUN / +UNVALIDATED**, nothing executed: + +| GPUs | Nodes x GPUs/node | Mode | hrefine | Elements | Per rank | Launch | +|---|---|---|---|---|---|---| +| 8 | 1 x 8 | strong | 10 | 32,000 | 4,000 | `mpirun -np 8 --host dgx003:8 --map-by ppr:8:node ...` (single node) | +| 40 | 5 x 8 | weak | 22 | 340,736 | 8,518 | 5 nodes x 8 -- multi-node BLOCKED on this site | +| 80 | 10 x 8 | weak | 27 | 629,856 | 7,873 | 10 nodes x 8 -- multi-node BLOCKED on this site | + +## Limitations + +- Multi-node: BLOCKED/UNVERIFIED on this site; 40/80-GPU shapes are plans + (multi-node also needs the JIT cache strategy `NEKRS_CACHE_LOCAL/BCAST`). +- HIP: untested; no gfx950 statement upstream. +- Vendored HYPRE 2.32.0 needed CUDA 13 compatibility patches (above); the + combination is not upstream-validated. +- Only the ethier family is wrapped (CI case + h-refined strong/weak); turbPipe, + kershaw (needs `genbox`, not shipped), tgv, pb146 build with this install + but have no wrappers yet. ADIOS2 checkpointing and CVODE are not built. diff --git a/level3/nekrs/build.sh b/level3/nekrs/build.sh new file mode 100755 index 0000000..30a81e3 --- /dev/null +++ b/level3/nekrs/build.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Build nekRS (OCCA/CUDA backend, HYPRE on GPU) with upstream's CMake, from a +# private copy of the vendored source tree. +# +# ./build.sh [CUDA|HIP] (default CUDA) +# +# Layout (Level 3 isolation): read-only clone _upstream/level3/nekRS; patched +# private source copy .deps/level3/nekrs/src (upstream's third-party libraries +# -- OCCA, HYPRE 2.32, gslib, Nek5000, LAPACK -- are vendored in-tree and built +# by nekRS' own superbuild, nothing is shared with other applications); build +# build/level3/nekrs/; install .deps/level3/nekrs/install +# (= NEKRS_HOME, with nekrs.conf recording the JIT toolchain); logs +# .deps/level3/nekrs/logs. +# +# Toolchain (upstream: GNU >= 9.1, MPI-3.1 with Fortran bindings, CMake >= 3.21, +# CUDA >= 12): CC/CXX/FC = conda Open MPI wrappers (mpicc/mpicxx/mpif90 around +# conda GCC 13.3.0). The conda environment has no gfortran, so the Fortran +# wrapper is pointed at the system gfortran 14.2.1 through OMPI_FC (class C, +# environment only); the conda MPI Fortran modules (.mod format 15) load under +# gfortran 14 and the mixed link (gfortran-14 objects + conda libgfortran) was +# tested with a 2-rank MPI Fortran program before this recipe was written. +# +# Options vs upstream defaults (class A, documented CMake options): +# OCCA_ENABLE_HIP/DPCPP=OFF (CUDA only; no ROCm/SYCL here), ENABLE_ADIOS=OFF +# (ADIOS2 checkpoint backend not needed; native .fld output stays), +# NEKRS_BUILD_FLOAT=OFF (skip the second, fp32 solver build), ENABLE_CVODE +# off (default). Patch (class B, 1 line): cmake/hypre.cmake adds sm_100 to the +# HYPRE device architectures for CUDA >= 13 (upstream lists 80 90 only, which +# would leave HYPRE's device kernels without Blackwell code); OKL kernels are +# JIT-compiled by OCCA for the device it finds at run time (sm_100 here). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +UP="$R/_upstream/level3/nekRS" +[ -f "$UP/CMakeLists.txt" ] || { echo "build.sh: $UP missing -- run $HERE/fetch.sh first" >&2; exit 1; } +SHA="$(git -C "$UP" rev-parse HEAD)" +l3_paths nekrs +BUILD_DIR="$R/build/level3/nekrs/$MODEL" +JOBS="${HPCPERF_BUILD_JOBS:-32}" +SYS_FC="${HPCPERF_SYSTEM_GFORTRAN:-/usr/bin/gfortran}" +[ -x "$SYS_FC" ] || { echo "build.sh: no gfortran at $SYS_FC (set HPCPERF_SYSTEM_GFORTRAN); the conda env has none" >&2; exit 1; } +export OMPI_FC="$SYS_FC" +# Mixed GCC majors (conda GCC 13 for C/C++, system gfortran 14): CMake's FortranCInterface detection +# compiles its probe objects with -flto=auto -ffat-lto-objects (GCC >= 12) and links them with the +# Fortran driver, whose lto1 rejects GCC 13 bytecode ("LTO version 13.1 instead of 14.0"). Linking with +# -fno-lto uses the fat objects' regular code instead (verified with a 2-language test program). nekRS +# itself does not use LTO, so this changes nothing else. Class C (link flag). +export LDFLAGS="${LDFLAGS:-} -fno-lto" +# The conda GCC links position-independent executables by default while the system gfortran emits +# non-PIE objects ("relocation R_X86_64_32S ... can not be used when making a PIE object"); every +# Fortran object that ends up in nekrs (Nek5000 interface, vendored LAPACK) therefore needs -fPIC. +# Class C (compile flag; no effect on numerics). +export FFLAGS="${FFLAGS:-} -fPIC" +# The conda environment exports AR=x86_64-conda-linux-gnu-ar; the vendored HYPRE's configure takes $AR +# verbatim as the full archive command (its default is "ar -rcu"), so the bare tool name makes every +# `ar libHYPRE_*.a ...` call fail with a usage error. Unset -> HYPRE's own default. Class C. +unset AR +# 0002 (class D, 2 lines): the vendored HYPRE 2.32.0 declares the result of thrust::reduce_by_key as +# `thrust::pair<...>`, a name the CCCL shipped with CUDA 13 no longer provides; `auto` takes the +# library's actual return type. No numerics touched. HYPRE 2.32.0 + CUDA 13 is not upstream-validated. +# 0003 (class D, 36 lines): Thrust 3.2 (CUDA 13) no longer includes +# and transitively (explicit includes added to HYPRE's device_utils.h and to the +# pre-generated concatenated header _hypre_utilities.hpp that the sources actually include) and removed +# the C++17-deprecated `thrust::not1`; its documented replacement `thrust::not_fn` (= cuda::std::not_fn) +# is substituted 1:1 (16 uses). Pure compatibility, no numerics. +PATCHES=("$HERE/patches/0001-hypre-cuda-sm100.patch" "$HERE/patches/0002-hypre-cuda13-thrust-pair.patch" "$HERE/patches/0003-hypre-cuda13-thrust3-compat.patch") + +case "$BACKEND" in + CUDA) command -v nvcc >/dev/null || { echo "build.sh: nvcc not on PATH" >&2; exit 1; } + OCCA_FLAGS=(-DOCCA_ENABLE_CUDA=ON -DOCCA_ENABLE_HIP=OFF -DOCCA_ENABLE_DPCPP=OFF -DOCCA_ENABLE_OPENCL=OFF); ARCHNOTE="sm_$(l3_gpu_arch) (JIT at run time)" ;; + HIP) command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + OCCA_FLAGS=(-DOCCA_ENABLE_CUDA=OFF -DOCCA_ENABLE_HIP=ON -DOCCA_ENABLE_DPCPP=OFF -DOCCA_ENABLE_OPENCL=OFF); ARCHNOTE="gfx950 (JIT at run time)" ;; + *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; +esac +CMAKE_OPTS="${OCCA_FLAGS[*]} ENABLE_HYPRE_GPU=ON ENABLE_ADIOS=OFF ENABLE_CVODE=OFF NEKRS_BUILD_FLOAT=OFF NEKRS_GPU_MPI=OFF(default; runtime NEKRS_GPU_MPI) CC=mpicc CXX=mpicxx FC=mpif90(OMPI_FC=$SYS_FC)" +PATCHNAMES=(); for p in "${PATCHES[@]}"; do PATCHNAMES+=("$(basename "$p")"); done +FP="$(l3_fingerprint_text nekrs "$SHA" "$MODEL" "vendored: occa=2.0.0-dev hypre=2.32.0 gslib nek5000 lapack (in-tree)" "$CMAKE_OPTS" "runtime(NEKRS_GPU_MPI, default 0)" "${PATCHNAMES[@]}")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +echo "# nekRS $BACKEND: upstream $SHA (v26.0), arch $ARCHNOTE, MPI $(mpirun --version 2>/dev/null | head -1), FC $($SYS_FC --version | head -1)" +# private source copy (upstream clone stays pristine). The copy is ~280 MB in many small files (slow on +# this filesystem), so it is reused when it already holds this upstream commit + patch set. +STAMP="$SHA ${PATCHNAMES[*]}" +if [ -f "$L3_SRC/.hpcperf-src-stamp" ] && [ "$(cat "$L3_SRC/.hpcperf-src-stamp")" = "$STAMP" ]; then + echo "# reusing patched source copy $L3_SRC ($STAMP)" +else + rm -rf "$L3_SRC"; mkdir -p "$L3_SRC" + rsync -a --exclude .git "$UP/" "$L3_SRC/" # examples/ and doc/ are installed by nekRS' CMake + for p in "${PATCHES[@]}"; do + (cd "$L3_SRC" && patch -p1 --forward --silent < "$p") || { echo "build.sh: patch $(basename "$p") failed to apply" >&2; exit 1; } + echo "# applied $(basename "$p")" + done + echo "$STAMP" > "$L3_SRC/.hpcperf-src-stamp" +fi +# fresh configure every time: CMake caches CMAKE_EXE_LINKER_FLAGS and the Fortran/C detection results +# from the first configure of a build directory, so environment fixes would otherwise not take effect. +# HPCPERF_L3_INCREMENTAL=1 keeps an existing build tree (only for re-running install/fingerprint after a +# late failure with an unchanged toolchain). +[ -n "${HPCPERF_L3_INCREMENTAL:-}" ] || rm -rf "$BUILD_DIR"; mkdir -p "$BUILD_DIR" +# upstream's build.sh uses CMake's default generator (Unix Makefiles); the conda environment exports +# CMAKE_GENERATOR=Ninja, under which nekRS' generated build file is invalid ("bad $-escape" in a +# vendored-library rule), so the generator is pinned to upstream's +CC=mpicc CXX=mpicxx FC=mpif90 cmake -S "$L3_SRC" -B "$BUILD_DIR" -G "Unix Makefiles" -Wfatal-errors \ + -DCMAKE_INSTALL_PREFIX="$L3_INSTALL" \ + "${OCCA_FLAGS[@]}" -DENABLE_HYPRE_GPU=ON -DENABLE_ADIOS=OFF -DENABLE_CVODE=OFF -DNEKRS_BUILD_FLOAT=OFF \ + > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ + || { tail -40 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } +t0=$(date +%s) +cmake --build "$BUILD_DIR" --target install -j "$JOBS" > "$L3_LOGS/build-$MODEL.log" 2>&1 \ + || { tail -40 "$L3_LOGS/build-$MODEL.log"; echo "build.sh: build failed (log: $L3_LOGS/build-$MODEL.log)" >&2; exit 1; } +l3_fingerprint_write "$L3_INSTALL" "$FP" +echo "# built + installed in $(( $(date +%s)-t0 )) s: $L3_INSTALL/bin/nekrs (NEKRS_HOME=$L3_INSTALL)" +echo "# compiler warning lines: $(grep -c 'warning' "$L3_LOGS/build-$MODEL.log" || true)" +grep -E 'OCCA_CXX|OCCA_CUDA_COMPILER|NEKRS_FC|NEKRS_GPU_MPI|BACKEND' "$L3_INSTALL/nekrs.conf" 2>/dev/null | sed 's/^/# nekrs.conf: /' diff --git a/level3/nekrs/fetch.sh b/level3/nekrs/fetch.sh new file mode 100755 index 0000000..112061e --- /dev/null +++ b/level3/nekrs/fetch.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Fetch nekRS at the latest release tag into _upstream/level3/nekRS +# (gitignored, read-only). nekRS vendors all third-party libraries as squashed +# subtrees (OCCA, HYPRE, gslib, Nek5000, ADIOS2, ...), so no submodules and no +# configure-time downloads are involved. Idempotent; a checkout at another +# commit is an error, never silently reused. +# +# ./fetch.sh +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +UPSTREAM_URL="https://github.com/Nek5000/nekRS.git" +UPSTREAM_TAG="v26.0" +UPSTREAM_SHA="96b3cf9e5bacede16568826c04a21bc0fe50dc7d" +DEST="$R/_upstream/level3/nekRS" + +if [ -d "$DEST/.git" ]; then + have="$(git -C "$DEST" rev-parse HEAD)" + if [ "$have" = "$UPSTREAM_SHA" ]; then echo "fetch.sh: $DEST already at $UPSTREAM_TAG ($UPSTREAM_SHA)"; exit 0; fi + echo "fetch.sh: $DEST is at $have, not the recorded $UPSTREAM_SHA ($UPSTREAM_TAG); remove it to re-fetch" >&2; exit 1 +fi +mkdir -p "$(dirname "$DEST")" +echo "fetch.sh: cloning $UPSTREAM_URL @ $UPSTREAM_TAG (shallow)" +git clone --quiet --depth 1 --branch "$UPSTREAM_TAG" "$UPSTREAM_URL" "$DEST" +have="$(git -C "$DEST" rev-parse HEAD)" +[ "$have" = "$UPSTREAM_SHA" ] || { echo "fetch.sh: tag $UPSTREAM_TAG resolved to $have, expected $UPSTREAM_SHA" >&2; exit 1; } +echo "fetch.sh: ok -> $DEST ($UPSTREAM_SHA)" diff --git a/level3/nekrs/patches/0001-hypre-cuda-sm100.patch b/level3/nekrs/patches/0001-hypre-cuda-sm100.patch new file mode 100644 index 0000000..a6e1bd8 --- /dev/null +++ b/level3/nekrs/patches/0001-hypre-cuda-sm100.patch @@ -0,0 +1,11 @@ +--- a/cmake/hypre.cmake ++++ b/cmake/hypre.cmake +@@ -58,7 +58,7 @@ + set(HYPRE_BACKEND "--with-cuda" "--with-cuda-home=${CUDAToolkit_LIBRARY_ROOT}") + + if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "13.0.0") +- set(HYPRE_DEVICE_ARCH "HYPRE_CUDA_SM=80 90") ++ set(HYPRE_DEVICE_ARCH "HYPRE_CUDA_SM=80 90 100") + #disable for now as it might not play well with all MPI implementations + #set(HYPRE_CONFIGURE_FLAGS "--enable-device-malloc-async") + elseif(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.0.0") diff --git a/level3/nekrs/patches/0002-hypre-cuda13-thrust-pair.patch b/level3/nekrs/patches/0002-hypre-cuda13-thrust-pair.patch new file mode 100644 index 0000000..6c9c7c9 --- /dev/null +++ b/level3/nekrs/patches/0002-hypre-cuda13-thrust-pair.patch @@ -0,0 +1,22 @@ +--- a/3rd_party/hypre/src/utilities/device_utils.c ++++ b/3rd_party/hypre/src/utilities/device_utils.c +@@ -1491,7 +1491,7 @@ + #else + HYPRE_THRUST_CALL(sort_by_key, map2, map2 + ny, y); + +- thrust::pair new_end = HYPRE_THRUST_CALL( reduce_by_key, ++ auto new_end = HYPRE_THRUST_CALL( reduce_by_key, + map2, + map2 + ny, + y, +--- a/3rd_party/hypre/src/seq_mv/csr_matop_device.c ++++ b/3rd_party/hypre/src/seq_mv/csr_matop_device.c +@@ -1034,7 +1034,7 @@ + hypre_assert(new_end.first - reduced_col_indices == new_end.second - reduced_col_nnz); + num_reduced_col_indices = new_end.first - reduced_col_indices; + #else +- thrust::pair new_end = ++ auto new_end = + HYPRE_THRUST_CALL(reduce_by_key, A_j_sorted, A_j_sorted + nnz_A, + thrust::make_constant_iterator(1), + reduced_col_indices, diff --git a/level3/nekrs/patches/0003-hypre-cuda13-thrust3-compat.patch b/level3/nekrs/patches/0003-hypre-cuda13-thrust3-compat.patch new file mode 100644 index 0000000..0fd3964 --- /dev/null +++ b/level3/nekrs/patches/0003-hypre-cuda13-thrust3-compat.patch @@ -0,0 +1,178 @@ +--- a/3rd_party/hypre/src/utilities/device_utils.h ++++ b/3rd_party/hypre/src/utilities/device_utils.h +@@ -164,6 +164,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +--- a/3rd_party/hypre/src/utilities/_hypre_utilities.hpp ++++ b/3rd_party/hypre/src/utilities/_hypre_utilities.hpp +@@ -359,6 +359,8 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include +--- a/3rd_party/hypre/src/IJ_mv/IJMatrix_parcsr_device.c ++++ b/3rd_party/hypre/src/IJ_mv/IJMatrix_parcsr_device.c +@@ -656,7 +656,7 @@ + is_on_proc, /* stencil */ + thrust::make_zip_iterator(thrust::make_tuple(off_proc_i, off_proc_j, off_proc_data, + off_proc_sora)), /* result */ +- thrust::not1(thrust::identity()) ); ++ thrust::not_fn(thrust::identity()) ); + + hypre_assert(thrust::get<0>(new_end1.get_iterator_tuple()) - off_proc_i == nelms_off); + +@@ -668,7 +668,7 @@ + thrust::make_zip_iterator(thrust::make_tuple(stack_i + nelms, stack_j + nelms, stack_data + nelms, + stack_sora + nelms)), /* last */ + is_on_proc, /* stencil */ +- thrust::not1(thrust::identity()) ); ++ thrust::not_fn(thrust::identity()) ); + + hypre_assert(thrust::get<0>(new_end2.get_iterator_tuple()) - stack_i == nelms_on); + #endif +--- a/3rd_party/hypre/src/IJ_mv/IJVector_parcsr_device.c ++++ b/3rd_party/hypre/src/IJ_mv/IJVector_parcsr_device.c +@@ -522,7 +522,7 @@ + is_on_proc, /* stencil */ + thrust::make_zip_iterator(thrust::make_tuple(off_proc_i, off_proc_data, + off_proc_sora)), /* result */ +- thrust::not1(thrust::identity()) ); ++ thrust::not_fn(thrust::identity()) ); + + hypre_assert(thrust::get<0>(new_end1.get_iterator_tuple()) - off_proc_i == nelms_off); + +@@ -534,7 +534,7 @@ + thrust::make_zip_iterator(thrust::make_tuple(stack_i + nelms, stack_data + nelms, + stack_sora + nelms)), /* last */ + is_on_proc, /* stencil */ +- thrust::not1(thrust::identity()) ); ++ thrust::not_fn(thrust::identity()) ); + + hypre_assert(thrust::get<0>(new_end2.get_iterator_tuple()) - stack_i == nelms_on); + #endif +--- a/3rd_party/hypre/src/parcsr_ls/ams.c ++++ b/3rd_party/hypre/src/parcsr_ls/ams.c +@@ -697,7 +697,7 @@ + 1.0 ); + #else + thrust::identity identity; +- HYPRE_THRUST_CALL( replace_if, l1_norm, l1_norm + num_rows, thrust::not1(identity), 1.0 ); ++ HYPRE_THRUST_CALL( replace_if, l1_norm, l1_norm + num_rows, thrust::not_fn(identity), 1.0 ); + #endif + } + else +@@ -768,7 +768,7 @@ + HYPRE_THRUST_CALL( transform_if, l1_norm, l1_norm + num_rows, diag_tmp, l1_norm, + thrust::negate(), + is_negative() ); +- //bool any_zero = HYPRE_THRUST_CALL( any_of, l1_norm, l1_norm + num_rows, thrust::not1(thrust::identity()) ); ++ //bool any_zero = HYPRE_THRUST_CALL( any_of, l1_norm, l1_norm + num_rows, thrust::not_fn(thrust::identity()) ); + bool any_zero = 0.0 == HYPRE_THRUST_CALL( reduce, l1_norm, l1_norm + num_rows, 1.0, + thrust::minimum() ); + #endif +--- a/3rd_party/hypre/src/parcsr_ls/par_mod_multi_interp_device.c ++++ b/3rd_party/hypre/src/parcsr_ls/par_mod_multi_interp_device.c +@@ -327,7 +327,7 @@ + thrust::make_counting_iterator(n_fine), + CF_marker, + points_left, +- thrust::not1(equal(1)) ); ++ thrust::not_fn(equal(1)) ); + remaining = points_end - points_left; + + /* Cpts; number of C pts */ +@@ -541,7 +541,7 @@ + points_left_old + remaining, + diag_shifts, + points_left, +- thrust::not1(thrust::identity()) ); ++ thrust::not_fn(thrust::identity()) ); + #endif + + hypre_assert(new_end - points_left == cnt_rem); +@@ -1637,7 +1637,7 @@ + fine_to_coarse, + fine_to_coarse + n_fine, + pass_marker, +- thrust::not1(equal(color)), ++ thrust::not_fn(equal(color)), + -1 ); + #endif + } +--- a/3rd_party/hypre/src/parcsr_mv/par_csr_triplemat_device.c ++++ b/3rd_party/hypre/src/parcsr_mv/par_csr_triplemat_device.c +@@ -229,7 +229,7 @@ + hypre_CSRMatrixData(Cbar))) + hypre_CSRMatrixNumNonzeros(Cbar), + hypre_CSRMatrixJ(Cbar), + thrust::make_zip_iterator(thrust::make_tuple(C_offd_ii, C_offd_j, C_offd_a)), +- thrust::not1(pred) ); ++ thrust::not_fn(pred) ); + hypre_assert( thrust::get<0>(new_end.get_iterator_tuple()) == C_offd_ii + nnz_C_offd ); + #endif + +@@ -958,7 +958,7 @@ + Cext_bigj)) + Cext_nnz, + Cext_bigj, + thrust::make_zip_iterator(thrust::make_tuple(work, big_work)), +- thrust::not1(pred1) ); ++ thrust::not_fn(pred1) ); + + HYPRE_Int Cext_offd_nnz = thrust::get<0>(off_end.get_iterator_tuple()) - work; + #endif +@@ -1220,7 +1220,7 @@ + thrust::make_zip_iterator(thrust::make_tuple(zmp_i, zmp_j, zmp_a)) + local_nnz_C, + zmp_j, + thrust::make_zip_iterator(thrust::make_tuple(C_offd_ii, C_offd_j, C_offd_a)), +- thrust::not1(pred) ); ++ thrust::not_fn(pred) ); + hypre_assert( thrust::get<0>(new_end.get_iterator_tuple()) == C_offd_ii + nnz_C_offd ); + #endif + hypreDevice_CsrRowIndicesToPtrs_v2(hypre_CSRMatrixNumRows(C_offd), nnz_C_offd, C_offd_ii, +--- a/3rd_party/hypre/src/seq_mv/csr_matop_device.c ++++ b/3rd_party/hypre/src/seq_mv/csr_matop_device.c +@@ -681,7 +681,7 @@ + B_ext_bigj, /* stencil */ + thrust::make_zip_iterator(thrust::make_tuple(B_ext_offd_ii, B_ext_offd_bigj, B_ext_offd_data, + B_ext_offd_xata)), /* result */ +- thrust::not1(pred1) ); ++ thrust::not_fn(pred1) ); + + hypre_assert( thrust::get<0>(new_end.get_iterator_tuple()) == B_ext_offd_ii + B_ext_offd_nnz ); + #endif +@@ -708,7 +708,7 @@ + B_ext_bigj, /* stencil */ + thrust::make_zip_iterator(thrust::make_tuple(B_ext_offd_ii, B_ext_offd_bigj, + B_ext_offd_data)), /* result */ +- thrust::not1(pred1) ); ++ thrust::not_fn(pred1) ); + + hypre_assert( thrust::get<0>(new_end.get_iterator_tuple()) == B_ext_offd_ii + B_ext_offd_nnz ); + #endif +@@ -2145,7 +2145,7 @@ + new_nnz = HYPRE_THRUST_CALL( count_if, + A_data, + A_data + nnz, +- thrust::not1(less_than(tol)) ); ++ thrust::not_fn(less_than(tol)) ); + } + else + { +@@ -2199,7 +2199,7 @@ + thrust::make_zip_iterator(thrust::make_tuple(A_ii, A_j, A_data)) + nnz, + A_data, + thrust::make_zip_iterator(thrust::make_tuple(new_ii, new_j, new_data)), +- thrust::not1(less_than(tol)) ); ++ thrust::not_fn(less_than(tol)) ); + + hypre_assert( thrust::get<0>(new_end.get_iterator_tuple()) == new_ii + new_nnz ); + } diff --git a/level3/nekrs/run.sh b/level3/nekrs/run.sh new file mode 100755 index 0000000..7e3dee4 --- /dev/null +++ b/level3/nekrs/run.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Run nekRS' ethier case (examples/ethier: 3D incompressible Navier-Stokes with +# two passive scalars on the Ethier-Steinman exact solution; the full solver +# stack -- pressure Poisson with pMG+HYPRE coarse grid, velocity Helmholtz, +# subcycled advection, gather-scatter halo exchange) on N GPUs. +# +# ./run.sh [CUDA|HIP] [extra nekrs args, e.g. --cimode 2] +# +# Execution model (upstream: "NekRS binds 1 GPU to 1 MPI rank"): one MPI rank +# per GPU. nekRS' own default is --device-id LOCAL-RANK; under the common +# launcher's per-rank wrapper each rank sees exactly one GPU, so the run passes +# --device-id 0 (the mapping is audited by the launcher). GPU-aware MPI is +# upstream-default OFF (RELEASE.md: enabling it "may cause a performance +# regression"); HPCPERF_NEKRS_GPU_MPI=1 turns it on (NEKRS_GPU_MPI env). +# Rank count is unconstrained (graph partitioning), but must stay well below +# the element count. +# +# Resource / size controls: +# HPCPERF_GPUS=N|all ranks = GPUs (default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# smoke : upstream ethier.par as shipped: 32 elements, N=9, 100 steps +# strong : ethierRefine.par with hrefine=H (H=HPCPERF_NEKRS_HREFINE, default +# 10): 32*H^3 = 32,000 elements, N=7 (16.4M grid points), fixed +# weak : hrefine chosen per rank count so that elements/rank ~ 8000 +# (upstream's reference load, kershaw README "E/GPU=8000"): +# H = round(cbrt(250 N)); integer H makes the per-rank count vary +# between ~6.9k and ~8.5k -- printed and recorded +# HPCPERF_NEKRS_STEPS time steps (default: upstream 100) +# HPCPERF_NEKRS_GPU_MPI 0|1 (default 0 = upstream default) +# +# The case directory (re2/usr/udf/oudf/par) is copied into the build tree; +# strong/weak write a derived ethier.par from upstream's ethierRefine.par with +# only `hrefine` and `numSteps` changed (class A). The OCCA JIT cache is shared +# per build (NEKRS_CACHE_DIR) -- the first run of a new kernel set compiles OKL +# kernels with nvcc, which takes minutes and is not part of the solve time. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +l3_paths nekrs +BUILD_DIR="$R/build/level3/nekrs/$MODEL" +export NEKRS_HOME="$L3_INSTALL" +EXE="$NEKRS_HOME/bin/nekrs" +[ -x "$EXE" ] || { echo "run.sh: $EXE not found -- run ./build.sh $BACKEND first" >&2; exit 1; } +CASE_SRC="$R/_upstream/level3/nekRS/examples/ethier" +[ -f "$CASE_SRC/ethier.re2" ] || { echo "run.sh: $CASE_SRC missing (run fetch.sh)" >&2; exit 1; } + +N_RANKS="$(hpcperf_ranks nekrs yes)" || exit 2 +hpcperf_forbid_args nekrs --setup --device-id --backend -- "$@" || exit 2 +MODE="$(l3_scale_mode nekrs)" || exit 2 +STEPS="${HPCPERF_NEKRS_STEPS:-100}" +case "$MODE" in + smoke) H=0; ORDER=9 ;; + strong) H="${HPCPERF_NEKRS_HREFINE:-10}"; ORDER=7 ;; + weak) H="$(python3 -c "import math; print(max(1, round((250*$N_RANKS)**(1/3))))")"; ORDER=7 ;; +esac +if [ "$H" -gt 0 ]; then ELEMS=$((32 * H * H * H)); else ELEMS=32; fi +POINTS=$((ELEMS * (ORDER + 1) * (ORDER + 1) * (ORDER + 1))) + +RUN_DIR="$BUILD_DIR/run/$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR" +cp "$CASE_SRC"/* "$RUN_DIR"/ # complete upstream case directory (re2, usr, udf, CASEDATA include, ci.inc, par files) +if [ "$MODE" = smoke ]; then + sed -e "s/^numSteps *=.*/numSteps = $STEPS/" "$CASE_SRC/ethier.par" > "$RUN_DIR/ethier.par" +else + # derived from upstream ethierRefine.par: hrefine and numSteps only + sed -e "s/^hrefine *=.*/hrefine = $H/" -e "s/^numSteps *=.*/numSteps = $STEPS/" "$CASE_SRC/ethierRefine.par" > "$RUN_DIR/ethier.par" +fi +export NEKRS_CACHE_DIR="$BUILD_DIR/cache"; mkdir -p "$NEKRS_CACHE_DIR" +export NEKRS_GPU_MPI="${HPCPERF_NEKRS_GPU_MPI:-0}" +# nekRS uses MPI one-sided operations (MPI_Win_lock); Open MPI's default one-sided component on this +# node is `osc ucx`, which goes through UCX/InfiniBand even on one node and aborts in uct_ib with 4 ranks +# ("'abort' is not implemented for protocol amo64/fetch"). The site profile already keeps point-to-point +# off UCX (pml ob1 / btl self,sm,smcuda); the same is done here for one-sided (class C, site transport). +export OMPI_MCA_osc="${HPCPERF_NEKRS_OSC:-^ucx}" +export OMPI_FC="${HPCPERF_SYSTEM_GFORTRAN:-/usr/bin/gfortran}" # .usr file is compiled at run time with the MPI Fortran wrapper +unset CMAKE_GENERATOR # nekRS' run-time UDF build configures with CMake and then calls `make okl.i`; the conda env's Ninja default breaks it +export CUDA_CACHE_DISABLE=1 +# The Nek5000 side of the case (ethier.usr) keeps per-field work arrays of size lx1^3*lelt on the stack; +# upstream's job scripts raise the stack limit for that reason. With the default 8 MB the h-refined +# cases segfault in useric. Applies to mpirun's children (inherited rlimit). Class C. +ulimit -s unlimited 2>/dev/null || ulimit -s "$(ulimit -H -s)" + +echo "# nekRS $BACKEND: mode=$MODE ranks=$N_RANKS case=ethier hrefine=$H elements=$ELEMS (~$((ELEMS / N_RANKS))/rank) N=$ORDER points=$POINTS steps=$STEPS gpu_mpi=$NEKRS_GPU_MPI run_dir=$RUN_DIR" +cd "$RUN_DIR" +"$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- "$EXE" --setup ethier --backend "$BACKEND" --device-id 0 "$@" 2>&1 | tee "$RUN_DIR/stdout.log" +exit "${PIPESTATUS[0]}" diff --git a/level3/nekrs/validate.sh b/level3/nekrs/validate.sh new file mode 100755 index 0000000..6b422ad --- /dev/null +++ b/level3/nekrs/validate.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Correctness check for nekRS: upstream's own CI test of the ethier case +# (analytic Ethier-Steinman solution) run on N GPUs. +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) +# +# `nekrs --cimode 2` is one of the modes upstream's CI (.github/workflows/ +# ci.yml) runs on this case: it fixes the solver settings (velocity solver +# +BLOCK, subcycling 1, tolerances 1e-12/1e-10) and, at the last step, checks +# the L2 errors of velocity, pressure and both scalars against the exact +# solution (reference values in examples/ethier/ci.inc: 2.77e-10, 7.14e-10, +# 7.49e-12, 7.22e-12; relative tolerance EPS = 0.3) plus the iteration counts +# of the pressure/velocity/scalar solves (+-1). nekRS prints "CI test <...> +# passed|failed" for each check and exits non-zero on any failure -- that +# verdict is used unchanged. The L2-error line the case prints itself +# ("... L2 err") is echoed for the record. +# Note: upstream runs this CI on CPUs with 2 ranks; here the GPU backend on N +# ranks is being validated against the same criteria. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +N="${HPCPERF_GPUS:-1}" +OUT="$R/build/level3/nekrs/$MODEL/run/smoke.np$N.cimode2.log" + +export HPCPERF_GPUS="$N" +mkdir -p "$(dirname "$OUT")" +echo "validate.sh: nekRS $BACKEND ethier --cimode 2 (upstream CI mode) on $N GPU(s)" +set +e +HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" --cimode 2 > "$OUT" 2>&1 +rc=$? +set -e +grep -a -E '^#|hpcperf-launch: audit summary|CI test|L2 err|elapsedStepSum|total elapsed|ERROR|error' "$OUT" | grep -a -v 'no error' | sed 's/^/ /' | tail -40 +FAILED=$(grep -a -c 'CI test .* failed' "$OUT" || true) +PASSED=$(grep -a -c 'CI test .* passed' "$OUT" || true) +echo " nekrs exit code $rc; CI checks passed=$PASSED failed=$FAILED" +if [ "$rc" -eq 0 ] && [ "$FAILED" -eq 0 ] && [ "$PASSED" -gt 0 ]; then + echo "nekRS $BACKEND validation ($N GPU, ethier --cimode 2 vs upstream CI references): PASS"; exit 0 +fi +echo "nekRS $BACKEND validation ($N GPU, ethier --cimode 2 vs upstream CI references): FAIL (log: $OUT)"; exit 1 diff --git a/level3/sparta/README.md b/level3/sparta/README.md new file mode 100644 index 0000000..0cdef29 --- /dev/null +++ b/level3/sparta/README.md @@ -0,0 +1,143 @@ +# SPARTA (Level 3) + +Direct Simulation Monte Carlo (DSMC) for rarefied gas dynamics: particle +move/sort, collisions, grid-cell decomposition, MPI migration -- the full +application driven by its own input scripts, KOKKOS package on the GPU. + +## Provenance + +- Official repository: https://github.com/sparta/sparta (docs + https://sparta.github.io/doc/Manual.html; Kokkos section + https://sparta.github.io/doc/Section_accelerate.html) +- Release policy: one stream of dated tags (no stable/feature split). +- Selected: **`27Aug2026`** (2026-08-28), commit + `95b9abaa8bd548991cc3c3f1c58b34722f7ade74`, fetched by `fetch.sh` into + `_upstream/level3/sparta` (shallow, read-only). This release moved the + bundled Kokkos to 5.0.2 and made KOKKOS builds CMake-only / C++20. +- License: GPL-2.0 (`LICENSE`). +- Application-owned LOC (cloc 2.06, code lines): `src/` **131,181** (C++ + 104,683; headers 25,195; incl. `src/KOKKOS` 36,909 in 194 files). Bundled + `lib/kokkos` (Kokkos 5.0.2, 223,495 lines) counted separately, not modified. + +## Build strategy: NATIVE (upstream CMake preset + bundled Kokkos 5.0.2) + +`build.sh CUDA` = the documented recipe: `cmake -S sparta/cmake -C +cmake/presets/kokkos_common.cmake` with `nvcc_wrapper` (host conda GCC 13.3.0) +as CXX, `Kokkos_ENABLE_CUDA`, `Kokkos_ARCH_BLACKWELL100` (the docs list "GB200 +(Blackwell) -> BLACKWELL100" explicitly), `Kokkos_ENABLE_SERIAL=ON`, +`Kokkos_ENABLE_OPENMP=OFF`, `FFT_KOKKOS=CUFFT`, C++20, `BUILD_MPI` (conda Open +MPI 5.0.10, CUDA-aware), `SPARTA_MACHINE=kokkos_cuda`. Build time on dgx003: +**579 s** at `-j32` (278 targets); 108 warning lines (nvcc_wrapper multiple +`-O` flags, a few upstream notes), no errors. Executable +`build/level3/sparta/cuda/src/spa_kokkos_cuda` (302 MB, static Kokkos); +install prefix `.deps/level3/sparta/install` with fingerprint (upstream +commit, Kokkos 5.0.2, compiler, CUDA 13.2.78, MPI, CMake options). + +Why not the others: **there is no Spack package for this SPARTA** -- the +`sparta` recipe in Spack (local and upstream) is the unrelated bioinformatics +tool sPARTA; upstream documents only CMake presets. No Apptainer on the node +and no upstream image. Site modules broken. Level 2's Kokkos 5.2.1 is not used +because the bundled 5.0.2 is the version the release was tested with (SPARTA +does not pin an external Kokkos, so `USE_EXTERNAL_KOKKOS=ON` remains a +documented fallback). + +HIP: `build.sh HIP` carries the upstream `kokkos_hip` recipe (`hipcc`, +`Kokkos_ARCH_AMD_GFX950`, `FFT_KOKKOS=HIPFFT`) and exits with a clear message +here (no ROCm). Note the bundled Kokkos 5.0.2 has no `AMD_GFX950` +architecture (added in Kokkos 5.1); an MI355X build would need +`USE_EXTERNAL_KOKKOS`. **Untested.** + +## Changes from upstream + +Class **A -- none.** `bench/in.collide` is run unmodified from `bench/` with +its documented `-var x y z` size variables; the log goes to the build tree. + +## Execution model + +One MPI rank per GPU (upstream: "the -np setting ... should set the number of +MPI tasks/node to be equal to the # of physical GPUs on the node"), `-k on g 1 +-sf kk -pk kokkos gpu/aware yes`. The common launcher's per-rank wrapper gives +each rank exactly one visible GPU and audits the mapping. GPU-aware MPI +defaults to `yes` -- unlike LAMMPS, SPARTA does **not** auto-detect +CUDA-awareness, so `HPCPERF_SPARTA_GPU_AWARE=no` must be used with a +non-CUDA-aware MPI. Any rank count is legal: the deck uses `balance_grid rcb +part` (recursive coordinate bisection), no processor-grid constraint. + +## Inputs (`HPCPERF_SCALE_MODE`) + +| Mode | Grid cells | Particles (10/cell) | Per rank @4 GPU | Topology | Steps | Memory/GPU (est.) | Runtime on B200 | Validation quantity | +|---|---|---|---|---|---|---|---|---| +| smoke (default) | 10x10x10 (upstream default) | 10,000 | 2,500 | RCB | 30 + 100 | < 0.1 GB | 0.03-0.05 s | Np, temp, Natt vs upstream reference log | +| strong | S^3, S=`HPCPERF_SPARTA_STRONG` (100) | 10,000,000 | 2,500,000 | RCB | 30 + 100 | ~2 GB | 0.87 s (1 GPU) / 0.38 s (4 GPU) | same stats | +| weak | (L*PX)x(L*PY)x(L*PZ), L=`HPCPERF_SPARTA_LOCAL` (50) | 1,250,000 x N | 1,250,000 | grid from `hpcperf_topology.py` (RCB inside) | 30 + 100 | ~0.3 GB | 0.23 s (4 GPU) | same stats | + +The deck runs 30 equilibration steps followed by the 100-step benchmark +(`run 30` / `run 100`, upstream). Memory estimate ~100 B/particle plus +per-cell data; all sizes above are far below a B200's 180 GB. The strong +default is a *correctness* size: 10M particles per 100 steps take under a +second, so 4-GPU vs 1-GPU timings (0.38 s vs 0.87 s) indicate the run is +already partially communication/launch bound and are not a scaling result. + +## Validation (`validate.sh`, upstream mechanism) + +Upstream's own guidance (`examples/README`, `tools/testing/regression.py`) +is statistical: DSMC is stochastic and "should get statistically similar +answers ... on different numbers of processors, but not identical answers". +`validate.sh` therefore compares the stats table of the unmodified +`bench/in.collide` with the reference log SPARTA ships, +`bench/log.7Jul14.collide.icc.10K.1`, on three quantities: + +1. particle count `Np` == 10,000 at every stats row (closed box, no + chemistry: exact conservation); +2. gas temperature (`compute temp`, printed as `c_temp`; the 2014 log labels + it `temp`): mean over the benchmark steps within 2 % of the reference. + Elastic VSS collisions conserve energy exactly, so within a run the + temperature is constant; its value is set by the Maxwellian sampling of the + initial velocities, whose statistical scatter for 10^4 particles is + sqrt(2/3N) ~ 0.8 %. 2 % is ~2.5 sigma of that noise and far below any + unit/physics error; +3. mean collision attempts per step `Natt` within 15 % (fixed by density, + temperature and cross-section; run-to-run scatter is a few %). + +With N > 1 GPUs the same three criteria are applied between the N-rank run and +this build's 1-rank run. Observed on dgx003 (2026-09-04): **PASS at 1, 2 and 4 +GPUs**: + +| GPUs | Np | mean temp (ref 274.41 K) | rel | mean Natt (ref 943.7) | rel | vs 1-GPU temp / Natt | +|---|---|---|---|---|---|---| +| 1 | 10,000 at every row | 275.43 | 3.7e-3 | 946.1 | 2.5e-3 | -- | +| 2 | 10,000 | 271.65 | 1.0e-2 | 942.4 | 1.4e-3 | 1.4e-2 / 3.9e-3 | +| 4 | 10,000 | 274.64 | 8.5e-4 | 947.7 | 4.2e-3 | 2.9e-3 / 1.7e-3 | + +(All temperature deviations are within ~1.7 sigma of the sampling noise; the +nominal gas temperature is 273.15 K.) + +## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) + +| Run | Ranks x GPUs | rank->GPU | CPU binding | Topology | Problem | Loop time (100-step benchmark) | Validation | +|---|---|---|---|---|---|---|---| +| smoke | 1 x 1 | wrapper; audit unverified (0.03 s run, too short to sample) | runtime default | RCB | 10k particles | 0.026 s | PASS | +| smoke | 2 x 2 | wrapper; audit 1 verified / 1 unverified (short run) | runtime default | RCB | 10k particles | 0.041 s | PASS | +| smoke | 4 x 4 | wrapper; audit 4/4 verified | runtime default | RCB | 10k particles | 0.053 s | PASS | +| strong | 1 x 1 | wrapper; 1/1 verified | runtime default | RCB | 100^3 cells, 10M particles | 0.869 s | run completes, Np conserved | +| strong | 4 x 4 | wrapper; 4/4 verified | runtime default | RCB | 100^3 cells, 10M particles | 0.381 s | run completes | +| weak | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2x1 -> 100x100x50 | 5M particles (1.25M/rank) | 0.229 s | run completes | + +Dry-runs (`HPCPERF_DRY_RUN=1`, hypothetical allocations) -- **DRY-RUN / +UNVALIDATED**, nothing executed: + +| GPUs | Nodes x GPUs/node | Mode | Grid | Particles | Per rank | Launch | +|---|---|---|---|---|---|---| +| 8 | 1 x 8 | strong | 100^3 | 10M | 1.25M | `mpirun -np 8 --host dgx003:8 --map-by ppr:8:node ...` (single node) | +| 40 | 5 x 8 | weak | 250x200x100 | 50M | 1.25M | 5 nodes x 8 -- multi-node BLOCKED on this site | +| 80 | 10 x 8 | weak | 250x200x200 | 100M | 1.25M | 10 nodes x 8 -- multi-node BLOCKED on this site | + +## Limitations + +- Multi-node: BLOCKED/UNVERIFIED on this site; 40/80-GPU shapes are plans. +- HIP: recipe present, untested; bundled Kokkos lacks gfx950. +- Only `bench/in.collide` is wrapped; `in.free` and `in.sphere` (surface + collisions, `fix balance`) build with this configuration but have no + wrappers yet. +- Reference logs are from 2014 (Intel CPU) and only for 1 and 8 ranks; the + comparison is statistical by design (upstream policy), not bitwise. diff --git a/level3/sparta/build.sh b/level3/sparta/build.sh new file mode 100755 index 0000000..ff6c1b2 --- /dev/null +++ b/level3/sparta/build.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Build SPARTA (KOKKOS package, CUDA or HIP) with upstream's native CMake and +# the Kokkos it bundles (lib/kokkos) -- upstream's own kokkos_cuda/kokkos_hip +# preset recipe, with the GPU architecture chosen for this node instead of the +# preset's hard-coded HOPPER90. +# +# ./build.sh [CUDA|HIP] (default CUDA) +# +# Layout (Level 3 isolation): source _upstream/level3/sparta, build +# build/level3/sparta/, install .deps/level3/sparta/install +# (+ .hpcperf-l3-fingerprint), logs .deps/level3/sparta/logs. +# +# Recipe = upstream cmake/presets/kokkos_common.cmake (PKG_KOKKOS, BUILD_MPI, +# -O3) loaded with -C, plus the settings of cmake/presets/kokkos_cuda.cmake +# given explicitly: nvcc_wrapper as CXX (host compiler = conda GCC 13.3.0), +# Kokkos_ENABLE_CUDA, Kokkos_ENABLE_SERIAL, FFT_KOKKOS=CUFFT, and +# Kokkos_ARCH_BLACKWELL100 (sm_100) instead of HOPPER90. SPARTA requires +# C++20 with the KOKKOS package (cmake/CMakeLists.txt). +# +# Modification class: A (no upstream file modified; the arch differs from the +# shipped preset only through command-line cache entries). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +SRC="$R/_upstream/level3/sparta" +[ -f "$SRC/cmake/CMakeLists.txt" ] || { echo "build.sh: SPARTA source missing -- run $HERE/fetch.sh first" >&2; exit 1; } +SHA="$(git -C "$SRC" rev-parse HEAD)" +KOKKOS_VER="$(sed -n 's/^set(Kokkos_VERSION_\(MAJOR\|MINOR\|PATCH\) \([0-9]*\))/\2/p' "$SRC/lib/kokkos/CMakeLists.txt" | paste -sd.)" +l3_paths sparta +BUILD_DIR="$R/build/level3/sparta/$MODEL" +JOBS="${HPCPERF_BUILD_JOBS:-32}" + +case "$BACKEND" in + CUDA) + ARCH="${HPCPERF_CUDA_ARCH:-$(l3_gpu_arch)}" + case "$ARCH" in + 100) KARCH=BLACKWELL100;; 120) KARCH=BLACKWELL120;; 90) KARCH=HOPPER90;; 80) KARCH=AMPERE80;; + *) echo "build.sh: no Kokkos arch mapping for compute capability '$ARCH' (set HPCPERF_CUDA_ARCH)" >&2; exit 2;; + esac + export NVCC_WRAPPER_DEFAULT_COMPILER="$CXX" + GPU_FLAGS=(-DCMAKE_CXX_COMPILER="$SRC/lib/kokkos/bin/nvcc_wrapper" + -DKokkos_ENABLE_CUDA=ON "-DKokkos_ARCH_$KARCH=ON" -DFFT_KOKKOS=CUFFT) + FFTK=CUFFT ;; + HIP) + command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + KARCH="${HPCPERF_HIP_ARCH:-AMD_GFX950}" + GPU_FLAGS=(-DCMAKE_CXX_COMPILER=hipcc -DKokkos_ENABLE_HIP=ON "-DKokkos_ARCH_$KARCH=ON" -DFFT_KOKKOS=HIPFFT) + FFTK=HIPFFT ;; + *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; +esac + +CMAKE_OPTS="preset=kokkos_common BUILD_MPI=ON PKG_KOKKOS=ON CXX_STANDARD=20 Kokkos_ENABLE_${BACKEND}=ON Kokkos_ARCH_${KARCH} Kokkos_ENABLE_SERIAL=ON Kokkos_ENABLE_OPENMP=OFF FFT_KOKKOS=$FFTK" +FP="$(l3_fingerprint_text sparta "$SHA" "$MODEL" "kokkos(bundled)=$KOKKOS_VER" "$CMAKE_OPTS" "runtime(-pk kokkos gpu/aware)")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +echo "# SPARTA $BACKEND: upstream $SHA, bundled Kokkos $KOKKOS_VER, arch $KARCH, MPI $(mpirun --version 2>/dev/null | head -1)" +mkdir -p "$BUILD_DIR" +cmake -S "$SRC/cmake" -B "$BUILD_DIR" -G Ninja -C "$SRC/cmake/presets/kokkos_common.cmake" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$L3_INSTALL" -DCMAKE_CXX_STANDARD=20 \ + -DSPARTA_MACHINE="kokkos_$MODEL" -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=OFF \ + "${GPU_FLAGS[@]}" > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ + || { tail -30 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } +t0=$(date +%s) +cmake --build "$BUILD_DIR" -j "$JOBS" > "$L3_LOGS/build-$MODEL.log" 2>&1 \ + || { tail -30 "$L3_LOGS/build-$MODEL.log"; echo "build.sh: build failed (log: $L3_LOGS/build-$MODEL.log)" >&2; exit 1; } +cmake --install "$BUILD_DIR" > "$L3_LOGS/install-$MODEL.log" 2>&1 || { echo "build.sh: install failed" >&2; exit 1; } +l3_fingerprint_write "$L3_INSTALL" "$FP" +EXE="$(find "$BUILD_DIR" -maxdepth 2 -name "spa_kokkos_$MODEL" -type f | head -1)" +echo "# built in $(( $(date +%s)-t0 )) s: ${EXE:-} (installed under $L3_INSTALL)" +echo "# compiler warning lines: $(grep -c 'warning' "$L3_LOGS/build-$MODEL.log" || true)" diff --git a/level3/sparta/fetch.sh b/level3/sparta/fetch.sh new file mode 100755 index 0000000..38116ab --- /dev/null +++ b/level3/sparta/fetch.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Fetch SPARTA at the recorded release into _upstream/level3/sparta +# (gitignored, read-only reference). Idempotent; a checkout at another commit +# is an error, never silently reused. +# +# ./fetch.sh +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +UPSTREAM_URL="https://github.com/sparta/sparta.git" +UPSTREAM_TAG="27Aug2026" +UPSTREAM_SHA="95b9abaa8bd548991cc3c3f1c58b34722f7ade74" +DEST="$R/_upstream/level3/sparta" +if [ -d "$DEST/.git" ]; then + have="$(git -C "$DEST" rev-parse HEAD)" + [ "$have" = "$UPSTREAM_SHA" ] && { echo "fetch.sh: $DEST already at $UPSTREAM_TAG ($UPSTREAM_SHA)"; exit 0; } + echo "fetch.sh: $DEST is at $have, not the recorded $UPSTREAM_SHA ($UPSTREAM_TAG); remove it to re-fetch" >&2; exit 1 +fi +mkdir -p "$(dirname "$DEST")" +echo "fetch.sh: cloning $UPSTREAM_URL @ $UPSTREAM_TAG (shallow)" +git clone --quiet --depth 1 --branch "$UPSTREAM_TAG" "$UPSTREAM_URL" "$DEST" +have="$(git -C "$DEST" rev-parse HEAD)" +[ "$have" = "$UPSTREAM_SHA" ] || { echo "fetch.sh: tag $UPSTREAM_TAG resolved to $have, expected $UPSTREAM_SHA" >&2; exit 1; } +echo "fetch.sh: ok -> $DEST ($UPSTREAM_SHA)" diff --git a/level3/sparta/run.sh b/level3/sparta/run.sh new file mode 100755 index 0000000..5920a96 --- /dev/null +++ b/level3/sparta/run.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Run the SPARTA collisional-flow benchmark (upstream bench/in.collide) on N GPUs. +# +# ./run.sh [CUDA|HIP] [extra spa args...] +# +# Execution model (upstream Section_accelerate): one MPI rank per GPU, KOKKOS +# package on the device (`-k on g 1 -sf kk`), particles/grid distributed by +# SPARTA's own `balance_grid rcb part` -- any rank count is legal. Ranks go +# through the common launcher with the per-rank GPU wrapper (each rank sees +# exactly one GPU; mapping audited). GPU-aware MPI (`-pk kokkos gpu/aware +# yes`, SPARTA's default on GPUs) matches this repository's CUDA-aware +# Open MPI; HPCPERF_SPARTA_GPU_AWARE=no disables it. +# +# Resource / size controls (common Level 3 parameters): +# HPCPERF_GPUS=N|all ranks = GPUs (default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# smoke : upstream deck as shipped: 10x10x10 cells, 10 particles/cell = +# 10,000 particles; 30 equilibration + 100 benchmark steps +# (reference log bench/log.7Jul14.collide.icc.10K.1) +# strong : ONE fixed global grid, S^3 cells (S=HPCPERF_SPARTA_STRONG, +# default 100: 1,000,000 cells = 10,000,000 particles), split by +# SPARTA over the ranks +# weak : fixed work per rank, L^3 cells per rank (L=HPCPERF_SPARTA_LOCAL, +# default 50: 125,000 cells = 1,250,000 particles/rank); grid +# L*PX x L*PY x L*PZ with PXxPYxPZ from hpcperf_topology.py +# HPCPERF_SPARTA_GPU_AWARE yes|no (default yes) +# The deck is upstream's bench/in.collide, unmodified; sizes enter through its +# own -var x/y/z variables (particles = 10 * cells by construction). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +BUILD_DIR="$R/build/level3/sparta/$MODEL" +EXE="$(find "$BUILD_DIR" -maxdepth 2 -name "spa_kokkos_$MODEL" -type f 2>/dev/null | head -1)" +[ -n "$EXE" ] && [ -x "$EXE" ] || { echo "run.sh: spa_kokkos_$MODEL not found under $BUILD_DIR -- run ./build.sh $BACKEND first" >&2; exit 1; } +SRC="$R/_upstream/level3/sparta" + +N_RANKS="$(hpcperf_ranks sparta yes)" || exit 2 +hpcperf_forbid_args sparta -in -i -var -v -k -kokkos -sf -suffix -pk -package -log -- "$@" || exit 2 +MODE="$(l3_scale_mode sparta)" || exit 2 +GAM="${HPCPERF_SPARTA_GPU_AWARE:-yes}" + +case "$MODE" in + smoke) X=10; Y=10; Z=10 ;; + strong) S="${HPCPERF_SPARTA_STRONG:-100}"; X=$S; Y=$S; Z=$S ;; + weak) L="${HPCPERF_SPARTA_LOCAL:-50}" + TOPO="$(hpcperf_topology sparta "$N_RANKS")" || exit 2 + read -r PX PY PZ <<< "$TOPO" + X=$((L * PX)); Y=$((L * PY)); Z=$((L * PZ)) ;; +esac +CELLS=$((X * Y * Z)); PARTS=$((10 * CELLS)) +RUN_DIR="$BUILD_DIR/run"; mkdir -p "$RUN_DIR" +LOG="$RUN_DIR/log.$MODE.np$N_RANKS.sparta" +echo "# SPARTA $BACKEND: mode=$MODE ranks=$N_RANKS grid=${X}x${Y}x${Z} = $CELLS cells, $PARTS particles ($((PARTS / N_RANKS))/rank), gpu-aware=$GAM, log=$LOG" +cd "$SRC/bench" # ar.species / ar.vss are referenced relative to the deck +exec "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ + "$EXE" -k on g 1 -sf kk -pk kokkos gpu/aware "$GAM" \ + -in in.collide -var x "$X" -var y "$Y" -var z "$Z" -log "$LOG" -echo none "$@" diff --git a/level3/sparta/validate.sh b/level3/sparta/validate.sh new file mode 100755 index 0000000..c79b036 --- /dev/null +++ b/level3/sparta/validate.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Correctness check for the SPARTA Kokkos build, using upstream's benchmark +# deck bench/in.collide as shipped (10,000 particles) and the reference log +# SPARTA ships for it, bench/log.7Jul14.collide.icc.10K.1. +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) selects the rank count +# +# DSMC is a stochastic method (random seed, random collision partners), and +# the particle distribution over ranks changes the random stream, so per-step +# collision counts cannot be compared exactly. What IS exact and what has a +# physically justified tolerance: +# * particle count Np at every stats row == 10 * cells (10,000 with the +# 10x10x10 deck: exact conservation -- no chemistry, reflecting walls); +# * gas temperature (compute temp): the equilibrated argon stays at the +# initial 273.15 K; the reference log shows 273.28 K. Tolerance 2 % on the +# mean over the benchmark steps (>= step 40): the statistical temperature +# noise of 10^4 particles is ~sqrt(2/3N) ~ 0.8 %, so 2 % is ~2.5 sigma of +# the sampling noise and far below any physics or unit error; +# * mean collision attempts per step (Natt) within 15 % of the reference +# mean: it is set by density/temperature/cross-section, so a wrong +# collision model or density would move it by far more; run-to-run +# statistical scatter is a few %. +# With HPCPERF_GPUS>1 the same three criteria are applied between the N-rank +# run and this build's 1-rank run (rank-count independence). Prints PASS/FAIL, +# exit 0/1. Nothing is loosened to pass; the deck is upstream's. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +N="${HPCPERF_GPUS:-1}" +REF="$R/_upstream/level3/sparta/bench/log.7Jul14.collide.icc.10K.1" +RUN_DIR="$R/build/level3/sparta/$MODEL/run" +[ -f "$REF" ] || { echo "validate.sh: reference log $REF missing (run fetch.sh)" >&2; exit 1; } + +export HPCPERF_GPUS="$N" +echo "validate.sh: SPARTA $BACKEND smoke (bench/in.collide 10x10x10, 10,000 particles) on $N GPU(s)" +HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit|Loop time|ERROR' || true +LOG="$RUN_DIR/log.smoke.np$N.sparta" +[ -f "$LOG" ] || { echo "validate.sh: FAIL -- no log produced ($LOG)"; exit 1; } +if [ "$N" -gt 1 ] && [ ! -f "$RUN_DIR/log.smoke.np1.sparta" ]; then + echo "validate.sh: producing the 1-GPU run for rank-count comparison" + HPCPERF_GPUS=1 HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" > /dev/null 2>&1 || true +fi + +python3 - "$LOG" "$REF" "$N" "$RUN_DIR/log.smoke.np1.sparta" <<'PY' +import re, sys +def stats(path): + """rows of the LAST stats block (the 100-step benchmark run) as dicts""" + blocks, cur, cols = [], [], None + for ln in open(path).read().splitlines(): + p = ln.split() + if p[:2] == ["Step", "CPU"]: + # the 2014 reference log labels the compute column "temp", current SPARTA prints "c_temp" + cols = ["temp" if c == "c_temp" else c for c in p]; cur = []; blocks.append(cur); continue + if cols and p and re.match(r'^\d+$', p[0]): + cur.append(dict(zip(cols, map(float, p)))) + elif cols and cur and not p: + cols = None + return blocks[-1] if blocks else [] +def mean(rows, k, minstep=40): + v = [r[k] for r in rows if r["Step"] >= minstep] + return sum(v) / len(v) if v else float("nan") +log, ref, n, log1 = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] +ok = True +def check(rows, base, label, npart): + global ok + if not rows: print(f" {label}: no stats rows"); ok = False; return + bad_np = [r["Step"] for r in rows if r["Np"] != npart] + print(f" {label}: Np == {npart} at every row: {'ok' if not bad_np else 'BAD at steps ' + str(bad_np)}"); ok &= not bad_np + for k, tol in (("temp", 0.02), ("Natt", 0.15)): + a, b = mean(rows, k), mean(base, k) + rel = abs(a - b) / abs(b) + print(f" {label}: mean {k:<5} {a:12.4f} vs {b:12.4f} rel {rel:.3e} (tol {tol}) {'ok' if rel <= tol else 'BAD'}") + ok &= rel <= tol +got, want = stats(log), stats(ref) +print(f"[1] {n}-GPU run vs upstream reference log (icc, 1 proc, 2014):") +NPART = 10.0 * 10 * 10 * 10 # deck: n = 10 * x*y*z particles = 10,000 for the 10x10x10 grid +check(got, want, "vs-ref", NPART) +if n > 1: + try: + one = stats(log1) + print(f"[2] {n}-GPU run vs this build's 1-GPU run:") + check(got, one, "vs-1gpu", NPART) + except FileNotFoundError: + print("[2] 1-GPU log unavailable; rank-count comparison skipped"); ok = False +print(f"SPARTA CUDA validation ({n} GPU, bench/in.collide vs log.7Jul14.collide.icc.10K.1): {'PASS' if ok else 'FAIL'}") +sys.exit(0 if ok else 1) +PY diff --git a/level3/specfem3d/README.md b/level3/specfem3d/README.md new file mode 100644 index 0000000..0b8e125 --- /dev/null +++ b/level3/specfem3d/README.md @@ -0,0 +1,130 @@ +# SPECFEM3D Cartesian (Level 3) + +Spectral-element seismic wave propagation: the complete workflow -- mesh +(CUBIT mesh + SCOTCH partitioner, or the in-house `xmeshfem3D`), per-slice +database generation, and the GPU time loop of `xspecfem3D` (stiffness, +MPI halo assembly, absorbing boundaries, seismogram output). + +## Provenance + +- Official repository: https://github.com/SPECFEM/specfem3d (moved from + geodynamics/specfem3d); docs https://specfem3d.readthedocs.io/ +- Release policy: tagged releases (v4.1.1 is the latest, 2024-03-15); + development happens on `devel`, ~2.5 years ahead of the release. +- Selected: **v4.1.1**, commit `c67d3ae7d4bfc5ac75cb9e5601d93afa262d3d8d`, + fetched by `fetch.sh` into `_upstream/level3/specfem3d` (shallow, read-only; + the `m4`/`flexwin`/`pyCMT3D` submodules are not needed). Two source + back-ports come from `devel` `cc2e9ffa7e7cb5338e05f5a7df81cfbe60e00683` + (2026-07-24). +- License: GPL-3.0. +- Application-owned LOC (cloc 2.06, code lines): `src/` **142,398** + (Fortran 90 119,057; CUDA 12,210; `src/gpu` 15,058 in 63 `.cu` files). + `utils/` (163,847) and the bundled `external_libs/` (SCOTCH 5.1.12b, + METIS, PaToH; 137,463) are counted separately. + +## Build strategy: NATIVE (upstream autotools + bundled SCOTCH) + +`build.sh CUDA` copies the sources into `.deps/level3/specfem3d/src` +(autotools builds in-tree), applies the two patches below and runs upstream's +recipe: `./configure --with-mpi --with-cuda=cuda12 FC=/usr/bin/gfortran +CC= MPIFC=mpif90 MPI_INC= CUDA_INC/CUDA_LIB +USE_BUNDLED_SCOTCH=1`, then `make -j all GENCODE="-gencode=arch=compute_100, +code=sm_100 -gencode=arch=compute_100,code=compute_100 -DGPU_DEVICE_Blackwell"`. +Toolchain: conda GCC 13.3.0 for C and as nvcc's host compiler, system +gfortran 14.2.1 for Fortran (the conda env has no gfortran), conda Open MPI +5.0.10 with `OMPI_FC=/usr/bin/gfortran`. Build time on dgx003: **21 s** at +`-j16` (428 objects; SCOTCH, 1,000+ Fortran units, 63 CUDA units), 7 +warning lines. 24 executables installed under `.deps/level3/specfem3d/install/bin` +with the fingerprint (upstream commit, SCOTCH 5.1.12b, compilers, CUDA +13.2.78, MPI, configure/make options, patch list). + +Why not the others: no Spack package exists for SPECFEM3D Cartesian (only +`specfem3d-globe`); no Apptainer on the node and no upstream image; site +modules broken. HIP: `build.sh HIP` carries `--with-hip` but v4.1.1 knows only +MI8..MI250 (gfx803..gfx90a); devel added MI300/MI350. **Untested.** + +## Changes from upstream (all recorded in `patches/` and `build.sh`) + +| Class | Change | Size / provenance | +|---|---|---| +| D | `0001-cuda13-deviceOverlap-guard.patch`: `src/gpu/initialize_gpu.cu` reads `cudaDeviceProp.deviceOverlap`, removed in CUDA 13 -> guarded, `asyncEngineCount` printed instead (diagnostic text only) | 10 lines, back-port of upstream devel | +| D | `0002-blackwell-device-block.patch`: `GPU_DEVICE_Blackwell` block in `src/gpu/mesh_constants_cuda.h` (`#undef USE_LAUNCH_BOUNDS`, identical to Hopper's) | 8 lines, back-port of upstream devel | +| B | make-time `GENCODE` override = devel's `--with-cuda=cuda13` value (sm_100 SASS + compute_100 PTX); v4.1.1's `configure` stops at `cuda12` and cannot be regenerated here (no autoreconf, empty `m4/`) | no file edited | +| B | bundled SCOTCH built without gzip support (generated `Makefile.inc`: `-DCOMMON_FILE_COMPRESS_GZ`/`-lz` removed; conda GCC has no `zlib.h`) | generated file only | +| C | `OMPI_FC=/usr/bin/gfortran`, `MPI_INC` (configure's `mpif90 -showme:incdirs` detection returns nothing with the conda wrapper) | environment | + +No numerics, physics or algorithm changed; `flags.guess`'s gfortran flags +(`-std=f2008 -pedantic-errors -ffpe-trap=invalid,zero,overflow`) are used as +shipped and compile cleanly with gfortran 14. + +## Execution model + +One MPI rank per GPU (upstream: `device = myrank % device_count`). The common +launcher's per-rank wrapper gives each rank one visible GPU (device 0), and +audits the mapping (4/4 verified). `NPROC` in `Par_file` is the number of mesh +slices and must equal the rank count -- it is fixed when the mesh is +partitioned, so every rank count gets its own mesh + databases (SPECFEM3D's +normal workflow; `run.sh` does all three stages through the launcher with the +same N). Halo exchange in v4.1.1 is host-staged (no GPU-aware MPI). CPU +binding: runtime default. + +## Inputs (`HPCPERF_SCALE_MODE`) + +| Mode | Mesh | Elements | Per rank @4 GPU | Topology | NSTEP / DT | GPU memory (est.) | Time loop on B200 | Validation quantity | +|---|---|---|---|---|---|---|---|---| +| smoke (default) | upstream `homogeneous_halfspace` CUBIT mesh, SCOTCH partition | 20,736 | 5,184 | SCOTCH (any N) | 5000 / 0.05 s | ~0.2 GB | 0.8-0.9 s | seismograms vs upstream `REF_SEIS` | +| strong | `xmeshfem3D`, same 134x134x60 km domain refined G x (G=`HPCPERF_SPECFEM_STRONG`, 2): 72x72x32 | 165,888 | 41,472 | `NPROC_XI x NPROC_ETA` from `hpcperf_topology.py --dims 2` (NEX divisible) | 1000 / 0.025 s | ~1.7 GB | 0.975 s (1 GPU) / 0.412 s (4 GPU) | run completes; seismograms written | +| weak | per-rank block (36F)x(36F)x(16F), F=`HPCPERF_SPECFEM_LOCAL` (2), domain extended PX x PY at fixed resolution | 165,888 x N | 165,888 | 2-D grid | 1000 / 0.025 s | ~1.7 GB | 1.035 s (4 GPU, 663,552 elements) | run completes | + +Memory estimate ~10 KB per element on the GPU (NGLL 5, single precision +fields). The weak deck keeps element shape, DT and per-rank work identical for +every N (the domain grows, source and stations stay in the first block). +Preprocessing cost is CPU-side and per slice: `xgenerate_databases` for +165,888 elements on one rank takes ~12 s; at G=4 (1.33M elements on one rank) +its serial neighbour search exceeded 20 minutes and was abandoned, which is +why the strong default is G=2. + +## Validation (`validate.sh`, upstream mechanism) + +Upstream ships reference seismograms for the homogeneous half-space +(`EXAMPLES/applications/homogeneous_halfspace/REF_SEIS`, 12 traces: 4 stations +x 3 components) and compares runs with them using +`utils/scripts/compare_seismogram_correlations.py` (per trace: correlation +coefficient, L2 misfit normalised by the reference energy, cross-correlation +time shift; upstream thresholds corr >= 0.8, misfit <= 1 %, shift <= 0.01 s). +The references are CPU/double-precision results; the GPU solver is single +precision, so upstream's tolerance-based comparison is the appropriate +criterion and is used unchanged. Observed on dgx003 (2026-09-04): **PASS at 1, +2 and 4 GPUs** -- correlation 1.00000 on all 12 traces, worst misfit 2.7e-4 / +2.3e-4 / 2.7e-4, worst time shift 2.5e-5 / 2.4e-5 / 2.7e-5 s. + +## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) + +| Run | Ranks x GPUs | rank->GPU | CPU binding | Topology | Problem | Time loop (`output_solver.txt`) | Wall incl. mesh+databases | Validation | +|---|---|---|---|---|---|---|---|---| +| smoke | 1 x 1 | wrapper; audit 1/1 verified | runtime default | 1 slice | 20,736 el., 5000 steps | 0.860 s | 4 s | PASS | +| smoke | 2 x 2 | wrapper; 2/2 verified | runtime default | SCOTCH 2 | same | 0.779 s | 6 s | PASS | +| smoke | 4 x 4 | wrapper; 4/4 verified | runtime default | SCOTCH 4 | same | 0.806 s | 7 s | PASS | +| strong | 1 x 1 | wrapper; 1/1 verified | runtime default | 1x1 | 165,888 el., 1000 steps | 0.975 s | 15 s | completes | +| strong | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2 | 165,888 el. (41,472/rank) | 0.412 s | 17 s | completes | +| weak | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2 | 663,552 el. (165,888/rank) | 1.035 s | 32 s | completes | + +Dry-runs (`HPCPERF_DRY_RUN=1`, hypothetical allocations) -- **DRY-RUN / +UNVALIDATED**, nothing executed (each of the three stages prints its plan): + +| GPUs | Nodes x GPUs/node | Mode | Mesh | Elements | Per rank | NPROC_XI x NPROC_ETA | Launch | +|---|---|---|---|---|---|---|---| +| 8 | 1 x 8 | strong | 72x72x32 on 134x134x60 km | 165,888 | 20,736 | 4x2 | `mpirun -np 8 --host dgx003:8 --map-by ppr:8:node ...` (single node) | +| 40 | 5 x 8 | weak | 576x360x32 on 1072x670x60 km | 6,635,520 | 165,888 | 8x5 | 5 nodes x 8 -- multi-node BLOCKED on this site | +| 80 | 10 x 8 | weak | 720x576x32 on 1340x1072x60 km | 13,271,040 | 165,888 | 10x8 | 10 nodes x 8 -- multi-node BLOCKED on this site | + +## Limitations + +- Multi-node: BLOCKED/UNVERIFIED on this site; 40/80-GPU shapes are plans. +- HIP: untested; v4.1.1 has no MI300/MI350 configure option. +- v4.1.1 + CUDA 13.2 + sm_100 is not an upstream-validated combination; it + needs the two back-ports and the make-time GENCODE above (all upstream + devel content). +- Only the homogeneous half-space family is wrapped; layered_halfspace, + Mount_StHelens, CPML and fault examples build with this configuration but + have no wrappers yet. diff --git a/level3/specfem3d/build.sh b/level3/specfem3d/build.sh new file mode 100755 index 0000000..f75dc8e --- /dev/null +++ b/level3/specfem3d/build.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Build SPECFEM3D Cartesian (meshfem3D, decompose_mesh, generate_databases, +# specfem3D) with upstream's autotools build, CUDA-enabled, from a private +# copy of the source tree. +# +# ./build.sh [CUDA|HIP] (default CUDA) +# +# Layout (Level 3 isolation): read-only clone _upstream/level3/specfem3d; +# patched private source copy .deps/level3/specfem3d/src (autotools builds +# in-tree: obj/ and bin/ live there); install .deps/level3/specfem3d/install/bin; +# logs .deps/level3/specfem3d/logs. Only dependency besides MPI/CUDA is SCOTCH, +# bundled (external_libs/scotch_5.1.12b) and built by the same make. +# +# Toolchain: CC = conda GCC 13.3.0 (also nvcc's host compiler, first `gcc` on +# PATH as upstream's Makefile expects), FC = system gfortran 14.2.1 (the conda +# env has no gfortran), MPIFC = conda Open MPI's mpif90 pointed at that +# gfortran via OMPI_FC (class C). The conda MPI Fortran module loads under +# gfortran 14 (.mod format 15; verified with a 2-rank MPI Fortran test). +# +# Modifications (all recorded in patches/, provenance upstream devel +# cc2e9ffa, 2026-07-24): +# class D 0001-cuda13-deviceOverlap-guard.patch -- v4.1.1 reads +# cudaDeviceProp.deviceOverlap, a field removed in CUDA 13; upstream +# devel guards it (CUDA_VERSION < 13000) and prints asyncEngineCount +# instead. Diagnostic output only, no numerics. 10 lines. +# class D 0002-blackwell-device-block.patch -- upstream devel's +# GPU_DEVICE_Blackwell block (same content as Hopper's: +# #undef USE_LAUNCH_BOUNDS). 8 lines. +# class B v4.1.1's configure knows --with-cuda=cuda4..cuda12 (cuda12 = +# sm_90 + GPU_DEVICE_Hopper); devel added cuda13 = sm_100 + +# GPU_DEVICE_Blackwell. Regenerating configure needs autoreconf and +# the m4 submodule (absent), so the same result is obtained by +# configuring with --with-cuda=cuda12 and overriding the GENCODE +# make variable on the command line with upstream devel's cuda13 +# value. No upstream build file is edited. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +UP="$R/_upstream/level3/specfem3d" +[ -f "$UP/configure" ] || { echo "build.sh: $UP missing -- run $HERE/fetch.sh first" >&2; exit 1; } +SHA="$(git -C "$UP" rev-parse HEAD)" +l3_paths specfem3d +JOBS="${HPCPERF_BUILD_JOBS:-32}" +SYS_FC="${HPCPERF_SYSTEM_GFORTRAN:-/usr/bin/gfortran}" +[ -x "$SYS_FC" ] || { echo "build.sh: no gfortran at $SYS_FC (set HPCPERF_SYSTEM_GFORTRAN); the conda env has none" >&2; exit 1; } +export OMPI_FC="$SYS_FC" +PATCHES=("$HERE/patches/0001-cuda13-deviceOverlap-guard.patch" "$HERE/patches/0002-blackwell-device-block.patch") + +case "$BACKEND" in + CUDA) + command -v nvcc >/dev/null || { echo "build.sh: nvcc not on PATH" >&2; exit 1; } + ARCH="${HPCPERF_CUDA_ARCH:-$(l3_gpu_arch)}" + CONF_GPU=(--with-cuda=cuda12) + # = upstream devel's cuda13 GENCODE (sm_100 SASS + compute_100 PTX), written as two -gencode flags so + # that no shell quoting travels through the make command line + GENCODE="-gencode=arch=compute_${ARCH},code=sm_${ARCH} -gencode=arch=compute_${ARCH},code=compute_${ARCH} -DGPU_DEVICE_Blackwell" + ARCHNOTE="sm_$ARCH" ;; + HIP) + command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + # v4.1.1 knows --with-hip=MI8..MI250 (gfx803..gfx90a) only; devel added MI300/MI350 (gfx942/gfx950) + CONF_GPU=(--with-hip=MI250); GENCODE=""; ARCHNOTE="gfx950 (UNTESTED; v4.1.1 has no MI350 option)" ;; + *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; +esac +CMAKE_OPTS="configure: FC=$SYS_FC CC=$CC MPIFC=mpif90(OMPI_FC=$SYS_FC) --with-mpi ${CONF_GPU[*]} USE_BUNDLED_SCOTCH=1; make GENCODE=${GENCODE:-default}" +PATCHNAMES=(); for p in "${PATCHES[@]}"; do PATCHNAMES+=("$(basename "$p")"); done +FP="$(l3_fingerprint_text specfem3d "$SHA" "$MODEL" "scotch=5.1.12b (bundled)" "$CMAKE_OPTS" "no (host-staged halo exchange in v4.1.1)" "${PATCHNAMES[@]}")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +echo "# SPECFEM3D $BACKEND: upstream $SHA (v4.1.1), arch $ARCHNOTE, MPI $(mpirun --version 2>/dev/null | head -1), FC $($SYS_FC --version | head -1), CC $($CC --version | head -1)" +rm -rf "$L3_SRC"; mkdir -p "$L3_SRC" +# private source copy (configure needs the top-level DATA/ defaults, which point into EXAMPLES/); doc/ stays in the clone +rsync -a --exclude .git --exclude doc "$UP/" "$L3_SRC/" +for p in "${PATCHES[@]}"; do + (cd "$L3_SRC" && patch -p1 --forward --silent < "$p") || { echo "build.sh: patch $(basename "$p") failed to apply" >&2; exit 1; } + echo "# applied $(basename "$p")" +done +cd "$L3_SRC" +CUDA_ROOT="${CUDA_HOME:-$(dirname "$(dirname "$(command -v nvcc)")")}" +# mpi.h for the nvcc-compiled GPU sources (configure's auto-detection via `mpif90 -showme:incdirs` did not +# reach the nvcc command line here); MPI_INC is the documented configure variable for this +MPI_INC_DIR="$(mpicc -showme:incdirs 2>/dev/null | awk '{print $1}')"; [ -f "$MPI_INC_DIR/mpi.h" ] || MPI_INC_DIR="$(dirname "$(dirname "$(command -v mpicc)")")/include" +[ -f "$MPI_INC_DIR/mpi.h" ] || { echo "build.sh: cannot locate mpi.h (looked in $MPI_INC_DIR)" >&2; exit 1; } +FC="$SYS_FC" CC="$CC" MPIFC=mpif90 MPI_INC="$MPI_INC_DIR" CUDA_INC="$CUDA_ROOT/include" CUDA_LIB="$CUDA_ROOT/lib64" USE_BUNDLED_SCOTCH=1 \ + ./configure --with-mpi "${CONF_GPU[@]}" > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ + || { tail -40 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } +# bundled SCOTCH: the generated Makefile.inc enables gzip-compressed mesh files (-DCOMMON_FILE_COMPRESS_GZ, +# -lz); the conda GCC has no zlib.h in its sysroot and the meshes here are uncompressed -> build SCOTCH +# without that optional feature (class B, generated file only; upstream sources untouched) +sed -i -e 's/ -DCOMMON_FILE_COMPRESS_GZ//' -e 's/ -lz\b//' external_libs/scotch/src/Makefile.inc +t0=$(date +%s) +if [ -n "$GENCODE" ]; then MAKEVARS=("GENCODE=$GENCODE"); else MAKEVARS=(); fi +make -j "$JOBS" "${MAKEVARS[@]}" all > "$L3_LOGS/build-$MODEL.log" 2>&1 \ + || { grep -n -i 'error' "$L3_LOGS/build-$MODEL.log" | head -20 || true; echo "build.sh: build failed (log: $L3_LOGS/build-$MODEL.log)" >&2; exit 1; } +mkdir -p "$L3_INSTALL/bin"; cp -f bin/x* "$L3_INSTALL/bin/" +l3_fingerprint_write "$L3_INSTALL" "$FP" +echo "# built in $(( $(date +%s)-t0 )) s: $(ls "$L3_INSTALL/bin" | tr '\n' ' ')" +echo "# compiler warning lines: $(grep -c -i 'warning' "$L3_LOGS/build-$MODEL.log" || true)" diff --git a/level3/specfem3d/fetch.sh b/level3/specfem3d/fetch.sh new file mode 100755 index 0000000..cd10663 --- /dev/null +++ b/level3/specfem3d/fetch.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Fetch SPECFEM3D Cartesian at the latest release tag into +# _upstream/level3/specfem3d (gitignored, read-only). Idempotent; a checkout +# at another commit is an error, never silently reused. +# +# ./fetch.sh +# +# The two source patches in patches/ are backports of upstream `devel` commits +# (CUDA 13 `deviceOverlap` guard, Blackwell device block); their provenance is +# the devel snapshot recorded in DEVEL_SNAPSHOT below (not checked out here). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +UPSTREAM_URL="https://github.com/SPECFEM/specfem3d.git" +UPSTREAM_TAG="v4.1.1" +UPSTREAM_SHA="c67d3ae7d4bfc5ac75cb9e5601d93afa262d3d8d" +DEVEL_SNAPSHOT="cc2e9ffa7e7cb5338e05f5a7df81cfbe60e00683" # devel @ 2026-07-24, source of the patches +DEST="$R/_upstream/level3/specfem3d" + +if [ -d "$DEST/.git" ]; then + have="$(git -C "$DEST" rev-parse HEAD)" + if [ "$have" = "$UPSTREAM_SHA" ]; then echo "fetch.sh: $DEST already at $UPSTREAM_TAG ($UPSTREAM_SHA)"; exit 0; fi + echo "fetch.sh: $DEST is at $have, not the recorded $UPSTREAM_SHA ($UPSTREAM_TAG); remove it to re-fetch" >&2; exit 1 +fi +mkdir -p "$(dirname "$DEST")" +echo "fetch.sh: cloning $UPSTREAM_URL @ $UPSTREAM_TAG (shallow; submodules m4/flexwin/pyCMT3D not needed)" +git clone --quiet --depth 1 --branch "$UPSTREAM_TAG" "$UPSTREAM_URL" "$DEST" +have="$(git -C "$DEST" rev-parse HEAD)" +[ "$have" = "$UPSTREAM_SHA" ] || { echo "fetch.sh: tag $UPSTREAM_TAG resolved to $have, expected $UPSTREAM_SHA" >&2; exit 1; } +echo "fetch.sh: ok -> $DEST ($UPSTREAM_SHA); patches backported from devel $DEVEL_SNAPSHOT" diff --git a/level3/specfem3d/patches/0001-cuda13-deviceOverlap-guard.patch b/level3/specfem3d/patches/0001-cuda13-deviceOverlap-guard.patch new file mode 100644 index 0000000..ed5eecd --- /dev/null +++ b/level3/specfem3d/patches/0001-cuda13-deviceOverlap-guard.patch @@ -0,0 +1,19 @@ +--- a/src/gpu/initialize_gpu.cu ++++ b/src/gpu/initialize_gpu.cu +@@ -241,11 +241,16 @@ + }else{ + fprintf(fp," canMapHostMemory: FALSE\n"); + } ++#if CUDA_VERSION < 13000 || (defined (__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ < 13)) + if (deviceProp.deviceOverlap){ + fprintf(fp," deviceOverlap: TRUE\n"); + }else{ + fprintf(fp," deviceOverlap: FALSE\n"); + } ++#else ++ // CUDA version >= 13, deviceOverlap deprecated, replaced by asyncEngineCount ++ fprintf(fp," asyncEngineCount: %d\n", deviceProp.asyncEngineCount); ++#endif + if (deviceProp.concurrentKernels){ + fprintf(fp," concurrentKernels: TRUE\n"); + }else{ diff --git a/level3/specfem3d/patches/0002-blackwell-device-block.patch b/level3/specfem3d/patches/0002-blackwell-device-block.patch new file mode 100644 index 0000000..ef77cf4 --- /dev/null +++ b/level3/specfem3d/patches/0002-blackwell-device-block.patch @@ -0,0 +1,17 @@ +--- a/src/gpu/mesh_constants_cuda.h ++++ b/src/gpu/mesh_constants_cuda.h +@@ -107,6 +107,14 @@ + #undef USE_LAUNCH_BOUNDS + #endif + ++#ifdef GPU_DEVICE_Blackwell ++// specifics see: https://docs.nvidia.com/cuda/blackwell-tuning-guide/index.html ++// register file size 64k 32-bit registers per SM ++// shared memory size 228KB per SM (for compute capability 10.0) or 128KB per SM (for compute capability 12.0) ++// maximum registers 255 per thread ++#undef USE_LAUNCH_BOUNDS ++#endif ++ + /* ----------------------------------------------------------------------------------------------- */ + + // CUDA specifics diff --git a/level3/specfem3d/run.sh b/level3/specfem3d/run.sh new file mode 100755 index 0000000..27d3e34 --- /dev/null +++ b/level3/specfem3d/run.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Run the full SPECFEM3D Cartesian workflow (mesh -> databases -> solver) for +# the homogeneous-halfspace problem on N GPUs. +# +# ./run.sh [CUDA|HIP] +# +# Workflow (upstream EXAMPLES/*/run_this_example.sh): the mesh is partitioned +# into NPROC slices, `xgenerate_databases` (NPROC MPI ranks, CPU) builds the +# per-slice databases, `xspecfem3D` (NPROC MPI ranks, GPU_MODE) runs the +# spectral-element time loop. NPROC is fixed at mesh time, so every rank count +# gets its own run directory and its own mesh/databases. +# +# Execution model: one MPI rank per GPU (upstream: device = myrank % +# device_count; the common launcher's per-rank wrapper gives each rank one +# visible GPU, so device 0 is that rank's GPU; audited). Halo exchange in +# v4.1.1 is host-staged (no GPU-aware MPI requirement). All three stages are +# launched through the common launcher with the same rank count. +# +# Resource / size controls: +# HPCPERF_GPUS=N|all ranks = GPUs = NPROC (default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# smoke : upstream case as shipped: CUBIT mesh MESH-default (36x36x16 = +# 20,736 HEX8 elements, 134x134x60 km, Vp 2.8 km/s), partitioned +# with the bundled SCOTCH by xdecompose_mesh (any NPROC), NSTEP +# 5000, DT 0.05 s -- the case whose reference seismograms ship +# strong : in-house mesher xmeshfem3D on the SAME domain refined G times +# (G=HPCPERF_SPECFEM_STRONG, default 2): (36G)x(36G)x(16G) = +# 165,888 elements, DT 0.05/G, NPROC = PX*PY from +# hpcperf_topology.py (2-D grid; NEX must be divisible by PX/PY). +# Larger G is legal but the serial per-slice neighbour search in +# xgenerate_databases dominates (G=4, 1.33M elements on one rank: +# > 20 min of CPU preprocessing before the GPU solver starts) +# weak : fixed per-rank block (36F)x(36F)x(16F) elements (F = +# HPCPERF_SPECFEM_LOCAL, default 2: 165,888/rank), the DOMAIN is +# extended PX x PY times at fixed resolution (element shape, DT +# and per-step work per rank identical for every N); source and +# stations stay in the first block +# HPCPERF_SPECFEM_STEPS NSTEP for strong/weak (default 1000; smoke keeps 5000) +# +# Derived inputs (class A): Par_file gets NPROC / GPU_MODE / NSTEP / DT (and +# SAVE_MESH_FILES=.false. for strong/weak to skip VTK mesh dumps); the mesher's +# Mesh_Par_file / interfaces.txt get the sizes above. Upstream files untouched. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +l3_paths specfem3d +BIN="$L3_INSTALL/bin" +for x in xdecompose_mesh xmeshfem3D xgenerate_databases xspecfem3D; do + [ -x "$BIN/$x" ] || { echo "run.sh: $BIN/$x missing -- run ./build.sh $BACKEND first" >&2; exit 1; } +done +EX="$R/_upstream/level3/specfem3d/EXAMPLES/applications/homogeneous_halfspace" +[ -f "$EX/DATA/Par_file" ] || { echo "run.sh: $EX missing (run fetch.sh)" >&2; exit 1; } + +N_RANKS="$(hpcperf_ranks specfem3d yes)" || exit 2 +MODE="$(l3_scale_mode specfem3d)" || exit 2 +BUILD_DIR="$R/build/level3/specfem3d/$MODEL" +RUN_DIR="$BUILD_DIR/run/$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR/OUTPUT_FILES/DATABASES_MPI" +cp -r "$EX/DATA" "$RUN_DIR/DATA" +PAR="$RUN_DIR/DATA/Par_file" +sed -i -e "s/^NPROC *=.*/NPROC = $N_RANKS/" -e "s/^GPU_MODE *=.*/GPU_MODE = .true./" "$PAR" + +case "$MODE" in + smoke) + NEX=36; NZ=16; STEPS=5000; DT=0.05; PX=1; PY=1; DESC="CUBIT mesh MESH-default, SCOTCH partition into $N_RANKS" ;; + strong) + G="${HPCPERF_SPECFEM_STRONG:-2}"; NEX=$((36 * G)); NZ=$((16 * G)); STEPS="${HPCPERF_SPECFEM_STEPS:-1000}" + DT="$(python3 -c "print(0.05/$G)")" + TOPO="$(hpcperf_topology specfem3d "$N_RANKS" --dims 2 --divides "$NEX,$NEX,0")" || exit 2 + read -r PX PY _ <<< "$TOPO"; NEX_XI=$NEX; NEX_ETA=$NEX; LX=134000.0; LY=134000.0 + DESC="xmeshfem3D ${NEX_XI}x${NEX_ETA}x${NZ} on 134x134x60 km, NPROC_XI x NPROC_ETA = ${PX}x${PY}" ;; + weak) + F="${HPCPERF_SPECFEM_LOCAL:-2}"; NZ=$((16 * F)); STEPS="${HPCPERF_SPECFEM_STEPS:-1000}" + DT="$(python3 -c "print(0.05/$F)")" + TOPO="$(hpcperf_topology specfem3d "$N_RANKS" --dims 2)" || exit 2 + read -r PX PY _ <<< "$TOPO"; NEX_XI=$((36 * F * PX)); NEX_ETA=$((36 * F * PY)) + LX="$(python3 -c "print(134000.0*$PX)")"; LY="$(python3 -c "print(134000.0*$PY)")" + DESC="xmeshfem3D ${NEX_XI}x${NEX_ETA}x${NZ} on $((134 * PX))x$((134 * PY))x60 km, NPROC_XI x NPROC_ETA = ${PX}x${PY}" ;; +esac +if [ "$MODE" != smoke ]; then + ELEMS=$((NEX_XI * NEX_ETA * NZ)) + sed -i -e "s/^NSTEP *=.*/NSTEP = $STEPS/" -e "s/^DT *=.*/DT = $DT/" \ + -e "s/^SAVE_MESH_FILES *=.*/SAVE_MESH_FILES = .false./" "$PAR" + mkdir -p "$RUN_DIR/DATA/meshfem3D_files" + cp "$EX/meshfem3D_files/interface1.txt" "$RUN_DIR/DATA/meshfem3D_files/" + sed -e "s/^ 16\$/ $NZ/" "$EX/meshfem3D_files/interfaces.txt" > "$RUN_DIR/DATA/meshfem3D_files/interfaces.txt" + sed -e "s/^LATITUDE_MAX *=.*/LATITUDE_MAX = $LY/" -e "s/^LONGITUDE_MAX *=.*/LONGITUDE_MAX = $LX/" \ + -e "s/^NEX_XI *=.*/NEX_XI = $NEX_XI/" -e "s/^NEX_ETA *=.*/NEX_ETA = $NEX_ETA/" \ + -e "s/^NPROC_XI *=.*/NPROC_XI = $PX/" -e "s/^NPROC_ETA *=.*/NPROC_ETA = $PY/" \ + -e "s/^CREATE_VTK_FILES *=.*/CREATE_VTK_FILES = .false./" \ + -e "s/^1 *36 *1 *36 *1 *16 *1\$/1 $NEX_XI 1 $NEX_ETA 1 $NZ 1/" \ + "$EX/meshfem3D_files/Mesh_Par_file" > "$RUN_DIR/DATA/meshfem3D_files/Mesh_Par_file" + grep -q "^1 $NEX_XI 1 $NEX_ETA 1 $NZ 1" "$RUN_DIR/DATA/meshfem3D_files/Mesh_Par_file" || { echo "run.sh: failed to rewrite the mesh region line" >&2; exit 1; } +else + ELEMS=20736 +fi + +echo "# SPECFEM3D $BACKEND: mode=$MODE ranks=$N_RANKS elements=$ELEMS (~$((ELEMS / N_RANKS))/rank) NSTEP=$STEPS DT=$DT mesh: $DESC run_dir=$RUN_DIR" +cd "$RUN_DIR" +LAUNCH=("$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper --) +if [ "$MODE" = smoke ]; then + if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + echo "# stage 1/3: xdecompose_mesh $N_RANKS (serial, CPU, SCOTCH)" + "$BIN/xdecompose_mesh" "$N_RANKS" "$EX/MESH-default" OUTPUT_FILES/DATABASES_MPI > OUTPUT_FILES/output_decompose_mesh.txt 2>&1 \ + || { tail -20 OUTPUT_FILES/output_decompose_mesh.txt; echo "run.sh: xdecompose_mesh failed" >&2; exit 1; } + else + echo "# stage 1/3: xdecompose_mesh $N_RANKS (serial, CPU) -- skipped in dry-run" + fi +else + echo "# stage 1/3: xmeshfem3D on $N_RANKS ranks (CPU)" + "${LAUNCH[@]}" "$BIN/xmeshfem3D" > OUTPUT_FILES/output_meshfem3D.log 2>&1 || { tail -30 OUTPUT_FILES/output_meshfem3D.log; echo "run.sh: xmeshfem3D failed" >&2; exit 1; } + grep -E 'hpcperf-launch: (actual|HYPOTHETICAL|launch|command|dry-run|NOTE)' OUTPUT_FILES/output_meshfem3D.log || true +fi +echo "# stage 2/3: xgenerate_databases on $N_RANKS ranks (CPU)" +"${LAUNCH[@]}" "$BIN/xgenerate_databases" > OUTPUT_FILES/output_generate_databases.log 2>&1 || { tail -30 OUTPUT_FILES/output_generate_databases.log; echo "run.sh: xgenerate_databases failed" >&2; exit 1; } +grep -E 'hpcperf-launch: (dry-run)' OUTPUT_FILES/output_generate_databases.log || true +echo "# stage 3/3: xspecfem3D on $N_RANKS ranks (GPU_MODE)" +t0=$(date +%s) +"${LAUNCH[@]}" "$BIN/xspecfem3D" 2>&1 | tee OUTPUT_FILES/output_specfem3D.log | grep -E 'hpcperf-launch|Error|ERROR|GPU|Time loop|Elapsed|End of' || true +rc=${PIPESTATUS[0]} +[ "$rc" -eq 0 ] || { echo "run.sh: xspecfem3D exited $rc (see $RUN_DIR/OUTPUT_FILES/output_specfem3D.log)" >&2; exit "$rc"; } +[ -n "${HPCPERF_DRY_RUN:-}" ] && exit 0 +echo "# solver wall time $(( $(date +%s)-t0 )) s; $(grep -E 'Total elapsed time in seconds|Time loop finished' OUTPUT_FILES/output_solver.txt 2>/dev/null | tr -s ' ' | tr '\n' ';')" +echo "# seismograms: $(ls OUTPUT_FILES/*.semd 2>/dev/null | wc -l) files in $RUN_DIR/OUTPUT_FILES" diff --git a/level3/specfem3d/validate.sh b/level3/specfem3d/validate.sh new file mode 100755 index 0000000..de5d263 --- /dev/null +++ b/level3/specfem3d/validate.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Correctness check for SPECFEM3D Cartesian on N GPUs, using upstream's own +# reference seismograms and comparison tool. +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) +# +# Case: EXAMPLES/applications/homogeneous_halfspace as shipped (36x36x16 = 20,736 +# HEX8 elements, CMT source at 30 km depth, 4 stations, NSTEP 5000, DT 0.05 s), +# run in GPU mode with NPROC = N. Its README (step 7) says to "check with 6 +# reference seismograms in REF_SEIS/"; upstream's BuildBot uses +# utils/scripts/compare_seismogram_correlations.py, which reports per trace the +# correlation coefficient, the L2 misfit normalised by the reference energy, +# and the cross-correlation time shift, with upstream's thresholds +# TOL_CORR = 0.8, TOL_ERR = 0.01 (1 %), TOL_SHIFT = 0.01 s. The reference +# traces were produced on CPUs (double precision, 4 ranks); the GPU solver is +# single precision, so bitwise equality is not expected -- upstream's +# tolerance-based comparison is the appropriate criterion and is used +# unchanged. PASS = every trace within all three thresholds. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +N="${HPCPERF_GPUS:-1}" +UP="$R/_upstream/level3/specfem3d" +REF="$UP/EXAMPLES/applications/homogeneous_halfspace/REF_SEIS" +CMP="$UP/utils/scripts/compare_seismogram_correlations.py" +RUN_DIR="$R/build/level3/specfem3d/$MODEL/run/smoke.np$N" +[ -d "$REF" ] && [ -f "$CMP" ] || { echo "validate.sh: $REF or $CMP missing (run fetch.sh)" >&2; exit 1; } + +export HPCPERF_GPUS="$N" +echo "validate.sh: SPECFEM3D $BACKEND homogeneous_halfspace (20,736 elements, 5000 steps) on $N GPU(s)" +HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Time loop|Elapsed time|End of the simulation|Error|ERROR' || true +OUT="$RUN_DIR/OUTPUT_FILES" +ls "$OUT"/*.semd >/dev/null 2>&1 || { echo "validate.sh: FAIL -- no seismograms under $OUT"; exit 1; } + +echo "validate.sh: comparing with upstream REF_SEIS (utils/scripts/compare_seismogram_correlations.py)" +CMP_OUT="$RUN_DIR/compare_ref_seis.log" +python3 "$CMP" "$OUT/" "$REF/" > "$CMP_OUT" 2>&1 || true +grep -E '^\|' "$CMP_OUT" | sed 's/^/ /' +grep -E 'seismograms compared|poor correlation|poor match|significant time shift|no poor|no significant' "$CMP_OUT" | sed 's/^/ /' +ok=1 +grep -q 'no poor correlations found' "$CMP_OUT" || ok=0 +grep -q 'no poor matches found' "$CMP_OUT" || ok=0 +grep -q 'no significant time shifts found' "$CMP_OUT" || ok=0 +NCMP="$(grep -oE '^[0-9]+ seismograms compared' "$CMP_OUT" | awk '{print $1}')" +[ "${NCMP:-0}" -gt 0 ] || ok=0 +if [ "$ok" -eq 1 ]; then + echo "SPECFEM3D $BACKEND validation ($N GPU, homogeneous_halfspace vs REF_SEIS, corr>=0.8 err<=1% shift<=0.01s): PASS"; exit 0 +fi +echo "SPECFEM3D $BACKEND validation ($N GPU, homogeneous_halfspace vs REF_SEIS): FAIL (see $CMP_OUT)"; exit 1 diff --git a/level3/tools/l3_common.sh b/level3/tools/l3_common.sh new file mode 100755 index 0000000..f0b23b1 --- /dev/null +++ b/level3/tools/l3_common.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# l3_common.sh -- shared helpers for Level 3 application wrappers. Source it. +# +# Dependency isolation: every Level 3 application owns a private tree +# $R/.deps/level3//{src,build,install,logs} +# (never a shared install root, so Kokkos/AMReX/MPI/hypre versions of +# different applications cannot pollute each other), plus its upstream +# checkout under $R/_upstream/level3/ and its own build tree under +# $R/build/level3//. Nothing here touches the Level 2 tree. +# +# Fingerprint: an install is stamped with .hpcperf-l3-fingerprint recording +# application, upstream commit, dependency versions, compiler, CUDA/ROCm, GPU +# arch, MPI, CMake options, GPU-aware-MPI option, patch hashes, site profile, +# Spack lock hash and container image hash where applicable. A recorded +# fingerprint that does not match the requested configuration FAILS FAST +# (l3_fingerprint_check) -- stale installs are never reused silently. +# +# Runtime: launches go through the common launcher. Until the shared runtime +# tools move to tools/runtime/ (proposal in tools/runtime/README.md) the +# location is a single variable, HPCPERF_RUNTIME_DIR, defaulting to +# level2/tools -- nothing is copied or moved, so Level 2 is not disturbed. + +L3_R="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HPCPERF_RUNTIME_DIR="${HPCPERF_RUNTIME_DIR:-$L3_R/level2/tools}" +L3_LAUNCHER="$HPCPERF_RUNTIME_DIR/hpcperf_mpi_launch.sh" +L3_TOPOLOGY="$HPCPERF_RUNTIME_DIR/hpcperf_topology.py" +# shellcheck disable=SC1091 +source "$HPCPERF_RUNTIME_DIR/hpcperf_launch_common.sh" + +# l3_paths : exports L3_APP, L3_UPSTREAM, L3_DEPS, L3_SRC, L3_BUILD_DEPS, L3_INSTALL, L3_LOGS +l3_paths() { + L3_APP="$1" + L3_DEPS="$L3_R/.deps/level3/$L3_APP" + L3_SRC="$L3_DEPS/src"; L3_BUILD_DEPS="$L3_DEPS/build"; L3_INSTALL="$L3_DEPS/install"; L3_LOGS="$L3_DEPS/logs" + mkdir -p "$L3_SRC" "$L3_BUILD_DEPS" "$L3_INSTALL" "$L3_LOGS" +} + +l3_first_line() { "$@" 2>/dev/null | head -n 1 || true; } +l3_cuda_version() { nvcc --version 2>/dev/null | sed -n 's/^Cuda compilation tools, release [^,]*, V\([0-9][0-9.]*\).*$/\1/p' | head -n 1; } +l3_gpu_arch() { # numeric compute capability of GPU 0, e.g. 100 + nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' .' +} +l3_site_profile() { + if [ -n "${HPCPERF_SITE_PROFILE:-}" ]; then echo "$HPCPERF_SITE_PROFILE"; return; fi + case "$(hostname -s)" in dgx003|hopper*|gpu0*) echo gmu-hopper ;; *) echo generic ;; esac +} + +# l3_fingerprint_text "" "" [patch files...] +# Prints the fingerprint for the configuration about to be built. +l3_fingerprint_text() { + local app=$1 sha=$2 backend=$3 deps=$4 cmakeopts=$5 gam=$6; shift 6 + local p + echo "schema=l3-1" + echo "application=$app" + echo "upstream_commit=$sha" + echo "backend=$backend arch=sm_$(l3_gpu_arch)" + echo "dependencies=$deps" + echo "compiler=${CXX:-c++} ($(l3_first_line "${CXX:-c++}" --version))" + echo "fortran=${FC:-gfortran} ($(l3_first_line "${FC:-gfortran}" --version))" + echo "cuda=$(l3_cuda_version)" + echo "rocm=${ROCM_VERSION:-none}" + echo "mpi=$(l3_first_line mpirun --version)" + echo "cmake_options=$cmakeopts" + echo "gpu_aware_mpi=$gam" + echo "site_profile=$(l3_site_profile)" + echo "spack_lock_sha256=${L3_SPACK_LOCK_SHA:-none}" + echo "container_image_sha256=${L3_CONTAINER_SHA:-none}" + for p in "$@"; do + [ -e "$p" ] || continue + echo "patch=$(basename "$p") sha256=$(sha256sum "$p" | cut -d' ' -f1)" + done +} + +# l3_fingerprint_check +# 0 = no fingerprint yet (fresh build) or identical; 1 = mismatch (prints diff, caller must fail). +l3_fingerprint_check() { + local dir=$1 expected=$2 fp="$1/.hpcperf-l3-fingerprint" diffout + [ -f "$fp" ] || return 0 + diffout="$(diff <(grep -v '^built=' "$fp") <(printf '%s\n' "$expected") || true)" + if [ -n "$diffout" ]; then + echo "l3: fingerprint mismatch for $dir (recorded < vs requested >):" >&2 + echo "$diffout" >&2 + echo "l3: refusing to reuse a differently-configured install; remove $dir or change the request" >&2 + return 1 + fi + return 0 +} + +# l3_fingerprint_write : written only after a successful build+install. +l3_fingerprint_write() { + local dir=$1 text=$2 + { printf '%s\n' "$text"; echo "built=$(date -u +%Y-%m-%dT%H:%MZ) (build-time record)"; } > "$dir/.hpcperf-l3-fingerprint.tmp" \ + && mv -f "$dir/.hpcperf-l3-fingerprint.tmp" "$dir/.hpcperf-l3-fingerprint" +} + +# l3_scale_mode : validated HPCPERF_SCALE_MODE (smoke|strong|weak; default smoke) +l3_scale_mode() { + local m="${HPCPERF_SCALE_MODE:-smoke}" + case "$m" in smoke|strong|weak) echo "$m";; *) echo "$1/run.sh: HPCPERF_SCALE_MODE must be smoke|strong|weak (got '$m')" >&2; return 2;; esac +} diff --git a/level3/warpx/README.md b/level3/warpx/README.md new file mode 100644 index 0000000..f14cbfe --- /dev/null +++ b/level3/warpx/README.md @@ -0,0 +1,156 @@ +# WarpX (Level 3) + +Full 3D electromagnetic particle-in-cell: charge/current deposition, +Yee/FDTD Maxwell solve, Boris push, particle and guard-cell exchange -- the +complete application driven by its own inputs files, on the GPU through AMReX. + +## Provenance + +- Official repository: **https://github.com/BLAST-WarpX/warpx** (the former + ECP-WarpX/WarpX URL redirects there); docs https://warpx.readthedocs.io/ +- Release policy: monthly `YY.MM` tags; the pinned AMReX release is recorded + in `dependencies.json`. +- Selected: **WarpX 26.09** (2026-09-03), commit + `0c62c75e53a9ad08241535444bd7e53fd1deba88`, with **AMReX 26.09** + `a52ca73324ac2c7b65ec04f131e6df99eec9c576` (the exact tag WarpX 26.09 + pins). Both fetched by `fetch.sh` into `_upstream/level3/{WarpX,amrex}` + (shallow, read-only). +- License: BSD-3-Clause-LBNL (`LICENSE.txt`, `LEGAL.txt`). +- Application-owned LOC (cloc 2.06, code lines): `Source/` **112,459** + (C++ 69,266 in 247 files; headers 37,146). AMReX `Src/` 273,313 counted + separately (dependency, not modified). No third-party source is vendored + in-tree; with the build options below nothing is downloaded at configure + time. + +## Build strategy: NATIVE (upstream CMake superbuild, local AMReX source) + +`build.sh CUDA` = upstream's documented CMake route: +`-DWarpX_COMPUTE=CUDA -DCMAKE_CUDA_ARCHITECTURES=100 -DWarpX_DIMS=3 +-DWarpX_MPI=ON -DWarpX_amrex_src= -DWarpX_OPENPMD=OFF +-DWarpX_QED=OFF -DWarpX_PYTHON=OFF -DWarpX_FFT=OFF -DWarpX_APP=ON +-DWarpX_LIB=OFF -DBUILD_TESTING=OFF`, host compiler conda GCC 13.3.0 (upstream +requires GCC 12+ / NVCC 12.4+; upstream's own Perlmutter profile pairs GCC 13 +with NVCC 13.2.78), conda Open MPI 5.0.10 (CUDA-aware), CMake 3.28.4 >= 3.25, +Ninja. openPMD (needs HDF5/ADIOS2 for useful output; AMReX plotfiles are +produced without it), QED (PICSAR download) and Python are off for the +bring-up; they are documented options, not modifications. Build time on +dgx003: **1219 s (20.3 min)** at `-j32` (367 targets, AMReX built by the +superbuild), **0 compiler warning lines**; executable +`build/level3/warpx/cuda/bin/warpx.3d.MPI.CUDA.DP.PDP.EB` (726 MB), installed +under `.deps/level3/warpx/install` with the fingerprint (upstream commit, +AMReX commit, compiler, CUDA 13.2.78, MPI, CMake options). + +Why not the others: the Spack `warpx` recipe stops at 26.08 and takes the +architecture only through the legacy `^amrex cuda_arch=` path; the local Spack +checkout is 2025-05 (warpx 25.04); no Apptainer on the node and the only +upstream container recipes are Perlmutter-specific (sm_80); site modules are +broken. HIP: `build.sh HIP` carries `-DWarpX_COMPUTE=HIP -DAMReX_AMD_ARCH=gfx950` +and exits with a clear message here (no ROCm). **Untested.** + +## Changes from upstream + +Class **A -- no source modification.** `run.sh` writes a derived inputs file +into the build tree: upstream's physics/numerics lines verbatim, plus +`warpx.numprocs` (one box per rank), the per-mode `amr.n_cell` / +`amr.max_grid_size` / `max_step`, `warpx.random_seed = 1`, and reduced +diagnostics instead of the plotfile/checkpoint diagnostics (I/O). For the +validation case only the openPMD diagnostic is dropped (openPMD not built). + +## Execution model + +One MPI rank per GPU (AMReX docs: "MPI ranks == Number of GPUs"). The common +launcher's per-rank wrapper gives each rank one visible GPU; AMReX then binds +device 0, and the launcher audits the mapping (4/4 verified at 4 GPUs). +`warpx.numprocs PX PY PZ` decomposes the domain into exactly one box per +rank, so the requested rank count is the decomposition (`PX*PY*PZ == ranks`, +each per-rank extent a multiple of the blocking factor; otherwise the run is +refused, nothing silently changed). GPU-aware MPI: AMReX auto-detects the +CUDA-aware Open MPI (`MPIX_Query_cuda_support`) and uses device buffers; +`HPCPERF_WARPX_GPU_AWARE=0` forces pinned-host staging (see the measurement +below). CPU binding: runtime default (no OpenMP threads used). + +## Inputs (`HPCPERF_SCALE_MODE`, case `uniform_plasma`) + +| Mode | Grid (cells) | Macroparticles (2/cell) | Per rank @4 GPU | Topology (numprocs) | Steps | Memory/GPU (est.) | Time/step on B200 | Validation quantity | +|---|---|---|---|---|---|---|---|---| +| smoke (default) | 64x32x32 (upstream) | 131,072 | 32,768 | 2x2x1 | 10 | < 0.1 GB | 0.02 s | particle number (exact) + energies recorded | +| strong | G^3, G=`HPCPERF_WARPX_GLOBAL` (256) | 33,554,432 | 8,388,608 | 2x2x1 (128x128x256 boxes) | 20 | ~4 GB | 0.029 s (1 GPU) / 0.081 s (4 GPU, GPU-aware MPI) / 0.026 s (4 GPU, host-staged) | run completes; reduced diagnostics | +| weak | (L*PX)x(L*PY)x(L*PZ), L=`HPCPERF_WARPX_LOCAL` (128) | 4,194,304 x N | 4,194,304 | `hpcperf_topology.py` grid | 20 | ~1 GB | 0.050 s (4 GPU) | run completes | + +Memory estimate ~100 B per particle plus 6 field components (double) per cell +-- far below 180 GB at all sizes. The strong default is a correctness/bring-up +size (33.6M particles, 20 steps in under a second); per-step times come from +WarpX's own `Evolve time ... Avg. per step` output and exclude initialisation +(`warpx.serialize_initial_conditions = 1`, upstream default in this deck, +serialises particle initialisation across ranks, so `Total Time` grows with +the rank count and must not be read as a scaling number). + +**Transport observation (single node, `pml ob1 / btl self,sm,smcuda`):** the +4-GPU strong run takes 0.081 s/step with AMReX's auto-enabled GPU-aware MPI +and 0.026 s/step with `amrex.use_gpu_aware_mpi = 0`; the 1-GPU run takes +0.029 s/step. Device-buffer MPI through `smcuda` is therefore the bottleneck +for WarpX's halo/particle exchange on this node (LAMMPS shows the opposite: +`gpu/aware on` 2.38 s vs `off` 5.87 s; SPARTA is indifferent). Recorded as a +site/transport finding; the default stays upstream's (auto-detect), correctness +is unaffected. + +## Validation (`validate.sh`, upstream mechanism) + +Upstream's regression mechanism is checksums with baselines that upstream +documents as architecture-dependent, so they cannot serve as a reference on a +B200. `validate.sh` therefore runs upstream's **analytic** regression test +`test_3d_langmuir_multi` (`Examples/Tests/langmuir/inputs_base_3d`: electron ++ positron Langmuir wave, 64^3 cells, 524,288 particles, 40 steps) and +re-implements `analysis_3d.py`'s checks on the final plotfile (read directly; +upstream's script needs yt/openPMD-viewer, absent here): + +1. `max|E_sim - E_th| / max|E_th| < 5e-2` for Ex, Ey, Ez against the + analytic solution `E = eps m_e c^2 k/e sin(kx) cos(ky) cos(kz) sin(wp t)` + (and cyclic), evaluated at cell centres exactly as upstream does; +2. charge conservation `max|divE - rho/eps0| / max|rho/eps0| < 1e-11` + (upstream's tolerance for Esirkepov deposition), with WarpX's own CODATA + 2022 constants (`Source/ablastr/constant.H`; using CODATA 2018 eps0 would + show a spurious 6.8e-10 offset); +3. the `uniform_plasma` smoke run: macroparticle number constant at every + step (exact); the particle+field energy series is recorded for information + only -- the shipped 2-particles-per-cell thermal plasma with zero initial + fields is not an energy-conservation test for the momentum-conserving Yee + scheme (a few percent change in the first plasma periods is expected). + +Observed on dgx003 (2026-09-04): **PASS at 1, 2 and 4 GPUs** -- +`error_rel = 3.351e-02` for all three components and all three rank counts +(the decomposition does not change the result), charge-conservation residual +1.8e-12 / 1.9e-12 / 1.3e-12, particle number 131,072 at every step; recorded +energy change -3.45 % / -3.56 % / -3.47 % over 10 steps (initial energies +differ by the per-rank random sampling, as expected). + +## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) + +| Run | Ranks x GPUs | rank->GPU | CPU binding | Topology | Problem | Time | Validation | +|---|---|---|---|---|---|---|---| +| langmuir | 1 x 1 | wrapper; audit 1/1 verified | runtime default | 1x1x1 | 64^3, 524k particles, 40 steps | Total 0.58 s | PASS | +| langmuir | 2 x 2 | wrapper; 2/2 verified | runtime default | 2x1x1 | same | Total 0.70 s | PASS | +| langmuir | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2x1 | same | Total 0.67 s | PASS | +| smoke | 1/2/4 | wrapper; verified where sampled | runtime default | 1x1x1 / 2x1x1 / 2x2x1 | 64x32x32, 131k particles, 10 steps | Total 0.21-0.35 s | particle number exact | +| strong | 1 x 1 | wrapper; 1/1 verified | runtime default | 1x1x1 | 256^3, 33.6M particles, 20 steps | 0.0289 s/step | completes | +| strong | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2x1 | 256^3, 33.6M particles | 0.0811 s/step (GPU-aware) / 0.0258 s/step (host-staged) | completes | +| weak | 4 x 4 | wrapper; 4/4 verified | runtime default | 2x2x1 | 256x256x128, 16.8M particles (4.2M/rank) | 0.0500 s/step | completes | + +Dry-runs (`HPCPERF_DRY_RUN=1`, hypothetical allocations) -- **DRY-RUN / +UNVALIDATED**, nothing executed: + +| GPUs | Nodes x GPUs/node | Mode | Grid | Particles | Per rank | numprocs | Launch | +|---|---|---|---|---|---|---|---| +| 8 | 1 x 8 | strong | 256^3 | 33.6M | 4.2M | 2x2x2 (128^3 boxes) | `mpirun -np 8 --host dgx003:8 --map-by ppr:8:node ...` (single node) | +| 40 | 5 x 8 | weak | 640x512x256 | 167.8M | 4.2M | 5x4x2 | 5 nodes x 8 -- multi-node BLOCKED on this site | +| 80 | 10 x 8 | weak | 640x512x512 | 335.5M | 4.2M | 5x4x4 | 10 nodes x 8 -- multi-node BLOCKED on this site | + +## Limitations + +- Multi-node: BLOCKED/UNVERIFIED on this site; 40/80-GPU shapes are plans. +- HIP: recipe present, untested (no AMD GPU); no gfx950 statement upstream. +- openPMD/HDF5 output, QED and Python bindings are not built (documented + options, off for bring-up). +- Upstream's checksum baselines are not used (platform-dependent by + upstream's own statement); validation is the analytic Langmuir test plus + charge and particle conservation. diff --git a/level3/warpx/build.sh b/level3/warpx/build.sh new file mode 100755 index 0000000..aebacc9 --- /dev/null +++ b/level3/warpx/build.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Build WarpX (3D, MPI, CUDA or HIP) with upstream's native CMake superbuild, +# using the AMReX 26.09 checkout WarpX pins (no configure-time downloads). +# +# ./build.sh [CUDA|HIP] (default CUDA) +# +# Layout (Level 3 isolation): sources _upstream/level3/{WarpX,amrex}, build +# build/level3/warpx/, install .deps/level3/warpx/install +# (+ .hpcperf-l3-fingerprint), logs .deps/level3/warpx/logs. AMReX is built +# by WarpX's superbuild from the local source (-DWarpX_amrex_src) -- it is +# WarpX's private copy, nothing is shared with other Level 3 applications. +# +# Configuration (bring-up subset of the documented options): WarpX_COMPUTE=CUDA, +# CMAKE_CUDA_ARCHITECTURES=100 (sm_100), WarpX_DIMS=3, WarpX_MPI=ON, +# WarpX_OPENPMD=OFF (plotfile output only; no HDF5/ADIOS2), WarpX_QED=OFF +# (PICSAR-QED not needed by the uniform-plasma/laser benchmarks; avoids a +# download), WarpX_PYTHON=OFF, WarpX_FFT=OFF (no PSATD). Host compiler conda +# GCC 13.3.0 (upstream: GCC 12+, NVCC 12.4+; upstream's Perlmutter profile +# uses GCC 13 with NVCC 13.2.78). Modification class: A (build options only). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +SRC="$R/_upstream/level3/WarpX"; AMREX="$R/_upstream/level3/amrex" +[ -f "$SRC/CMakeLists.txt" ] && [ -f "$AMREX/CMakeLists.txt" ] || { echo "build.sh: sources missing -- run $HERE/fetch.sh first" >&2; exit 1; } +SHA="$(git -C "$SRC" rev-parse HEAD)"; AMREX_SHA="$(git -C "$AMREX" rev-parse HEAD)" +l3_paths warpx +BUILD_DIR="$R/build/level3/warpx/$MODEL" +JOBS="${HPCPERF_BUILD_JOBS:-32}" + +case "$BACKEND" in + CUDA) + ARCH="${HPCPERF_CUDA_ARCH:-$(l3_gpu_arch)}" + GPU_FLAGS=(-DWarpX_COMPUTE=CUDA "-DCMAKE_CUDA_ARCHITECTURES=$ARCH" "-DCMAKE_CUDA_HOST_COMPILER=$CXX") + ARCHNOTE="sm_$ARCH" ;; + HIP) + command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + ARCH="${HPCPERF_HIP_ARCH:-gfx950}" + GPU_FLAGS=(-DWarpX_COMPUTE=HIP "-DAMReX_AMD_ARCH=$ARCH" -DCMAKE_CXX_COMPILER=hipcc) + ARCHNOTE="$ARCH" ;; + *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; +esac + +CMAKE_OPTS="WarpX_COMPUTE=$BACKEND arch=$ARCHNOTE WarpX_DIMS=3 WarpX_MPI=ON WarpX_OPENPMD=OFF WarpX_QED=OFF WarpX_PYTHON=OFF WarpX_FFT=OFF WarpX_amrex_src=local BUILD_TESTING=OFF" +FP="$(l3_fingerprint_text warpx "$SHA" "$MODEL" "amrex=26.09($AMREX_SHA) picsar-qed=off openpmd=off" "$CMAKE_OPTS" "runtime(amrex.use_gpu_aware_mpi auto)")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +echo "# WarpX $BACKEND: upstream $SHA, AMReX $AMREX_SHA (26.09), arch $ARCHNOTE, MPI $(mpirun --version 2>/dev/null | head -1)" +mkdir -p "$BUILD_DIR" +cmake -S "$SRC" -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$L3_INSTALL" \ + -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ + -DWarpX_DIMS=3 -DWarpX_MPI=ON -DWarpX_OPENPMD=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=OFF -DWarpX_FFT=OFF \ + -DWarpX_APP=ON -DWarpX_LIB=OFF -DWarpX_amrex_src="$AMREX" -DBUILD_TESTING=OFF \ + "${GPU_FLAGS[@]}" > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ + || { tail -40 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } +t0=$(date +%s) +cmake --build "$BUILD_DIR" -j "$JOBS" > "$L3_LOGS/build-$MODEL.log" 2>&1 \ + || { tail -40 "$L3_LOGS/build-$MODEL.log"; echo "build.sh: build failed (log: $L3_LOGS/build-$MODEL.log)" >&2; exit 1; } +cmake --install "$BUILD_DIR" > "$L3_LOGS/install-$MODEL.log" 2>&1 || { echo "build.sh: install failed" >&2; exit 1; } +l3_fingerprint_write "$L3_INSTALL" "$FP" +EXE="$(find "$BUILD_DIR/bin" -maxdepth 1 -name 'warpx.3d*' -type f 2>/dev/null | head -1)" +echo "# built in $(( $(date +%s)-t0 )) s: ${EXE:-} (installed under $L3_INSTALL)" +echo "# compiler warning lines: $(grep -c 'warning' "$L3_LOGS/build-$MODEL.log" || true)" diff --git a/level3/warpx/fetch.sh b/level3/warpx/fetch.sh new file mode 100755 index 0000000..2838cef --- /dev/null +++ b/level3/warpx/fetch.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Fetch WarpX and the AMReX release it pins into _upstream/level3/{WarpX,amrex} +# (gitignored, read-only). WarpX 26.09 pins AMReX 26.09 (cmake/dependencies/ +# AMReX.cmake / dependencies.json); the AMReX checkout is used as +# -DWarpX_amrex_src so no configure-time download is needed. Idempotent; a +# checkout at another commit is an error, never silently reused. +# +# ./fetch.sh +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +# Official repository (ECP-WarpX/WarpX redirects here since the project moved). +WARPX_URL="https://github.com/BLAST-WarpX/warpx.git"; WARPX_TAG="26.09"; WARPX_SHA="0c62c75e53a9ad08241535444bd7e53fd1deba88" +AMREX_URL="https://github.com/AMReX-Codes/amrex.git"; AMREX_TAG="26.09"; AMREX_SHA="a52ca73324ac2c7b65ec04f131e6df99eec9c576" + +fetch_one() { # url tag sha dest + local url=$1 tag=$2 sha=$3 dest=$4 have + if [ -d "$dest/.git" ]; then + have="$(git -C "$dest" rev-parse HEAD)" + [ "$have" = "$sha" ] && { echo "fetch.sh: $dest already at $tag ($sha)"; return 0; } + echo "fetch.sh: $dest is at $have, not the recorded $sha ($tag); remove it to re-fetch" >&2; return 1 + fi + mkdir -p "$(dirname "$dest")" + echo "fetch.sh: cloning $url @ $tag (shallow)" + git clone --quiet --depth 1 --branch "$tag" "$url" "$dest" + have="$(git -C "$dest" rev-parse HEAD)" + [ "$have" = "$sha" ] || { echo "fetch.sh: tag $tag resolved to $have, expected $sha" >&2; return 1; } + echo "fetch.sh: ok -> $dest ($sha)" +} +fetch_one "$WARPX_URL" "$WARPX_TAG" "$WARPX_SHA" "$R/_upstream/level3/WarpX" +fetch_one "$AMREX_URL" "$AMREX_TAG" "$AMREX_SHA" "$R/_upstream/level3/amrex" diff --git a/level3/warpx/run.sh b/level3/warpx/run.sh new file mode 100755 index 0000000..2aec1d8 --- /dev/null +++ b/level3/warpx/run.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Run WarpX (full 3D electromagnetic PIC: deposition, Maxwell/Yee solve, +# particle push, periodic halo/particle exchange) on N GPUs. +# +# ./run.sh [CUDA|HIP] [extra WarpX inputs overrides...] +# +# Cases (HPCPERF_WARPX_CASE): +# uniform_plasma (default) -- Examples/Physics_applications/uniform_plasma/ +# inputs_base_3d, upstream's "commonly used to study performance" case: +# thermal electron plasma, 2 particles/cell. Sized by HPCPERF_SCALE_MODE. +# langmuir -- Examples/Tests/langmuir/inputs_base_3d (test_3d_langmuir_multi): +# electron/positron Langmuir wave with an analytic solution, 64^3 cells, +# 40 steps; the correctness case used by validate.sh (sizes fixed). +# +# Execution model (AMReX/WarpX docs): one MPI rank per GPU. The common +# launcher's per-rank wrapper gives each rank exactly one visible GPU (AMReX +# then binds device 0 = that GPU; mapping audited). `warpx.numprocs PX PY PZ` +# assigns exactly one box per rank, so the requested rank count is what is +# decomposed: PX*PY*PZ must equal HPCPERF_GPUS and every dimension of +# amr.n_cell must be divisible by PX*blocking_factor -- otherwise the run is +# refused with the nearby legal rank counts (nothing is silently changed). +# GPU-aware MPI: AMReX auto-detects it from the (CUDA-aware) Open MPI; +# HPCPERF_WARPX_GPU_AWARE=0 forces host-staged communication. +# +# Resource / size controls (uniform_plasma): +# HPCPERF_GPUS=N|all ranks = GPUs (default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# smoke : upstream grid 64x32x32 cells (131,072 macroparticles), 10 steps +# strong : ONE fixed global grid G^3 (G=HPCPERF_WARPX_GLOBAL, default 256: +# 16.8M cells, 33.6M particles), decomposed over the ranks +# weak : fixed per-rank block L^3 (L=HPCPERF_WARPX_LOCAL, default 128: +# 2.1M cells, 4.2M particles per rank); grid L*PX x L*PY x L*PZ +# HPCPERF_WARPX_STEPS time steps (default 10; strong/weak 20) +# HPCPERF_WARPX_GPU_AWARE 1|0 (default: AMReX auto-detect) +# +# Derived inputs (class A, written into the build tree; upstream files +# untouched): upstream physics/numerics lines are kept verbatim; +# uniform_plasma drops the plotfile/checkpoint diagnostics (I/O), sets +# amr.n_cell / amr.max_grid_size / max_step / warpx.numprocs per mode, +# warpx.random_seed = 1 (reproducible sampling) and adds reduced diagnostics +# (ParticleEnergy, FieldEnergy, ParticleNumber); langmuir keeps upstream's +# diag1 plotfile (the analysis input), drops only the openPMD diagnostic +# (openPMD is not built) and adds warpx.numprocs. stdout is copied to +# /stdout.log. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +BUILD_DIR="$R/build/level3/warpx/$MODEL" +EXE="$(find "$BUILD_DIR/bin" -maxdepth 1 -name 'warpx.3d*' -type f 2>/dev/null | head -1)" +[ -n "$EXE" ] && [ -x "$EXE" ] || { echo "run.sh: warpx.3d* not found under $BUILD_DIR/bin -- run ./build.sh $BACKEND first" >&2; exit 1; } +CASE="${HPCPERF_WARPX_CASE:-uniform_plasma}" +case "$CASE" in + uniform_plasma) BASE="$R/_upstream/level3/WarpX/Examples/Physics_applications/uniform_plasma/inputs_base_3d" ;; + langmuir) BASE="$R/_upstream/level3/WarpX/Examples/Tests/langmuir/inputs_base_3d" ;; + *) echo "run.sh: HPCPERF_WARPX_CASE must be uniform_plasma or langmuir" >&2; exit 2 ;; +esac +[ -f "$BASE" ] || { echo "run.sh: $BASE missing (run fetch.sh)" >&2; exit 1; } + +N_RANKS="$(hpcperf_ranks warpx yes)" || exit 2 +hpcperf_forbid_args warpx amr.n_cell amr.max_grid_size amr.blocking_factor warpx.numprocs max_step warpx.random_seed -- "$@" || exit 2 + +if [ "$CASE" = langmuir ]; then + MODE=validate; NX=64; NY=64; NZ=64; STEPS=40; BF=8 # upstream test as shipped (default blocking factor) + TOPO="$(hpcperf_topology warpx "$N_RANKS" --divides "$((NX / BF)),$((NY / BF)),$((NZ / BF))")" || exit 2 +else + MODE="$(l3_scale_mode warpx)" || exit 2 + BF=16 + case "$MODE" in + smoke) NX=64; NY=32; NZ=32; STEPS="${HPCPERF_WARPX_STEPS:-10}" ;; + strong) G="${HPCPERF_WARPX_GLOBAL:-256}"; NX=$G; NY=$G; NZ=$G; STEPS="${HPCPERF_WARPX_STEPS:-20}" ;; + weak) L="${HPCPERF_WARPX_LOCAL:-128}"; STEPS="${HPCPERF_WARPX_STEPS:-20}" + [ $((L % BF)) -eq 0 ] || { echo "run.sh: HPCPERF_WARPX_LOCAL=$L must be a multiple of the blocking factor $BF" >&2; exit 2; } ;; + esac + if [ "$MODE" = weak ]; then + TOPO="$(hpcperf_topology warpx "$N_RANKS")" || exit 2 + read -r PX PY PZ <<< "$TOPO"; NX=$((L * PX)); NY=$((L * PY)); NZ=$((L * PZ)) + else + TOPO="$(hpcperf_topology warpx "$N_RANKS" --divides "$((NX / BF)),$((NY / BF)),$((NZ / BF))")" || { + echo "run.sh: $N_RANKS ranks cannot tile the ${NX}x${NY}x${NZ} grid into one ${BF}-aligned box per rank (see feasible counts above)" >&2; exit 2; } + fi +fi +read -r PX PY PZ <<< "$TOPO" +BX=$((NX / PX)); BY=$((NY / PY)); BZ=$((NZ / PZ)) +MGS=$BX; [ "$BY" -gt "$MGS" ] && MGS=$BY; [ "$BZ" -gt "$MGS" ] && MGS=$BZ +CELLS=$((NX * NY * NZ)) +if [ "$CASE" = langmuir ]; then PARTS=$((2 * CELLS)); else PARTS=$((2 * CELLS)); fi + +RUN_DIR="$BUILD_DIR/run/$CASE.$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR" +IN="$RUN_DIR/inputs" +{ + echo "# derived from upstream $(realpath --relative-to="$R/_upstream/level3/WarpX" "$BASE") (HPC-Performance-AI level3/warpx/run.sh)" + if [ "$CASE" = langmuir ]; then + grep -vE '^\s*(amr\.max_grid_size|diagnostics\.diags_names|openpmd\.)' "$BASE" + echo "diagnostics.diags_names = diag1" + echo "amr.max_grid_size = $MGS" + else + grep -vE '^\s*(max_step|amr\.n_cell|amr\.max_grid_size|amr\.blocking_factor|diagnostics\.|diag1\.|chk\.)' "$BASE" + echo "max_step = $STEPS" + echo "amr.n_cell = $NX $NY $NZ" + echo "amr.max_grid_size = $MGS" + echo "amr.blocking_factor = $BF" + echo "warpx.random_seed = 1" + # no plotfile/checkpoint diagnostics (the upstream diag1/chk lines were dropped above) + echo "warpx.reduced_diags_names = EP EF NP" + echo "EP.type = ParticleEnergy"; echo "EP.intervals = 1" + echo "EF.type = FieldEnergy"; echo "EF.intervals = 1" + echo "NP.type = ParticleNumber"; echo "NP.intervals = 1" + fi + echo "warpx.numprocs = $PX $PY $PZ" + [ -n "${HPCPERF_WARPX_GPU_AWARE:-}" ] && echo "amrex.use_gpu_aware_mpi = $HPCPERF_WARPX_GPU_AWARE" +} > "$IN" + +echo "# WarpX $BACKEND: case=$CASE mode=$MODE ranks=$N_RANKS grid=${NX}x${NY}x${NZ} ($CELLS cells, $PARTS particles, $((PARTS / N_RANKS))/rank) numprocs=${PX}x${PY}x${PZ} box=${BX}x${BY}x${BZ} steps=$STEPS run_dir=$RUN_DIR" +cd "$RUN_DIR" +"$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- "$EXE" "$IN" "$@" 2>&1 | tee "$RUN_DIR/stdout.log" +exit "${PIPESTATUS[0]}" diff --git a/level3/warpx/validate.sh b/level3/warpx/validate.sh new file mode 100755 index 0000000..47d794e --- /dev/null +++ b/level3/warpx/validate.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Correctness check for WarpX on N GPUs, using upstream's analytic Langmuir-wave +# regression test plus an invariant of the performance case. +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) +# +# [1] test_3d_langmuir_multi (Examples/Tests/langmuir): an electron/positron +# plasma wave whose fields are known analytically, +# Ex = eps m_e c^2 kx/e sin(kx x) cos(ky y) cos(kz z) sin(wp t) (and cyclic), +# 64^3 cells, 40 steps. Upstream's analysis_3d.py compares the cell-centred +# Ex/Ey/Ez of the final plotfile with this solution and requires +# max|E_sim - E_th| / max|E_th| < 5e-2 for each component, and (Esirkepov +# deposition) charge conservation max|divE - rho/eps0| / max|rho/eps0| < +# 1e-11. The same checks are re-implemented here (upstream's script needs +# yt/openPMD-viewer, not available in this environment): the plotfile is +# read directly (AMReX native format), the formulas, grid positions and +# tolerances are upstream's. This is architecture-independent, unlike +# upstream's checksum baselines (documented as platform-dependent). +# [2] uniform_plasma smoke run (the performance case): the macroparticle count +# must be constant at every step (periodic box, no ionisation) -- exact; the +# particle+field energy time series is recorded for information (the +# shipped 2-particles-per-cell thermal plasma with E=0 initial fields is +# not an energy-conservation test: a few % change over the first plasma +# periods is expected for the momentum-conserving Yee scheme). +# PASS = [1] both criteria on N GPUs and [2] exact particle conservation. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # conda python3 + numpy for the analysis +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" +N="${HPCPERF_GPUS:-1}" +RUNS="$R/build/level3/warpx/$MODEL/run" +python3 -c 'import numpy' 2>/dev/null || { echo "validate.sh: python3 with numpy required for the plotfile analysis" >&2; exit 1; } +export HPCPERF_GPUS="$N" +ok=1 + +echo "validate.sh: [1] WarpX $BACKEND langmuir_multi (64^3, 40 steps, analytic solution) on $N GPU(s)" +HPCPERF_WARPX_CASE=langmuir "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' || true +PLT="$RUNS/langmuir.validate.np$N/diags/diag1000040" +[ -f "$PLT/Header" ] || { echo "validate.sh: FAIL -- plotfile $PLT not produced"; exit 1; } +python3 - "$PLT" <<'PY' || ok=0 +import sys, re, numpy as np +# WarpX's own constants (Source/ablastr/constant.H, CODATA 2022; scipy >= 1.15 as used by upstream's +# analysis_3d.py carries the same values). With CODATA 2018 eps0 the divE - rho/eps0 residual would show a +# spurious uniform 6.8e-10 offset (= the eps0 revision), 68x upstream's 1e-11 tolerance. +c, e, epsilon_0, m_e = 299792458.0, 1.602176634e-19, 8.8541878188e-12, 9.1093837139e-31 +plt = sys.argv[1] +# ---- AMReX plotfile reader (single level, cell-centred data) ---- +hdr = open(f"{plt}/Header").read().split("\n") +ncomp = int(hdr[1]); names = hdr[2:2 + ncomp]; i = 2 + ncomp +dim = int(hdr[i]); time = float(hdr[i + 1]); i += 3 +lo = [float(v) for v in hdr[i].split()]; hi = [float(v) for v in hdr[i + 1].split()] +dom = re.search(r"\(\((\d+),(\d+),(\d+)\) \((\d+),(\d+),(\d+)\)", hdr[i + 3]) +n = [int(dom.group(k + 4)) - int(dom.group(k + 1)) + 1 for k in range(3)] +ch = open(f"{plt}/Level_0/Cell_H").read().split("\n") +boxes = [tuple(int(v) for v in m.groups()) for m in re.finditer(r"\(\((-?\d+),(-?\d+),(-?\d+)\) \((-?\d+),(-?\d+),(-?\d+)\) \(", "\n".join(ch))] +fabs = [(m.group(1), int(m.group(2))) for m in re.finditer(r"FabOnDisk: (\S+) (\d+)", "\n".join(ch))] +assert len(boxes) == len(fabs) > 0, (len(boxes), len(fabs)) +data = np.zeros((ncomp, n[0], n[1], n[2])) +for (lx, ly, lz, hx, hy, hz), (fname, off) in zip(boxes, fabs): + with open(f"{plt}/Level_0/{fname}", "rb") as f: + f.seek(off); line = b"" + while not line.endswith(b"\n"): line += f.read(1) + h = line.decode() + # "FAB ((8, (64 11 52 0 1 12 0 1023)),(8, (8 7 6 5 4 3 2 1)))((lo) (hi) (0,0,0)) ncomp": the second + # descriptor is the byte order of the 8-byte reals (8 7 ... 1 = little endian) + order = re.search(r"\(\d+, \((\d)(?: \d){7}\)\)\)", h).group(1) + dt = "f8" + nc = int(h.strip().split()[-1]) + shape = (hx - lx + 1, hy - ly + 1, hz - lz + 1) + arr = np.frombuffer(f.read(8 * nc * np.prod(shape)), dtype=dt).reshape((nc, shape[2], shape[1], shape[0])).transpose(0, 3, 2, 1) + data[:, lx:hx + 1, ly:hy + 1, lz:hz + 1] = arr +comp = {nm: data[k] for k, nm in enumerate(names)} +# ---- upstream analysis_3d.py, verbatim parameters ---- +epsilon, nden = 0.01, 4.0e24 +Ncell = n +kx, ky, kz = [2.0 * np.pi * 2 / (hi[d] - lo[d]) for d in range(3)] +wp = np.sqrt(nden * e**2 / (m_e * epsilon_0)) +k = {"Ex": kx, "Ey": ky, "Ez": kz}; cos = {"Ex": (0, 1, 1), "Ey": (1, 0, 1), "Ez": (1, 1, 0)} +def contrib(is_cos, kk, d): + du = (hi[d] - lo[d]) / Ncell[d]; u = lo[d] + du * (0.5 + np.arange(Ncell[d])) + return np.cos(kk * u) if is_cos else np.sin(kk * u) +def theory(field, t): + amp = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) + cf = cos[field] + return amp * contrib(cf[0], kx, 0)[:, None, None] * contrib(cf[1], ky, 1)[None, :, None] * contrib(cf[2], kz, 2)[None, None, :] +print(f" plotfile time t = {time:.6e} s (wp t = {wp*time:.4f}), grid {n}, {len(boxes)} box(es), fields {names[:6]}...") +ok = True; err = 0.0 +for fld in ("Ex", "Ey", "Ez"): + th = theory(fld, time); m = abs(comp[fld] - th).max() / abs(th).max(); err = max(err, m) + print(f" {fld}: max|E_sim-E_th|/max|E_th| = {m:.3e}") +print(f" error_rel = {err:.3e} (upstream tolerance_rel 5e-2) {'ok' if err < 5e-2 else 'BAD'}"); ok &= err < 5e-2 +rho, divE = comp["rho"], comp["divE"] +ce = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax(np.abs(rho / epsilon_0)) +print(f" charge conservation max|divE-rho/eps0|/max|rho/eps0| = {ce:.3e} (upstream tolerance 1e-11) {'ok' if ce < 1e-11 else 'BAD'}"); ok &= ce < 1e-11 +sys.exit(0 if ok else 1) +PY + +echo "validate.sh: [2] WarpX $BACKEND uniform_plasma smoke (64x32x32, 131,072 particles) on $N GPU(s)" +HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' || true +D="$RUNS/uniform_plasma.smoke.np$N/diags/reducedfiles" +[ -f "$D/NP.txt" ] || { echo "validate.sh: FAIL -- reduced diagnostics not produced under $D"; exit 1; } +python3 - "$D" <<'PY' || ok=0 +import sys +d = sys.argv[1] +def load(p): return [[float(x) for x in l.split()] for l in open(p) if l.strip() and not l.startswith('#')] +npart = load(f"{d}/NP.txt"); ep = load(f"{d}/EP.txt"); ef = load(f"{d}/EF.txt") +vals = sorted(set(r[2] for r in npart)) +print(f" ParticleNumber over steps {int(npart[0][0])}..{int(npart[-1][0])}: {vals} -> {'ok (exact)' if len(vals) == 1 else 'BAD'}") +e0 = ep[0][2] + ef[0][2]; e1 = ep[-1][2] + ef[-1][2] +print(f" for the record: E_particles+E_fields = {e0:.6e} J at step {int(ep[0][0])}, {e1:.6e} J at step {int(ep[-1][0])} (rel change {(e1-e0)/e0:+.3e}; not a pass/fail criterion, see header)") +sys.exit(0 if len(vals) == 1 else 1) +PY + +if [ "$ok" -eq 1 ]; then echo "WarpX $BACKEND validation ($N GPU, langmuir_multi analytic + charge conservation, particle conservation): PASS"; exit 0; fi +echo "WarpX $BACKEND validation ($N GPU): FAIL"; exit 1 diff --git a/tools/runtime/README.md b/tools/runtime/README.md new file mode 100644 index 0000000..65ded28 --- /dev/null +++ b/tools/runtime/README.md @@ -0,0 +1,34 @@ +# Shared runtime tools -- commonization proposal (not yet moved) + +The GPU-count-aware launcher and its helpers currently live under +`level2/tools/` and are validated there: + +| Tool | Role | +|---|---| +| `level2/tools/hpcperf_mpi_launch.sh` | allocation parser, `HPCPERF_GPUS` selection, CPU binding, site profiles, GPU binding audit | +| `level2/tools/mpi_gpu_bind.sh` | scheduler-safe one-GPU-per-rank wrapper | +| `level2/tools/hpcperf_topology.py` | process-grid helper with constraint checking | +| `level2/tools/hpcperf_launch_common.sh` | `run.sh` helpers (rank resolution, argument guards) | +| `level2/tools/site/*.sh` | site profiles (transport, launcher choice) | +| `level2/tools/tests/` | regression tests | + +Level 3 uses the same semantics (`HPCPERF_GPUS=N|all`, `HPCPERF_NODES`, +`HPCPERF_GPUS_PER_NODE`, `HPCPERF_CPUS_PER_RANK`, `HPCPERF_SCALE_MODE`, +`HPCPERF_SITE_PROFILE`, `HPCPERF_DRY_RUN=1`) and MUST not fork them. + +## Minimal commonization plan + +1. **Now (this round):** Level 3 scripts reference the tools through one + variable, `HPCPERF_RUNTIME_DIR` (default `level2/tools`), set in + `level3/tools/l3_common.sh`. No file is moved or copied; Level 2 is not + disturbed. +2. **Next (separate PR):** `git mv level2/tools/{hpcperf_mpi_launch.sh, + mpi_gpu_bind.sh,hpcperf_topology.py,hpcperf_launch_common.sh,site,tests} + tools/runtime/`, leave thin forwarding shims at the old paths for one + release (`exec "$(dirname "$0")/../../tools/runtime/" "$@"`), switch + the `HPCPERF_RUNTIME_DIR` default to `tools/runtime`, re-run + `tools/runtime/tests/run_all.sh` plus the Level 2 4-GPU smoke set before + merging. +3. Only then remove the shims. + +Until step 2 lands, `tools/runtime/` holds this note only. From 366b72f7b6d0ac9b94d04b3619adbfb5468059e5 Mon Sep 17 00:00:00 2001 From: SWE-bench Date: Sat, 5 Sep 2026 18:40:00 -0400 Subject: [PATCH 02/52] Level 3 first batch: correctness/reproducibility fixes and nekRS coarse-solver decision Validators capture the real run exit code (timeout/nonzero/missing output -> FAIL), reject NaN/Inf (level3/tools/l3_check.py), require the complete step/field/trace/check sets, and write a per-run manifest. Dry-runs are routed to a .dryrun/ scratch directory and can no longer clobber real results. Fingerprint schema l3-2 records ordered patch-content hashes; the nekRS source-cache key is SHA + patch series hash. Level 3 builds strip the Level 2 .deps/install prefixes (LAMMPS/SPARTA rebuilt isolated, identical results). CPU-only negative tests: level3/tools/tests/run_all.sh (13/13). nekRS: the Ethier workload runs its HYPRE coarse solve on the CPU by default (cimode 2); the three HYPRE/Thrust patches are needed only for GPU HYPRE, which is now verified separately with cimode 3 (DEVICE coarse, 9/9 at 1/4 GPU, hypregpu variant). A patch-free cpucoarse variant (ENABLE_HYPRE_GPU=OFF) passes cimode 2 at 1/2/4 GPU and rejects a DEVICE request explicitly. Both variants are kept, isolated per variant. See level3/CORRECTNESS_FIXES.md and level3/nekrs/COMPATIBILITY.md. --- level3/APPLICATION_AUDIT.md | 2 +- level3/BUILD_STRATEGY.md | 13 +- level3/CORRECTNESS_FIXES.md | 101 ++++++++++++++ level3/README.md | 11 +- level3/lammps/build.sh | 1 + level3/lammps/run.sh | 20 ++- level3/lammps/validate.sh | 103 ++++++++------ level3/nekrs/COMPATIBILITY.md | 200 ++++++++++++++++++++++++++++ level3/nekrs/README.md | 51 ++++++- level3/nekrs/build.sh | 57 +++++--- level3/nekrs/run.sh | 25 +++- level3/nekrs/validate.sh | 77 +++++++---- level3/sparta/build.sh | 1 + level3/sparta/run.sh | 18 ++- level3/sparta/validate.sh | 109 ++++++++------- level3/specfem3d/build.sh | 1 + level3/specfem3d/run.sh | 19 ++- level3/specfem3d/validate.sh | 64 ++++++--- level3/tools/l3_check.py | 42 ++++++ level3/tools/l3_common.sh | 113 +++++++++++++++- level3/tools/tests/run_all.sh | 15 +++ level3/tools/tests/test_l3_infra.sh | 77 +++++++++++ level3/warpx/build.sh | 1 + level3/warpx/run.sh | 15 ++- level3/warpx/validate.sh | 187 ++++++++++++++------------ 25 files changed, 1068 insertions(+), 255 deletions(-) create mode 100644 level3/CORRECTNESS_FIXES.md create mode 100644 level3/nekrs/COMPATIBILITY.md create mode 100644 level3/tools/l3_check.py create mode 100755 level3/tools/tests/run_all.sh create mode 100755 level3/tools/tests/test_l3_infra.sh diff --git a/level3/APPLICATION_AUDIT.md b/level3/APPLICATION_AUDIT.md index 1878415..ce18a1b 100644 --- a/level3/APPLICATION_AUDIT.md +++ b/level3/APPLICATION_AUDIT.md @@ -219,7 +219,7 @@ candidates have an officially supported native CUDA path. - container_availability: none official; JIT couples to host toolchain anyway - spack_availability: `nekrs` package stale (23.0, 21.0; option names do not match v26.0) - recommended_integration_priority: FIRST_BATCH -- blocker: (1) conda `mpif90` needs `OMPI_FC=/usr/bin/gfortran` (class C, verified); (2) hypre SM list (class B, 1 line); (3) `build.sh` interactive -> cmake called directly; (4) `genbox` missing for kershaw weak scaling. **Found during bring-up (not visible in the audit):** (5) the vendored HYPRE 2.32.0 does not compile against the Thrust 3.2 shipped with CUDA 13 (`thrust::pair` result types, non-transitive `reverse_iterator`/`pair` headers, removed `thrust::not1`) -- ~20 mechanical class-D lines; (6) with conda GCC 13 for C/C++ and system gfortran 14, CMake's FortranCInterface detection fails on LTO bytecode versions (`-fno-lto` at link) and on PIE (`-fPIC` for Fortran); (7) HYPRE's configure takes the conda `AR` variable as the full archive command (`unset AR`); (8) the conda `CMAKE_GENERATOR=Ninja` produces an invalid rule for the HYPRE ExternalProject and breaks the run-time UDF build (upstream's Makefiles generator pinned / env unset at run time); (9) Open MPI's `osc ucx` is selected for nekRS' `MPI_Win_lock` calls and aborts in `uct_ib` with 4 ranks (`OMPI_MCA_osc=^ucx`); (10) the default 8 MB stack limit segfaults the h-refined cases in `useric` (`ulimit -s unlimited`, as upstream's job scripts). All resolved; nekRS validated at 1/2/4 GPUs +- blocker: (1) conda `mpif90` needs `OMPI_FC=/usr/bin/gfortran` (class C, verified); (2) hypre SM list (class B, 1 line); (3) `build.sh` interactive -> cmake called directly; (4) `genbox` missing for kershaw weak scaling. **Found during bring-up (not visible in the audit):** (5) the vendored HYPRE 2.32.0 does not compile against the Thrust 3.2 shipped with CUDA 13 (`thrust::pair` result types, non-transitive `reverse_iterator`/`pair` headers, removed `thrust::not1`) -- ~20 mechanical class-D lines; (6) with conda GCC 13 for C/C++ and system gfortran 14, CMake's FortranCInterface detection fails on LTO bytecode versions (`-fno-lto` at link) and on PIE (`-fPIC` for Fortran); (7) HYPRE's configure takes the conda `AR` variable as the full archive command (`unset AR`); (8) the conda `CMAKE_GENERATOR=Ninja` produces an invalid rule for the HYPRE ExternalProject and breaks the run-time UDF build (upstream's Makefiles generator pinned / env unset at run time); (9) Open MPI's `osc ucx` is selected for nekRS' `MPI_Win_lock` calls and aborts in `uct_ib` with 4 ranks (`OMPI_MCA_osc=^ucx`); (10) the default 8 MB stack limit segfaults the h-refined cases in `useric` (`ulimit -s unlimited`, as upstream's job scripts). All resolved; nekRS validated at 1/2/4 GPUs. **CUDA/dependency decision (post-review, see `level3/nekrs/COMPATIBILITY.md`):** the ethier benchmark runs the HYPRE BoomerAMG coarse solve on the CPU by default (`COARSE SOLVER LOCATION = CPU`; cimode 2) with the main app on the GPU; issues (5)-(8) are needed ONLY to build/exercise GPU HYPRE. GPU coarse is now separately verified (cimode 3 = DEVICE, 9/9, 1/4 GPU, `hypregpu` variant); a minimal `cpucoarse` variant (`ENABLE_HYPRE_GPU=OFF`, 0 patches, 113 s) runs the current workload and rejects a DEVICE request rather than silently down-grading - build_strategy_notes: **NATIVE** ## CP2K diff --git a/level3/BUILD_STRATEGY.md b/level3/BUILD_STRATEGY.md index 4a00cbe..f3aaa5f 100644 --- a/level3/BUILD_STRATEGY.md +++ b/level3/BUILD_STRATEGY.md @@ -16,7 +16,7 @@ APPTAINER | SPACK+APPTAINER | SITE_NATIVE | DEFER`. | SPARTA | feasible, documented (`cmake -C presets/kokkos_cuda.cmake -DKokkos_ARCH_BLACKWELL100=ON`); deps = MPI + CUDA; **built in 579 s** | **no package** (Spack's `sparta` is a bioinformatics tool) | no runtime; no recipe | lmod broken | **NATIVE** | | WarpX | feasible, documented superbuild; small graph (AMReX local checkout, PICSAR/openPMD off); CUDA vs HIP one switch; upstream itself builds with CUDA 13.2 | possible only with a fresh spack-packages (26.09 missing; arch via `^amrex cuda_arch=100` legacy path); graph balloons with `+openpmd +python` | no runtime; only Perlmutter Containerfiles (sm_80) | lmod broken | **NATIVE** | | SPECFEM3D Cartesian | feasible, only documented route (autotools; bundled SCOTCH); needs two devel back-ports for CUDA 13 + make-time `GENCODE` for sm_100; Fortran via system gfortran + `OMPI_FC` | no package | none; tiny dependency graph, nothing to gain | lmod broken | **NATIVE** | -| nekRS | feasible, only documented route (CMake; all TPLs vendored); CUDA/HIP separable by OCCA options; JIT needs host g++/nvcc/gfortran at run time anyway; 1-line hypre SM patch | recipe stale (23.0, wrong option names); deps vendored -> nothing for Spack to provide | none official; JIT couples to host toolchain | lmod broken | **NATIVE** | +| nekRS | feasible, only documented route (CMake; all TPLs vendored); CUDA/HIP separable by OCCA options; two build variants (see below) | recipe stale (23.0, wrong option names); deps vendored -> nothing for Spack to provide | none official; JIT couples to host toolchain | lmod broken | **NATIVE** | | CP2K | feasible (CMake + `install_cp2k_toolchain.sh`); ~15 packages for a GPU-DFT build; needs the DBCSR B200 sed upstream master applies; 3-6 h | officially recommended (`make_cp2k.sh`, `spack install cp2k+cuda`) **but `cp2k` and `dbcsr` recipes hard-reject `cuda_arch=100`**; 60-100 packages; local checkout too old | official `cp2k/cp2k` images stop at H100; multi-node needs host MPI; no runtime here | lmod broken | **NATIVE+SPACK_DEPS** | | Nyx | feasible, only documented path (CMake superbuild or GNU make); can consume the WarpX AMReX 26.09 checkout (`AMREX_MINIMUM_VERSION 20.11`); SUNDIALS CUDA superbuild for HEATCOOL | no package | none | lmod broken | **NATIVE** | | QMCPACK | feasible today only for `QMC_GPU=cuda` (partial GPU); the recommended `openmp;cuda` needs Clang with NVPTX offload (absent) + Boost (absent) + tested HDF5 | package's `+cuda` is inert for 4.x (no `QMC_GPU`), no offload variant -> use Spack only for `llvm+cuda`, `boost`, `hdf5@1.14` | CI dependency images only (CPU) | lmod broken | **NATIVE+SPACK_DEPS** | @@ -27,6 +27,17 @@ No candidate gets `DEFER`: every one has an officially supported native CUDA path on this toolchain. `APPTAINER`/`SPACK+APPTAINER`/`SITE_NATIVE` are unavailable on this node regardless of application. +nekRS has two verified variants (both NATIVE; select with +`HPCPERF_NEKRS_HYPRE_GPU`/`HPCPERF_NEKRS_VARIANT`; isolated src/build/install/JIT +cache; see `level3/nekrs/COMPATIBILITY.md`): +- `hypregpu` (`ENABLE_HYPRE_GPU=ON`, 3 patches) -- required for a GPU (DEVICE) + HYPRE coarse solve; verified with cimode 3 (GPU coarse) at 1/4 GPU. +- `cpucoarse` (`ENABLE_HYPRE_GPU=OFF`, 0 patches, 113 s build) -- covers the + current Ethier CPU-coarse workload; a DEVICE-coarse request is rejected by + nekRS itself, not silently down-graded. Recommended default only once the + workload's coarse-solver placement is fixed by review; the CPU-coarse option's + scalability at 40/80 GPUs is UNVERIFIED. + ## Spack policy (Level 3) - Spack is used **only** where upstream documents it as a supported route and diff --git a/level3/CORRECTNESS_FIXES.md b/level3/CORRECTNESS_FIXES.md new file mode 100644 index 0000000..338159b --- /dev/null +++ b/level3/CORRECTNESS_FIXES.md @@ -0,0 +1,101 @@ +# Level 3 first batch -- correctness / reproducibility fixes + +Round after the da35285 review. Base for this work: HEAD was +`da352857561daa8f754161a856d0d53875d6f3ad` (verified; working tree clean at +start). No application source, patch, input, or tolerance was changed to obtain +a PASS; the five applications, their patches, inputs and prior results are kept. + +Shared mechanism lives in `level3/tools/l3_common.sh` (+ `l3_check.py`); the +CPU-only tests are `level3/tools/tests/` (`run_all.sh` -> `test_l3_infra.sh`, +13/13 passing). + +## 1. False PASS / exit codes + +| Review point | Fix | Where | +|---|---|---| +| A failed run must not PASS on a stale log | validators delete/rewrite only this-run output; run.sh removes its target log before launching; validators require the run's real exit code == 0 | all `validate.sh`; `lammps/sparta run.sh` `rm -f "$LOG"` | +| SPECFEM solver failure swallowed by `\| tee \| grep \|\| true` | solver now runs to a file and its exit code is captured directly (`rc=0; cmd > log \|\| rc=$?`); the grep is display-only afterwards | `specfem3d/run.sh` | +| Separate execution from log filtering; capture launcher/app/validator real exit codes | validators run the app into a stdout file and gate on `rc`; `l3_capture` returns the command's status, not tee's; LAMMPS/SPARTA run.sh use `\|\| rc=$?` (not `; rc=$?` which `set -e` would abort) so the code and manifest are always recorded | `l3_common.sh` `l3_capture`; every `run.sh`/`validate.sh` | +| timeout / missing file / analysis exception / nonzero -> FAIL; validate only new output | `timeout` wraps each run (rc 124 -> FAIL); missing output -> FAIL; python raises `ValidationError` -> FAIL | every `validate.sh` | + +Negative test: `test_l3_infra.sh` case 3 shows a failed run (rc=1) FAILs the gate +even when a PASS-looking stale log is present; case 2 shows `l3_capture` returns +the real code (7), not tee's 0. + +## 2. Numerical finiteness / completeness + +| Review point | Fix | +|---|---| +| Reject NaN/Inf in data/reference/error | `l3_check.require_finite` names and rejects any non-finite compared quantity; used by LAMMPS/SPARTA/WarpX validators and the SPECFEM sample scan | +| LAMMPS: expected thermo fields + final step | requires Step 0 and Step 100 rows and the fields Temp/E_pair/TotEng/Press present, else FAIL | +| SPARTA: final step, complete stats interval, fields | requires the benchmark block to span steps 30..130 (equilibration boundary to final) and the fields Np/temp/Natt present | +| SPECFEM: all required reference traces, sampling range, comparison | requires the compared-trace count == number of REF_SEIS traces (12), and scans every produced `.semd` for non-finite samples before trusting the correlation | +| nekRS: complete cimode check set (not passed>0) | requires `passed+failed == EXPECT_CHECKS` (9) AND failed==0 AND rc==0 AND coarse-location matches the cimode | +| WarpX: final step, expected particle count, field completeness, reader robustness | reader FAILs on missing field, box/fab mismatch, a truncated FAB, or boxes not covering 100% of the domain; requires the langmuir plotfile at step 40 and the uniform-plasma NP series to reach step 10 with a constant finite count | +| Adapted-subset labelling | LAMMPS/SPARTA validators print "adapted subset" and name exactly what upstream check they re-implement | +| CPU-only negative tests | `test_l3_infra.sh` (NaN/Inf rejection, rc-gate, dry-run sentinel, patch/cache) | + +Thresholds unchanged (LAMMPS 1e-8/1e-5; SPARTA Np-exact/2%/15%; WarpX 5e-2/1e-11; +SPECFEM 0.8/1%/0.01s; nekRS EPS 0.3). + +## 3. Result management + +| Review point | Fix | +|---|---| +| Unique run_id; full stdout/stderr, command, exit code, source/binary/input hash | `l3_run_id` + `l3_manifest` write `run_manifest.txt` per real run with run_id, exit_code, binary+input sha256, backend, ranks, sizes, timer; the launcher already logs the command and per-rank GPU audit into the captured stdout | +| backend, GPU/rank/node, CPU/GPU binding, transport, timer, validation | recorded in the manifest and in the launcher lines of the captured stdout | +| dry-run must not delete/overwrite/rewrite real results; sentinel test | `l3_rundir` routes a dry-run to a `.dryrun/` scratch dir and refuses paths outside `build/level3/`; WarpX/SPECFEM/nekRS run.sh (which `rm -rf`'d the real dir before the dry-run check) now go through it; LAMMPS/SPARTA redirect their log into `.dryrun/`. Verified live: an 8-GPU dry-run left a real `log.smoke.np1` untouched and used `.dryrun/` (`test_l3_infra` case 4 + the live sentinel run) | +| Historical UNKNOWN exit codes stay UNKNOWN | not back-filled; the review bundle already labels them UNKNOWN | + +## 4. Fingerprint / cache + +| Review point | Fix | +|---|---| +| Patch full path + ordered content hash, not basename | `l3_fingerprint_text` records `patch[i]= sha256=` in order and a `patch_series_sha256` over the ordered contents (schema bumped l3-1 -> l3-2) | +| Missing / unhashable patch -> error | `l3_fingerprint_text` returns non-zero on a missing patch; `nekrs/build.sh` also checks each patch exists before building | +| Source-cache key includes upstream SHA + patch content hash; renamed-but-changed invalidates | nekRS src stamp is now `SHA ` (was basenames) | +| build/install/cache isolated by backend/toolchain/dependency/config | per-app `.deps/level3//{src,build,install,logs}`; nekRS is further split per variant (`hypregpu` legacy paths, others under `.deps/level3/nekrs//` with their own build dir and JIT cache) | +| Verify binary backend before running (no HIP request on a CUDA install) | `l3_binary_backend_check` (libcudart vs libamdhip64) available in `l3_common.sh` | +| Post-hoc fingerprints marked | `l3_fingerprint_write` stamps `built= (build-time record)`; nothing back-dates | + +Negative test: `test_l3_infra` case 5 (missing patch -> error; same-name changed +content -> different series hash; empty series -> `none`). + +## 5. Dependency isolation + +| Review point | Fix / finding | +|---|---| +| Check LAMMPS/SPARTA actual Kokkos helper source | The recorded (polluted-env) `CMakeCache.txt` had `Kokkos_NVCC_WRAPPER`/`Kokkos_COMPILE_LAUNCHER` pointing at **Level 2's** `.deps/install/kokkos`; the actual compile/link commands had **0** Level 2 references and used the bundled `nvcc_wrapper`+includes (so the binaries were clean, the cache entry was an inert stale detection). | +| Reconfigure without Level 2 prefixes; rebuild only affected apps | `l3_isolate_build_env` strips `$R/.deps/install/` from CMAKE_PREFIX_PATH/LD_LIBRARY_PATH in every build.sh. LAMMPS + SPARTA rebuilt in the isolated env (221 s / 570 s): `CMakeCache.txt` now has 0 Level 2 refs and `Kokkos_NVCC_WRAPPER` points at the bundled Kokkos. WarpX/nekRS/SPECFEM already had 0 refs (CXX = conda/mpicxx, autotools); the isolation call was added to their build.sh too but they were not rebuilt. Re-validated LAMMPS/SPARTA 1/2/4 -> identical numbers, PASS. | +| env_profiles base/head 9/11 tracked separately, not green | unchanged Level 2 issue (conda cmake activation drops a user CMAKE_PREFIX_PATH); documented in the review bundle `50_issues/env_profiles`; NOT a Level 3 regression and NOT marked passing here. | + +## 6. Status / build strategy + +| Review point | Fix | +|---|---| +| Separate PASS (smoke/analytic) from COMPLETED (strong/weak) | status table distinguishes validated-correctness runs from run-completed runs; strong/weak remain COMPLETED unless a numerical criterion applies | +| printed-values-equal != full bitwise | wording corrected: LAMMPS reports the four state variables agree to printed precision, not full-state bitwise identity | +| Don't widen to untested algorithm paths | LAMMPS stays LJ; WarpX stays FFT=OFF Yee PIC; no new solver paths added | +| Spack doc vs local facts; concretization NOT_RUN | recorded in the review bundle `50_issues/build_strategy` (local Spack 1.0.0.dev0, recipe versions) and `COMPATIBILITY.md`; no concretization run this round | +| Missing container runtime is an environment limit, not a feasibility verdict | stated as such; only podman present, not evaluated | + +## nekRS-specific (see COMPATIBILITY.md) + +- Confirmed the six logs' `COARSE SOLVER LOCATION: CPU` against upstream source + defaults and `ci.inc`: cimode 2 = CPU coarse (GPU main app), cimode 3 = DEVICE + (GPU) coarse. +- The cimode-2 "9/9" validates the CUDA main app + CPU coarse only. GPU HYPRE is + now separately verified with cimode 3 (`hypregpu` variant): 9/9, coarse=DEVICE, + at 1 and 4 GPUs. +- The `pair`/`reverse_iterator` build errors are missing-include (visibility) + issues; only `thrust::not1` is a genuinely removed API. Patches are labelled + project-local (no upstream backport SHA located). +- Minimal candidate `cpucoarse` (`ENABLE_HYPRE_GPU=OFF`, **0 patches**, isolated + variant tree, 113 s build): cimode 2 (CPU coarse) PASS 1/2/4 GPU (9/9, + coarse=CPU, main app on distinct GPUs); cimode 3 (DEVICE requested) is + **explicitly rejected** by nekRS (`HYPRE+DEVICE not enabled! Recompile with + -DENABLE_HYPRE_GPU=ON`, exit 1) -- no silent CPU fallback. So the three patches + are needed only for GPU HYPRE; the current CPU-coarse workload needs none. +- `hypregpu` (patched) cimode 3 (DEVICE/GPU coarse) PASS 1/4 GPU (9/9, + coarse=DEVICE): the GPU HYPRE coarse solve the patches enable is verified + correct, not merely compiled. diff --git a/level3/README.md b/level3/README.md index e7717b7..ed383f3 100644 --- a/level3/README.md +++ b/level3/README.md @@ -13,6 +13,15 @@ dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083, 2026-09-04). Nothing is claimed validated beyond what the per-application README records for runs that actually happened on this node. +Correctness / reproducibility hardening (post-review, 2026-09-05): +[CORRECTNESS_FIXES.md](CORRECTNESS_FIXES.md) -- validators now capture the real +exit code and fail on timeout/missing/non-finite output, reject NaN/Inf, require +the expected steps/fields/traces, and write a per-run manifest; dry-runs can no +longer overwrite real results; fingerprints record ordered patch-content hashes; +Level 3 builds are isolated from Level 2 prefixes. CPU-only negative tests: +`level3/tools/tests/run_all.sh` (13/13). nekRS CUDA/dependency decision: +[nekrs/COMPATIBILITY.md](nekrs/COMPATIBILITY.md). + ## Status | Application | Version | Build Strategy | CUDA Build | 1 GPU | 2 GPU | 4 GPU | HIP | Strong | Weak | Multi-node | Source Mod | Status | @@ -21,7 +30,7 @@ that actually happened on this node. | [SPARTA](sparta/README.md) | 27Aug2026 | NATIVE (bundled Kokkos 5.0.2) | OK, 579 s | PASS | PASS | PASS | untested | 10M particles, 1/4 GPU run | 1.25M particles/rank, 4 GPU run | BLOCKED/UNVERIFIED | A | FIRST_BATCH done | | [WarpX](warpx/README.md) | 26.09 (+AMReX 26.09) | NATIVE (local AMReX source) | OK, 1219 s | PASS | PASS | PASS | untested | 33.6M particles, 1/4 GPU run | 4.2M particles/rank, 4 GPU run | BLOCKED/UNVERIFIED | A (derived inputs) | FIRST_BATCH done | | [SPECFEM3D Cartesian](specfem3d/README.md) | v4.1.1 (+2 devel back-ports) | NATIVE (autotools, bundled SCOTCH) | OK, 21 s | PASS | PASS | PASS | untested | 165,888 elements, 1/4 GPU run | 165,888 elements/rank, 4 GPU run | BLOCKED/UNVERIFIED | B+C+D (18 lines, upstream devel) | FIRST_BATCH done | -| [nekRS](nekrs/README.md) | v26.0 | NATIVE (vendored OCCA/HYPRE) | OK, ~30 min | PASS | PASS | PASS | untested | 32,000 elements N=7, 1/4 GPU run | 8,000 elements/rank, 4 GPU run | BLOCKED/UNVERIFIED | B+C+D (39 lines; vendored HYPRE 2.32.0 vs CUDA 13) | FIRST_BATCH done | +| [nekRS](nekrs/README.md) | v26.0 | NATIVE (vendored OCCA/HYPRE) | OK (hypregpu ~30 min; cpucoarse 113 s) | PASS | PASS | PASS | untested | 32,000 elements N=7, 1/4 GPU run | 8,000 elements/rank, 4 GPU run | BLOCKED/UNVERIFIED | B+C+D, hypregpu variant (39 lines; vendored HYPRE 2.32.0 vs CUDA 13); cpucoarse variant 0 patches | FIRST_BATCH done; GPU-coarse verified (cimode 3), CPU-coarse candidate verified -- see [COMPATIBILITY.md](nekrs/COMPATIBILITY.md) | | CP2K | v2026.2 | NATIVE+SPACK_DEPS | not started | -- | -- | -- | -- | H2O-N series (upstream) | QS_DM_LS NREP (upstream) | -- | -- | SECOND_BATCH (deps 3-6 h; DBCSR B200 patch) | | Nyx | 26.09 | NATIVE (shared AMReX 26.09) | not started | -- | -- | -- | -- | Exec/Scaling (upstream) | RandomPerCell init | -- | -- | SECOND_BATCH | | QMCPACK | v4.4.0 | NATIVE+SPACK_DEPS | not started | -- | -- | -- | -- | NiO S-series (download) | walkers_per_rank | -- | -- | SECOND_BATCH (needs Clang offload, Boost) | diff --git a/level3/lammps/build.sh b/level3/lammps/build.sh index fd4e42a..4ed409c 100755 --- a/level3/lammps/build.sh +++ b/level3/lammps/build.sh @@ -29,6 +29,7 @@ set +u; # shellcheck disable=SC1091 source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # shellcheck disable=SC1091 source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # Level 3 builds must not see Level 2 .deps/install prefixes BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" diff --git a/level3/lammps/run.sh b/level3/lammps/run.sh index c1efc71..d74e1fe 100755 --- a/level3/lammps/run.sh +++ b/level3/lammps/run.sh @@ -61,8 +61,13 @@ case "$MODE" in PROCS=(-var px "$PX" -var py "$PY" -var pz "$PZ") ;; esac ATOMS=$(( 4 * 20 * X * 20 * Y * 20 * Z )) -RUN_DIR="$R/build/level3/lammps/$MODEL/run"; mkdir -p "$RUN_DIR" +RUN_DIR="$R/build/level3/lammps/$MODEL/run" +# A dry-run must never touch real results: it writes its derived deck and would-be +# log into a throwaway .dryrun/ subdir instead of the real run directory. +[ -n "${HPCPERF_DRY_RUN:-}" ] && RUN_DIR="$RUN_DIR/.dryrun" +mkdir -p "$RUN_DIR" LOG="$RUN_DIR/log.$MODE.np$N_RANKS.lammps" +rm -f "$LOG" # validate only against THIS run's output; never a stale log left by a failed run # Derived deck (upstream bench/in.lj untouched): `run 100` -> `run ${steps}`, # and in weak mode a `processors ${px} ${py} ${pz}` line before create_box so # the rank grid matches the box shape. With steps=100 and no processors line @@ -77,7 +82,16 @@ IN="$RUN_DIR/in.lj.$MODE" } > "$IN" echo "# LAMMPS $BACKEND: mode=$MODE ranks=$N_RANKS box=$((20*X))x$((20*Y))x$((20*Z)) fcc cells = $ATOMS atoms ($((ATOMS / N_RANKS))/rank), $STEPS steps, gpu-aware=$GAM, log=$LOG" -exec "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ +RUN_ID="$(l3_run_id)" +rc=0 +"$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ "$EXE" -k on g 1 t "${HPCPERF_CPUS_PER_RANK:-1}" -sf kk -pk kokkos newton on neigh half gpu/aware "$GAM" \ -in "$IN" -var x "$X" -var y "$Y" -var z "$Z" "${PROCS[@]}" -var steps "$STEPS" \ - -log "$LOG" -echo none "$@" + -log "$LOG" -echo none "$@" || rc=$? +if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + l3_manifest "$RUN_DIR" "run_id=$RUN_ID" "app=lammps" "backend=$BACKEND" "mode=$MODE" \ + "ranks=$N_RANKS" "atoms=$ATOMS" "steps=$STEPS" "gpu_aware=$GAM" "exit_code=$rc" \ + "binary=$EXE" "binary_sha256=$(l3_sha_file "$EXE")" "input=$IN" "input_sha256=$(l3_sha_file "$IN")" \ + "log=$LOG" "utc=$(date -u +%FT%TZ)" +fi +exit "$rc" diff --git a/level3/lammps/validate.sh b/level3/lammps/validate.sh index ac6547a..5a96ad7 100755 --- a/level3/lammps/validate.sh +++ b/level3/lammps/validate.sh @@ -8,78 +8,103 @@ # ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) selects the rank count # # What is compared (Step 0 and Step 100 rows: Temp, E_pair, TotEng, Press): -# Step 0 : relative tolerance 1e-8 -- the initial energies are deterministic -# (same lattice, same geometric velocities) and must agree to -# double-precision reduction-order noise. -# Step 100: relative tolerance 1e-5 -- after 100 NVE steps the trajectory has -# accumulated floating-point differences from the different -# force-summation order (GPU vs CPU, N ranks vs 1), but the -# thermodynamic averages of a 32k-atom LJ liquid are insensitive to -# that at the 1e-6 level; 1e-5 is ten times the largest difference -# observed on this node and far below any physics change. -# Additionally, with HPCPERF_GPUS>1 the N-rank result is compared against the -# 1-rank GPU result of the same build with the same tolerances (rank-count -# independence). Prints PASS/FAIL; exit 0/1. No tolerance is loosened to pass. +# Step 0 : relative tolerance 1e-8 -- deterministic initial state. +# Step 100: relative tolerance 1e-5 -- reduction-order divergence only. +# With HPCPERF_GPUS>1 the N-rank result is also compared with this build's 1-GPU +# result (rank-count independence). Adapted subset of upstream's regression +# check: upstream's tools/regression-tests/run_tests.py compares every thermo +# column of every logged step with per-quantity tolerances; here the four +# state variables at the first and last step are checked (the quantities that +# move if the physics or force summation is wrong). No tolerance is loosened. +# +# Reproducibility rules enforced here: +# * the run's real exit code is captured; a nonzero exit, a timeout, a missing +# log or a non-finite value is a FAIL (never a PASS on a stale log); +# * run.sh removes its target log before launching, so only THIS run's output +# is validated; +# * NaN/Inf in any compared quantity is rejected explicitly (l3_check). set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" R="$(cd "$HERE/../.." && pwd)" +set +u; source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" N="${HPCPERF_GPUS:-1}" REF="$R/_upstream/level3/lammps/bench/log.15Jul25.lj.fixed.g++.1" RUN_DIR="$R/build/level3/lammps/$MODEL/run" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-900}" [ -f "$REF" ] || { echo "validate.sh: reference log $REF missing (run fetch.sh)" >&2; exit 1; } unset HPCPERF_SCALE_MODE export HPCPERF_GPUS="$N" echo "validate.sh: LAMMPS $BACKEND smoke (bench/in.lj, 32000 atoms, 100 steps) on $N GPU(s)" -HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit|Loop time|ERROR' || true + +# run once, on N GPUs, capturing the real exit code (no pipe swallows it) +run_once() { # + local ng=$1 out=$2 rc=0 + HPCPERF_GPUS="$ng" HPCPERF_SCALE_MODE=smoke timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" > "$out" 2>&1 || rc=$? + return $rc +} +VOUT="$RUN_DIR/validate.smoke.np$N.stdout"; mkdir -p "$RUN_DIR" +rc=0; run_once "$N" "$VOUT" || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|Loop time|ERROR|abort' "$VOUT" || true +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- run timed out after ${TIMEOUT}s"; exit 1; fi +[ "$rc" -eq 0 ] || { echo "validate.sh: FAIL -- run.sh exited $rc (see $VOUT)"; exit 1; } LOG="$RUN_DIR/log.smoke.np$N.lammps" -[ -f "$LOG" ] || { echo "validate.sh: FAIL -- no log produced ($LOG)" ; exit 1; } +[ -f "$LOG" ] || { echo "validate.sh: FAIL -- no log produced ($LOG)"; exit 1; } if [ "$N" -gt 1 ] && [ ! -f "$RUN_DIR/log.smoke.np1.lammps" ]; then echo "validate.sh: producing the 1-GPU reference run for rank-count comparison" - HPCPERF_GPUS=1 HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" > /dev/null 2>&1 || true + r1=0; run_once 1 "$RUN_DIR/validate.smoke.np1.stdout" || r1=$? + [ "$r1" -eq 0 ] && [ -f "$RUN_DIR/log.smoke.np1.lammps" ] || { echo "validate.sh: FAIL -- 1-GPU reference run failed (rc=$r1)"; exit 1; } fi python3 - "$LOG" "$REF" "$N" "$RUN_DIR/log.smoke.np1.lammps" <<'PY' -import re, sys +import re, sys, os +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import require_finite, ValidationError def thermo(path): rows = {} - with open(path) as f: - lines = f.read().splitlines() + lines = open(path).read().splitlines() for i, ln in enumerate(lines): if ln.split()[:2] == ["Step", "Temp"]: cols = ln.split() for row in lines[i+1:]: p = row.split() if not p or not re.match(r'^\d+$', p[0]): break - rows[int(p[0])] = dict(zip(cols[1:], map(float, p[1:]))) + rows[int(p[0])] = dict(zip(cols[1:], p[1:])) return rows log, ref, n, log1 = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] -got, want = thermo(log), thermo(ref) -tol = {0: 1e-8, 100: 1e-5} -keys = ["Temp", "E_pair", "TotEng", "Press"] -ok = True -def cmp(a, b, label): - global ok - for step, t in tol.items(): - if step not in a or step not in b: - print(f" {label}: step {step} missing (got {sorted(a)} vs {sorted(b)})"); ok = False; continue +keys = ["Temp", "E_pair", "TotEng", "Press"]; tol = {0: 1e-8, 100: 1e-5} +try: + got, want = thermo(log), thermo(ref) + for step in tol: + if step not in got: raise ValidationError(f"run log has no Step {step} row (got {sorted(got)}); run did not complete") + if step not in want: raise ValidationError(f"reference log has no Step {step} row") for k in keys: - x, y = a[step][k], b[step][k] - rel = abs(x - y) / max(abs(y), 1e-30) - flag = "ok " if rel <= t else "BAD" - if rel > t: ok = False - print(f" {label}: step {step:>3} {k:<7} got {x: .10g} ref {y: .10g} rel {rel:.2e} (tol {t:.0e}) {flag}") -print(f"[1] {n}-GPU run vs upstream CPU reference log:") -cmp(got, want, "vs-ref") -if n > 1: - try: + if k not in got[step]: raise ValidationError(f"field '{k}' missing at step {step} in run log") + ok = True + def cmp(a, b, label): + global ok + for step, t in tol.items(): + for k in keys: + x = require_finite(f"{label} step{step} {k} (got)", a[step][k]) + y = require_finite(f"{label} step{step} {k} (ref)", b[step][k]) + rel = abs(x - y) / max(abs(y), 1e-30) + if rel > t: ok = False + print(f" {label}: step {step:>3} {k:<7} got {x: .10g} ref {y: .10g} rel {rel:.2e} (tol {t:.0e}) {'ok ' if rel <= t else 'BAD'}") + print(f"[1] {n}-GPU run vs upstream CPU reference log (adapted subset: 4 state vars at step 0 and 100):") + cmp(got, want, "vs-ref") + if n > 1: g1 = thermo(log1) + for step in tol: + if step not in g1: raise ValidationError(f"1-GPU log has no Step {step} row") print(f"[2] {n}-GPU run vs this build's 1-GPU run (rank-count independence):") cmp(got, g1, "vs-1gpu") - except FileNotFoundError: - print("[2] 1-GPU log unavailable; rank-count comparison skipped"); ok = False +except ValidationError as ex: + print(f" VALIDATION ERROR: {ex}") + print(f"LAMMPS {sys.argv and ''}validation ({n} GPU): FAIL"); sys.exit(1) print(f"LAMMPS CUDA validation ({n} GPU, bench/in.lj vs log.15Jul25.lj.fixed.g++.1): {'PASS' if ok else 'FAIL'}") sys.exit(0 if ok else 1) PY diff --git a/level3/nekrs/COMPATIBILITY.md b/level3/nekrs/COMPATIBILITY.md new file mode 100644 index 0000000..2a5f46b --- /dev/null +++ b/level3/nekrs/COMPATIBILITY.md @@ -0,0 +1,200 @@ +# nekRS -- CUDA / dependency compatibility (v26.0 on dgx003, B200, CUDA 13.2.78) + +Scope: the exact software *combination* used for the Level 3 nekRS bring-up, and +what is and is not verified about it. This document separates four things the +review asked never to be conflated: + +1. what upstream **documents**; +2. what upstream **CI / site testing** actually exercises; +3. the **local build** result for this exact combination; +4. the **local run** result for this exact case. + +"Verified" below means *observed on this node*, never "endorsed by upstream". +Where no upstream evidence was found, the entry is `NOT_FOUND` / `UNVERIFIED` -- +not "upstream forbids it". + +Upstream references consulted (read at the pinned commit / current docs): +`github.com/Nek5000/nekRS` and `/releases`; the pinned +`CMakeLists.txt` and `cmake/hypre.cmake` at +`96b3cf9e5bacede16568826c04a21bc0fe50dc7d`; `nekrs.readthedocs.io/quickstart`; +`Nek5000/nekRS_HPCsupport`; `hypre-space/hypre`; the CCCL 3.0 migration guide +(`nvidia.github.io/cccl/.../3.0_migration_guide.html`); the Blackwell +compatibility guide (`docs.nvidia.com/cuda/blackwell-compatibility-guide`). + +## The combination + +| Component | Value | +|---|---| +| nekRS | v26.0, commit `96b3cf9e5bacede16568826c04a21bc0fe50dc7d` | +| vendored HYPRE | 2.32.0 (`3rd_party/hypre`, squashed subtree) | +| vendored OCCA | 2.0.0-dev (`OCCA_VERSION_STR`) | +| CUDA toolkit | 13.2.78 (`/usr/local/cuda` -> cuda-13.2); Thrust/CCCL 3.2 (`THRUST_VERSION 300200`) | +| GPU / arch | B200, sm_100 | +| host compiler | conda GCC 13.3.0 (C/C++), system gfortran 14.2.1 (Fortran) | +| MPI | conda Open MPI 5.0.10 (CUDA-aware) | +| default build option | `OCCA_ENABLE_CUDA=ON`, `ENABLE_HYPRE_GPU=ON` (variant `hypregpu`) | + +## `cmake/hypre.cmake` at this commit already has a CUDA >= 13 branch + +The pinned `cmake/hypre.cmake` contains: + +``` +if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "13.0.0") + set(HYPRE_DEVICE_ARCH "HYPRE_CUDA_SM=80 90") +elseif(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.0.0") + set(HYPRE_DEVICE_ARCH "HYPRE_CUDA_SM=70 80") +endif() +``` + +So nekRS v26.0 **is aware of CUDA 13** (a dedicated branch exists) -- it is wrong +to say "nekRS does not support CUDA 13". Equally, the branch only sets the SASS +list to `80 90` (no sm_100, no PTX) and the presence of a branch is **not** +evidence that CUDA 13.2.78 + B200 + vendored HYPRE 2.32.0 passed any upstream CI: +upstream CI (`.github/workflows/ci.yml`) builds the serial/CPU backend with +MPICH + gfortran on ubuntu; the nekRS_HPCsupport machine files target +Frontier/Perlmutter/Polaris/etc., none of them a B200/CUDA-13 combination +(NOT_FOUND for this exact stack). + +## Four problem classes, separated + +### A. HYPRE GPU source vs CCCL/Thrust 3.2 API (compile, GPU-only code) + +`ENABLE_HYPRE_GPU=ON` compiles HYPRE's device sources with nvcc against CUDA 13's +Thrust 3.2. Three incompatibilities were hit and are addressed by the three +patches (all confined to HYPRE's *device* code / device headers): + +| Symptom (build error) | Cause | Real API change? | Patch | +|---|---|---|---| +| `namespace "thrust" has no member "make_reverse_iterator"` / `"pair"` | `` and `` are no longer pulled in transitively by other thrust headers in CCCL 3.x | **No** -- the names still exist (`thrust::make_reverse_iterator`/`reverse_iterator` are re-exported from `cuda::std` in `thrust/iterator/reverse_iterator.h`); they were merely not *visible* without an explicit include | 0003 (add the includes to `device_utils.h` and the pre-generated `_hypre_utilities.hpp`) | +| `thrust::pair<...>` as a declared result type of `reduce_by_key` | same non-transitive-include issue at the point of use | **No** (visibility) -- resolved either by the include or by letting `auto` take the library's real return type | 0002 (`auto`) + 0003 (include) | +| `thrust::not1` undefined | `thrust::not1` (C++17-deprecated) was **removed** in CCCL 3.x | **Yes -- genuinely removed**; documented replacement is `thrust::not_fn` (= `cuda::std::not_fn`) | 0003 (`not1` -> `not_fn`, 16 sites) | + +So only `not1` is a true API removal; the `pair`/`reverse_iterator` errors are +missing explicit includes, not deleted APIs. This distinction is recorded per the +review's instruction. + +Upstream fix status: **UNVERIFIED / NOT looked up in detail.** HYPRE's own master +is understood to build under CUDA 13, and the CCCL migration guide documents the +`not1 -> not_fn` change, but no specific HYPRE commit/PR or nekRS-vendored-HYPRE +bump was located and pinned. The patches are therefore labelled **project-local +compatibility patches**, not backports of a known upstream commit. Whether nekRS +`next` already vendors a CUDA-13-ready HYPRE was not checked (route A, below). + +### B. HYPRE prebuilt device library arch list (sm_100 coverage) + +Patch 0001 changes `HYPRE_CUDA_SM=80 90` to `80 90 100` so the device library +carries Blackwell SASS. Without it HYPRE's device kernels would have no sm_100 +code (and the branch emits no PTX to JIT from). This matters **only when GPU +HYPRE is actually used** (see the coarse-solver finding below). + +### C. OCCA CUDA / JIT + +OCCA derives `-arch=sm_` at run time and JIT-compiles the OKL kernels +with the host nvcc; no patch needed. Verified locally: `active occa mode: CUDA`, +kernels compiled, the main solver runs on the GPU (the launcher's per-rank GPU +audit reports the ranks on distinct GPUs). + +### D. MPI one-sided (OSC) UCX runtime failure + +Independent of the above: nekRS uses `MPI_Win_lock`, and Open MPI's default +`osc ucx` aborts in `uct_ib` at 4 ranks on this node. Worked around at run time +with `OMPI_MCA_osc=^ucx` in `run.sh` (nekRS-local; not a global default). This is +a transport/runtime issue, not a CUDA/HYPRE compatibility issue. + +Fixing any one of A/B/C/D does not fix the others; they are tracked separately. + +## The load-bearing finding: the benchmark case runs the HYPRE coarse solve on the CPU + +For the ethier case (the only nekRS case wired up), the six recorded logs +(`smoke.np{1,2,4}.cimode2`, `strong.np{1,4}`, `weak.np4`) all show: + +``` +FLUID PRESSURE MULTIGRID COARSE SOLVER: BOOMERAMG +FLUID PRESSURE MULTIGRID COARSE SOLVER LOCATION: CPU +FLUID PRESSURE MULTIGRID COARSE SOLVER PRECISION: FP32 +``` + +Cross-checked against the source, not just the logs: +- `src/platform/par/parsePreconditioner.hpp` sets the default + `... MULTIGRID COARSE SOLVER LOCATION = CPU` and `... PRECISION = FP32`; +- `src/core/elliptic/elliptic.cpp` sets `... COARSE SOLVER LOCATION = CPU`; +- `examples/ethier/ethier.par` does **not** set a coarse-solver location, and + `--cimode 2` (`examples/ethier/ci.inc`) does **not** set one either -> the + default (CPU) applies; +- only `--cimode 3` sets `FLUID PRESSURE PRECONDITIONER = MULTIGRID+SEMFEM` and + `... COARSE SOLVER LOCATION = DEVICE` (falling back to CPU only when the run is + serial / CPU-backend). + +Consequences (stated precisely): +- The nekRS **main application does run on the GPU** (OCCA CUDA) -- this is NOT a + CPU-only application, and multi-GPU main-solve is real (per-rank GPU audit). +- The HYPRE **BoomerAMG coarse solve for the ethier cimode-2 case runs on the + CPU**. The GPU HYPRE device library built by `ENABLE_HYPRE_GPU=ON` is compiled + but **not exercised** by this case. +- Therefore the earlier "9/9 CI checks passed" for cimode 2 validated *the CUDA + main application + a CPU coarse solve*. It did **not** validate GPU HYPRE. A + case that selects `COARSE SOLVER LOCATION = DEVICE` (cimode 3) is required to + exercise GPU HYPRE on the target GPU; that verification is tracked in the + matrix below and must not be claimed from the cimode-2 result. + +("Library compiled/loaded" is not "kernel executed" -- the judgement here is from +the solver-configuration lines the run prints and the source defaults, not from a +source grep alone.) + +## Compatibility / verification matrix + +| variant | ENABLE_HYPRE_GPU | patches | cimode | runtime coarse | build (dgx003, CUDA 13.2.78, sm_100) | main app CUDA multi-GPU | GPU HYPRE coarse | official CI for this stack | +|---|---|---|---|---|---|---|---|---| +| hypregpu (default) | ON | 0001+0002+0003 | 2 | CPU | ok (~30 min, prior round) | VERIFIED 1/2/4 GPU, 9/9 | not exercised (CPU coarse) | NOT_FOUND | +| hypregpu | ON | 0001+0002+0003 | 3 | DEVICE (GPU) | (same install) | VERIFIED 1/4 GPU, 9/9 | **VERIFIED** 1/4 GPU (9/9, coarse=DEVICE) | NOT_FOUND | +| cpucoarse (candidate) | OFF | **none** | 2 | CPU | ok, **113 s, 0 patches** | VERIFIED 1/2/4 GPU, 9/9 | not built | n/a | +| cpucoarse | OFF | none | 3 | (DEVICE requested) | (same install) | n/a | **explicitly rejected** -- nekRS aborts: `HYPRE+DEVICE not enabled! Recompile with -DENABLE_HYPRE_GPU=ON` (exit 1); NO silent CPU fallback | n/a | + +Results (dgx003, 2026-09-05), all through the common launcher (one rank per GPU, +per-rank GPU wrapper, audit verified on distinct GPUs), analytic Ethier solution, +EPS 0.3, stricter validator (real exit code, complete 9/9 check set, coarse +location asserted, NaN/Inf rejected): + +- hypregpu cimode 2: PASS 1/2/4 GPU, coarse=CPU. +- hypregpu cimode 3: PASS 1/4 GPU, coarse=DEVICE -> **the GPU HYPRE coarse solve + enabled by the three patches is verified correct on B200/CUDA 13.2**, not just + compiled. +- cpucoarse cimode 2: PASS 1/2/4 GPU, coarse=CPU, main app on distinct GPUs -> + the entire current Ethier workload runs correctly with GPU HYPRE OFF and NO + patches (build 113 s vs the ~30 min patched GPU-HYPRE build). +- cpucoarse cimode 3: nekRS's own `HYPRE+DEVICE not enabled!` check aborts the + run (exit 1); the validator FAILs (coarse != DEVICE). No silent downgrade. + +Finding: the three patches are needed ONLY to build/exercise GPU HYPRE. For the +current CPU-coarse Ethier workload the `cpucoarse` variant is a smaller, +patch-free, faster-to-build configuration that runs the same GPU main +application. The `hypregpu` variant remains the one that can run (and is verified +to run) the GPU coarse solve. + +## What is and is not claimed for the default (hypregpu) variant + +- CLAIMED (verified locally): CUDA main application, multi-GPU (1/2/4), on + distinct GPUs; ethier cimode-2 correctness (analytic solution, 9/9 checks) with + a **CPU** HYPRE coarse solve. +- NOT CLAIMED: that GPU HYPRE coarse solve is correct (cimode 2 does not run it); + that the CUDA 13.2.78 + B200 + HYPRE 2.32.0 combination is upstream-CI-certified + (NOT_FOUND); that CPU-coarse is the best configuration at 40/80 GPUs + (UNVERIFIED -- CPU coarse scalability is a separate, unmeasured question). + +## Routes if GPU HYPRE coarse is actually required (audited, not all built) + +- **A -- upstream integration**: pin a newer nekRS release / a fixed + master/next commit / an upstream HYPRE bump that resolves CCCL-3 + sm_100. + STATUS: NOT_RUN (not audited to a specific SHA this round). +- **B -- keep CUDA 13.2 + HYPRE 2.32.0 + the local patches** as a + project-maintained compatibility variant, and verify GPU coarse with cimode 3. + STATUS: the cimode-3 verification is attempted this round (matrix). +- **C -- an earlier B200-capable CUDA (12.8/12.9) for nekRS only**: would still + need HYPRE_CUDA_SM to include sm_100 and re-verification; does not + automatically avoid the arch-target问题. Must not touch the system CUDA symlink + or the other four apps. STATUS: NOT_RUN. +- **D -- replace the vendored HYPRE**: dependency-upgrade experiment; needs + host/device wrapper, single/mixedint, precision, HYPRE_Int/BigInt and link + checks. Not to be recorded as an LLM/source optimisation. STATUS: NOT_RUN. + +Routes A/C/D are UNVERIFIED this round by design (no extra large builds started). diff --git a/level3/nekrs/README.md b/level3/nekrs/README.md index 9cf37c3..b2bffd5 100644 --- a/level3/nekrs/README.md +++ b/level3/nekrs/README.md @@ -90,6 +90,40 @@ Memory estimate ~60 KB per element at N=7 (velocity, pressure, two scalars, multistep history, preconditioner). Derived `ethier.par` files change only `hrefine` and `numSteps` (class A). +## Coarse-solver location: what runs on the GPU (read `COMPATIBILITY.md`) + +The ethier case's HYPRE BoomerAMG coarse solve runs where the cimode selects: + +- `--cimode 2` (and the default `.par`): `FLUID PRESSURE MULTIGRID COARSE SOLVER + LOCATION = CPU` -- the coarse solve is on the **host**. The nekRS main + application (advection, Helmholtz, pressure pMG smoother, gather-scatter) runs + on the **GPU** via OCCA/CUDA. So this is not a CPU-only application, but a + cimode-2 PASS does **not** exercise GPU HYPRE. +- `--cimode 3`: `FLUID PRESSURE PRECONDITIONER = MULTIGRID+SEMFEM`, `COARSE + SOLVER LOCATION = DEVICE` -- the coarse solve runs on the **GPU** (this is the + mode that exercises the GPU HYPRE built by `ENABLE_HYPRE_GPU=ON` and the three + patches). + +Two build variants exist (isolated src/build/install/JIT-cache; select with +`HPCPERF_NEKRS_HYPRE_GPU`/`HPCPERF_NEKRS_VARIANT`): + +| variant | `ENABLE_HYPRE_GPU` | patches | GPU HYPRE coarse (cimode 3) | CPU coarse (cimode 2) | +|---|---|---|---|---| +| `hypregpu` (default) | ON | 0001+0002+0003 | built + **verified** (cimode 3, 1/4 GPU, 9/9, coarse=DEVICE) | verified 1/2/4 | +| `cpucoarse` (candidate) | OFF | none | not built; a DEVICE request is **explicitly rejected** by nekRS (`HYPRE+DEVICE not enabled!`, exit 1), no silent fallback | verified 1/2/4 (candidate, 0 patches, 113 s build) | + +Build/run isolation per variant: `hypregpu` keeps the legacy paths +(`.deps/level3/nekrs/{src,install}`, `build/level3/nekrs/cuda`); other variants +use `.deps/level3/nekrs//{src,install}` and `build/level3/nekrs/.` +with their own OCCA/nekRS JIT cache. The source-copy cache key is the upstream +SHA plus the ordered patch-content hash, so the two variants never share a +patched/unpatched tree. + +Which to make default is a decision for review: `cpucoarse` is minimal (no +patches, fast build) and covers the current CPU-coarse workload; `hypregpu` is +required if a case selects GPU (DEVICE) coarse. CPU-coarse scalability at 40/80 +GPUs is UNVERIFIED and is not claimed to be optimal at all scales. + ## Validation (`validate.sh`, upstream mechanism) `nekrs --setup ethier --cimode 2` is one of the modes upstream's CI runs on this @@ -101,10 +135,19 @@ relative tolerance EPS = 0.3) plus the iteration counts of the pressure, velocity and scalar solves. nekRS prints `CI test <...> passed|failed` per check and exits non-zero on failure; `validate.sh` uses that verdict unchanged (upstream runs it on CPUs with 2 ranks; here the CUDA backend on 1, 2 and 4 -ranks). Observed on dgx003 (2026-09-05): **PASS at 1, 2 and 4 GPUs, 9/9 checks -each**; final L2 errors velocity 2.776e-10, pressure 6.983e-10, scalar00 -6.672e-12, scalar01 7.495e-12 -- identical to 5 significant digits across the -three rank counts (CI references 2.77e-10 / 7.14e-10 / 7.49e-12 / 7.22e-12). +ranks). The validator now (a) captures the run's real exit code (nonzero/timeout -> +FAIL), (b) requires the COMPLETE set of CI checks (9 for cimode 2/3, not merely +"some passed"), (c) asserts the coarse-solver LOCATION recorded in the log +matches the cimode (CPU for 2, DEVICE for 3 -- no silent fallback), and (d) +rejects NaN/Inf. `HPCPERF_NEKRS_CIMODE` selects the mode. + +Observed on dgx003 (2026-09-05), `hypregpu` variant: +- `--cimode 2` (CPU coarse): **PASS at 1, 2, 4 GPUs, 9/9 checks, coarse=CPU**. +- `--cimode 3` (DEVICE / GPU HYPRE coarse): **PASS at 1 and 4 GPUs, 9/9 checks, + coarse=DEVICE** -- this is the run that actually exercises the GPU HYPRE coarse + solve the three patches enable, verified against the analytic solution. +Earlier CI L2 errors (cimode 2): velocity 2.78e-10, pressure 6.98e-10, scalars +6.67e-12 / 7.49e-12 (CI references 2.77e-10 / 7.14e-10 / 7.49e-12 / 7.22e-12). ## Results on dgx003 (4x B200, CUDA 13.2.78, Slurm job 9552083) diff --git a/level3/nekrs/build.sh b/level3/nekrs/build.sh index 30a81e3..cece0a2 100755 --- a/level3/nekrs/build.sh +++ b/level3/nekrs/build.sh @@ -35,6 +35,7 @@ set +u; # shellcheck disable=SC1091 source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # shellcheck disable=SC1091 source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # Level 3 builds must not see Level 2 .deps/install prefixes BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" @@ -42,7 +43,25 @@ UP="$R/_upstream/level3/nekRS" [ -f "$UP/CMakeLists.txt" ] || { echo "build.sh: $UP missing -- run $HERE/fetch.sh first" >&2; exit 1; } SHA="$(git -C "$UP" rev-parse HEAD)" l3_paths nekrs -BUILD_DIR="$R/build/level3/nekrs/$MODEL" + +# --- variant selection (multi-variant build/install/cache isolation) ---------- +# HPCPERF_NEKRS_HYPRE_GPU=ON|OFF selects whether the vendored HYPRE is built with +# its CUDA device backend (GPU coarse solve possible) or host-only (CPU coarse +# only). This is INDEPENDENT of OCCA_ENABLE_CUDA: the main application is on the +# GPU either way. The default variant 'hypregpu' (ENABLE_HYPRE_GPU=ON) keeps the +# existing legacy paths so the already-validated install is untouched; any other +# variant gets a fully separate src/build/install/logs and its own JIT cache. +HYPRE_GPU="${HPCPERF_NEKRS_HYPRE_GPU:-ON}" +case "$HYPRE_GPU" in ON|OFF) : ;; *) echo "build.sh: HPCPERF_NEKRS_HYPRE_GPU must be ON or OFF" >&2; exit 2 ;; esac +VARIANT="${HPCPERF_NEKRS_VARIANT:-$([ "$HYPRE_GPU" = ON ] && echo hypregpu || echo cpucoarse)}" +if [ "$VARIANT" = hypregpu ]; then + BUILD_DIR="$R/build/level3/nekrs/$MODEL" # legacy layout (unchanged) +else + L3_DEPS="$L3_R/.deps/level3/nekrs/$VARIANT" + L3_SRC="$L3_DEPS/src"; L3_INSTALL="$L3_DEPS/install"; L3_LOGS="$L3_DEPS/logs" + mkdir -p "$L3_SRC" "$L3_INSTALL" "$L3_LOGS" + BUILD_DIR="$R/build/level3/nekrs/$VARIANT.$MODEL" +fi JOBS="${HPCPERF_BUILD_JOBS:-32}" SYS_FC="${HPCPERF_SYSTEM_GFORTRAN:-/usr/bin/gfortran}" [ -x "$SYS_FC" ] || { echo "build.sh: no gfortran at $SYS_FC (set HPCPERF_SYSTEM_GFORTRAN); the conda env has none" >&2; exit 1; } @@ -62,15 +81,19 @@ export FFLAGS="${FFLAGS:-} -fPIC" # verbatim as the full archive command (its default is "ar -rcu"), so the bare tool name makes every # `ar libHYPRE_*.a ...` call fail with a usage error. Unset -> HYPRE's own default. Class C. unset AR -# 0002 (class D, 2 lines): the vendored HYPRE 2.32.0 declares the result of thrust::reduce_by_key as -# `thrust::pair<...>`, a name the CCCL shipped with CUDA 13 no longer provides; `auto` takes the -# library's actual return type. No numerics touched. HYPRE 2.32.0 + CUDA 13 is not upstream-validated. -# 0003 (class D, 36 lines): Thrust 3.2 (CUDA 13) no longer includes -# and transitively (explicit includes added to HYPRE's device_utils.h and to the -# pre-generated concatenated header _hypre_utilities.hpp that the sources actually include) and removed -# the C++17-deprecated `thrust::not1`; its documented replacement `thrust::not_fn` (= cuda::std::not_fn) -# is substituted 1:1 (16 uses). Pure compatibility, no numerics. -PATCHES=("$HERE/patches/0001-hypre-cuda-sm100.patch" "$HERE/patches/0002-hypre-cuda13-thrust-pair.patch" "$HERE/patches/0003-hypre-cuda13-thrust3-compat.patch") +# The three HYPRE patches are needed ONLY to compile HYPRE's CUDA device backend against CUDA 13: +# 0001 (class B, 1 line): add sm_100 to HYPRE_CUDA_SM (device SASS list). +# 0002 (class D, 2 lines): thrust::reduce_by_key result type -> auto (device_utils.c/csr_matop_device.c). +# 0003 (class D, 36 lines): explicit + includes and +# thrust::not1 -> thrust::not_fn, all in HYPRE's *device* sources / device_utils headers. +# All three touch code that is compiled by nvcc ONLY when ENABLE_HYPRE_GPU=ON. With ENABLE_HYPRE_GPU=OFF +# HYPRE is host-only and that code is not compiled, so NO patch is applied (the candidate tests whether +# the unused GPU component -- and therefore the patches -- can be dropped entirely). +if [ "$HYPRE_GPU" = ON ]; then + PATCHES=("$HERE/patches/0001-hypre-cuda-sm100.patch" "$HERE/patches/0002-hypre-cuda13-thrust-pair.patch" "$HERE/patches/0003-hypre-cuda13-thrust3-compat.patch") +else + PATCHES=() +fi case "$BACKEND" in CUDA) command -v nvcc >/dev/null || { echo "build.sh: nvcc not on PATH" >&2; exit 1; } @@ -79,15 +102,19 @@ case "$BACKEND" in OCCA_FLAGS=(-DOCCA_ENABLE_CUDA=OFF -DOCCA_ENABLE_HIP=ON -DOCCA_ENABLE_DPCPP=OFF -DOCCA_ENABLE_OPENCL=OFF); ARCHNOTE="gfx950 (JIT at run time)" ;; *) echo "usage: $0 [CUDA|HIP]" >&2; exit 2 ;; esac -CMAKE_OPTS="${OCCA_FLAGS[*]} ENABLE_HYPRE_GPU=ON ENABLE_ADIOS=OFF ENABLE_CVODE=OFF NEKRS_BUILD_FLOAT=OFF NEKRS_GPU_MPI=OFF(default; runtime NEKRS_GPU_MPI) CC=mpicc CXX=mpicxx FC=mpif90(OMPI_FC=$SYS_FC)" +CMAKE_OPTS="variant=$VARIANT ${OCCA_FLAGS[*]} ENABLE_HYPRE_GPU=$HYPRE_GPU ENABLE_ADIOS=OFF ENABLE_CVODE=OFF NEKRS_BUILD_FLOAT=OFF NEKRS_GPU_MPI=OFF(default; runtime NEKRS_GPU_MPI) CC=mpicc CXX=mpicxx FC=mpif90(OMPI_FC=$SYS_FC)" PATCHNAMES=(); for p in "${PATCHES[@]}"; do PATCHNAMES+=("$(basename "$p")"); done FP="$(l3_fingerprint_text nekrs "$SHA" "$MODEL" "vendored: occa=2.0.0-dev hypre=2.32.0 gslib nek5000 lapack (in-tree)" "$CMAKE_OPTS" "runtime(NEKRS_GPU_MPI, default 0)" "${PATCHNAMES[@]}")" l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 -echo "# nekRS $BACKEND: upstream $SHA (v26.0), arch $ARCHNOTE, MPI $(mpirun --version 2>/dev/null | head -1), FC $($SYS_FC --version | head -1)" +echo "# nekRS $BACKEND: variant=$VARIANT ENABLE_HYPRE_GPU=$HYPRE_GPU patches=${#PATCHES[@]} upstream $SHA (v26.0), arch $ARCHNOTE, install=$L3_INSTALL" # private source copy (upstream clone stays pristine). The copy is ~280 MB in many small files (slow on -# this filesystem), so it is reused when it already holds this upstream commit + patch set. -STAMP="$SHA ${PATCHNAMES[*]}" +# this filesystem), so it is reused only when it already holds this upstream commit AND this exact patch +# series. The cache key is the upstream SHA plus the ORDERED patch-CONTENT hash, so editing a patch (even +# without renaming it) invalidates the copy. A missing patch is a hard error. +for p in "${PATCHES[@]}"; do [ -f "$p" ] || { echo "build.sh: patch $p missing" >&2; exit 1; }; done +PATCH_SERIES_HASH="$(for p in "${PATCHES[@]}"; do sha256sum "$p" | cut -d' ' -f1; done | sha256sum | cut -d' ' -f1)" +STAMP="$SHA $PATCH_SERIES_HASH" if [ -f "$L3_SRC/.hpcperf-src-stamp" ] && [ "$(cat "$L3_SRC/.hpcperf-src-stamp")" = "$STAMP" ]; then echo "# reusing patched source copy $L3_SRC ($STAMP)" else @@ -109,7 +136,7 @@ fi # vendored-library rule), so the generator is pinned to upstream's CC=mpicc CXX=mpicxx FC=mpif90 cmake -S "$L3_SRC" -B "$BUILD_DIR" -G "Unix Makefiles" -Wfatal-errors \ -DCMAKE_INSTALL_PREFIX="$L3_INSTALL" \ - "${OCCA_FLAGS[@]}" -DENABLE_HYPRE_GPU=ON -DENABLE_ADIOS=OFF -DENABLE_CVODE=OFF -DNEKRS_BUILD_FLOAT=OFF \ + "${OCCA_FLAGS[@]}" -DENABLE_HYPRE_GPU="$HYPRE_GPU" -DENABLE_ADIOS=OFF -DENABLE_CVODE=OFF -DNEKRS_BUILD_FLOAT=OFF \ > "$L3_LOGS/configure-$MODEL.log" 2>&1 \ || { tail -40 "$L3_LOGS/configure-$MODEL.log"; echo "build.sh: configure failed (log: $L3_LOGS/configure-$MODEL.log)" >&2; exit 1; } t0=$(date +%s) diff --git a/level3/nekrs/run.sh b/level3/nekrs/run.sh index 7e3dee4..43cd189 100755 --- a/level3/nekrs/run.sh +++ b/level3/nekrs/run.sh @@ -44,10 +44,18 @@ source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" l3_paths nekrs -BUILD_DIR="$R/build/level3/nekrs/$MODEL" +# variant selection must match build.sh: the default 'hypregpu' uses the legacy layout; any other +# variant (e.g. cpucoarse = ENABLE_HYPRE_GPU=OFF) has its own install and JIT cache. +VARIANT="${HPCPERF_NEKRS_VARIANT:-$([ "${HPCPERF_NEKRS_HYPRE_GPU:-ON}" = ON ] && echo hypregpu || echo cpucoarse)}" +if [ "$VARIANT" = hypregpu ]; then + BUILD_DIR="$R/build/level3/nekrs/$MODEL" +else + L3_INSTALL="$L3_R/.deps/level3/nekrs/$VARIANT/install" + BUILD_DIR="$R/build/level3/nekrs/$VARIANT.$MODEL" +fi export NEKRS_HOME="$L3_INSTALL" EXE="$NEKRS_HOME/bin/nekrs" -[ -x "$EXE" ] || { echo "run.sh: $EXE not found -- run ./build.sh $BACKEND first" >&2; exit 1; } +[ -x "$EXE" ] || { echo "run.sh: $EXE not found for variant '$VARIANT' -- run HPCPERF_NEKRS_VARIANT=$VARIANT ./build.sh $BACKEND first" >&2; exit 1; } CASE_SRC="$R/_upstream/level3/nekRS/examples/ethier" [ -f "$CASE_SRC/ethier.re2" ] || { echo "run.sh: $CASE_SRC missing (run fetch.sh)" >&2; exit 1; } @@ -63,7 +71,8 @@ esac if [ "$H" -gt 0 ]; then ELEMS=$((32 * H * H * H)); else ELEMS=32; fi POINTS=$((ELEMS * (ORDER + 1) * (ORDER + 1) * (ORDER + 1))) -RUN_DIR="$BUILD_DIR/run/$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR" +# l3_rundir: dry-run gets a throwaway dir instead of rm -rf'ing the real run directory. +RUN_DIR="$(l3_rundir "$BUILD_DIR/run/$MODE.np$N_RANKS")" || exit 2 cp "$CASE_SRC"/* "$RUN_DIR"/ # complete upstream case directory (re2, usr, udf, CASEDATA include, ci.inc, par files) if [ "$MODE" = smoke ]; then sed -e "s/^numSteps *=.*/numSteps = $STEPS/" "$CASE_SRC/ethier.par" > "$RUN_DIR/ethier.par" @@ -88,5 +97,13 @@ ulimit -s unlimited 2>/dev/null || ulimit -s "$(ulimit -H -s)" echo "# nekRS $BACKEND: mode=$MODE ranks=$N_RANKS case=ethier hrefine=$H elements=$ELEMS (~$((ELEMS / N_RANKS))/rank) N=$ORDER points=$POINTS steps=$STEPS gpu_mpi=$NEKRS_GPU_MPI run_dir=$RUN_DIR" cd "$RUN_DIR" +RUN_ID="$(l3_run_id)" "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- "$EXE" --setup ethier --backend "$BACKEND" --device-id 0 "$@" 2>&1 | tee "$RUN_DIR/stdout.log" -exit "${PIPESTATUS[0]}" +rc=${PIPESTATUS[0]} +if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + l3_manifest "$RUN_DIR" "run_id=$RUN_ID" "app=nekrs" "variant=$VARIANT" "backend=$BACKEND" "mode=$MODE" "ranks=$N_RANKS" \ + "elements=$ELEMS" "order=$ORDER" "points=$POINTS" "steps=$STEPS" "gpu_mpi=$NEKRS_GPU_MPI" \ + "osc=$OMPI_MCA_osc" "extra_args=$*" "exit_code=$rc" "binary=$EXE" "binary_sha256=$(l3_sha_file "$EXE")" \ + "par_sha256=$(l3_sha_file "$RUN_DIR/ethier.par")" "stdout=$RUN_DIR/stdout.log" "utc=$(date -u +%FT%TZ)" +fi +exit "$rc" diff --git a/level3/nekrs/validate.sh b/level3/nekrs/validate.sh index 6b422ad..ec6f855 100755 --- a/level3/nekrs/validate.sh +++ b/level3/nekrs/validate.sh @@ -3,39 +3,68 @@ # (analytic Ethier-Steinman solution) run on N GPUs. # # ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) +# HPCPERF_NEKRS_CIMODE=C upstream CI mode (default 2) # -# `nekrs --cimode 2` is one of the modes upstream's CI (.github/workflows/ -# ci.yml) runs on this case: it fixes the solver settings (velocity solver -# +BLOCK, subcycling 1, tolerances 1e-12/1e-10) and, at the last step, checks -# the L2 errors of velocity, pressure and both scalars against the exact -# solution (reference values in examples/ethier/ci.inc: 2.77e-10, 7.14e-10, -# 7.49e-12, 7.22e-12; relative tolerance EPS = 0.3) plus the iteration counts -# of the pressure/velocity/scalar solves (+-1). nekRS prints "CI test <...> -# passed|failed" for each check and exits non-zero on any failure -- that -# verdict is used unchanged. The L2-error line the case prints itself -# ("... L2 err") is echoed for the record. -# Note: upstream runs this CI on CPUs with 2 ranks; here the GPU backend on N -# ranks is being validated against the same criteria. +# `nekrs --cimode C` fixes the solver settings for CI and, at the last step, +# checks the L2 errors of velocity/pressure/scalars against the exact solution +# (references in examples/ethier/ci.inc, EPS = 0.3) plus solver iteration +# counts. nekRS prints "CI test passed|failed" per check and exits +# non-zero on any failure. This validator requires ALL of: +# * run exited 0 (no timeout, no abort); +# * the COMPLETE expected set of CI checks was produced (count matches the +# per-cimode table below) -- not merely "some passed"; +# * zero failed checks; +# * the coarse-solver LOCATION recorded in the log matches what the cimode +# selects (CPU for mode 2; DEVICE for mode 3 -- i.e. the run must really +# have exercised GPU HYPRE, not silently fallen back). +# The coarse-solver location/precision are extracted and printed for the record. +# +# IMPORTANT SCOPE NOTE: cimode 2 runs the HYPRE BoomerAMG coarse solve on the +# CPU (nekRS default). A PASS here validates the CUDA main application + a +# CPU coarse solve; it does NOT validate GPU HYPRE. Use HPCPERF_NEKRS_CIMODE=3 +# (DEVICE coarse) to exercise the GPU HYPRE coarse solve. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" R="$(cd "$HERE/../.." && pwd)" +set +u; source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" N="${HPCPERF_GPUS:-1}" -OUT="$R/build/level3/nekrs/$MODEL/run/smoke.np$N.cimode2.log" +CIMODE="${HPCPERF_NEKRS_CIMODE:-2}" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-2400}" +VARIANT="${HPCPERF_NEKRS_VARIANT:-$([ "${HPCPERF_NEKRS_HYPRE_GPU:-ON}" = ON ] && echo hypregpu || echo cpucoarse)}" +if [ "$VARIANT" = hypregpu ]; then VBD="$R/build/level3/nekrs/$MODEL"; else VBD="$R/build/level3/nekrs/$VARIANT.$MODEL"; fi +OUT="$VBD/run/validate.cimode$CIMODE.np$N.log" + +# complete CI-check counts and required coarse-solver location, per cimode +case "$CIMODE" in + 2) EXPECT_CHECKS=9; WANT_COARSE=CPU ;; + 3) EXPECT_CHECKS=9; WANT_COARSE=DEVICE ;; + *) echo "validate.sh: expected CI-check count for cimode $CIMODE is not recorded here; add it after observing one run (refusing to guess)" >&2; exit 2 ;; +esac export HPCPERF_GPUS="$N" mkdir -p "$(dirname "$OUT")" -echo "validate.sh: nekRS $BACKEND ethier --cimode 2 (upstream CI mode) on $N GPU(s)" -set +e -HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" --cimode 2 > "$OUT" 2>&1 -rc=$? -set -e -grep -a -E '^#|hpcperf-launch: audit summary|CI test|L2 err|elapsedStepSum|total elapsed|ERROR|error' "$OUT" | grep -a -v 'no error' | sed 's/^/ /' | tail -40 -FAILED=$(grep -a -c 'CI test .* failed' "$OUT" || true) +echo "validate.sh: nekRS $BACKEND variant=$VARIANT ethier --cimode $CIMODE (upstream CI mode; expect $EXPECT_CHECKS checks, coarse=$WANT_COARSE) on $N GPU(s)" +rc=0 +HPCPERF_SCALE_MODE=smoke timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" --cimode "$CIMODE" > "$OUT" 2>&1 || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|CI test|L2 err|COARSE SOLVER LOCATION|COARSE SOLVER PRECISION|elapsedStepSum|ERROR|Abort|abort' "$OUT" | grep -aiv 'no error' | sed 's/^/ /' | tail -50 +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- run timed out after ${TIMEOUT}s"; exit 1; fi + +COARSE="$(grep -a 'COARSE SOLVER LOCATION' "$OUT" | head -1 | sed 's/.*value: *//' | tr -d ' ' || true)" PASSED=$(grep -a -c 'CI test .* passed' "$OUT" || true) -echo " nekrs exit code $rc; CI checks passed=$PASSED failed=$FAILED" -if [ "$rc" -eq 0 ] && [ "$FAILED" -eq 0 ] && [ "$PASSED" -gt 0 ]; then - echo "nekRS $BACKEND validation ($N GPU, ethier --cimode 2 vs upstream CI references): PASS"; exit 0 +FAILED=$(grep -a -c 'CI test .* failed' "$OUT" || true) +TOTAL=$((PASSED + FAILED)) +echo " nekrs exit code $rc; CI checks passed=$PASSED failed=$FAILED total=$TOTAL (expected $EXPECT_CHECKS); coarse solver location=${COARSE:-UNKNOWN}" + +ok=1 +[ "$rc" -eq 0 ] || { echo " run exited $rc"; ok=0; } +[ "$FAILED" -eq 0 ] || { echo " $FAILED CI check(s) failed"; ok=0; } +[ "$TOTAL" -eq "$EXPECT_CHECKS" ] || { echo " produced $TOTAL CI checks, expected the complete set of $EXPECT_CHECKS (incomplete run or changed CI)"; ok=0; } +[ "${COARSE:-}" = "$WANT_COARSE" ] || { echo " coarse solver location is '${COARSE:-UNKNOWN}', expected '$WANT_COARSE' -- the intended solve path did not run (no silent fallback allowed)"; ok=0; } +if [ "$ok" -eq 1 ]; then + echo "nekRS $BACKEND validation ($N GPU, ethier --cimode $CIMODE, $EXPECT_CHECKS/$EXPECT_CHECKS checks, coarse=$COARSE): PASS"; exit 0 fi -echo "nekRS $BACKEND validation ($N GPU, ethier --cimode 2 vs upstream CI references): FAIL (log: $OUT)"; exit 1 +echo "nekRS $BACKEND validation ($N GPU, ethier --cimode $CIMODE): FAIL (log: $OUT)"; exit 1 diff --git a/level3/sparta/build.sh b/level3/sparta/build.sh index ff6c1b2..5a338a3 100755 --- a/level3/sparta/build.sh +++ b/level3/sparta/build.sh @@ -26,6 +26,7 @@ set +u; # shellcheck disable=SC1091 source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # shellcheck disable=SC1091 source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # Level 3 builds must not see Level 2 .deps/install prefixes BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" diff --git a/level3/sparta/run.sh b/level3/sparta/run.sh index 5920a96..bb63152 100755 --- a/level3/sparta/run.sh +++ b/level3/sparta/run.sh @@ -55,10 +55,22 @@ case "$MODE" in X=$((L * PX)); Y=$((L * PY)); Z=$((L * PZ)) ;; esac CELLS=$((X * Y * Z)); PARTS=$((10 * CELLS)) -RUN_DIR="$BUILD_DIR/run"; mkdir -p "$RUN_DIR" +RUN_DIR="$BUILD_DIR/run" +[ -n "${HPCPERF_DRY_RUN:-}" ] && RUN_DIR="$RUN_DIR/.dryrun" # dry-run never overwrites real results +mkdir -p "$RUN_DIR" LOG="$RUN_DIR/log.$MODE.np$N_RANKS.sparta" +rm -f "$LOG" # validate only against THIS run's output; never a stale log echo "# SPARTA $BACKEND: mode=$MODE ranks=$N_RANKS grid=${X}x${Y}x${Z} = $CELLS cells, $PARTS particles ($((PARTS / N_RANKS))/rank), gpu-aware=$GAM, log=$LOG" cd "$SRC/bench" # ar.species / ar.vss are referenced relative to the deck -exec "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ +RUN_ID="$(l3_run_id)" +rc=0 +"$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- \ "$EXE" -k on g 1 -sf kk -pk kokkos gpu/aware "$GAM" \ - -in in.collide -var x "$X" -var y "$Y" -var z "$Z" -log "$LOG" -echo none "$@" + -in in.collide -var x "$X" -var y "$Y" -var z "$Z" -log "$LOG" -echo none "$@" || rc=$? +if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + l3_manifest "$RUN_DIR" "run_id=$RUN_ID" "app=sparta" "backend=$BACKEND" "mode=$MODE" \ + "ranks=$N_RANKS" "cells=$CELLS" "particles=$PARTS" "gpu_aware=$GAM" "exit_code=$rc" \ + "binary=$EXE" "binary_sha256=$(l3_sha_file "$EXE")" "input=$SRC/bench/in.collide" \ + "input_sha256=$(l3_sha_file "$SRC/bench/in.collide")" "log=$LOG" "utc=$(date -u +%FT%TZ)" +fi +exit "$rc" diff --git a/level3/sparta/validate.sh b/level3/sparta/validate.sh index c79b036..4dd47d9 100755 --- a/level3/sparta/validate.sh +++ b/level3/sparta/validate.sh @@ -5,85 +5,100 @@ # # ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) selects the rank count # -# DSMC is a stochastic method (random seed, random collision partners), and -# the particle distribution over ranks changes the random stream, so per-step -# collision counts cannot be compared exactly. What IS exact and what has a -# physically justified tolerance: -# * particle count Np at every stats row == 10 * cells (10,000 with the -# 10x10x10 deck: exact conservation -- no chemistry, reflecting walls); -# * gas temperature (compute temp): the equilibrated argon stays at the -# initial 273.15 K; the reference log shows 273.28 K. Tolerance 2 % on the -# mean over the benchmark steps (>= step 40): the statistical temperature -# noise of 10^4 particles is ~sqrt(2/3N) ~ 0.8 %, so 2 % is ~2.5 sigma of -# the sampling noise and far below any physics or unit error; -# * mean collision attempts per step (Natt) within 15 % of the reference -# mean: it is set by density/temperature/cross-section, so a wrong -# collision model or density would move it by far more; run-to-run -# statistical scatter is a few %. -# With HPCPERF_GPUS>1 the same three criteria are applied between the N-rank -# run and this build's 1-rank run (rank-count independence). Prints PASS/FAIL, -# exit 0/1. Nothing is loosened to pass; the deck is upstream's. +# DSMC is stochastic, so per-step collision counts cannot match exactly. What +# is checked (adapted subset of upstream's tolerance-based regression): +# * the benchmark stats block is COMPLETE -- it must span the equilibration +# boundary (step 30) through the final step (130); a truncated run FAILs; +# * particle count Np == 10,000 at every stats row (exact conservation); +# * mean gas temperature over steps >= 40 within 2 % of the reference +# (statistical noise ~0.8 % for 10^4 particles); +# * mean collision attempts (Natt) within 15 %. +# With HPCPERF_GPUS>1 the same criteria compare the N-rank run with this build's +# 1-rank run. Reproducibility: the run's real exit code is captured (nonzero / +# timeout / missing log / non-finite -> FAIL), run.sh removes its target log +# first so only this run's output is used, and NaN/Inf is rejected explicitly. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" R="$(cd "$HERE/../.." && pwd)" +set +u; source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" N="${HPCPERF_GPUS:-1}" REF="$R/_upstream/level3/sparta/bench/log.7Jul14.collide.icc.10K.1" RUN_DIR="$R/build/level3/sparta/$MODEL/run" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-900}" [ -f "$REF" ] || { echo "validate.sh: reference log $REF missing (run fetch.sh)" >&2; exit 1; } export HPCPERF_GPUS="$N" echo "validate.sh: SPARTA $BACKEND smoke (bench/in.collide 10x10x10, 10,000 particles) on $N GPU(s)" -HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit|Loop time|ERROR' || true +run_once() { local ng=$1 out=$2 rc=0; HPCPERF_GPUS="$ng" HPCPERF_SCALE_MODE=smoke timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" > "$out" 2>&1 || rc=$?; return $rc; } +mkdir -p "$RUN_DIR" +VOUT="$RUN_DIR/validate.smoke.np$N.stdout" +rc=0; run_once "$N" "$VOUT" || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|Loop time|ERROR|abort' "$VOUT" || true +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- run timed out after ${TIMEOUT}s"; exit 1; fi +[ "$rc" -eq 0 ] || { echo "validate.sh: FAIL -- run.sh exited $rc (see $VOUT)"; exit 1; } LOG="$RUN_DIR/log.smoke.np$N.sparta" [ -f "$LOG" ] || { echo "validate.sh: FAIL -- no log produced ($LOG)"; exit 1; } if [ "$N" -gt 1 ] && [ ! -f "$RUN_DIR/log.smoke.np1.sparta" ]; then echo "validate.sh: producing the 1-GPU run for rank-count comparison" - HPCPERF_GPUS=1 HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" > /dev/null 2>&1 || true + r1=0; run_once 1 "$RUN_DIR/validate.smoke.np1.stdout" || r1=$? + [ "$r1" -eq 0 ] && [ -f "$RUN_DIR/log.smoke.np1.sparta" ] || { echo "validate.sh: FAIL -- 1-GPU run failed (rc=$r1)"; exit 1; } fi python3 - "$LOG" "$REF" "$N" "$RUN_DIR/log.smoke.np1.sparta" <<'PY' -import re, sys +import re, sys, os +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import require_finite, ValidationError def stats(path): - """rows of the LAST stats block (the 100-step benchmark run) as dicts""" blocks, cur, cols = [], [], None for ln in open(path).read().splitlines(): p = ln.split() if p[:2] == ["Step", "CPU"]: - # the 2014 reference log labels the compute column "temp", current SPARTA prints "c_temp" cols = ["temp" if c == "c_temp" else c for c in p]; cur = []; blocks.append(cur); continue if cols and p and re.match(r'^\d+$', p[0]): - cur.append(dict(zip(cols, map(float, p)))) - elif cols and cur and not p: - cols = None + cur.append(dict(zip(cols, p))); continue + if cols and cur and not p: cols = None return blocks[-1] if blocks else [] +EXPECT_FIRST, EXPECT_LAST = 30, 130 # benchmark block: run 30 (equilibrate) then run 100 +FIELDS = ["Np", "temp", "Natt"] def mean(rows, k, minstep=40): - v = [r[k] for r in rows if r["Step"] >= minstep] - return sum(v) / len(v) if v else float("nan") + v = [require_finite(f"{k}@{int(r['Step'])}", r[k]) for r in rows if int(r["Step"]) >= minstep] + if not v: raise ValidationError(f"no rows with Step>={minstep} for {k}") + return sum(v) / len(v) log, ref, n, log1 = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] +NPART = 10.0 * 10 * 10 * 10 ok = True -def check(rows, base, label, npart): - global ok - if not rows: print(f" {label}: no stats rows"); ok = False; return - bad_np = [r["Step"] for r in rows if r["Np"] != npart] - print(f" {label}: Np == {npart} at every row: {'ok' if not bad_np else 'BAD at steps ' + str(bad_np)}"); ok &= not bad_np - for k, tol in (("temp", 0.02), ("Natt", 0.15)): - a, b = mean(rows, k), mean(base, k) - rel = abs(a - b) / abs(b) - print(f" {label}: mean {k:<5} {a:12.4f} vs {b:12.4f} rel {rel:.3e} (tol {tol}) {'ok' if rel <= tol else 'BAD'}") - ok &= rel <= tol -got, want = stats(log), stats(ref) -print(f"[1] {n}-GPU run vs upstream reference log (icc, 1 proc, 2014):") -NPART = 10.0 * 10 * 10 * 10 # deck: n = 10 * x*y*z particles = 10,000 for the 10x10x10 grid -check(got, want, "vs-ref", NPART) -if n > 1: - try: +try: + got, want = stats(log), stats(ref) + for label, rows in (("run", got), ("reference", want)): + if not rows: raise ValidationError(f"{label} log has no benchmark stats block") + for f in FIELDS: + if f not in rows[0]: raise ValidationError(f"{label} log missing field '{f}' (have {list(rows[0])})") + steps = [int(r["Step"]) for r in got] + if steps[0] != EXPECT_FIRST or steps[-1] != EXPECT_LAST: + raise ValidationError(f"run stats block spans steps {steps[0]}..{steps[-1]}, expected {EXPECT_FIRST}..{EXPECT_LAST} (truncated/incomplete run)") + def check(rows, base, label): + global ok + bad_np = [int(r["Step"]) for r in rows if require_finite("Np", r["Np"]) != NPART] + print(f" {label}: Np == {NPART:.0f} at every row: {'ok' if not bad_np else 'BAD at ' + str(bad_np)}"); ok &= not bad_np + for k, tol in (("temp", 0.02), ("Natt", 0.15)): + a, b = mean(rows, k), mean(base, k) + rel = abs(a - b) / abs(b) + print(f" {label}: mean {k:<5} {a:12.4f} vs {b:12.4f} rel {rel:.3e} (tol {tol}) {'ok' if rel <= tol else 'BAD'}") + ok &= rel <= tol + print(f"[1] {n}-GPU run vs upstream reference log (icc, 1 proc, 2014):") + check(got, want, "vs-ref") + if n > 1: one = stats(log1) + if not one: raise ValidationError("1-GPU log has no benchmark stats block") print(f"[2] {n}-GPU run vs this build's 1-GPU run:") - check(got, one, "vs-1gpu", NPART) - except FileNotFoundError: - print("[2] 1-GPU log unavailable; rank-count comparison skipped"); ok = False + check(got, one, "vs-1gpu") +except ValidationError as ex: + print(f" VALIDATION ERROR: {ex}") + print(f"SPARTA CUDA validation ({n} GPU): FAIL"); sys.exit(1) print(f"SPARTA CUDA validation ({n} GPU, bench/in.collide vs log.7Jul14.collide.icc.10K.1): {'PASS' if ok else 'FAIL'}") sys.exit(0 if ok else 1) PY diff --git a/level3/specfem3d/build.sh b/level3/specfem3d/build.sh index f75dc8e..ecee2e6 100755 --- a/level3/specfem3d/build.sh +++ b/level3/specfem3d/build.sh @@ -40,6 +40,7 @@ set +u; # shellcheck disable=SC1091 source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # shellcheck disable=SC1091 source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # Level 3 builds must not see Level 2 .deps/install prefixes BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" diff --git a/level3/specfem3d/run.sh b/level3/specfem3d/run.sh index 27d3e34..4af1fcd 100755 --- a/level3/specfem3d/run.sh +++ b/level3/specfem3d/run.sh @@ -61,7 +61,10 @@ EX="$R/_upstream/level3/specfem3d/EXAMPLES/applications/homogeneous_halfspace" N_RANKS="$(hpcperf_ranks specfem3d yes)" || exit 2 MODE="$(l3_scale_mode specfem3d)" || exit 2 BUILD_DIR="$R/build/level3/specfem3d/$MODEL" -RUN_DIR="$BUILD_DIR/run/$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR/OUTPUT_FILES/DATABASES_MPI" +# l3_rundir: dry-run gets a throwaway dir (the old code rm -rf'd the real run dir before the +# per-stage dry-run checks, deleting real seismograms/databases when only a plan was requested). +RUN_DIR="$(l3_rundir "$BUILD_DIR/run/$MODE.np$N_RANKS")" || exit 2 +mkdir -p "$RUN_DIR/OUTPUT_FILES/DATABASES_MPI" cp -r "$EX/DATA" "$RUN_DIR/DATA" PAR="$RUN_DIR/DATA/Par_file" sed -i -e "s/^NPROC *=.*/NPROC = $N_RANKS/" -e "s/^GPU_MODE *=.*/GPU_MODE = .true./" "$PAR" @@ -122,9 +125,17 @@ echo "# stage 2/3: xgenerate_databases on $N_RANKS ranks (CPU)" grep -E 'hpcperf-launch: (dry-run)' OUTPUT_FILES/output_generate_databases.log || true echo "# stage 3/3: xspecfem3D on $N_RANKS ranks (GPU_MODE)" t0=$(date +%s) -"${LAUNCH[@]}" "$BIN/xspecfem3D" 2>&1 | tee OUTPUT_FILES/output_specfem3D.log | grep -E 'hpcperf-launch|Error|ERROR|GPU|Time loop|Elapsed|End of' || true -rc=${PIPESTATUS[0]} +# Capture the solver's real exit code directly (a `... | tee | grep || true` pipeline resets +# PIPESTATUS via the trailing `true` and would report success even when the solver aborted). +rc=0 +"${LAUNCH[@]}" "$BIN/xspecfem3D" > OUTPUT_FILES/output_specfem3D.log 2>&1 || rc=$? +grep -E 'hpcperf-launch|Error|ERROR|GPU|Time loop|Elapsed|End of' OUTPUT_FILES/output_specfem3D.log || true [ "$rc" -eq 0 ] || { echo "run.sh: xspecfem3D exited $rc (see $RUN_DIR/OUTPUT_FILES/output_specfem3D.log)" >&2; exit "$rc"; } [ -n "${HPCPERF_DRY_RUN:-}" ] && exit 0 -echo "# solver wall time $(( $(date +%s)-t0 )) s; $(grep -E 'Total elapsed time in seconds|Time loop finished' OUTPUT_FILES/output_solver.txt 2>/dev/null | tr -s ' ' | tr '\n' ';')" +WALL=$(( $(date +%s)-t0 )) +echo "# solver wall time $WALL s; $(grep -E 'Total elapsed time in seconds|Time loop finished' OUTPUT_FILES/output_solver.txt 2>/dev/null | tr -s ' ' | tr '\n' ';')" echo "# seismograms: $(ls OUTPUT_FILES/*.semd 2>/dev/null | wc -l) files in $RUN_DIR/OUTPUT_FILES" +l3_manifest "$RUN_DIR" "run_id=$(l3_run_id)" "app=specfem3d" "backend=$BACKEND" "mode=$MODE" \ + "ranks=$N_RANKS" "elements=$ELEMS" "nstep=$STEPS" "dt=$DT" "solver_exit_code=$rc" \ + "solver_wall_s=$WALL" "binary=$BIN/xspecfem3D" "binary_sha256=$(l3_sha_file "$BIN/xspecfem3D")" \ + "par_file_sha256=$(l3_sha_file "$PAR")" "seismograms=$(ls OUTPUT_FILES/*.semd 2>/dev/null | wc -l)" "utc=$(date -u +%FT%TZ)" diff --git a/level3/specfem3d/validate.sh b/level3/specfem3d/validate.sh index de5d263..40e0ba9 100755 --- a/level3/specfem3d/validate.sh +++ b/level3/specfem3d/validate.sh @@ -4,21 +4,22 @@ # # ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) # -# Case: EXAMPLES/applications/homogeneous_halfspace as shipped (36x36x16 = 20,736 -# HEX8 elements, CMT source at 30 km depth, 4 stations, NSTEP 5000, DT 0.05 s), -# run in GPU mode with NPROC = N. Its README (step 7) says to "check with 6 -# reference seismograms in REF_SEIS/"; upstream's BuildBot uses -# utils/scripts/compare_seismogram_correlations.py, which reports per trace the -# correlation coefficient, the L2 misfit normalised by the reference energy, -# and the cross-correlation time shift, with upstream's thresholds -# TOL_CORR = 0.8, TOL_ERR = 0.01 (1 %), TOL_SHIFT = 0.01 s. The reference -# traces were produced on CPUs (double precision, 4 ranks); the GPU solver is -# single precision, so bitwise equality is not expected -- upstream's -# tolerance-based comparison is the appropriate criterion and is used -# unchanged. PASS = every trace within all three thresholds. +# Case: EXAMPLES/applications/homogeneous_halfspace as shipped (20,736 HEX8 +# elements, NSTEP 5000, DT 0.05 s), GPU_MODE, NPROC = N. Upstream's +# utils/scripts/compare_seismogram_correlations.py reports per trace the +# correlation, the L2 misfit (normalised by the reference energy) and the +# cross-correlation time shift; thresholds corr>=0.8, err<=1%, shift<=0.01 s. +# The GPU solver is single precision, so tolerance comparison (not bitwise) is +# the right criterion and is used unchanged. PASS requires: run.sh exited 0, +# EVERY reference trace was compared (count == number of REF_SEIS traces), all +# produced seismograms are finite, and the comparison reports no poor +# correlation / no poor match / no significant time shift. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" R="$(cd "$HERE/../.." && pwd)" +set +u; source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" N="${HPCPERF_GPUS:-1}" @@ -26,26 +27,53 @@ UP="$R/_upstream/level3/specfem3d" REF="$UP/EXAMPLES/applications/homogeneous_halfspace/REF_SEIS" CMP="$UP/utils/scripts/compare_seismogram_correlations.py" RUN_DIR="$R/build/level3/specfem3d/$MODEL/run/smoke.np$N" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-1800}" [ -d "$REF" ] && [ -f "$CMP" ] || { echo "validate.sh: $REF or $CMP missing (run fetch.sh)" >&2; exit 1; } +NREF="$(ls "$REF"/*.semd 2>/dev/null | wc -l)" +[ "$NREF" -ge 1 ] || { echo "validate.sh: no reference traces in $REF" >&2; exit 1; } export HPCPERF_GPUS="$N" -echo "validate.sh: SPECFEM3D $BACKEND homogeneous_halfspace (20,736 elements, 5000 steps) on $N GPU(s)" -HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Time loop|Elapsed time|End of the simulation|Error|ERROR' || true +echo "validate.sh: SPECFEM3D $BACKEND homogeneous_halfspace (20,736 elements, 5000 steps) on $N GPU(s); $NREF reference traces" +VOUT="$R/build/level3/specfem3d/$MODEL/run/validate.smoke.np$N.stdout"; mkdir -p "$(dirname "$VOUT")" +rc=0; HPCPERF_SCALE_MODE=smoke timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" > "$VOUT" 2>&1 || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|Time loop|Elapsed time|End of the simulation|Error|ERROR' "$VOUT" || true +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- run timed out after ${TIMEOUT}s"; exit 1; fi +[ "$rc" -eq 0 ] || { echo "validate.sh: FAIL -- run.sh exited $rc (see $VOUT)"; exit 1; } OUT="$RUN_DIR/OUTPUT_FILES" -ls "$OUT"/*.semd >/dev/null 2>&1 || { echo "validate.sh: FAIL -- no seismograms under $OUT"; exit 1; } +NGOT="$(ls "$OUT"/*.semd 2>/dev/null | wc -l)" +[ "$NGOT" -ge "$NREF" ] || { echo "validate.sh: FAIL -- produced $NGOT seismograms, need >= $NREF"; exit 1; } + +# reject non-finite samples in the produced traces before trusting the correlation +python3 - "$OUT" <<'PY' || { echo "validate.sh: FAIL -- non-finite sample in a produced seismogram"; exit 1; } +import sys, os, glob, math +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import ValidationError +try: + for f in sorted(glob.glob(sys.argv[1] + "/*.semd")): + nrow = 0 + for ln in open(f): + parts = ln.split() + if len(parts) < 2: continue + for v in parts[:2]: + if not math.isfinite(float(v)): raise ValidationError(f"{os.path.basename(f)}: non-finite sample {v}") + nrow += 1 + if nrow == 0: raise ValidationError(f"{os.path.basename(f)}: no samples") +except ValidationError as ex: + print(f" VALIDATION ERROR: {ex}"); sys.exit(1) +PY echo "validate.sh: comparing with upstream REF_SEIS (utils/scripts/compare_seismogram_correlations.py)" CMP_OUT="$RUN_DIR/compare_ref_seis.log" python3 "$CMP" "$OUT/" "$REF/" > "$CMP_OUT" 2>&1 || true grep -E '^\|' "$CMP_OUT" | sed 's/^/ /' grep -E 'seismograms compared|poor correlation|poor match|significant time shift|no poor|no significant' "$CMP_OUT" | sed 's/^/ /' +NCMP="$(grep -oE '^[0-9]+ seismograms compared' "$CMP_OUT" | awk '{print $1}')" ok=1 +[ "${NCMP:-0}" -eq "$NREF" ] || { echo " only ${NCMP:-0} of $NREF reference traces were compared"; ok=0; } grep -q 'no poor correlations found' "$CMP_OUT" || ok=0 grep -q 'no poor matches found' "$CMP_OUT" || ok=0 grep -q 'no significant time shifts found' "$CMP_OUT" || ok=0 -NCMP="$(grep -oE '^[0-9]+ seismograms compared' "$CMP_OUT" | awk '{print $1}')" -[ "${NCMP:-0}" -gt 0 ] || ok=0 if [ "$ok" -eq 1 ]; then - echo "SPECFEM3D $BACKEND validation ($N GPU, homogeneous_halfspace vs REF_SEIS, corr>=0.8 err<=1% shift<=0.01s): PASS"; exit 0 + echo "SPECFEM3D $BACKEND validation ($N GPU, homogeneous_halfspace vs REF_SEIS, $NREF/$NREF traces, corr>=0.8 err<=1% shift<=0.01s): PASS"; exit 0 fi echo "SPECFEM3D $BACKEND validation ($N GPU, homogeneous_halfspace vs REF_SEIS): FAIL (see $CMP_OUT)"; exit 1 diff --git a/level3/tools/l3_check.py b/level3/tools/l3_check.py new file mode 100644 index 0000000..08cf234 --- /dev/null +++ b/level3/tools/l3_check.py @@ -0,0 +1,42 @@ +"""l3_check -- shared helpers for the Level 3 validators. + +Every numeric quantity that enters a pass/fail decision goes through require_finite, +so a NaN/Inf (a diverged solve, an uninitialised field, a truncated reference) is an +explicit FAIL with a named quantity -- never a silently-swallowed comparison. The +validators import this via sys.path.insert(0, os.environ["L3_TOOLS"]). +""" +import math +import sys + + +class ValidationError(Exception): + pass + + +def require_finite(name, x): + """Return float(x) if finite; raise ValidationError naming the quantity otherwise.""" + try: + v = float(x) + except (TypeError, ValueError): + raise ValidationError(f"{name} is not a number ({x!r})") + if not math.isfinite(v): + raise ValidationError(f"{name} is not finite ({v})") + return v + + +def require_finite_seq(name, xs): + xs = list(xs) + if not xs: + raise ValidationError(f"{name}: empty sequence (no data)") + return [require_finite(f"{name}[{i}]", x) for i, x in enumerate(xs)] + + +def rel_error(name, got, ref): + g = require_finite(f"{name}(got)", got) + r = require_finite(f"{name}(ref)", ref) + return abs(g - r) / max(abs(r), 1e-300) + + +def fail(msg): + print(f" VALIDATION ERROR: {msg}") + sys.exit(1) diff --git a/level3/tools/l3_common.sh b/level3/tools/l3_common.sh index f0b23b1..1e168af 100755 --- a/level3/tools/l3_common.sh +++ b/level3/tools/l3_common.sh @@ -21,6 +21,7 @@ # level2/tools -- nothing is copied or moved, so Level 2 is not disturbed. L3_R="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +L3_TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export L3_TOOLS # for the validators' python (l3_check.py) HPCPERF_RUNTIME_DIR="${HPCPERF_RUNTIME_DIR:-$L3_R/level2/tools}" L3_LAUNCHER="$HPCPERF_RUNTIME_DIR/hpcperf_mpi_launch.sh" L3_TOPOLOGY="$HPCPERF_RUNTIME_DIR/hpcperf_topology.py" @@ -35,6 +36,20 @@ l3_paths() { mkdir -p "$L3_SRC" "$L3_BUILD_DEPS" "$L3_INSTALL" "$L3_LOGS" } +# l3_isolate_build_env: remove the Level 2 dependency prefixes (everything under +# $L3_R/.deps/install/, the validated Level 2 tree) from CMAKE_PREFIX_PATH and +# LD_LIBRARY_PATH before a Level 3 configure, so a Level 3 build can never pick +# up a Level 2 Kokkos/RAJA/hypre. Each Level 3 app owns its dependencies +# (bundled, or under .deps/level3/). Call once in build.sh after sourcing +# hpcperf_env.sh. Does not touch the launcher (level2/tools, not .deps/install). +l3_isolate_build_env() { + local before_c="${CMAKE_PREFIX_PATH:-}" before_l="${LD_LIBRARY_PATH:-}" + [ -n "$before_c" ] && export CMAKE_PREFIX_PATH="$(tr ':' '\n' <<<"$before_c" | grep -v "$L3_R/.deps/install/" | paste -sd:)" + [ -n "$before_l" ] && export LD_LIBRARY_PATH="$(tr ':' '\n' <<<"$before_l" | grep -v "$L3_R/.deps/install/" | paste -sd:)" + local n; n="$(tr ':' '\n' <<<"${CMAKE_PREFIX_PATH:-}" | grep -c "$L3_R/.deps/install/" || true)" + echo "# l3: build env isolated from Level 2 prefixes (CMAKE_PREFIX_PATH .deps/install entries remaining: $n)" +} + l3_first_line() { "$@" 2>/dev/null | head -n 1 || true; } l3_cuda_version() { nvcc --version 2>/dev/null | sed -n 's/^Cuda compilation tools, release [^,]*, V\([0-9][0-9.]*\).*$/\1/p' | head -n 1; } l3_gpu_arch() { # numeric compute capability of GPU 0, e.g. 100 @@ -46,11 +61,16 @@ l3_site_profile() { } # l3_fingerprint_text "" "" [patch files...] -# Prints the fingerprint for the configuration about to be built. +# Prints the fingerprint for the configuration about to be built. Patches are +# recorded IN THE ORDER GIVEN with their content sha256 (the source-cache key +# depends on both the upstream SHA and this ordered patch-content hash, so a +# same-named patch whose bytes change invalidates the cache). A patch path +# that does not exist, or whose hash cannot be taken, is a hard error -- the +# fingerprint is never written with a silently-missing patch. l3_fingerprint_text() { local app=$1 sha=$2 backend=$3 deps=$4 cmakeopts=$5 gam=$6; shift 6 - local p - echo "schema=l3-1" + local p h + echo "schema=l3-2" echo "application=$app" echo "upstream_commit=$sha" echo "backend=$backend arch=sm_$(l3_gpu_arch)" @@ -65,10 +85,20 @@ l3_fingerprint_text() { echo "site_profile=$(l3_site_profile)" echo "spack_lock_sha256=${L3_SPACK_LOCK_SHA:-none}" echo "container_image_sha256=${L3_CONTAINER_SHA:-none}" + local idx=0 ordered="" for p in "$@"; do - [ -e "$p" ] || continue - echo "patch=$(basename "$p") sha256=$(sha256sum "$p" | cut -d' ' -f1)" + idx=$((idx+1)) + if [ ! -f "$p" ]; then + echo "l3: patch file '$p' not found -- refusing to fingerprint a build with a missing patch" >&2 + return 1 + fi + h="$(sha256sum "$p" 2>/dev/null | cut -d' ' -f1)" + [ -n "$h" ] || { echo "l3: could not hash patch '$p'" >&2; return 1; } + echo "patch[$idx]=$(basename "$p") sha256=$h" + ordered="$ordered$h" done + # ordered content hash of the whole patch series (empty series -> the literal 'none') + echo "patch_series_sha256=$( [ -n "$ordered" ] && printf '%s' "$ordered" | sha256sum | cut -d' ' -f1 || echo none )" } # l3_fingerprint_check @@ -98,3 +128,76 @@ l3_scale_mode() { local m="${HPCPERF_SCALE_MODE:-smoke}" case "$m" in smoke|strong|weak) echo "$m";; *) echo "$1/run.sh: HPCPERF_SCALE_MODE must be smoke|strong|weak (got '$m')" >&2; return 2;; esac } + +# --------------------------------------------------------------------------- +# Result management (correctness / reproducibility) +# --------------------------------------------------------------------------- + +# l3_run_id: a unique id for one real execution (UTC, pid, random). +l3_run_id() { echo "$(date -u +%Y%m%dT%H%M%SZ)-$$-${RANDOM}"; } + +# l3_rundir +# Echoes the directory run.sh should actually write into, and prepares it. +# In a real run: rm -rf the intended dir and recreate it (fresh output only). +# In a dry-run (HPCPERF_DRY_RUN set): NEVER touch the real dir -- a throwaway +# sibling under .dryrun/ is used, so planning can never delete or overwrite a +# real result. Refuses to operate on a path that is not under a Level 3 +# build/ tree (guards against an accidental rm of the wrong directory). +l3_rundir() { + local real=$1 base parent + case "$real" in + "$L3_R"/build/level3/*) : ;; + *) echo "l3_rundir: refusing to manage '$real' (not under $L3_R/build/level3/)" >&2; return 2;; + esac + if [ -n "${HPCPERF_DRY_RUN:-}" ]; then + parent="$(dirname "$real")"; base="$(basename "$real")" + real="$parent/.dryrun/$base" + rm -rf "$real"; mkdir -p "$real" + else + rm -rf "$real"; mkdir -p "$real" + fi + printf '%s\n' "$real" +} + +# l3_capture -- +# Runs the command, copies combined stdout+stderr to , and returns +# the command's real exit status (NOT tee's). Nothing is swallowed; callers +# check the status. Use this instead of `cmd | grep ... || true`. +l3_capture() { + local log=$1; shift + [ "${1:-}" = -- ] && shift + mkdir -p "$(dirname "$log")" + set -o pipefail + "$@" 2>&1 | tee "$log" + local rc=${PIPESTATUS[0]} + set +o pipefail + return "$rc" +} + +# l3_manifest key=value ... +# Appends structured provenance for one real run. Records the run_id once. +l3_manifest() { + local dir=$1; shift + local f="$dir/run_manifest.txt" + { for kv in "$@"; do echo "$kv"; done; } >> "$f" +} + +# l3_sha_file : sha256 of a file, or the literal MISSING. +l3_sha_file() { [ -f "$1" ] && sha256sum "$1" 2>/dev/null | cut -d' ' -f1 || echo MISSING; } + +# l3_binary_backend_check +# Fails if the binary's GPU backend does not match what was requested (so a +# HIP request can never run a CUDA install and vice versa). Uses the linked +# runtime libraries (libcudart / libamdhip64) as the evidence. +l3_binary_backend_check() { + local exe=$1 want=$2 libs + [ -x "$exe" ] || { echo "l3: $exe not executable" >&2; return 1; } + libs="$(ldd "$exe" 2>/dev/null || true)" + case "$want" in + cuda) grep -q 'libcudart' <<<"$libs" || { echo "l3: $exe is not a CUDA binary (no libcudart linked) but CUDA was requested" >&2; return 1; } + grep -q 'libamdhip64' <<<"$libs" && { echo "l3: $exe links libamdhip64 (HIP) but CUDA was requested" >&2; return 1; } ;; + hip) grep -q 'libamdhip64' <<<"$libs" || { echo "l3: $exe is not a HIP binary (no libamdhip64 linked) but HIP was requested" >&2; return 1; } ;; + *) return 0 ;; + esac + return 0 +} diff --git a/level3/tools/tests/run_all.sh b/level3/tools/tests/run_all.sh new file mode 100755 index 0000000..5ea99e9 --- /dev/null +++ b/level3/tools/tests/run_all.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Level 3 CPU-only regression tests (no GPU, no application, no build): +# test_l3_infra.sh -- correctness/reproducibility helpers (l3_common.sh, l3_check.py): +# NaN/Inf rejection, real-exit-code capture, the failed-run +# gate that prevents a stale-log false PASS, the dry-run +# result-directory sentinel, and patch fingerprint / cache +# invalidation. +# Usage: level3/tools/tests/run_all.sh +set -u +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +rc=0 +run() { echo "=== $1"; shift; bash "$@" || { echo "=== FAILED: $*"; rc=1; }; echo; } +run "l3 infra (correctness/reproducibility)" "$HERE/test_l3_infra.sh" +[ $rc -eq 0 ] && echo "ALL LEVEL3 TEST GROUPS PASSED" || echo "SOME LEVEL3 TEST GROUPS FAILED" +exit $rc diff --git a/level3/tools/tests/test_l3_infra.sh b/level3/tools/tests/test_l3_infra.sh new file mode 100755 index 0000000..927d89d --- /dev/null +++ b/level3/tools/tests/test_l3_infra.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# CPU-only negative/positive tests for the Level 3 correctness & reproducibility +# helpers in level3/tools/l3_common.sh and l3_check.py. No GPU, no application, +# no build -- these check that the mechanisms which decide PASS/FAIL behave. +set -u +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../../.." && pwd)" +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" +TOOLS="$R/level3/tools" +pass=0; failn=0 +ok() { echo "ok $*"; pass=$((pass+1)); } +bad() { echo "FAIL $*"; failn=$((failn+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# 1. l3_check.require_finite rejects NaN/Inf/non-numbers, accepts finite +py_check() { python3 - "$1" <<'PY' +import os, sys +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import require_finite, ValidationError +try: + require_finite("x", sys.argv[1]); print("ACCEPT") +except ValidationError: + print("REJECT") +PY +} +[ "$(L3_TOOLS=$TOOLS py_check nan)" = REJECT ] && ok "1a: NaN rejected" || bad "1a: NaN not rejected" +[ "$(L3_TOOLS=$TOOLS py_check inf)" = REJECT ] && ok "1b: Inf rejected" || bad "1b: Inf not rejected" +[ "$(L3_TOOLS=$TOOLS py_check abc)" = REJECT ] && ok "1c: non-number rejected" || bad "1c: non-number not rejected" +[ "$(L3_TOOLS=$TOOLS py_check 1.5)" = ACCEPT ] && ok "1d: finite accepted" || bad "1d: finite rejected" + +# 2. l3_capture returns the COMMAND's exit code (not tee's), and saves output +rc=0; l3_capture "$TMP/cap.log" -- bash -c 'echo hello; exit 7' >/dev/null 2>&1 || rc=$? +[ "$rc" -eq 7 ] && ok "2a: l3_capture propagates real exit code (7)" || bad "2a: got rc=$rc, expected 7" +grep -q hello "$TMP/cap.log" && ok "2b: l3_capture saved stdout" || bad "2b: output not saved" + +# 3. rc-gate pattern: a failed run must FAIL even if a stale log is present +# (this is the logic every validator relies on). +echo "OLD PASS-looking log" > "$TMP/stale.log" +fake_validate() { # simulates: run fails (rc=1) but an old log exists + local rc=0 + bash -c 'exit 1' || rc=$? + [ "$rc" -eq 0 ] || return 1 # gate: nonzero run -> FAIL, regardless of stale log + return 0 +} +if fake_validate; then bad "3: stale-log gate let a failed run pass"; else ok "3: failed run FAILs even with a stale log present"; fi + +# 4. l3_rundir: dry-run must not touch a real result dir (sentinel test) +real="$R/build/level3/__selftest__/run/case.np1" +mkdir -p "$real"; echo SENTINEL > "$real/keep.txt" +d_real="$(l3_rundir "$real")" +[ "$d_real" = "$real" ] && [ ! -e "$real/keep.txt" ] && ok "4a: real run recreates the dir fresh" || bad "4a: real run did not refresh ($d_real)" +echo SENTINEL > "$real/keep.txt" +d_dry="$(HPCPERF_DRY_RUN=1 l3_rundir "$real")" +if [ "$d_dry" != "$real" ] && [ -f "$real/keep.txt" ] && [ "$(cat "$real/keep.txt")" = SENTINEL ]; then + ok "4b: dry-run used a scratch dir ($(basename "$(dirname "$d_dry")")/$(basename "$d_dry")) and left the real result untouched" +else bad "4b: dry-run touched the real result dir (d_dry=$d_dry)"; fi +d_bad=0; l3_rundir "/tmp/not-under-build" >/dev/null 2>&1 || d_bad=$? +[ "$d_bad" -ne 0 ] && ok "4c: l3_rundir refuses a path outside build/level3/" || bad "4c: accepted an out-of-tree path" +rm -rf "$R/build/level3/__selftest__" + +# 5. fingerprint patch handling: missing patch is an error; changed content +# changes the ordered series hash (cache-invalidation), same content stable. +export CXX=/bin/true FC=/bin/true +fp_missing=0; l3_fingerprint_text app sha cuda deps opts gam "$TMP/nope.patch" >/dev/null 2>&1 || fp_missing=$? +[ "$fp_missing" -ne 0 ] && ok "5a: missing patch is a hard error" || bad "5a: missing patch accepted" +printf 'A\n' > "$TMP/p.patch" +h1="$(l3_fingerprint_text app sha cuda deps opts gam "$TMP/p.patch" 2>/dev/null | sed -n 's/^patch_series_sha256=//p')" +printf 'B\n' > "$TMP/p.patch" # same name, different content +h2="$(l3_fingerprint_text app sha cuda deps opts gam "$TMP/p.patch" 2>/dev/null | sed -n 's/^patch_series_sha256=//p')" +[ -n "$h1" ] && [ "$h1" != "$h2" ] && ok "5b: same-named patch with changed content changes the series hash (cache invalidated)" || bad "5b: series hash did not change ($h1 vs $h2)" +hn="$(l3_fingerprint_text app sha cuda deps opts gam 2>/dev/null | sed -n 's/^patch_series_sha256=//p')" +[ "$hn" = none ] && ok "5c: empty patch series -> 'none'" || bad "5c: empty series hash '$hn'" + +echo +echo "test_l3_infra: $pass passed, $failn failed" +[ "$failn" -eq 0 ] diff --git a/level3/warpx/build.sh b/level3/warpx/build.sh index aebacc9..9c3e841 100755 --- a/level3/warpx/build.sh +++ b/level3/warpx/build.sh @@ -24,6 +24,7 @@ set +u; # shellcheck disable=SC1091 source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # shellcheck disable=SC1091 source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # Level 3 builds must not see Level 2 .deps/install prefixes BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" diff --git a/level3/warpx/run.sh b/level3/warpx/run.sh index 2aec1d8..2e011b4 100755 --- a/level3/warpx/run.sh +++ b/level3/warpx/run.sh @@ -92,7 +92,10 @@ MGS=$BX; [ "$BY" -gt "$MGS" ] && MGS=$BY; [ "$BZ" -gt "$MGS" ] && MGS=$BZ CELLS=$((NX * NY * NZ)) if [ "$CASE" = langmuir ]; then PARTS=$((2 * CELLS)); else PARTS=$((2 * CELLS)); fi -RUN_DIR="$BUILD_DIR/run/$CASE.$MODE.np$N_RANKS"; rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR" +# l3_rundir: real runs get a fresh dir; a dry-run gets a throwaway .dryrun/ dir so it +# can never delete or overwrite a real result directory (the old code rm -rf'd the real +# dir before the launcher's dry-run check ever ran). +RUN_DIR="$(l3_rundir "$BUILD_DIR/run/$CASE.$MODE.np$N_RANKS")" || exit 2 IN="$RUN_DIR/inputs" { echo "# derived from upstream $(realpath --relative-to="$R/_upstream/level3/WarpX" "$BASE") (HPC-Performance-AI level3/warpx/run.sh)" @@ -119,5 +122,13 @@ IN="$RUN_DIR/inputs" echo "# WarpX $BACKEND: case=$CASE mode=$MODE ranks=$N_RANKS grid=${NX}x${NY}x${NZ} ($CELLS cells, $PARTS particles, $((PARTS / N_RANKS))/rank) numprocs=${PX}x${PY}x${PZ} box=${BX}x${BY}x${BZ} steps=$STEPS run_dir=$RUN_DIR" cd "$RUN_DIR" +RUN_ID="$(l3_run_id)" "$L3_LAUNCHER" --gpus "$N_RANKS" --bind wrapper -- "$EXE" "$IN" "$@" 2>&1 | tee "$RUN_DIR/stdout.log" -exit "${PIPESTATUS[0]}" +rc=${PIPESTATUS[0]} +if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + l3_manifest "$RUN_DIR" "run_id=$RUN_ID" "app=warpx" "backend=$BACKEND" "case=$CASE" "mode=$MODE" \ + "ranks=$N_RANKS" "grid=${NX}x${NY}x${NZ}" "numprocs=${PX}x${PY}x${PZ}" "particles=$PARTS" "steps=$STEPS" \ + "exit_code=$rc" "binary=$EXE" "binary_sha256=$(l3_sha_file "$EXE")" "input=$IN" "input_sha256=$(l3_sha_file "$IN")" \ + "stdout=$RUN_DIR/stdout.log" "utc=$(date -u +%FT%TZ)" +fi +exit "$rc" diff --git a/level3/warpx/validate.sh b/level3/warpx/validate.sh index 47d794e..91b2cdc 100755 --- a/level3/warpx/validate.sh +++ b/level3/warpx/validate.sh @@ -4,114 +4,133 @@ # # ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1) # -# [1] test_3d_langmuir_multi (Examples/Tests/langmuir): an electron/positron -# plasma wave whose fields are known analytically, -# Ex = eps m_e c^2 kx/e sin(kx x) cos(ky y) cos(kz z) sin(wp t) (and cyclic), -# 64^3 cells, 40 steps. Upstream's analysis_3d.py compares the cell-centred -# Ex/Ey/Ez of the final plotfile with this solution and requires -# max|E_sim - E_th| / max|E_th| < 5e-2 for each component, and (Esirkepov -# deposition) charge conservation max|divE - rho/eps0| / max|rho/eps0| < -# 1e-11. The same checks are re-implemented here (upstream's script needs -# yt/openPMD-viewer, not available in this environment): the plotfile is -# read directly (AMReX native format), the formulas, grid positions and -# tolerances are upstream's. This is architecture-independent, unlike -# upstream's checksum baselines (documented as platform-dependent). -# [2] uniform_plasma smoke run (the performance case): the macroparticle count -# must be constant at every step (periodic box, no ionisation) -- exact; the -# particle+field energy time series is recorded for information (the -# shipped 2-particles-per-cell thermal plasma with E=0 initial fields is -# not an energy-conservation test: a few % change over the first plasma -# periods is expected for the momentum-conserving Yee scheme). -# PASS = [1] both criteria on N GPUs and [2] exact particle conservation. +# [1] test_3d_langmuir_multi: an e-/e+ plasma wave with an analytic field +# solution. Adapted subset of upstream analysis_3d.py (which needs +# yt/openPMD-viewer, not installed): the final plotfile's Ex/Ey/Ez are +# compared with the exact solution -- require max|E_sim-E_th|/max|E_th|<5e-2 +# each -- and (Esirkepov) charge conservation max|divE-rho/eps0|/ +# max|rho/eps0|<1e-11, with WarpX's own CODATA-2022 constants. Reader is +# hardened: it FAILs on a missing/short plotfile, a box set that does not +# cover the domain (truncation), or any non-finite field value. +# [2] uniform_plasma smoke: macroparticle count constant AND finite at every +# step, and the run must reach the final step; the energy series is recorded +# for information only (not a pass/fail criterion, see header of run.sh). +# Reproducibility: the run's real exit code is captured (nonzero/timeout -> +# FAIL); run.sh writes a fresh per-run directory (dry-run never touches it). set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" R="$(cd "$HERE/../.." && pwd)" -set +u; # shellcheck disable=SC1091 -source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u # conda python3 + numpy for the analysis +set +u; source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" MODEL="$(echo "$BACKEND" | tr '[:upper:]' '[:lower:]')" N="${HPCPERF_GPUS:-1}" RUNS="$R/build/level3/warpx/$MODEL/run" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-1800}" python3 -c 'import numpy' 2>/dev/null || { echo "validate.sh: python3 with numpy required for the plotfile analysis" >&2; exit 1; } export HPCPERF_GPUS="$N" ok=1 +run_case() { local case=$1 mode=$2 out=$3 rc=0; HPCPERF_WARPX_CASE="$case" HPCPERF_SCALE_MODE="$mode" timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" > "$out" 2>&1 || rc=$?; return $rc; } + echo "validate.sh: [1] WarpX $BACKEND langmuir_multi (64^3, 40 steps, analytic solution) on $N GPU(s)" -HPCPERF_WARPX_CASE=langmuir "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' || true +mkdir -p "$RUNS"; L1="$RUNS/validate.langmuir.np$N.stdout" +rc=0; run_case langmuir validate "$L1" || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' "$L1" || true +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- langmuir run timed out after ${TIMEOUT}s"; exit 1; fi +[ "$rc" -eq 0 ] || { echo "validate.sh: FAIL -- langmuir run.sh exited $rc (see $L1)"; exit 1; } PLT="$RUNS/langmuir.validate.np$N/diags/diag1000040" -[ -f "$PLT/Header" ] || { echo "validate.sh: FAIL -- plotfile $PLT not produced"; exit 1; } +[ -f "$PLT/Header" ] || { echo "validate.sh: FAIL -- plotfile $PLT not produced (run did not reach step 40)"; exit 1; } python3 - "$PLT" <<'PY' || ok=0 -import sys, re, numpy as np -# WarpX's own constants (Source/ablastr/constant.H, CODATA 2022; scipy >= 1.15 as used by upstream's -# analysis_3d.py carries the same values). With CODATA 2018 eps0 the divE - rho/eps0 residual would show a -# spurious uniform 6.8e-10 offset (= the eps0 revision), 68x upstream's 1e-11 tolerance. +import sys, re, os +import numpy as np +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import require_finite, ValidationError c, e, epsilon_0, m_e = 299792458.0, 1.602176634e-19, 8.8541878188e-12, 9.1093837139e-31 plt = sys.argv[1] -# ---- AMReX plotfile reader (single level, cell-centred data) ---- -hdr = open(f"{plt}/Header").read().split("\n") -ncomp = int(hdr[1]); names = hdr[2:2 + ncomp]; i = 2 + ncomp -dim = int(hdr[i]); time = float(hdr[i + 1]); i += 3 -lo = [float(v) for v in hdr[i].split()]; hi = [float(v) for v in hdr[i + 1].split()] -dom = re.search(r"\(\((\d+),(\d+),(\d+)\) \((\d+),(\d+),(\d+)\)", hdr[i + 3]) -n = [int(dom.group(k + 4)) - int(dom.group(k + 1)) + 1 for k in range(3)] -ch = open(f"{plt}/Level_0/Cell_H").read().split("\n") -boxes = [tuple(int(v) for v in m.groups()) for m in re.finditer(r"\(\((-?\d+),(-?\d+),(-?\d+)\) \((-?\d+),(-?\d+),(-?\d+)\) \(", "\n".join(ch))] -fabs = [(m.group(1), int(m.group(2))) for m in re.finditer(r"FabOnDisk: (\S+) (\d+)", "\n".join(ch))] -assert len(boxes) == len(fabs) > 0, (len(boxes), len(fabs)) -data = np.zeros((ncomp, n[0], n[1], n[2])) -for (lx, ly, lz, hx, hy, hz), (fname, off) in zip(boxes, fabs): - with open(f"{plt}/Level_0/{fname}", "rb") as f: - f.seek(off); line = b"" - while not line.endswith(b"\n"): line += f.read(1) - h = line.decode() - # "FAB ((8, (64 11 52 0 1 12 0 1023)),(8, (8 7 6 5 4 3 2 1)))((lo) (hi) (0,0,0)) ncomp": the second - # descriptor is the byte order of the 8-byte reals (8 7 ... 1 = little endian) - order = re.search(r"\(\d+, \((\d)(?: \d){7}\)\)\)", h).group(1) - dt = "f8" - nc = int(h.strip().split()[-1]) - shape = (hx - lx + 1, hy - ly + 1, hz - lz + 1) - arr = np.frombuffer(f.read(8 * nc * np.prod(shape)), dtype=dt).reshape((nc, shape[2], shape[1], shape[0])).transpose(0, 3, 2, 1) - data[:, lx:hx + 1, ly:hy + 1, lz:hz + 1] = arr -comp = {nm: data[k] for k, nm in enumerate(names)} -# ---- upstream analysis_3d.py, verbatim parameters ---- -epsilon, nden = 0.01, 4.0e24 -Ncell = n -kx, ky, kz = [2.0 * np.pi * 2 / (hi[d] - lo[d]) for d in range(3)] -wp = np.sqrt(nden * e**2 / (m_e * epsilon_0)) -k = {"Ex": kx, "Ey": ky, "Ez": kz}; cos = {"Ex": (0, 1, 1), "Ey": (1, 0, 1), "Ez": (1, 1, 0)} -def contrib(is_cos, kk, d): - du = (hi[d] - lo[d]) / Ncell[d]; u = lo[d] + du * (0.5 + np.arange(Ncell[d])) - return np.cos(kk * u) if is_cos else np.sin(kk * u) -def theory(field, t): - amp = epsilon * (m_e * c**2 * k[field]) / e * np.sin(wp * t) - cf = cos[field] - return amp * contrib(cf[0], kx, 0)[:, None, None] * contrib(cf[1], ky, 1)[None, :, None] * contrib(cf[2], kz, 2)[None, None, :] -print(f" plotfile time t = {time:.6e} s (wp t = {wp*time:.4f}), grid {n}, {len(boxes)} box(es), fields {names[:6]}...") -ok = True; err = 0.0 -for fld in ("Ex", "Ey", "Ez"): - th = theory(fld, time); m = abs(comp[fld] - th).max() / abs(th).max(); err = max(err, m) - print(f" {fld}: max|E_sim-E_th|/max|E_th| = {m:.3e}") -print(f" error_rel = {err:.3e} (upstream tolerance_rel 5e-2) {'ok' if err < 5e-2 else 'BAD'}"); ok &= err < 5e-2 -rho, divE = comp["rho"], comp["divE"] -ce = np.amax(np.abs(divE - rho / epsilon_0)) / np.amax(np.abs(rho / epsilon_0)) -print(f" charge conservation max|divE-rho/eps0|/max|rho/eps0| = {ce:.3e} (upstream tolerance 1e-11) {'ok' if ce < 1e-11 else 'BAD'}"); ok &= ce < 1e-11 -sys.exit(0 if ok else 1) +try: + hdr = open(f"{plt}/Header").read().split("\n") + ncomp = int(hdr[1]); names = hdr[2:2 + ncomp]; i = 2 + ncomp + time = float(hdr[i + 1]); i += 3 + lo = [float(v) for v in hdr[i].split()]; hi = [float(v) for v in hdr[i + 1].split()] + dom = re.search(r"\(\((\d+),(\d+),(\d+)\) \((\d+),(\d+),(\d+)\)", hdr[i + 3]) + n = [int(dom.group(k + 4)) - int(dom.group(k + 1)) + 1 for k in range(3)] + for fld in ("Ex", "Ey", "Ez", "rho", "divE"): + if fld not in names: raise ValidationError(f"plotfile missing field '{fld}' (have {names})") + ch = open(f"{plt}/Level_0/Cell_H").read().split("\n") + boxes = [tuple(int(v) for v in m.groups()) for m in re.finditer(r"\(\((-?\d+),(-?\d+),(-?\d+)\) \((-?\d+),(-?\d+),(-?\d+)\) \(", "\n".join(ch))] + fabs = [(m.group(1), int(m.group(2))) for m in re.finditer(r"FabOnDisk: (\S+) (\d+)", "\n".join(ch))] + if not (len(boxes) == len(fabs) > 0): raise ValidationError(f"plotfile box/fab mismatch ({len(boxes)} boxes, {len(fabs)} fabs)") + data = np.full((ncomp, n[0], n[1], n[2]), np.nan) # nan-init: any uncovered cell trips the finite check + covered = np.zeros((n[0], n[1], n[2]), bool) + for (lx, ly, lz, hx, hy, hz), (fname, off) in zip(boxes, fabs): + with open(f"{plt}/Level_0/{fname}", "rb") as f: + f.seek(off); line = b"" + while not line.endswith(b"\n"): line += f.read(1) + h = line.decode() + order = re.search(r"\(\d+, \((\d)(?: \d){7}\)\)\)", h).group(1) + dt = "f8" + nc = int(h.strip().split()[-1]) + shape = (hx - lx + 1, hy - ly + 1, hz - lz + 1) + raw = np.frombuffer(f.read(8 * nc * int(np.prod(shape))), dtype=dt) + if raw.size != nc * int(np.prod(shape)): raise ValidationError(f"FAB {fname} truncated: {raw.size} of {nc*int(np.prod(shape))} reals") + arr = raw.reshape((nc, shape[2], shape[1], shape[0])).transpose(0, 3, 2, 1) + data[:, lx:hx + 1, ly:hy + 1, lz:hz + 1] = arr + covered[lx:hx + 1, ly:hy + 1, lz:hz + 1] = True + if not covered.all(): raise ValidationError(f"plotfile boxes cover only {covered.mean()*100:.1f}% of the {n} domain (missing boxes)") + comp = {nm: data[k] for k, nm in enumerate(names)} + epsilon, nden = 0.01, 4.0e24 + kx, ky, kz = [2.0 * np.pi * 2 / (hi[d] - lo[d]) for d in range(3)] + wp = np.sqrt(nden * e**2 / (m_e * epsilon_0)) + kmap = {"Ex": kx, "Ey": ky, "Ez": kz}; cosf = {"Ex": (0, 1, 1), "Ey": (1, 0, 1), "Ez": (1, 1, 0)} + def contrib(is_cos, kk, d): + du = (hi[d] - lo[d]) / n[d]; u = lo[d] + du * (0.5 + np.arange(n[d])) + return np.cos(kk * u) if is_cos else np.sin(kk * u) + def theory(field, t): + amp = epsilon * (m_e * c**2 * kmap[field]) / e * np.sin(wp * t); cf = cosf[field] + return amp * contrib(cf[0], kx, 0)[:, None, None] * contrib(cf[1], ky, 1)[None, :, None] * contrib(cf[2], kz, 2)[None, None, :] + require_finite("plotfile time", time) + print(f" plotfile time t = {time:.6e} s (wp t = {wp*time:.4f}), grid {n}, {len(boxes)} box(es), full coverage") + err = 0.0 + for fld in ("Ex", "Ey", "Ez"): + th = theory(fld, time) + m = require_finite(f"max|{fld}_sim-{fld}_th|/max|{fld}_th|", abs(comp[fld] - th).max() / abs(th).max()) + err = max(err, m); print(f" {fld}: max|E_sim-E_th|/max|E_th| = {m:.3e}") + print(f" error_rel = {err:.3e} (upstream tolerance_rel 5e-2) {'ok' if err < 5e-2 else 'BAD'}") + ce = require_finite("charge-conservation residual", np.amax(np.abs(comp["divE"] - comp["rho"]/epsilon_0)) / np.amax(np.abs(comp["rho"]/epsilon_0))) + print(f" charge conservation max|divE-rho/eps0|/max|rho/eps0| = {ce:.3e} (upstream tolerance 1e-11) {'ok' if ce < 1e-11 else 'BAD'}") + sys.exit(0 if (err < 5e-2 and ce < 1e-11) else 1) +except ValidationError as ex: + print(f" VALIDATION ERROR: {ex}"); sys.exit(1) PY echo "validate.sh: [2] WarpX $BACKEND uniform_plasma smoke (64x32x32, 131,072 particles) on $N GPU(s)" -HPCPERF_SCALE_MODE=smoke "$HERE/run.sh" "$BACKEND" 2>&1 | grep -E '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' || true +L2="$RUNS/validate.uniform_plasma.np$N.stdout" +rc=0; run_case uniform_plasma smoke "$L2" || rc=$? +grep -aE '^#|hpcperf-launch: audit summary|Total Time|ERROR|abort' "$L2" || true +if [ "$rc" -eq 124 ]; then echo "validate.sh: FAIL -- uniform_plasma run timed out"; exit 1; fi +[ "$rc" -eq 0 ] || { echo "validate.sh: FAIL -- uniform_plasma run.sh exited $rc (see $L2)"; ok=0; } D="$RUNS/uniform_plasma.smoke.np$N/diags/reducedfiles" [ -f "$D/NP.txt" ] || { echo "validate.sh: FAIL -- reduced diagnostics not produced under $D"; exit 1; } python3 - "$D" <<'PY' || ok=0 -import sys +import sys, os +sys.path.insert(0, os.environ["L3_TOOLS"]) +from l3_check import require_finite, ValidationError d = sys.argv[1] def load(p): return [[float(x) for x in l.split()] for l in open(p) if l.strip() and not l.startswith('#')] -npart = load(f"{d}/NP.txt"); ep = load(f"{d}/EP.txt"); ef = load(f"{d}/EF.txt") -vals = sorted(set(r[2] for r in npart)) -print(f" ParticleNumber over steps {int(npart[0][0])}..{int(npart[-1][0])}: {vals} -> {'ok (exact)' if len(vals) == 1 else 'BAD'}") -e0 = ep[0][2] + ef[0][2]; e1 = ep[-1][2] + ef[-1][2] -print(f" for the record: E_particles+E_fields = {e0:.6e} J at step {int(ep[0][0])}, {e1:.6e} J at step {int(ep[-1][0])} (rel change {(e1-e0)/e0:+.3e}; not a pass/fail criterion, see header)") -sys.exit(0 if len(vals) == 1 else 1) +try: + npart, ep, ef = load(f"{d}/NP.txt"), load(f"{d}/EP.txt"), load(f"{d}/EF.txt") + if not npart: raise ValidationError("NP.txt has no rows") + steps = [int(r[0]) for r in npart] + if steps[-1] != 10: raise ValidationError(f"uniform_plasma reached step {steps[-1]}, expected final step 10 (incomplete run)") + vals = sorted(set(require_finite(f"Np@{int(r[0])}", r[2]) for r in npart)) + print(f" ParticleNumber over steps {steps[0]}..{steps[-1]}: {vals} -> {'ok (exact, finite)' if len(vals) == 1 else 'BAD'}") + e0 = require_finite("E0", ep[0][2] + ef[0][2]); e1 = require_finite("E1", ep[-1][2] + ef[-1][2]) + print(f" for the record: E_particles+E_fields = {e0:.6e} J at step {int(ep[0][0])}, {e1:.6e} J at step {int(ep[-1][0])} (rel change {(e1-e0)/e0:+.3e}; informational, see header)") + sys.exit(0 if len(vals) == 1 else 1) +except ValidationError as ex: + print(f" VALIDATION ERROR: {ex}"); sys.exit(1) PY if [ "$ok" -eq 1 ]; then echo "WarpX $BACKEND validation ($N GPU, langmuir_multi analytic + charge conservation, particle conservation): PASS"; exit 0; fi From 9327c264a14939273417647ea496ba437c7ff6c4 Mon Sep 17 00:00:00 2001 From: SWE-bench Date: Sat, 5 Sep 2026 20:53:19 -0400 Subject: [PATCH 03/52] Level 3 tools: per-profile paths and static-cudart backend check for the second batch l3_paths_profile gives every second-batch configuration its own .deps/level3///{src,build,install,logs,cache} tree and build directory (profiles never share a mutable source tree or install); l3_version_mm derives the profile name components. l3_binary_backend_check now accepts a CUDA binary that links cudart statically (CMake's default for AMReX-based applications) when cuobjdump finds embedded device code, instead of reporting it as non-CUDA. Fingerprint schema unchanged (l3-2). --- level3/tools/l3_common.sh | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/level3/tools/l3_common.sh b/level3/tools/l3_common.sh index 1e168af..9768896 100755 --- a/level3/tools/l3_common.sh +++ b/level3/tools/l3_common.sh @@ -36,6 +36,24 @@ l3_paths() { mkdir -p "$L3_SRC" "$L3_BUILD_DEPS" "$L3_INSTALL" "$L3_LOGS" } +# l3_paths_profile : second-batch layout -- one private tree per +# *configuration profile* (compiler/Toolkit/backend/key-dependency variant): +# $R/.deps/level3///{src,build,install,logs,cache} +# exports L3_APP, L3_PROFILE, L3_DEPS, L3_SRC, L3_BUILD_DEPS, L3_INSTALL, L3_LOGS, +# L3_CACHE and L3_BUILD (= $R/build/level3//, the application build +# tree). Different profiles never share a mutable source tree or an install. +l3_paths_profile() { + L3_APP="$1"; L3_PROFILE="$2" + case "$L3_PROFILE" in ""|*/*|.*) echo "l3_paths_profile: invalid profile name '$L3_PROFILE'" >&2; return 2;; esac + L3_DEPS="$L3_R/.deps/level3/$L3_APP/$L3_PROFILE" + L3_SRC="$L3_DEPS/src"; L3_BUILD_DEPS="$L3_DEPS/build"; L3_INSTALL="$L3_DEPS/install"; L3_LOGS="$L3_DEPS/logs"; L3_CACHE="$L3_DEPS/cache" + L3_BUILD="$L3_R/build/level3/$L3_APP/$L3_PROFILE" + mkdir -p "$L3_SRC" "$L3_BUILD_DEPS" "$L3_INSTALL" "$L3_LOGS" "$L3_CACHE" +} + +# l3_version_mm : "13.2.78" -> "132", "13.3.0" -> "133" (profile-name component) +l3_version_mm() { echo "$1" | awk -F. '{printf "%s%s", $1, $2}'; } + # l3_isolate_build_env: remove the Level 2 dependency prefixes (everything under # $L3_R/.deps/install/, the validated Level 2 tree) from CMAKE_PREFIX_PATH and # LD_LIBRARY_PATH before a Level 3 configure, so a Level 3 build can never pick @@ -187,14 +205,22 @@ l3_sha_file() { [ -f "$1" ] && sha256sum "$1" 2>/dev/null | cut -d' ' -f1 || ech # l3_binary_backend_check # Fails if the binary's GPU backend does not match what was requested (so a -# HIP request can never run a CUDA install and vice versa). Uses the linked -# runtime libraries (libcudart / libamdhip64) as the evidence. +# HIP request can never run a CUDA install and vice versa). Evidence: the +# linked runtime libraries (libcudart / libamdhip64); a binary that links the +# CUDA runtime statically (CMake's default CUDA_RUNTIME_LIBRARY=Static, e.g. +# AMReX-based apps) is accepted when cuobjdump finds embedded device code. l3_binary_backend_check() { local exe=$1 want=$2 libs [ -x "$exe" ] || { echo "l3: $exe not executable" >&2; return 1; } libs="$(ldd "$exe" 2>/dev/null || true)" case "$want" in - cuda) grep -q 'libcudart' <<<"$libs" || { echo "l3: $exe is not a CUDA binary (no libcudart linked) but CUDA was requested" >&2; return 1; } + cuda) if ! grep -q 'libcudart' <<<"$libs"; then + if command -v cuobjdump >/dev/null 2>&1 && cuobjdump --list-elf "$exe" 2>/dev/null | grep -q 'sm_'; then + : # static cudart with embedded CUDA device code + else + echo "l3: $exe is not a CUDA binary (no libcudart linked, no embedded CUDA ELF) but CUDA was requested" >&2; return 1 + fi + fi grep -q 'libamdhip64' <<<"$libs" && { echo "l3: $exe links libamdhip64 (HIP) but CUDA was requested" >&2; return 1; } ;; hip) grep -q 'libamdhip64' <<<"$libs" || { echo "l3: $exe is not a HIP binary (no libamdhip64 linked) but HIP was requested" >&2; return 1; } ;; *) return 0 ;; From 769482f0d7f21416ba9c204f38a26f3af9b9cd7b Mon Sep 17 00:00:00 2001 From: SWE-bench Date: Sat, 5 Sep 2026 20:54:18 -0400 Subject: [PATCH 04/52] Level 3 second batch: Nyx 26.09 (CUDA sm_100) built, validated at 1/2/4 GPUs Native CMake build of Nyx 26.09 against a private AMReX 26.09 install (the AMReX commit Nyx pins cannot emit sm_100 through CMake: its convert_cuda_archs drops SM >= 10.0 and autodetects 8.6+PTX on this node; 26.09 is a strict descendant and resolves sm_100 correctly). Profiles cuda132-gcc133-adiabatic and a cpu-gcc133-adiabatic reference (with AMReX plotfile tools and particle_compare); double-precision particles as upstream's regression builds. Cases are upstream's decks: MiniSB (nightly GPU regression test, inputs.32 + ppm_type=0), LyA-adiabatic (inputs.rt.garuda), the 64^3 LyA science deck as a named adiabatic derivative for strong scaling, and the Scaling deck (RandomPerCell, labelled synthetic) for strong/weak. Fixed BoxArray across rank counts; ranks > boxes refused. validate.sh: completeness/finiteness, upstream's fcompare tolerance (2e-10) against a same-configuration rerun (1 GPU) or the 1-GPU run (2/4 GPUs), a CPU-backend reference at a pre-fixed 1e-8, baryon-mass conservation and exact DM counts. AMReX's particle_compare cannot compare across rank counts (header equality incl. next_id; exit code 0 even on "FAIL"), so nyx_particle_compare.py matches particles through their exact t=0 positions (checkpoints at step 0 and the final step) and applies the same norms. Results: VALIDATED_PASS at 1/2/4 GPUs for both official decks (max rel err 1.4e-10 vs 1-GPU, <= 1.8e-10 vs CPU, particles <= 2e-15, mass exact); strong/weak runs completed; 8/40/80 dry-runs planned or refused as designed. --- level3/nyx/README.md | 146 ++++++++++++++++++ level3/nyx/build.sh | 191 ++++++++++++++++++++++++ level3/nyx/fetch.sh | 43 ++++++ level3/nyx/nyx_particle_compare.py | 132 +++++++++++++++++ level3/nyx/run.sh | 189 ++++++++++++++++++++++++ level3/nyx/validate.sh | 229 +++++++++++++++++++++++++++++ 6 files changed, 930 insertions(+) create mode 100644 level3/nyx/README.md create mode 100755 level3/nyx/build.sh create mode 100755 level3/nyx/fetch.sh create mode 100644 level3/nyx/nyx_particle_compare.py create mode 100755 level3/nyx/run.sh create mode 100755 level3/nyx/validate.sh diff --git a/level3/nyx/README.md b/level3/nyx/README.md new file mode 100644 index 0000000..c97b039 --- /dev/null +++ b/level3/nyx/README.md @@ -0,0 +1,146 @@ +# Nyx (AMReX-Astro) -- Level 3 second batch + +Cosmological N-body + baryon hydrodynamics (dark-matter particles, Poisson +gravity by AMReX MLMG multigrid, PPM hydro, comoving coordinates), full +application workflow: IC read, gravity solve, hydro/particle advance, particle +redistribution, plotfile/checkpoint I/O. + +## Provenance / versions + +| Item | Value | +|---|---| +| Nyx | tag `26.09`, `e06eabc1b9dbcad5612db9529aced682402daede` (2026-08-26), BSD-3-Clause-LBNL | +| AMReX (used) | tag `26.09`, `a52ca73324ac2c7b65ec04f131e6df99eec9c576` (2026-09-01) -- **external, private build** | +| AMReX (Nyx submodule pin) | `6e875b7cc1a4eec78e22ae4cdaa79f88acf5169e` (development 2026-08-12) -- **not used**, see below | +| SUNDIALS (heatcool variant only) | Nyx submodule pin `5c53be85c88f63c5201c130b8cb2c686615cfb03` = v7.2.1 | +| Build strategy | NATIVE (CMake): private AMReX 26.09 install + Nyx via `find_package(AMReX CONFIG)`; upstream's GPU CI options (`Nyx_HYDRO=YES Nyx_MPI=YES Nyx_OMP=NO`, C++17); double-precision particles as upstream's nightly regression builds | +| Compiler / Toolkit / MPI | conda GCC 13.3.0 (host), CUDA 13.2.78 sm_100, conda Open MPI 5.0.10 (site profile gmu-hopper: `pml ob1 / btl self,sm,smcuda`) | +| Profiles | `cuda132-gcc133-adiabatic` (Nyx_HEATCOOL=NO), `cpu-gcc133-adiabatic` (Nyx_GPU_BACKEND=NONE reference + AMReX plotfile tools + particle_compare), `cuda132-gcc133-heatcool` (planned: SUNDIALS 7.2.1 CVODE, see status) | +| Source changes | **none** (class A: build options; class A derived decks written at run time) | + +Layout: `.deps/level3/nyx//{src,build,install,logs,cache}`, application +build tree `build/level3/nyx/`, run dirs +`build/level3/nyx//run/..np` (dry-runs under `.dryrun/`). +Fingerprint `.deps/level3/nyx//install/.hpcperf-l3-fingerprint` (schema +l3-2) + `BUILD_INFO.txt` (binary sha256, `AMREX_CUDA_ARCHS`, `cuobjdump` archs). + +### Why AMReX 26.09 and not the submodule pin + +The first build against the pin produced an **sm_86** binary: that AMReX's +`convert_cuda_archs()` (Tools/CMake/AMReXUtils.cmake) drops every SM >= 10.0 +("CMake 3.30 does not support SM 10.0+ in cuda_select_nvcc_arch_flags"), the +list becomes empty, autodetection runs and CMake 3.28's table maps this B200 +(compute capability 10.0, confirmed by the detection program itself) to +`8.6+PTX`. No option of that AMReX yields sm_100. AMReX 26.09 rewrote the +resolution (`AMReXCUDAArchs`, `nvcc --list-gpu-arch`) and produced +`AMREX_CUDA_ARCHS=100` / `cuobjdump: sm_100` -- the same path the first-batch +WarpX 26.09 build already used. 26.09 is 21 commits ahead of / 0 behind the +pin (GitHub compare), Nyx's minimum is AMReX 20.11, and Nyx consumes an +external AMReX through `find_package(AMReX CONFIG)` with the component set its +own superbuild would request (3D, DOUBLE, PARTICLES/PDOUBLE, MPI, CUDA, +LSOLVERS). The sm_86 attempt was removed +(`.deps/level3/nyx/ATTEMPT-1-sm86-removed.txt`). WarpX's AMReX *binary* is not +reused (different component set: EB, no linear solvers, FFT); only the same +read-only source checkout tag. + +## Cases (all upstream decks, used from the read-only checkout) + +| Case | Deck | What | Mode | +|---|---|---|---| +| `minisb` | `Exec/MiniSB/inputs.32` + `nyx.ppm_type=0` | Santa Barbara cluster, 32^3 cells, 32,686 DM particles (shipped ASCII IC), 10 steps; **exactly upstream's nightly GPU regression test "MiniSB"** (2 ranks there) | smoke | +| `lya_adiabatic` smoke | `Exec/LyA/inputs.rt.garuda` | upstream's GPU regression deck **"LyA-adiabatic"** (heat_cool_type=0, strang_split=1), 32^3, shipped `32.nyx` IC, z=100, 10 steps | smoke | +| `lya_adiabatic` strong | `Exec/LyA/inputs` with heating/cooling OFF | the flagship 64^3 Lyman-alpha science deck (shipped `64sssss_20mpc.nyx` IC, z=159) as a **named adiabatic derivative**: `nyx.heat_cool_type=0 sdc_split=0 strang_split=1`. Does **not** cover the heating/cooling LyA workload | strong | +| `scaling_synthetic` | `Exec/Scaling/inputs` (RandomPerCell) | upstream's scaling deck; `RandomPerCell` is documented as a testing-only initialisation -> **synthetic scaling/communication test, not a science IC** | strong (fixed G^3, default 256^3) / weak (64^3 cells per rank, box and total DM mass scaled with the tiles) | +| `lya_heatcool` | `Exec/LyA/inputs` as shipped (heat_cool_type=11, CVODE) | the heating/cooling LyA workload -- needs the `heatcool` profile (SUNDIALS) | smoke/strong -- **not built yet (status below)** | + +Decomposition: `amr.max_grid_size` is fixed per case (16 for 32^3/64^3 decks, +64 for the synthetic decks) and `amr.refine_grid_layout=0` (as upstream's MiniSB +deck), so the BoxArray is identical for every rank count and only the +distribution changes; ranks > boxes is refused (never a silent idle rank), +boxes % ranks != 0 is reported as imbalanced. One MPI rank per GPU; AMReX binds +device 0 of the one GPU the launcher wrapper exposes; the launcher audits the +expected/observed mapping (all runs below: every rank `verified`). + +Derived deck = upstream lines verbatim minus the I/O cadence / decomposition +lines listed in the file header, plus `amrex.the_arena_init_size=0` (as the +official test command) and checkpoints at step 0 and the final step +(`chk00000`, `chk`: needed for particle identity across rank counts; +upstream's test command disables checkpoints -- I/O only). + +## Validation (`validate.sh`, pre-fixed criteria) + +Per case, for the N-GPU run: + +1. **Completeness**: exit code 0 (timeout -> FAIL), `plt00000` + final plotfile, + runlog reaches `max_step`, every plotfile variable finite (`amrex_fextrema` + through `l3_check.require_finite`), DM particle count == IC count (exact). +2. **Official regression comparison at upstream's tolerance** (nightly GPU + suite: `fcompare -n 0 --rel_tol 2e-10 --abort_if_not_all_found`): N=1 vs a + second independent 1-GPU run (same-configuration reproducibility, what the + nightly test measures); N>1 vs the 1-GPU plotfile of the same binary/deck. + Particles: AMReX's `particle_compare` needs identical headers (incl. + `next_id` and per-file layout, i.e. the same rank count) and returns 0 even + when it prints "FAIL - Particle data headers do not agree" -> across rank + counts `nyx_particle_compare.py` matches particles by their exact t=0 + position (chk00000 -> chk by (id,cpu) within a run) and reports + particle_compare's abs/rel norms per component at the same tolerance. +3. **Cross-backend reference**: CPU-profile binary (Nyx_GPU_BACKEND=NONE, same + Nyx/AMReX/deck), rel_tol **1e-8** fixed before any run (FMA/libm/reduction + differences host vs device over 10 steps). +4. **Conservation**: comoving baryon mass `sum(density*dV)` (`amrex_fvolumesum`) + plt00000 -> final, |dM/M| <= 1e-9; DM count exact. + +No tolerance was changed after seeing results. + +## Results (dgx003, 2026-09-06; logs under `build/level3/nyx/cuda132-gcc133-adiabatic/run/`) + +Build: AMReX 26.09 CUDA + Nyx = 121 s (nyx 115 s) at -j32; CPU profile 13 s +(after AMReX); `cuobjdump` sm_100 only; cudart static (`l3_binary_backend_check` +accepts embedded CUDA ELF). + +| GPUs | Case | fcompare vs reference (max rel err, tol 2e-10) | DM particles (rel, tol 2e-10) | vs CPU (tol 1e-8) | mass, count | audit | Result | +|---|---|---|---|---|---|---|---| +| 1 | minisb | rerun: 3.0e-11 (Temp) | 6.5e-16 | 3.9e-11 / 7.0e-16 | exact / 32,686 | 1 verified | **VALIDATED_PASS** | +| 1 | lya_adiabatic | rerun: 6.5e-14 | 7.9e-16 | 9.6e-14 / 7.0e-16 | exact / 32,768 | 1 verified | **VALIDATED_PASS** | +| 2 | minisb | vs 1 GPU: 3.3e-12 | 6.5e-16 | 3.6e-11 / 6.9e-16 | exact | 2 verified | **VALIDATED_PASS** | +| 2 | lya_adiabatic | vs 1 GPU: 6.4e-14 | 8.8e-16 | 8.7e-14 / 7.0e-16 | exact | 2 verified | **VALIDATED_PASS** | +| 4 | minisb | vs 1 GPU: 1.38e-10 | 1.3e-15 | 1.77e-10 / 1.7e-15 | exact | 4 verified | **VALIDATED_PASS** | +| 4 | lya_adiabatic | vs 1 GPU: 6.7e-14 | 6.1e-16 | 9.3e-14 / 7.0e-16 | exact | 4 verified | **VALIDATED_PASS** | + +(The `Temp` field carries the largest relative differences; MiniSB at 4 GPUs is +the closest to the official tolerance, 1.4e-10 vs 2e-10.) + +Scaling runs (COMPLETED, 10 steps, not correctness-validated; timing = Nyx +"Run time", includes IC read and I/O; too short for a performance statement): + +| Mode | Case | 1 GPU | 2 GPU | 4 GPU | +|---|---|---|---|---| +| strong | lya_adiabatic 64^3 (science IC, adiabatic) | 2.06 s | 2.35 s | 1.93 s | +| strong | scaling_synthetic 256^3 (synthetic) | 39.5 s | 41.5 s | 29.7 s | +| weak | scaling_synthetic 64^3 cells/rank (synthetic) | 1.63 s | 2.20 s | 3.52 s | + +No science IC larger than 64^3 is shipped (256^3/1024^3 exist only at OLCF +paths); meaningful strong scaling of a science case needs such an IC +(**blocked by data availability**, documented, not worked around). + +Dry-runs (`HPCPERF_DRY_RUN=1`, `HPCPERF_NODES` hypothetical): 8 GPUs (2 nodes) +planned for all three decks; 40/80 GPUs planned for `scaling_synthetic` weak +(40/80 boxes) and 40 GPUs for `lya_adiabatic` strong (64 boxes over 40 ranks = +imbalanced, reported); **refused** for `minisb` (8 boxes) and for 80 ranks on +the 64-box strong deck -- as designed. Multi-node remains BLOCKED/UNVERIFIED +on this site (launcher note); HIP untested (no ROCm). + +## Status of the heating/cooling variant + +`build.sh` with `HPCPERF_NYX_HEATCOOL=YES` builds SUNDIALS 7.2.1 (ENABLE_CUDA, +index 32, fused kernels -- Nyx's own `NyxSetupSUNDIALS.cmake` options) and AMReX +with `AMReX_SUNDIALS=ON`, then Nyx with `Nyx_HEATCOOL=YES`; `lya_heatcool` runs +`Exec/LyA/inputs` / `inputs.rt` as shipped. **Not executed in this round** (see +SECOND_BATCH_STATUS.md for the current state); the adiabatic results above make +no claim about heating/cooling. + +## Files + +`fetch.sh` (pinned clones + SHA checks), `build.sh` (staged, profile-aware, +fingerprinted), `run.sh` (cases/modes, decomposition guard, manifest), +`validate.sh` + `nyx_particle_compare.py` (criteria above), this README. diff --git a/level3/nyx/build.sh b/level3/nyx/build.sh new file mode 100755 index 0000000..44b0b5e --- /dev/null +++ b/level3/nyx/build.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Build Nyx 26.09 natively (CMake) against a PRIVATE AMReX 26.09 install and, for +# the heating/cooling variant, a private SUNDIALS 7.2.1 (the commit Nyx pins). +# Staged build, all inside one profile tree -- nothing shared with WarpX/Level 2: +# [1] SUNDIALS (heatcool only) .deps/level3/nyx//install/sundials +# [2] AMReX 26.09 .deps/level3/nyx//install/amrex +# [3] Nyx build/level3/nyx/ -> install/bin/nyx_* +# +# ./build.sh [CUDA|HIP|CPU] (default CUDA) +# HPCPERF_NYX_HEATCOOL=NO|YES (default NO -> adiabatic variant; YES -> SUNDIALS CVODE + CUDA fused kernels) +# HPCPERF_NYX_PROFILE= (override the derived profile name) +# HPCPERF_BUILD_JOBS=N (default 32) +# +# Profiles: -gcc-, e.g. +# cuda132-gcc133-adiabatic CUDA 13.2 / conda GCC 13.3 / Nyx_HEATCOOL=NO, sm_100 +# cuda132-gcc133-heatcool same + Nyx_HEATCOOL=YES (SUNDIALS 7.2.1, ENABLE_CUDA, fused kernels) +# cpu-gcc133-adiabatic Nyx_GPU_BACKEND=NONE reference build; its AMReX also builds the +# plotfile tools (amrex_fcompare/fnan/fvolumesum/...) and this script +# adds particle_compare -- the official comparison tools validate.sh uses +# +# Why an external AMReX 26.09 instead of Nyx's submodule pin (6e875b7c): the +# pin's CMake drops SM >= 10.0 (convert_cuda_archs) and autodetects 8.6+PTX on +# this node -> an sm_86 binary (first attempt, removed; see README). AMReX 26.09 +# (a52ca73, 21 commits ahead / 0 behind the pin) resolves sm_100 correctly. Nyx +# requires AMReX >= 20.11 and consumes it through find_package(AMReX CONFIG). +# AMReX options = exactly what Nyx's superbuild would set for the same Nyx options +# (cmake/NyxSetupAMReX.cmake: 3D, DOUBLE, PARTICLES(PDOUBLE), MPI, no OMP, no +# Fortran/PROBINIT, LINEAR_SOLVERS, SUNDIALS iff HEATCOOL, GPU backend); SUNDIALS +# options = cmake/NyxSetupSUNDIALS.cmake (CVODE only, index 32, fused kernels). +# Nyx options mirror upstream's GPU CI (Nyx_HYDRO=YES Nyx_MPI=YES Nyx_OMP=NO, +# CMAKE_CXX_STANDARD=17). Modification class: A (build options; out-of-source +# builds; no file of any checkout is modified). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" +l3_isolate_build_env # never see Level 2 .deps/install prefixes + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +HC="$(echo "${HPCPERF_NYX_HEATCOOL:-NO}" | tr '[:lower:]' '[:upper:]')" +case "$HC" in YES|NO) ;; *) echo "build.sh: HPCPERF_NYX_HEATCOOL must be YES or NO" >&2; exit 2;; esac +VARIANT=adiabatic; [ "$HC" = YES ] && VARIANT=heatcool +SRC="$R/_upstream/level3/Nyx"; AMREX_SRC="$R/_upstream/level3/amrex" +[ -f "$SRC/CMakeLists.txt" ] && [ -f "$AMREX_SRC/CMakeLists.txt" ] || { echo "build.sh: sources missing -- run $HERE/fetch.sh first" >&2; exit 1; } +[ "$HC" = NO ] || [ -f "$SRC/subprojects/sundials/CMakeLists.txt" ] || { echo "build.sh: sundials submodule missing -- run $HERE/fetch.sh" >&2; exit 1; } +SHA="$(git -C "$SRC" rev-parse HEAD)"; AMREX_SHA="$(git -C "$AMREX_SRC" rev-parse HEAD)" +SUNDIALS_SHA="$(git -C "$SRC/subprojects/sundials" rev-parse HEAD 2>/dev/null || echo none)" +GCC_MM="$(l3_version_mm "$("$CXX" -dumpfullversion 2>/dev/null || "$CXX" -dumpversion)")" + +case "$BACKEND" in + CUDA) + ARCH="${HPCPERF_CUDA_ARCH:-$(l3_gpu_arch)}"; [ -n "$ARCH" ] || { echo "build.sh: cannot determine GPU arch (no GPU?) -- set HPCPERF_CUDA_ARCH" >&2; exit 1; } + MODEL=cuda; ARCHNOTE="sm_$ARCH"; AMREX_GPU=CUDA + ARCH_FLAGS=("-DCMAKE_CUDA_ARCHITECTURES=$ARCH" "-DCMAKE_CUDA_HOST_COMPILER=$CXX") + PROFILE_DEFAULT="cuda$(l3_version_mm "$(l3_cuda_version)")-gcc${GCC_MM}-${VARIANT}" ;; + HIP) + command -v hipcc >/dev/null 2>&1 || { echo "build.sh: HIP requested but hipcc not found -- HIP build is UNTESTED on this machine (no ROCm)" >&2; exit 1; } + ARCH="${HPCPERF_HIP_ARCH:-gfx950}"; MODEL=hip; ARCHNOTE="$ARCH"; AMREX_GPU=HIP + ARCH_FLAGS=("-DAMReX_AMD_ARCH=$ARCH" -DCMAKE_CXX_COMPILER=hipcc) + PROFILE_DEFAULT="hip-${ARCH}-${VARIANT}" ;; + CPU|NONE) + MODEL=cpu; ARCHNOTE=host; AMREX_GPU=NONE; ARCH_FLAGS=() + PROFILE_DEFAULT="cpu-gcc${GCC_MM}-${VARIANT}" ;; + *) echo "usage: $0 [CUDA|HIP|CPU]" >&2; exit 2 ;; +esac +PROFILE="${HPCPERF_NYX_PROFILE:-$PROFILE_DEFAULT}" +l3_paths_profile nyx "$PROFILE" +BUILD_DIR="$L3_BUILD" +JOBS="${HPCPERF_BUILD_JOBS:-32}" +AMREX_PREFIX="$L3_INSTALL/amrex"; SUND_PREFIX="$L3_INSTALL/sundials" +TOOLS=OFF; [ "$MODEL" = cpu ] && TOOLS=ON + +AMREX_OPTS="AMReX_SPACEDIM=3 AMReX_PRECISION=DOUBLE AMReX_PARTICLES=ON AMReX_PARTICLES_PRECISION=DOUBLE AMReX_MPI=ON AMReX_OMP=OFF AMReX_FORTRAN=OFF AMReX_PROBINIT=OFF AMReX_LINEAR_SOLVERS=ON AMReX_EB=OFF AMReX_FFT=OFF AMReX_SUNDIALS=$( [ "$HC" = YES ] && echo ON || echo OFF) AMReX_GPU_BACKEND=$AMREX_GPU arch=$ARCHNOTE AMReX_PLOTFILE_TOOLS=$TOOLS" +CMAKE_OPTS="Nyx_GPU_BACKEND=$AMREX_GPU arch=$ARCHNOTE Nyx_HYDRO=YES Nyx_HEATCOOL=$HC Nyx_MPI=YES Nyx_OMP=NO Nyx_SINGLE_PRECISION_PARTICLES=NO CMAKE_CXX_STANDARD=17 CMAKE_BUILD_TYPE=Release amrex=external($AMREX_OPTS) sundials=$( [ "$HC" = YES ] && echo "external(ENABLE_CUDA=$( [ "$MODEL" = cuda ] && echo ON || echo OFF) INDEX_SIZE=32 FUSED_KERNELS=$( [ "$MODEL" = cuda ] && echo ON || echo OFF) CVODE only)" || echo off)" +DEPS="amrex=26.09($AMREX_SHA) [Nyx submodule pin $(git -C "$SRC/subprojects/amrex" rev-parse HEAD 2>/dev/null || echo unknown) not used: no sm_100 through CMake] sundials=$( [ "$HC" = YES ] && echo "7.2.1($SUNDIALS_SHA)" || echo off) profile=$PROFILE" +FP="$(l3_fingerprint_text nyx "$SHA" "$MODEL" "$DEPS" "$CMAKE_OPTS" "runtime(amrex.use_gpu_aware_mpi default)")" +l3_fingerprint_check "$L3_INSTALL" "$FP" || exit 1 + +COMMON=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX") +[ "$MODEL" = hip ] && COMMON=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 -DCMAKE_C_COMPILER="$CC") +echo "# Nyx $BACKEND profile=$PROFILE: Nyx $SHA (26.09), AMReX $AMREX_SHA (26.09, external), sundials $( [ "$HC" = YES ] && echo "$SUNDIALS_SHA (7.2.1)" || echo off), arch $ARCHNOTE, host $CXX, MPI $(mpirun --version 2>/dev/null | head -1)" +echo "# resources: -j$JOBS; expected AMReX 5-10 min + Nyx 2-5 min (+ SUNDIALS ~3 min); trees $L3_BUILD_DEPS, $BUILD_DIR" +t0=$(date +%s) + +stage() { # stage [cmake options...] + local name=$1 src=$2 bld=$3 log=$4; shift 4 + mkdir -p "$bld" + cmake -S "$src" -B "$bld" "${COMMON[@]}" "$@" > "$L3_LOGS/$log-configure.log" 2>&1 \ + || { tail -40 "$L3_LOGS/$log-configure.log"; echo "build.sh: $name configure failed (log: $L3_LOGS/$log-configure.log)" >&2; exit 1; } + cmake --build "$bld" -j "$JOBS" > "$L3_LOGS/$log-build.log" 2>&1 \ + || { tail -40 "$L3_LOGS/$log-build.log"; echo "build.sh: $name build failed (log: $L3_LOGS/$log-build.log)" >&2; exit 1; } +} + +# [1] SUNDIALS 7.2.1 (Nyx's pinned submodule commit), options from cmake/NyxSetupSUNDIALS.cmake +if [ "$HC" = YES ] && [ ! -f "$SUND_PREFIX/.hpcperf-stage-done" ]; then + SUND_GPU=(-DENABLE_CUDA=OFF) + [ "$MODEL" = cuda ] && SUND_GPU=(-DENABLE_CUDA=ON -DSUNDIALS_INDEX_SIZE=32 -DSUNDIALS_BUILD_PACKAGE_FUSED_KERNELS=ON "${ARCH_FLAGS[@]}") + [ "$MODEL" = hip ] && SUND_GPU=(-DENABLE_HIP=ON -DSUNDIALS_BUILD_PACKAGE_FUSED_KERNELS=ON) + stage SUNDIALS "$SRC/subprojects/sundials" "$L3_BUILD_DEPS/sundials" sundials \ + -DCMAKE_INSTALL_PREFIX="$SUND_PREFIX" -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ + -DEXAMPLES_ENABLE_C=OFF -DEXAMPLES_ENABLE_CXX=OFF -DEXAMPLES_INSTALL=OFF -DENABLE_MPI=OFF -DENABLE_OPENMP=OFF \ + -DBUILD_ARKODE=OFF -DBUILD_KINSOL=OFF -DBUILD_IDA=OFF -DBUILD_IDAS=OFF -DBUILD_CVODES=OFF -DBUILD_TESTING=OFF \ + "${SUND_GPU[@]}" + cmake --install "$L3_BUILD_DEPS/sundials" > "$L3_LOGS/sundials-install.log" 2>&1 || { echo "build.sh: SUNDIALS install failed" >&2; exit 1; } + touch "$SUND_PREFIX/.hpcperf-stage-done" +fi +t1=$(date +%s) + +# [2] AMReX 26.09, component set identical to what Nyx's superbuild sets for these Nyx options +if [ ! -f "$AMREX_PREFIX/.hpcperf-stage-done" ]; then + AMREX_SUND=(-DAMReX_SUNDIALS=OFF); [ "$HC" = YES ] && AMREX_SUND=(-DAMReX_SUNDIALS=ON "-DSUNDIALS_ROOT=$SUND_PREFIX") + stage AMReX "$AMREX_SRC" "$L3_BUILD_DEPS/amrex" amrex \ + -DCMAKE_INSTALL_PREFIX="$AMREX_PREFIX" -DBUILD_SHARED_LIBS=OFF \ + -DAMReX_SPACEDIM=3 -DAMReX_PRECISION=DOUBLE -DAMReX_PARTICLES=ON -DAMReX_PARTICLES_PRECISION=DOUBLE \ + -DAMReX_MPI=ON -DAMReX_OMP=OFF -DAMReX_FORTRAN=OFF -DAMReX_PROBINIT=OFF -DAMReX_LINEAR_SOLVERS=ON \ + -DAMReX_EB=OFF -DAMReX_FFT=OFF -DAMReX_AMRDATA=OFF -DAMReX_BUILD_TUTORIALS=OFF -DAMReX_INSTALL=ON \ + "-DAMReX_GPU_BACKEND=$AMREX_GPU" -DAMReX_PLOTFILE_TOOLS="$TOOLS" "${ARCH_FLAGS[@]}" "${AMREX_SUND[@]}" + cmake --install "$L3_BUILD_DEPS/amrex" > "$L3_LOGS/amrex-install.log" 2>&1 || { echo "build.sh: AMReX install failed" >&2; exit 1; } + if [ "$MODEL" = cuda ]; then + archs="$(/usr/bin/grep -h 'AMREX_CUDA_ARCHS:INTERNAL' "$L3_BUILD_DEPS/amrex/CMakeCache.txt" | cut -d= -f2)" + [ "$archs" = "$ARCH" ] || { echo "build.sh: AMReX resolved CUDA archs '$archs', expected '$ARCH' -- refusing" >&2; exit 1; } + fi + touch "$AMREX_PREFIX/.hpcperf-stage-done" +fi +t2=$(date +%s) + +# [3] Nyx against the private AMReX (+SUNDIALS). ENABLE_CUDA=ON makes Nyx include +# AMReXTargetHelpers (setup_target_for_cuda_compilation) in the external-AMReX branch. +NYX_GPU=(-DNyx_GPU_BACKEND="$AMREX_GPU") +[ "$MODEL" = cuda ] && NYX_GPU+=(-DENABLE_CUDA=ON "${ARCH_FLAGS[@]}") +[ "$MODEL" = hip ] && NYX_GPU+=("${ARCH_FLAGS[@]}") +NYX_SUND=(); [ "$HC" = YES ] && NYX_SUND=("-DSUNDIALS_ROOT=$SUND_PREFIX") +# Nyx_SINGLE_PRECISION_PARTICLES=NO: double-precision particles, as upstream's nightly +# regression builds (GNU make default; LyA-adiabatic passes USE_SINGLE_PRECISION_PARTICLES=FALSE +# explicitly) -- the CMake default would be single precision (PSINGLE). +stage Nyx "$SRC" "$BUILD_DIR" nyx \ + "-DAMReX_ROOT=$AMREX_PREFIX" "-DCMAKE_PREFIX_PATH=$AMREX_PREFIX" \ + -DNyx_HYDRO=YES -DNyx_HEATCOOL="$HC" -DNyx_MPI=YES -DNyx_OMP=NO -DNyx_SINGLE_PRECISION_PARTICLES=NO \ + "${NYX_GPU[@]}" "${NYX_SUND[@]}" +/usr/bin/grep -q 'AMReX found: configuration file located at' "$L3_LOGS/nyx-configure.log" \ + || { echo "build.sh: Nyx did not pick up the external AMReX (would have fallen back to the submodule)" >&2; exit 1; } +t3=$(date +%s) + +mkdir -p "$L3_INSTALL/bin" +for exe in Exec/MiniSB/nyx_MiniSB Exec/LyA/nyx_LyA Exec/AMR-density/nyx_AMR-density; do + [ -x "$BUILD_DIR/$exe" ] && cp -f "$BUILD_DIR/$exe" "$L3_INSTALL/bin/" +done +[ -x "$L3_INSTALL/bin/nyx_MiniSB" ] && [ -x "$L3_INSTALL/bin/nyx_LyA" ] || { echo "build.sh: nyx_MiniSB / nyx_LyA not produced under $BUILD_DIR/Exec" >&2; exit 1; } +if [ "$MODEL" = cuda ]; then + for exe in "$L3_INSTALL"/bin/nyx_*; do + got="$(cuobjdump --list-elf "$exe" 2>/dev/null | /usr/bin/grep -o 'sm_[0-9a-z]*' | sort -u | paste -sd,)" + [ "$got" = "sm_$ARCH" ] || { echo "build.sh: $(basename "$exe") embeds '$got', expected sm_$ARCH -- refusing to install" >&2; exit 1; } + done +fi +if [ "$MODEL" = cpu ]; then + for t in fcompare fnan fvolumesum fextrema fvarnames ftime; do + [ -x "$AMREX_PREFIX/bin/amrex_$t" ] && cp -f "$AMREX_PREFIX/bin/amrex_$t" "$L3_INSTALL/bin/" + done + [ -x "$L3_INSTALL/bin/amrex_fcompare" ] || { echo "build.sh: amrex_fcompare not installed by AMReX (AMReX_PLOTFILE_TOOLS)" >&2; exit 1; } + # particle_compare (AMReX Tools/Postprocessing/C_Src) against the installed AMReX + PC_SRC="$L3_SRC/particle_compare"; rm -rf "$PC_SRC"; mkdir -p "$PC_SRC" + cp "$AMREX_SRC/Tools/Postprocessing/C_Src/particle_compare.cpp" "$PC_SRC/" + cat > "$PC_SRC/CMakeLists.txt" <<'EOF' +cmake_minimum_required(VERSION 3.24) +project(particle_compare C CXX) # AMReXConfig's find_dependency(MPI) needs the C language enabled +find_package(AMReX REQUIRED CONFIG) +add_executable(particle_compare particle_compare.cpp) +target_link_libraries(particle_compare PRIVATE AMReX::amrex_3d) +EOF + stage particle_compare "$PC_SRC" "$L3_BUILD_DEPS/particle_compare" particle_compare "-DAMReX_ROOT=$AMREX_PREFIX" + cp -f "$L3_BUILD_DEPS/particle_compare/particle_compare" "$L3_INSTALL/bin/" +fi +l3_fingerprint_write "$L3_INSTALL" "$FP" +{ + echo "profile=$PROFILE backend=$MODEL variant=$VARIANT jobs=$JOBS utc=$(date -u +%FT%TZ)" + echo "seconds: sundials=$((t1 - t0)) amrex=$((t2 - t1)) nyx=$((t3 - t2)) total=$(( $(date +%s) - t0 ))" + echo "amrex=$AMREX_SHA (26.09) options: $AMREX_OPTS" + [ "$HC" = YES ] && echo "sundials=$SUNDIALS_SHA (7.2.1)" + for b in "$L3_INSTALL"/bin/*; do echo "$(basename "$b") sha256=$(l3_sha_file "$b")"; done + if [ "$MODEL" = cuda ]; then + echo "AMREX_CUDA_ARCHS=$(/usr/bin/grep -h 'AMREX_CUDA_ARCHS:INTERNAL' "$L3_BUILD_DEPS/amrex/CMakeCache.txt" | cut -d= -f2)" + echo "cuobjdump(nyx_MiniSB): $(cuobjdump --list-elf "$L3_INSTALL/bin/nyx_MiniSB" 2>/dev/null | /usr/bin/grep -o 'sm_[0-9a-z]*' | sort -u | paste -sd,) (cudart static; ldd shows $(ldd "$L3_INSTALL/bin/nyx_MiniSB" | /usr/bin/grep -oE 'lib(cudart|cuda|cusparse|cublas|curand)[^ ]*' | sort -u | paste -sd, || echo none))" + fi +} > "$L3_INSTALL/BUILD_INFO.txt" +echo "# built in $(( $(date +%s) - t0 )) s (sundials $((t1 - t0)), amrex $((t2 - t1)), nyx $((t3 - t2))): $(ls "$L3_INSTALL/bin" | paste -sd' ') (installed under $L3_INSTALL)" +echo "# compiler warning lines (nyx): $(/usr/bin/grep -c 'warning' "$L3_LOGS/nyx-build.log" || true)" +[ "$MODEL" = cuda ] && /usr/bin/grep -E 'AMREX_CUDA_ARCHS|cuobjdump' "$L3_INSTALL/BUILD_INFO.txt" +exit 0 diff --git a/level3/nyx/fetch.sh b/level3/nyx/fetch.sh new file mode 100755 index 0000000..3f23972 --- /dev/null +++ b/level3/nyx/fetch.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Fetch Nyx (AMReX-Astro) at the pinned release plus the AMReX release it is +# built against here. Read-only checkouts under _upstream/level3/; nothing is +# committed to this repository. +# +# Nyx 26.09 e06eabc1b9dbcad5612db9529aced682402daede (tag 26.09, 2026-08-26) +# subprojects/amrex 6e875b7cc1a4eec78e22ae4cdaa79f88acf5169e (development 2026-08-12; the pin) +# subprojects/sundials 5c53be85c88f63c5201c130b8cb2c686615cfb03 (v7.2.1; used by the heatcool profile) +# AMReX 26.09 a52ca73324ac2c7b65ec04f131e6df99eec9c576 (tag 26.09, 2026-09-01; 21 commits +# ahead of / 0 behind the Nyx pin -- a strict descendant) +# +# Why AMReX 26.09 and not the submodule pin: the pinned AMReX resolves CUDA +# architectures through CMake's legacy cuda_select_nvcc_arch_flags and its +# convert_cuda_archs() drops every SM >= 10.0 ("CMake 3.30 does not support SM +# 10.0+"), then autodetects 8.6+PTX on this B200 node -> an sm_86 binary (first +# attempt, removed). AMReX 26.09 resolves the architecture through nvcc +# --list-gpu-arch into CMAKE_CUDA_ARCHITECTURES (the path WarpX 26.09 already +# verified here with sm_100). Nyx's own minimum is AMREX_MINIMUM_VERSION 20.11 +# and it consumes an external AMReX through find_package(AMReX CONFIG). This is a +# private build for Nyx: WarpX's AMReX *binary* is not reused (WarpX's superbuild +# builds AMReX with a different component set: EB, no linear solvers, FFT ...). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +UP="$R/_upstream/level3" +NYX_TAG=26.09; NYX_SHA=e06eabc1b9dbcad5612db9529aced682402daede +AMREX_PIN_SHA=6e875b7cc1a4eec78e22ae4cdaa79f88acf5169e +SUNDIALS_SHA=5c53be85c88f63c5201c130b8cb2c686615cfb03 +AMREX_TAG=26.09; AMREX_SHA=a52ca73324ac2c7b65ec04f131e6df99eec9c576 + +mkdir -p "$UP" +if [ ! -d "$UP/Nyx/.git" ]; then git clone -q --branch "$NYX_TAG" --depth 1 https://github.com/AMReX-Astro/Nyx.git "$UP/Nyx"; fi +got="$(git -C "$UP/Nyx" rev-parse HEAD)" +[ "$got" = "$NYX_SHA" ] || { echo "fetch.sh: $UP/Nyx is at $got, expected $NYX_SHA (tag $NYX_TAG)" >&2; exit 1; } +git -C "$UP/Nyx" submodule update --init --depth 1 subprojects/amrex subprojects/sundials +a="$(git -C "$UP/Nyx/subprojects/amrex" rev-parse HEAD)"; s="$(git -C "$UP/Nyx/subprojects/sundials" rev-parse HEAD)" +[ "$a" = "$AMREX_PIN_SHA" ] || { echo "fetch.sh: Nyx amrex submodule at $a, expected $AMREX_PIN_SHA" >&2; exit 1; } +[ "$s" = "$SUNDIALS_SHA" ] || { echo "fetch.sh: Nyx sundials submodule at $s, expected $SUNDIALS_SHA" >&2; exit 1; } + +if [ ! -d "$UP/amrex/.git" ]; then git clone -q --branch "$AMREX_TAG" --depth 1 https://github.com/AMReX-Codes/amrex.git "$UP/amrex"; fi +x="$(git -C "$UP/amrex" rev-parse HEAD)" +[ "$x" = "$AMREX_SHA" ] || { echo "fetch.sh: $UP/amrex is at $x, expected $AMREX_SHA (tag $AMREX_TAG)" >&2; exit 1; } +echo "# Nyx $NYX_TAG $got (submodules amrex $a, sundials $s); AMReX $AMREX_TAG $x -> $UP" diff --git a/level3/nyx/nyx_particle_compare.py b/level3/nyx/nyx_particle_compare.py new file mode 100644 index 0000000..ece5433 --- /dev/null +++ b/level3/nyx/nyx_particle_compare.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Compare Nyx dark-matter particles between two runs that may have used a +DIFFERENT number of MPI ranks. + +AMReX's own particle_compare (Tools/Postprocessing/C_Src) compares particle +files chunk by chunk and requires identical headers, including `next_id` and the +per-file layout -- i.e. the same process count. Across rank counts Nyx assigns +particle (id, cpu) pairs per reading rank, so ids differ as well. Particle +identity is therefore established through the initial checkpoint: every particle +of a run is keyed by its exact (bit-identical) t=0 position, read from the +checkpoint written at step 0 with the same (id, cpu) as the final checkpoint. + + nyx_particle_compare.py [--rel_tol R] [--abs_tol A] + +Reads /chk00000/DM and /chk/DM (checkpoint format: ids present). +Reports, per real component (position_x/y/z, mass, xvel, yvel, zvel), the maximum +absolute difference and the relative difference max|a-b| / max|a| (the same +definition AMReX's particle_compare prints), and exits 0 only if every component +satisfies abs <= abs_tol or rel <= rel_tol (particle_compare semantics: with +abs_tol 0 the relative tolerance alone decides). Exit 2 on any structural problem +(count mismatch, unmatched particle, unreadable file, non-finite data). +""" +import argparse, math, os, struct, sys +import numpy as np + + +def read_particle_dir(pdir): + """Return (header dict, ints[n, num_int] int32, reals[n, num_real] float64).""" + lines = [l.strip() for l in open(os.path.join(pdir, "Header")) if l.strip()] + i = 0 + version = lines[i]; i += 1 + ndim = int(lines[i]); i += 1 + nreal_extra = int(lines[i]); i += 1 + real_names = lines[i:i + nreal_extra]; i += nreal_extra + nint_extra = int(lines[i]); i += 1 + int_names = lines[i:i + nint_extra]; i += nint_extra + is_chk = int(lines[i]); i += 1 + nparticles = int(lines[i]); i += 1 + next_id = int(lines[i]); i += 1 + finest = int(lines[i]); i += 1 + ngrids = [int(lines[i + l]) for l in range(finest + 1)]; i += finest + 1 + grids = [] # (level, which, count, where) + for lev in range(finest + 1): + for g in range(ngrids[lev]): + w, c, o = lines[i].split(); i += 1 + grids.append((lev, int(w), int(c), int(o))) + if "single" in version: + raise SystemExit(f"{pdir}: single-precision particle files are not expected here") + num_int = 2 * is_chk + nint_extra + num_real = ndim + nreal_extra + ints = np.zeros((nparticles, num_int), dtype=np.int32) + reals = np.zeros((nparticles, num_real), dtype=np.float64) + pos = 0 + for lev, which, count, where in grids: + if count == 0: + continue + fn = os.path.join(pdir, f"Level_{lev}", f"DATA_{which:05d}") + with open(fn, "rb") as f: + f.seek(where) + ib = f.read(4 * num_int * count) + rb = f.read(8 * num_real * count) + if len(ib) != 4 * num_int * count or len(rb) != 8 * num_real * count: + raise SystemExit(f"{fn}: short read at offset {where} (grid count {count})") + if num_int: + ints[pos:pos + count] = np.frombuffer(ib, dtype=" its final real components (via id,cpu).""" + h0, i0, r0 = read_particle_dir(os.path.join(run, "chk00000", "DM")) + h1, i1, r1 = read_particle_dir(os.path.join(run, f"chk{final_step:05d}", "DM")) + for h, tag in ((h0, "chk00000"), (h1, f"chk{final_step:05d}")): + if not h["is_checkpoint"]: + raise SystemExit(f"{run}/{tag}: not a checkpoint (no particle ids) -- run.sh must write checkpoints") + if h0["nparticles"] != h1["nparticles"]: + raise SystemExit(f"{run}: particle count changed {h0['nparticles']} -> {h1['nparticles']}") + if not (np.isfinite(r0).all() and np.isfinite(r1).all()): + raise SystemExit(f"{run}: non-finite particle data") + # (id, cpu) -> row, then order the final data by the initial-position key + key0 = i0[:, 0].astype(np.int64) * (1 << 32) + i0[:, 1].astype(np.int64) + key1 = i1[:, 0].astype(np.int64) * (1 << 32) + i1[:, 1].astype(np.int64) + if len(np.unique(key0)) != len(key0) or len(np.unique(key1)) != len(key1): + raise SystemExit(f"{run}: duplicate (id,cpu) pairs") + order1 = np.argsort(key1); k1s = key1[order1] + idx = np.searchsorted(k1s, key0) + if not np.array_equal(k1s[idx], key0): + raise SystemExit(f"{run}: a particle of chk00000 is missing in the final checkpoint") + final_by_initial_row = r1[order1[idx]] # row j: final data of the particle that was row j initially + init_pos = r0[:, :h0["ndim"]] + # canonical order: lexicographic on the exact initial position (identical across runs by construction) + canon = np.lexsort([init_pos[:, d] for d in reversed(range(h0["ndim"]))]) + return h1, init_pos[canon], final_by_initial_row[canon] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("run_a"); ap.add_argument("run_b"); ap.add_argument("final_step", type=int) + ap.add_argument("--rel_tol", type=float, default=0.0); ap.add_argument("--abs_tol", type=float, default=0.0) + a = ap.parse_args() + ha, pa, fa = keyed_final(a.run_a, a.final_step) + hb, pb, fb = keyed_final(a.run_b, a.final_step) + if ha["names"] != hb["names"] or ha["nparticles"] != hb["nparticles"]: + raise SystemExit(f"component/count mismatch: {ha['names']} ({ha['nparticles']}) vs {hb['names']} ({hb['nparticles']})") + if not np.array_equal(pa, pb): + raise SystemExit("initial (t=0) particle positions differ between the runs -- cannot establish identity") + print(f" {ha['nparticles']} particles matched through their exact initial positions; comparing chk{a.final_step:05d}") + print(f" {'component':<20} {'abs error':>16} {'rel error':>16}") + ok = True + for j, name in enumerate(ha["names"]): + d = np.abs(fa[:, j] - fb[:, j]).max() + ref = np.abs(fa[:, j]).max() + rel = d / ref if ref > 0 else d + print(f" {name:<20} {d:16.8e} {rel:16.8e}") + if d > a.abs_tol and rel > a.rel_tol: + ok = False + if ok: + print(f" PARTICLES AGREE to relative tolerance {a.rel_tol:g}" + (f" and/or absolute tolerance {a.abs_tol:g}" if a.abs_tol > 0 else "")) + return 0 + print(f" PARTICLES DISAGREE to relative tolerance {a.rel_tol:g}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/level3/nyx/run.sh b/level3/nyx/run.sh new file mode 100755 index 0000000..fd03f7e --- /dev/null +++ b/level3/nyx/run.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Run Nyx (full cosmological N-body + baryon hydrodynamics workflow: dark-matter +# particle IC read, Poisson gravity via AMReX MLMG, PPM hydro, particle push, +# periodic redistribution, plotfile I/O) on N GPUs, one MPI rank per GPU. +# +# ./run.sh [CUDA|HIP|CPU] [extra Nyx inputs overrides...] +# +# Cases (HPCPERF_NYX_CASE), all upstream decks from the read-only checkout: +# minisb (default) Exec/MiniSB/inputs.32 -- Santa Barbara cluster, 32^3 cells, +# 32,686 DM particles from the shipped ASCII IC, 10 steps; +# exactly upstream's nightly GPU regression test "MiniSB" +# (run there as `inputs.32 nyx.ppm_type=0`, 2 MPI ranks). +# Adiabatic (no heating/cooling in the deck). smoke only. +# lya_adiabatic smoke : Exec/LyA/inputs.rt.garuda -- upstream's GPU regression +# test "LyA-adiabatic" deck (heat_cool_type=0, +# strang_split=1, 32^3 cells, shipped 32.nyx IC, z=100). +# strong: Exec/LyA/inputs (the flagship 64^3 Lyman-alpha science +# deck with the shipped 64sssss_20mpc.nyx IC, z=159) with +# heating/cooling turned OFF (heat_cool_type=0, sdc_split=0, +# strang_split=1, UVB table unused). This is a NAMED +# ADIABATIC DERIVATIVE of LyA; it does not cover the +# heating/cooling LyA workload (see lya_heatcool). +# lya_heatcool Exec/LyA/inputs as shipped (heat_cool_type=11, CVODE via +# SUNDIALS) -- requires a *heatcool* profile (HPCPERF_NYX_HEATCOOL=YES build). +# scaling_synthetic weak: Exec/Scaling/inputs physics with nyx.particle_init_type=RandomPerCell +# (upstream labels this initialisation as testing-only): a +# SYNTHETIC scaling/communication test, NOT a science IC. +# +# Decomposition: `amr.max_grid_size` fixes the box size so the BoxArray is the +# same for every rank count (amr.refine_grid_layout=0, as upstream's MiniSB deck); +# ranks are refused when boxes < ranks (a rank without a box is never launched +# silently) and reported when boxes % ranks != 0 (imbalance). Nothing changes N. +# Rank -> GPU: AMReX binds device 0 of what the launcher's per-rank wrapper +# exposes (one visible GPU per rank, audited by the launcher). +# +# Controls: +# HPCPERF_GPUS=N|all ranks (= GPUs, default 1) +# HPCPERF_SCALE_MODE smoke | strong | weak (default smoke) +# HPCPERF_NYX_STEPS max_step (default: 10 = upstream's MiniSB test length) +# HPCPERF_NYX_PROFILE build profile (default: cuda-gcc-adiabatic|heatcool) +# HPCPERF_NYX_HEATCOOL=YES select the heatcool profile +# HPCPERF_NYX_GPU_AWARE=0|1 amrex.use_gpu_aware_mpi (default: AMReX auto) +# HPCPERF_NYX_MGS override amr.max_grid_size (decomposition control only) +# HPCPERF_NYX_WEAK_CELLS weak: cells per rank per dimension (default 64) +# Derived deck (class A): upstream lines kept verbatim except the parameters listed +# in the header of the written file (I/O cadence, decomposition, and -- for the named +# adiabatic derivative -- heat_cool_type/sdc_split/strang_split); the official test +# command's `amrex.the_arena_init_size=0 amr.checkpoint_files_output=0` are applied. +# stdout is copied to /stdout.log; a run_manifest.txt records provenance. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" + +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')"; [ $# -gt 0 ] && shift +HC="$(echo "${HPCPERF_NYX_HEATCOOL:-NO}" | tr '[:lower:]' '[:upper:]')" +VARIANT=adiabatic; [ "$HC" = YES ] && VARIANT=heatcool +GCC_MM="$(l3_version_mm "$("$CXX" -dumpfullversion 2>/dev/null || "$CXX" -dumpversion)")" +case "$BACKEND" in + CUDA) MODEL=cuda; PROFILE_DEFAULT="cuda$(l3_version_mm "$(l3_cuda_version)")-gcc${GCC_MM}-${VARIANT}" ;; + HIP) MODEL=hip; PROFILE_DEFAULT="hip-${HPCPERF_HIP_ARCH:-gfx950}-${VARIANT}" ;; + CPU) MODEL=cpu; PROFILE_DEFAULT="cpu-gcc${GCC_MM}-${VARIANT}" ;; + *) echo "usage: $0 [CUDA|HIP|CPU] [inputs overrides]" >&2; exit 2 ;; +esac +PROFILE="${HPCPERF_NYX_PROFILE:-$PROFILE_DEFAULT}" +l3_paths_profile nyx "$PROFILE" +SRC="$R/_upstream/level3/Nyx" +CASE="${HPCPERF_NYX_CASE:-minisb}" +MODE="$(l3_scale_mode nyx)" || exit 2 +N_RANKS="$(hpcperf_ranks nyx yes)" || exit 2 +STEPS="${HPCPERF_NYX_STEPS:-10}" +hpcperf_forbid_args nyx max_step amr.n_cell amr.max_grid_size amr.refine_grid_layout amr.plot_int amr.plot_file amr.check_int \ + amr.checkpoint_files_output nyx.heat_cool_type nyx.particle_init_type nyx.binary_particle_file nyx.ascii_particle_file \ + geometry.prob_hi nyx.particle_initrandom_mass_total -- "$@" || exit 2 + +# ----------------------------------------------------------------------------- case -> deck +EXTRA=() # deck lines appended (documented in the derived file header) +DROP='^\s*(max_step|amr\.plot_int|amr\.check_int|amr\.checkpoint_files_output|amr\.plot_file|amr\.check_file|amr\.max_grid_size|amr\.refine_grid_layout)\s*=' +LINKS=() # upstream data files the deck references by relative name +case "$CASE" in + minisb) + [ "$MODE" = smoke ] || { echo "run.sh: case minisb is the 32^3 official regression deck: smoke only (use lya_adiabatic for strong, scaling_synthetic for weak)" >&2; exit 2; } + EXE="$L3_INSTALL/bin/nyx_MiniSB"; DECK="$SRC/Exec/MiniSB/inputs.32"; LINKS=(ic_sb_32.ascii) + NX=32; MGS="${HPCPERF_NYX_MGS:-16}" + DROP="$DROP|^\s*nyx\.ppm_type\s*="; EXTRA+=("nyx.ppm_type = 0 # as upstream's nightly GPU test command") ;; + lya_adiabatic) + EXE="$L3_INSTALL/bin/nyx_LyA" + [ "$HC" = NO ] || { echo "run.sh: lya_adiabatic is defined for the adiabatic profile (HPCPERF_NYX_HEATCOOL=NO)" >&2; exit 2; } + case "$MODE" in + smoke) DECK="$SRC/Exec/LyA/inputs.rt.garuda"; LINKS=(32.nyx); NX=32; MGS="${HPCPERF_NYX_MGS:-16}" ;; + strong) DECK="$SRC/Exec/LyA/inputs"; LINKS=(64sssss_20mpc.nyx); NX=64; MGS="${HPCPERF_NYX_MGS:-16}" + DROP="$DROP|^\s*nyx\.(heat_cool_type|sdc_split|strang_split|uvb_rates_file)\s*=" + EXTRA+=("nyx.heat_cool_type = 0 # NAMED ADIABATIC DERIVATIVE of Exec/LyA/inputs (heating/cooling OFF)" \ + "nyx.sdc_split = 0" "nyx.strang_split = 1 # required by Nyx without SDC/HEATCOOL (as inputs.rt.garuda)") ;; + weak) echo "run.sh: weak scaling with a science IC is not defined for Nyx (the shipped ICs are two different cosmologies); use HPCPERF_NYX_CASE=scaling_synthetic (labelled synthetic)" >&2; exit 2 ;; + esac ;; + lya_heatcool) + EXE="$L3_INSTALL/bin/nyx_LyA" + [ "$HC" = YES ] || { echo "run.sh: lya_heatcool needs the heatcool profile: HPCPERF_NYX_HEATCOOL=YES (build.sh + run.sh)" >&2; exit 2; } + [ "$MODE" != weak ] || { echo "run.sh: weak not defined for lya_heatcool" >&2; exit 2; } + case "$MODE" in + smoke) DECK="$SRC/Exec/LyA/inputs.rt"; LINKS=(32.nyx TREECOOL_middle); NX=32; MGS="${HPCPERF_NYX_MGS:-16}" ;; + strong) DECK="$SRC/Exec/LyA/inputs"; LINKS=(64sssss_20mpc.nyx TREECOOL_middle); NX=64; MGS="${HPCPERF_NYX_MGS:-16}" ;; + esac ;; + scaling_synthetic) + EXE="$L3_INSTALL/bin/nyx_LyA"; DECK="$SRC/Exec/Scaling/inputs"; LINKS=(TREECOOL_middle) + [ "$MODE" != smoke ] || { echo "run.sh: scaling_synthetic is a scaling deck (HPCPERF_SCALE_MODE=strong|weak); correctness cases are minisb / lya_adiabatic" >&2; exit 2; } + L="${HPCPERF_NYX_WEAK_CELLS:-64}" + # upstream deck: 64^3 cells in a 28.49002849 Mpc box, RandomPerCell, total DM mass 869658119634944.0 + BOX0=28.49002849; MASS0=869658119634944.0 + if [ "$MODE" = weak ]; then + TOPO="$(hpcperf_topology nyx "$N_RANKS")" || exit 2; read -r PX PY PZ <<< "$TOPO" + NX=$((L * PX)); NY=$((L * PY)); NZ=$((L * PZ)); MGS="${HPCPERF_NYX_MGS:-$L}"; WHAT="weak: ${L}^3 cells per rank, ${PX}x${PY}x${PZ} tiles" + else + G="${HPCPERF_NYX_GLOBAL:-256}"; [ $((G % 64)) -eq 0 ] || { echo "run.sh: HPCPERF_NYX_GLOBAL=$G must be a multiple of 64" >&2; exit 2; } + PX=$((G / 64)); PY=$PX; PZ=$PX; NX=$G; NY=$G; NZ=$G; MGS="${HPCPERF_NYX_MGS:-64}"; WHAT="strong: fixed ${G}^3 global grid (upstream 64^3 deck scaled x$PX per dimension)" + fi + PHX="$(python3 -c "print(f'{$BOX0*$PX:.8f}')")"; PHY="$(python3 -c "print(f'{$BOX0*$PY:.8f}')")"; PHZ="$(python3 -c "print(f'{$BOX0*$PZ:.8f}')")" + MASS="$(python3 -c "print(f'{$MASS0*$PX*$PY*$PZ:.1f}')")" + DROP="$DROP|^\s*(amr\.n_cell|geometry\.prob_hi|nyx\.particle_initrandom_mass_total)\s*=" + EXTRA+=("amr.n_cell = $NX $NY $NZ # $WHAT" \ + "geometry.prob_hi = $PHX $PHY $PHZ # box scaled with the tiles (mean density unchanged)" \ + "nyx.particle_initrandom_mass_total = $MASS # total DM mass scaled with the volume") + if [ "$HC" = NO ]; then + DROP="$DROP|^\s*nyx\.(heat_cool_type|sdc_split|strang_split|uvb_rates_file)\s*=" + EXTRA+=("nyx.heat_cool_type = 0 # adiabatic profile" "nyx.sdc_split = 0" "nyx.strang_split = 1") + fi ;; + *) echo "run.sh: HPCPERF_NYX_CASE must be minisb | lya_adiabatic | lya_heatcool | scaling_synthetic (got '$CASE')" >&2; exit 2 ;; +esac +[ -x "$EXE" ] || { echo "run.sh: $EXE not found -- run ./build.sh $BACKEND (profile $PROFILE) first" >&2; exit 1; } +[ -f "$DECK" ] || { echo "run.sh: deck $DECK missing (run fetch.sh)" >&2; exit 1; } +[ "$MODEL" = cpu ] || l3_binary_backend_check "$EXE" "$MODEL" || exit 1 +: "${NY:=$NX}"; : "${NZ:=$NX}" +# boxes with the fixed layout (n_cell / max_grid_size per dimension) +for d in "$NX" "$NY" "$NZ"; do [ $((d % MGS)) -eq 0 ] || { echo "run.sh: amr.n_cell $d not divisible by amr.max_grid_size $MGS" >&2; exit 2; }; done +BOXES=$(( (NX / MGS) * (NY / MGS) * (NZ / MGS) )) +if [ "$N_RANKS" -gt "$BOXES" ]; then + echo "run.sh: $N_RANKS ranks requested but the ${NX}x${NY}x${NZ} grid with amr.max_grid_size=$MGS has only $BOXES boxes -- a rank without work is never launched silently; lower HPCPERF_GPUS or set HPCPERF_NYX_MGS (e.g. $((MGS / 2)) -> $((BOXES * 8)) boxes)" >&2; exit 2 +fi +BALANCE=balanced; [ $((BOXES % N_RANKS)) -eq 0 ] || BALANCE="IMBALANCED ($BOXES boxes over $N_RANKS ranks)" + +RUN_DIR="$(l3_rundir "$L3_BUILD/run/$CASE.$MODE.np$N_RANKS")" || exit 2 +IN="$RUN_DIR/inputs" +{ + echo "# derived from upstream $(realpath --relative-to="$SRC" "$DECK") (HPC-Performance-AI level3/nyx/run.sh; profile $PROFILE)" + echo "# upstream lines removed and replaced below: max_step, amr.plot_int/plot_file/check_int/check_file/checkpoint_files_output, amr.max_grid_size, amr.refine_grid_layout$( [ ${#EXTRA[@]} -gt 0 ] && echo ', plus the case-specific lines marked with a comment')" + grep -vE "$DROP" "$DECK" + echo "" + echo "# --- level3/nyx/run.sh ---" + echo "max_step = $STEPS" + echo "amr.max_grid_size = $MGS # fixed BoxArray ($BOXES boxes) for every rank count" + echo "amr.refine_grid_layout = 0" + echo "amr.plot_file = plt" + echo "amr.plot_int = $STEPS # plt00000 (initial) and plt$(printf '%05d' "$STEPS") (final)" + echo "amr.plot_vars = ALL" + echo "amr.check_file = chk" + echo "amr.check_int = $STEPS # chk00000 and chk$(printf '%05d' "$STEPS"): particle ids for the cross-rank-count" + echo "amr.checkpoint_files_output = 1 # particle comparison (nyx_particle_compare.py); upstream's test command disables checkpoints" + echo "amrex.the_arena_init_size = 0 # as upstream's nightly test command" + for e in "${EXTRA[@]}"; do echo "$e"; done + [ -n "${HPCPERF_NYX_GPU_AWARE:-}" ] && echo "amrex.use_gpu_aware_mpi = $HPCPERF_NYX_GPU_AWARE" +} > "$IN" +for f in "${LINKS[@]}"; do + src="$(dirname "$DECK")/$f"; [ -f "$src" ] || { echo "run.sh: upstream data file $src missing" >&2; exit 1; } + ln -sfn "$src" "$RUN_DIR/$f" +done + +BIND=wrapper; [ "$MODEL" = cpu ] && BIND=none +echo "# Nyx $BACKEND profile=$PROFILE case=$CASE mode=$MODE ranks=$N_RANKS grid=${NX}x${NY}x${NZ} max_grid_size=$MGS boxes=$BOXES ($BALANCE) steps=$STEPS exe=$(basename "$EXE") deck=$(realpath --relative-to="$SRC" "$DECK") run_dir=$RUN_DIR" +cd "$RUN_DIR" +RUN_ID="$(l3_run_id)" +# real exit code of the launcher/application (pipefail: tee's 0 never masks it; set +e so it is recorded, not aborted on) +set +e; set -o pipefail +"$L3_LAUNCHER" --gpus "$N_RANKS" --bind "$BIND" -- "$EXE" "$IN" "$@" 2>&1 | tee "$RUN_DIR/stdout.log" +rc=$? +set +o pipefail; set -e +if [ -z "${HPCPERF_DRY_RUN:-}" ]; then + ICS=""; for f in "${LINKS[@]}"; do ICS="$ICS $f=$(l3_sha_file "$(dirname "$DECK")/$f")"; done + l3_manifest "$RUN_DIR" "run_id=$RUN_ID" "app=nyx" "backend=$BACKEND" "profile=$PROFILE" "case=$CASE" "mode=$MODE" \ + "ranks=$N_RANKS" "grid=${NX}x${NY}x${NZ}" "max_grid_size=$MGS" "boxes=$BOXES" "steps=$STEPS" \ + "exit_code=$rc" "binary=$EXE" "binary_sha256=$(l3_sha_file "$EXE")" "deck=$DECK" "deck_sha256=$(l3_sha_file "$DECK")" \ + "input=$IN" "input_sha256=$(l3_sha_file "$IN")" "ic_sha256=${ICS# }" \ + "fingerprint=$L3_INSTALL/.hpcperf-l3-fingerprint" "fingerprint_sha256=$(l3_sha_file "$L3_INSTALL/.hpcperf-l3-fingerprint")" \ + "stdout=$RUN_DIR/stdout.log" "utc=$(date -u +%FT%TZ)" +fi +exit "$rc" diff --git a/level3/nyx/validate.sh b/level3/nyx/validate.sh new file mode 100755 index 0000000..d222190 --- /dev/null +++ b/level3/nyx/validate.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# Correctness check for Nyx on N GPUs with upstream's own comparison tools and +# pre-fixed criteria (no tolerance is derived from the results). +# +# ./validate.sh [CUDA|HIP] HPCPERF_GPUS=N (default 1); HPCPERF_NYX_CASES="minisb lya_adiabatic" +# +# For every case (default: the two official GPU-regression decks, MiniSB and +# LyA-adiabatic, both 10 steps as upstream) the N-GPU run must satisfy ALL of: +# [1] completeness: run.sh exit code 0 (timeout -> FAIL), plt00000 and the final +# plotfile exist, the runlog reaches max_step, every plotfile variable is +# finite (amrex_fextrema min/max through l3_check.require_finite), the DM +# particle count in the final plotfile equals the IC count. +# [2] official regression comparison, upstream tolerance (nightly GPU suite: +# `fcompare -n 0 --rel_tol 2e-10 --abort_if_not_all_found`, plus +# particle_compare on the DM particles at the same tolerance): +# N=1 : against a second, independent 1-GPU run of the same binary and deck +# (same-configuration reproducibility, exactly what the nightly test measures); +# N>1 : against the 1-GPU plotfile of the same binary/deck (rank-count +# independence; the BoxArray is fixed by run.sh, only the distribution changes). +# [3] cross-backend reference: the CPU-profile binary (Nyx_GPU_BACKEND=NONE, same +# Nyx/AMReX/deck) run on 1 rank; fcompare/particle_compare with rel_tol 1e-8, +# a tolerance FIXED HERE before any run (two orders above the same-platform +# tolerance to allow for FMA contraction, libm and reduction-order differences +# between host and device code over 10 steps). +# [4] conservation: total comoving baryon mass sum(density*dV) (amrex_fvolumesum) +# between plt00000 and the final plotfile: |dM/M| <= 1e-9 (adiabatic, periodic: +# no mass source); DM particle count exact. +# Exit codes of run.sh/launcher/application and every tool are captured; a +# missing tool, plotfile or reference is FAIL, never skipped silently. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +R="$(cd "$HERE/../.." && pwd)" +set +u; # shellcheck disable=SC1091 +source "$R/hpcperf_env.sh" 2>/dev/null || true; set -u +# shellcheck disable=SC1091 +source "$R/level3/tools/l3_common.sh" +BACKEND="$(echo "${1:-CUDA}" | tr '[:lower:]' '[:upper:]')" +[ "$BACKEND" != CPU ] || { echo "validate.sh: validates a GPU backend against the CPU reference; use CUDA or HIP" >&2; exit 2; } +N="${HPCPERF_GPUS:-1}" +CASES="${HPCPERF_NYX_CASES:-minisb lya_adiabatic}" +STEPS="${HPCPERF_NYX_STEPS:-10}" +TIMEOUT="${HPCPERF_VALIDATE_TIMEOUT:-1800}" +REL_TOL_SAME=2e-10 # upstream nightly GPU regression tolerance (fcompare --rel_tol) +REL_TOL_XBACKEND=1e-8 # pre-fixed CPU-vs-GPU tolerance (see header) +MASS_TOL=1e-9 # pre-fixed baryon mass conservation tolerance +python3 -c 'import numpy' 2>/dev/null || { echo "validate.sh: python3 with numpy required" >&2; exit 1; } +export HPCPERF_GPUS="$N" HPCPERF_SCALE_MODE=smoke HPCPERF_NYX_STEPS="$STEPS" + +HC="$(echo "${HPCPERF_NYX_HEATCOOL:-NO}" | tr '[:lower:]' '[:upper:]')"; VARIANT=adiabatic; [ "$HC" = YES ] && VARIANT=heatcool +GCC_MM="$(l3_version_mm "$("$CXX" -dumpfullversion 2>/dev/null || "$CXX" -dumpversion)")" +case "$BACKEND" in + CUDA) GPU_PROFILE="${HPCPERF_NYX_PROFILE:-cuda$(l3_version_mm "$(l3_cuda_version)")-gcc${GCC_MM}-${VARIANT}}" ;; + HIP) GPU_PROFILE="${HPCPERF_NYX_PROFILE:-hip-${HPCPERF_HIP_ARCH:-gfx950}-${VARIANT}}" ;; +esac +CPU_PROFILE="${HPCPERF_NYX_CPU_PROFILE:-cpu-gcc${GCC_MM}-${VARIANT}}" +GPU_ROOT="$R/.deps/level3/nyx/$GPU_PROFILE"; CPU_ROOT="$R/.deps/level3/nyx/$CPU_PROFILE" +GPU_RUNS="$R/build/level3/nyx/$GPU_PROFILE/run"; CPU_RUNS="$R/build/level3/nyx/$CPU_PROFILE/run" +TOOLS="$CPU_ROOT/install/bin" +for t in amrex_fcompare amrex_fextrema amrex_fvolumesum particle_compare; do + [ -x "$TOOLS/$t" ] || { echo "validate.sh: FAIL -- $TOOLS/$t missing: build the CPU reference profile first (./build.sh CPU)"; exit 1; } +done +[ -x "$GPU_ROOT/install/bin/nyx_MiniSB" ] || { echo "validate.sh: FAIL -- GPU profile $GPU_PROFILE not built (./build.sh $BACKEND)"; exit 1; } +mkdir -p "$GPU_RUNS" "$CPU_RUNS" +ok=1 +fail() { echo "validate.sh: FAIL -- $*"; ok=0; } + +run_gpu() { # run_gpu [run-dir-suffix] + local case=$1 n=$2 out=$3 rc=0 + HPCPERF_NYX_CASE="$case" HPCPERF_GPUS="$n" timeout "$TIMEOUT" "$HERE/run.sh" "$BACKEND" > "$out" 2>&1 || rc=$? + return $rc +} +run_cpu() { + local case=$1 out=$2 rc=0 + HPCPERF_NYX_CASE="$case" HPCPERF_GPUS=1 HPCPERF_NYX_PROFILE="$CPU_PROFILE" timeout "$TIMEOUT" "$HERE/run.sh" CPU > "$out" 2>&1 || rc=$? + return $rc +} +manifest_val() { /usr/bin/grep -m1 "^$2=" "$1/run_manifest.txt" 2>/dev/null | cut -d= -f2- || true; } +final_plt() { printf 'plt%05d' "$STEPS"; } + +# [1] completeness + finiteness + particle count +check_complete() { # check_complete