Run the whole parallel directory, and at four ranks as well as two

scripts/test.sh

@@ -4,11 +4,13 @@
 #
 # Usage: ./test.sh [OPTIONS]
 #   --p N            Run parallel tests with N MPI ranks (default: skip parallel tests)
+#   --full-parallel  Add a second parallel pass at 4 ranks (see below; slow on CI)
 #   --parallel-only  Run ONLY parallel tests (skip all serial tests)
 #
 # Examples:
 #   ./test.sh                     # All serial tests only
 #   ./test.sh --p 2               # All serial + parallel (2 ranks)
+#   ./test.sh --p 2 --full-parallel   # ... and again at 4 ranks
 #   ./test.sh --parallel-only --p 2   # Only parallel tests (debugging)
 #
 # We do not run one monolithic pytest because tests produce a large number of
@@ -20,19 +22,27 @@ status=0
 # Parse arguments
 PARALLEL_RANKS=0
 PARALLEL_ONLY=0
+# Second parallel pass at four ranks. OFF by default: on a 2-core CI runner
+# np=4 is oversubscribed and the pass costs far more than the ~3 minutes it
+# takes on a workstation — enough to exceed the 120-minute job cap (#573).
+FULL_PARALLEL=0
 while [[ $# -gt 0 ]]; do
     case $1 in
         --p)
             PARALLEL_RANKS="$2"
             shift 2
             ;;
+        --full-parallel)
+            FULL_PARALLEL=1
+            shift
+            ;;
         --parallel-only)
             PARALLEL_ONLY=1
             shift
             ;;
         *)
             echo "Unknown option: $1"
-            echo "Usage: $0 [--p N] [--parallel-only]"
+            echo "Usage: $0 [--p N] [--full-parallel] [--parallel-only]"
             exit 1
             ;;
     esac
@@ -148,19 +158,53 @@ if [ $PARALLEL_RANKS -gt 0 ]; then
     # - Solver operations
     # - Global evaluations
 
-    echo "Testing global statistics and parallel operations..."
-    mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_075*py || status=1
-
-    # Parallel SOLVER tests. This line was commented out, so test_1017 and
-    # test_1062..test_1069 — the whole rotated / constrained / MG parallel set,
-    # including the partition-independence guards for the rotated nodal normal
-    # (#560) and the mesh boundary normal (#564) — executed at NO rank count
-    # in CI.
-    echo "Testing parallel solvers..."
-    mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_10*py || status=1
-
-    # echo "Testing parallel I/O..."
-    # mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_io*py || status=1
+    # Every file under tests/parallel/, in BATCHES. Two things matter here and
+    # they pull in opposite directions.
+    #
+    # Coverage: the batches used to be `test_075*py` and `test_10*py`, which
+    # between them named 14 of the 32 files and left a hole from 0760 to 0999 —
+    # test_0760, test_0765..test_0790, test_0855 and test_0873 ran in parallel
+    # at NO rank count, in CI or locally. That is how the np=4 hang in #611
+    # survived unnoticed, and it is the #570 class: a glob naming ranges grows
+    # holes as files are added between them. So the list is ENUMERATED, and
+    # covers the directory by construction.
+    #
+    # Batching: this script does not run one monolithic pytest, for the reason
+    # in its header — PETSc objects accumulate across files and tests start
+    # interacting. Handing the whole directory to a single mpirun took the CI
+    # job past its 120-minute cap while the same set in batches costs a couple
+    # of minutes. So the enumeration is chunked rather than passed in one go.
+    PARALLEL_BATCH=${PARALLEL_BATCH:-6}
+    PARALLEL_FILES=(tests/parallel/test_*.py)
+    echo "Testing parallel operations, swarms and solvers"
+    echo "  ${#PARALLEL_FILES[@]} files in batches of $PARALLEL_BATCH"
+    for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do
+      mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi \
+        "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" || status=1
+    done
+
+    # A SECOND pass at four ranks, because two is a special case: the failure
+    # this suite exists to catch is a collective entered by some ranks and not
+    # others, and with two ranks the mismatched pair often still meets. Both
+    # recent instances passed at np=2 and hung at np=4 (#609's conditional
+    # collective, and #611).
+    #
+    # Opt-in via --full-parallel. A CI runner has two cores, so np=4 is
+    # oversubscribed there; this belongs in a nightly or a separate job (#573).
+    #
+    # The deselection is #611: that test passes at np=2 and hangs at np=4 on
+    # development. The node id has no `tests/` prefix because tests/pytest.ini
+    # puts rootdir at `tests/`, and a deselect that does not match is ignored in
+    # silence — confirm "1 deselected" in the output when changing it.
+    if [ $FULL_PARALLEL -eq 1 ] && [ "$PARALLEL_RANKS" -ne 4 ]; then
+      echo "Testing the same set at 4 ranks (np=2 is a special case)..."
+      for ((i = 0; i < ${#PARALLEL_FILES[@]}; i += PARALLEL_BATCH)); do
+        mpirun -n 4 python -m pytest --with-mpi \
+          "${PARALLEL_FILES[@]:i:PARALLEL_BATCH}" \
+          --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \
+          || status=1
+      done
+    fi
 
     echo "Parallel tests complete"
     echo "=========================================="

scripts/test_levels.sh

@@ -212,9 +212,16 @@ run_level_2() {
             echo "Running parallel tests (MPI)"
             echo "=========================================="
 
-            # Parallel tests with specified number of ranks
+            # The WHOLE directory, not `test_07*py`. That glob covered
+            # test_0700..test_0790 and so missed test_0005, test_0855,
+            # test_0873 and the entire test_10* solver set. `scripts/test.sh`
+            # had a different hole (test_075* and test_10*, missing
+            # test_0760..test_0790), so between the two scripts three files ran
+            # at no rank count anywhere: test_0005, test_0855 and test_0873.
+            # A glob naming ranges grows holes as files are added between them
+            # (#570, #611).
             echo "Testing with $PARALLEL_RANKS MPI ranks..."
-            if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then
+            if mpirun -n $PARALLEL_RANKS python -m pytest --with-mpi tests/parallel/ $VERBOSE; then
                 echo "✅ PASSED: Parallel tests ($PARALLEL_RANKS ranks)"
             else
                 echo "❌ FAILED: Parallel tests ($PARALLEL_RANKS ranks)"
@@ -224,8 +231,14 @@ run_level_2() {
             # Optional: Test with 4 ranks if --full-parallel specified
             if [ $FULL_PARALLEL -eq 1 ]; then
                 echo ""
+                # The deselection is #611: this test passes at np=2 and hangs
+                # at np=4 on development. The node id carries no `tests/`
+                # prefix because tests/pytest.ini puts rootdir at `tests/`, and
+                # a deselect that does not match is ignored in silence.
                 echo "Running extended parallel tests (4 ranks)..."
-                if mpirun -n 4 python -m pytest --with-mpi tests/parallel/test_07*py $VERBOSE; then
+                if mpirun -n 4 python -m pytest --with-mpi tests/parallel/ \
+                    --deselect "parallel/test_0760_swarm_cache_migration.py::test_global_evaluate_after_migration" \
+                    $VERBOSE; then
                     echo "✅ PASSED: Parallel tests (4 ranks)"
                 else
                     echo "❌ FAILED: Parallel tests (4 ranks)"