CI: shard tests across 4 runners; heavy tests on main and nightly (#350)

* CI: build tests once and run them in 4 nextest shards Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01RjGw4gA5ZvKrU8MQP6sBgT * CI: quote the shard arguments Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01RjGw4gA5ZvKrU8MQP6sBgT * CI: extract the test archive into the checkout; skip the heaviest tests on pull requests Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01RjGw4gA5ZvKrU8MQP6sBgT * Docs: CI shards and the tests pull requests skip Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01RjGw4gA5ZvKrU8MQP6sBgT --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
2026-10-12 14:30:35 -04:00
1 parent 511d105dcb
commit d15869ab85
@@ -0,0 +1,29 @@
+# cargo-nextest settings for CI (`cargo xtask test` does not use nextest).
+# PR runs build the tests once and run them in 4 shards (.github/workflows/ci.yml).
+
+[profile.ci]
+# Heavy tests (one to four minutes each on a CI runner) that pull requests skip. They run on
+# every push to main and nightly (`--ignore-default-filter`), and in `cargo xtask test`.
+# Keep this list in sync with docs/book/src/dev/testing.md and CONTRIBUTING.md.
+default-filter = """
+not (
+    (package(solvecraft-engine) & (
+        test(=tests::hostile_params_never_panic)
+        | test(/^tests::robustness::/)
+        | test(=pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen)
+        | test(=pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs)
+        | test(=cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall)
+    ))
+    | (package(solvecraft-kernel) & test(=tests::boolean_identity_on_tangent_primitives))
+    | (package(solvecraft-ui-egui) & binary(scenarios) & test(=acceptance_sensor_enclosure))
+)
+"""
+fail-fast = false
+# Every test's time in the log, so the slow ones are easy to find.
+status-level = "pass"
+final-status-level = "slow"
+failure-output = "immediate-final"
+slow-timeout = { period = "60s" }
+
+[profile.ci.junit]
+path = "junit.xml"
@@ -1,9 +1,15 @@
 # Fast checks always run; native builds run for code/CI changes.
+# Tests are built once and run in 4 shards. A few heavy tests (see .config/nextest.toml) are
+# skipped on pull requests and run on pushes to main and nightly.
 # Cross-compilation is opt-in through Run workflow; release builds remain in release.yml.
 name: CI

 on:
  pull_request:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "17 6 * * *"
  workflow_dispatch:
    inputs:
      cross_compile:
@@ -16,7 +22,8 @@ permissions:

 concurrency:
  group: ci-${{ github.ref }}
-  cancel-in-progress: true
+  # Every push to main gets its full run; a new PR push replaces the old run.
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

 env:
  CARGO_TERM_COLOR: always
@@ -39,7 +46,7 @@ jobs:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          native=false
-          if [ "$EVENT_NAME" = workflow_dispatch ]; then
+          if [ "$EVENT_NAME" != pull_request ]; then
            native=true
          else
            # Compare both old and new paths of renames; unknown paths run native checks.
@@ -92,12 +99,12 @@ jobs:
          key: native-clippy
      - run: cargo clippy --workspace --all-targets --profile ci --locked -- -D warnings

-  tests:
-    name: Native Linux tests
+  build-tests:
+    name: Build tests
    needs: changes
    if: needs.changes.outputs.native == 'true'
    runs-on: ubuntu-24.04
-    timeout-minutes: 90
+    timeout-minutes: 45
    env:
      CARGO_INCREMENTAL: "0"
      RUSTC_WRAPPER: sccache
@@ -113,15 +120,82 @@ jobs:
        uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
        with:
          version: v0.16.0
-      - name: Compile tests
-        run: cargo test --workspace --profile ci --locked --no-run
-      - name: Run tests
-        # Every test binary at once (cargo test runs them one after another).
-        run: cargo xtask test --locked
+      - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28
+        with:
+          tool: cargo-nextest
+      - name: Compile tests into a nextest archive
+        run: cargo nextest archive --workspace --cargo-profile ci --locked --archive-file tests.tar.zst
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: nextest-archive
+          path: tests.tar.zst
+          retention-days: 1
+          compression-level: 0
      - name: Compiler cache statistics
        if: always()
        run: sccache --show-stats

+  tests:
+    name: Native Linux tests (${{ matrix.shard }}/4)
+    needs: build-tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28
+        with:
+          tool: cargo-nextest
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: nextest-archive
+      - name: Run tests
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          SHARD: ${{ matrix.shard }}
+        run: |
+          # Pull requests skip the heavy tests (default-filter in .config/nextest.toml).
+          all=()
+          if [ "$EVENT_NAME" != pull_request ]; then all=(--ignore-default-filter); fi
+          # Extract into ./target: tests that run a built binary (env!("CARGO_BIN_EXE_*")) have
+          # the build job's path, which is the same checkout path here.
+          cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \
+            --profile ci --partition "slice:$SHARD/4" "${all[@]}"
+      - name: Upload test timings
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: junit-${{ matrix.shard }}
+          path: target/nextest/ci/junit.xml
+          retention-days: 7
+
+  doc-tests:
+    name: Doc tests
+    needs: changes
+    if: needs.changes.outputs.native == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    env:
+      CARGO_INCREMENTAL: "0"
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          key: doc-tests
+      - name: Set up compiler cache
+        uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
+        with:
+          version: v0.16.0
+      # nextest does not run doc tests.
+      - run: cargo test --workspace --profile ci --locked --doc
+
  oracle:
    name: Reference tests
    needs: changes
@@ -156,7 +230,7 @@ jobs:

  # Preserve the existing check name for branch protection, including docs-only PRs.
  ci:
-    needs: [changes, checks, clippy, tests, oracle]
+    needs: [changes, checks, clippy, build-tests, tests, doc-tests, oracle]
    if: always()
    runs-on: ubuntu-24.04
    steps:
@@ -166,14 +240,18 @@ jobs:
          CHECKS: ${{ needs.checks.result }}
          NATIVE: ${{ needs.changes.outputs.native }}
          CLIPPY: ${{ needs.clippy.result }}
+          BUILD: ${{ needs.build-tests.result }}
          TESTS: ${{ needs.tests.result }}
+          DOCS: ${{ needs.doc-tests.result }}
          ORACLE: ${{ needs.oracle.result }}
        run: |
          test "$CHANGES" = success
          test "$CHECKS" = success
          if [ "$NATIVE" = true ]; then
            test "$CLIPPY" = success
+            test "$BUILD" = success
            test "$TESTS" = success
+            test "$DOCS" = success
            test "$ORACLE" = success
          fi

@@ -153,6 +153,19 @@ in the PR. `cargo xtask test` builds the tests with it and then runs every test
 `cargo xtask test -- shell`. Tests run beside each other on a busy machine, so a test must not
 depend on wall-clock timing.

+**CI shards the tests.** One job builds them into a cargo-nextest archive and four jobs run a
+quarter each, so every test runs in its own process: don't rely on state left by another test.
+Pull requests skip a few heavy tests (one to four minutes each), which run on every push to
+`main` and nightly; check the `main` run after merging. They are listed in
+`.config/nextest.toml` and the [testing page](docs/book/src/dev/testing.md):
+
+- `solvecraft-engine`: `tests::hostile_params_never_panic`, everything in `tests::robustness`,
+  `pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen`,
+  `pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs`,
+  `cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall`
+- `solvecraft-kernel`: `tests::boolean_identity_on_tangent_primitives`
+- `solvecraft-ui-egui` scenarios: `acceptance_sensor_enclosure`
+
 **The reference tests** are 76 parts in [`tests/oracle/`](tests/oracle/), each a `recipe.json`
 in our own format with an `expected.json` beside it. SolidCraft rebuilds the recipe and compares
 body count, volume, area, centre of mass and bounding box (and, where there is a `reference.stl`,
@@ -45,3 +45,23 @@ Run `cargo xtask test` to build with this profile and run the test binaries conc
 with doc tests alongside. Arguments after `--` go to each test binary, for example
 `cargo xtask test -- shell`. To run the test binaries sequentially instead, use
 `cargo test --workspace --profile ci`.
+
+## CI sharding and heavy tests
+
+On GitHub, one job builds the tests into a
+[cargo-nextest](https://nexte.st/) archive and four jobs run a quarter of the tests each
+(`--partition slice:N/4`, settings in `.config/nextest.toml`); doc tests run in their own job.
+Pull requests skip a few heavy tests (each takes one to four minutes); they run on every push to
+`main`, nightly, and in `cargo xtask test`:
+
+- `solvecraft-engine`: `tests::hostile_params_never_panic`, everything in `tests::robustness`,
+  `pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen`,
+  `pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs`,
+  `cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall`
+- `solvecraft-kernel`: `tests::boolean_identity_on_tangent_primitives`
+- `solvecraft-ui-egui` scenarios: `acceptance_sensor_enclosure`
+
+To run exactly what a pull request runs: `cargo nextest run --cargo-profile ci --profile ci`;
+add `--ignore-default-filter` for the full set. A new test that takes over a minute on CI can
+join the list (the `default-filter` in `.config/nextest.toml`) if it guards robustness rather
+than everyday behaviour.