diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000..ec67145f --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,29 @@ +# cargo-nextest settings for CI (`cargo xtask test` does not use nextest). +# PR runs build the tests once and run them in 4 shards (.github/workflows/ci.yml). + +[profile.ci] +# Heavy tests (one to four minutes each on a CI runner) that pull requests skip. They run on +# every push to main and nightly (`--ignore-default-filter`), and in `cargo xtask test`. +# Keep this list in sync with docs/book/src/dev/testing.md and CONTRIBUTING.md. +default-filter = """ +not ( + (package(solvecraft-engine) & ( + test(=tests::hostile_params_never_panic) + | test(/^tests::robustness::/) + | test(=pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen) + | test(=pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs) + | test(=cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall) + )) + | (package(solvecraft-kernel) & test(=tests::boolean_identity_on_tangent_primitives)) + | (package(solvecraft-ui-egui) & binary(scenarios) & test(=acceptance_sensor_enclosure)) +) +""" +fail-fast = false +# Every test's time in the log, so the slow ones are easy to find. +status-level = "pass" +final-status-level = "slow" +failure-output = "immediate-final" +slow-timeout = { period = "60s" } + +[profile.ci.junit] +path = "junit.xml" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10e90df8..5ab041b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,15 @@ # Fast checks always run; native builds run for code/CI changes. +# Tests are built once and run in 4 shards. A few heavy tests (see .config/nextest.toml) are +# skipped on pull requests and run on pushes to main and nightly. # Cross-compilation is opt-in through Run workflow; release builds remain in release.yml. name: CI on: pull_request: + push: + branches: [main] + schedule: + - cron: "17 6 * * *" workflow_dispatch: inputs: cross_compile: @@ -16,7 +22,8 @@ permissions: concurrency: group: ci-${{ github.ref }} - cancel-in-progress: true + # Every push to main gets its full run; a new PR push replaces the old run. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: CARGO_TERM_COLOR: always @@ -39,7 +46,7 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | native=false - if [ "$EVENT_NAME" = workflow_dispatch ]; then + if [ "$EVENT_NAME" != pull_request ]; then native=true else # Compare both old and new paths of renames; unknown paths run native checks. @@ -92,12 +99,12 @@ jobs: key: native-clippy - run: cargo clippy --workspace --all-targets --profile ci --locked -- -D warnings - tests: - name: Native Linux tests + build-tests: + name: Build tests needs: changes if: needs.changes.outputs.native == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 45 env: CARGO_INCREMENTAL: "0" RUSTC_WRAPPER: sccache @@ -113,15 +120,82 @@ jobs: uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 with: version: v0.16.0 - - name: Compile tests - run: cargo test --workspace --profile ci --locked --no-run - - name: Run tests - # Every test binary at once (cargo test runs them one after another). - run: cargo xtask test --locked + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 + with: + tool: cargo-nextest + - name: Compile tests into a nextest archive + run: cargo nextest archive --workspace --cargo-profile ci --locked --archive-file tests.tar.zst + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nextest-archive + path: tests.tar.zst + retention-days: 1 + compression-level: 0 - name: Compiler cache statistics if: always() run: sccache --show-stats + tests: + name: Native Linux tests (${{ matrix.shard }}/4) + needs: build-tests + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: taiki-e/install-action@1168f6810a3cb30a3514bf5756a3b7f0422c6df8 # v2.87.28 + with: + tool: cargo-nextest + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: nextest-archive + - name: Run tests + env: + EVENT_NAME: ${{ github.event_name }} + SHARD: ${{ matrix.shard }} + run: | + # Pull requests skip the heavy tests (default-filter in .config/nextest.toml). + all=() + if [ "$EVENT_NAME" != pull_request ]; then all=(--ignore-default-filter); fi + # Extract into ./target: tests that run a built binary (env!("CARGO_BIN_EXE_*")) have + # the build job's path, which is the same checkout path here. + cargo nextest run --archive-file tests.tar.zst --workspace-remap . --extract-to . --extract-overwrite \ + --profile ci --partition "slice:$SHARD/4" "${all[@]}" + - name: Upload test timings + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: junit-${{ matrix.shard }} + path: target/nextest/ci/junit.xml + retention-days: 7 + + doc-tests: + name: Doc tests + needs: changes + if: needs.changes.outputs.native == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + CARGO_INCREMENTAL: "0" + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_IGNORE_SERVER_IO_ERROR: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: doc-tests + - name: Set up compiler cache + uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 + with: + version: v0.16.0 + # nextest does not run doc tests. + - run: cargo test --workspace --profile ci --locked --doc + oracle: name: Reference tests needs: changes @@ -156,7 +230,7 @@ jobs: # Preserve the existing check name for branch protection, including docs-only PRs. ci: - needs: [changes, checks, clippy, tests, oracle] + needs: [changes, checks, clippy, build-tests, tests, doc-tests, oracle] if: always() runs-on: ubuntu-24.04 steps: @@ -166,14 +240,18 @@ jobs: CHECKS: ${{ needs.checks.result }} NATIVE: ${{ needs.changes.outputs.native }} CLIPPY: ${{ needs.clippy.result }} + BUILD: ${{ needs.build-tests.result }} TESTS: ${{ needs.tests.result }} + DOCS: ${{ needs.doc-tests.result }} ORACLE: ${{ needs.oracle.result }} run: | test "$CHANGES" = success test "$CHECKS" = success if [ "$NATIVE" = true ]; then test "$CLIPPY" = success + test "$BUILD" = success test "$TESTS" = success + test "$DOCS" = success test "$ORACLE" = success fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4e15972..20982b91 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,6 +153,19 @@ in the PR. `cargo xtask test` builds the tests with it and then runs every test `cargo xtask test -- shell`. Tests run beside each other on a busy machine, so a test must not depend on wall-clock timing. +**CI shards the tests.** One job builds them into a cargo-nextest archive and four jobs run a +quarter each, so every test runs in its own process: don't rely on state left by another test. +Pull requests skip a few heavy tests (one to four minutes each), which run on every push to +`main` and nightly; check the `main` run after merging. They are listed in +`.config/nextest.toml` and the [testing page](docs/book/src/dev/testing.md): + +- `solvecraft-engine`: `tests::hostile_params_never_panic`, everything in `tests::robustness`, + `pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen`, + `pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs`, + `cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall` +- `solvecraft-kernel`: `tests::boolean_identity_on_tangent_primitives` +- `solvecraft-ui-egui` scenarios: `acceptance_sensor_enclosure` + **The reference tests** are 76 parts in [`tests/oracle/`](tests/oracle/), each a `recipe.json` in our own format with an `expected.json` beside it. SolidCraft rebuilds the recipe and compares body count, volume, area, centre of mass and bounding box (and, where there is a `reference.stl`, diff --git a/docs/book/src/dev/testing.md b/docs/book/src/dev/testing.md index 1d8ec605..e8fba22c 100644 --- a/docs/book/src/dev/testing.md +++ b/docs/book/src/dev/testing.md @@ -45,3 +45,23 @@ Run `cargo xtask test` to build with this profile and run the test binaries conc with doc tests alongside. Arguments after `--` go to each test binary, for example `cargo xtask test -- shell`. To run the test binaries sequentially instead, use `cargo test --workspace --profile ci`. + +## CI sharding and heavy tests + +On GitHub, one job builds the tests into a +[cargo-nextest](https://nexte.st/) archive and four jobs run a quarter of the tests each +(`--partition slice:N/4`, settings in `.config/nextest.toml`); doc tests run in their own job. +Pull requests skip a few heavy tests (each takes one to four minutes); they run on every push to +`main`, nightly, and in `cargo xtask test`: + +- `solvecraft-engine`: `tests::hostile_params_never_panic`, everything in `tests::robustness`, + `pattern_suppress_tests::enclosure_arc_extinction_keeps_a_real_sharp_inner_lip_after_resize_and_reopen`, + `pattern_suppress_tests::enclosure_datums_follow_wall_height_and_board_depth_with_material_proofs`, + `cmd::plastic::tests::enclosure_snap_fit_follows_resized_front_wall` +- `solvecraft-kernel`: `tests::boolean_identity_on_tangent_primitives` +- `solvecraft-ui-egui` scenarios: `acceptance_sensor_enclosure` + +To run exactly what a pull request runs: `cargo nextest run --cargo-profile ci --profile ci`; +add `--ignore-default-filter` for the full set. A new test that takes over a minute on CI can +join the list (the `default-filter` in `.config/nextest.toml`) if it guards robustness rather +than everyday behaviour.