diff --git a/overlays/patches/README.md b/overlays/patches/README.md index 45f902f..d254048 100644 --- a/overlays/patches/README.md +++ b/overlays/patches/README.md @@ -19,7 +19,7 @@ change independently of Nix, and `default.nix` preserves existing patches. | [GnuTLS](gnutls/README.md) | Wait for the UDP server socket before connecting | | [Jupyter Server](jupyter-server/README.md) | Exercise the correct shared future during reconnect | | [Prometheus](prometheus/README.md) | Complete parsing before inspecting the test editor state | -| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements despite concurrent crashes | +| [pytest-xdist](pytest-xdist/README.md) | Check worker replacements and allow startup on loaded builders | | [SciPy](scipy/README.md) | Account for floating-point rounding in STFT tests | | [Sentry SDK](sentry-sdk/README.md) | Isolate SDK thread mocks from Python's threading module | | [Torchaudio](torchaudio/README.md) | Compare pitch-shift batches at appropriate precision | @@ -33,8 +33,8 @@ package set. Abseil repairs several vendored copies and is gated on `x86-64-v3`; Prometheus patches its separate assets derivation; Python packages use `pythonPackagesExtensions`. -[`../test-exclusions.nix`](../test-exclusions.nix) retains only pytest-xdist's -outer-worker limit and inner-worker startup allowance. It adds no skipped +The [pytest-xdist directory](pytest-xdist/README.md) also owns its outer-worker +limit and remote-worker event timeout. These package overrides add no skipped tests. Existing nixpkgs exclusions remain separate from these repairs. The test-exclusion review used Python 3.14.7 and the pinned x86-64-v3 package diff --git a/overlays/patches/default.nix b/overlays/patches/default.nix index 0029556..fbc0c5e 100644 --- a/overlays/patches/default.nix +++ b/overlays/patches/default.nix @@ -6,10 +6,9 @@ _final: prev: pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ (_pythonFinal: pythonPrev: { - backrefs = import ./backrefs { inherit (pythonPrev) backrefs; }; pytest-xdist = import ./pytest-xdist { inherit (pythonPrev) pytest-xdist; }; + scipy = import ./scipy { inherit (pythonPrev) scipy; }; sentry-sdk = import ./sentry-sdk { inherit (pythonPrev) sentry-sdk; }; - torchcodec = import ./torchcodec { inherit (pythonPrev) torchcodec; }; }) ]; } diff --git a/overlays/patches/pytest-xdist/README.md b/overlays/patches/pytest-xdist/README.md index aa47a94..07fd295 100644 --- a/overlays/patches/pytest-xdist/README.md +++ b/overlays/patches/pytest-xdist/README.md @@ -1,4 +1,4 @@ -# pytest-xdist concurrent worker crashes +# pytest-xdist test fixes With two workers and a restart limit of three, the fourth worker crash requests shutdown while another test can still be running. That test may @@ -13,6 +13,10 @@ even though five failures can occur without exceeding the replacement limit. five failed tests, the failed-tests exit status, the limit message, and no internal error. It retains the two-worker workload and ten queued tests. +`worker-startup-timeout.patch` changes the remote-test helper's event timeout +from 10 to 60 seconds so loaded builders have time to start workers. The +helper returns immediately when an event arrives and still has a bounded wait. + The existing nixpkgs pytest-9 compatibility patches remain in place. Production scheduling and worker-restart behavior are unchanged. @@ -23,9 +27,11 @@ the nixpkgs pytest-9 compatibility patches where required. From this directory: ```sh patch --fuzz=0 -d /path/to/pytest-xdist -p1 < concurrent-worker-crashes.patch +patch --fuzz=0 -d /path/to/pytest-xdist -p1 < worker-startup-timeout.patch cd /path/to/pytest-xdist python -m pytest testing/acceptance_test.py \ -k test_max_worker_restart_tests_queued -q +python -m pytest testing/test_remote.py -q ``` Twenty unmodified runs passed during the review. To force the failing @@ -36,22 +42,23 @@ tests when shutdown starts. Bound the marker wait so a reproduction failure cannot hang the suite. The original assertion fails on five reported failures; the patched test passes. -## Remaining resource settings +## Worker startup and outer concurrency -[`../../test-exclusions.nix`](../../test-exclusions.nix) runs the outer suite -with one worker and sets the inner-worker wait to 60 seconds. These settings -limit nested process pools and allow worker startup on loaded builders. +[`default.nix`](default.nix) runs the outer suite with one worker to limit +nested process pools. This is a Nix test-runner setting; the source timeout +change lives in [`worker-startup-timeout.patch`](worker-startup-timeout.patch). A separate reproduction inserts an 11-second `pytest_sessionstart` delay into the child created by `test_basic_collect_and_runtests` in `testing/test_remote.py`. The original 10-second channel wait fails; the -60-second wait passes. This is a worker-startup bound, not a product deadline. +60-second wait passes. This bounds waits for test worker events, including +startup, rather than changing a product deadline. ## Upstream status -This is a standalone test patch for pytest-xdist 3.8.0. No upstream submission -was made during this work. Recheck the allowed in-flight failures and -replacement count when updating the scheduler or shutdown behavior. +These are standalone test patches for pytest-xdist 3.8.0. No upstream submission +was made during this work. Recheck the allowed in-flight failures, replacement +count, and remote-test wait when updating the scheduler or worker behavior. ## Local NixOS integration and build results @@ -62,6 +69,11 @@ replacement count when updating the scheduler or shutdown behavior. nix build --no-link -L .#nixosConfigurations.jeeves.pkgs.python314Packages.pytest-xdist ``` -The patched x86-64-v3 package passed 185 tests, with 6 existing skips and -10 expected failures. The forced concurrent-crash reproduction passed after -the fix, and the focused test passed again after formatting the assertion. +After consolidating the settings in this directory, the full x86-64-v3 package +build passed 185 tests, with 6 existing skips and 10 expected failures. Nix +evaluation confirmed the same outer-worker limit and preserved existing +patches, with the timeout now applied as a source patch. + +The earlier forced concurrent-crash and delayed-startup reproductions passed +after their fixes; the focused crash test also passed after formatting its +assertion. diff --git a/overlays/patches/pytest-xdist/default.nix b/overlays/patches/pytest-xdist/default.nix index 1f81ee6..25c068e 100644 --- a/overlays/patches/pytest-xdist/default.nix +++ b/overlays/patches/pytest-xdist/default.nix @@ -1,4 +1,12 @@ { pytest-xdist }: pytest-xdist.overridePythonAttrs (old: { - patches = (old.patches or [ ]) ++ [ ./concurrent-worker-crashes.patch ]; + patches = (old.patches or [ ]) ++ [ + ./concurrent-worker-crashes.patch + ./worker-startup-timeout.patch + ]; + + # The suite exercises its own worker pools. Limit the outer suite to one worker. + preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] ( + old.preCheck or "" + ); }) diff --git a/overlays/patches/pytest-xdist/worker-startup-timeout.patch b/overlays/patches/pytest-xdist/worker-startup-timeout.patch new file mode 100644 index 0000000..0e83bdc --- /dev/null +++ b/overlays/patches/pytest-xdist/worker-startup-timeout.patch @@ -0,0 +1,19 @@ +Subject: [PATCH] tests: allow more time for remote worker events + +Worker startup can exceed ten seconds on heavily loaded builders. Allow +the remote-test helper to wait up to sixty seconds for worker events. +The wait still returns as soon as an event arrives and remains bounded. +Production worker timeouts and test assertions are unchanged. + +--- a/testing/test_remote.py ++++ b/testing/test_remote.py +@@ -17,7 +17,8 @@ + from xdist.workermanage import WorkerController + + +-WAIT_TIMEOUT = 10.0 ++# Allow worker events extra time on heavily loaded builders. ++WAIT_TIMEOUT = 60.0 + + + def check_marshallable(d: object) -> None: diff --git a/overlays/test-exclusions.nix b/overlays/test-exclusions.nix deleted file mode 100644 index 95aac73..0000000 --- a/overlays/test-exclusions.nix +++ /dev/null @@ -1,27 +0,0 @@ -# Test resource settings for the locally rebuilt x86-64-v3 package set. -# -# Selecting x86-64-v3 changes every affected derivation, so the normal -# nixpkgs binary cache cannot be used and upstream test suites run locally. -# The jeeves builder uses /tmp/nix-builds so filesystem tests run on tmpfs -# instead of ZFS with normalization=formD and utf8only=on; those tests remain -# enabled. This overlay no longer excludes any tests. The remaining settings -# bound nested worker concurrency and allow time for worker startup under load. -# Test repairs and their validation are indexed in patches/README.md. -_final: prev: { - pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ - (_pythonFinal: pythonPrev: { - pytest-xdist = pythonPrev.pytest-xdist.overridePythonAttrs (old: { - # The suite exercises its own worker pools. Run the outer suite with one - # worker and allow inner workers more time on heavily loaded builders. - postPatch = (old.postPatch or "") + '' - substituteInPlace testing/test_remote.py \ - --replace-fail "WAIT_TIMEOUT = 10.0" "WAIT_TIMEOUT = 60.0" - ''; - preCheck = builtins.replaceStrings [ "--numprocesses=$NIX_BUILD_CORES" ] [ "--numprocesses=1" ] ( - old.preCheck or "" - ); - }); - - }) - ]; -}