diff --git a/doc/api/cli.md b/doc/api/cli.md index 065439a777e5dc..e4a0614c0c8fa3 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -2762,6 +2762,38 @@ changes: Configures the test runner to only execute top level tests that have the `only` option set. This flag is not necessary when test isolation is disabled. +### `--test-random-seed` + + + +Set the seed used to randomize test execution order. This applies to both test +file execution order and queued tests within each file. Providing this flag +enables randomization implicitly, even without `--test-randomize`. + +The value must be an integer between `0` and `4294967295`. + +This flag cannot be used with `--watch` or `--test-rerun-failures`. + +### `--test-randomize` + + + +Randomize test execution order. This applies to both test file execution order +and queued tests within each file. This can help detect tests that rely on +shared state or execution order. + +The seed used for randomization is printed in the test summary and can be +reused with `--test-random-seed`. + +For detailed behavior and examples, see +[randomizing tests execution order][]. + +This flag cannot be used with `--watch` or `--test-rerun-failures`. + ### `--test-reporter` + +> Stability: 1.0 - Early development + +The test runner can randomize execution order to help detect +order-dependent tests. When enabled, the runner randomizes both discovered +test files and queued tests within each file. Use `--test-randomize` to +enable this mode. + +```bash +node --test --test-randomize +``` + +When randomization is enabled, the test runner prints the seed used for the run +as a diagnostic message: + +```text +Randomized test order seed: 12345 +``` + +Use `--test-random-seed=` to replay the same randomized order +deterministically. Supplying `--test-random-seed` also enables randomization, +so `--test-randomize` is optional when a seed is provided: + +```bash +node --test --test-randomize --test-random-seed=12345 +``` + +In most test files, randomization works automatically. One important exception +is when subtests are awaited one by one. In that pattern, each subtest starts +only after the previous one finishes, so the runner keeps declaration order +instead of randomizing it. + +Example: this runs sequentially and is **not** randomized. + +```mjs +import test from 'node:test'; + +test('math', async (t) => { + for (const name of ['adds', 'subtracts', 'multiplies']) { + // Sequentially awaiting each subtest preserves declaration order. + await t.test(name, async () => {}); + } +}); +``` + +```cjs +const test = require('node:test'); + +test('math', async (t) => { + for (const name of ['adds', 'subtracts', 'multiplies']) { + // Sequentially awaiting each subtest preserves declaration order. + await t.test(name, async () => {}); + } +}); +``` + +Using suite-style APIs such as `describe()`/`it()` or `suite()`/`test()` +still allows randomization, because sibling tests are queued together. + +Example: this remains eligible for randomization. + +```mjs +import { describe, it } from 'node:test'; + +describe('math', () => { + it('adds', () => {}); + it('subtracts', () => {}); + it('multiplies', () => {}); +}); +``` + +```cjs +const { describe, it } = require('node:test'); + +describe('math', () => { + it('adds', () => {}); + it('subtracts', () => {}); + it('multiplies', () => {}); +}); +``` + +`--test-randomize` and `--test-random-seed` are not supported with `--watch` mode. + Matching files are executed as test files. More information on the test file execution can be found in the [test runner execution model][] section. @@ -625,6 +713,10 @@ test runner functionality: * `--test-reporter` - Reporting is managed by the parent process * `--test-reporter-destination` - Output destinations are controlled by the parent * `--experimental-config-file` - Config file paths are managed by the parent +* `--test-randomize` - Randomization is managed by the parent process and + propagated to child processes +* `--test-random-seed` - Randomization seed is managed by the parent process and + propagated to child processes All other Node.js options from command line arguments, environment variables, and configuration files are inherited by the child processes. @@ -1531,6 +1623,13 @@ changes: that specifies the index of the shard to run. This option is _required_. * `total` {number} is a positive integer that specifies the total number of shards to split the test files to. This option is _required_. + * `randomize` {boolean} Randomize execution order for test files and queued tests. + This option is not supported with `watch: true`. + **Default:** `false`. + * `randomSeed` {number} Seed used when randomizing execution order. If this + option is set, runs can replay the same randomized order deterministically, + and setting this option also enables randomization. + **Default:** `undefined`. * `rerunFailuresFilePath` {string} A file path where the test runner will store the state of the tests to allow rerunning only the failed tests on a next run. see \[Rerunning failed tests]\[] for more information. diff --git a/doc/node-config-schema.json b/doc/node-config-schema.json index 8894e1a410a28a..8dfc59da7c7fec 100644 --- a/doc/node-config-schema.json +++ b/doc/node-config-schema.json @@ -528,6 +528,14 @@ "type": "boolean", "description": "run tests with 'only' option set" }, + "test-random-seed": { + "type": "number", + "description": "seed used to randomize test execution order" + }, + "test-randomize": { + "type": "boolean", + "description": "run tests in a random order" + }, "test-reporter": { "oneOf": [ { @@ -910,6 +918,14 @@ "type": "boolean", "description": "run tests with 'only' option set" }, + "test-random-seed": { + "type": "number", + "description": "seed used to randomize test execution order" + }, + "test-randomize": { + "type": "boolean", + "description": "run tests in a random order" + }, "test-reporter": { "oneOf": [ { diff --git a/doc/node.1 b/doc/node.1 index 6d16ef84c45df0..268c077a061055 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -1357,6 +1357,22 @@ tests must satisfy \fBboth\fR requirements in order to be executed. Configures the test runner to only execute top level tests that have the \fBonly\fR option set. This flag is not necessary when test isolation is disabled. . +.It Fl -test-random-seed +Set the seed used to randomize test execution order. +This applies to both test file execution order and queued tests within each file. +Providing this flag enables randomization implicitly, even without +\fB--test-randomize\fR. +The value must be an integer between 0 and 4294967295. +This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR. +. +.It Fl -test-randomize +Randomize test execution order. +This applies to both test file execution order and queued tests within each file. +This can help detect tests that rely on shared state or execution order. +The seed used for randomization is printed in the test summary and can be +reused with \fB--test-random-seed\fR. +This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR. +. .It Fl -test-reporter A test reporter to use when running tests. See the documentation on test reporters for more details. @@ -2034,6 +2050,10 @@ one is included in the list below. .It \fB--test-reporter-destination\fR .It +\fB--test-randomize\fR +.It +\fB--test-random-seed\fR +.It \fB--test-reporter\fR .It \fB--test-rerun-failures\fR diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index df2c85bdaed8de..30780bff4131ab 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -56,6 +56,7 @@ const { validateObject, validateOneOf, validateInteger, + validateUint32, validateString, validateStringArray, } = require('internal/validators'); @@ -81,10 +82,12 @@ const { const { FastBuffer } = require('internal/buffer'); const { + createRandomSeed, convertStringToRegExp, countCompletedTest, kDefaultPattern, parseCommandLine, + shuffleArrayWithSeed, } = require('internal/test_runner/utils'); const { Glob } = require('internal/fs/glob'); const { once } = require('events'); @@ -102,12 +105,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName'); const kFilterArgs = [ '--test', '--experimental-test-coverage', + '--test-randomize', '--watch', '--experimental-default-config-file', ]; const kFilterArgValues = [ '--test-reporter', '--test-reporter-destination', + '--test-random-seed', '--experimental-config-file', ]; const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms']; @@ -150,6 +155,8 @@ function getRunArgs(path, { forceExit, argv: suppliedArgs, execArgv, rerunFailuresFilePath, + randomize, + randomSeed, root: { timeout }, cwd }) { const processNodeOptions = getOptionsAsFlagsFromBinding(); @@ -191,6 +198,12 @@ function getRunArgs(path, { forceExit, if (rerunFailuresFilePath) { ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`); } + if (randomize === true) { + ArrayPrototypePush(runArgs, '--test-randomize'); + } + if (randomSeed != null) { + ArrayPrototypePush(runArgs, `--test-random-seed=${randomSeed}`); + } ArrayPrototypePushApply(runArgs, execArgv); @@ -619,6 +632,8 @@ function run(options = kEmptyObject) { lineCoverage = 0, branchCoverage = 0, functionCoverage = 0, + randomize: suppliedRandomize, + randomSeed: suppliedRandomSeed, execArgv = [], argv = [], cwd = process.cwd(), @@ -647,6 +662,56 @@ function run(options = kEmptyObject) { if (globPatterns != null) { validateArray(globPatterns, 'options.globPatterns'); } + if (suppliedRandomize != null) { + validateBoolean(suppliedRandomize, 'options.randomize'); + } + if (suppliedRandomSeed != null) { + validateUint32(suppliedRandomSeed, 'options.randomSeed'); + } + let randomize = suppliedRandomize; + let randomSeed = suppliedRandomSeed; + + if (randomSeed != null) { + randomize = true; + } + if (watch) { + if (randomSeed != null) { + throw new ERR_INVALID_ARG_VALUE( + 'options.randomSeed', + randomSeed, + 'is not supported with watch mode', + ); + } + if (randomize) { + throw new ERR_INVALID_ARG_VALUE( + 'options.randomize', + randomize, + 'is not supported with watch mode', + ); + } + } + if (rerunFailuresFilePath) { + validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath'); + // TODO(pmarchini): Support rerun-failures with randomization by + // persisting the randomization seed in the rerun state file. + if (randomSeed != null) { + throw new ERR_INVALID_ARG_VALUE( + 'options.randomSeed', + randomSeed, + 'is not supported with rerun failures mode', + ); + } + if (randomize) { + throw new ERR_INVALID_ARG_VALUE( + 'options.randomize', + randomize, + 'is not supported with rerun failures mode', + ); + } + } + if (randomize) { + randomSeed ??= createRandomSeed(); + } validateString(cwd, 'options.cwd'); @@ -656,10 +721,6 @@ function run(options = kEmptyObject) { ); } - if (rerunFailuresFilePath) { - validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath'); - } - if (shard != null) { validateObject(shard, 'options.shard'); // Avoid re-evaluating the shard object in case it's a getter @@ -756,11 +817,19 @@ function run(options = kEmptyObject) { functionCoverage: functionCoverage, cwd, globalSetupPath, + randomize, + randomSeed, }; + const root = createTestTree(rootTestOptions, globalOptions); let testFiles = files ?? createTestFileList(globPatterns, cwd); const { isTestRunner } = globalOptions; + if (randomize) { + testFiles = shuffleArrayWithSeed(testFiles, randomSeed); + root.diagnostic(`Randomized test order seed: ${randomSeed}`); + } + if (shard) { testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1); } @@ -786,6 +855,8 @@ function run(options = kEmptyObject) { execArgv, rerunFailuresFilePath, env, + randomize, + randomSeed, }; if (isolation === 'process') { diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index beeb49c1763473..3f4c7b4c4cca99 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -10,6 +10,7 @@ const { ArrayPrototypeUnshiftApply, Error, FunctionPrototype, + MathFloor, MathMax, Number, NumberPrototypeToFixed, @@ -47,6 +48,7 @@ const { MockTracker } = require('internal/test_runner/mock/mock'); const { TestsStream } = require('internal/test_runner/tests_stream'); const { createDeferredCallback, + createSeededGenerator, countCompletedTest, isTestFailureError, reporterScope, @@ -648,11 +650,16 @@ class Test extends AsyncResource { this.message = typeof skip === 'string' ? skip : typeof todo === 'string' ? todo : null; this.activeSubtests = 0; + this.subtestQueueRandom = this.config.randomize ? + createSeededGenerator(this.config.randomSeed) : + null; this.pendingSubtests = []; this.readySubtests = new SafeMap(); this.unfinishedSubtests = new SafeSet(); this.subtestsPromise = null; this.subtests = []; + this.nextReportOrder = 1; + this.reportOrder = 0; this.waitingOn = 0; this.finished = false; this.hooks = { @@ -776,13 +783,37 @@ class Test extends AsyncResource { ArrayPrototypePush(this.pendingSubtests, deferred); } + /** + * Ensure each subtest has a contiguous, per-parent reporting order. + * This is assigned at dequeue time for randomized runs, but tests that are + * cancelled before dequeue still need an order to be reported. + * @param {Test} subtest + * @returns {void} + */ + assignReportOrder(subtest) { + if (subtest.reportOrder === 0) { + subtest.reportOrder = this.nextReportOrder++; + } + } + + dequeuePendingSubtest() { + if (!this.subtestQueueRandom || this.pendingSubtests.length < 2) { + return ArrayPrototypeShift(this.pendingSubtests); + } + + // Pick a uniformly random pending sibling when randomization is enabled. + const index = MathFloor(this.subtestQueueRandom() * this.pendingSubtests.length); + return ArrayPrototypeSplice(this.pendingSubtests, index, 1)[0]; + } + /** * @returns {Promise} */ async processPendingSubtests() { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { - const deferred = ArrayPrototypeShift(this.pendingSubtests); + const deferred = this.dequeuePendingSubtest(); const test = deferred.test; + this.assignReportOrder(test); test.reporter.dequeue(test.nesting, test.loc, test.name, this.reportedType); await test.run(); deferred.resolve(); @@ -794,7 +825,8 @@ class Test extends AsyncResource { * @returns {void} */ addReadySubtest(subtest) { - this.readySubtests.set(subtest.childNumber, subtest); + this.assignReportOrder(subtest); + this.readySubtests.set(subtest.reportOrder, subtest); if (this.unfinishedSubtests.delete(subtest) && this.unfinishedSubtests.size === 0) { @@ -1011,6 +1043,7 @@ class Test extends AsyncResource { return deferred.promise; } + this.parent.assignReportOrder(this); this.reporter.dequeue(this.nesting, this.loc, this.name, this.reportedType); return this.run(); } @@ -1315,7 +1348,7 @@ class Test extends AsyncResource { isClearToSend() { return this.parent === null || ( - this.parent.waitingOn === this.childNumber && this.parent.isClearToSend() + this.parent.waitingOn === this.reportOrder && this.parent.isClearToSend() ); } diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 5b53342933cdcb..246eb29a4411b3 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -7,11 +7,14 @@ const { ArrayPrototypePop, ArrayPrototypePush, ArrayPrototypeReduce, + ArrayPrototypeSlice, ArrayPrototypeSome, JSONParse, MathFloor, + MathImul, MathMax, MathMin, + MathRandom, NumberParseInt, NumberPrototypeToFixed, ObjectGetOwnPropertyDescriptor, @@ -45,6 +48,7 @@ const { compose } = require('stream'); const { validateInteger, validateFunction, + validateUint32, } = require('internal/validators'); const { validatePath } = require('internal/fs/utils'); const { kEmptyObject } = require('internal/util'); @@ -58,6 +62,7 @@ const coverageColors = { const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; const kRegExpPattern = /^\/(.*)\/([a-z]*)$/; +const kMaxRandomSeed = 0xFFFF_FFFF; const kPatterns = ['test', 'test/**/*', 'test-*', '*[._-]test']; const kFileExtensions = ['js', 'mjs', 'cjs']; @@ -133,6 +138,54 @@ const kBuiltinReporters = new SafeMap([ const kDefaultReporter = 'spec'; const kDefaultDestination = 'stdout'; +/** + * Create a random uint32 seed. + * @returns {number} + */ +function createRandomSeed() { + return MathFloor(MathRandom() * (kMaxRandomSeed + 1)); +} + +/** + * Create a Mulberry32 pseudo-random number generator from a uint32 seed. + * @param {number} seed + * @returns {() => number} + */ +function createSeededGenerator(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6D2B79F5) | 0; + let value = MathImul(state ^ state >>> 15, 1 | state); + value ^= value + MathImul(value ^ value >>> 7, 61 | value); + return ((value ^ value >>> 14) >>> 0) / 4_294_967_296; + }; +} + +/** + * Return a deterministically shuffled copy of an array. + * @template T + * @param {T[]} values + * @param {number} seed + * @returns {T[]} + */ +function shuffleArrayWithSeed(values, seed) { + if (values.length < 2) { + return values; + } + + const randomized = ArrayPrototypeSlice(values); + const random = createSeededGenerator(seed); + + for (let i = randomized.length - 1; i > 0; i--) { + const j = MathFloor(random() * (i + 1)); + const tmp = randomized[i]; + randomized[i] = randomized[j]; + randomized[j] = tmp; + } + + return randomized; +} + function tryBuiltinReporter(name) { const builtinPath = kBuiltinReporters.get(name); @@ -217,6 +270,10 @@ function parseCommandLine() { const updateSnapshots = getOptionValue('--test-update-snapshots'); const watch = getOptionValue('--watch'); const timeout = getOptionValue('--test-timeout') || Infinity; + let randomize = getOptionValue('--test-randomize'); + const hasRandomSeedOption = getOptionValue('[has_test_random_seed]'); + const randomSeedOption = getOptionValue('--test-random-seed'); + let randomSeed; const rerunFailuresFilePath = getOptionValue('--test-rerun-failures'); const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child'; const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8'; @@ -328,6 +385,12 @@ function parseCommandLine() { validatePath(rerunFailuresFilePath, '--test-rerun-failures'); } + if (hasRandomSeedOption) { + validateUint32(randomSeedOption, '--test-random-seed'); + randomSeed = randomSeedOption; + randomize = true; + } + const setup = reporterScope.bind(async (rootReporter) => { const reportersMap = await getReportersMap(reporters, destinations); for (let i = 0; i < reportersMap.length; i++) { @@ -362,6 +425,8 @@ function parseCommandLine() { timeout, updateSnapshots, watch, + randomize, + randomSeed, rerunFailuresFilePath, }; @@ -649,10 +714,14 @@ async function setupGlobalSetupTeardownFunctions(globalSetupPath, cwd) { module.exports = { convertStringToRegExp, countCompletedTest, + createRandomSeed, + createSeededGenerator, createDeferredCallback, isTestFailureError, kDefaultPattern, + kMaxRandomSeed, parseCommandLine, + shuffleArrayWithSeed, reporterScope, shouldColorizeTestFiles, getCoverageReport, diff --git a/src/node_options.cc b/src/node_options.cc index 22fe0f61da8c69..a27b0608895a1e 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -987,6 +987,20 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { &EnvironmentOptions::test_global_setup_path, kAllowedInEnvvar, OptionNamespaces::kTestRunnerNamespace); + AddOption("--test-randomize", + "run tests in a random order", + &EnvironmentOptions::test_randomize, + kAllowedInEnvvar, + false, + OptionNamespaces::kTestRunnerNamespace); + AddOption( + "[has_test_random_seed]", "", &EnvironmentOptions::has_test_random_seed); + AddOption("--test-random-seed", + "seed used to randomize test execution order", + &EnvironmentOptions::test_random_seed, + kAllowedInEnvvar, + OptionNamespaces::kTestRunnerNamespace); + Implies("--test-random-seed", "[has_test_random_seed]"); AddOption("--test-rerun-failures", "specifies the path to the rerun state file", &EnvironmentOptions::test_rerun_failures_path, diff --git a/src/node_options.h b/src/node_options.h index 21f43946ea8a1e..80fa8d4d540a99 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -204,6 +204,9 @@ class EnvironmentOptions : public Options { std::string test_rerun_failures_path; std::vector test_reporter_destination; std::string test_global_setup_path; + bool test_randomize = false; + bool has_test_random_seed = false; + uint64_t test_random_seed = 0; bool test_only = false; bool test_udp_no_try_send = false; std::string test_isolation = "process"; diff --git a/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.js b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.js new file mode 100644 index 00000000000000..21f582756f6f26 --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.js @@ -0,0 +1,17 @@ +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn( + process.execPath, + [ + '--no-warnings', + '--test-reporter', 'spec', + '--test-random-seed=1', + '--test-isolation=none', + '--test', + fixtures.path('test-runner/randomize/internal-order-nested-scenarios.cjs'), + ], + { stdio: 'inherit' }, +); diff --git a/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.snapshot b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.snapshot new file mode 100644 index 00000000000000..1ab24b0e6c3132 --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_none.snapshot @@ -0,0 +1,213 @@ +▶ nested-scenarios + ▶ scenario describe-it + ▶ describe-it block + ▶ describe-it 1 + ▶ describe-it 1-1 + ✔ describe-it 1-1-1 (*ms) + ✔ describe-it 1-1-3 (*ms) + ✔ describe-it 1-1-2 (*ms) + ✔ describe-it 1-1 (*ms) + ▶ describe-it 1-3 + ✔ describe-it 1-3-1 (*ms) + ✔ describe-it 1-3-3 (*ms) + ✔ describe-it 1-3-2 (*ms) + ✔ describe-it 1-3 (*ms) + ▶ describe-it 1-2 + ✔ describe-it 1-2-1 (*ms) + ✔ describe-it 1-2-3 (*ms) + ✔ describe-it 1-2-2 (*ms) + ✔ describe-it 1-2 (*ms) + ✔ describe-it 1 (*ms) + ▶ describe-it 3 + ▶ describe-it 3-1 + ✔ describe-it 3-1-1 (*ms) + ✔ describe-it 3-1-3 (*ms) + ✔ describe-it 3-1-2 (*ms) + ✔ describe-it 3-1 (*ms) + ▶ describe-it 3-3 + ✔ describe-it 3-3-1 (*ms) + ✔ describe-it 3-3-3 (*ms) + ✔ describe-it 3-3-2 (*ms) + ✔ describe-it 3-3 (*ms) + ▶ describe-it 3-2 + ✔ describe-it 3-2-1 (*ms) + ✔ describe-it 3-2-3 (*ms) + ✔ describe-it 3-2-2 (*ms) + ✔ describe-it 3-2 (*ms) + ✔ describe-it 3 (*ms) + ▶ describe-it 2 + ▶ describe-it 2-1 + ✔ describe-it 2-1-1 (*ms) + ✔ describe-it 2-1-3 (*ms) + ✔ describe-it 2-1-2 (*ms) + ✔ describe-it 2-1 (*ms) + ▶ describe-it 2-3 + ✔ describe-it 2-3-1 (*ms) + ✔ describe-it 2-3-3 (*ms) + ✔ describe-it 2-3-2 (*ms) + ✔ describe-it 2-3 (*ms) + ▶ describe-it 2-2 + ✔ describe-it 2-2-1 (*ms) + ✔ describe-it 2-2-3 (*ms) + ✔ describe-it 2-2-2 (*ms) + ✔ describe-it 2-2 (*ms) + ✔ describe-it 2 (*ms) + ✔ describe-it block (*ms) + ✔ scenario describe-it (*ms) + ▶ scenario static-no-await + ▶ static-no-await block + ▶ static-no-await 1 + ▶ static-no-await 1-1 + ✔ static-no-await 1-1-1 (*ms) + ✔ static-no-await 1-1-3 (*ms) + ✔ static-no-await 1-1-2 (*ms) + ✔ static-no-await 1-1 (*ms) + ▶ static-no-await 1-3 + ✔ static-no-await 1-3-1 (*ms) + ✔ static-no-await 1-3-3 (*ms) + ✔ static-no-await 1-3-2 (*ms) + ✔ static-no-await 1-3 (*ms) + ▶ static-no-await 1-2 + ✔ static-no-await 1-2-1 (*ms) + ✔ static-no-await 1-2-3 (*ms) + ✔ static-no-await 1-2-2 (*ms) + ✔ static-no-await 1-2 (*ms) + ✔ static-no-await 1 (*ms) + ▶ static-no-await 3 + ▶ static-no-await 3-1 + ✔ static-no-await 3-1-1 (*ms) + ✔ static-no-await 3-1-3 (*ms) + ✔ static-no-await 3-1-2 (*ms) + ✔ static-no-await 3-1 (*ms) + ▶ static-no-await 3-3 + ✔ static-no-await 3-3-1 (*ms) + ✔ static-no-await 3-3-3 (*ms) + ✔ static-no-await 3-3-2 (*ms) + ✔ static-no-await 3-3 (*ms) + ▶ static-no-await 3-2 + ✔ static-no-await 3-2-1 (*ms) + ✔ static-no-await 3-2-3 (*ms) + ✔ static-no-await 3-2-2 (*ms) + ✔ static-no-await 3-2 (*ms) + ✔ static-no-await 3 (*ms) + ▶ static-no-await 2 + ▶ static-no-await 2-1 + ✔ static-no-await 2-1-1 (*ms) + ✔ static-no-await 2-1-3 (*ms) + ✔ static-no-await 2-1-2 (*ms) + ✔ static-no-await 2-1 (*ms) + ▶ static-no-await 2-3 + ✔ static-no-await 2-3-1 (*ms) + ✔ static-no-await 2-3-3 (*ms) + ✔ static-no-await 2-3-2 (*ms) + ✔ static-no-await 2-3 (*ms) + ▶ static-no-await 2-2 + ✔ static-no-await 2-2-1 (*ms) + ✔ static-no-await 2-2-3 (*ms) + ✔ static-no-await 2-2-2 (*ms) + ✔ static-no-await 2-2 (*ms) + ✔ static-no-await 2 (*ms) + ✔ static-no-await block (*ms) + ✔ scenario static-no-await (*ms) + ▶ scenario static-await + ▶ static-await block + ▶ static-await 1 + ▶ static-await 1-1 + ✔ static-await 1-1-1 (*ms) + ✔ static-await 1-1-2 (*ms) + ✔ static-await 1-1-3 (*ms) + ✔ static-await 1-1 (*ms) + ▶ static-await 1-2 + ✔ static-await 1-2-1 (*ms) + ✔ static-await 1-2-2 (*ms) + ✔ static-await 1-2-3 (*ms) + ✔ static-await 1-2 (*ms) + ▶ static-await 1-3 + ✔ static-await 1-3-1 (*ms) + ✔ static-await 1-3-2 (*ms) + ✔ static-await 1-3-3 (*ms) + ✔ static-await 1-3 (*ms) + ✔ static-await 1 (*ms) + ▶ static-await 2 + ▶ static-await 2-1 + ✔ static-await 2-1-1 (*ms) + ✔ static-await 2-1-2 (*ms) + ✔ static-await 2-1-3 (*ms) + ✔ static-await 2-1 (*ms) + ▶ static-await 2-2 + ✔ static-await 2-2-1 (*ms) + ✔ static-await 2-2-2 (*ms) + ✔ static-await 2-2-3 (*ms) + ✔ static-await 2-2 (*ms) + ▶ static-await 2-3 + ✔ static-await 2-3-1 (*ms) + ✔ static-await 2-3-2 (*ms) + ✔ static-await 2-3-3 (*ms) + ✔ static-await 2-3 (*ms) + ✔ static-await 2 (*ms) + ▶ static-await 3 + ▶ static-await 3-1 + ✔ static-await 3-1-1 (*ms) + ✔ static-await 3-1-2 (*ms) + ✔ static-await 3-1-3 (*ms) + ✔ static-await 3-1 (*ms) + ▶ static-await 3-2 + ✔ static-await 3-2-1 (*ms) + ✔ static-await 3-2-2 (*ms) + ✔ static-await 3-2-3 (*ms) + ✔ static-await 3-2 (*ms) + ▶ static-await 3-3 + ✔ static-await 3-3-1 (*ms) + ✔ static-await 3-3-2 (*ms) + ✔ static-await 3-3-3 (*ms) + ✔ static-await 3-3 (*ms) + ✔ static-await 3 (*ms) + ✔ static-await block (*ms) + ✔ scenario static-await (*ms) + ▶ scenario mixed-await-normal + ▶ mixed-await-normal block + ▶ mixed-await-normal 1 + ✔ mixed-await-normal 1-1-sync-before (*ms) + ✔ mixed-await-normal 1-1-awaited (*ms) + ✔ mixed-await-normal 1-1-sync-after (*ms) + ✔ mixed-await-normal 1-2-awaited (*ms) + ✔ mixed-await-normal 1-2-sync-before (*ms) + ✔ mixed-await-normal 1-2-sync-after (*ms) + ✔ mixed-await-normal 1-3-awaited (*ms) + ✔ mixed-await-normal 1-3-sync-before (*ms) + ✔ mixed-await-normal 1-3-sync-after (*ms) + ✔ mixed-await-normal 1 (*ms) + ▶ mixed-await-normal 3 + ✔ mixed-await-normal 3-1-sync-before (*ms) + ✔ mixed-await-normal 3-1-awaited (*ms) + ✔ mixed-await-normal 3-1-sync-after (*ms) + ✔ mixed-await-normal 3-2-awaited (*ms) + ✔ mixed-await-normal 3-2-sync-before (*ms) + ✔ mixed-await-normal 3-2-sync-after (*ms) + ✔ mixed-await-normal 3-3-awaited (*ms) + ✔ mixed-await-normal 3-3-sync-before (*ms) + ✔ mixed-await-normal 3-3-sync-after (*ms) + ✔ mixed-await-normal 3 (*ms) + ▶ mixed-await-normal 2 + ✔ mixed-await-normal 2-1-sync-before (*ms) + ✔ mixed-await-normal 2-1-awaited (*ms) + ✔ mixed-await-normal 2-1-sync-after (*ms) + ✔ mixed-await-normal 2-2-awaited (*ms) + ✔ mixed-await-normal 2-2-sync-before (*ms) + ✔ mixed-await-normal 2-2-sync-after (*ms) + ✔ mixed-await-normal 2-3-awaited (*ms) + ✔ mixed-await-normal 2-3-sync-before (*ms) + ✔ mixed-await-normal 2-3-sync-after (*ms) + ✔ mixed-await-normal 2 (*ms) + ✔ mixed-await-normal block (*ms) + ✔ scenario mixed-await-normal (*ms) +✔ nested-scenarios (*ms) +ℹ Randomized test order seed: 1 +ℹ tests 144 +ℹ suites 12 +ℹ pass 144 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms * diff --git a/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.js b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.js new file mode 100644 index 00000000000000..5b0d40d01fa21f --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.js @@ -0,0 +1,17 @@ +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn( + process.execPath, + [ + '--no-warnings', + '--test-reporter', 'spec', + '--test-random-seed=1', + '--test-isolation=process', + '--test', + fixtures.path('test-runner/randomize/internal-order-nested-scenarios.cjs'), + ], + { stdio: 'inherit' }, +); diff --git a/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.snapshot b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.snapshot new file mode 100644 index 00000000000000..1ab24b0e6c3132 --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_nested_scenarios_output_cli_process.snapshot @@ -0,0 +1,213 @@ +▶ nested-scenarios + ▶ scenario describe-it + ▶ describe-it block + ▶ describe-it 1 + ▶ describe-it 1-1 + ✔ describe-it 1-1-1 (*ms) + ✔ describe-it 1-1-3 (*ms) + ✔ describe-it 1-1-2 (*ms) + ✔ describe-it 1-1 (*ms) + ▶ describe-it 1-3 + ✔ describe-it 1-3-1 (*ms) + ✔ describe-it 1-3-3 (*ms) + ✔ describe-it 1-3-2 (*ms) + ✔ describe-it 1-3 (*ms) + ▶ describe-it 1-2 + ✔ describe-it 1-2-1 (*ms) + ✔ describe-it 1-2-3 (*ms) + ✔ describe-it 1-2-2 (*ms) + ✔ describe-it 1-2 (*ms) + ✔ describe-it 1 (*ms) + ▶ describe-it 3 + ▶ describe-it 3-1 + ✔ describe-it 3-1-1 (*ms) + ✔ describe-it 3-1-3 (*ms) + ✔ describe-it 3-1-2 (*ms) + ✔ describe-it 3-1 (*ms) + ▶ describe-it 3-3 + ✔ describe-it 3-3-1 (*ms) + ✔ describe-it 3-3-3 (*ms) + ✔ describe-it 3-3-2 (*ms) + ✔ describe-it 3-3 (*ms) + ▶ describe-it 3-2 + ✔ describe-it 3-2-1 (*ms) + ✔ describe-it 3-2-3 (*ms) + ✔ describe-it 3-2-2 (*ms) + ✔ describe-it 3-2 (*ms) + ✔ describe-it 3 (*ms) + ▶ describe-it 2 + ▶ describe-it 2-1 + ✔ describe-it 2-1-1 (*ms) + ✔ describe-it 2-1-3 (*ms) + ✔ describe-it 2-1-2 (*ms) + ✔ describe-it 2-1 (*ms) + ▶ describe-it 2-3 + ✔ describe-it 2-3-1 (*ms) + ✔ describe-it 2-3-3 (*ms) + ✔ describe-it 2-3-2 (*ms) + ✔ describe-it 2-3 (*ms) + ▶ describe-it 2-2 + ✔ describe-it 2-2-1 (*ms) + ✔ describe-it 2-2-3 (*ms) + ✔ describe-it 2-2-2 (*ms) + ✔ describe-it 2-2 (*ms) + ✔ describe-it 2 (*ms) + ✔ describe-it block (*ms) + ✔ scenario describe-it (*ms) + ▶ scenario static-no-await + ▶ static-no-await block + ▶ static-no-await 1 + ▶ static-no-await 1-1 + ✔ static-no-await 1-1-1 (*ms) + ✔ static-no-await 1-1-3 (*ms) + ✔ static-no-await 1-1-2 (*ms) + ✔ static-no-await 1-1 (*ms) + ▶ static-no-await 1-3 + ✔ static-no-await 1-3-1 (*ms) + ✔ static-no-await 1-3-3 (*ms) + ✔ static-no-await 1-3-2 (*ms) + ✔ static-no-await 1-3 (*ms) + ▶ static-no-await 1-2 + ✔ static-no-await 1-2-1 (*ms) + ✔ static-no-await 1-2-3 (*ms) + ✔ static-no-await 1-2-2 (*ms) + ✔ static-no-await 1-2 (*ms) + ✔ static-no-await 1 (*ms) + ▶ static-no-await 3 + ▶ static-no-await 3-1 + ✔ static-no-await 3-1-1 (*ms) + ✔ static-no-await 3-1-3 (*ms) + ✔ static-no-await 3-1-2 (*ms) + ✔ static-no-await 3-1 (*ms) + ▶ static-no-await 3-3 + ✔ static-no-await 3-3-1 (*ms) + ✔ static-no-await 3-3-3 (*ms) + ✔ static-no-await 3-3-2 (*ms) + ✔ static-no-await 3-3 (*ms) + ▶ static-no-await 3-2 + ✔ static-no-await 3-2-1 (*ms) + ✔ static-no-await 3-2-3 (*ms) + ✔ static-no-await 3-2-2 (*ms) + ✔ static-no-await 3-2 (*ms) + ✔ static-no-await 3 (*ms) + ▶ static-no-await 2 + ▶ static-no-await 2-1 + ✔ static-no-await 2-1-1 (*ms) + ✔ static-no-await 2-1-3 (*ms) + ✔ static-no-await 2-1-2 (*ms) + ✔ static-no-await 2-1 (*ms) + ▶ static-no-await 2-3 + ✔ static-no-await 2-3-1 (*ms) + ✔ static-no-await 2-3-3 (*ms) + ✔ static-no-await 2-3-2 (*ms) + ✔ static-no-await 2-3 (*ms) + ▶ static-no-await 2-2 + ✔ static-no-await 2-2-1 (*ms) + ✔ static-no-await 2-2-3 (*ms) + ✔ static-no-await 2-2-2 (*ms) + ✔ static-no-await 2-2 (*ms) + ✔ static-no-await 2 (*ms) + ✔ static-no-await block (*ms) + ✔ scenario static-no-await (*ms) + ▶ scenario static-await + ▶ static-await block + ▶ static-await 1 + ▶ static-await 1-1 + ✔ static-await 1-1-1 (*ms) + ✔ static-await 1-1-2 (*ms) + ✔ static-await 1-1-3 (*ms) + ✔ static-await 1-1 (*ms) + ▶ static-await 1-2 + ✔ static-await 1-2-1 (*ms) + ✔ static-await 1-2-2 (*ms) + ✔ static-await 1-2-3 (*ms) + ✔ static-await 1-2 (*ms) + ▶ static-await 1-3 + ✔ static-await 1-3-1 (*ms) + ✔ static-await 1-3-2 (*ms) + ✔ static-await 1-3-3 (*ms) + ✔ static-await 1-3 (*ms) + ✔ static-await 1 (*ms) + ▶ static-await 2 + ▶ static-await 2-1 + ✔ static-await 2-1-1 (*ms) + ✔ static-await 2-1-2 (*ms) + ✔ static-await 2-1-3 (*ms) + ✔ static-await 2-1 (*ms) + ▶ static-await 2-2 + ✔ static-await 2-2-1 (*ms) + ✔ static-await 2-2-2 (*ms) + ✔ static-await 2-2-3 (*ms) + ✔ static-await 2-2 (*ms) + ▶ static-await 2-3 + ✔ static-await 2-3-1 (*ms) + ✔ static-await 2-3-2 (*ms) + ✔ static-await 2-3-3 (*ms) + ✔ static-await 2-3 (*ms) + ✔ static-await 2 (*ms) + ▶ static-await 3 + ▶ static-await 3-1 + ✔ static-await 3-1-1 (*ms) + ✔ static-await 3-1-2 (*ms) + ✔ static-await 3-1-3 (*ms) + ✔ static-await 3-1 (*ms) + ▶ static-await 3-2 + ✔ static-await 3-2-1 (*ms) + ✔ static-await 3-2-2 (*ms) + ✔ static-await 3-2-3 (*ms) + ✔ static-await 3-2 (*ms) + ▶ static-await 3-3 + ✔ static-await 3-3-1 (*ms) + ✔ static-await 3-3-2 (*ms) + ✔ static-await 3-3-3 (*ms) + ✔ static-await 3-3 (*ms) + ✔ static-await 3 (*ms) + ✔ static-await block (*ms) + ✔ scenario static-await (*ms) + ▶ scenario mixed-await-normal + ▶ mixed-await-normal block + ▶ mixed-await-normal 1 + ✔ mixed-await-normal 1-1-sync-before (*ms) + ✔ mixed-await-normal 1-1-awaited (*ms) + ✔ mixed-await-normal 1-1-sync-after (*ms) + ✔ mixed-await-normal 1-2-awaited (*ms) + ✔ mixed-await-normal 1-2-sync-before (*ms) + ✔ mixed-await-normal 1-2-sync-after (*ms) + ✔ mixed-await-normal 1-3-awaited (*ms) + ✔ mixed-await-normal 1-3-sync-before (*ms) + ✔ mixed-await-normal 1-3-sync-after (*ms) + ✔ mixed-await-normal 1 (*ms) + ▶ mixed-await-normal 3 + ✔ mixed-await-normal 3-1-sync-before (*ms) + ✔ mixed-await-normal 3-1-awaited (*ms) + ✔ mixed-await-normal 3-1-sync-after (*ms) + ✔ mixed-await-normal 3-2-awaited (*ms) + ✔ mixed-await-normal 3-2-sync-before (*ms) + ✔ mixed-await-normal 3-2-sync-after (*ms) + ✔ mixed-await-normal 3-3-awaited (*ms) + ✔ mixed-await-normal 3-3-sync-before (*ms) + ✔ mixed-await-normal 3-3-sync-after (*ms) + ✔ mixed-await-normal 3 (*ms) + ▶ mixed-await-normal 2 + ✔ mixed-await-normal 2-1-sync-before (*ms) + ✔ mixed-await-normal 2-1-awaited (*ms) + ✔ mixed-await-normal 2-1-sync-after (*ms) + ✔ mixed-await-normal 2-2-awaited (*ms) + ✔ mixed-await-normal 2-2-sync-before (*ms) + ✔ mixed-await-normal 2-2-sync-after (*ms) + ✔ mixed-await-normal 2-3-awaited (*ms) + ✔ mixed-await-normal 2-3-sync-before (*ms) + ✔ mixed-await-normal 2-3-sync-after (*ms) + ✔ mixed-await-normal 2 (*ms) + ✔ mixed-await-normal block (*ms) + ✔ scenario mixed-await-normal (*ms) +✔ nested-scenarios (*ms) +ℹ Randomized test order seed: 1 +ℹ tests 144 +ℹ suites 12 +ℹ pass 144 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms * diff --git a/test/fixtures/test-runner/output/randomize_output_cli.js b/test/fixtures/test-runner/output/randomize_output_cli.js new file mode 100644 index 00000000000000..0dda8ed79caa71 --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_output_cli.js @@ -0,0 +1,16 @@ +'use strict'; +require('../../../common'); +const fixtures = require('../../../common/fixtures'); +const spawn = require('node:child_process').spawn; + +spawn( + process.execPath, + [ + '--no-warnings', + '--test-reporter', 'spec', + '--test-random-seed=12345', + '--test', + fixtures.path('test-runner/shards/*.cjs'), + ], + { stdio: 'inherit' }, +); diff --git a/test/fixtures/test-runner/output/randomize_output_cli.snapshot b/test/fixtures/test-runner/output/randomize_output_cli.snapshot new file mode 100644 index 00000000000000..ae613bc67799bb --- /dev/null +++ b/test/fixtures/test-runner/output/randomize_output_cli.snapshot @@ -0,0 +1,19 @@ +✔ g.cjs this should pass (*ms) +✔ e.cjs this should pass (*ms) +✔ i.cjs this should pass (*ms) +✔ a.cjs this should pass (*ms) +✔ b.cjs this should pass (*ms) +✔ h.cjs this should pass (*ms) +✔ f.cjs this should pass (*ms) +✔ d.cjs this should pass (*ms) +✔ c.cjs this should pass (*ms) +✔ j.cjs this should pass (*ms) +ℹ Randomized test order seed: 12345 +ℹ tests 10 +ℹ suites 0 +ℹ pass 10 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms * diff --git a/test/fixtures/test-runner/randomize/internal-order-nested-scenarios.cjs b/test/fixtures/test-runner/randomize/internal-order-nested-scenarios.cjs new file mode 100644 index 00000000000000..de37d44b7ec319 --- /dev/null +++ b/test/fixtures/test-runner/randomize/internal-order-nested-scenarios.cjs @@ -0,0 +1,82 @@ +'use strict'; + +const { describe, it, test } = require('node:test'); +const { setImmediate: setImmediatePromise } = require('node:timers/promises'); + +const levelOne = ['1', '2', '3']; +const levelTwo = ['1', '2', '3']; +const levelThree = ['1', '2', '3']; + +test('nested-scenarios', async (t) => { + await t.test('scenario describe-it', async (scenario) => { + await scenario.test('describe-it block', () => { + for (const outer of levelOne) { + describe(`describe-it ${outer}`, () => { + for (const middle of levelTwo) { + describe(`describe-it ${outer}-${middle}`, () => { + for (const inner of levelThree) { + it(`describe-it ${outer}-${middle}-${inner}`, () => {}); + } + }); + } + }); + } + }); + }); + + await t.test('scenario static-no-await', async (scenario) => { + await scenario.test('static-no-await block', (block) => { + for (const outer of levelOne) { + block.test(`static-no-await ${outer}`, (outerBlock) => { + for (const middle of levelTwo) { + outerBlock.test(`static-no-await ${outer}-${middle}`, (middleBlock) => { + for (const inner of levelThree) { + middleBlock.test(`static-no-await ${outer}-${middle}-${inner}`, () => {}); + } + }); + } + }); + } + }); + }); + + await t.test('scenario static-await', async (scenario) => { + await scenario.test('static-await block', async (block) => { + for (const outer of levelOne) { + await block.test(`static-await ${outer}`, async (outerBlock) => { + await setImmediatePromise(); + + for (const middle of levelTwo) { + await outerBlock.test(`static-await ${outer}-${middle}`, async (middleBlock) => { + await setImmediatePromise(); + + for (const inner of levelThree) { + await middleBlock.test(`static-await ${outer}-${middle}-${inner}`, async () => { + await setImmediatePromise(); + }); + } + }); + } + }); + } + }); + }); + + await t.test('scenario mixed-await-normal', async (scenario) => { + await scenario.test('mixed-await-normal block', (block) => { + for (const outer of levelOne) { + block.test(`mixed-await-normal ${outer}`, async (outerBlock) => { + for (const middle of levelTwo) { + outerBlock.test(`mixed-await-normal ${outer}-${middle}-sync-before`, () => {}); + + await outerBlock.test(`mixed-await-normal ${outer}-${middle}-awaited`, async () => { + await setImmediatePromise(); + }); + + outerBlock.test(`mixed-await-normal ${outer}-${middle}-sync-after`, () => {}); + } + }); + } + }); + }); +}); diff --git a/test/fixtures/test-runner/randomize/internal-order.cjs b/test/fixtures/test-runner/randomize/internal-order.cjs new file mode 100644 index 00000000000000..7b0fb0a5c656bd --- /dev/null +++ b/test/fixtures/test-runner/randomize/internal-order.cjs @@ -0,0 +1,16 @@ +'use strict'; + +const test = require('node:test'); +const { setImmediate: setImmediatePromise } = require('node:timers/promises'); +const executionOrder = []; + +for (const name of ['a', 'b', 'c', 'd', 'e']) { + test(`internal ${name}`, async () => { + executionOrder.push(name); + await setImmediatePromise(); + }); +} + +process.on('exit', () => { + process.stdout.write(`EXECUTION_ORDER:${executionOrder.join(',')}\n`); +}); diff --git a/test/parallel/test-runner-cli-randomize.js b/test/parallel/test-runner-cli-randomize.js new file mode 100644 index 00000000000000..e35b2546714594 --- /dev/null +++ b/test/parallel/test-runner-cli-randomize.js @@ -0,0 +1,228 @@ +'use strict'; + +require('../common'); +const assert = require('node:assert'); +const { spawnSync } = require('node:child_process'); +const { join } = require('node:path'); +const { describe, it } = require('node:test'); +const fixtures = require('../common/fixtures'); + +const testFixtures = fixtures.path('test-runner'); +const internalOrderFixture = fixtures.path('test-runner', 'randomize', 'internal-order.cjs'); +const rerunStateFile = fixtures.path('test-runner', 'rerun-state.json'); +const kShardFiles = ['a.cjs', 'b.cjs', 'c.cjs', 'd.cjs', 'e.cjs', 'f.cjs', 'g.cjs', 'h.cjs', 'i.cjs', 'j.cjs']; +const kInternalTests = ['a', 'b', 'c', 'd', 'e']; + +function getShardOrder(stdout) { + return Array.from(stdout.matchAll(/ok \d+ - ([a-j]\.cjs) this should pass/g), ({ 1: name }) => name); +} + +function getInternalExecutionOrder(stdout) { + const match = stdout.match(/EXECUTION_ORDER:([a-e](?:,[a-e])*)/); + assert(match, `Missing EXECUTION_ORDER marker in output: ${stdout}`); + return match[1].split(','); +} + +describe('test runner randomize flags via command line', () => { + it('should be deterministic with --test-random-seed', () => { + const args = [ + '--test', + '--test-reporter=tap', + '--test-concurrency=1', + '--test-randomize', + '--test-random-seed=12345', + join(testFixtures, 'shards/*.cjs'), + ]; + const first = spawnSync(process.execPath, args); + const second = spawnSync(process.execPath, args); + + assert.strictEqual(first.stderr.toString(), ''); + assert.strictEqual(second.stderr.toString(), ''); + assert.strictEqual(first.status, 0); + assert.strictEqual(second.status, 0); + + const firstOrder = getShardOrder(first.stdout.toString()); + const secondOrder = getShardOrder(second.stdout.toString()); + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), kShardFiles); + }); + + it('should use different orders for different seeds', () => { + const first = spawnSync(process.execPath, [ + '--test', + '--test-reporter=tap', + '--test-concurrency=1', + '--test-randomize', + '--test-random-seed=11111', + join(testFixtures, 'shards/*.cjs'), + ]); + const second = spawnSync(process.execPath, [ + '--test', + '--test-reporter=tap', + '--test-concurrency=1', + '--test-randomize', + '--test-random-seed=22222', + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(first.stderr.toString(), ''); + assert.strictEqual(second.stderr.toString(), ''); + assert.strictEqual(first.status, 0); + assert.strictEqual(second.status, 0); + + const firstOrder = getShardOrder(first.stdout.toString()); + const secondOrder = getShardOrder(second.stdout.toString()); + assert.notDeepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), kShardFiles); + assert.deepStrictEqual([...secondOrder].sort(), kShardFiles); + }); + + it('should randomize deterministically with --test-random-seed alone', () => { + const args = [ + '--test', + '--test-reporter=tap', + '--test-concurrency=1', + '--test-random-seed=24680', + join(testFixtures, 'shards/*.cjs'), + ]; + const first = spawnSync(process.execPath, args); + const second = spawnSync(process.execPath, args); + + assert.strictEqual(first.stderr.toString(), ''); + assert.strictEqual(second.stderr.toString(), ''); + assert.strictEqual(first.status, 0); + assert.strictEqual(second.status, 0); + + const firstOrder = getShardOrder(first.stdout.toString()); + const secondOrder = getShardOrder(second.stdout.toString()); + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), kShardFiles); + assert.match(first.stdout.toString(), /# Randomized test order seed: 24680/); + }); + + it('should print the randomization seed when --test-randomize is used', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--test-reporter=tap', + '--test-concurrency=1', + '--test-randomize', + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.status, 0); + assert.match(child.stdout.toString(), /# Randomized test order seed: \d+/); + }); + + it('should randomize internal test order deterministically with --test-random-seed', () => { + const args = [ + '--test', + '--test-reporter=tap', + '--test-random-seed=12345', + internalOrderFixture, + ]; + const first = spawnSync(process.execPath, args); + const second = spawnSync(process.execPath, args); + + assert.strictEqual(first.stderr.toString(), ''); + assert.strictEqual(second.stderr.toString(), ''); + assert.strictEqual(first.status, 0); + assert.strictEqual(second.status, 0); + + const firstOrder = getInternalExecutionOrder(first.stdout.toString()); + const secondOrder = getInternalExecutionOrder(second.stdout.toString()); + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), kInternalTests); + }); + + it('should randomize internal test order differently across seeds', () => { + const orders = []; + for (const seed of [11111, 22222, 33333, 44444]) { + const child = spawnSync(process.execPath, [ + '--test', + '--test-reporter=tap', + `--test-random-seed=${seed}`, + internalOrderFixture, + ]); + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.status, 0); + + const order = getInternalExecutionOrder(child.stdout.toString()); + assert.deepStrictEqual([...order].sort(), kInternalTests); + orders.push(order.join(',')); + } + + assert.notStrictEqual(new Set(orders).size, 1); + }); + + it('should reject --test-randomize with --watch', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--watch', + '--test-randomize', + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /The property 'options\.randomize' is not supported with watch mode\./); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + }); + + it('should reject --test-random-seed with --watch', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--watch', + '--test-random-seed=12345', + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /The property 'options\.randomSeed' is not supported with watch mode\./); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + }); + + it('should reject --test-randomize with --test-rerun-failures', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--test-randomize', + '--test-rerun-failures', + rerunStateFile, + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /The property 'options\.randomize' is not supported with rerun failures mode\./); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + }); + + it('should reject --test-random-seed with --test-rerun-failures', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--test-random-seed=12345', + '--test-rerun-failures', + rerunStateFile, + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /The property 'options\.randomSeed' is not supported with rerun failures mode\./); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + }); + + it('should reject out of range --test-random-seed values', () => { + const child = spawnSync(process.execPath, [ + '--test', + '--test-random-seed=4294967296', + join(testFixtures, 'shards/*.cjs'), + ]); + + assert.strictEqual(child.stdout.toString(), ''); + assert.match(child.stderr.toString(), /The value of "--test-random-seed" is out of range\. It must be >= 0 && <= 4294967295\. Received 4294967296/); + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + }); +}); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index e9bb6c4a260160..36b96e5934d3ee 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -1,6 +1,6 @@ import * as common from '../common/index.mjs'; import * as fixtures from '../common/fixtures.mjs'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { describe, it, run } from 'node:test'; import { dot, spec, tap } from 'node:test/reporters'; import consumers from 'node:stream/consumers'; @@ -8,6 +8,7 @@ import assert from 'node:assert'; import util from 'node:util'; const testFixtures = fixtures.path('test-runner'); +const rerunStateFile = join(testFixtures, 'rerun-state.json'); describe('require(\'node:test\').run', { concurrency: true }, () => { it('should run with no tests', async () => { @@ -343,20 +344,64 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { }); }); + const shardsTestsFixtures = fixtures.path('test-runner', 'shards'); + const internalOrderTestFile = join(testFixtures, 'randomize', 'internal-order.cjs'); + const shardFileNames = [ + 'a.cjs', + 'b.cjs', + 'c.cjs', + 'd.cjs', + 'e.cjs', + 'f.cjs', + 'g.cjs', + 'h.cjs', + 'i.cjs', + 'j.cjs', + ]; + const internalTestNames = ['a', 'b', 'c', 'd', 'e']; + const shardsTestsFiles = shardFileNames.map((file) => join(shardsTestsFixtures, file)); + + async function getExecutedShardOrder(options = {}) { + const stream = run({ + files: shardsTestsFiles, + concurrency: false, + ...options, + }); + const executedTestFiles = []; + + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', (passedTest) => { + if (passedTest.nesting === 0) { + executedTestFiles.push(basename(passedTest.file)); + } + }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream) ; + + return executedTestFiles; + } + + async function getExecutedInternalOrder(options = {}) { + const stream = run({ + files: [internalOrderTestFile], + concurrency: false, + ...options, + }); + const executionOrder = []; + + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', (passedTest) => { + if (passedTest.file === internalOrderTestFile && passedTest.name.startsWith('internal ')) { + executionOrder.push(passedTest.name.slice('internal '.length)); + } + }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream) ; + + return executionOrder; + } + describe('sharding', () => { - const shardsTestsFixtures = fixtures.path('test-runner', 'shards'); - const shardsTestsFiles = [ - 'a.cjs', - 'b.cjs', - 'c.cjs', - 'd.cjs', - 'e.cjs', - 'f.cjs', - 'g.cjs', - 'h.cjs', - 'i.cjs', - 'j.cjs', - ].map((file) => join(shardsTestsFixtures, file)); describe('validation', () => { it('should require shard.total when having shard option', () => { @@ -517,6 +562,73 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { assert.deepStrictEqual(executedTestFiles.sort(), [...shardsTestsFiles].sort()); }); + + }); + + describe('randomization', () => { + it('should randomize file order deterministically when using the same seed', async () => { + const firstOrder = await getExecutedShardOrder({ randomize: true, randomSeed: 12345 }); + const secondOrder = await getExecutedShardOrder({ randomize: true, randomSeed: 12345 }); + + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), [...shardFileNames].sort()); + }); + + it('should randomize file order differently with different seeds', async () => { + const firstOrder = await getExecutedShardOrder({ randomize: true, randomSeed: 11111 }); + const secondOrder = await getExecutedShardOrder({ randomize: true, randomSeed: 22222 }); + + assert.notDeepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), [...shardFileNames].sort()); + assert.deepStrictEqual([...secondOrder].sort(), [...shardFileNames].sort()); + }); + + it('should randomize file order when only randomSeed is provided', async () => { + const firstOrder = await getExecutedShardOrder({ randomSeed: 24680 }); + const secondOrder = await getExecutedShardOrder({ randomSeed: 24680 }); + + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), [...shardFileNames].sort()); + }); + + it('should randomize internal test order deterministically when using the same seed', async () => { + const firstOrder = await getExecutedInternalOrder({ randomSeed: 12345 }); + const secondOrder = await getExecutedInternalOrder({ randomSeed: 12345 }); + + assert.deepStrictEqual(firstOrder, secondOrder); + assert.deepStrictEqual([...firstOrder].sort(), [...internalTestNames].sort()); + }); + + it('should randomize internal test order differently across seeds', async () => { + const orders = []; + for (const seed of [11111, 22222, 33333, 44444]) { + const order = await getExecutedInternalOrder({ randomSeed: seed }); + assert.deepStrictEqual([...order].sort(), [...internalTestNames].sort()); + orders.push(order.join(',')); + } + + assert.notStrictEqual(new Set(orders).size, 1); + }); + + it('should emit the randomization seed as a diagnostic message', async () => { + const stream = run({ + files: shardsTestsFiles, + concurrency: false, + randomize: true, + }); + const diagnostics = []; + + stream.on('test:diagnostic', ({ message }) => { + diagnostics.push(message); + }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream) ; + + assert( + diagnostics.some((message) => /Randomized test order seed: \d+/.test(message)), + `Missing randomization seed diagnostic. Received diagnostics: ${diagnostics.join(', ')}`, + ); + }); }); describe('validation', () => { @@ -540,6 +652,40 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { }); }); + it('should not allow randomize with watch mode', () => { + assert.throws(() => run({ watch: true, randomize: true }), { + code: 'ERR_INVALID_ARG_VALUE', + message: /The property 'options\.randomize' is not supported with watch mode\./, + }); + }); + + it('should not allow randomSeed with watch mode', () => { + assert.throws(() => run({ watch: true, randomSeed: 12345 }), { + code: 'ERR_INVALID_ARG_VALUE', + message: /The property 'options\.randomSeed' is not supported with watch mode\./, + }); + }); + + it('should not allow randomize with rerunFailuresFilePath', () => { + assert.throws(() => run({ randomize: true, rerunFailuresFilePath: rerunStateFile }), { + code: 'ERR_INVALID_ARG_VALUE', + message: /The property 'options\.randomize' is not supported with rerun failures mode\./, + }); + }); + + it('should not allow randomSeed with rerunFailuresFilePath', () => { + assert.throws(() => run({ randomSeed: 12345, rerunFailuresFilePath: rerunStateFile }), { + code: 'ERR_INVALID_ARG_VALUE', + message: /The property 'options\.randomSeed' is not supported with rerun failures mode\./, + }); + }); + + it('should not allow decimal randomSeed values', () => { + assert.throws(() => run({ randomSeed: 1.5 }), { + code: 'ERR_OUT_OF_RANGE', + }); + }); + it('should only allow a string in options.cwd', async () => { [Symbol(), {}, [], () => {}, 0, 1, 0n, 1n, true, false] .forEach((cwd) => assert.throws(() => run({ cwd }), { diff --git a/test/test-runner/test-output-randomize-nested-scenarios-output-cli-none.mjs b/test/test-runner/test-output-randomize-nested-scenarios-output-cli-none.mjs new file mode 100644 index 00000000000000..b585f921a90586 --- /dev/null +++ b/test/test-runner/test-output-randomize-nested-scenarios-output-cli-none.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/randomize_nested_scenarios_output_cli_none.js +// matches test-runner/output/randomize_nested_scenarios_output_cli_none.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, specTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/randomize_nested_scenarios_output_cli_none.js'), + specTransform, +); diff --git a/test/test-runner/test-output-randomize-nested-scenarios-output-cli-process.mjs b/test/test-runner/test-output-randomize-nested-scenarios-output-cli-process.mjs new file mode 100644 index 00000000000000..2a2d6db1187527 --- /dev/null +++ b/test/test-runner/test-output-randomize-nested-scenarios-output-cli-process.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/randomize_nested_scenarios_output_cli_process.js +// matches test-runner/output/randomize_nested_scenarios_output_cli_process.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, specTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/randomize_nested_scenarios_output_cli_process.js'), + specTransform, +); diff --git a/test/test-runner/test-output-randomize-output-cli.mjs b/test/test-runner/test-output-randomize-output-cli.mjs new file mode 100644 index 00000000000000..204117f31a2d21 --- /dev/null +++ b/test/test-runner/test-output-randomize-output-cli.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/randomize_output_cli.js matches +// test-runner/output/randomize_output_cli.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, specTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/randomize_output_cli.js'), + specTransform, +);