diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7f52065..06a06bf44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO ### Added - Cucumber Expressions now support a wider array of parameter types (see [documentation](https://github.com/cucumber/cucumber-expressions#parameter-types)) - Improved styling and usability on report from `html` formatter +- Support for customising work assignment when running in parallel ([#1044](https://github.com/cucumber/cucumber-js/issues/1044) + [#1588](https://github.com/cucumber/cucumber-js/pull/1588)) - Add a new option to `--format-options`: `printAttachments`. See [./docs/cli.md#printing-attachments-details](https://github.com/cucumber/cucumber-js/blob/main/docs/cli.md#printing-attachments-details) for more info. ([#1136](https://github.com/cucumber/cucumber-js/issues/1136) diff --git a/docs/parallel.md b/docs/parallel.md index 889c027d8..a730f1c5e 100644 --- a/docs/parallel.md +++ b/docs/parallel.md @@ -27,3 +27,42 @@ When using parallel mode, the last line of the summary output differentiates bet ### Hooks When using parallel mode, any `BeforeAll` and `AfterAll` hooks you have defined will run _once per worker_. + +### Custom work assignment + +If you would like to prevent specific sets of scenarios from running in parallel you can use `setParallelCanAssign`. + +Example: +```javascript +setParallelCanAssign(function(pickleInQuestion, picklesInProgress) { + // Only one pickle with the word example in the name can run at a time + if (pickleInQuestion.name.includes("example")) { + return picklesInProgress.every(p => !p.name.includes("example")); + } + // No other restrictions + return true; +}) +``` + +For convenience, the following helpers exist to build a `canAssignFn`: + +```javascript +import { setParallelCanAssign } from '@cucumber/cucumber' +import { atMostOnePicklePerTag } from '@cucumber/cucumber/lib/support_code_library_builder/parallel_can_assign_helpers' + +const myTagRule = atMostOnePicklePerTag(["@tag1", "@tag2"]); + +// Only one pickle with @tag1 can run at a time +// AND only one pickle with @tag2 can run at a time +setParallelCanAssign(myTagRule) + +// If you want to join a tag rule with other rules you can compose them like so: +const myCustomRule = function(pickleInQuestion, picklesInProgress) { + // ... +}; + +setParallelCanAssign(function(pickleInQuestion, picklesInProgress) { + return myCustomRule(pickleInQuestion, picklesInProgress) && + myTagRule(pickleInQuestion, picklesInProgress); +}) +``` diff --git a/docs/support_files/api_reference.md b/docs/support_files/api_reference.md index c3fa28d45..dee006a94 100644 --- a/docs/support_files/api_reference.md +++ b/docs/support_files/api_reference.md @@ -164,6 +164,21 @@ When used, the result is wrapped again to ensure it has the same length of the o --- +### setParallelCanAssign(canAssignFn) + +Set the function used to determine if a pickle can be executed based on currently executing pickles. + +The `canAssignFn` function is expected to take 2 arguments: + +- `pickleInQuestion` is the a pickle we are checking if its okay to run +- `picklesInProgress` is an array of pickles currently being executed + +And returns true if the pickle can be executed, false otherwise. + +See examples in our [parallel](../parallel.md) documentation. + +--- + #### `setWorldConstructor(constructor)` Set a custom world constructor, to override the default world constructor: diff --git a/features/parallel_custom_assign.feature b/features/parallel_custom_assign.feature new file mode 100644 index 000000000..1594e6769 --- /dev/null +++ b/features/parallel_custom_assign.feature @@ -0,0 +1,69 @@ +Feature: Running scenarios in parallel with custom assignment + + @spawn + Scenario: Bad parallel assignment helper uses 1 worker + Given a file named "features/step_definitions/cucumber_steps.js" with: + """ + const {Given, setParallelCanAssign} = require('@cucumber/cucumber') + + setParallelCanAssign(() => false) + + Given('slow step', (done) => setTimeout(done, 50)) + """ + And a file named "features/a.feature" with: + """ + Feature: only one worker works + Scenario: someone must do work + Given slow step + + Scenario: even if it's all the work + Given slow step + """ + When I run cucumber-js with `--parallel 2` + Then the error output contains the text: + """ + WARNING: All workers went idle 2 time(s). Consider revising handler passed to setParallelCanAssign. + """ + And no pickles run at the same time + + Scenario: assignment is appropriately applied + Given a file named "features/step_definitions/cucumber_steps.js" with: + """ + const {Given, setParallelCanAssign} = require('@cucumber/cucumber') + const {atMostOnePicklePerTag} = require('@cucumber/cucumber/lib/support_code_library_builder/parallel_can_assign_helpers') + + setParallelCanAssign(atMostOnePicklePerTag(["@complex", "@simple"])) + + Given('complex step', (done) => setTimeout(done, 3000)) + Given('simple step', (done) => setTimeout(done, 2000)) + """ + And a file named "features/a.feature" with: + """ + Feature: adheres to setParallelCanAssign handler + @complex + Scenario: complex1 + Given complex step + + @complex + Scenario: complex2 + Given complex step + + @simple + Scenario: simple1 + Given simple step + + @simple + Scenario: simple2 + Given simple step + + @simple + Scenario: simple3 + Given simple step + """ + When I run cucumber-js with `--parallel 2` + Then it passes + And the following sets of pickles execute at the same time: + | complex1, simple1 | + | complex1, simple2 | + | complex2, simple2 | + | complex2, simple3 | \ No newline at end of file diff --git a/features/step_definitions/parallel_steps.ts b/features/step_definitions/parallel_steps.ts new file mode 100644 index 000000000..e97c156f7 --- /dev/null +++ b/features/step_definitions/parallel_steps.ts @@ -0,0 +1,57 @@ +import { DataTable, Then } from '../../' +import { World } from '../support/world' +import messages from '@cucumber/messages' +import { expect } from 'chai' + +function getSetsOfPicklesRunningAtTheSameTime( + envelopes: messages.Envelope[] +): string[] { + const pickleIdToName: Record = {} + const testCaseIdToPickleId: Record = {} + const testCaseStarteIdToPickleId: Record = {} + let currentRunningPickleIds: string[] = [] + const result: string[] = [] + envelopes.forEach((envelope) => { + if (envelope.pickle != null) { + pickleIdToName[envelope.pickle.id] = envelope.pickle.name + } else if (envelope.testCase != null) { + testCaseIdToPickleId[envelope.testCase.id] = envelope.testCase.pickleId + } else if (envelope.testCaseStarted != null) { + const pickleId = testCaseIdToPickleId[envelope.testCaseStarted.testCaseId] + testCaseStarteIdToPickleId[envelope.testCaseStarted.id] = pickleId + currentRunningPickleIds.push(pickleId) + if (currentRunningPickleIds.length > 1) { + const setOfPickleNames = currentRunningPickleIds + .map((x) => pickleIdToName[x]) + .sort() + .join(', ') + result.push(setOfPickleNames) + } + } else if (envelope.testCaseFinished != null) { + const pickleId = + testCaseStarteIdToPickleId[envelope.testCaseFinished.testCaseStartedId] + currentRunningPickleIds = currentRunningPickleIds.filter( + (x) => x != pickleId + ) + } + }) + return result +} + +Then('no pickles run at the same time', function (this: World) { + const actualSets = getSetsOfPicklesRunningAtTheSameTime( + this.lastRun.envelopes + ) + expect(actualSets).to.eql([]) +}) + +Then( + 'the following sets of pickles execute at the same time:', + function (this: World, dataTable: DataTable) { + const expectedSets = dataTable.raw().map((row) => row[0]) + const actualSets = getSetsOfPicklesRunningAtTheSameTime( + this.lastRun.envelopes + ) + expect(actualSets).to.eql(expectedSets) + } +) diff --git a/src/cli/helpers_spec.ts b/src/cli/helpers_spec.ts index c5c151657..5e9e19af0 100644 --- a/src/cli/helpers_spec.ts +++ b/src/cli/helpers_spec.ts @@ -80,6 +80,7 @@ function testEmitSupportCodeMessages( parameterTypeRegistry: new ParameterTypeRegistry(), undefinedParameterTypes: [], World: null, + parallelCanAssign: () => true, }, supportCode ), diff --git a/src/index.ts b/src/index.ts index efa1e813e..5db98a80d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,6 +41,7 @@ export const Given = methods.Given export const setDefaultTimeout = methods.setDefaultTimeout export const setDefinitionFunctionWrapper = methods.setDefinitionFunctionWrapper export const setWorldConstructor = methods.setWorldConstructor +export const setParallelCanAssign = methods.setParallelCanAssign export const Then = methods.Then export const When = methods.When export { diff --git a/src/runtime/parallel/README.md b/src/runtime/parallel/README.md index 08f2e084e..826c20a68 100644 --- a/src/runtime/parallel/README.md +++ b/src/runtime/parallel/README.md @@ -1,11 +1,32 @@ Parallelization is achieved by having multiple child processes running scenarios. +#### Customizable work assignment +Cucumber exposes customization of worker assignment via `setParallelCanAssign`. +This can be used to prevent specific test cases from running at the same time. + +The coordinator doesn't reorder work as it skips un-assignable tests. Also, it always +returns to the beginning of the unprocessed list when attempting to make assignments +to an idle worker. + +Custom work assignment prioritizes your definition of assignable work over efficiency. +The exception to this rule is if all remaining work is un-assignable, such that all +workers are idle. In this case Cucumber assigns the next test to the first worker +before continuing to utilize the handler to determine assignable work. Workers become +idle after checking all remaining test cases against the handler. Assignment is +attempted on all idle workers when a busy worker becomes `ready`. + #### Coordinator - load all features, generate test cases - broadcast `test-run-started` - create workers and for each worker - send an `initialize` command - - when a worker outputs a `ready` command, send it a `run` command with a test case. If there are no more test cases, send a `finalize` command + - when a worker outputs a `ready` command + - if there are no more test cases, send a `finalize` command + - identify the next processable test case (the next test by default) + - when there are no processable test cases all idle workers remain idle + - send a `run` command with the test case to an idle worker + - repeat if there are still idle workers + - if all workers become idle and there are more tests, process the next test case - when a worker outputs an `event` command, broadcast the event to the formatters, and on `test-case-finished` update the overall result diff --git a/src/runtime/parallel/coordinator.ts b/src/runtime/parallel/coordinator.ts index a4672dd7d..1ac910d4e 100644 --- a/src/runtime/parallel/coordinator.ts +++ b/src/runtime/parallel/coordinator.ts @@ -27,9 +27,22 @@ export interface INewCoordinatorOptions { numberOfWorkers: number } +const enum WorkerState { + 'idle', + 'closed', + 'running', + 'new', +} + interface IWorker { - closed: boolean + state: WorkerState process: ChildProcess + id: string +} + +interface IPicklePlacement { + index: number + pickle: messages.Pickle } export default class Coordinator implements IRuntime { @@ -38,17 +51,18 @@ export default class Coordinator implements IRuntime { private readonly eventDataCollector: EventDataCollector private readonly stopwatch: ITestRunStopwatch private onFinish: (success: boolean) => void - private nextPickleIdIndex: number private readonly options: IRuntimeOptions private readonly newId: IdGenerator.NewId private readonly pickleIds: string[] private assembledTestCases: IAssembledTestCases + private inProgressPickles: Record private workers: Record private readonly supportCodeLibrary: ISupportCodeLibrary private readonly supportCodePaths: string[] private readonly supportCodeRequiredModules: string[] private readonly numberOfWorkers: number private success: boolean + private idleInterventions: number constructor({ cwd, @@ -71,20 +85,23 @@ export default class Coordinator implements IRuntime { this.supportCodeLibrary = supportCodeLibrary this.supportCodePaths = supportCodePaths this.supportCodeRequiredModules = supportCodeRequiredModules - this.pickleIds = pickleIds + this.pickleIds = Array.from(pickleIds) this.numberOfWorkers = numberOfWorkers - this.nextPickleIdIndex = 0 this.success = true this.workers = {} + this.inProgressPickles = {} + this.idleInterventions = 0 } parseWorkerMessage(worker: IWorker, message: ICoordinatorReport): void { if (message.ready) { - this.giveWork(worker) + worker.state = WorkerState.idle + this.awakenWorkers(worker) } else if (doesHaveValue(message.jsonEnvelope)) { const envelope = messages.parseEnvelope(message.jsonEnvelope) this.eventBroadcaster.emit('envelope', envelope) if (doesHaveValue(envelope.testCaseFinished)) { + delete this.inProgressPickles[worker.id] this.parseTestCaseResult(envelope.testCaseFinished) } } else { @@ -94,6 +111,26 @@ export default class Coordinator implements IRuntime { } } + awakenWorkers(triggeringWorker: IWorker): void { + Object.values(this.workers).forEach((worker) => { + if (worker.state === WorkerState.idle) { + this.giveWork(worker) + } + return worker.state !== WorkerState.idle + }) + + let wip: Boolean = false + for (const p in this.inProgressPickles) { + wip = true + break + } + + if (!wip && this.pickleIds.length > 0) { + this.giveWork(triggeringWorker, true) + this.idleInterventions++ + } + } + startWorker(id: string, total: number): void { const workerProcess = fork(runWorkerPath, [], { cwd: this.cwd, @@ -105,13 +142,13 @@ export default class Coordinator implements IRuntime { }, stdio: ['inherit', 'inherit', 'inherit', 'ipc'], }) - const worker = { closed: false, process: workerProcess } + const worker = { state: WorkerState.new, process: workerProcess, id } this.workers[id] = worker worker.process.on('message', (message: ICoordinatorReport) => { this.parseWorkerMessage(worker, message) }) worker.process.on('close', (exitCode) => { - worker.closed = true + worker.state = WorkerState.closed this.onWorkerProcessClose(exitCode) }) const initializeCommand: IWorkerCommand = { @@ -143,7 +180,10 @@ export default class Coordinator implements IRuntime { if (!success) { this.success = false } - if (Object.values(this.workers).every((x) => x.closed)) { + + if ( + Object.values(this.workers).every((x) => x.state === WorkerState.closed) + ) { const envelope: messages.Envelope = { testRunFinished: { timestamp: this.stopwatch.timestamp(), @@ -184,23 +224,65 @@ export default class Coordinator implements IRuntime { supportCodeLibrary: this.supportCodeLibrary, }) return await new Promise((resolve) => { - for (let i = 0; i <= this.numberOfWorkers; i++) { + for (let i = 0; i < this.numberOfWorkers; i++) { this.startWorker(i.toString(), this.numberOfWorkers) } - this.onFinish = resolve + this.onFinish = (status) => { + if (this.idleInterventions > 0) { + console.warn( + `WARNING: All workers went idle ${this.idleInterventions} time(s). Consider revising handler passed to setParallelCanAssign.` + ) + } + + resolve(status) + } }) } - giveWork(worker: IWorker): void { - if (this.nextPickleIdIndex === this.pickleIds.length) { + nextPicklePlacement(): IPicklePlacement { + for (let index = 0; index < this.pickleIds.length; index++) { + const placement = this.placementAt(index) + if ( + this.supportCodeLibrary.parallelCanAssign( + placement.pickle, + Object.values(this.inProgressPickles) + ) + ) { + return placement + } + } + + return null + } + + placementAt(index: number): IPicklePlacement { + return { + index, + pickle: this.eventDataCollector.getPickle(this.pickleIds[index]), + } + } + + giveWork(worker: IWorker, force: boolean = false): void { + if (this.pickleIds.length < 1) { const finalizeCommand: IWorkerCommand = { finalize: true } + worker.state = WorkerState.running worker.process.send(finalizeCommand) return } - const pickleId = this.pickleIds[this.nextPickleIdIndex] - this.nextPickleIdIndex += 1 - const pickle = this.eventDataCollector.getPickle(pickleId) - const testCase = this.assembledTestCases[pickleId] + + const picklePlacement = force + ? this.placementAt(0) + : this.nextPicklePlacement() + + if (picklePlacement === null) { + return + } + + const { index: nextPickleIndex, pickle } = picklePlacement + + this.pickleIds.splice(nextPickleIndex, 1) + this.inProgressPickles[worker.id] = pickle + const testCase = this.assembledTestCases[pickle.id] const gherkinDocument = this.eventDataCollector.getGherkinDocument( pickle.uri ) @@ -216,6 +298,7 @@ export default class Coordinator implements IRuntime { gherkinDocument, }, } + worker.state = WorkerState.running worker.process.send(runCommand) } } diff --git a/src/support_code_library_builder/index.ts b/src/support_code_library_builder/index.ts index f0061f6a9..d297a9e48 100644 --- a/src/support_code_library_builder/index.ts +++ b/src/support_code_library_builder/index.ts @@ -27,6 +27,7 @@ import { ISupportCodeLibrary, TestCaseHookFunction, TestStepHookFunction, + ParallelAssignmentValidator, } from './types' import World from './world' import { ICanonicalSupportCodeIds } from '../runtime/parallel/command_types' @@ -90,6 +91,7 @@ export class SupportCodeLibraryBuilder { private parameterTypeRegistry: ParameterTypeRegistry private stepDefinitionConfigs: IStepDefinitionConfig[] private World: any + private parallelCanAssign: ParallelAssignmentValidator constructor() { const defineStep = this.defineStep.bind(this) @@ -124,6 +126,9 @@ export class SupportCodeLibraryBuilder { setWorldConstructor: (fn) => { this.World = fn }, + setParallelCanAssign: (fn: ParallelAssignmentValidator): void => { + this.parallelCanAssign = fn + }, Then: defineStep, When: defineStep, } @@ -414,6 +419,7 @@ export class SupportCodeLibraryBuilder { undefinedParameterTypes: stepDefinitionsResult.undefinedParameterTypes, stepDefinitions: stepDefinitionsResult.stepDefinitions, World: this.World, + parallelCanAssign: this.parallelCanAssign, } } @@ -430,6 +436,7 @@ export class SupportCodeLibraryBuilder { this.defaultTimeout = 5000 this.parameterTypeRegistry = new ParameterTypeRegistry() this.stepDefinitionConfigs = [] + this.parallelCanAssign = () => true this.World = World } } diff --git a/src/support_code_library_builder/parallel_can_assign_helpers.ts b/src/support_code_library_builder/parallel_can_assign_helpers.ts new file mode 100644 index 000000000..f697d2998 --- /dev/null +++ b/src/support_code_library_builder/parallel_can_assign_helpers.ts @@ -0,0 +1,19 @@ +import * as messages from '@cucumber/messages' +import { ParallelAssignmentValidator } from './types' + +function hasTag(pickle: messages.Pickle, tagName: string): boolean { + return pickle.tags.some((t) => t.name == tagName) +} + +export function atMostOnePicklePerTag( + tagNames: string[] +): ParallelAssignmentValidator { + return (inQuestion: messages.Pickle, inProgress: messages.Pickle[]) => { + return tagNames.every((tagName) => { + return ( + !hasTag(inQuestion, tagName) || + inProgress.every((p) => !hasTag(p, tagName)) + ) + }) + } +} diff --git a/src/support_code_library_builder/parallel_can_assign_helpers_spec.ts b/src/support_code_library_builder/parallel_can_assign_helpers_spec.ts new file mode 100644 index 000000000..f6d819b49 --- /dev/null +++ b/src/support_code_library_builder/parallel_can_assign_helpers_spec.ts @@ -0,0 +1,72 @@ +import { atMostOnePicklePerTag } from './parallel_can_assign_helpers' +import * as messages from '@cucumber/messages' +import { expect } from 'chai' + +function pickleWithTags(tagNames: string[]): messages.Pickle { + return { + id: 'test', + name: '', + uri: '', + steps: [], + language: null, + astNodeIds: [], + tags: tagNames.map((tagName) => ({ name: tagName, astNodeId: null })), + } +} + +describe('parallel can assign helpers', () => { + describe('atMostOnePicklePerTag()', () => { + const testCanAssignFn = atMostOnePicklePerTag(['@complex', '@simple']) + + it('returns true if no pickles in progress', () => { + // Arrange + const inQuestion = pickleWithTags(['@complex']) + const inProgress: messages.Pickle[] = [] + + // Act + const result = testCanAssignFn(inQuestion, inProgress) + + // Assert + expect(result).to.eql(true) + }) + + it('returns true if pickle in question does not any of the given tags', () => { + // Arrange + const inQuestion = pickleWithTags([]) + const inProgress: messages.Pickle[] = [ + pickleWithTags(['@complex']), + pickleWithTags(['@simple']), + ] + + // Act + const result = testCanAssignFn(inQuestion, inProgress) + + // Assert + expect(result).to.eql(true) + }) + + it('returns true if pickle in question has one of the given tags but no other pickles in progress do', () => { + // Arrange + const inQuestion = pickleWithTags(['@complex']) + const inProgress: messages.Pickle[] = [pickleWithTags(['@simple'])] + + // Act + const result = testCanAssignFn(inQuestion, inProgress) + + // Assert + expect(result).to.eql(true) + }) + + it('returns false if pickle in question has one of the given tags and a pickle in progress also has that tag', () => { + // Arrange + const inQuestion = pickleWithTags(['@complex']) + const inProgress: messages.Pickle[] = [pickleWithTags(['@complex'])] + + // Act + const result = testCanAssignFn(inQuestion, inProgress) + + // Assert + expect(result).to.eql(false) + }) + }) +}) diff --git a/src/support_code_library_builder/types.ts b/src/support_code_library_builder/types.ts index 9f24f0ccb..9caf8b99c 100644 --- a/src/support_code_library_builder/types.ts +++ b/src/support_code_library_builder/types.ts @@ -7,7 +7,10 @@ import { ParameterTypeRegistry } from '@cucumber/cucumber-expressions' import { IWorld } from './world' export type DefineStepPattern = string | RegExp - +export type ParallelAssignmentValidator = ( + pickle: messages.Pickle, + runningPickles: messages.Pickle[] +) => boolean export interface ITestCaseHookParameter { gherkinDocument: messages.GherkinDocument pickle: messages.Pickle @@ -79,6 +82,7 @@ export interface IDefineSupportCodeMethods { ) => void) setDefaultTimeout: (milliseconds: number) => void setDefinitionFunctionWrapper: (fn: Function) => void + setParallelCanAssign: (fn: ParallelAssignmentValidator) => void setWorldConstructor: (fn: any) => void After: ((code: TestCaseHookFunction) => void) & (( @@ -167,4 +171,5 @@ export interface ISupportCodeLibrary { readonly undefinedParameterTypes: messages.UndefinedParameterType[] readonly parameterTypeRegistry: ParameterTypeRegistry readonly World: any + readonly parallelCanAssign: ParallelAssignmentValidator }