chore: task-runner e2e (#3457)

.github/workflows/ci.yml

@@ -43,8 +43,12 @@ jobs:
           # TODO: update this once project targets are converted
           cmd5: npx nx run-many --target=test --projects=\"libs-*,nx-*\" --parallel=3
 
+      # e2e tests for everything except the primary task runner
       - run: PUBLISHED_VERSION=999.9.9-e2e.0 npx nx run-many --target=run-e2e-tests-process --parallel=2
 
+      # Isolated e2e tests for the task runner which become too flaky if nested through further node child processes
+      - run: e2e/run/task-runner/src/run-tests.sh
+
       - name: Stop all running agents for this CI run
         # It's important that we always run this step, otherwise in the case of any failures in preceding non-Nx steps, the agents will keep running and waste billable minutes
         if: ${{ always() }}

CONTRIBUTING.md

@@ -154,6 +154,10 @@ In addition to our lower level testing, we also have a suite of e2e tests which
 
 Because of this high-value nature of the tests, they are also much slower than unit tests. Therefore they are split up into different e2e projects in the workspace which can be run independently and can benefit from more granular caching (thanks to Nx).
 
+The core concepts of building and publishing the packages to a local registry and then invoking the lerna CLI just look a user would remain the same across all areas, but there are some slightly different instructions for the task-runner tests.
+
+#### All Projects (except `e2e-run-task-runner`)
+
 To run the e2e tests for a particular project, such as `e2e/info`, which tests the `lerna info` CLI command, you can run:
 
 ```sh
@@ -169,6 +173,39 @@ npx nx e2e e2e-info -t qqqq -u
 
 > NOTE: The building, versioning and publishing of the packages will be the same regardless of the jest flags passed
 
+#### Testing the task-runner (`e2e-run-task-runner`)
+
+Because the task-runner itself handles spawning multiple nested child processes in node, it becomes hard to wrap that in further node child processes for the purposes of collecting the stdout and stderr outputs in order to assert things about them in test files. This is because node offers no guarantees around the ordering of writes to those stdout and stderr streams, meaning that, even though all the CLI output lines are the same as what a user would see, the ordering of those lines could be non-deterministic in our tests.
+
+Therefore in our task-runner tests we use bash to invoke the lerna CLI and first write the outputs to disk. Then our node tests read those files and assert things about them. We've found by doing it this way we remove most if not all of the non-determinism from the tests.
+
+To run the e2e tests for the task runner, you first need to make sure the local registry is running and has had the packages published to it. For that you can run:
+
+```sh
+npx nx prepare-for-e2e e2e-run-task-runner
+```
+
+Then you can actually execute the tests by running the bash script:
+
+```sh
+e2e/run/task-runner/src/run-tests.sh
+```
+
+This bash script takes the name of a subdirectory within `e2e/run/task-runner/src` as an argument, allowing you to focus on a smaller subset of tests. E.g. to just run the tests in `e2e/run/task-runner/src/env-files`, you can run:
+
+```sh
+e2e/run/task-runner/src/run-tests.sh env-files
+```
+
+If you pass `--update-snapshots` to the shell script it will run jest with the `-u` option to update any existing snapshots.
+
+E.g.
+
+```sh
+e2e/run/task-runner/src/run-tests.sh --update-snapshots # to update all test snapshots
+e2e/run/task-runner/src/run-tests.sh env-files --update-snapshots # to update just the snapshots env-files
+```
+
 ### Releasing
 
 If you are a member of Lerna's [GitHub org](https://github.com/orgs/lerna/people) and have read-write privileges in Lerna's [npm org](https://www.npmjs.com/org/lerna) _with 2-factor auth enabled_, congratulations, you can cut a release!

e2e/run/legacy-task-runner/src/include-dependencies.spec.ts

@@ -0,0 +1,80 @@
+import { Fixture, normalizeCommandOutput, normalizeEnvironment } from "@lerna/e2e-utils";
+
+expect.addSnapshotSerializer({
+  serialize(str: string) {
+    return normalizeCommandOutput(normalizeEnvironment(str));
+  },
+  test(val: string) {
+    return val != null && typeof val === "string";
+  },
+});
+
+describe("lerna-run-legacy-task-runner-include-dependencies", () => {
+  let fixture: Fixture;
+
+  beforeEach(async () => {
+    fixture = await Fixture.create({
+      name: "lerna-run-legacy-task-runner-include-dependencies",
+      packageManager: "npm",
+      initializeGit: true,
+      runLernaInit: true,
+      installDependencies: true,
+      /**
+       * Because lerna run involves spawning further child processes, the tests would be too flaky
+       * if we didn't force deterministic terminal output by appending stderr to stdout instead
+       * of interleaving them.
+       */
+      forceDeterministicTerminalOutput: true,
+    });
+
+    await fixture.lerna("create package-1 -y");
+    await fixture.addScriptsToPackage({
+      packagePath: "packages/package-1",
+      scripts: {
+        "print-name": "echo test-package-1",
+      },
+    });
+    await fixture.lerna("create package-2 -y");
+    await fixture.addScriptsToPackage({
+      packagePath: "packages/package-2",
+      scripts: {
+        "print-name": "echo test-package-2",
+      },
+    });
+    await fixture.lerna("create package-3 -y --dependencies package-1 package-2");
+    await fixture.addScriptsToPackage({
+      packagePath: "packages/package-3",
+      scripts: {
+        "print-name": "echo test-package-3",
+      },
+    });
+
+    await fixture.updateJson("lerna.json", (json) => ({
+      ...json,
+      loglevel: "verbose",
+    }));
+  });
+  afterEach(() => fixture.destroy());
+
+  it("should exclude dependencies by default", async () => {
+    // Enable legacy task runner
+    await fixture.overrideLernaConfig({
+      useNx: false,
+    });
+
+    const output = await fixture.lerna("run print-name --scope package-3 -- --silent");
+
+    expect(output.combinedOutput).toMatchInlineSnapshot(`
+        test-package-X
+        lerna notice cli v999.9.9-e2e.0
+        lerna verb rootPath /tmp/lerna-e2e/lerna-run-legacy-task-runner-include-dependencies/lerna-workspace
+        lerna notice filter including "package-X"
+        lerna info filter [ 'package-X' ]
+        lerna info Executing command in 1 package: "npm run print-name --silent"
+        lerna info run Ran npm script 'print-name' in 'package-X' in X.Xs:
+        lerna success run Ran npm script 'print-name' in 1 package in X.Xs:
+        lerna success - package-X
+
+      `);
+  });
+});