fix(nm): Properly hoist nested workspaces (#3438)

* Properly hoist nested workspaces

* Update pnp.cjs

* Runs lint

* Adds nested internal portals support

* Adds a test that checks if symlinks are created for anonymous workspaces

* Adds a test for nested workspace hoisting

* Don't try to meet workspace peer dependency constraints for now

* Forces Babel repo to use Yarn from sources

* Removes console.log

* Allows direct workspace dependencies hoisting to the root workspace only

* Keep the orig reason

* Adds a unit test to check multi-parent workspace hoisting

* Fixes test

* Improve nested workspaces hoisting unit test

* Fixes reasons binding

* Improves means of hoister troubleshooting by adding hoistedTo and hoistedFrom to the parent of each node

* Outputs detailed troubleshooting information for broken require promise self check

* Adds additional hoisting checks for the round > 1

* Fixes lint issues

Co-authored-by: Maël Nison <nison.mael@gmail.com>

.github/workflows/e2e-nm-babel-workflow.yml

@@ -51,5 +51,6 @@ jobs:
         source scripts/e2e-setup-ci.sh nm
         git clone https://github.com/babel/babel.git
         cd babel
+        yarn config unset yarnPath
         NM_DEBUG_LEVEL=1 yarn
       shell: bash

.yarn/versions/64d8f84b.yml

@@ -0,0 +1,26 @@
+releases:
+  "@yarnpkg/cli": patch
+  "@yarnpkg/nm": patch
+  "@yarnpkg/plugin-nm": patch
+  "@yarnpkg/pnpify": patch
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/core"
+  - "@yarnpkg/doctor"

CHANGELOG.md

@@ -15,6 +15,8 @@ Yarn now accepts sponsorships! Please give a look at our [OpenCollective](https:
 - Hoisting algorithm speedup, impacts recurrent `node_modules` installs time.
 - CLI bundles built from sources output `commit` hash instead of `tree` hash as part of their version
 - `workspaces foreach run` now handles the fact that a script containing `:` only becomes global if it exists in one workspace.
+- Nested workspaces are properly hoisted by `node-modules` linker.
+- Self-referencing symlinks are not created for anonymous workspaces by `node-modules` linker, since they cannot be used anyway from the code.
 - The PnP compatibility patch for `resolve` will no longer resolve missing modules to a file with the same name located next to the issuer
 - `logFilters` using `pattern` matchers now match any part of the log entry
 - The cache is now fully atomic when moving files across devices and in general more efficient.

packages/acceptance-tests/pkg-tests-specs/sources/node-modules.test.ts

@@ -690,19 +690,17 @@ describe(`Node_Modules`, () => {
       {
         nodeLinker: `node-modules`,
       },
-      async ({path, run, source}) => {
+      async ({path, run}) => {
         await writeJson(npath.toPortablePath(`${path}/foo/package.json`), {
           name: `foo`,
-          version: `1.0.0`,
           workspaces: [`bar`],
           dependencies: {
+            bar: `workspace:*`,
             'no-deps': `1.0.0`,
           },
         });
         await writeJson(npath.toPortablePath(`${path}/foo/bar/package.json`), {
           name: `bar`,
-          version: `1.0.0`,
-          workspaces: [`bar`],
           peerDependencies: {
             'no-deps': `*`,
           },
@@ -1339,13 +1337,13 @@ describe(`Node_Modules`, () => {
 
 
   test(
-    `it should fallback to dependencies if the parent doesn't provide the peer dependency`,
+    `should fallback to dependencies if the parent doesn't provide the peer dependency`,
     makeTemporaryEnv(
       {},
       {
         nodeLinker: `node-modules`,
       },
-      async ({path, run, source}) => {
+      async ({path, run}) => {
         const appPath = ppath.join(path, `lib-1` as Filename);
         const libPath = ppath.join(path, `lib-2` as Filename);
 
@@ -1381,4 +1379,76 @@ describe(`Node_Modules`, () => {
       },
     ),
   );
+
+  it(`should not create self-referencing symlinks for anonymous workspaces`,
+    makeTemporaryEnv(
+      {
+      },
+      {
+        nodeLinker: `node-modules`,
+      },
+      async ({path, run}) => {
+        await run(`install`);
+
+        const entries = await xfs.readdirPromise(ppath.join(path, `node_modules` as Filename), {withFileTypes: true});
+        let symlinkCount = 0;
+        for (const entry of entries) {
+          if (entry.isSymbolicLink()) {
+            symlinkCount++;
+          }
+        }
+
+        expect(symlinkCount).toBe(0);
+      },
+    ),
+  );
+
+  it(`should properly hoist nested workspaces`,
+    makeTemporaryEnv(
+      {
+        workspaces: [`ws`, `ws/nested1`, `ws/nested1/nested2`],
+        dependencies: {
+          ws: `workspace:*`,
+          nested1: `workspace:*`,
+          nested2: `workspace:*`,
+        },
+      },
+      {
+        nodeLinker: `node-modules`,
+      },
+      async ({path, run, source}) => {
+        await xfs.mkdirpPromise(ppath.join(path, `ws/nested1/nested2` as PortablePath));
+
+        await xfs.writeJsonPromise(ppath.join(path, `ws/${Filename.manifest}` as PortablePath), {
+          name: `ws`,
+          dependencies: {
+            [`no-deps`]: `1.0.0`,
+          },
+        });
+
+        await xfs.writeJsonPromise(ppath.join(path, `ws/nested1/${Filename.manifest}` as PortablePath), {
+          name: `nested1`,
+          dependencies: {
+            [`no-deps`]: `2.0.0`,
+          },
+        });
+
+        await xfs.writeJsonPromise(ppath.join(path, `ws/nested1/nested2/${Filename.manifest}` as PortablePath), {
+          name: `nested2`,
+          dependencies: {
+            [`no-deps`]: `2.0.0`,
+          },
+        });
+
+        await run(`install`);
+
+        await expect(source(`require('no-deps')`)).resolves.toMatchObject({
+          version: `1.0.0`,
+        });
+        await expect(source(`require('module').createRequire(require.resolve('nested1/package.json') + '/..')('no-deps')`)).resolves.toMatchObject({
+          version: `2.0.0`,
+        });
+      },
+    ),
+  );
 });