Merge pull request #4701 from iclanton/fix-rush-install

[rush] Fix an issue with rush install in large repos on Windows.

.github/workflows/ci.yml

@@ -26,37 +26,64 @@ jobs:
     name: Node.js v${{ matrix.NodeVersionDisplayName }} (${{ matrix.OS }})
     runs-on: ${{ matrix.OS }}
     steps:
+      - name: Create ~/.rush-user/settings.json
+        shell: pwsh
+        run: |
+          mkdir -p $HOME/.rush-user
+          @{ buildCacheFolder = Join-Path ${{ github.workspace }} rush-cache } | ConvertTo-Json > $HOME/.rush-user/settings.json
+
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
+          path: repo-a
 
       - name: Git config user
         run: |
           git config --local user.name "Rushbot"
           git config --local user.email "rushbot@users.noreply.github.com"
+        working-directory: repo-a
 
       - uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.NodeVersion }}
 
       - name: Verify Change Logs
         run: node common/scripts/install-run-rush.js change --verify
+        working-directory: repo-a
 
       - name: Rush Install
         run: node common/scripts/install-run-rush.js install
+        working-directory: repo-a
 
       # - if: runner.os == 'Linux'
       #   name: Start xvfb
       #   run: /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+      #   working-directory: repo-a
 
       - name: Rush retest (install-run-rush)
         run: node common/scripts/install-run-rush.js retest --verbose --production
+        working-directory: repo-a
 
       - name: Ensure repo README is up-to-date
         run: node repo-scripts/repo-toolbox/lib/start.js readme --verify
+        working-directory: repo-a
+
+      - name: Clone another copy of the repo to test the build cache
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+          path: repo-b
+
+      - name: Git config user
+        run: |
+          git config --local user.name "Rushbot"
+          git config --local user.email "rushbot@users.noreply.github.com"
+        working-directory: repo-b
 
       - name: Rush install (rush-lib)
-        run: node apps/rush/lib/start-dev.js install
+        run: node ${{ github.workspace }}/repo-a/apps/rush/lib/start-dev.js install
+        working-directory: repo-b
 
       - name: Rush test (rush-lib)
-        run: node apps/rush/lib/start-dev.js test --verbose --production --timeline
+        run: node ${{ github.workspace }}/repo-a/apps/rush/lib/start-dev.js test --verbose --production --timeline
+        working-directory: repo-b

common/changes/@microsoft/rush/fix-rush-install_2024-05-13-23-10.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Fix an issue where `rush install` and `rush update` will fail with an `ENAMETOOLONG` error on Windows in repos with a large number of projects.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -1170,7 +1170,7 @@ export class RushConfiguration {
     // @beta (undocumented)
     getSubspace(subspaceName: string): Subspace;
     // @beta
-    getSubspacesForProjects(projects: ReadonlySet<RushConfigurationProject>): ReadonlySet<Subspace>;
+    getSubspacesForProjects(projects: Iterable<RushConfigurationProject>): ReadonlySet<Subspace>;
     readonly gitAllowedEmailRegExps: string[];
     readonly gitChangefilesCommitMessage: string | undefined;
     readonly gitChangeLogUpdateCommitMessage: string | undefined;

libraries/rush-lib/src/api/RushConfiguration.ts

@@ -1295,14 +1295,16 @@ export class RushConfiguration {
    * Returns the set of subspaces that the given projects belong to
    * @beta
    */
-  public getSubspacesForProjects(projects: ReadonlySet<RushConfigurationProject>): ReadonlySet<Subspace> {
+  public getSubspacesForProjects(projects: Iterable<RushConfigurationProject>): ReadonlySet<Subspace> {
     if (!this._projects) {
       this._initializeAndValidateLocalProjects();
     }
+
     const subspaceSet: Set<Subspace> = new Set();
     for (const project of projects) {
       subspaceSet.add(project.subspace);
     }
+
     return subspaceSet;
   }
 

libraries/rush-lib/src/cli/RushPnpmCommandLineParser.ts

@@ -510,7 +510,8 @@ export class RushPnpmCommandLineParser {
       offline: false,
       collectLogFile: false,
       maxInstallAttempts: RushConstants.defaultMaxInstallAttempts,
-      filteredProjects: [],
+      pnpmFilterArguments: [],
+      selectedProjects: new Set(this._rushConfiguration.projects),
       checkOnly: false,
       // TODO: Support subspaces
       subspace: this._rushConfiguration.defaultSubspace,

libraries/rush-lib/src/cli/actions/BaseInstallAction.ts

@@ -24,6 +24,12 @@ import type { SelectionParameterSet } from '../parsing/SelectionParameterSet';
 import type { RushConfigurationProject } from '../../api/RushConfigurationProject';
 import type { Subspace } from '../../api/Subspace';
 
+interface ISubspaceSelectedProjectsMetadata {
+  selectedProjects: Set<RushConfigurationProject>;
+  pnpmFilterArguments: string[];
+  alwaysFullInstall: boolean;
+}
+
 /**
  * This is the common base class for InstallAction and UpdateAction.
  */
@@ -142,23 +148,38 @@ export abstract class BaseInstallAction extends BaseRushAction {
     }
 
     // If we are doing a filtered install and subspaces is enabled, we need to find the affected subspaces and install for all of them.
-    let selectedSubspaces: ReadonlySet<Subspace> | undefined;
-    const filteredProjectsForSubspace: Map<Subspace, RushConfigurationProject[]> = new Map();
+    let selectedSubspaces: Set<Subspace> | undefined;
+    const selectedProjectsMetadataBySubspace: Map<Subspace, ISubspaceSelectedProjectsMetadata> = new Map();
     if (this.rushConfiguration.subspacesFeatureEnabled) {
-      if (installManagerOptions.filteredProjects.length) {
-        // Go through each project, add it to it's subspace's pnpm filter arguments
-        for (const project of installManagerOptions.filteredProjects) {
-          let subspaceFilteredProjects: RushConfigurationProject[] | undefined =
-            filteredProjectsForSubspace.get(project.subspace);
-          if (!subspaceFilteredProjects) {
-            subspaceFilteredProjects = [];
-            filteredProjectsForSubspace.set(project.subspace, subspaceFilteredProjects);
+      selectedSubspaces = new Set();
+      const { selectedProjects } = installManagerOptions;
+      if (selectedProjects.size !== this.rushConfiguration.projects.length) {
+        // This is a filtered install. Go through each project, add its subspace's pnpm filter arguments
+        for (const project of selectedProjects) {
+          const { subspace: projectSubspace } = project;
+          let subspaceSelectedProjectsMetadata: ISubspaceSelectedProjectsMetadata | undefined =
+            selectedProjectsMetadataBySubspace.get(projectSubspace);
+          if (!subspaceSelectedProjectsMetadata) {
+            selectedSubspaces.add(projectSubspace);
+            const alwaysFullInstall: boolean = projectSubspace.getPnpmOptions()?.alwaysFullInstall ?? false;
+            subspaceSelectedProjectsMetadata = {
+              selectedProjects: new Set(alwaysFullInstall ? projectSubspace.getProjects() : undefined),
+              pnpmFilterArguments: [],
+              alwaysFullInstall
+            };
+            selectedProjectsMetadataBySubspace.set(projectSubspace, subspaceSelectedProjectsMetadata);
+          }
+
+          const {
+            pnpmFilterArguments,
+            selectedProjects: subspaceSelectedProjects,
+            alwaysFullInstall
+          } = subspaceSelectedProjectsMetadata;
+          if (!alwaysFullInstall) {
+            subspaceSelectedProjects.add(project);
+            pnpmFilterArguments.push('--filter', project.packageName);
           }
-          subspaceFilteredProjects.push(project);
         }
-        selectedSubspaces = this.rushConfiguration.getSubspacesForProjects(
-          new Set(installManagerOptions.filteredProjects)
-        );
       } else if (this._subspaceParameter.value) {
         // Selecting a single subspace
         const selectedSubspace: Subspace = this.rushConfiguration.getSubspace(this._subspaceParameter.value);
@@ -241,16 +262,28 @@ export abstract class BaseInstallAction extends BaseRushAction {
     try {
       if (selectedSubspaces) {
         // Run the install for each affected subspace
-        for (const selectedSubspace of selectedSubspaces) {
-          installManagerOptions.subspace = selectedSubspace;
-          if (selectedSubspace.getPnpmOptions()?.alwaysFullInstall) {
-            installManagerOptions.filteredProjects = [];
+        for (const subspace of selectedSubspaces) {
+          const selectedProjectsMetadata: ISubspaceSelectedProjectsMetadata | undefined =
+            selectedProjectsMetadataBySubspace.get(subspace);
+          // eslint-disable-next-line no-console
+          console.log(Colorize.green(`Installing for subspace: ${subspace.subspaceName}`));
+          let installManagerOptionsForInstall: IInstallManagerOptions;
+          if (selectedProjectsMetadata) {
+            const { selectedProjects, pnpmFilterArguments } = selectedProjectsMetadata;
+            installManagerOptionsForInstall = {
+              ...installManagerOptions,
+              selectedProjects,
+              pnpmFilterArguments,
+              subspace
+            };
           } else {
-            installManagerOptions.filteredProjects = filteredProjectsForSubspace.get(selectedSubspace) || [];
+            installManagerOptionsForInstall = {
+              ...installManagerOptions,
+              subspace
+            };
           }
-          // eslint-disable-next-line no-console
-          console.log(Colorize.green(`Installing for subspace: ${selectedSubspace.subspaceName}`));
-          await this._doInstall(installManagerFactoryModule, purgeManager, installManagerOptions);
+
+          await this._doInstall(installManagerFactoryModule, purgeManager, installManagerOptionsForInstall);
         }
       } else {
         await this._doInstall(installManagerFactoryModule, purgeManager, installManagerOptions);

libraries/rush-lib/src/cli/actions/InstallAction.ts

@@ -7,6 +7,7 @@ import { BaseInstallAction } from './BaseInstallAction';
 import type { IInstallManagerOptions } from '../../logic/base/BaseInstallManagerTypes';
 import type { RushCommandLineParser } from '../RushCommandLineParser';
 import { SelectionParameterSet } from '../parsing/SelectionParameterSet';
+import type { RushConfigurationProject } from '../../api/RushConfigurationProject';
 
 export class InstallAction extends BaseInstallAction {
   private readonly _checkOnlyParameter: CommandLineFlagParameter;
@@ -44,6 +45,10 @@ export class InstallAction extends BaseInstallAction {
   }
 
   protected async buildInstallOptionsAsync(): Promise<IInstallManagerOptions> {
+    const selectedProjects: Set<RushConfigurationProject> =
+      (await this._selectionParameters?.getSelectedProjectsAsync(this._terminal)) ??
+      new Set(this.rushConfiguration.projects);
+
     return {
       debug: this.parser.isDebug,
       allowShrinkwrapUpdates: false,
@@ -59,7 +64,9 @@ export class InstallAction extends BaseInstallAction {
       // it is safe to assume that the value is not null
       maxInstallAttempts: this._maxInstallAttempts.value!,
       // These are derived independently of the selection for command line brevity
-      filteredProjects: Array.from(await this._selectionParameters!.getSelectedProjectsAsync(this._terminal)),
+      selectedProjects,
+      pnpmFilterArguments:
+        (await this._selectionParameters?.getPnpmFilterArgumentsAsync(this._terminal)) ?? [],
       checkOnly: this._checkOnlyParameter.value,
       subspace: this.getTargetSubspace(),
       beforeInstallAsync: () => this.rushSession.hooks.beforeInstall.promise(this),

libraries/rush-lib/src/cli/actions/UpdateAction.ts

@@ -7,6 +7,7 @@ import { BaseInstallAction } from './BaseInstallAction';
 import type { IInstallManagerOptions } from '../../logic/base/BaseInstallManagerTypes';
 import type { RushCommandLineParser } from '../RushCommandLineParser';
 import { SelectionParameterSet } from '../parsing/SelectionParameterSet';
+import type { RushConfigurationProject } from '../../api/RushConfigurationProject';
 
 export class UpdateAction extends BaseInstallAction {
   private readonly _fullParameter: CommandLineFlagParameter;
@@ -75,6 +76,10 @@ export class UpdateAction extends BaseInstallAction {
   }
 
   protected async buildInstallOptionsAsync(): Promise<IInstallManagerOptions> {
+    const selectedProjects: Set<RushConfigurationProject> =
+      (await this._selectionParameters?.getSelectedProjectsAsync(this._terminal)) ??
+      new Set(this.rushConfiguration.projects);
+
     return {
       debug: this.parser.isDebug,
       allowShrinkwrapUpdates: true,
@@ -90,9 +95,9 @@ export class UpdateAction extends BaseInstallAction {
       // it is safe to assume that the value is not null
       maxInstallAttempts: this._maxInstallAttempts.value!,
       // These are derived independently of the selection for command line brevity
-      filteredProjects: Array.from(
-        (await this._selectionParameters?.getSelectedProjectsAsync(this._terminal)) || []
-      ),
+      selectedProjects,
+      pnpmFilterArguments:
+        (await this._selectionParameters?.getPnpmFilterArgumentsAsync(this._terminal)) ?? [],
       checkOnly: false,
       subspace: this.getTargetSubspace(),
 

libraries/rush-lib/src/logic/PackageJsonUpdater.ts

@@ -232,7 +232,7 @@ export class PackageJsonUpdater {
     if (!skipUpdate) {
       if (this._rushConfiguration.subspacesFeatureEnabled) {
         const subspaceSet: ReadonlySet<Subspace> = this._rushConfiguration.getSubspacesForProjects(
-          new Set(options.projects)
+          options.projects
         );
         for (const subspace of subspaceSet) {
           await this._doUpdate(debugInstall, subspace);
@@ -262,7 +262,7 @@ export class PackageJsonUpdater {
     if (!skipUpdate) {
       if (this._rushConfiguration.subspacesFeatureEnabled) {
         const subspaceSet: ReadonlySet<Subspace> = this._rushConfiguration.getSubspacesForProjects(
-          new Set(options.projects)
+          options.projects
         );
         for (const subspace of subspaceSet) {
           await this._doUpdate(debugInstall, subspace);
@@ -290,7 +290,8 @@ export class PackageJsonUpdater {
       offline: false,
       collectLogFile: false,
       maxInstallAttempts: RushConstants.defaultMaxInstallAttempts,
-      filteredProjects: [],
+      pnpmFilterArguments: [],
+      selectedProjects: new Set(this._rushConfiguration.projects),
       checkOnly: false,
       subspace: subspace,
       terminal: this._terminal
@@ -326,9 +327,7 @@ export class PackageJsonUpdater {
     );
 
     const allPackageUpdates: IUpdateProjectOptions[] = [];
-    const subspaceSet: ReadonlySet<Subspace> = this._rushConfiguration.getSubspacesForProjects(
-      new Set(projects)
-    );
+    const subspaceSet: ReadonlySet<Subspace> = this._rushConfiguration.getSubspacesForProjects(projects);
     for (const subspace of subspaceSet) {
       // Projects for this subspace
       allPackageUpdates.push(...(await this._updateProjects(subspace, dependencyAnalyzer, options)));