fix(core): fix yarn and npm direct node reference

packages/nx/src/lock-file/lock-file.spec.ts

@@ -24,6 +24,8 @@ import {
   lockFile as yarnLockFile,
 } from './__fixtures__/yarn.lock';
 import { vol } from 'memfs';
+import { ProjectGraph } from '../config/project-graph';
+import { createPackageJson } from '../utils/create-package-json';
 
 jest.mock('fs', () => require('memfs').fs);
 
@@ -193,32 +195,105 @@ describe('lock-file', () => {
   });
 
   describe('package aliases and direct urls', () => {
+    function expandGraph(partialGraph: ProjectGraph) {
+      partialGraph.nodes['lib1'] = {
+        type: 'lib',
+        name: 'lib1',
+        data: {
+          root: 'libs/lib1',
+        },
+      };
+      partialGraph.dependencies['lib1'] = [
+        {
+          type: 'static',
+          source: 'lib1',
+          target: 'npm:postgres',
+        },
+      ];
+      return partialGraph;
+    }
+
+    const fileSys = {
+      'package.json': JSON.stringify({
+        name: 'test',
+        version: '0.0.0',
+        license: 'MIT',
+        dependencies: {
+          '@nrwl/devkit': '15.0.13',
+          yargs: '17.6.2',
+          postgres: 'charsleysa/postgres#fix-errors-compiled',
+        },
+        devDependencies: {
+          react: '18.2.0',
+        },
+        peerDependencies: {
+          typescript: '4.8.4',
+        },
+      }),
+    };
+    beforeEach(() => {
+      vol.fromJSON(fileSys, '/root');
+    });
+
     it('should properly parse, map and stringify npm', () => {
       const lockFileData = parseNpmLockFile(npmLockFileWithAliases);
-
-      const partialGraph = mapLockFileDataToPartialGraph(lockFileData, 'npm');
-
       const lockFile = stringifyNpmLockFile(lockFileData);
 
       expect(lockFile).toEqual(npmLockFileWithAliases);
+
+      const partialGraph = expandGraph(
+        mapLockFileDataToPartialGraph(lockFileData, 'npm')
+      );
+      const newPackage = createPackageJson('lib1', partialGraph, {});
+      expect(newPackage).toMatchInlineSnapshot(`
+        Object {
+          "dependencies": Object {
+            "postgres": "git+ssh://git@github.com/charsleysa/postgres.git#3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+          },
+          "name": "lib1",
+          "version": "0.0.1",
+        }
+      `);
     });
     it('should properly parse, map and stringify yarn', () => {
       const lockFileData = parseYarnLockFile(yarnLockFileWithAliases);
-
-      const partialGraph = mapLockFileDataToPartialGraph(lockFileData, 'yarn');
-
       const lockFile = stringifyYarnLockFile(lockFileData);
 
       expect(lockFile).toEqual(yarnLockFileWithAliases);
+
+      const partialGraph = expandGraph(
+        mapLockFileDataToPartialGraph(lockFileData, 'yarn')
+      );
+      const newPackage = createPackageJson('lib1', partialGraph, {});
+      expect(newPackage).toMatchInlineSnapshot(`
+        Object {
+          "dependencies": Object {
+            "postgres": "charsleysa/postgres#fix-errors-compiled",
+          },
+          "name": "lib1",
+          "version": "0.0.1",
+        }
+      `);
     });
     it('should properly parse, map and stringify pnpm', () => {
       const lockFileData = parsePnpmLockFile(pnpmLockFileWithAliases);
-
-      const partialGraph = mapLockFileDataToPartialGraph(lockFileData, 'pnpm');
-
       const lockFile = stringifyPnpmLockFile(lockFileData);
 
       expect(lockFile).toEqual(pnpmLockFileWithAliases);
+
+      const partialGraph = expandGraph(
+        mapLockFileDataToPartialGraph(lockFileData, 'pnpm')
+      );
+      const newPackage = createPackageJson('lib1', partialGraph, {});
+      expect(newPackage).toMatchInlineSnapshot(`
+        Object {
+          "dependencies": Object {
+            "postgres": "github.com/charsleysa/postgres/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+          },
+          "name": "lib1",
+          "version": "0.0.1",
+        }
+      `);
     });
   });
 });

packages/nx/src/lock-file/npm.ts

@@ -40,7 +40,7 @@ type NpmLockFile = {
   name?: string;
   lockfileVersion: number;
   requires?: boolean;
-  packages?: Dependencies;
+  packages?: Record<string, NpmDependency>;
   dependencies?: Record<string, NpmDependency>;
 };
 
@@ -67,7 +67,7 @@ export function parseNpmLockFile(lockFile: string): LockFileData {
 // Maps /node_modules/@abc/def with version 1.2.3 => @abc/def > @abc/dev@1.2.3
 function mapPackages(
   dependencies: Record<string, NpmDependency>,
-  packages: Dependencies,
+  packages: Record<string, NpmDependency>,
   lockfileVersion: number
 ): LockFileData['dependencies'] {
   const mappedPackages: LockFileData['dependencies'] = {};
@@ -76,7 +76,7 @@ function mapPackages(
     Object.entries(dependencies).forEach(([packageName, value]) => {
       const { newKey, packagePath } = prepareDependency(
         packageName,
-        value.version,
+        value,
         mappedPackages
       );
 
@@ -104,7 +104,7 @@ function mapPackages(
         const packageName = packagePath.split('node_modules/').pop();
         const { newKey } = prepareDependency(
           packageName,
-          value.version,
+          value,
           mappedPackages,
           undefined,
           packagePath
@@ -127,12 +127,15 @@ function mapPackages(
 
 function prepareDependency(
   packageName: string,
-  version: string,
+  dependency: NpmDependency,
   mappedPackages: LockFileData['dependencies'],
   pathPrefix: string = '',
   path?: string
 ) {
   mappedPackages[packageName] = mappedPackages[packageName] || {};
+  const version = dependency.integrity
+    ? dependency.version
+    : dependency.resolved;
   const newKey = packageName + '@' + version;
   const packagePath =
     path || pathPrefix
@@ -173,6 +176,10 @@ function mapPackageDependency(
 
     mappedPackages[packageName][key] = {
       ...(value as Omit<PackageDependency, 'packageMeta'>),
+      ...(!value.integrity && {
+        actualVersion: value.version,
+        version: value.resolved,
+      }),
       packageMeta: [],
       rootVersion,
     };
@@ -194,7 +201,7 @@ function mapPackageDependencies(
   Object.entries(dependencies).forEach(([packageName, value]) => {
     const { newKey, packagePath } = prepareDependency(
       packageName,
-      value.version,
+      value,
       mappedPackages,
       parentPath
     );
@@ -273,6 +280,7 @@ function unmapPackage(packages: Dependencies, dependency: PackageDependency) {
     packageMeta,
     rootVersion,
     version,
+    actualVersion,
     resolved,
     integrity,
     dev,
@@ -286,7 +294,7 @@ function unmapPackage(packages: Dependencies, dependency: PackageDependency) {
     const { path, dev, peer, optional } = packageMeta[i];
     // we are sorting the properties to get as close as possible to the original package-lock.json
     packages[path] = {
-      version,
+      version: actualVersion || version,
       resolved,
       integrity,
       dev,

packages/nx/src/lock-file/utils/hashing.ts

@@ -40,16 +40,12 @@ function traverseExternalNodesDependencies(
 ) {
   graph.dependencies[projectName].forEach((d) => {
     const target = graph.externalNodes[d.target];
-    try {
-      const targetKey = `${target.data.packageName}@${target.data.version}`;
-      if (visited.indexOf(targetKey) === -1) {
-        visited.push(targetKey);
-        if (graph.dependencies[d.target]) {
-          traverseExternalNodesDependencies(d.target, graph, visited);
-        }
+    const targetKey = `${target.data.packageName}@${target.data.version}`;
+    if (visited.indexOf(targetKey) === -1) {
+      visited.push(targetKey);
+      if (graph.dependencies[d.target]) {
+        traverseExternalNodesDependencies(d.target, graph, visited);
       }
-    } catch (e) {
-      console.log(d.target, Object.keys(graph.externalNodes));
     }
   });
 }

packages/nx/src/lock-file/yarn.ts

@@ -26,8 +26,11 @@ export function parseYarnLockFile(lockFile: string): LockFileData {
   const { __metadata, ...dependencies } = parseSyml(lockFile);
 
   // Yarn Berry has workspace packages includes, so we need to extract those to metadata
-  const [mappedPackages, workspacePackages] = mapPackages(dependencies);
   const isBerry = !!__metadata;
+  const [mappedPackages, workspacePackages] = mapPackages(
+    dependencies,
+    isBerry
+  );
   const hash = hashString(lockFile);
   if (isBerry) {
     return {
@@ -43,9 +46,34 @@ export function parseYarnLockFile(lockFile: string): LockFileData {
   }
 }
 
+function extendVersionProperty(
+  key: string,
+  value: Omit<PackageDependency, 'packageMeta'>,
+  isBerry: boolean
+) {
+  if (isBerry) {
+    const packageName = key.slice(0, key.lastIndexOf('@'));
+    if (value.resolution !== `${packageName}@npm:${value.version}`) {
+      return {
+        version: value.resolution.slice(packageName.length + 1),
+        actualVersion: value.version,
+      };
+    }
+    return;
+  }
+  if (value.integrity) {
+    return;
+  }
+  return {
+    version: key.slice(key.lastIndexOf('@') + 1),
+    actualVersion: value.version,
+  };
+}
+
 // map original yarn packages to the LockFileData structure
 function mapPackages(
-  packages: LockFileDependencies
+  packages: LockFileDependencies,
+  isBerry: boolean
 ): [LockFileData['dependencies'], LockFileDependencies] {
   const mappedPackages: LockFileData['dependencies'] = {};
   const workspacePackages: LockFileDependencies = {};
@@ -65,6 +93,7 @@ function mapPackages(
       if (!mappedPackages[packageName][newKey]) {
         mappedPackages[packageName][newKey] = {
           ...value,
+          ...extendVersionProperty(keys[0], value, isBerry),
           packageMeta: keys,
         };
       } else {
@@ -137,7 +166,10 @@ function unmapPackages(
 
   Object.values(dependencies).forEach((packageVersions) => {
     Object.values(packageVersions).forEach((value) => {
-      const { packageMeta, rootVersion, ...rest } = value;
+      const { packageMeta, rootVersion, actualVersion, ...rest } = value;
+      if (actualVersion) {
+        rest.version = actualVersion;
+      }
       if (isBerry) {
         // berry's `stringifySyml` does not combine packages
         // we have to do it manually