Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(core): add project sizes to project graph metadata #22772

Closed
wants to merge 3 commits into from
Closed

feat(core): add project sizes to project graph metadata #22772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3cfe55f
feat(core): add file sizes to the files archive
Cammisuli Apr 10, 2024
fe98558
feat(core): make size optional in the FileData interface for js
Cammisuli Apr 10, 2024
c2d77c1
feat(core): add a metadata plugin for project file sizes
Cammisuli May 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/generated/devkit/FileData.md
View file
Open in desktop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't love that this changes public API, but reckon its fine

Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Some metadata about a file
- [deps](../../devkit/documents/FileData#deps): FileDataDependency[]
- [file](../../devkit/documents/FileData#file): string
- [hash](../../devkit/documents/FileData#hash): string
- [size](../../devkit/documents/FileData#size): number

## Properties

Expand All @@ -33,3 +34,11 @@ If the tuple has 3 elements, the first is preceded by a source.
### hash

• **hash**: `string`

---

### size

• `Optional` **size**: `number`

The size in bytes of this file on the file system \*
2 changes: 2 additions & 0 deletions packages/nx/src/config/project-graph.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import { NxJsonConfiguration } from './nx-json';
export interface FileData {
file: string;
hash: string;
/** The size in bytes of this file on the file system **/
size?: number;
/**
* An array of dependencies. If an element is just a string,
* the dependency is assumed to be a static dependency targetting
Expand Down
7 changes: 5 additions & 2 deletions packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ import {
import { workspaceRoot } from '../../utils/workspace-root';
import { notifyFileWatcherSockets } from './file-watching/file-watcher-sockets';
import { serverLogger } from './logger';
import { NxWorkspaceFilesExternals } from '../../native';
import {
FileData as NativeFileData,
NxWorkspaceFilesExternals,
} from '../../native';
import { ConfigurationResult } from '../../project-graph/utils/project-configuration-utils';
import { LoadedNxPlugin } from '../../project-graph/plugins/internal-api';
import { getPlugins } from './plugins';
Expand Down Expand Up @@ -160,7 +163,7 @@ function computeWorkspaceConfigHash(

async function processCollectedUpdatedAndDeletedFiles(
{ projects, externalNodes, projectRootMap }: ConfigurationResult,
updatedFileHashes: Record<string, string>,
updatedFileHashes: NativeFileData[],
deletedFiles: string[]
) {
try {
Expand Down
108 changes: 59 additions & 49 deletions packages/nx/src/hasher/task-hasher.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@ describe('TaskHasher', () => {
},
});
const allWorkspaceFiles = [
{ file: 'yarn.lock', hash: 'yarn.lock.hash' },
{ file: 'nx.json', hash: 'nx.json.hash' },
{ file: 'package-lock.json', hash: 'package-lock.json.hash' },
{ file: 'package.json', hash: 'package.json.hash' },
{ file: 'pnpm-lock.yaml', hash: 'pnpm-lock.yaml.hash' },
{ file: 'tsconfig.base.json', hash: tsConfigBaseJson },
{ file: 'workspace.json', hash: 'workspace.json.hash' },
{ file: 'global1', hash: 'global1.hash' },
{ file: 'global2', hash: 'global2.hash' },
{ file: 'yarn.lock', hash: 'yarn.lock.hash', size: 0 },
{ file: 'nx.json', hash: 'nx.json.hash', size: 0 },
{ file: 'package-lock.json', hash: 'package-lock.json.hash', size: 0 },
{ file: 'package.json', hash: 'package.json.hash', size: 0 },
{ file: 'pnpm-lock.yaml', hash: 'pnpm-lock.yaml.hash', size: 0 },
{ file: 'tsconfig.base.json', hash: tsConfigBaseJson, size: 0 },
{ file: 'workspace.json', hash: 'workspace.json.hash', size: 0 },
{ file: 'global1', hash: 'global1.hash', size: 0 },
{ file: 'global2', hash: 'global2.hash', size: 0 },
];

beforeEach(async () => {
Expand All @@ -55,8 +55,10 @@ describe('TaskHasher', () => {
it('should create task hash', async () => {
const hasher = new InProcessTaskHasher(
{
parent: [{ file: '/file', hash: 'file.hash' }],
unrelated: [{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash' }],
parent: [{ file: '/file', hash: 'file.hash', size: 0 }],
unrelated: [
{ file: 'libs/unrelated/filec.ts', hash: 'filec.hash', size: 0 },
],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -144,12 +146,12 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
{ file: '/filea.ts', hash: 'a.hash', size: 0 },
{ file: '/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: '/fileb.ts', hash: 'b.hash', size: 0 },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -219,12 +221,12 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: 'libs/child/fileb.ts', hash: 'b.hash' },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: 'libs/child/fileb.ts', hash: 'b.hash', size: 0 },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -306,8 +308,8 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -390,12 +392,12 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: 'libs/child/fileb.ts', hash: 'b.hash' },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: 'libs/child/fileb.ts', hash: 'b.hash', size: 0 },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -503,12 +505,12 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: 'libs/child/fileb.ts', hash: 'b.hash' },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: 'libs/child/fileb.ts', hash: 'b.hash', size: 0 },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -588,7 +590,7 @@ describe('TaskHasher', () => {
it('should be able to include only a part of the base tsconfig', async () => {
const hasher = new InProcessTaskHasher(
{
parent: [{ file: '/file', hash: 'file.hash' }],
parent: [{ file: '/file', hash: 'file.hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -644,8 +646,8 @@ describe('TaskHasher', () => {
it('should hash tasks where the project graph has circular dependencies', async () => {
const hasher = new InProcessTaskHasher(
{
parent: [{ file: '/filea.ts', hash: 'a.hash' }],
child: [{ file: '/fileb.ts', hash: 'b.hash' }],
parent: [{ file: '/filea.ts', hash: 'a.hash', size: 0 }],
child: [{ file: '/fileb.ts', hash: 'b.hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -730,7 +732,7 @@ describe('TaskHasher', () => {
it('should throw an error when failed to execute runtimeCacheInputs', async () => {
const hasher = new InProcessTaskHasher(
{
parent: [{ file: '/file', hash: 'some-hash' }],
parent: [{ file: '/file', hash: 'some-hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -791,7 +793,7 @@ describe('TaskHasher', () => {
it('should hash npm project versions', async () => {
const hasher = new InProcessTaskHasher(
{
app: [{ file: '/filea.ts', hash: 'a.hash' }],
app: [{ file: '/filea.ts', hash: 'a.hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -855,7 +857,7 @@ describe('TaskHasher', () => {
it('should hash missing dependent npm project versions', async () => {
const hasher = new InProcessTaskHasher(
{
app: [{ file: '/filea.ts', hash: 'a.hash' }],
app: [{ file: '/filea.ts', hash: 'a.hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -984,8 +986,8 @@ describe('TaskHasher', () => {
function createHasher() {
return new InProcessTaskHasher(
{
a: [{ file: 'a/filea.ts', hash: 'a.hash' }],
b: [{ file: 'b/fileb.ts', hash: 'b.hash' }],
a: [{ file: 'a/filea.ts', hash: 'a.hash', size: 0 }],
b: [{ file: 'b/fileb.ts', hash: 'b.hash', size: 0 }],
},
allWorkspaceFiles,
{
Expand Down Expand Up @@ -1609,16 +1611,20 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: 'libs/child/fileb.ts', hash: 'b.hash' },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: 'libs/child/fileb.ts', hash: 'b.hash', size: 0 },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
grandchild: [
{ file: 'libs/grandchild/filec.ts', hash: 'c.hash' },
{ file: 'libs/grandchild/filec.spec.ts', hash: 'c.spec.hash' },
{ file: 'libs/grandchild/filec.ts', hash: 'c.hash', size: 0 },
{
file: 'libs/grandchild/filec.spec.ts',
hash: 'c.spec.hash',
size: 0,
},
],
},
allWorkspaceFiles,
Expand Down Expand Up @@ -1748,16 +1754,20 @@ describe('TaskHasher', () => {
const hasher = new InProcessTaskHasher(
{
parent: [
{ file: 'libs/parent/filea.ts', hash: 'a.hash' },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash' },
{ file: 'libs/parent/filea.ts', hash: 'a.hash', size: 0 },
{ file: 'libs/parent/filea.spec.ts', hash: 'a.spec.hash', size: 0 },
],
child: [
{ file: 'libs/child/fileb.ts', hash: 'b.hash' },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash' },
{ file: 'libs/child/fileb.ts', hash: 'b.hash', size: 0 },
{ file: 'libs/child/fileb.spec.ts', hash: 'b.spec.hash', size: 0 },
],
grandchild: [
{ file: 'libs/grandchild/filec.ts', hash: 'c.hash' },
{ file: 'libs/grandchild/filec.spec.ts', hash: 'c.spec.hash' },
{ file: 'libs/grandchild/filec.ts', hash: 'c.hash', size: 0 },
{
file: 'libs/grandchild/filec.spec.ts',
hash: 'c.spec.hash',
size: 0,
},
],
},
allWorkspaceFiles,
Expand Down
5 changes: 3 additions & 2 deletions packages/nx/src/native/index.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ export interface TaskGraph {
export interface FileData {
file: string
hash: string
size: number
}
export interface InputsInput {
input: string
Expand Down Expand Up @@ -185,8 +186,8 @@ export class WorkspaceContext {
getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
hashFilesMatchingGlob(globs: Array<string>, exclude?: Array<string> | undefined | null): string
incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Record<string, string>
updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Record<string, string>, deletedFiles: Array<string>): UpdatedWorkspaceFiles
incrementalUpdate(updatedFiles: Array<string>, deletedFiles: Array<string>): Array<FileData>
updateProjectFiles(projectRootMappings: ProjectRootMappings, projectFiles: ExternalObject<ProjectFiles>, globalFiles: ExternalObject<Array<FileData>>, updatedFiles: Array<FileData>, deletedFiles: Array<string>): UpdatedWorkspaceFiles
allFileData(): Array<FileData>
getFilesInDirectory(directory: string): Array<string>
}
14 changes: 14 additions & 0 deletions packages/nx/src/native/tasks/hashers/hash_project_files.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,18 +76,22 @@ mod tests {
let tsfile_1 = FileData {
file: "test/root/test1.ts".into(),
hash: Default::default(),
size: 0,
};
let testfile_1 = FileData {
file: "test/root/test.spec.ts".into(),
hash: Default::default(),
size: 0,
};
let tsfile_2 = FileData {
file: "test/root/src/module/test3.ts".into(),
hash: Default::default(),
size: 0,
};
let testfile_2 = FileData {
file: "test/root/test.spec.tsx.snap".into(),
hash: Default::default(),
size: 0,
};
file_map.insert(
String::from(proj_name),
Expand Down Expand Up @@ -121,6 +125,7 @@ mod tests {
}

#[test]

fn should_hash_deterministically() {
let proj_name = "test_project";
let proj_root = "test/root";
Expand All @@ -131,19 +136,24 @@ mod tests {
let mut file_map = HashMap::new();
let file_data1 = FileData {
file: "test/root/test1.ts".into(),

hash: "file_data1".into(),
size: 0,
};
let file_data2 = FileData {
file: "test/root/test.spec.ts".into(),
hash: "file_data2".into(),
size: 0,
};
let file_data3 = FileData {
file: "test/root/test3.ts".into(),
hash: "file_data3".into(),
size: 0,
};
let file_data4 = FileData {
file: "test/root/test.spec.tsx.snap".into(),
hash: "file_data4".into(),
size: 0,
};
file_map.insert(
String::from(proj_name),
Expand Down Expand Up @@ -174,18 +184,22 @@ mod tests {
let file_data1 = FileData {
file: "test/root/test1.ts".into(),
hash: "file_data1".into(),
size: 0,
};
let file_data2 = FileData {
file: "test/root/test.spec.ts".into(),
hash: "file_data2".into(),
size: 0,
};
let file_data3 = FileData {
file: "test/root/test3.ts".into(),
hash: "file_data3".into(),
size: 0,
};
let file_data4 = FileData {
file: "test/root/test.spec.tsx.snap".into(),
hash: "file_data4".into(),
size: 0,
};
file_map.insert(
String::from(proj_name),
Expand Down
4 changes: 4 additions & 0 deletions packages/nx/src/native/tasks/hashers/hash_workspace_files.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,22 @@ mod test {
let gitignore_file = FileData {
file: ".gitignore".into(),
hash: "123".into(),
size: 0
};
let nxignore_file = FileData {
file: ".nxignore".into(),
hash: "456".into(),
size: 0
};
let package_json_file = FileData {
file: "package.json".into(),
hash: "789".into(),
size: 0
};
let project_file = FileData {
file: "packages/project/project.json".into(),
hash: "abc".into(),
size: 0
};
let result = hash_workspace_files(
"{workspaceRoot}/.gitignore",
Expand Down