/
binary-cleanup.js
193 lines (171 loc) · 8.77 KB
/
binary-cleanup.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
const fs = require('fs-extra')
const path = require('path')
const { consolidateDeps } = require('@tooling/v8-snapshot')
const del = require('del')
const esbuild = require('esbuild')
const snapshotMetadata = require('@tooling/v8-snapshot/cache/prod-darwin/snapshot-meta.cache.json')
const tempDir = require('temp-dir')
const workingDir = path.join(tempDir, 'binary-cleanup-workdir')
fs.ensureDirSync(workingDir)
async function removeEmptyDirectories (directory) {
// lstat does not follow symlinks (in contrast to stat)
const fileStats = await fs.lstat(directory)
if (!fileStats.isDirectory()) {
return
}
let fileNames = await fs.readdir(directory)
if (fileNames.length > 0) {
const recursiveRemovalPromises = fileNames.map(
(fileName) => removeEmptyDirectories(path.join(directory, fileName)),
)
await Promise.all(recursiveRemovalPromises)
// re-evaluate fileNames; after deleting subdirectory
// we may have parent directory empty now
fileNames = await fs.readdir(directory)
}
if (fileNames.length === 0) {
await fs.rmdir(directory)
}
}
const getDependencyPathsToKeep = async (buildAppDir) => {
let entryPoints = new Set([
// This is the entry point for the server bundle. It will not have access to the snapshot yet. It needs to be kept in the binary
require.resolve('@packages/server/index.js', { paths: [buildAppDir] }),
// This is a dynamic import that is used to load the snapshot require logic. It will not have access to the snapshot yet. It needs to be kept in the binary
require.resolve('@packages/server/hook-require.js', { paths: [buildAppDir] }),
// These dependencies are started in a new process or thread and will not have access to the snapshot. They need to be kept in the binary
require.resolve('@packages/server/lib/plugins/child/require_async_child.js', { paths: [buildAppDir] }),
require.resolve('@packages/server/lib/plugins/child/register_ts_node.js', { paths: [buildAppDir] }),
require.resolve('@packages/rewriter/lib/threads/worker.ts', { paths: [buildAppDir] }),
// These dependencies use the `require.resolve(<dependency>, { paths: [<path>] })` pattern where <path> is a path within the cypress monorepo. These will not be
// pulled in by esbuild but still need to be kept in the binary.
require.resolve('webpack', { paths: [buildAppDir] }),
require.resolve('webpack-dev-server', { paths: [buildAppDir] }),
require.resolve('html-webpack-plugin-4', { paths: [buildAppDir] }),
require.resolve('html-webpack-plugin-5', { paths: [buildAppDir] }),
// These dependencies are completely dynamic using the pattern `require(`./${name}`)` and will not be pulled in by esbuild but still need to be kept in the binary.
...['ibmi',
'sunos',
'android',
'darwin',
'freebsd',
'linux',
'openbsd',
'sunos',
'win32'].map((platform) => require.resolve(`default-gateway/${platform}`, { paths: [buildAppDir] })),
])
let esbuildResult
let newEntryPointsFound = true
// The general idea here is to run esbuild on entry points that are used outside of the snapshot. If, during the process,
// we find places where we do a require.resolve on a module, that should be treated as an additional entry point and we run
// esbuild again. We do this until we no longer find any new entry points. The resulting metafile inputs are
// the dependency paths that we need to ensure stay in the snapshot.
while (newEntryPointsFound) {
esbuildResult = await esbuild.build({
entryPoints: [...entryPoints],
bundle: true,
outdir: workingDir,
platform: 'node',
metafile: true,
absWorkingDir: buildAppDir,
external: [
'./packages/packherd-require/dist/transpile-ts',
'./packages/server/server-entry',
'fsevents',
'pnpapi',
'@swc/core',
'emitter',
],
})
newEntryPointsFound = false
esbuildResult.warnings.forEach((warning) => {
const matches = warning.text.match(/"(.*)" should be marked as external for use with "require.resolve"/)
const warningSubject = matches && matches[1]
if (warningSubject) {
let entryPoint
if (warningSubject.startsWith('.')) {
entryPoint = path.join(buildAppDir, path.dirname(warning.location.file), warningSubject)
} else {
entryPoint = require.resolve(warningSubject, { paths: [path.join(buildAppDir, path.dirname(warning.location.file))] })
}
if (path.extname(entryPoint) !== '' && !entryPoints.has(entryPoint)) {
newEntryPointsFound = true
entryPoints.add(entryPoint)
}
}
})
}
return [...Object.keys(esbuildResult.metafile.inputs), ...entryPoints]
}
const cleanup = async (buildAppDir) => {
// 1. Retrieve all dependencies that still need to be kept in the binary. In theory, we could use the bundles generated here as single files within the binary,
// but for now, we just track on the dependencies that get pulled in
const keptDependencies = [...await getDependencyPathsToKeep(buildAppDir), 'package.json', 'packages/server/server-entry.js']
// 2. Gather the dependencies that could potentially be removed from the binary due to being in the snapshot
const potentiallyRemovedDependencies = [...snapshotMetadata.healthy, ...snapshotMetadata.deferred, ...snapshotMetadata.norewrite]
// 3. Remove all dependencies that are in the snapshot but not in the list of kept dependencies from the binary
await Promise.all(potentiallyRemovedDependencies.map(async (dependency) => {
// marionette-client requires all of its dependencies in a very non-standard dynamic way. We will keep anything in marionette-client
if (!keptDependencies.includes(dependency.slice(2)) && !dependency.includes('marionette-client')) {
await fs.remove(path.join(buildAppDir, dependency.replace(/.ts$/, '.js')))
}
}))
// 4. Consolidate dependencies that are safe to consolidate (`lodash` and `bluebird`)
await consolidateDeps({ projectBaseDir: buildAppDir })
// 5. Remove various unnecessary files from the binary to further clean things up. Likely, there is additional work that can be done here
await del([
// Remove test files
path.join(buildAppDir, '**', 'test'),
path.join(buildAppDir, '**', 'tests'),
// What we need of prettier is entirely encapsulated within the v8 snapshot, but has a few leftover large files
path.join(buildAppDir, '**', 'prettier', 'esm'),
path.join(buildAppDir, '**', 'prettier', 'standalone.js'),
path.join(buildAppDir, '**', 'prettier', 'bin-prettier.js'),
// ESM files are mostly not needed currently
path.join(buildAppDir, '**', '@babel', '**', 'esm'),
path.join(buildAppDir, '**', 'ramda', 'es'),
path.join(buildAppDir, '**', 'jimp', 'es'),
path.join(buildAppDir, '**', '@jimp', '**', 'es'),
path.join(buildAppDir, '**', 'nexus', 'dist-esm'),
path.join(buildAppDir, '**', '@graphql-tools', '**', '*.mjs'),
path.join(buildAppDir, '**', 'graphql', '**', '*.mjs'),
// We currently do not use any map files
path.join(buildAppDir, '**', '*js.map'),
// License files need to be kept
path.join(buildAppDir, '**', '!(LICENSE|license|License).md'),
// These are type related files that are not used within the binary
path.join(buildAppDir, '**', '*.d.ts'),
path.join(buildAppDir, '**', 'ajv', 'lib', '**', '*.ts'),
path.join(buildAppDir, '**', '*.flow'),
// Example files are not needed
path.join(buildAppDir, '**', 'jimp', 'browser', 'examples'),
// Documentation files are not needed
path.join(buildAppDir, '**', 'JSV', 'jsdoc-toolkit'),
path.join(buildAppDir, '**', 'JSV', 'docs'),
path.join(buildAppDir, '**', 'fluent-ffmpeg', 'doc'),
// Files used as part of prebuilding are not necessary
path.join(buildAppDir, '**', 'registry-js', 'prebuilds'),
path.join(buildAppDir, '**', '*.cc'),
path.join(buildAppDir, '**', '*.o'),
path.join(buildAppDir, '**', '*.c'),
path.join(buildAppDir, '**', '*.h'),
// Remove distributions that are not needed in the binary
path.join(buildAppDir, '**', 'ramda', 'dist'),
path.join(buildAppDir, '**', 'jimp', 'browser'),
path.join(buildAppDir, '**', '@jimp', '**', 'src'),
path.join(buildAppDir, '**', 'nexus', 'src'),
path.join(buildAppDir, '**', 'source-map', 'dist'),
path.join(buildAppDir, '**', 'source-map-js', 'dist'),
path.join(buildAppDir, '**', 'pako', 'dist'),
path.join(buildAppDir, '**', 'node-forge', 'dist'),
path.join(buildAppDir, '**', 'pngjs', 'browser.js'),
path.join(buildAppDir, '**', 'plist', 'dist'),
// Remove yarn locks
path.join(buildAppDir, '**', 'yarn.lock'),
], { force: true })
// 6. Remove any empty directories as a result of the rest of the cleanup
await removeEmptyDirectories(buildAppDir)
}
module.exports = {
cleanup,
}