/
binary-cleanup.js
240 lines (209 loc) · 10.1 KB
/
binary-cleanup.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
const fs = require('fs-extra')
const path = require('path')
const { consolidateDeps } = require('@tooling/v8-snapshot')
const del = require('del')
const esbuild = require('esbuild')
const snapshotMetadata = require('@tooling/v8-snapshot/cache/prod-darwin/snapshot-meta.cache.json')
const tempDir = require('temp-dir')
const workingDir = path.join(tempDir, 'binary-cleanup-workdir')
const bytenode = require('bytenode')
fs.ensureDirSync(workingDir)
async function removeEmptyDirectories (directory) {
// lstat does not follow symlinks (in contrast to stat)
const fileStats = await fs.lstat(directory)
if (!fileStats.isDirectory()) {
return
}
let fileNames = await fs.readdir(directory)
if (fileNames.length > 0) {
const recursiveRemovalPromises = fileNames.map(
(fileName) => removeEmptyDirectories(path.join(directory, fileName)),
)
await Promise.all(recursiveRemovalPromises)
// re-evaluate fileNames; after deleting subdirectory
// we may have parent directory empty now
fileNames = await fs.readdir(directory)
}
if (fileNames.length === 0) {
await fs.rmdir(directory)
}
}
const getDependencyPathsToKeep = async (buildAppDir) => {
const unixBuildAppDir = buildAppDir.split(path.sep).join(path.posix.sep)
const startingEntryPoints = [
'packages/server/lib/plugins/child/require_async_child.js',
'packages/server/lib/plugins/child/register_ts_node.js',
'packages/rewriter/lib/threads/worker.js',
'node_modules/webpack/lib/webpack.js',
'node_modules/webpack-dev-server/lib/Server.js',
'node_modules/html-webpack-plugin-4/index.js',
'node_modules/html-webpack-plugin-5/index.js',
'node_modules/mocha-7.0.1/index.js',
]
let entryPoints = new Set([
...startingEntryPoints.map((entryPoint) => path.join(unixBuildAppDir, entryPoint)),
// These dependencies are completely dynamic using the pattern `require(`./${name}`)` and will not be pulled in by esbuild but still need to be kept in the binary.
...['ibmi',
'sunos',
'android',
'darwin',
'freebsd',
'linux',
'openbsd',
'sunos',
'win32'].map((platform) => path.join(unixBuildAppDir, `node_modules/default-gateway/${platform}.js`)),
])
let esbuildResult
let newEntryPointsFound = true
// The general idea here is to run esbuild on entry points that are used outside of the snapshot. If, during the process,
// we find places where we do a require.resolve on a module, that should be treated as an additional entry point and we run
// esbuild again. We do this until we no longer find any new entry points. The resulting metafile inputs are
// the dependency paths that we need to ensure stay in the snapshot.
while (newEntryPointsFound) {
esbuildResult = await esbuild.build({
entryPoints: [...entryPoints],
bundle: true,
outdir: workingDir,
platform: 'node',
metafile: true,
absWorkingDir: unixBuildAppDir,
external: [
'./transpile-ts',
'./start-cypress',
'fsevents',
'pnpapi',
'@swc/core',
'emitter',
],
})
newEntryPointsFound = false
esbuildResult.warnings.forEach((warning) => {
const matches = warning.text.match(/"(.*)" should be marked as external for use with "require.resolve"/)
const warningSubject = matches && matches[1]
if (warningSubject) {
let entryPoint
if (warningSubject.startsWith('.')) {
entryPoint = path.join(unixBuildAppDir, path.dirname(warning.location.file), warningSubject)
} else {
entryPoint = require.resolve(warningSubject, { paths: [path.join(unixBuildAppDir, path.dirname(warning.location.file))] })
}
if (path.extname(entryPoint) !== '' && !entryPoints.has(entryPoint)) {
newEntryPointsFound = true
entryPoints.add(entryPoint)
}
}
})
}
return [...Object.keys(esbuildResult.metafile.inputs), ...entryPoints, 'package.json']
}
const createServerEntryPointBundle = async (buildAppDir) => {
const unixBuildAppDir = buildAppDir.split(path.sep).join(path.posix.sep)
const entryPoints = [path.join(unixBuildAppDir, 'packages/server/index.js')]
// Build the binary entry point ignoring anything that happens in start-cypress since that will be in the v8 snapshot
const esbuildResult = await esbuild.build({
entryPoints,
bundle: true,
outdir: workingDir,
platform: 'node',
metafile: true,
absWorkingDir: unixBuildAppDir,
external: [
'./transpile-ts',
'./start-cypress',
],
})
console.log(`copying server entry point bundle from ${path.join(workingDir, 'index.js')} to ${path.join(buildAppDir, 'packages', 'server', 'index.js')}`)
await fs.copy(path.join(workingDir, 'index.js'), path.join(buildAppDir, 'packages', 'server', 'index.js'))
console.log(`compiling server entry point bundle to ${path.join(buildAppDir, 'packages', 'server', 'index.jsc')}`)
// Use bytenode to compile the entry point bundle. This will save time on the v8 compile step and ensure the integrity of the entry point
await bytenode.compileFile({
filename: path.join(buildAppDir, 'packages', 'server', 'index.js'),
output: path.join(buildAppDir, 'packages', 'server', 'index.jsc'),
electron: true,
})
// Convert these inputs to a relative file path. Note that these paths are posix paths.
return [...Object.keys(esbuildResult.metafile.inputs)].map((input) => `./${input}`)
}
const buildEntryPointAndCleanup = async (buildAppDir) => {
const [keptDependencies, serverEntryPointBundleDependencies] = await Promise.all([
// 1. Retrieve all dependencies that still need to be kept in the binary. In theory, we could use the bundles generated here as single files within the binary,
// but for now, we just track on the dependencies that get pulled in
getDependencyPathsToKeep(buildAppDir),
// 2. Create a bundle for the server entry point. This will be used to start the server in the binary. It returns the dependencies that are pulled in by this bundle that potentially can now be removed
createServerEntryPointBundle(buildAppDir),
])
// 3. Gather the dependencies that could potentially be removed from the binary due to being in the snapshot or in the entry point bundle
const potentiallyRemovedDependencies = [
...snapshotMetadata.healthy,
...snapshotMetadata.deferred,
...snapshotMetadata.norewrite,
...serverEntryPointBundleDependencies,
]
console.log(`potentially removing ${potentiallyRemovedDependencies.length} dependencies`)
// 4. Remove all dependencies that are in the snapshot but not in the list of kept dependencies from the binary
await Promise.all(potentiallyRemovedDependencies.map(async (dependency) => {
const typeScriptlessDependency = dependency.replace(/\.ts$/, '.js')
// marionette-client and babel/runtime require all of their dependencies in a very non-standard dynamic way. We will keep anything in marionette-client and babel/runtime
if (!keptDependencies.includes(typeScriptlessDependency.slice(2)) && !typeScriptlessDependency.includes('marionette-client') && !typeScriptlessDependency.includes('@babel/runtime')) {
await fs.remove(path.join(buildAppDir, typeScriptlessDependency))
}
}))
// 5. Consolidate dependencies that are safe to consolidate (`lodash` and `bluebird`)
await consolidateDeps({ projectBaseDir: buildAppDir })
// 6. Remove various unnecessary files from the binary to further clean things up. Likely, there is additional work that can be done here
await del([
// Remove test files
path.join(buildAppDir, '**', 'test'),
path.join(buildAppDir, '**', 'tests'),
// What we need of prettier is entirely encapsulated within the v8 snapshot, but has a few leftover large files
path.join(buildAppDir, '**', 'prettier', 'esm'),
path.join(buildAppDir, '**', 'prettier', 'standalone.js'),
path.join(buildAppDir, '**', 'prettier', 'bin-prettier.js'),
// ESM files are mostly not needed currently
path.join(buildAppDir, '**', '@babel', '**', 'esm'),
path.join(buildAppDir, '**', 'ramda', 'es'),
path.join(buildAppDir, '**', 'jimp', 'es'),
path.join(buildAppDir, '**', '@jimp', '**', 'es'),
path.join(buildAppDir, '**', 'nexus', 'dist-esm'),
path.join(buildAppDir, '**', '@graphql-tools', '**', '*.mjs'),
path.join(buildAppDir, '**', 'graphql', '**', '*.mjs'),
// We currently do not use any map files
path.join(buildAppDir, '**', '*js.map'),
// License files need to be kept
path.join(buildAppDir, '**', '!(LICENSE|license|License).md'),
// These are type related files that are not used within the binary
path.join(buildAppDir, '**', '*.d.ts'),
path.join(buildAppDir, '**', 'ajv', 'lib', '**', '*.ts'),
path.join(buildAppDir, '**', '*.flow'),
// Example files are not needed
path.join(buildAppDir, '**', 'jimp', 'browser', 'examples'),
// Documentation files are not needed
path.join(buildAppDir, '**', 'JSV', 'jsdoc-toolkit'),
path.join(buildAppDir, '**', 'JSV', 'docs'),
path.join(buildAppDir, '**', 'fluent-ffmpeg', 'doc'),
// Files used as part of prebuilding are not necessary
path.join(buildAppDir, '**', 'registry-js', 'prebuilds'),
path.join(buildAppDir, '**', '*.cc'),
path.join(buildAppDir, '**', '*.o'),
path.join(buildAppDir, '**', '*.c'),
path.join(buildAppDir, '**', '*.h'),
// Remove distributions that are not needed in the binary
path.join(buildAppDir, '**', 'ramda', 'dist'),
path.join(buildAppDir, '**', 'jimp', 'browser'),
path.join(buildAppDir, '**', '@jimp', '**', 'src'),
path.join(buildAppDir, '**', 'nexus', 'src'),
path.join(buildAppDir, '**', 'source-map', 'dist'),
path.join(buildAppDir, '**', 'source-map-js', 'dist'),
path.join(buildAppDir, '**', 'pako', 'dist'),
path.join(buildAppDir, '**', 'node-forge', 'dist'),
path.join(buildAppDir, '**', 'pngjs', 'browser.js'),
path.join(buildAppDir, '**', 'plist', 'dist'),
// Remove yarn locks
path.join(buildAppDir, '**', 'yarn.lock'),
], { force: true })
// 7. Remove any empty directories as a result of the rest of the cleanup
await removeEmptyDirectories(buildAppDir)
}
module.exports = {
buildEntryPointAndCleanup,
}