From 684417e7353d3681851839aa285215e774fb6d93 Mon Sep 17 00:00:00 2001 From: Yaraslau Kurmyza Date: Fri, 22 Mar 2024 16:30:50 +0100 Subject: [PATCH] feat(auth,w-m): Use separate @googleapis/* packages Single big `googleapis` package contained service definitions for all or most of google services, that counted 100mb+ in source files to read during each startup. By using smaller libraries we load only what we need and reduce startup time --- changelog/issue-6761.md | 6 + package.json | 4 +- services/auth/src/main.js | 2 +- .../worker-manager/src/providers/google.js | 11 +- services/worker-manager/test/fakes/google.js | 9 +- yarn.lock | 155 +++++++++--------- 6 files changed, 103 insertions(+), 84 deletions(-) create mode 100644 changelog/issue-6761.md diff --git a/changelog/issue-6761.md b/changelog/issue-6761.md new file mode 100644 index 00000000000..9ad78d0272f --- /dev/null +++ b/changelog/issue-6761.md @@ -0,0 +1,6 @@ +audience: developers +level: patch +reference: issue 6761 +--- + +Switching from `googleapis` package to a smaller `@googleapis/*` libraries to reduce startup time and avoid loading APIs we don't use. diff --git a/package.json b/package.json index a8700ab966c..bdc9a929a50 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,9 @@ "@azure/ms-rest-azure-js": "^2.0.1", "@azure/ms-rest-js": "^2.6.6", "@azure/ms-rest-nodeauth": "^3.0.5", + "@googleapis/compute": "^10.0.0", + "@googleapis/iam": "^16.0.0", + "@googleapis/iamcredentials": "^3.0.0", "@graphql-tools/schema": "^10.0.3", "@octokit/auth-app": "^4.0.7", "@octokit/core": "^3.0.0", @@ -56,7 +59,6 @@ "fast-azure-storage": "^3.1.2", "fast-json-stable-stringify": "^2.0.0", "generate-password": "^1.5.0", - "googleapis": "^73.0.0", "got": "^14.2.1", "graphql": "^16.8.1", "graphql-depth-limit": "^1.1.0", diff --git a/services/auth/src/main.js b/services/auth/src/main.js index 306a0771453..fd58289bf68 100755 --- a/services/auth/src/main.js +++ b/services/auth/src/main.js @@ -15,7 +15,7 @@ import createSignatureValidator from './signaturevalidator.js'; import taskcluster from 'taskcluster-client'; import makeSentryManager from './sentrymanager.js'; import * as libPulse from 'taskcluster-lib-pulse'; -import { google as googleapis } from 'googleapis'; +import googleapis from '@googleapis/iamcredentials'; import assert from 'assert'; import { fileURLToPath } from 'url'; import { syncStaticClients } from './static-clients.js'; diff --git a/services/worker-manager/src/providers/google.js b/services/worker-manager/src/providers/google.js index 14e452ce0ed..74a47691d16 100644 --- a/services/worker-manager/src/providers/google.js +++ b/services/worker-manager/src/providers/google.js @@ -3,7 +3,8 @@ import slugid from 'slugid'; import _ from 'lodash'; import taskcluster from 'taskcluster-client'; import * as uuid from 'uuid'; -import { google } from 'googleapis'; +import gcpCompute from '@googleapis/compute'; +import gcpIam from '@googleapis/iam'; import { ApiError, Provider } from './provider.js'; import { CloudAPI } from './cloudapi.js'; import { WorkerPool, Worker } from '../data.js'; @@ -64,20 +65,20 @@ export class GoogleProvider extends Provider { } this.ownClientEmail = creds.client_email; - const client = google.auth.fromJSON(creds); + const client = gcpCompute.auth.fromJSON(creds); client.scopes = [ 'https://www.googleapis.com/auth/compute', // For configuring instance templates, groups, etc 'https://www.googleapis.com/auth/iam', // For fetching service account name ]; - this.compute = google.compute({ + this.compute = gcpCompute.compute({ version: 'v1', auth: client, }); - this.iam = google.iam({ + this.iam = gcpIam.iam({ version: 'v1', auth: client, }); - this.oauth2 = new google.auth.OAuth2(); + this.oauth2 = new gcpCompute.auth.OAuth2(); } async setup() { diff --git a/services/worker-manager/test/fakes/google.js b/services/worker-manager/test/fakes/google.js index e0016da6846..96fa7f0b9d0 100644 --- a/services/worker-manager/test/fakes/google.js +++ b/services/worker-manager/test/fakes/google.js @@ -1,7 +1,8 @@ import { FakeCloud } from './fake.js'; import { strict as assert } from 'assert'; import slugid from 'slugid'; -import { google } from 'googleapis'; +import google from '@googleapis/compute'; +import gcpIam from '@googleapis/iam'; const WORKER_SERVICE_ACCOUNT_ID = '12345'; const PROJECT = 'testy'; @@ -10,8 +11,8 @@ const PROJECT = 'testy'; * Fake the Google SDK. * * This fakes `google.auth`, `google.compute`, where `google` is imported from the - * `googleapis` package. The results of the fake `google.compute()` and - * `google.iam()` calls are `fake.compute` and `fake.iam`. + * `@googleapis/compute` package. The results of the fake `google.compute()` and + * `gcpIam.iam()` calls are `fake.compute` and `fake.iam`. * * The `google.auth.OAuth2` class is defined by the `FakeOAuth2` class below, and * the instance returned from the constructor is available at `fake.oauth2`. @@ -45,7 +46,7 @@ export class FakeGoogle extends FakeCloud { return this.compute; }); - this.sinon.stub(google, 'iam').callsFake(({ version, auth }) => { + this.sinon.stub(gcpIam, 'iam').callsFake(({ version, auth }) => { assert.equal(version, 'v1'); assert(auth.fake); assert.deepEqual(auth.scopes, [ diff --git a/yarn.lock b/yarn.lock index 3b57ffeae9e..084ef79cd08 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1503,6 +1503,33 @@ __metadata: languageName: node linkType: hard +"@googleapis/compute@npm:^10.0.0": + version: 10.0.0 + resolution: "@googleapis/compute@npm:10.0.0" + dependencies: + googleapis-common: "npm:^7.0.0" + checksum: 10c0/8291c14a6d8c3016fa359e01b4b2d29c15403d7fb6c55f517804b0d1c6f2a3c899666997cb16c95aa1d991bb9f5a5a43733b5de4d71982a25deadc03dd1c51fd + languageName: node + linkType: hard + +"@googleapis/iam@npm:^16.0.0": + version: 16.0.0 + resolution: "@googleapis/iam@npm:16.0.0" + dependencies: + googleapis-common: "npm:^7.0.0" + checksum: 10c0/2cb7478294455c44b60252fd36e12b345c5209f54193234b571347317168010ae64a18c5cd4c789c92c2693c63b87229dfc08622f8be5e0947986e2078026334 + languageName: node + linkType: hard + +"@googleapis/iamcredentials@npm:^3.0.0": + version: 3.0.0 + resolution: "@googleapis/iamcredentials@npm:3.0.0" + dependencies: + googleapis-common: "npm:^7.0.0" + checksum: 10c0/4fad799eef11d21b8a7e06616358ceff3866f4b43e18351b55f3a6dc560b98743d5d02d276b8759429e98db3098cd4465d9fc5ebf898e6cb716d61e5f33d3629 + languageName: node + linkType: hard + "@graphql-tools/merge@npm:^8.4.1": version: 8.4.2 resolution: "@graphql-tools/merge@npm:8.4.2" @@ -3852,7 +3879,7 @@ __metadata: languageName: node linkType: hard -"arrify@npm:^2.0.0, arrify@npm:^2.0.1": +"arrify@npm:^2.0.1": version: 2.0.1 resolution: "arrify@npm:2.0.1" checksum: 10c0/3fb30b5e7c37abea1907a60b28a554d2f0fc088757ca9bf5b684786e583fdf14360721eb12575c1ce6f995282eab936712d3c4389122682eafab0e0b57f78dbb @@ -6113,13 +6140,6 @@ __metadata: languageName: node linkType: hard -"fast-text-encoding@npm:^1.0.0": - version: 1.0.6 - resolution: "fast-text-encoding@npm:1.0.6" - checksum: 10c0/e1d0381bda229c92c7906f63308f3b9caca8c78b732768b1ee16f560089ed21bc159bbe1434138ccd3815931ec8d4785bdade1ad1c45accfdf27ac6606ac67d2 - languageName: node - linkType: hard - "fast-xml-parser@npm:4.2.5": version: 4.2.5 resolution: "fast-xml-parser@npm:4.2.5" @@ -6434,26 +6454,38 @@ __metadata: languageName: node linkType: hard -"gaxios@npm:^4.0.0": - version: 4.3.3 - resolution: "gaxios@npm:4.3.3" +"gaxios@npm:^6.0.0, gaxios@npm:^6.1.1": + version: 6.3.0 + resolution: "gaxios@npm:6.3.0" dependencies: - abort-controller: "npm:^3.0.0" extend: "npm:^3.0.2" - https-proxy-agent: "npm:^5.0.0" + https-proxy-agent: "npm:^7.0.1" is-stream: "npm:^2.0.0" - node-fetch: "npm:^2.6.7" - checksum: 10c0/661001bb428dfdb8fabeb0d4b8290edd5ceff4fa7615ef45f447049a38c6379422eafe537c408c0bde333cdf3249fa9673cf8ee66a0658ee174fb85a728efc04 + node-fetch: "npm:^2.6.9" + checksum: 10c0/89b390d6a3776fcb706dc9ef9728c424595fca15afb6a7c6fb8a164a06708203189cd71e39f1617ece39e1d7ec6964f8882b2cafaf1b823a02175f758c1b8738 languageName: node linkType: hard -"gcp-metadata@npm:^4.2.0": - version: 4.3.1 - resolution: "gcp-metadata@npm:4.3.1" +"gaxios@npm:^6.0.3": + version: 6.5.0 + resolution: "gaxios@npm:6.5.0" dependencies: - gaxios: "npm:^4.0.0" + extend: "npm:^3.0.2" + https-proxy-agent: "npm:^7.0.1" + is-stream: "npm:^2.0.0" + node-fetch: "npm:^2.6.9" + uuid: "npm:^9.0.1" + checksum: 10c0/4c377ab27045aef82228dd41c81c53cf4c0c8de848065a150b82912979bd4f0a804d1270c6a98557f8a37a6c227254b51f7767d80a85baaa866f2c4fa9a7778a + languageName: node + linkType: hard + +"gcp-metadata@npm:^6.1.0": + version: 6.1.0 + resolution: "gcp-metadata@npm:6.1.0" + dependencies: + gaxios: "npm:^6.0.0" json-bigint: "npm:^1.0.0" - checksum: 10c0/1fca413fea44b103443c490895e103e972eab1f18453eabc140de10031ce5f35542d3349f09fa57bd629adf2c202005eda004b7c950272837d55fc7da040ed7d + checksum: 10c0/0f84f8c0b974e79d0da0f3063023486e53d7982ce86c4b5871e4ee3b1fc4e7f76fcc05f6342aa0ded5023f1a499c21ab97743a498b31f3aa299905226d1f66ab languageName: node linkType: hard @@ -6625,55 +6657,31 @@ __metadata: languageName: node linkType: hard -"google-auth-library@npm:^7.0.2, google-auth-library@npm:^7.14.0": - version: 7.14.1 - resolution: "google-auth-library@npm:7.14.1" +"google-auth-library@npm:^9.7.0": + version: 9.9.0 + resolution: "google-auth-library@npm:9.9.0" dependencies: - arrify: "npm:^2.0.0" base64-js: "npm:^1.3.0" ecdsa-sig-formatter: "npm:^1.0.11" - fast-text-encoding: "npm:^1.0.0" - gaxios: "npm:^4.0.0" - gcp-metadata: "npm:^4.2.0" - gtoken: "npm:^5.0.4" + gaxios: "npm:^6.1.1" + gcp-metadata: "npm:^6.1.0" + gtoken: "npm:^7.0.0" jws: "npm:^4.0.0" - lru-cache: "npm:^6.0.0" - checksum: 10c0/30b632cfbc312701b4d1b750effbf3575bd826dc2241c8ee4526e8632cef9ee4593d9b3c0b0076a264a79864b13805b6c74c1b13171dfb8eb2ea0bd5c7aa0d43 - languageName: node - linkType: hard - -"google-p12-pem@npm:^3.1.3": - version: 3.1.4 - resolution: "google-p12-pem@npm:3.1.4" - dependencies: - node-forge: "npm:^1.3.1" - bin: - gp12-pem: build/src/bin/gp12-pem.js - checksum: 10c0/d40c7fa17bebd60f18a4eb7355582ab25513b641703bc3f39ddf2f90deb051d5a9622eeb46d4e45d925dc2bf0841d61e8ba34ff20f214019f270a03f39b28fd8 + checksum: 10c0/4dcd2b3542fb8cf68db00c46b65a386bdfdc3e4f6e626197e497fe9fbc1b92d9ea1ff2c28fcfe9653a79828e5cc90f904afbb48a6b189e19ea54d209c71efbd6 languageName: node linkType: hard -"googleapis-common@npm:^5.0.2": - version: 5.1.0 - resolution: "googleapis-common@npm:5.1.0" +"googleapis-common@npm:^7.0.0": + version: 7.1.0 + resolution: "googleapis-common@npm:7.1.0" dependencies: extend: "npm:^3.0.2" - gaxios: "npm:^4.0.0" - google-auth-library: "npm:^7.14.0" + gaxios: "npm:^6.0.3" + google-auth-library: "npm:^9.7.0" qs: "npm:^6.7.0" url-template: "npm:^2.0.8" - uuid: "npm:^8.0.0" - checksum: 10c0/c2ebed3e643cb493af917487edb72318ead16f5e1eb27f4ef95b8252fc33bed054de9ea5f9fdbcf844cadf7c4af328f9431679048dd2c9e04db5f816feafac46 - languageName: node - linkType: hard - -"googleapis@npm:^73.0.0": - version: 73.0.0 - resolution: "googleapis@npm:73.0.0" - dependencies: - google-auth-library: "npm:^7.0.2" - googleapis-common: "npm:^5.0.2" - checksum: 10c0/f90b28e3007a4ad52767f61392fd41f14a7d799ffc270b886219ea7a04bfb758ae21631aecfb60a9f071d35e1a01e66263effe151e51facf42f485a84ad755eb + uuid: "npm:^9.0.0" + checksum: 10c0/6ef780c3b26a41cb4d4934978271c39154264d1843901021593c4ef72f9fce02bfbab6a837bf45019ce1612cab3c57f1742da5f28ce50e0f4d1ff22404d8b45e languageName: node linkType: hard @@ -6820,14 +6828,13 @@ __metadata: languageName: node linkType: hard -"gtoken@npm:^5.0.4": - version: 5.3.2 - resolution: "gtoken@npm:5.3.2" +"gtoken@npm:^7.0.0": + version: 7.1.0 + resolution: "gtoken@npm:7.1.0" dependencies: - gaxios: "npm:^4.0.0" - google-p12-pem: "npm:^3.1.3" + gaxios: "npm:^6.0.0" jws: "npm:^4.0.0" - checksum: 10c0/c4a6893cda5a4abe3967e15e0b14f292ebbcc6c0d186bb062ff947cb41f7a2e440c1a451ac94e944aa903a51db1be86e5c2ebf837d9a70a66ae1088bcf5fa3ff + checksum: 10c0/0a3dcacb1a3c4578abe1ee01c7d0bf20bffe8ded3ee73fc58885d53c00f6eb43b4e1372ff179f0da3ed5cfebd5b7c6ab8ae2776f1787e90d943691b4fe57c716 languageName: node linkType: hard @@ -8901,7 +8908,7 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:^2.6.0, node-fetch@npm:^2.6.7": +"node-fetch@npm:^2.6.0, node-fetch@npm:^2.6.7, node-fetch@npm:^2.6.9": version: 2.7.0 resolution: "node-fetch@npm:2.7.0" dependencies: @@ -8915,7 +8922,7 @@ __metadata: languageName: node linkType: hard -"node-forge@npm:^1.0.0, node-forge@npm:^1.3.1": +"node-forge@npm:^1.0.0": version: 1.3.1 resolution: "node-forge@npm:1.3.1" checksum: 10c0/e882819b251a4321f9fc1d67c85d1501d3004b4ee889af822fd07f64de3d1a8e272ff00b689570af0465d65d6bf5074df9c76e900e0aff23e60b847f2a46fbe8 @@ -10005,7 +10012,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.11.0, qs@npm:^6.7.0, qs@npm:^6.9.4": +"qs@npm:^6.11.0, qs@npm:^6.9.4": version: 6.11.2 resolution: "qs@npm:6.11.2" dependencies: @@ -10014,12 +10021,12 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.12.0": - version: 6.12.0 - resolution: "qs@npm:6.12.0" +"qs@npm:^6.12.0, qs@npm:^6.7.0": + version: 6.12.1 + resolution: "qs@npm:6.12.1" dependencies: side-channel: "npm:^1.0.6" - checksum: 10c0/e165a77ac5f3ca60c15c5f3d51b321ddec7aa438804436b29d160117bc6fb7bf7dab94abd0c7d7c0785890d3a75ae41e1d6346e158aaf1540c6fe53a31f11675 + checksum: 10c0/439e6d7c6583e7c69f2cab2c39c55b97db7ce576e4c7c469082b938b7fc8746e8d547baacb69b4cd2b6666484776c3f4840ad7163a4c5326300b0afa0acdd84b languageName: node linkType: hard @@ -11604,6 +11611,9 @@ __metadata: "@azure/ms-rest-azure-js": "npm:^2.0.1" "@azure/ms-rest-js": "npm:^2.6.6" "@azure/ms-rest-nodeauth": "npm:^3.0.5" + "@googleapis/compute": "npm:^10.0.0" + "@googleapis/iam": "npm:^16.0.0" + "@googleapis/iamcredentials": "npm:^3.0.0" "@graphql-tools/schema": "npm:^10.0.3" "@octokit/auth-app": "npm:^4.0.7" "@octokit/core": "npm:^3.0.0" @@ -11650,7 +11660,6 @@ __metadata: generate-password: "npm:^1.5.0" github-slugger: "npm:^2.0.0" glob: "npm:^8.0.3" - googleapis: "npm:^73.0.0" got: "npm:^14.2.1" graphql: "npm:^16.8.1" graphql-depth-limit: "npm:^1.1.0" @@ -12175,7 +12184,7 @@ __metadata: languageName: node linkType: hard -"uuid@npm:^8.0.0, uuid@npm:^8.3.2": +"uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" bin: