From 8717527d3cac853e8953da297d28ec36eded1126 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Mon, 15 Aug 2022 19:12:13 +0000 Subject: [PATCH 01/25] feat: implement parallel operations --- src/transfer-manager.ts | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/transfer-manager.ts diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts new file mode 100644 index 000000000..29c2fc618 --- /dev/null +++ b/src/transfer-manager.ts @@ -0,0 +1,42 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {UploadCallback, UploadResponse} from './bucket'; +import {File} from './file'; + +export interface UploadMultiOptions { + concurrencyLimit?: number; +} + +export class TransferManager { + constructor() {} + + uploadMulti( + files: File[], + options?: UploadMultiOptions + ): Promise; + uploadMulti(files: File[], callback: UploadCallback): void; + uploadMulti( + files: File[], + options: UploadMultiOptions, + callback: UploadCallback + ): void; + uploadMulti( + files: File[], + optionsOrCallback?: UploadMultiOptions | UploadCallback, + callback?: UploadCallback + ): Promise | void {} +} From 22097dfcc71a616817be9f5ed1c852e44e7755ac Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 15 Sep 2022 20:55:59 +0000 Subject: [PATCH 02/25] add more parallel operations --- src/index.ts | 6 + src/transfer-manager.ts | 275 +++++++++++++++++++++++++++++++-- test/transfer-manager.ts | 319 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 585 insertions(+), 15 deletions(-) create mode 100644 test/transfer-manager.ts diff --git a/src/index.ts b/src/index.ts index 0f8210aa9..e09060711 100644 --- a/src/index.ts +++ b/src/index.ts @@ -256,3 +256,9 @@ export { StorageOptions, } from './storage'; export {GetSignedUrlCallback, GetSignedUrlResponse} from './signer'; +export { + TransferManager, + UploadMultiOptions, + UploadMultiCallback, + LargeFileDownloadOptions, +} from './transfer-manager'; diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 29c2fc618..4da384483 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -14,29 +14,274 @@ * limitations under the License. */ -import {UploadCallback, UploadResponse} from './bucket'; -import {File} from './file'; +import {Bucket, UploadOptions, UploadResponse} from './bucket'; +import { + DownloadCallback, + DownloadOptions, + DownloadResponse, + File, +} from './file'; +import * as pLimit from 'p-limit'; +import {Metadata} from './nodejs-common'; +import * as path from 'path'; +import * as extend from 'extend'; +import * as fs from 'fs/promises'; +const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; +const DEFAULT_PARALLEL_LARGE_FILE_DOWNLOAD_LIMIT = 2; +const LARGE_FILE_SIZE_THRESHOLD = 256 * 1024 * 1024; +const LARGE_FILE_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; +const EMPTY_REGEX = '(?:)'; export interface UploadMultiOptions { concurrencyLimit?: number; + skipIfExists?: boolean; + prefix?: string; + passthroughOptions?: UploadOptions; +} + +export interface DownloadMultiOptions { + concurrencyLimit?: number; + prefix?: string; + stripPrefix?: string; + passthroughOptions?: DownloadOptions; +} + +export interface LargeFileDownloadOptions { + concurrencyLimit?: number; + chunkSizeBytes?: number; +} + +export interface UploadMultiCallback { + (err: Error | null, files?: File[], metadata?: Metadata[]): void; +} + +export interface DownloadMultiCallback { + (err: Error | null, contents?: Buffer[]): void; } export class TransferManager { - constructor() {} + bucket: Bucket; + constructor(bucket: Bucket) { + this.bucket = bucket; + } - uploadMulti( - files: File[], + async uploadMulti( + filePaths: string[], options?: UploadMultiOptions - ): Promise; - uploadMulti(files: File[], callback: UploadCallback): void; - uploadMulti( - files: File[], + ): Promise; + async uploadMulti( + filePaths: string[], + callback: UploadMultiCallback + ): Promise; + async uploadMulti( + filePaths: string[], options: UploadMultiOptions, - callback: UploadCallback - ): void; - uploadMulti( + callback: UploadMultiCallback + ): Promise; + async uploadMulti( + filePaths: string[], + optionsOrCallback?: UploadMultiOptions | UploadMultiCallback, + callback?: UploadMultiCallback + ): Promise { + let options: UploadMultiOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + if (options.skipIfExists && options.passthroughOptions?.preconditionOpts) { + options.passthroughOptions.preconditionOpts.ifGenerationMatch = 0; + } else if ( + options.skipIfExists && + options.passthroughOptions === undefined + ) { + options.passthroughOptions = { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }; + } + + const limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT + ); + const promises = []; + + for (const filePath of filePaths) { + const baseFileName = path.basename(filePath); + const passThroughOptionsCopy = extend( + true, + {}, + options.passthroughOptions + ); + if (options.prefix) { + passThroughOptionsCopy.destination = path.join( + options.prefix, + baseFileName + ); + } + promises.push( + limit(() => this.bucket.upload(filePath, passThroughOptionsCopy)) + ); + } + + if (callback) { + try { + const results = await Promise.all(promises); + const files = results.map(fileAndMetadata => fileAndMetadata[0]); + const metadata = results.map(fileAndMetadata => fileAndMetadata[1]); + callback(null, files, metadata); + } catch (e) { + callback(e as Error); + } + return; + } + + return Promise.all(promises); + } + + async downloadMulti( + files: File[], + options?: DownloadMultiOptions + ): Promise; + async downloadMulti( + files: File[], + callback: DownloadMultiCallback + ): Promise; + async downloadMulti( + files: File[], + options: DownloadMultiOptions, + callback: DownloadMultiCallback + ): Promise; + async downloadMulti( files: File[], - optionsOrCallback?: UploadMultiOptions | UploadCallback, - callback?: UploadCallback - ): Promise | void {} + optionsOrCallback?: DownloadMultiOptions | DownloadMultiCallback, + callback?: DownloadMultiCallback + ): Promise { + let options: DownloadMultiOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + const limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT + ); + const promises = []; + + const stripRegexString = options.stripPrefix + ? `^${options.stripPrefix}` + : EMPTY_REGEX; + const regex = new RegExp(stripRegexString, 'g'); + + for (const file of files) { + const passThroughOptionsCopy = extend( + true, + {}, + options.passthroughOptions + ); + if (options.prefix) { + passThroughOptionsCopy.destination = path.join( + options.prefix || '', + passThroughOptionsCopy.destination || '', + file.name + ); + } + if (options.stripPrefix) { + passThroughOptionsCopy.destination = file.name.replace(regex, ''); + } + promises.push(limit(() => file.download(passThroughOptionsCopy))); + } + + if (callback) { + try { + const results = await Promise.all(promises); + callback(null, ...results); + } catch (e) { + callback(e as Error); + } + return; + } + + return Promise.all(promises); + } + + async downloadLargeFile( + file: File, + options?: LargeFileDownloadOptions + ): Promise; + async downloadLargeFile( + file: File, + callback: DownloadCallback + ): Promise; + async downloadLargeFile( + file: File, + options: LargeFileDownloadOptions, + callback: DownloadCallback + ): Promise; + async downloadLargeFile( + file: File, + optionsOrCallback?: LargeFileDownloadOptions | DownloadCallback, + callback?: DownloadCallback + ): Promise { + let options: LargeFileDownloadOptions = {}; + if (typeof optionsOrCallback === 'function') { + callback = optionsOrCallback; + } else if (optionsOrCallback) { + options = optionsOrCallback; + } + + let chunkSize = options.chunkSizeBytes || LARGE_FILE_DEFAULT_CHUNK_SIZE; + let limit = pLimit( + options.concurrencyLimit || DEFAULT_PARALLEL_LARGE_FILE_DOWNLOAD_LIMIT + ); + const promises = []; + + const fileInfo = await file.get(); + const size = parseInt(fileInfo[0].metadata.size); + // If the file size does not meet the threshold download it as a single chunk. + if (size < LARGE_FILE_SIZE_THRESHOLD) { + limit = pLimit(1); + chunkSize = size; + } + + let start = 0; + const fileToWrite = await fs.open(path.basename(file.name), 'w+'); + while (start < size) { + const chunkStart = start; + let chunkEnd = start + chunkSize - 1; + chunkEnd = chunkEnd > size ? size : chunkEnd; + promises.push( + limit(() => + file.download({start: chunkStart, end: chunkEnd}).then(resp => { + return fileToWrite.write(resp[0], 0, resp[0].length, chunkStart); + }) + ) + ); + + start += chunkSize; + } + + if (callback) { + try { + const results = await Promise.all(promises); + callback(null, Buffer.concat(results.map(result => result.buffer))); + } catch (e) { + callback(e as Error, Buffer.alloc(0)); + } + await fileToWrite.close(); + return; + } + + return Promise.all(promises) + .then(results => { + return results.map(result => result.buffer) as DownloadResponse; + }) + .finally(async () => { + await fileToWrite.close(); + }); + } } diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts new file mode 100644 index 000000000..c3d63f009 --- /dev/null +++ b/test/transfer-manager.ts @@ -0,0 +1,319 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { + Metadata, + ServiceObject, + ServiceObjectConfig, + util, +} from '../src/nodejs-common'; +import * as pLimit from 'p-limit'; +import * as proxyquire from 'proxyquire'; +import { + Bucket, + CRC32C, + CreateWriteStreamOptions, + DownloadOptions, + File, + FileOptions, + IdempotencyStrategy, + UploadOptions, +} from '../src'; +import * as assert from 'assert'; +import * as path from 'path'; +import * as stream from 'stream'; +import * as extend from 'extend'; +import * as fs from 'fs/promises'; + +const fakeUtil = Object.assign({}, util); +fakeUtil.noop = util.noop; + +class FakeServiceObject extends ServiceObject { + calledWith_: IArguments; + constructor(config: ServiceObjectConfig) { + super(config); + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + } +} + +class FakeAcl { + calledWith_: Array<{}>; + constructor(...args: Array<{}>) { + this.calledWith_ = args; + } +} + +class FakeFile { + calledWith_: IArguments; + bucket: Bucket; + name: string; + options: FileOptions; + metadata: {}; + createWriteStream: Function; + isSameFile = () => false; + constructor(bucket: Bucket, name: string, options?: FileOptions) { + // eslint-disable-next-line prefer-rest-params + this.calledWith_ = arguments; + this.bucket = bucket; + this.name = name; + this.options = options || {}; + this.metadata = {}; + + this.createWriteStream = (options: CreateWriteStreamOptions) => { + this.metadata = options.metadata; + const ws = new stream.Writable(); + ws.write = () => { + ws.emit('complete'); + ws.end(); + return true; + }; + return ws; + }; + } +} + +class HTTPError extends Error { + code: number; + constructor(message: string, code: number) { + super(message); + this.code = code; + } +} + +let pLimitOverride: Function | null; +const fakePLimit = (limit: number) => (pLimitOverride || pLimit)(limit); +const fakeFs = extend(true, {}, fs, { + open: () => { + return { + close: () => {}, + write: (buffer: Buffer) => { + return Promise.resolve({buffer}); + }, + }; + }, +}); + +describe('Transfer Manager', () => { + let TransferManager: any; + let transferManager: any; + let Bucket: any; + let bucket: any; + let File: any; + + const STORAGE: any = { + createBucket: util.noop, + retryOptions: { + autoRetry: true, + maxRetries: 3, + retryDelayMultipier: 2, + totalTimeout: 600, + maxRetryDelay: 60, + retryableErrorFn: (err: HTTPError) => { + return err.code === 500; + }, + idempotencyStrategy: IdempotencyStrategy.RetryConditional, + }, + crc32cGenerator: () => new CRC32C(), + }; + const BUCKET_NAME = 'test-bucket'; + + before(() => { + Bucket = proxyquire('../src/bucket.js', { + 'p-limit': fakePLimit, + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + './acl.js': {Acl: FakeAcl}, + './file.js': {File: FakeFile}, + }).Bucket; + + File = proxyquire('../src/file.js', { + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + }).File; + + TransferManager = proxyquire('../src/transfer-manager.js', { + 'p-limit': fakePLimit, + './nodejs-common': { + ServiceObject: FakeServiceObject, + util: fakeUtil, + }, + './acl.js': {Acl: FakeAcl}, + './file.js': {File: FakeFile}, + 'fs/promises': fakeFs, + }).TransferManager; + }); + + beforeEach(() => { + bucket = new Bucket(STORAGE, BUCKET_NAME); + transferManager = new TransferManager(bucket); + }); + + describe('instantiation', () => { + it('should correctly set the bucket', () => { + assert.strictEqual(transferManager.bucket, bucket); + }); + }); + + describe('uploadMulti', () => { + it('calls upload with the provided file paths', async () => { + const paths = ['/a/b/c', '/d/e/f', '/h/i/j']; + let count = 0; + + bucket.upload = (path: string) => { + count++; + assert(paths.includes(path)); + }; + + await transferManager.uploadMulti(paths); + assert.strictEqual(count, paths.length); + }); + + it('sets ifGenerationMatch to 0 if skipIfExists is set', async () => { + const paths = ['/a/b/c']; + + bucket.upload = (_path: string, options: UploadOptions) => { + assert.strictEqual(options.preconditionOpts?.ifGenerationMatch, 0); + }; + + await transferManager.uploadMulti(paths, {skipIfExists: true}); + }); + + it('sets destination to prefix + filename when prefix is supplied', async () => { + const paths = ['/a/b/foo/bar.txt']; + const expectedDestination = 'hello/world/bar.txt'; + + bucket.upload = (_path: string, options: UploadOptions) => { + assert.strictEqual(options.destination, expectedDestination); + }; + + await transferManager.uploadMulti(paths, {prefix: 'hello/world'}); + }); + + it('invokes the callback if one is provided', done => { + const basename = 'testfile.json'; + const paths = [path.join(__dirname, '../../test/testdata/' + basename)]; + + transferManager.uploadMulti( + paths, + (err: Error | null, files?: File[], metadata?: Metadata[]) => { + assert.ifError(err); + assert(files); + assert(metadata); + assert.strictEqual(files[0].name, basename); + done(); + } + ); + }); + + it('returns a promise with the uploaded file if there is no callback', async () => { + const basename = 'testfile.json'; + const paths = [path.join(__dirname, '../../test/testdata/' + basename)]; + const result = await transferManager.uploadMulti(paths); + assert.strictEqual(result[0][0].name, basename); + }); + }); + + describe('downloadMulti', () => { + it('calls download for each provided file', async () => { + let count = 0; + const download = () => { + count++; + }; + const firstFile = new File(bucket, 'first.txt'); + firstFile.download = download; + const secondFile = new File(bucket, 'second.txt'); + secondFile.download = download; + + const files = [firstFile, secondFile]; + await transferManager.downloadMulti(files); + assert.strictEqual(count, 2); + }); + + it('sets the destination correctly when provided a prefix', async () => { + const prefix = 'test-prefix'; + const filename = 'first.txt'; + const expectedDestination = `${prefix}/${filename}`; + const download = (options: DownloadOptions) => { + assert.strictEqual(options.destination, expectedDestination); + }; + + const file = new File(bucket, filename); + file.download = download; + await transferManager.downloadMulti([file], {prefix}); + }); + + it('sets the destination correctly when provided a strip prefix', async () => { + const stripPrefix = 'should-be-removed/'; + const filename = 'should-be-removed/first.txt'; + const expectedDestination = 'first.txt'; + const download = (options: DownloadOptions) => { + assert.strictEqual(options.destination, expectedDestination); + }; + + const file = new File(bucket, filename); + file.download = download; + await transferManager.downloadMulti([file], {stripPrefix}); + }); + + it('invokes the callback if one if provided', done => { + const file = new File(bucket, 'first.txt'); + file.download = () => { + return Promise.resolve(Buffer.alloc(100)); + }; + transferManager.downloadMulti( + [file], + (err: Error | null, contents: Buffer[]) => { + assert.strictEqual(err, null); + assert.strictEqual(contents.length, 100); + done(); + } + ); + }); + }); + + describe('downloadLargeFile', () => { + let file: any; + + beforeEach(() => { + file = new File(bucket, 'some-large-file'); + file.get = () => { + return [ + { + metadata: { + size: 1024, + }, + }, + ]; + }; + }); + + it('should download a single chunk if file size is below threshold', async () => { + let downloadCallCount = 0; + file.download = () => { + downloadCallCount++; + return Promise.resolve([Buffer.alloc(100)]); + }; + + await transferManager.downloadLargeFile(file); + assert.strictEqual(downloadCallCount, 1); + }); + + it('invokes the callback if one is provided', done => { + file.download = () => { + return Promise.resolve([Buffer.alloc(100)]); + }; + + transferManager.downloadLargeFile( + file, + (err: Error, contents: Buffer) => { + assert.equal(err, null); + assert.strictEqual(contents.length, 100); + done(); + } + ); + }); + }); +}); From 9a59f1f61b952d9dddc92afade9dc6f5989a40f4 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 15 Sep 2022 20:58:57 +0000 Subject: [PATCH 03/25] add header to test file --- test/transfer-manager.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index c3d63f009..8f2a1e12f 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -1,3 +1,19 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /* eslint-disable @typescript-eslint/no-explicit-any */ import { Metadata, From d2b241f92576c5f2e3dee470333d5aec2dfc7c11 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 15 Sep 2022 21:12:10 +0000 Subject: [PATCH 04/25] update import of fs/promises --- src/transfer-manager.ts | 2 +- test/transfer-manager.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 4da384483..766b26200 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -25,7 +25,7 @@ import * as pLimit from 'p-limit'; import {Metadata} from './nodejs-common'; import * as path from 'path'; import * as extend from 'extend'; -import * as fs from 'fs/promises'; +import {promises as fs} from 'fs'; const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 8f2a1e12f..c4add8b3c 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -37,7 +37,7 @@ import * as assert from 'assert'; import * as path from 'path'; import * as stream from 'stream'; import * as extend from 'extend'; -import * as fs from 'fs/promises'; +import {promises as fs} from 'fs'; const fakeUtil = Object.assign({}, util); fakeUtil.noop = util.noop; From 30b45d36e71701035d2fd148ebf17cd532ec607b Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Fri, 16 Sep 2022 14:44:47 +0000 Subject: [PATCH 05/25] fix pathing on windows, fix mocking of fs promises --- src/transfer-manager.ts | 4 ++-- test/transfer-manager.ts | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 766b26200..d28fca0dd 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -25,7 +25,7 @@ import * as pLimit from 'p-limit'; import {Metadata} from './nodejs-common'; import * as path from 'path'; import * as extend from 'extend'; -import {promises as fs} from 'fs'; +import {promises as fsp} from 'fs'; const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; @@ -249,7 +249,7 @@ export class TransferManager { } let start = 0; - const fileToWrite = await fs.open(path.basename(file.name), 'w+'); + const fileToWrite = await fsp.open(path.basename(file.name), 'w+'); while (start < size) { const chunkStart = start; let chunkEnd = start + chunkSize - 1; diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index c4add8b3c..f96f4aac2 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -37,7 +37,7 @@ import * as assert from 'assert'; import * as path from 'path'; import * as stream from 'stream'; import * as extend from 'extend'; -import {promises as fs} from 'fs'; +import * as fs from 'fs'; const fakeUtil = Object.assign({}, util); fakeUtil.noop = util.noop; @@ -98,11 +98,15 @@ class HTTPError extends Error { let pLimitOverride: Function | null; const fakePLimit = (limit: number) => (pLimitOverride || pLimit)(limit); const fakeFs = extend(true, {}, fs, { - open: () => { + get promises() { return { - close: () => {}, - write: (buffer: Buffer) => { - return Promise.resolve({buffer}); + open: () => { + return { + close: () => {}, + write: (buffer: Buffer) => { + return Promise.resolve({buffer}); + }, + }; }, }; }, @@ -158,7 +162,8 @@ describe('Transfer Manager', () => { }, './acl.js': {Acl: FakeAcl}, './file.js': {File: FakeFile}, - 'fs/promises': fakeFs, + fs: fakeFs, + fsp: fakeFs, }).TransferManager; }); @@ -199,7 +204,7 @@ describe('Transfer Manager', () => { it('sets destination to prefix + filename when prefix is supplied', async () => { const paths = ['/a/b/foo/bar.txt']; - const expectedDestination = 'hello/world/bar.txt'; + const expectedDestination = path.normalize('hello/world/bar.txt'); bucket.upload = (_path: string, options: UploadOptions) => { assert.strictEqual(options.destination, expectedDestination); @@ -251,7 +256,7 @@ describe('Transfer Manager', () => { it('sets the destination correctly when provided a prefix', async () => { const prefix = 'test-prefix'; const filename = 'first.txt'; - const expectedDestination = `${prefix}/${filename}`; + const expectedDestination = path.normalize(`${prefix}/${filename}`); const download = (options: DownloadOptions) => { assert.strictEqual(options.destination, expectedDestination); }; From 16a5f05a8161e5a8264e7db364073889a45eca33 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Fri, 16 Sep 2022 21:29:38 +0000 Subject: [PATCH 06/25] add jsdoc headers to class and uploadMulti --- src/transfer-manager.ts | 62 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index d28fca0dd..eb3bc07be 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -60,6 +60,14 @@ export interface DownloadMultiCallback { (err: Error | null, contents?: Buffer[]): void; } +/** + * Create a TransferManager object to perform parallel transfer operations on a Cloud Storage bucket. + * + * @class + * @hideconstructor + * + * @param {Bucket} bucket A {@link Bucket} instance + */ export class TransferManager { bucket: Bucket; constructor(bucket: Bucket) { @@ -79,6 +87,60 @@ export class TransferManager { options: UploadMultiOptions, callback: UploadMultiCallback ): Promise; + /** + * @typedef {object} UploadMultiOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when uploading the files. + * @property {boolean} [skipIfExists] Do not upload the file if it already exists in + * the bucket. This will set the precondition ifGenerationMatch = 0. + * @property {string} [prefix] A prefix to append to all of the uploaded files. + * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through + * to each individual upload operation. + */ + /** + * @typedef {array} UploadResponse + * @property {object} The uploaded {@link File} + * @property {object} The uploaded {@link Metadata} + */ + /** + * @callback UploadMultiCallback + * @param {?Error} err Rewuest error if any + * @param {array} files Array of uploaded {@link File}. + * @param {array} metadata Array of uploaded {@link Metadata} + */ + /** + * Upload multiple files in parallel to the bucket. This is a convenience method + * that utilizes {@link Bucket#upload} to perform the upload. + * + * @param {array} [filePaths] An array of fully qualified paths to the files. + * you wish to upload to the bucket + * @param {UploadMultiOptions} [options] Configuration options. + * @param {UploadMultiCallback} [callback] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Upload multiple files. + * //- + * transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt'], function(err, files, metadata) { + * // Your bucket now contains: + * // - "file1.txt" (with the contents of '/local/path/file1.txt') + * // - "file2.txt" (with the contents of '/local/path/file2.txt') + * // `files` is an array of instances of File objects that refers to the new files. + * }); + * + * //- + * // If the callback if omitted, we will return a Promise. + * //- + * const response = transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt']); + * ``` + */ async uploadMulti( filePaths: string[], optionsOrCallback?: UploadMultiOptions | UploadMultiCallback, From 5a94aa7849047164c8ce9f82f0a778280bc86370 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Fri, 30 Sep 2022 17:40:13 +0000 Subject: [PATCH 07/25] add jsdoc comments to remaining functions --- src/transfer-manager.ts | 99 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index eb3bc07be..a3c190ea6 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -97,16 +97,11 @@ export class TransferManager { * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through * to each individual upload operation. */ - /** - * @typedef {array} UploadResponse - * @property {object} The uploaded {@link File} - * @property {object} The uploaded {@link Metadata} - */ /** * @callback UploadMultiCallback - * @param {?Error} err Rewuest error if any - * @param {array} files Array of uploaded {@link File}. - * @param {array} metadata Array of uploaded {@link Metadata} + * @param {?Error} [err] Request error, if any. + * @param {array} [files] Array of uploaded {@link File}. + * @param {array} [metadata] Array of uploaded {@link Metadata} */ /** * Upload multiple files in parallel to the bucket. This is a convenience method @@ -126,19 +121,19 @@ export class TransferManager { * const transferManager = new TransferManager(bucket); * * //- - * // Upload multiple files. + * // Upload multiple files in parallel. * //- * transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt'], function(err, files, metadata) { - * // Your bucket now contains: + * // Your bucket now contains: * // - "file1.txt" (with the contents of '/local/path/file1.txt') * // - "file2.txt" (with the contents of '/local/path/file2.txt') * // `files` is an array of instances of File objects that refers to the new files. * }); * * //- - * // If the callback if omitted, we will return a Promise. + * // If the callback is omitted, we will return a promise. * //- - * const response = transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt']); + * const response = await transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt']); * ``` */ async uploadMulti( @@ -217,6 +212,51 @@ export class TransferManager { options: DownloadMultiOptions, callback: DownloadMultiCallback ): Promise; + /** + * @typedef {object} DownloadMultiOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when downloading the files. + * @property {string} [prefix] A prefix to append to all of the downloaded files. + * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. + * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through + * to each individual download operation. + */ + /** + * @callback DownloadMultiCallback + * @param {?Error} [err] Request error, if any. + * @param {array} [contents] Contents of the downloaded files. + */ + /** + * Download multiple files in parallel to the local filesystem. This is a convenience method + * that utilizes {@link File#download} to perform the download. + * + * @param {array} [files] An array of file objects you wish to download. + * @param {DownloadMultiOptions} [options] Configuration options. + * @param {DownloadMultiCallback} {callback} Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Download multiple files in parallel. + * //- + * transferManager.downloadMulti([bucket.file('file1.txt'), bucket.file('file2.txt')], function(err, contents){ + * // Your local directory now contains: + * // - "file1.txt" (with the contents from my-bucket.file1.txt) + * // - "file2.txt" (with the contents from my-bucket.file2.txt) + * // `contents` is an array containing the file data for each downloaded file. + * }); + * + * //- + * // If the callback is omitted, we will return a promise. + * //- + * const response = await transferManager.downloadMulti(bucket.File('file1.txt'), bucket.File('file2.txt')]); + */ async downloadMulti( files: File[], optionsOrCallback?: DownloadMultiOptions | DownloadMultiCallback, @@ -284,6 +324,41 @@ export class TransferManager { options: LargeFileDownloadOptions, callback: DownloadCallback ): Promise; + /** + * @typedef {object} LargeFileDownloadOptions + * @property {number} [concurrencyLimit] The number of concurrently executing promises + * to use when downloading the file. + * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded. + */ + /** + * Download a large file in chunks utilizing parallel download operations. This is a convenience method + * that utilizes {@link File#download} to perform the download. + * + * @param {object} [file] {@link File} to download. + * @param {LargeFileDownloadOptions} [options] Configuration options. + * @param {DownloadCallback} [callbac] Callback function. + * @returns {Promise} + * + * @example + * ``` + * const {Storage} = require('@google-cloud/storage'); + * const storage = new Storage(); + * const bucket = storage.bucket('my-bucket'); + * const transferManager = new TransferManager(bucket); + * + * //- + * // Download a large file in chunks utilizing parallel operations. + * //- + * transferManager.downloadLargeFile(bucket.file('large-file.txt'), function(err, contents) { + * // Your local directory now contains: + * // - "large-file.txt" (with the contents from my-bucket.large-file.txt) + * }); + * + * //- + * // If the callback is omitted, we will return a promise. + * //- + * const response = await transferManager.downloadLargeFile(bucket.file('large-file.txt'); + */ async downloadLargeFile( file: File, optionsOrCallback?: LargeFileDownloadOptions | DownloadCallback, From 1cc8baeeda2b01a1bd6c7d7ec1be2a67fd0e635e Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Mon, 24 Oct 2022 15:24:43 +0000 Subject: [PATCH 08/25] update comment wording --- src/transfer-manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index a3c190ea6..dae925480 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -108,7 +108,7 @@ export class TransferManager { * that utilizes {@link Bucket#upload} to perform the upload. * * @param {array} [filePaths] An array of fully qualified paths to the files. - * you wish to upload to the bucket + * to be uploaded to the bucket * @param {UploadMultiOptions} [options] Configuration options. * @param {UploadMultiCallback} [callback] Callback function. * @returns {Promise} @@ -230,7 +230,7 @@ export class TransferManager { * Download multiple files in parallel to the local filesystem. This is a convenience method * that utilizes {@link File#download} to perform the download. * - * @param {array} [files] An array of file objects you wish to download. + * @param {array} [files] An array of file objects to be downloaded. * @param {DownloadMultiOptions} [options] Configuration options. * @param {DownloadMultiCallback} {callback} Callback function. * @returns {Promise} From face55f3b32e15afe92501c477d6a8cf968d46af Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 27 Oct 2022 20:15:51 +0000 Subject: [PATCH 09/25] add experimental jsdoc tags --- src/transfer-manager.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index dae925480..2ce5f125f 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -67,6 +67,7 @@ export interface DownloadMultiCallback { * @hideconstructor * * @param {Bucket} bucket A {@link Bucket} instance + * @experimental */ export class TransferManager { bucket: Bucket; @@ -96,12 +97,14 @@ export class TransferManager { * @property {string} [prefix] A prefix to append to all of the uploaded files. * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through * to each individual upload operation. + * @experimental */ /** * @callback UploadMultiCallback * @param {?Error} [err] Request error, if any. * @param {array} [files] Array of uploaded {@link File}. * @param {array} [metadata] Array of uploaded {@link Metadata} + * @experimental */ /** * Upload multiple files in parallel to the bucket. This is a convenience method @@ -135,6 +138,7 @@ export class TransferManager { * //- * const response = await transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt']); * ``` + * @experimental */ async uploadMulti( filePaths: string[], @@ -220,11 +224,13 @@ export class TransferManager { * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through * to each individual download operation. + * @experimental */ /** * @callback DownloadMultiCallback * @param {?Error} [err] Request error, if any. * @param {array} [contents] Contents of the downloaded files. + * @experimental */ /** * Download multiple files in parallel to the local filesystem. This is a convenience method @@ -256,6 +262,7 @@ export class TransferManager { * // If the callback is omitted, we will return a promise. * //- * const response = await transferManager.downloadMulti(bucket.File('file1.txt'), bucket.File('file2.txt')]); + * @experimental */ async downloadMulti( files: File[], @@ -329,6 +336,7 @@ export class TransferManager { * @property {number} [concurrencyLimit] The number of concurrently executing promises * to use when downloading the file. * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded. + * @experimental */ /** * Download a large file in chunks utilizing parallel download operations. This is a convenience method @@ -358,6 +366,7 @@ export class TransferManager { * // If the callback is omitted, we will return a promise. * //- * const response = await transferManager.downloadLargeFile(bucket.file('large-file.txt'); + * @experimental */ async downloadLargeFile( file: File, From 524a31074a60bd5beaf25cd267f940a2b18069b7 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 3 Nov 2022 15:35:09 +0000 Subject: [PATCH 10/25] feat: add directory generator to performance test framework --- internal-tooling/performPerformanceTest.ts | 58 +++-------- internal-tooling/performanceUtils.ts | 108 +++++++++++++++++++++ 2 files changed, 119 insertions(+), 47 deletions(-) create mode 100644 internal-tooling/performanceUtils.ts diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index 946339daa..c0cf0cdc8 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -15,22 +15,25 @@ */ import yargs from 'yargs'; -import * as uuid from 'uuid'; -import {execSync} from 'child_process'; import {unlinkSync} from 'fs'; import {Storage} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; import path = require('path'); +import { + BLOCK_SIZE_IN_BYTES, + DEFAULT_LARGE_FILE_SIZE_BYTES, + DEFAULT_SMALL_FILE_SIZE_BYTES, + generateRandomFile, + generateRandomFileName, + randomInteger, +} from './performanceUtils'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; const DEFAULT_NUMBER_OF_WRITES = 1; const DEFAULT_NUMBER_OF_READS = 3; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics'; -const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; -const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; -const BLOCK_SIZE_IN_BYTES = 1024; const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; export interface TestResult { @@ -46,20 +49,6 @@ export interface TestResult { status: '[OK]'; } -/** - * Create a uniformly distributed random integer beween the inclusive min and max provided. - * - * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. - * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. - * @returns {number} returns a random integer between minInclusive and maxInclusive - */ -const randomInteger = (minInclusive: number, maxInclusive: number) => { - // Utilizing Math.random will generate uniformly distributed random numbers. - return ( - Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive - ); -}; - const argv = yargs(process.argv.slice(2)) .options({ bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, @@ -85,8 +74,8 @@ async function main() { */ async function performWriteReadTest(): Promise { const results: TestResult[] = []; - const fileName = generateRandomFileName(); - const sizeInBytes = generateRandomFile(fileName); + const fileName = generateRandomFileName(TEST_NAME_STRING); + const sizeInBytes = generateRandomFile(fileName, argv.small, argv.large); const checkType = randomInteger(0, 2); const stg = new Storage({ @@ -159,7 +148,7 @@ async function performWriteReadTest(): Promise { status: '[OK]', }; - const destinationFileName = generateRandomFileName(); + const destinationFileName = generateRandomFileName(TEST_NAME_STRING); const destination = path.join(__dirname, destinationFileName); if (checkType === 0) { start = performance.now(); @@ -186,31 +175,6 @@ async function performWriteReadTest(): Promise { return results; } -/** - * Creates a file with a size between the small (default 5120 bytes) and large (2.147e9 bytes) parameters. - * The file is filled with random data. - * - * @param {string} fileName name of the file to generate. - * @returns {number} the size of the file generated. - */ -function generateRandomFile(fileName: string) { - const fileSizeBytes = randomInteger(argv.small, argv.large); - const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES); - const cmd = `dd if=/dev/urandom of=${__dirname}/${fileName} bs=${BLOCK_SIZE_IN_BYTES} count=${numberNeeded} status=none iflag=fullblock`; - execSync(cmd); - - return fileSizeBytes; -} - -/** - * Creates a random file name by appending a UUID to the TEST_NAME_STRING. - * - * @returns {string} random file name that was generated. - */ -function generateRandomFileName(): string { - return `${TEST_NAME_STRING}.${uuid.v4()}`; -} - /** * Deletes the file specified by the fileName parameter. * diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts new file mode 100644 index 000000000..77fab8043 --- /dev/null +++ b/internal-tooling/performanceUtils.ts @@ -0,0 +1,108 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {execSync} from 'child_process'; +import {mkdirSync} from 'fs'; +import path = require('path'); +import * as uuid from 'uuid'; + +export const BLOCK_SIZE_IN_BYTES = 1024; +export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; +export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; + +const CREATE_DIRECTORY = 1; + +/** + * Create a uniformly distributed random integer beween the inclusive min and max provided. + * + * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. + * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. + * @returns {number} returns a random integer between minInclusive and maxInclusive + */ +export function randomInteger(minInclusive: number, maxInclusive: number) { + // Utilizing Math.random will generate uniformly distributed random numbers. + return ( + Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive + ); +} + +/** + * Creates a random file name by appending a UUID to the TEST_NAME_STRING. + * + * @returns {string} random file name that was generated. + */ +export function generateRandomFileName(testName: string): string { + return `${testName}.${uuid.v4()}`; +} + +/** + * Creates a file with a size between the small (default 5120 bytes) and large (2.147e9 bytes) parameters. + * The file is filled with random data. + * + * @param {string} fileName name of the file to generate. + * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. + * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. + * @param {string} currentDirectory the directory in which to generate the file. + * + * @returns {number} the size of the file generated. + */ +export function generateRandomFile( + fileName: string, + fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, + fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES, + currentDirectory: string = __dirname +) { + const fileSizeBytes = randomInteger( + fileSizeLowerBoundBytes, + fileSizeUpperBoundBytes + ); + const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES); + const cmd = `dd if=/dev/urandom of=${currentDirectory}/${fileName} bs=${BLOCK_SIZE_IN_BYTES} count=${numberNeeded} status=none iflag=fullblock`; + execSync(cmd); + + return fileSizeBytes; +} + +/** + * Creates a random directory structure consisting of subdirectories and random files. + * + * @param {number} maxObjects the total number of subdirectories and files to generate. + * @param {string} testName the test name. + * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. + * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. + */ +export function generateRandomDirectoryStructure( + maxObjects: number, + testName: string, + fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, + fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES +) { + let curPath = testName; + for (let i = 0; i < maxObjects; i++) { + const dirOrFile = randomInteger(0, 1); + if (dirOrFile === CREATE_DIRECTORY) { + curPath = path.join(curPath, uuid.v4()); + mkdirSync(curPath, {recursive: true}); + } else { + generateRandomFile( + generateRandomFileName(testName), + fileSizeLowerBoundBytes, + fileSizeUpperBoundBytes, + curPath + ); + } + } +} From bac3ed8bf676aaacfced1d2884b4751e5cb81228 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 3 Nov 2022 17:10:29 +0000 Subject: [PATCH 11/25] clarify variable names and comments --- internal-tooling/performanceUtils.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index 77fab8043..d9d7222bd 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -40,12 +40,14 @@ export function randomInteger(minInclusive: number, maxInclusive: number) { } /** - * Creates a random file name by appending a UUID to the TEST_NAME_STRING. + * Creates a random file name by appending a UUID to the baseName. + * + * @param {string} baseName the base file name. A random uuid will be appended to this value. * * @returns {string} random file name that was generated. */ -export function generateRandomFileName(testName: string): string { - return `${testName}.${uuid.v4()}`; +export function generateRandomFileName(baseName: string): string { + return `${baseName}.${uuid.v4()}`; } /** @@ -80,17 +82,17 @@ export function generateRandomFile( * Creates a random directory structure consisting of subdirectories and random files. * * @param {number} maxObjects the total number of subdirectories and files to generate. - * @param {string} testName the test name. + * @param {string} baseName The starting directory under which everything else is added. File names will have this value prepended. * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. */ export function generateRandomDirectoryStructure( maxObjects: number, - testName: string, + baseName: string, fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES ) { - let curPath = testName; + let curPath = baseName; for (let i = 0; i < maxObjects; i++) { const dirOrFile = randomInteger(0, 1); if (dirOrFile === CREATE_DIRECTORY) { @@ -98,7 +100,7 @@ export function generateRandomDirectoryStructure( mkdirSync(curPath, {recursive: true}); } else { generateRandomFile( - generateRandomFileName(testName), + generateRandomFileName(baseName), fileSizeLowerBoundBytes, fileSizeUpperBoundBytes, curPath From b4bc33324b608c8c4251d2bbdf74ecb04047177c Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 3 Nov 2022 17:15:02 +0000 Subject: [PATCH 12/25] capitalization --- internal-tooling/performanceUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index d9d7222bd..c717afdb8 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -82,7 +82,7 @@ export function generateRandomFile( * Creates a random directory structure consisting of subdirectories and random files. * * @param {number} maxObjects the total number of subdirectories and files to generate. - * @param {string} baseName The starting directory under which everything else is added. File names will have this value prepended. + * @param {string} baseName the starting directory under which everything else is added. File names will have this value prepended. * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. */ From 46687c6407752b0bf8d398bfca0d0a1ab27b0728 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 10 Nov 2022 18:15:04 +0000 Subject: [PATCH 13/25] wip: transfer manager performance tests --- internal-tooling/performPerformanceTest.ts | 12 +- .../performTransferManagerTest.ts | 144 ++++++++++++++++++ internal-tooling/performanceTest.ts | 32 +++- internal-tooling/performanceUtils.ts | 54 +++++-- src/index.ts | 7 +- src/transfer-manager.ts | 7 + 6 files changed, 228 insertions(+), 28 deletions(-) create mode 100644 internal-tooling/performTransferManagerTest.ts diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index c0cf0cdc8..7fa8ad20e 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -23,6 +23,7 @@ import {parentPort} from 'worker_threads'; import path = require('path'); import { BLOCK_SIZE_IN_BYTES, + cleanupFile, DEFAULT_LARGE_FILE_SIZE_BYTES, DEFAULT_SMALL_FILE_SIZE_BYTES, generateRandomFile, @@ -70,7 +71,7 @@ async function main() { /** * Performs an iteration of the Write 1 / Read 3 performance measuring test. * - * @returns {Promise} Promise that resolves to an array of test results for the iteration. */ async function performWriteReadTest(): Promise { const results: TestResult[] = []; @@ -175,13 +176,4 @@ async function performWriteReadTest(): Promise { return results; } -/** - * Deletes the file specified by the fileName parameter. - * - * @param {string} fileName name of the file to delete. - */ -function cleanupFile(fileName: string) { - unlinkSync(`${__dirname}/${fileName}`); -} - main(); diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts new file mode 100644 index 000000000..fb2b24b6f --- /dev/null +++ b/internal-tooling/performTransferManagerTest.ts @@ -0,0 +1,144 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// eslint-disable-next-line node/no-unsupported-features/node-builtins +import {parentPort} from 'worker_threads'; +import yargs from 'yargs'; +import {Bucket, Storage, TransferManager} from '../src'; +import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; +import { + cleanupFile, + DEFAULT_LARGE_FILE_SIZE_BYTES, + DEFAULT_SMALL_FILE_SIZE_BYTES, + generateRandomDirectoryStructure, + generateRandomFile, + generateRandomFileName, +} from './performanceUtils'; + +const TEST_NAME_STRING = 'transfer-manager-perf-metrics'; +const DEFAULT_BUCKET_NAME = 'nodejs-transfer-manager-perf-metrics'; +const DEFAULT_NUMBER_OF_PROMISES = 2; +const DEFAULT_NUMBER_OF_OBJECTS = 1000; +const DEFAULT_CHUNK_SIZE_BYTES = 16 * 1024 * 1024; + +export interface TransferManagerTestResult { + numberOfObjects: number; +} + +let stg: Storage; +let bucket: Bucket; +let tm: TransferManager; + +const argv = yargs(process.argv.slice(2)) + .options({ + bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, + small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, + large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, + numpromises: {type: 'number', default: DEFAULT_NUMBER_OF_PROMISES}, + numobjects: {type: 'number', default: DEFAULT_NUMBER_OF_OBJECTS}, + chunksize: {type: 'number', default: DEFAULT_CHUNK_SIZE_BYTES}, + projectid: {type: 'string'}, + testtype: { + type: 'string', + choices: [ + TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD, + ], + }, + }) + .parseSync(); + +async function main() { + let results: TransferManagerTestResult[] = []; + await performTestSetup(); + + switch (argv.testtype) { + case TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS: + results = await performUploadMultipleObjectsTest(); + break; + case TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS: + results = await performDownloadMultipleObjectsTest(); + break; + case TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD: + results = await performDownloadLargeFileTest(); + break; + default: + break; + } + parentPort?.postMessage(results); +} + +async function performTestSetup() { + stg = new Storage({projectId: argv.projectid}); + bucket = stg.bucket(argv.bucket); + if (!(await bucket.exists())[0]) { + await bucket.create(); + } + tm = new TransferManager(bucket); +} + +async function performUploadMultipleObjectsTest(): Promise< + TransferManagerTestResult[] +> { + const results: TransferManagerTestResult[] = []; + const paths = generateRandomDirectoryStructure( + argv.numobjects, + TEST_NAME_STRING, + argv.small, + argv.large + ); + const uploadResults = await tm.uploadMulti(paths, { + concurrencyLimit: argv.numpromises, + }); + + return results; +} + +async function performDownloadMultipleObjectsTest(): Promise< + TransferManagerTestResult[] +> { + const results: TransferManagerTestResult[] = []; + + return results; +} + +async function performDownloadLargeFileTest(): Promise< + TransferManagerTestResult[] +> { + const results: TransferManagerTestResult[] = []; + try { + const fileName = generateRandomFileName(TEST_NAME_STRING); + generateRandomFile(fileName, argv.small, argv.large, __dirname); + const file = bucket.file(`${fileName}`); + + await bucket.upload(`${__dirname}/${fileName}`); + cleanupFile(fileName); + const start = performance.now(); + await tm.downloadLargeFile(file, { + concurrencyLimit: argv.numpromises, + chunkSizeBytes: argv.chunksize, + }); + const end = performance.now(); + console.log(Math.round((end - start) * 1000)); + } catch (e) { + console.error(e); + } + + return results; +} + +main(); diff --git a/internal-tooling/performanceTest.ts b/internal-tooling/performanceTest.ts index 55c2b6baf..a124eac91 100644 --- a/internal-tooling/performanceTest.ts +++ b/internal-tooling/performanceTest.ts @@ -27,11 +27,27 @@ const DEFAULT_THREADS = 1; const CSV_HEADERS = 'Op,ObjectSize,AppBufferSize,LibBufferSize,Crc32cEnabled,MD5Enabled,ApiName,ElapsedTimeUs,CpuTimeUs,Status\n'; const START_TIME = Date.now(); +export const enum TRANSFER_MANAGER_TEST_TYPES { + WRITE_ONE_READ_THREE = 'w1r3', + UPLOAD_MULTIPLE_OBJECTS = 'upload', + DOWNLOAD_MULTIPLE_OBJECTS = 'download', + LARGE_FILE_DOWNLOAD = 'large', +} const argv = yargs(process.argv.slice(2)) .options({ iterations: {type: 'number', default: DEFAULT_ITERATIONS}, numthreads: {type: 'number', default: DEFAULT_THREADS}, + testtype: { + type: 'string', + choices: [ + TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE, + TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD, + ], + default: TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE, + }, }) .parseSync(); @@ -51,6 +67,9 @@ function main() { ); numThreads = iterationsRemaining; } + if (argv.testtype !== TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE) { + numThreads = 1; + } for (let i = 0; i < numThreads; i++) { createWorker(); } @@ -65,9 +84,20 @@ function createWorker() { console.log( `Starting new iteration. Current iterations remaining: ${iterationsRemaining}` ); - const w = new Worker(__dirname + '/performPerformanceTest.js', { + let testPath = ''; + if (argv.testtype === TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE) { + testPath = `${__dirname}/performPerformanceTest.js`; + } else if ( + argv.testtype === TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS || + argv.testtype === TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD + ) { + testPath = `${__dirname}/performTransferManagerTest.js`; + } + + const w = new Worker(testPath, { argv: process.argv.slice(2), }); + w.on('message', data => { console.log('Successfully completed iteration.'); appendResultToCSV(data); diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index c717afdb8..b7b658dc0 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import {execSync} from 'child_process'; -import {mkdirSync} from 'fs'; +import {execFileSync} from 'child_process'; +import {mkdirSync, mkdtempSync, unlinkSync} from 'fs'; import path = require('path'); import * as uuid from 'uuid'; @@ -23,8 +23,6 @@ export const BLOCK_SIZE_IN_BYTES = 1024; export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; -const CREATE_DIRECTORY = 1; - /** * Create a uniformly distributed random integer beween the inclusive min and max provided. * @@ -39,6 +37,15 @@ export function randomInteger(minInclusive: number, maxInclusive: number) { ); } +/** + * Return a random boolean + * + * @returns {boolean} a random boolean value + */ +export function randomBoolean() { + return !!randomInteger(0, 1); +} + /** * Creates a random file name by appending a UUID to the baseName. * @@ -65,15 +72,22 @@ export function generateRandomFile( fileName: string, fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES, - currentDirectory: string = __dirname + currentDirectory: string = mkdtempSync(uuid.v4()) ) { const fileSizeBytes = randomInteger( fileSizeLowerBoundBytes, fileSizeUpperBoundBytes ); const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES); - const cmd = `dd if=/dev/urandom of=${currentDirectory}/${fileName} bs=${BLOCK_SIZE_IN_BYTES} count=${numberNeeded} status=none iflag=fullblock`; - execSync(cmd); + const args = [ + 'if=/dev/urandom', + `of=${currentDirectory}/${fileName}`, + `bs=${BLOCK_SIZE_IN_BYTES}`, + `count=${numberNeeded}`, + 'status=none', + 'iflag=fullblock', + ]; + execFileSync('dd', args); return fileSizeBytes; } @@ -85,26 +99,44 @@ export function generateRandomFile( * @param {string} baseName the starting directory under which everything else is added. File names will have this value prepended. * @param {number} fileSizeLowerBoundBytes minimum size of file to generate. * @param {number} fileSizeUpperBoundBytes maximum size of file to generate. + * + * @returns {array} an array of all the generated paths */ export function generateRandomDirectoryStructure( maxObjects: number, baseName: string, fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES -) { +): string[] { let curPath = baseName; + mkdirSync(curPath); + const generatedPaths: string[] = []; + for (let i = 0; i < maxObjects; i++) { - const dirOrFile = randomInteger(0, 1); - if (dirOrFile === CREATE_DIRECTORY) { + if (randomBoolean()) { curPath = path.join(curPath, uuid.v4()); mkdirSync(curPath, {recursive: true}); + generatedPaths.push(curPath); } else { + const randomName = generateRandomFileName(baseName); generateRandomFile( - generateRandomFileName(baseName), + randomName, fileSizeLowerBoundBytes, fileSizeUpperBoundBytes, curPath ); + generatedPaths.push(path.join(curPath, randomName)); } } + + return generatedPaths; +} + +/** + * Deletes the file specified by the fileName parameter. + * + * @param {string} fileName name of the file to delete. + */ +export function cleanupFile(fileName: string) { + unlinkSync(`${__dirname}/${fileName}`); } diff --git a/src/index.ts b/src/index.ts index e09060711..c2e6b9df6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -256,9 +256,4 @@ export { StorageOptions, } from './storage'; export {GetSignedUrlCallback, GetSignedUrlResponse} from './signer'; -export { - TransferManager, - UploadMultiOptions, - UploadMultiCallback, - LargeFileDownloadOptions, -} from './transfer-manager'; +export * from './transfer-manager'; diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 2ce5f125f..121d84f56 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -171,15 +171,22 @@ export class TransferManager { const promises = []; for (const filePath of filePaths) { + const stat = await fsp.lstat(filePath); + if (stat.isDirectory()) { + continue; + } const baseFileName = path.basename(filePath); const passThroughOptionsCopy = extend( true, {}, options.passthroughOptions ); + passThroughOptionsCopy.destination = filePath; + //console.log(passThroughOptionsCopy.destination); if (options.prefix) { passThroughOptionsCopy.destination = path.join( options.prefix, + passThroughOptionsCopy.destination?.toString() || '', baseFileName ); } From 721aab649d76585b0e490aac3ef4406493f148f4 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 10 Nov 2022 10:36:09 -0800 Subject: [PATCH 14/25] feat: merged in application performance tests (#2100) --- .../performApplicationPerformanceTest.ts | 208 ++++++++++++++++++ internal-tooling/performPerformanceTest.ts | 14 +- internal-tooling/performanceTest.ts | 18 +- internal-tooling/performanceUtils.ts | 14 ++ 4 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 internal-tooling/performApplicationPerformanceTest.ts diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts new file mode 100644 index 000000000..56789c22a --- /dev/null +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -0,0 +1,208 @@ +/*! + * Copyright 2022 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import yargs from 'yargs'; +import * as uuid from 'uuid'; +import {execSync} from 'child_process'; +import {unlinkSync, opendirSync} from 'fs'; +import {Bucket, DownloadOptions, DownloadResponse, File, Storage} from '../src'; +import {performance} from 'perf_hooks'; +// eslint-disable-next-line node/no-unsupported-features/node-builtins +import {parentPort} from 'worker_threads'; +import path = require('path'); +import { generateRandomDirectoryStructure, generateRandomFileName, TestResult } from './performanceUtils'; + +const TEST_NAME_STRING = 'nodejs-perf-metrics'; +const DEFAULT_NUMBER_OF_WRITES = 1; +const DEFAULT_NUMBER_OF_READS = 3; +const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics-shaffeeullah'; +const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; +const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; +const BLOCK_SIZE_IN_BYTES = 1024; +const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; + + +/** + * Create a uniformly distributed random integer beween the inclusive min and max provided. + * + * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. + * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. + * @returns {number} returns a random integer between minInclusive and maxInclusive + */ +const randomInteger = (minInclusive: number, maxInclusive: number) => { + // Utilizing Math.random will generate uniformly distributed random numbers. + return ( + Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive + ); +}; + +const argv = yargs(process.argv.slice(2)) + .options({ + bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, + small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, + large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, + projectid: {type: 'string'}, + }) + .parseSync(); + +/** + * Main entry point. This function performs a test iteration and posts the message back + * to the parent thread. + */ +async function main() { + const results = await performWriteReadTest(); + parentPort?.postMessage(results); +} + +async function uploadInParallel(bucket: Bucket, directory: string, validation: Object) { + + const promises = []; + let openedDir = opendirSync(directory); + console.log("\nPath of the directory:", openedDir.path); + console.log("Files Present in directory:"); + let filesLeft = true; + while (filesLeft) { + // Read a file as fs.Dirent object + let fileDirent = openedDir.readSync(); + + // If readSync() does not return null + // print its filename + if (fileDirent != null) { + console.log("Name:", fileDirent.name); + promises.push(bucket.upload(`${directory}/${fileDirent!.name}`, validation)) + } + // If the readSync() returns null + // stop the loop + else filesLeft = false; + } + await Promise.all(promises).catch(console.error); +} + +async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { + const promises: Promise [] = []; + const [files] = await bucket.getFiles(); + files.forEach(file => { + promises.push(file.download(options)); + }); + await Promise.all(promises).catch(console.error); +} + +/** + * Performs an iteration of the Write 1 / Read 3 performance measuring test. + * + * @returns {Promise { + const results: TestResult[] = []; + const directory = TEST_NAME_STRING;//"/node-test-files" + generateRandomDirectoryStructure(10, directory); + const checkType = randomInteger(0, 2); + + const stg = new Storage({ + projectId: argv.projectid, + }); + + let bucket = stg.bucket(argv.bucket); + if (!(await bucket.exists())[0]) { + await bucket.create(); + } + + for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { + let start = 0; + let end = 0; + + const iterationResult: TestResult = { + op: 'WRITE', + objectSize: BLOCK_SIZE_IN_BYTES, //note this is wrong + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: '[OK]', + }; + + bucket = stg.bucket(argv.bucket, { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }); + + await bucket.deleteFiles(); + + if (checkType === 0) { + start = performance.now(); + await uploadInParallel(bucket, `${directory}`, {validation: false}); + end = performance.now(); + } else if (checkType === 1) { + iterationResult.crc32Enabled = true; + start = performance.now(); + await uploadInParallel(bucket, `${directory}`, {validation: 'crc32c'}); + end = performance.now(); + } else { + iterationResult.md5Enabled = true; + start = performance.now(); + await uploadInParallel(bucket, `${directory}`, {validation: 'md5'}); + end = performance.now(); + } + + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(iterationResult); + } + + bucket = stg.bucket(argv.bucket); + for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { + let start = 0; + let end = 0; + const iterationResult: TestResult = { + op: `READ[${j}]`, + objectSize: 0, //this is wrong + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: '[OK]', + }; + + const destinationFileName = "TODO" + const destination = path.join(__dirname, destinationFileName); + if (checkType === 0) { + start = performance.now(); + await downloadInParallel(bucket, {validation: false, destination}); + end = performance.now(); + } else if (checkType === 1) { + iterationResult.crc32Enabled = true; + start = performance.now(); + await downloadInParallel(bucket, {validation: 'crc32c', destination}); + end = performance.now(); + } else { + iterationResult.md5Enabled = true; + start = performance.now(); + await downloadInParallel(bucket, {validation: 'md5', destination}); + end = performance.now(); + } + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(iterationResult); + } + return results; +} + +main(); diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index 7fa8ad20e..03d85134c 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -29,6 +29,7 @@ import { generateRandomFile, generateRandomFileName, randomInteger, + TestResult } from './performanceUtils'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; @@ -37,19 +38,6 @@ const DEFAULT_NUMBER_OF_READS = 3; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics'; const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; -export interface TestResult { - op: string; - objectSize: number; - appBufferSize: number; - libBufferSize: number; - crc32Enabled: boolean; - md5Enabled: boolean; - apiName: 'JSON' | 'XML'; - elapsedTimeUs: number; - cpuTimeUs: number; - status: '[OK]'; -} - const argv = yargs(process.argv.slice(2)) .options({ bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, diff --git a/internal-tooling/performanceTest.ts b/internal-tooling/performanceTest.ts index a124eac91..77c393dbf 100644 --- a/internal-tooling/performanceTest.ts +++ b/internal-tooling/performanceTest.ts @@ -18,7 +18,7 @@ import {appendFile} from 'fs/promises'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {Worker} from 'worker_threads'; import yargs = require('yargs'); -import {TestResult} from './performPerformanceTest'; +import {TestResult} from './performanceUtils'; import {existsSync} from 'fs'; import {writeFile} from 'fs/promises'; @@ -32,6 +32,9 @@ export const enum TRANSFER_MANAGER_TEST_TYPES { UPLOAD_MULTIPLE_OBJECTS = 'upload', DOWNLOAD_MULTIPLE_OBJECTS = 'download', LARGE_FILE_DOWNLOAD = 'large', + APPLICATION_LARGE_FILE_DOWNLOAD = 'application-large', + APPLICATION_UPLOAD_MULTIPLE_OBJECTS = 'application-upload', + APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS = 'application-download' } const argv = yargs(process.argv.slice(2)) @@ -45,6 +48,9 @@ const argv = yargs(process.argv.slice(2)) TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS, TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS, TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD, + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD, + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS ], default: TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE, }, @@ -93,6 +99,13 @@ function createWorker() { ) { testPath = `${__dirname}/performTransferManagerTest.js`; } + else if ( + argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS || + argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD || + argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS + ) { + testPath = `${__dirname}/performApplicationPerformanceTest.js`; + } const w = new Worker(testPath, { argv: process.argv.slice(2), @@ -105,8 +118,9 @@ function createWorker() { createWorker(); } }); - w.on('error', () => { + w.on('error', (e) => { console.log('An error occurred.'); + console.log(e); }); } diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index b7b658dc0..1308389bc 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -23,6 +23,20 @@ export const BLOCK_SIZE_IN_BYTES = 1024; export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; +export interface TestResult { + op: string; + objectSize: number; + appBufferSize: number; + libBufferSize: number; + crc32Enabled: boolean; + md5Enabled: boolean; + apiName: 'JSON' | 'XML'; + elapsedTimeUs: number; + cpuTimeUs: number; + status: '[OK]'; +} + + /** * Create a uniformly distributed random integer beween the inclusive min and max provided. * From f5e81212f872500f00593188e0f67a098975438e Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Tue, 15 Nov 2022 12:23:54 -0800 Subject: [PATCH 15/25] fix: fixed many bugs (#2102) * fix: cleaning up bugs * fix: fixed many bugs --- .../performApplicationPerformanceTest.ts | 111 ++++++++++-------- 1 file changed, 64 insertions(+), 47 deletions(-) diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts index 56789c22a..1a2c62b30 100644 --- a/internal-tooling/performApplicationPerformanceTest.ts +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -15,15 +15,14 @@ */ import yargs from 'yargs'; -import * as uuid from 'uuid'; -import {execSync} from 'child_process'; -import {unlinkSync, opendirSync} from 'fs'; -import {Bucket, DownloadOptions, DownloadResponse, File, Storage} from '../src'; +import {promises as fsp} from 'fs'; +import {Bucket, DownloadOptions, DownloadResponse, Storage, UploadOptions} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; -import path = require('path'); -import { generateRandomDirectoryStructure, generateRandomFileName, TestResult } from './performanceUtils'; +import { generateRandomDirectoryStructure, TestResult } from './performanceUtils'; +import { TransferManagerTestResult } from './performTransferManagerTest'; +import { TRANSFER_MANAGER_TEST_TYPES } from './performanceTest'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; const DEFAULT_NUMBER_OF_WRITES = 1; @@ -34,6 +33,8 @@ const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; const BLOCK_SIZE_IN_BYTES = 1024; const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; +let stg: Storage; +let bucket: Bucket; /** * Create a uniformly distributed random integer beween the inclusive min and max provided. @@ -49,6 +50,8 @@ const randomInteger = (minInclusive: number, maxInclusive: number) => { ); }; +const checkType = randomInteger(0, 2); + const argv = yargs(process.argv.slice(2)) .options({ bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, @@ -63,30 +66,47 @@ const argv = yargs(process.argv.slice(2)) * to the parent thread. */ async function main() { - const results = await performWriteReadTest(); + let results: TestResult[] = []; + await performTestSetup(); + + switch (argv.testtype) { + case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: + results = await performWriteTest(); + break; + case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: + results = await performReadTest(); + break; + // case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD: + // results = await performLargeReadTest(); + // break; + default: + break; + } parentPort?.postMessage(results); } -async function uploadInParallel(bucket: Bucket, directory: string, validation: Object) { +async function performTestSetup() { + stg = new Storage({ + projectId: argv.projectid, + }); + + bucket = stg.bucket(argv.bucket); + if (!(await bucket.exists())[0]) { + await bucket.create(); + } +} + +async function uploadInParallel(bucket: Bucket, paths: string[], options: UploadOptions) { const promises = []; - let openedDir = opendirSync(directory); - console.log("\nPath of the directory:", openedDir.path); - console.log("Files Present in directory:"); - let filesLeft = true; - while (filesLeft) { - // Read a file as fs.Dirent object - let fileDirent = openedDir.readSync(); - - // If readSync() does not return null - // print its filename - if (fileDirent != null) { - console.log("Name:", fileDirent.name); - promises.push(bucket.upload(`${directory}/${fileDirent!.name}`, validation)) + for (const index in paths) { + const path = paths[index]; + const stat = await fsp.lstat(path); + if (stat.isDirectory()){ + continue } - // If the readSync() returns null - // stop the loop - else filesLeft = false; + options.destination = path; + promises.push(bucket.upload(path, options)) } await Promise.all(promises).catch(console.error); } @@ -101,24 +121,14 @@ async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { } /** - * Performs an iteration of the Write 1 / Read 3 performance measuring test. + * Performs an iteration of the Write multiple objects test. * * @returns {Promise { +async function performWriteTest(): Promise { const results: TestResult[] = []; - const directory = TEST_NAME_STRING;//"/node-test-files" - generateRandomDirectoryStructure(10, directory); - const checkType = randomInteger(0, 2); - - const stg = new Storage({ - projectId: argv.projectid, - }); - - let bucket = stg.bucket(argv.bucket); - if (!(await bucket.exists())[0]) { - await bucket.create(); - } + const directory = TEST_NAME_STRING; + const directories = generateRandomDirectoryStructure(10, directory); for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { let start = 0; @@ -143,28 +153,37 @@ async function performWriteReadTest(): Promise { }, }); - await bucket.deleteFiles(); + await bucket.deleteFiles(); //cleanup anything old if (checkType === 0) { start = performance.now(); - await uploadInParallel(bucket, `${directory}`, {validation: false}); + await uploadInParallel(bucket, directories, {validation: false}); end = performance.now(); } else if (checkType === 1) { iterationResult.crc32Enabled = true; start = performance.now(); - await uploadInParallel(bucket, `${directory}`, {validation: 'crc32c'}); + await uploadInParallel(bucket, directories, {validation: 'crc32c'}); end = performance.now(); } else { iterationResult.md5Enabled = true; start = performance.now(); - await uploadInParallel(bucket, `${directory}`, {validation: 'md5'}); + await uploadInParallel(bucket, directories, {validation: 'md5'}); end = performance.now(); } iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); results.push(iterationResult); } + return results; +} +/** + * Performs an iteration of the read multiple objects test. + * + * @returns {Promise} Promise that resolves to an array of test results for the iteration. + */ + async function performReadTest(): Promise { + const results: TestResult[] = []; bucket = stg.bucket(argv.bucket); for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { let start = 0; @@ -182,21 +201,19 @@ async function performWriteReadTest(): Promise { status: '[OK]', }; - const destinationFileName = "TODO" - const destination = path.join(__dirname, destinationFileName); if (checkType === 0) { start = performance.now(); - await downloadInParallel(bucket, {validation: false, destination}); + await downloadInParallel(bucket, {validation: false}); end = performance.now(); } else if (checkType === 1) { iterationResult.crc32Enabled = true; start = performance.now(); - await downloadInParallel(bucket, {validation: 'crc32c', destination}); + await downloadInParallel(bucket, {validation: 'crc32c'}); end = performance.now(); } else { iterationResult.md5Enabled = true; start = performance.now(); - await downloadInParallel(bucket, {validation: 'md5', destination}); + await downloadInParallel(bucket, {validation: 'md5'}); end = performance.now(); } iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); From 886dc03c34b6a3e7369068a9d64b9acd7bc32276 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:19:47 -0500 Subject: [PATCH 16/25] fix: more work on transfer manager perf metrics (#2103) * fix: more work on transfer manager perf metrics * fix unit tests for tm --- .../performApplicationPerformanceTest.ts | 36 ++-- internal-tooling/performPerformanceTest.ts | 5 +- .../performTransferManagerTest.ts | 181 +++++++++++++----- internal-tooling/performanceTest.ts | 45 +++-- internal-tooling/performanceUtils.ts | 81 ++++---- src/transfer-manager.ts | 16 +- test/transfer-manager.ts | 19 +- 7 files changed, 251 insertions(+), 132 deletions(-) diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts index 1a2c62b30..752e49dd0 100644 --- a/internal-tooling/performApplicationPerformanceTest.ts +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -16,13 +16,18 @@ import yargs from 'yargs'; import {promises as fsp} from 'fs'; -import {Bucket, DownloadOptions, DownloadResponse, Storage, UploadOptions} from '../src'; +import { + Bucket, + DownloadOptions, + DownloadResponse, + Storage, + UploadOptions, +} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; -import { generateRandomDirectoryStructure, TestResult } from './performanceUtils'; -import { TransferManagerTestResult } from './performTransferManagerTest'; -import { TRANSFER_MANAGER_TEST_TYPES } from './performanceTest'; +import {generateRandomDirectoryStructure, TestResult} from './performanceUtils'; +import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; const DEFAULT_NUMBER_OF_WRITES = 1; @@ -96,23 +101,26 @@ async function performTestSetup() { } } - -async function uploadInParallel(bucket: Bucket, paths: string[], options: UploadOptions) { +async function uploadInParallel( + bucket: Bucket, + paths: string[], + options: UploadOptions +) { const promises = []; for (const index in paths) { const path = paths[index]; const stat = await fsp.lstat(path); - if (stat.isDirectory()){ - continue + if (stat.isDirectory()) { + continue; } options.destination = path; - promises.push(bucket.upload(path, options)) + promises.push(bucket.upload(path, options)); } await Promise.all(promises).catch(console.error); } async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { - const promises: Promise [] = []; + const promises: Promise[] = []; const [files] = await bucket.getFiles(); files.forEach(file => { promises.push(file.download(options)); @@ -157,17 +165,17 @@ async function performWriteTest(): Promise { if (checkType === 0) { start = performance.now(); - await uploadInParallel(bucket, directories, {validation: false}); + await uploadInParallel(bucket, directories.paths, {validation: false}); end = performance.now(); } else if (checkType === 1) { iterationResult.crc32Enabled = true; start = performance.now(); - await uploadInParallel(bucket, directories, {validation: 'crc32c'}); + await uploadInParallel(bucket, directories.paths, {validation: 'crc32c'}); end = performance.now(); } else { iterationResult.md5Enabled = true; start = performance.now(); - await uploadInParallel(bucket, directories, {validation: 'md5'}); + await uploadInParallel(bucket, directories.paths, {validation: 'md5'}); end = performance.now(); } @@ -182,7 +190,7 @@ async function performWriteTest(): Promise { * * @returns {Promise} Promise that resolves to an array of test results for the iteration. */ - async function performReadTest(): Promise { +async function performReadTest(): Promise { const results: TestResult[] = []; bucket = stg.bucket(argv.bucket); for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index 03d85134c..489368569 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -15,7 +15,6 @@ */ import yargs from 'yargs'; -import {unlinkSync} from 'fs'; import {Storage} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins @@ -28,15 +27,15 @@ import { DEFAULT_SMALL_FILE_SIZE_BYTES, generateRandomFile, generateRandomFileName, + NODE_DEFAULT_HIGHWATER_MARK_BYTES, randomInteger, - TestResult + TestResult, } from './performanceUtils'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; const DEFAULT_NUMBER_OF_WRITES = 1; const DEFAULT_NUMBER_OF_READS = 3; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics'; -const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; const argv = yargs(process.argv.slice(2)) .options({ diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index fb2b24b6f..b4a1464a5 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -20,23 +20,25 @@ import yargs from 'yargs'; import {Bucket, Storage, TransferManager} from '../src'; import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; import { + BLOCK_SIZE_IN_BYTES, cleanupFile, DEFAULT_LARGE_FILE_SIZE_BYTES, DEFAULT_SMALL_FILE_SIZE_BYTES, generateRandomDirectoryStructure, generateRandomFile, generateRandomFileName, + NODE_DEFAULT_HIGHWATER_MARK_BYTES, + TestResult, } from './performanceUtils'; +import {performance} from 'perf_hooks'; +import {rmSync} from 'fs'; -const TEST_NAME_STRING = 'transfer-manager-perf-metrics'; +const TEST_NAME_STRING = 'tm-perf-metrics'; const DEFAULT_BUCKET_NAME = 'nodejs-transfer-manager-perf-metrics'; const DEFAULT_NUMBER_OF_PROMISES = 2; const DEFAULT_NUMBER_OF_OBJECTS = 1000; const DEFAULT_CHUNK_SIZE_BYTES = 16 * 1024 * 1024; - -export interface TransferManagerTestResult { - numberOfObjects: number; -} +const DIRECTORY_PROBABILITY = 0.1; let stg: Storage; let bucket: Bucket; @@ -54,32 +56,44 @@ const argv = yargs(process.argv.slice(2)) testtype: { type: 'string', choices: [ - TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS, - TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS, - TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD, ], }, }) .parseSync(); async function main() { - let results: TransferManagerTestResult[] = []; + let result: TestResult = { + op: '', + objectSize: 0, + appBufferSize: 0, + libBufferSize: 0, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: 0, + status: '[OK]', + }; await performTestSetup(); switch (argv.testtype) { - case TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS: - results = await performUploadMultipleObjectsTest(); + case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS: + result = await performUploadMultipleObjectsTest(); break; - case TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS: - results = await performDownloadMultipleObjectsTest(); + case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS: + result = await performDownloadMultipleObjectsTest(); break; - case TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD: - results = await performDownloadLargeFileTest(); + case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD: + result = await performDownloadLargeFileTest(); break; default: break; } - parentPort?.postMessage(results); + parentPort?.postMessage(result); + await performTestCleanup(); } async function performTestSetup() { @@ -91,54 +105,119 @@ async function performTestSetup() { tm = new TransferManager(bucket); } -async function performUploadMultipleObjectsTest(): Promise< - TransferManagerTestResult[] -> { - const results: TransferManagerTestResult[] = []; - const paths = generateRandomDirectoryStructure( +async function performTestCleanup() { + await bucket.deleteFiles(); +} + +async function performUploadMultipleObjectsTest(): Promise { + const result: TestResult = { + op: 'WRITE', + objectSize: 0, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: '[OK]', + }; + const creationInfo = generateRandomDirectoryStructure( argv.numobjects, TEST_NAME_STRING, argv.small, - argv.large + argv.large, + DIRECTORY_PROBABILITY ); - const uploadResults = await tm.uploadMulti(paths, { + result.objectSize = creationInfo.totalSizeInBytes; + const start = performance.now(); + await tm.uploadMulti(creationInfo.paths, { concurrencyLimit: argv.numpromises, }); + const end = performance.now(); + + result.elapsedTimeUs = Math.round((end - start) * 1000); + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); - return results; + return result; } -async function performDownloadMultipleObjectsTest(): Promise< - TransferManagerTestResult[] -> { - const results: TransferManagerTestResult[] = []; +async function performDownloadMultipleObjectsTest(): Promise { + const result: TestResult = { + op: 'READ', + objectSize: 0, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: '[OK]', + }; + + const creationInfo = generateRandomDirectoryStructure( + argv.numobjects, + TEST_NAME_STRING, + argv.small, + argv.large, + DIRECTORY_PROBABILITY + ); + result.objectSize = creationInfo.totalSizeInBytes; + await tm.uploadMulti(creationInfo.paths, { + concurrencyLimit: argv.numpromises, + }); + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + const getFilesResult = await bucket.getFiles(); + const start = performance.now(); + await tm.downloadMulti(getFilesResult[0], { + concurrencyLimit: argv.numpromises, + prefix: __dirname, + }); + const end = performance.now(); + + result.elapsedTimeUs = Math.round((end - start) * 1000); + //rmSync(TEST_NAME_STRING, {recursive: true, force: true}); - return results; + return result; } -async function performDownloadLargeFileTest(): Promise< - TransferManagerTestResult[] -> { - const results: TransferManagerTestResult[] = []; - try { - const fileName = generateRandomFileName(TEST_NAME_STRING); - generateRandomFile(fileName, argv.small, argv.large, __dirname); - const file = bucket.file(`${fileName}`); - - await bucket.upload(`${__dirname}/${fileName}`); - cleanupFile(fileName); - const start = performance.now(); - await tm.downloadLargeFile(file, { - concurrencyLimit: argv.numpromises, - chunkSizeBytes: argv.chunksize, - }); - const end = performance.now(); - console.log(Math.round((end - start) * 1000)); - } catch (e) { - console.error(e); - } +async function performDownloadLargeFileTest(): Promise { + const fileName = generateRandomFileName(TEST_NAME_STRING); + const sizeInBytes = generateRandomFile( + fileName, + argv.small, + argv.large, + __dirname + ); + const result: TestResult = { + op: 'READ', + objectSize: sizeInBytes, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: '[OK]', + }; + const file = bucket.file(`${fileName}`); + + await bucket.upload(`${__dirname}/${fileName}`); + cleanupFile(fileName); + const start = performance.now(); + await tm.downloadLargeFile(file, { + concurrencyLimit: argv.numpromises, + chunkSizeBytes: argv.chunksize, + path: `${__dirname}`, + }); + const end = performance.now(); + + result.elapsedTimeUs = Math.round((end - start) * 1000); + cleanupFile(fileName); - return results; + return result; } main(); diff --git a/internal-tooling/performanceTest.ts b/internal-tooling/performanceTest.ts index 77c393dbf..408d2f7f3 100644 --- a/internal-tooling/performanceTest.ts +++ b/internal-tooling/performanceTest.ts @@ -29,12 +29,12 @@ const CSV_HEADERS = const START_TIME = Date.now(); export const enum TRANSFER_MANAGER_TEST_TYPES { WRITE_ONE_READ_THREE = 'w1r3', - UPLOAD_MULTIPLE_OBJECTS = 'upload', - DOWNLOAD_MULTIPLE_OBJECTS = 'download', - LARGE_FILE_DOWNLOAD = 'large', + TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS = 'tm-upload', + TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS = 'tm-download', + TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD = 'tm-large', APPLICATION_LARGE_FILE_DOWNLOAD = 'application-large', APPLICATION_UPLOAD_MULTIPLE_OBJECTS = 'application-upload', - APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS = 'application-download' + APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS = 'application-download', } const argv = yargs(process.argv.slice(2)) @@ -45,12 +45,12 @@ const argv = yargs(process.argv.slice(2)) type: 'string', choices: [ TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE, - TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS, - TRANSFER_MANAGER_TEST_TYPES.DOWNLOAD_MULTIPLE_OBJECTS, - TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS, + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD, TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS, TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD, - TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS, ], default: TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE, }, @@ -94,15 +94,21 @@ function createWorker() { if (argv.testtype === TRANSFER_MANAGER_TEST_TYPES.WRITE_ONE_READ_THREE) { testPath = `${__dirname}/performPerformanceTest.js`; } else if ( - argv.testtype === TRANSFER_MANAGER_TEST_TYPES.UPLOAD_MULTIPLE_OBJECTS || - argv.testtype === TRANSFER_MANAGER_TEST_TYPES.LARGE_FILE_DOWNLOAD + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS || + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD || + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS ) { testPath = `${__dirname}/performTransferManagerTest.js`; - } - else if ( - argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS || - argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD || - argv.testtype === TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS + } else if ( + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS || + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD || + argv.testtype === + TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS ) { testPath = `${__dirname}/performApplicationPerformanceTest.js`; } @@ -118,7 +124,7 @@ function createWorker() { createWorker(); } }); - w.on('error', (e) => { + w.on('error', e => { console.log('An error occurred.'); console.log(e); }); @@ -129,13 +135,16 @@ function createWorker() { * * @param {TestResult[]} results */ -async function appendResultToCSV(results: TestResult[]) { +async function appendResultToCSV(results: TestResult[] | TestResult) { const fileName = `nodejs-perf-metrics-${START_TIME}-${argv.iterations}.csv`; + const resultsToAppend: TestResult[] = Array.isArray(results) + ? results + : [results]; if (!existsSync(fileName)) { await writeFile(fileName, CSV_HEADERS); } - const csv = results.map(result => Object.values(result)); + const csv = resultsToAppend.map(result => Object.values(result)); const csvString = csv.join('\n'); await appendFile(fileName, `${csvString}\n`); } diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index 1308389bc..794f31c44 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -14,14 +14,15 @@ * limitations under the License. */ -import {execFileSync} from 'child_process'; +import {execSync} from 'child_process'; import {mkdirSync, mkdtempSync, unlinkSync} from 'fs'; import path = require('path'); -import * as uuid from 'uuid'; export const BLOCK_SIZE_IN_BYTES = 1024; export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; +export const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; +export const DEFAULT_DIRECTORY_PROBABILITY = 0.5; export interface TestResult { op: string; @@ -36,6 +37,10 @@ export interface TestResult { status: '[OK]'; } +export interface RandomDirectoryCreationInformation { + paths: string[]; + totalSizeInBytes: number; +} /** * Create a uniformly distributed random integer beween the inclusive min and max provided. @@ -52,12 +57,23 @@ export function randomInteger(minInclusive: number, maxInclusive: number) { } /** - * Return a random boolean + * Returns a boolean value with the provided probability + * + * @param {number} trueProbablity the probability the value will be true + * + * @returns {boolean} a boolean value with the probablity provided. + */ +export function weightedRandomBoolean(trueProbablity: number): boolean { + return Math.random() <= trueProbablity ? true : false; +} + +/** + * Return a string of 6 random characters * - * @returns {boolean} a random boolean value + * @returns {string} a random string value with length of 6 */ -export function randomBoolean() { - return !!randomInteger(0, 1); +export function randomString(): string { + return Math.random().toString(36).slice(-6); } /** @@ -68,7 +84,7 @@ export function randomBoolean() { * @returns {string} random file name that was generated. */ export function generateRandomFileName(baseName: string): string { - return `${baseName}.${uuid.v4()}`; + return `${baseName}.${randomString()}`; } /** @@ -86,22 +102,16 @@ export function generateRandomFile( fileName: string, fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES, - currentDirectory: string = mkdtempSync(uuid.v4()) -) { + currentDirectory: string = mkdtempSync(randomString()) +): number { const fileSizeBytes = randomInteger( fileSizeLowerBoundBytes, fileSizeUpperBoundBytes ); - const numberNeeded = Math.ceil(fileSizeBytes / BLOCK_SIZE_IN_BYTES); - const args = [ - 'if=/dev/urandom', - `of=${currentDirectory}/${fileName}`, - `bs=${BLOCK_SIZE_IN_BYTES}`, - `count=${numberNeeded}`, - 'status=none', - 'iflag=fullblock', - ]; - execFileSync('dd', args); + + execSync( + `head --bytes=${fileSizeBytes} /dev/urandom > ${currentDirectory}/${fileName}` + ); return fileSizeBytes; } @@ -120,30 +130,34 @@ export function generateRandomDirectoryStructure( maxObjects: number, baseName: string, fileSizeLowerBoundBytes: number = DEFAULT_SMALL_FILE_SIZE_BYTES, - fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES -): string[] { + fileSizeUpperBoundBytes: number = DEFAULT_LARGE_FILE_SIZE_BYTES, + directoryProbability: number = DEFAULT_DIRECTORY_PROBABILITY +): RandomDirectoryCreationInformation { let curPath = baseName; - mkdirSync(curPath); - const generatedPaths: string[] = []; + const creationInfo: RandomDirectoryCreationInformation = { + paths: [], + totalSizeInBytes: 0, + }; + mkdirSync(curPath); for (let i = 0; i < maxObjects; i++) { - if (randomBoolean()) { - curPath = path.join(curPath, uuid.v4()); + if (weightedRandomBoolean(directoryProbability)) { + curPath = path.join(curPath, randomString()); mkdirSync(curPath, {recursive: true}); - generatedPaths.push(curPath); + creationInfo.paths.push(curPath); } else { - const randomName = generateRandomFileName(baseName); - generateRandomFile( + const randomName = randomString(); + creationInfo.totalSizeInBytes += generateRandomFile( randomName, fileSizeLowerBoundBytes, fileSizeUpperBoundBytes, curPath ); - generatedPaths.push(path.join(curPath, randomName)); + creationInfo.paths.push(path.join(curPath, randomName)); } } - return generatedPaths; + return creationInfo; } /** @@ -151,6 +165,9 @@ export function generateRandomDirectoryStructure( * * @param {string} fileName name of the file to delete. */ -export function cleanupFile(fileName: string) { - unlinkSync(`${__dirname}/${fileName}`); +export function cleanupFile( + fileName: string, + directoryName: string = __dirname +): void { + unlinkSync(`${directoryName}/${fileName}`); } diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 121d84f56..97ff92c37 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -37,7 +37,7 @@ export interface UploadMultiOptions { concurrencyLimit?: number; skipIfExists?: boolean; prefix?: string; - passthroughOptions?: UploadOptions; + passthroughOptions?: Omit; } export interface DownloadMultiOptions { @@ -50,6 +50,7 @@ export interface DownloadMultiOptions { export interface LargeFileDownloadOptions { concurrencyLimit?: number; chunkSizeBytes?: number; + path?: string; } export interface UploadMultiCallback { @@ -175,19 +176,16 @@ export class TransferManager { if (stat.isDirectory()) { continue; } - const baseFileName = path.basename(filePath); - const passThroughOptionsCopy = extend( + const passThroughOptionsCopy: UploadOptions = extend( true, {}, options.passthroughOptions ); passThroughOptionsCopy.destination = filePath; - //console.log(passThroughOptionsCopy.destination); if (options.prefix) { passThroughOptionsCopy.destination = path.join( options.prefix, - passThroughOptionsCopy.destination?.toString() || '', - baseFileName + passThroughOptionsCopy.destination ); } promises.push( @@ -402,7 +400,11 @@ export class TransferManager { } let start = 0; - const fileToWrite = await fsp.open(path.basename(file.name), 'w+'); + const filePath = path.join( + options.path || __dirname, + path.basename(file.name) + ); + const fileToWrite = await fsp.open(filePath, 'w+'); while (start < size) { const chunkStart = start; let chunkEnd = start + chunkSize - 1; diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index f96f4aac2..1386cc6a2 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -108,6 +108,13 @@ const fakeFs = extend(true, {}, fs, { }, }; }, + lstat: () => { + return { + isDirectory: () => { + return false; + }, + }; + }, }; }, }); @@ -204,7 +211,7 @@ describe('Transfer Manager', () => { it('sets destination to prefix + filename when prefix is supplied', async () => { const paths = ['/a/b/foo/bar.txt']; - const expectedDestination = path.normalize('hello/world/bar.txt'); + const expectedDestination = path.normalize('hello/world/a/b/foo/bar.txt'); bucket.upload = (_path: string, options: UploadOptions) => { assert.strictEqual(options.destination, expectedDestination); @@ -214,8 +221,7 @@ describe('Transfer Manager', () => { }); it('invokes the callback if one is provided', done => { - const basename = 'testfile.json'; - const paths = [path.join(__dirname, '../../test/testdata/' + basename)]; + const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; transferManager.uploadMulti( paths, @@ -223,17 +229,16 @@ describe('Transfer Manager', () => { assert.ifError(err); assert(files); assert(metadata); - assert.strictEqual(files[0].name, basename); + assert.strictEqual(files[0].name, paths[0]); done(); } ); }); it('returns a promise with the uploaded file if there is no callback', async () => { - const basename = 'testfile.json'; - const paths = [path.join(__dirname, '../../test/testdata/' + basename)]; + const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; const result = await transferManager.uploadMulti(paths); - assert.strictEqual(result[0][0].name, basename); + assert.strictEqual(result[0][0].name, paths[0]); }); }); From 94b1c02acff561ab517570b6bb69533cc37f336a Mon Sep 17 00:00:00 2001 From: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> Date: Wed, 16 Nov 2022 13:17:44 -0500 Subject: [PATCH 17/25] fix: performance test refactoring, comments (#2104) --- .../performApplicationPerformanceTest.ts | 79 +++++-------------- internal-tooling/performPerformanceTest.ts | 74 +++++------------ .../performTransferManagerTest.ts | 77 ++++++++++++------ internal-tooling/performanceUtils.ts | 52 ++++++++++++ 4 files changed, 143 insertions(+), 139 deletions(-) diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts index 752e49dd0..7ff67e07f 100644 --- a/internal-tooling/performApplicationPerformanceTest.ts +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -26,7 +26,13 @@ import { import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; -import {generateRandomDirectoryStructure, TestResult} from './performanceUtils'; +import { + DEFAULT_PROJECT_ID, + generateRandomDirectoryStructure, + getValidationType, + performanceTestSetup, + TestResult, +} from './performanceUtils'; import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; @@ -41,28 +47,14 @@ const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; let stg: Storage; let bucket: Bucket; -/** - * Create a uniformly distributed random integer beween the inclusive min and max provided. - * - * @param {number} minInclusive lower bound (inclusive) of the range of random integer to return. - * @param {number} maxInclusive upper bound (inclusive) of the range of random integer to return. - * @returns {number} returns a random integer between minInclusive and maxInclusive - */ -const randomInteger = (minInclusive: number, maxInclusive: number) => { - // Utilizing Math.random will generate uniformly distributed random numbers. - return ( - Math.floor(Math.random() * (maxInclusive - minInclusive + 1)) + minInclusive - ); -}; - -const checkType = randomInteger(0, 2); +const checkType = getValidationType(); const argv = yargs(process.argv.slice(2)) .options({ bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, - projectid: {type: 'string'}, + projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, }) .parseSync(); @@ -72,7 +64,7 @@ const argv = yargs(process.argv.slice(2)) */ async function main() { let results: TestResult[] = []; - await performTestSetup(); + ({bucket} = await performanceTestSetup(argv.projectid, argv.bucket)); switch (argv.testtype) { case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: @@ -90,17 +82,6 @@ async function main() { parentPort?.postMessage(results); } -async function performTestSetup() { - stg = new Storage({ - projectId: argv.projectid, - }); - - bucket = stg.bucket(argv.bucket); - if (!(await bucket.exists())[0]) { - await bucket.create(); - } -} - async function uploadInParallel( bucket: Bucket, paths: string[], @@ -131,7 +112,7 @@ async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { /** * Performs an iteration of the Write multiple objects test. * - * @returns {Promise} Promise that resolves to an array of test results for the iteration. */ async function performWriteTest(): Promise { const results: TestResult[] = []; @@ -162,22 +143,9 @@ async function performWriteTest(): Promise { }); await bucket.deleteFiles(); //cleanup anything old - - if (checkType === 0) { - start = performance.now(); - await uploadInParallel(bucket, directories.paths, {validation: false}); - end = performance.now(); - } else if (checkType === 1) { - iterationResult.crc32Enabled = true; - start = performance.now(); - await uploadInParallel(bucket, directories.paths, {validation: 'crc32c'}); - end = performance.now(); - } else { - iterationResult.md5Enabled = true; - start = performance.now(); - await uploadInParallel(bucket, directories.paths, {validation: 'md5'}); - end = performance.now(); - } + start = performance.now(); + await uploadInParallel(bucket, directories.paths, {validation: checkType}); + end = performance.now(); iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); results.push(iterationResult); @@ -209,21 +177,10 @@ async function performReadTest(): Promise { status: '[OK]', }; - if (checkType === 0) { - start = performance.now(); - await downloadInParallel(bucket, {validation: false}); - end = performance.now(); - } else if (checkType === 1) { - iterationResult.crc32Enabled = true; - start = performance.now(); - await downloadInParallel(bucket, {validation: 'crc32c'}); - end = performance.now(); - } else { - iterationResult.md5Enabled = true; - start = performance.now(); - await downloadInParallel(bucket, {validation: 'md5'}); - end = performance.now(); - } + start = performance.now(); + await downloadInParallel(bucket, {validation: checkType}); + end = performance.now(); + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); results.push(iterationResult); } diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index 489368569..eb125b1e6 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -15,7 +15,6 @@ */ import yargs from 'yargs'; -import {Storage} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; @@ -24,25 +23,31 @@ import { BLOCK_SIZE_IN_BYTES, cleanupFile, DEFAULT_LARGE_FILE_SIZE_BYTES, + DEFAULT_PROJECT_ID, DEFAULT_SMALL_FILE_SIZE_BYTES, generateRandomFile, generateRandomFileName, + getValidationType, NODE_DEFAULT_HIGHWATER_MARK_BYTES, - randomInteger, + performanceTestSetup, TestResult, } from './performanceUtils'; +import {Bucket} from '../src'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; const DEFAULT_NUMBER_OF_WRITES = 1; const DEFAULT_NUMBER_OF_READS = 3; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics'; +let bucket: Bucket; +const checkType = getValidationType(); + const argv = yargs(process.argv.slice(2)) .options({ bucket: {type: 'string', default: DEFAULT_BUCKET_NAME}, small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, - projectid: {type: 'string'}, + projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, }) .parseSync(); @@ -64,16 +69,8 @@ async function performWriteReadTest(): Promise { const results: TestResult[] = []; const fileName = generateRandomFileName(TEST_NAME_STRING); const sizeInBytes = generateRandomFile(fileName, argv.small, argv.large); - const checkType = randomInteger(0, 2); - - const stg = new Storage({ - projectId: argv.projectid, - }); - let bucket = stg.bucket(argv.bucket); - if (!(await bucket.exists())[0]) { - await bucket.create(); - } + ({bucket} = await performanceTestSetup(argv.projectid, argv.bucket)); for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { let start = 0; @@ -84,41 +81,22 @@ async function performWriteReadTest(): Promise { objectSize: sizeInBytes, appBufferSize: BLOCK_SIZE_IN_BYTES, libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', apiName: 'JSON', elapsedTimeUs: 0, cpuTimeUs: -1, status: '[OK]', }; - bucket = stg.bucket(argv.bucket, { - preconditionOpts: { - ifGenerationMatch: 0, - }, - }); - - if (checkType === 0) { - start = performance.now(); - await bucket.upload(`${__dirname}/${fileName}`, {validation: false}); - end = performance.now(); - } else if (checkType === 1) { - iterationResult.crc32Enabled = true; - start = performance.now(); - await bucket.upload(`${__dirname}/${fileName}`, {validation: 'crc32c'}); - end = performance.now(); - } else { - iterationResult.md5Enabled = true; - start = performance.now(); - await bucket.upload(`${__dirname}/${fileName}`, {validation: 'md5'}); - end = performance.now(); - } + start = performance.now(); + await bucket.upload(`${__dirname}/${fileName}`, {validation: checkType}); + end = performance.now(); iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); results.push(iterationResult); } - bucket = stg.bucket(argv.bucket); for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { let start = 0; let end = 0; @@ -128,8 +106,8 @@ async function performWriteReadTest(): Promise { objectSize: sizeInBytes, appBufferSize: BLOCK_SIZE_IN_BYTES, libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', apiName: 'JSON', elapsedTimeUs: 0, cpuTimeUs: -1, @@ -138,21 +116,11 @@ async function performWriteReadTest(): Promise { const destinationFileName = generateRandomFileName(TEST_NAME_STRING); const destination = path.join(__dirname, destinationFileName); - if (checkType === 0) { - start = performance.now(); - await file.download({validation: false, destination}); - end = performance.now(); - } else if (checkType === 1) { - iterationResult.crc32Enabled = true; - start = performance.now(); - await file.download({validation: 'crc32c', destination}); - end = performance.now(); - } else { - iterationResult.md5Enabled = true; - start = performance.now(); - await file.download({validation: 'md5', destination}); - end = performance.now(); - } + + start = performance.now(); + await file.download({validation: checkType, destination}); + end = performance.now(); + cleanupFile(destinationFileName); iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); results.push(iterationResult); diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index b4a1464a5..80e685c5f 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -17,17 +17,20 @@ // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; import yargs from 'yargs'; -import {Bucket, Storage, TransferManager} from '../src'; +import {Bucket, TransferManager} from '../src'; import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; import { BLOCK_SIZE_IN_BYTES, cleanupFile, DEFAULT_LARGE_FILE_SIZE_BYTES, + DEFAULT_PROJECT_ID, DEFAULT_SMALL_FILE_SIZE_BYTES, generateRandomDirectoryStructure, generateRandomFile, generateRandomFileName, + getValidationType, NODE_DEFAULT_HIGHWATER_MARK_BYTES, + performanceTestSetup, TestResult, } from './performanceUtils'; import {performance} from 'perf_hooks'; @@ -40,9 +43,9 @@ const DEFAULT_NUMBER_OF_OBJECTS = 1000; const DEFAULT_CHUNK_SIZE_BYTES = 16 * 1024 * 1024; const DIRECTORY_PROBABILITY = 0.1; -let stg: Storage; let bucket: Bucket; -let tm: TransferManager; +let transferManager: TransferManager; +const checkType = getValidationType(); const argv = yargs(process.argv.slice(2)) .options({ @@ -52,7 +55,7 @@ const argv = yargs(process.argv.slice(2)) numpromises: {type: 'number', default: DEFAULT_NUMBER_OF_PROMISES}, numobjects: {type: 'number', default: DEFAULT_NUMBER_OF_OBJECTS}, chunksize: {type: 'number', default: DEFAULT_CHUNK_SIZE_BYTES}, - projectid: {type: 'string'}, + projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, testtype: { type: 'string', choices: [ @@ -64,6 +67,10 @@ const argv = yargs(process.argv.slice(2)) }) .parseSync(); +/** + * Main entry point. This function performs a test iteration and posts the message back + * to the parent thread. + */ async function main() { let result: TestResult = { op: '', @@ -77,7 +84,11 @@ async function main() { cpuTimeUs: 0, status: '[OK]', }; - await performTestSetup(); + + ({bucket, transferManager} = await performanceTestSetup( + argv.projectid, + argv.bucket + )); switch (argv.testtype) { case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS: @@ -96,27 +107,26 @@ async function main() { await performTestCleanup(); } -async function performTestSetup() { - stg = new Storage({projectId: argv.projectid}); - bucket = stg.bucket(argv.bucket); - if (!(await bucket.exists())[0]) { - await bucket.create(); - } - tm = new TransferManager(bucket); -} - +/** + * Cleans up after a test is complete by removing all files from the bucket + */ async function performTestCleanup() { await bucket.deleteFiles(); } +/** + * Performs a test where multiple objects are uploaded in parallel to a bucket. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ async function performUploadMultipleObjectsTest(): Promise { const result: TestResult = { op: 'WRITE', objectSize: 0, appBufferSize: BLOCK_SIZE_IN_BYTES, libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', apiName: 'JSON', elapsedTimeUs: 0, cpuTimeUs: -1, @@ -131,8 +141,11 @@ async function performUploadMultipleObjectsTest(): Promise { ); result.objectSize = creationInfo.totalSizeInBytes; const start = performance.now(); - await tm.uploadMulti(creationInfo.paths, { + await transferManager.uploadMulti(creationInfo.paths, { concurrencyLimit: argv.numpromises, + passthroughOptions: { + validation: checkType, + }, }); const end = performance.now(); @@ -142,14 +155,19 @@ async function performUploadMultipleObjectsTest(): Promise { return result; } +/** + * Performs a test where multiple objects are downloaded in parallel from a bucket. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ async function performDownloadMultipleObjectsTest(): Promise { const result: TestResult = { op: 'READ', objectSize: 0, appBufferSize: BLOCK_SIZE_IN_BYTES, libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', apiName: 'JSON', elapsedTimeUs: 0, cpuTimeUs: -1, @@ -164,24 +182,33 @@ async function performDownloadMultipleObjectsTest(): Promise { DIRECTORY_PROBABILITY ); result.objectSize = creationInfo.totalSizeInBytes; - await tm.uploadMulti(creationInfo.paths, { + await transferManager.uploadMulti(creationInfo.paths, { concurrencyLimit: argv.numpromises, + passthroughOptions: { + validation: checkType, + }, }); - rmSync(TEST_NAME_STRING, {recursive: true, force: true}); const getFilesResult = await bucket.getFiles(); const start = performance.now(); - await tm.downloadMulti(getFilesResult[0], { + await transferManager.downloadMulti(getFilesResult[0], { concurrencyLimit: argv.numpromises, - prefix: __dirname, + passthroughOptions: { + validation: checkType, + }, }); const end = performance.now(); result.elapsedTimeUs = Math.round((end - start) * 1000); - //rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); return result; } +/** + * Performs a test where a large file is downloaded as chunks in parallel. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ async function performDownloadLargeFileTest(): Promise { const fileName = generateRandomFileName(TEST_NAME_STRING); const sizeInBytes = generateRandomFile( @@ -207,7 +234,7 @@ async function performDownloadLargeFileTest(): Promise { await bucket.upload(`${__dirname}/${fileName}`); cleanupFile(fileName); const start = performance.now(); - await tm.downloadLargeFile(file, { + await transferManager.downloadLargeFile(file, { concurrencyLimit: argv.numpromises, chunkSizeBytes: argv.chunksize, path: `${__dirname}`, diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index 794f31c44..2109ccc55 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -17,12 +17,14 @@ import {execSync} from 'child_process'; import {mkdirSync, mkdtempSync, unlinkSync} from 'fs'; import path = require('path'); +import {Bucket, Storage, TransferManager} from '../src'; export const BLOCK_SIZE_IN_BYTES = 1024; export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; export const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; export const DEFAULT_DIRECTORY_PROBABILITY = 0.5; +export const DEFAULT_PROJECT_ID = 'GCS_NODE_PERFORMANCE_METRICS'; export interface TestResult { op: string; @@ -42,6 +44,12 @@ export interface RandomDirectoryCreationInformation { totalSizeInBytes: number; } +export interface PerformanceTestSetupResults { + storage: Storage; + bucket: Bucket; + transferManager: TransferManager; +} + /** * Create a uniformly distributed random integer beween the inclusive min and max provided. * @@ -171,3 +179,47 @@ export function cleanupFile( ): void { unlinkSync(`${directoryName}/${fileName}`); } + +/** + * Creates the necessary structures for performing a performance test. + * + * @param {string} projectId the project ID to use. + * @param {string} bucketName the name of the bucket to use. + * @returns {object} object containing the created storage, bucket, and transfer manager instance. + */ +export async function performanceTestSetup( + projectId: string, + bucketName: string +): Promise { + const storage = new Storage({projectId}); + const bucket = storage.bucket(bucketName, { + preconditionOpts: { + ifGenerationMatch: 0, + }, + }); + if (!(await bucket.exists())[0]) { + await bucket.create(); + } + const transferManager = new TransferManager(bucket); + return { + storage, + bucket, + transferManager, + }; +} + +/** + * Randomly returns the type of validation check to run on upload / download + * + * @returns {string | boolean | undefined} the type of validation to run (crc32c, md5, or none). + */ +export function getValidationType(): 'md5' | 'crc32c' | boolean | undefined { + const checkType = randomInteger(0, 2); + if (checkType === 0) { + return false; + } else if (checkType === 1) { + return 'crc32c'; + } else { + return 'md5'; + } +} From 9793cc697f43db0d7f41f4d9a36d68e8b587f538 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Wed, 16 Nov 2022 15:10:55 -0500 Subject: [PATCH 18/25] refactor: refactor constants (#2105) * refactor: refactor constants * bug fixes * bug fixes * bug fixes --- .../performApplicationPerformanceTest.ts | 166 +++++++++--------- .../performTransferManagerTest.ts | 82 +++++---- internal-tooling/performanceUtils.ts | 3 +- 3 files changed, 129 insertions(+), 122 deletions(-) diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts index 7ff67e07f..f5d8b615d 100644 --- a/internal-tooling/performApplicationPerformanceTest.ts +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -15,7 +15,7 @@ */ import yargs from 'yargs'; -import {promises as fsp} from 'fs'; +import {promises as fsp, rmSync} from 'fs'; import { Bucket, DownloadOptions, @@ -27,7 +27,12 @@ import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; import { + BLOCK_SIZE_IN_BYTES, DEFAULT_PROJECT_ID, + DEFAULT_NUMBER_OF_OBJECTS, + DEFAULT_SMALL_FILE_SIZE_BYTES, + DEFAULT_LARGE_FILE_SIZE_BYTES, + NODE_DEFAULT_HIGHWATER_MARK_BYTES, generateRandomDirectoryStructure, getValidationType, performanceTestSetup, @@ -35,14 +40,8 @@ import { } from './performanceUtils'; import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; -const TEST_NAME_STRING = 'nodejs-perf-metrics'; -const DEFAULT_NUMBER_OF_WRITES = 1; -const DEFAULT_NUMBER_OF_READS = 3; +const TEST_NAME_STRING = 'nodejs-perf-metrics-application'; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics-shaffeeullah'; -const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; -const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; -const BLOCK_SIZE_IN_BYTES = 1024; -const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; let stg: Storage; let bucket: Bucket; @@ -55,6 +54,7 @@ const argv = yargs(process.argv.slice(2)) small: {type: 'number', default: DEFAULT_SMALL_FILE_SIZE_BYTES}, large: {type: 'number', default: DEFAULT_LARGE_FILE_SIZE_BYTES}, projectid: {type: 'string', default: DEFAULT_PROJECT_ID}, + numobjects: {type: 'number', default: DEFAULT_NUMBER_OF_OBJECTS}, }) .parseSync(); @@ -63,23 +63,35 @@ const argv = yargs(process.argv.slice(2)) * to the parent thread. */ async function main() { - let results: TestResult[] = []; + let result: TestResult = { + op: '', + objectSize: 0, + appBufferSize: 0, + libBufferSize: 0, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: 0, + status: '[OK]', + }; + ({bucket} = await performanceTestSetup(argv.projectid, argv.bucket)); switch (argv.testtype) { case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS: - results = await performWriteTest(); + result = await performWriteTest(); break; case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS: - results = await performReadTest(); + result = await performReadTest(); break; // case TRANSFER_MANAGER_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD: - // results = await performLargeReadTest(); + // result = await performLargeReadTest(); // break; default: break; } - parentPort?.postMessage(results); + parentPort?.postMessage(result); } async function uploadInParallel( @@ -112,79 +124,75 @@ async function downloadInParallel(bucket: Bucket, options: DownloadOptions) { /** * Performs an iteration of the Write multiple objects test. * - * @returns {Promise} Promise that resolves to an array of test results for the iteration. + * @returns {Promise} Promise that resolves to a test result of an iteration. */ -async function performWriteTest(): Promise { - const results: TestResult[] = []; - const directory = TEST_NAME_STRING; - const directories = generateRandomDirectoryStructure(10, directory); - - for (let j = 0; j < DEFAULT_NUMBER_OF_WRITES; j++) { - let start = 0; - let end = 0; - - const iterationResult: TestResult = { - op: 'WRITE', - objectSize: BLOCK_SIZE_IN_BYTES, //note this is wrong - appBufferSize: BLOCK_SIZE_IN_BYTES, - libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, - apiName: 'JSON', - elapsedTimeUs: 0, - cpuTimeUs: -1, - status: '[OK]', - }; - - bucket = stg.bucket(argv.bucket, { - preconditionOpts: { - ifGenerationMatch: 0, - }, - }); - - await bucket.deleteFiles(); //cleanup anything old - start = performance.now(); - await uploadInParallel(bucket, directories.paths, {validation: checkType}); - end = performance.now(); - - iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); - results.push(iterationResult); - } - return results; +async function performWriteTest(): Promise { + await bucket.deleteFiles(); //start clean + + const creationInfo = generateRandomDirectoryStructure( + argv.numobjects, + TEST_NAME_STRING, + argv.small, + argv.large + ); + + const start = performance.now(); + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); + const end = performance.now(); + + await bucket.deleteFiles(); //cleanup files + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + + const result: TestResult = { + op: 'WRITE', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + apiName: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: '[OK]', + }; + return result; } /** * Performs an iteration of the read multiple objects test. * - * @returns {Promise} Promise that resolves to an array of test results for the iteration. + * @returns {Promise} Promise that resolves to an array of test results for the iteration. */ -async function performReadTest(): Promise { - const results: TestResult[] = []; - bucket = stg.bucket(argv.bucket); - for (let j = 0; j < DEFAULT_NUMBER_OF_READS; j++) { - let start = 0; - let end = 0; - const iterationResult: TestResult = { - op: `READ[${j}]`, - objectSize: 0, //this is wrong - appBufferSize: BLOCK_SIZE_IN_BYTES, - libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, - apiName: 'JSON', - elapsedTimeUs: 0, - cpuTimeUs: -1, - status: '[OK]', - }; - - start = performance.now(); - await downloadInParallel(bucket, {validation: checkType}); - end = performance.now(); - - iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); - results.push(iterationResult); - } - return results; +async function performReadTest(): Promise { + await bucket.deleteFiles(); // start clean + const creationInfo = generateRandomDirectoryStructure( + argv.numobjects, + TEST_NAME_STRING, + argv.small, + argv.large + ); + await uploadInParallel(bucket, creationInfo.paths, {validation: checkType}); + + const start = performance.now(); + await downloadInParallel(bucket, {validation: checkType}); + const end = performance.now(); + + const result: TestResult = { + op: 'READ', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + apiName: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: '[OK]', + }; + + rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + await bucket.deleteFiles(); //cleanup + return result; } main(); diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index 80e685c5f..5d31515c8 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -30,6 +30,7 @@ import { generateRandomFileName, getValidationType, NODE_DEFAULT_HIGHWATER_MARK_BYTES, + DEFAULT_NUMBER_OF_OBJECTS, performanceTestSetup, TestResult, } from './performanceUtils'; @@ -39,7 +40,6 @@ import {rmSync} from 'fs'; const TEST_NAME_STRING = 'tm-perf-metrics'; const DEFAULT_BUCKET_NAME = 'nodejs-transfer-manager-perf-metrics'; const DEFAULT_NUMBER_OF_PROMISES = 2; -const DEFAULT_NUMBER_OF_OBJECTS = 1000; const DEFAULT_CHUNK_SIZE_BYTES = 16 * 1024 * 1024; const DIRECTORY_PROBABILITY = 0.1; @@ -120,18 +120,6 @@ async function performTestCleanup() { * @returns {Promise} A promise that resolves containing information about the test results. */ async function performUploadMultipleObjectsTest(): Promise { - const result: TestResult = { - op: 'WRITE', - objectSize: 0, - appBufferSize: BLOCK_SIZE_IN_BYTES, - libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: checkType === 'crc32c', - md5Enabled: checkType === 'md5', - apiName: 'JSON', - elapsedTimeUs: 0, - cpuTimeUs: -1, - status: '[OK]', - }; const creationInfo = generateRandomDirectoryStructure( argv.numobjects, TEST_NAME_STRING, @@ -139,7 +127,7 @@ async function performUploadMultipleObjectsTest(): Promise { argv.large, DIRECTORY_PROBABILITY ); - result.objectSize = creationInfo.totalSizeInBytes; + const start = performance.now(); await transferManager.uploadMulti(creationInfo.paths, { concurrencyLimit: argv.numpromises, @@ -149,31 +137,30 @@ async function performUploadMultipleObjectsTest(): Promise { }); const end = performance.now(); - result.elapsedTimeUs = Math.round((end - start) * 1000); rmSync(TEST_NAME_STRING, {recursive: true, force: true}); - return result; -} - -/** - * Performs a test where multiple objects are downloaded in parallel from a bucket. - * - * @returns {Promise} A promise that resolves containing information about the test results. - */ -async function performDownloadMultipleObjectsTest(): Promise { const result: TestResult = { - op: 'READ', - objectSize: 0, + op: 'WRITE', + objectSize: creationInfo.totalSizeInBytes, appBufferSize: BLOCK_SIZE_IN_BYTES, libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, crc32Enabled: checkType === 'crc32c', md5Enabled: checkType === 'md5', apiName: 'JSON', - elapsedTimeUs: 0, + elapsedTimeUs: Math.round((end - start) * 1000), cpuTimeUs: -1, status: '[OK]', }; + return result; +} + +/** + * Performs a test where multiple objects are downloaded in parallel from a bucket. + * + * @returns {Promise} A promise that resolves containing information about the test results. + */ +async function performDownloadMultipleObjectsTest(): Promise { const creationInfo = generateRandomDirectoryStructure( argv.numobjects, TEST_NAME_STRING, @@ -181,7 +168,7 @@ async function performDownloadMultipleObjectsTest(): Promise { argv.large, DIRECTORY_PROBABILITY ); - result.objectSize = creationInfo.totalSizeInBytes; + await transferManager.uploadMulti(creationInfo.paths, { concurrencyLimit: argv.numpromises, passthroughOptions: { @@ -198,9 +185,20 @@ async function performDownloadMultipleObjectsTest(): Promise { }); const end = performance.now(); - result.elapsedTimeUs = Math.round((end - start) * 1000); rmSync(TEST_NAME_STRING, {recursive: true, force: true}); + const result: TestResult = { + op: 'READ', + objectSize: creationInfo.totalSizeInBytes, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + apiName: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: '[OK]', + }; return result; } @@ -217,18 +215,6 @@ async function performDownloadLargeFileTest(): Promise { argv.large, __dirname ); - const result: TestResult = { - op: 'READ', - objectSize: sizeInBytes, - appBufferSize: BLOCK_SIZE_IN_BYTES, - libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, - crc32Enabled: false, - md5Enabled: false, - apiName: 'JSON', - elapsedTimeUs: 0, - cpuTimeUs: -1, - status: '[OK]', - }; const file = bucket.file(`${fileName}`); await bucket.upload(`${__dirname}/${fileName}`); @@ -241,9 +227,21 @@ async function performDownloadLargeFileTest(): Promise { }); const end = performance.now(); - result.elapsedTimeUs = Math.round((end - start) * 1000); cleanupFile(fileName); + const result: TestResult = { + op: 'READ', + objectSize: sizeInBytes, + appBufferSize: BLOCK_SIZE_IN_BYTES, + libBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32Enabled: false, + md5Enabled: false, + apiName: 'JSON', + elapsedTimeUs: Math.round((end - start) * 1000), + cpuTimeUs: -1, + status: '[OK]', + }; + return result; } diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index 2109ccc55..df8f69b08 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -23,8 +23,9 @@ export const BLOCK_SIZE_IN_BYTES = 1024; export const DEFAULT_SMALL_FILE_SIZE_BYTES = 5120; export const DEFAULT_LARGE_FILE_SIZE_BYTES = 2.147e9; export const NODE_DEFAULT_HIGHWATER_MARK_BYTES = 16384; -export const DEFAULT_DIRECTORY_PROBABILITY = 0.5; +export const DEFAULT_DIRECTORY_PROBABILITY = 0.1; export const DEFAULT_PROJECT_ID = 'GCS_NODE_PERFORMANCE_METRICS'; +export const DEFAULT_NUMBER_OF_OBJECTS = 1000; export interface TestResult { op: string; From 89e8204a35394533030aba4efab807143b166641 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 29 Nov 2022 14:56:48 +0000 Subject: [PATCH 19/25] linter fixes, download to disk for performance test --- internal-tooling/performApplicationPerformanceTest.ts | 9 +-------- internal-tooling/performTransferManagerTest.ts | 2 ++ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/internal-tooling/performApplicationPerformanceTest.ts b/internal-tooling/performApplicationPerformanceTest.ts index f5d8b615d..3ece2633e 100644 --- a/internal-tooling/performApplicationPerformanceTest.ts +++ b/internal-tooling/performApplicationPerformanceTest.ts @@ -16,13 +16,7 @@ import yargs from 'yargs'; import {promises as fsp, rmSync} from 'fs'; -import { - Bucket, - DownloadOptions, - DownloadResponse, - Storage, - UploadOptions, -} from '../src'; +import {Bucket, DownloadOptions, DownloadResponse, UploadOptions} from '../src'; import {performance} from 'perf_hooks'; // eslint-disable-next-line node/no-unsupported-features/node-builtins import {parentPort} from 'worker_threads'; @@ -43,7 +37,6 @@ import {TRANSFER_MANAGER_TEST_TYPES} from './performanceTest'; const TEST_NAME_STRING = 'nodejs-perf-metrics-application'; const DEFAULT_BUCKET_NAME = 'nodejs-perf-metrics-shaffeeullah'; -let stg: Storage; let bucket: Bucket; const checkType = getValidationType(); diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index 5d31515c8..80465e4f3 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -36,6 +36,7 @@ import { } from './performanceUtils'; import {performance} from 'perf_hooks'; import {rmSync} from 'fs'; +import * as path from 'path'; const TEST_NAME_STRING = 'tm-perf-metrics'; const DEFAULT_BUCKET_NAME = 'nodejs-transfer-manager-perf-metrics'; @@ -178,6 +179,7 @@ async function performDownloadMultipleObjectsTest(): Promise { const getFilesResult = await bucket.getFiles(); const start = performance.now(); await transferManager.downloadMulti(getFilesResult[0], { + prefix: path.join(__dirname, '..', '..'), concurrencyLimit: argv.numpromises, passthroughOptions: { validation: checkType, From c153ab6feea14443cc8e86a17932128b7f24c037 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 29 Nov 2022 16:29:25 +0000 Subject: [PATCH 20/25] rename transfer manager functions --- .../performTransferManagerTest.ts | 22 ++-- src/transfer-manager.ts | 105 +++++++++--------- test/transfer-manager.ts | 28 ++--- 3 files changed, 76 insertions(+), 79 deletions(-) diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index 80465e4f3..c1e432fa7 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -93,13 +93,13 @@ async function main() { switch (argv.testtype) { case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MULTIPLE_OBJECTS: - result = await performUploadMultipleObjectsTest(); + result = await performUploadManyFilesTest(); break; case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MULTIPLE_OBJECTS: - result = await performDownloadMultipleObjectsTest(); + result = await performDownloadManyFilesTest(); break; case TRANSFER_MANAGER_TEST_TYPES.TRANSFER_MANAGER_LARGE_FILE_DOWNLOAD: - result = await performDownloadLargeFileTest(); + result = await performDownloadFileInChunksTest(); break; default: break; @@ -120,7 +120,7 @@ async function performTestCleanup() { * * @returns {Promise} A promise that resolves containing information about the test results. */ -async function performUploadMultipleObjectsTest(): Promise { +async function performUploadManyFilesTest(): Promise { const creationInfo = generateRandomDirectoryStructure( argv.numobjects, TEST_NAME_STRING, @@ -130,7 +130,7 @@ async function performUploadMultipleObjectsTest(): Promise { ); const start = performance.now(); - await transferManager.uploadMulti(creationInfo.paths, { + await transferManager.uploadManyFiles(creationInfo.paths, { concurrencyLimit: argv.numpromises, passthroughOptions: { validation: checkType, @@ -161,7 +161,7 @@ async function performUploadMultipleObjectsTest(): Promise { * * @returns {Promise} A promise that resolves containing information about the test results. */ -async function performDownloadMultipleObjectsTest(): Promise { +async function performDownloadManyFilesTest(): Promise { const creationInfo = generateRandomDirectoryStructure( argv.numobjects, TEST_NAME_STRING, @@ -170,7 +170,7 @@ async function performDownloadMultipleObjectsTest(): Promise { DIRECTORY_PROBABILITY ); - await transferManager.uploadMulti(creationInfo.paths, { + await transferManager.uploadManyFiles(creationInfo.paths, { concurrencyLimit: argv.numpromises, passthroughOptions: { validation: checkType, @@ -178,7 +178,7 @@ async function performDownloadMultipleObjectsTest(): Promise { }); const getFilesResult = await bucket.getFiles(); const start = performance.now(); - await transferManager.downloadMulti(getFilesResult[0], { + await transferManager.downloadManyFiles(getFilesResult[0], { prefix: path.join(__dirname, '..', '..'), concurrencyLimit: argv.numpromises, passthroughOptions: { @@ -209,7 +209,7 @@ async function performDownloadMultipleObjectsTest(): Promise { * * @returns {Promise} A promise that resolves containing information about the test results. */ -async function performDownloadLargeFileTest(): Promise { +async function performDownloadFileInChunksTest(): Promise { const fileName = generateRandomFileName(TEST_NAME_STRING); const sizeInBytes = generateRandomFile( fileName, @@ -222,10 +222,10 @@ async function performDownloadLargeFileTest(): Promise { await bucket.upload(`${__dirname}/${fileName}`); cleanupFile(fileName); const start = performance.now(); - await transferManager.downloadLargeFile(file, { + await transferManager.downloadFileInChunks(file, { concurrencyLimit: argv.numpromises, chunkSizeBytes: argv.chunksize, - path: `${__dirname}`, + destination: path.join(__dirname, fileName), }); const end = performance.now(); diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 97ff92c37..426c79948 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -33,31 +33,31 @@ const DEFAULT_PARALLEL_LARGE_FILE_DOWNLOAD_LIMIT = 2; const LARGE_FILE_SIZE_THRESHOLD = 256 * 1024 * 1024; const LARGE_FILE_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; const EMPTY_REGEX = '(?:)'; -export interface UploadMultiOptions { +export interface UploadManyFilesOptions { concurrencyLimit?: number; skipIfExists?: boolean; prefix?: string; passthroughOptions?: Omit; } -export interface DownloadMultiOptions { +export interface DownloadManyFilesOptions { concurrencyLimit?: number; prefix?: string; stripPrefix?: string; passthroughOptions?: DownloadOptions; } -export interface LargeFileDownloadOptions { +export interface DownloadFileInChunksOptions { concurrencyLimit?: number; chunkSizeBytes?: number; - path?: string; + destination?: string; } -export interface UploadMultiCallback { +export interface UploadManyFilesCallback { (err: Error | null, files?: File[], metadata?: Metadata[]): void; } -export interface DownloadMultiCallback { +export interface DownloadManyFilesCallback { (err: Error | null, contents?: Buffer[]): void; } @@ -76,21 +76,21 @@ export class TransferManager { this.bucket = bucket; } - async uploadMulti( + async uploadManyFiles( filePaths: string[], - options?: UploadMultiOptions + options?: UploadManyFilesOptions ): Promise; - async uploadMulti( + async uploadManyFiles( filePaths: string[], - callback: UploadMultiCallback + callback: UploadManyFilesCallback ): Promise; - async uploadMulti( + async uploadManyFiles( filePaths: string[], - options: UploadMultiOptions, - callback: UploadMultiCallback + options: UploadManyFilesOptions, + callback: UploadManyFilesCallback ): Promise; /** - * @typedef {object} UploadMultiOptions + * @typedef {object} UploadManyFilesOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises * to use when uploading the files. * @property {boolean} [skipIfExists] Do not upload the file if it already exists in @@ -101,7 +101,7 @@ export class TransferManager { * @experimental */ /** - * @callback UploadMultiCallback + * @callback UploadManyFilesCallback * @param {?Error} [err] Request error, if any. * @param {array} [files] Array of uploaded {@link File}. * @param {array} [metadata] Array of uploaded {@link Metadata} @@ -113,8 +113,8 @@ export class TransferManager { * * @param {array} [filePaths] An array of fully qualified paths to the files. * to be uploaded to the bucket - * @param {UploadMultiOptions} [options] Configuration options. - * @param {UploadMultiCallback} [callback] Callback function. + * @param {UploadManyFilesOptions} [options] Configuration options. + * @param {UploadManyFilesCallback} [callback] Callback function. * @returns {Promise} * * @example @@ -127,7 +127,7 @@ export class TransferManager { * //- * // Upload multiple files in parallel. * //- - * transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt'], function(err, files, metadata) { + * transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt'], function(err, files, metadata) { * // Your bucket now contains: * // - "file1.txt" (with the contents of '/local/path/file1.txt') * // - "file2.txt" (with the contents of '/local/path/file2.txt') @@ -137,16 +137,16 @@ export class TransferManager { * //- * // If the callback is omitted, we will return a promise. * //- - * const response = await transferManager.uploadMulti(['/local/path/file1.txt, 'local/path/file2.txt']); + * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']); * ``` * @experimental */ - async uploadMulti( + async uploadManyFiles( filePaths: string[], - optionsOrCallback?: UploadMultiOptions | UploadMultiCallback, - callback?: UploadMultiCallback + optionsOrCallback?: UploadManyFilesOptions | UploadManyFilesCallback, + callback?: UploadManyFilesCallback ): Promise { - let options: UploadMultiOptions = {}; + let options: UploadManyFilesOptions = {}; if (typeof optionsOrCallback === 'function') { callback = optionsOrCallback; } else if (optionsOrCallback) { @@ -208,21 +208,21 @@ export class TransferManager { return Promise.all(promises); } - async downloadMulti( + async downloadManyFiles( files: File[], - options?: DownloadMultiOptions + options?: DownloadManyFilesOptions ): Promise; - async downloadMulti( + async downloadManyFiles( files: File[], - callback: DownloadMultiCallback + callback: DownloadManyFilesCallback ): Promise; - async downloadMulti( + async downloadManyFiles( files: File[], - options: DownloadMultiOptions, - callback: DownloadMultiCallback + options: DownloadManyFilesOptions, + callback: DownloadManyFilesCallback ): Promise; /** - * @typedef {object} DownloadMultiOptions + * @typedef {object} DownloadManyFilesOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises * to use when downloading the files. * @property {string} [prefix] A prefix to append to all of the downloaded files. @@ -232,7 +232,7 @@ export class TransferManager { * @experimental */ /** - * @callback DownloadMultiCallback + * @callback DownloadManyFilesCallback * @param {?Error} [err] Request error, if any. * @param {array} [contents] Contents of the downloaded files. * @experimental @@ -242,8 +242,8 @@ export class TransferManager { * that utilizes {@link File#download} to perform the download. * * @param {array} [files] An array of file objects to be downloaded. - * @param {DownloadMultiOptions} [options] Configuration options. - * @param {DownloadMultiCallback} {callback} Callback function. + * @param {DownloadManyFilesOptions} [options] Configuration options. + * @param {DownloadManyFilesCallback} {callback} Callback function. * @returns {Promise} * * @example @@ -256,7 +256,7 @@ export class TransferManager { * //- * // Download multiple files in parallel. * //- - * transferManager.downloadMulti([bucket.file('file1.txt'), bucket.file('file2.txt')], function(err, contents){ + * transferManager.downloadManyFiles([bucket.file('file1.txt'), bucket.file('file2.txt')], function(err, contents){ * // Your local directory now contains: * // - "file1.txt" (with the contents from my-bucket.file1.txt) * // - "file2.txt" (with the contents from my-bucket.file2.txt) @@ -266,15 +266,15 @@ export class TransferManager { * //- * // If the callback is omitted, we will return a promise. * //- - * const response = await transferManager.downloadMulti(bucket.File('file1.txt'), bucket.File('file2.txt')]); + * const response = await transferManager.downloadManyFiles(bucket.File('file1.txt'), bucket.File('file2.txt')]); * @experimental */ - async downloadMulti( + async downloadManyFiles( files: File[], - optionsOrCallback?: DownloadMultiOptions | DownloadMultiCallback, - callback?: DownloadMultiCallback + optionsOrCallback?: DownloadManyFilesOptions | DownloadManyFilesCallback, + callback?: DownloadManyFilesCallback ): Promise { - let options: DownloadMultiOptions = {}; + let options: DownloadManyFilesOptions = {}; if (typeof optionsOrCallback === 'function') { callback = optionsOrCallback; } else if (optionsOrCallback) { @@ -323,21 +323,21 @@ export class TransferManager { return Promise.all(promises); } - async downloadLargeFile( + async downloadFileInChunks( file: File, - options?: LargeFileDownloadOptions + options?: DownloadFileInChunksOptions ): Promise; - async downloadLargeFile( + async downloadFileInChunks( file: File, callback: DownloadCallback ): Promise; - async downloadLargeFile( + async downloadFileInChunks( file: File, - options: LargeFileDownloadOptions, + options: DownloadFileInChunksOptions, callback: DownloadCallback ): Promise; /** - * @typedef {object} LargeFileDownloadOptions + * @typedef {object} DownloadFileInChunksOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises * to use when downloading the file. * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded. @@ -348,7 +348,7 @@ export class TransferManager { * that utilizes {@link File#download} to perform the download. * * @param {object} [file] {@link File} to download. - * @param {LargeFileDownloadOptions} [options] Configuration options. + * @param {DownloadFileInChunksOptions} [options] Configuration options. * @param {DownloadCallback} [callbac] Callback function. * @returns {Promise} * @@ -373,12 +373,12 @@ export class TransferManager { * const response = await transferManager.downloadLargeFile(bucket.file('large-file.txt'); * @experimental */ - async downloadLargeFile( + async downloadFileInChunks( file: File, - optionsOrCallback?: LargeFileDownloadOptions | DownloadCallback, + optionsOrCallback?: DownloadFileInChunksOptions | DownloadCallback, callback?: DownloadCallback ): Promise { - let options: LargeFileDownloadOptions = {}; + let options: DownloadFileInChunksOptions = {}; if (typeof optionsOrCallback === 'function') { callback = optionsOrCallback; } else if (optionsOrCallback) { @@ -400,10 +400,7 @@ export class TransferManager { } let start = 0; - const filePath = path.join( - options.path || __dirname, - path.basename(file.name) - ); + const filePath = options.destination || path.basename(file.name); const fileToWrite = await fsp.open(filePath, 'w+'); while (start < size) { const chunkStart = start; diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 1386cc6a2..43d09c5c9 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -185,7 +185,7 @@ describe('Transfer Manager', () => { }); }); - describe('uploadMulti', () => { + describe('uploadManyFiles', () => { it('calls upload with the provided file paths', async () => { const paths = ['/a/b/c', '/d/e/f', '/h/i/j']; let count = 0; @@ -195,7 +195,7 @@ describe('Transfer Manager', () => { assert(paths.includes(path)); }; - await transferManager.uploadMulti(paths); + await transferManager.uploadManyFiles(paths); assert.strictEqual(count, paths.length); }); @@ -206,7 +206,7 @@ describe('Transfer Manager', () => { assert.strictEqual(options.preconditionOpts?.ifGenerationMatch, 0); }; - await transferManager.uploadMulti(paths, {skipIfExists: true}); + await transferManager.uploadManyFiles(paths, {skipIfExists: true}); }); it('sets destination to prefix + filename when prefix is supplied', async () => { @@ -217,13 +217,13 @@ describe('Transfer Manager', () => { assert.strictEqual(options.destination, expectedDestination); }; - await transferManager.uploadMulti(paths, {prefix: 'hello/world'}); + await transferManager.uploadManyFiles(paths, {prefix: 'hello/world'}); }); it('invokes the callback if one is provided', done => { const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; - transferManager.uploadMulti( + transferManager.uploadManyFiles( paths, (err: Error | null, files?: File[], metadata?: Metadata[]) => { assert.ifError(err); @@ -237,12 +237,12 @@ describe('Transfer Manager', () => { it('returns a promise with the uploaded file if there is no callback', async () => { const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; - const result = await transferManager.uploadMulti(paths); + const result = await transferManager.uploadManyFiles(paths); assert.strictEqual(result[0][0].name, paths[0]); }); }); - describe('downloadMulti', () => { + describe('downloadManyFiles', () => { it('calls download for each provided file', async () => { let count = 0; const download = () => { @@ -254,7 +254,7 @@ describe('Transfer Manager', () => { secondFile.download = download; const files = [firstFile, secondFile]; - await transferManager.downloadMulti(files); + await transferManager.downloadManyFiles(files); assert.strictEqual(count, 2); }); @@ -268,7 +268,7 @@ describe('Transfer Manager', () => { const file = new File(bucket, filename); file.download = download; - await transferManager.downloadMulti([file], {prefix}); + await transferManager.downloadManyFiles([file], {prefix}); }); it('sets the destination correctly when provided a strip prefix', async () => { @@ -281,7 +281,7 @@ describe('Transfer Manager', () => { const file = new File(bucket, filename); file.download = download; - await transferManager.downloadMulti([file], {stripPrefix}); + await transferManager.downloadManyFiles([file], {stripPrefix}); }); it('invokes the callback if one if provided', done => { @@ -289,7 +289,7 @@ describe('Transfer Manager', () => { file.download = () => { return Promise.resolve(Buffer.alloc(100)); }; - transferManager.downloadMulti( + transferManager.downloadManyFiles( [file], (err: Error | null, contents: Buffer[]) => { assert.strictEqual(err, null); @@ -300,7 +300,7 @@ describe('Transfer Manager', () => { }); }); - describe('downloadLargeFile', () => { + describe('downloadFileInChunks', () => { let file: any; beforeEach(() => { @@ -323,7 +323,7 @@ describe('Transfer Manager', () => { return Promise.resolve([Buffer.alloc(100)]); }; - await transferManager.downloadLargeFile(file); + await transferManager.downloadFileInChunks(file); assert.strictEqual(downloadCallCount, 1); }); @@ -332,7 +332,7 @@ describe('Transfer Manager', () => { return Promise.resolve([Buffer.alloc(100)]); }; - transferManager.downloadLargeFile( + transferManager.downloadFileInChunks( file, (err: Error, contents: Buffer) => { assert.equal(err, null); From 4feb1c2c7a0fb8778b4a67b312454960f6c6ee70 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 29 Nov 2022 18:00:48 +0000 Subject: [PATCH 21/25] remove callbacks from transfer manager --- src/transfer-manager.ts | 185 +++++---------------------------------- test/transfer-manager.ts | 53 +---------- 2 files changed, 22 insertions(+), 216 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 426c79948..874531aa9 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -15,23 +15,17 @@ */ import {Bucket, UploadOptions, UploadResponse} from './bucket'; -import { - DownloadCallback, - DownloadOptions, - DownloadResponse, - File, -} from './file'; +import {DownloadOptions, DownloadResponse, File} from './file'; import * as pLimit from 'p-limit'; -import {Metadata} from './nodejs-common'; import * as path from 'path'; import * as extend from 'extend'; import {promises as fsp} from 'fs'; const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; -const DEFAULT_PARALLEL_LARGE_FILE_DOWNLOAD_LIMIT = 2; -const LARGE_FILE_SIZE_THRESHOLD = 256 * 1024 * 1024; -const LARGE_FILE_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; +const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 2; +const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 256 * 1024 * 1024; +const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; const EMPTY_REGEX = '(?:)'; export interface UploadManyFilesOptions { concurrencyLimit?: number; @@ -53,14 +47,6 @@ export interface DownloadFileInChunksOptions { destination?: string; } -export interface UploadManyFilesCallback { - (err: Error | null, files?: File[], metadata?: Metadata[]): void; -} - -export interface DownloadManyFilesCallback { - (err: Error | null, contents?: Buffer[]): void; -} - /** * Create a TransferManager object to perform parallel transfer operations on a Cloud Storage bucket. * @@ -76,19 +62,6 @@ export class TransferManager { this.bucket = bucket; } - async uploadManyFiles( - filePaths: string[], - options?: UploadManyFilesOptions - ): Promise; - async uploadManyFiles( - filePaths: string[], - callback: UploadManyFilesCallback - ): Promise; - async uploadManyFiles( - filePaths: string[], - options: UploadManyFilesOptions, - callback: UploadManyFilesCallback - ): Promise; /** * @typedef {object} UploadManyFilesOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises @@ -100,13 +73,6 @@ export class TransferManager { * to each individual upload operation. * @experimental */ - /** - * @callback UploadManyFilesCallback - * @param {?Error} [err] Request error, if any. - * @param {array} [files] Array of uploaded {@link File}. - * @param {array} [metadata] Array of uploaded {@link Metadata} - * @experimental - */ /** * Upload multiple files in parallel to the bucket. This is a convenience method * that utilizes {@link Bucket#upload} to perform the upload. @@ -114,8 +80,7 @@ export class TransferManager { * @param {array} [filePaths] An array of fully qualified paths to the files. * to be uploaded to the bucket * @param {UploadManyFilesOptions} [options] Configuration options. - * @param {UploadManyFilesCallback} [callback] Callback function. - * @returns {Promise} + * @returns {Promise} * * @example * ``` @@ -127,32 +92,17 @@ export class TransferManager { * //- * // Upload multiple files in parallel. * //- - * transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt'], function(err, files, metadata) { + * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']); * // Your bucket now contains: * // - "file1.txt" (with the contents of '/local/path/file1.txt') * // - "file2.txt" (with the contents of '/local/path/file2.txt') - * // `files` is an array of instances of File objects that refers to the new files. - * }); - * - * //- - * // If the callback is omitted, we will return a promise. - * //- - * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']); * ``` * @experimental */ async uploadManyFiles( filePaths: string[], - optionsOrCallback?: UploadManyFilesOptions | UploadManyFilesCallback, - callback?: UploadManyFilesCallback - ): Promise { - let options: UploadManyFilesOptions = {}; - if (typeof optionsOrCallback === 'function') { - callback = optionsOrCallback; - } else if (optionsOrCallback) { - options = optionsOrCallback; - } - + options: UploadManyFilesOptions = {} + ): Promise { if (options.skipIfExists && options.passthroughOptions?.preconditionOpts) { options.passthroughOptions.preconditionOpts.ifGenerationMatch = 0; } else if ( @@ -193,34 +143,9 @@ export class TransferManager { ); } - if (callback) { - try { - const results = await Promise.all(promises); - const files = results.map(fileAndMetadata => fileAndMetadata[0]); - const metadata = results.map(fileAndMetadata => fileAndMetadata[1]); - callback(null, files, metadata); - } catch (e) { - callback(e as Error); - } - return; - } - return Promise.all(promises); } - async downloadManyFiles( - files: File[], - options?: DownloadManyFilesOptions - ): Promise; - async downloadManyFiles( - files: File[], - callback: DownloadManyFilesCallback - ): Promise; - async downloadManyFiles( - files: File[], - options: DownloadManyFilesOptions, - callback: DownloadManyFilesCallback - ): Promise; /** * @typedef {object} DownloadManyFilesOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises @@ -231,20 +156,13 @@ export class TransferManager { * to each individual download operation. * @experimental */ - /** - * @callback DownloadManyFilesCallback - * @param {?Error} [err] Request error, if any. - * @param {array} [contents] Contents of the downloaded files. - * @experimental - */ /** * Download multiple files in parallel to the local filesystem. This is a convenience method * that utilizes {@link File#download} to perform the download. * * @param {array} [files] An array of file objects to be downloaded. * @param {DownloadManyFilesOptions} [options] Configuration options. - * @param {DownloadManyFilesCallback} {callback} Callback function. - * @returns {Promise} + * @returns {Promise} * * @example * ``` @@ -256,31 +174,17 @@ export class TransferManager { * //- * // Download multiple files in parallel. * //- - * transferManager.downloadManyFiles([bucket.file('file1.txt'), bucket.file('file2.txt')], function(err, contents){ - * // Your local directory now contains: + * const response = await transferManager.downloadManyFiles(bucket.File('file1.txt'), bucket.File('file2.txt')]); + * // The following files have been downloaded: * // - "file1.txt" (with the contents from my-bucket.file1.txt) * // - "file2.txt" (with the contents from my-bucket.file2.txt) - * // `contents` is an array containing the file data for each downloaded file. - * }); - * - * //- - * // If the callback is omitted, we will return a promise. - * //- - * const response = await transferManager.downloadManyFiles(bucket.File('file1.txt'), bucket.File('file2.txt')]); + * ``` * @experimental */ async downloadManyFiles( files: File[], - optionsOrCallback?: DownloadManyFilesOptions | DownloadManyFilesCallback, - callback?: DownloadManyFilesCallback + options: DownloadManyFilesOptions = {} ): Promise { - let options: DownloadManyFilesOptions = {}; - if (typeof optionsOrCallback === 'function') { - callback = optionsOrCallback; - } else if (optionsOrCallback) { - options = optionsOrCallback; - } - const limit = pLimit( options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT ); @@ -310,32 +214,9 @@ export class TransferManager { promises.push(limit(() => file.download(passThroughOptionsCopy))); } - if (callback) { - try { - const results = await Promise.all(promises); - callback(null, ...results); - } catch (e) { - callback(e as Error); - } - return; - } - return Promise.all(promises); } - async downloadFileInChunks( - file: File, - options?: DownloadFileInChunksOptions - ): Promise; - async downloadFileInChunks( - file: File, - callback: DownloadCallback - ): Promise; - async downloadFileInChunks( - file: File, - options: DownloadFileInChunksOptions, - callback: DownloadCallback - ): Promise; /** * @typedef {object} DownloadFileInChunksOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises @@ -349,8 +230,7 @@ export class TransferManager { * * @param {object} [file] {@link File} to download. * @param {DownloadFileInChunksOptions} [options] Configuration options. - * @param {DownloadCallback} [callbac] Callback function. - * @returns {Promise} + * @returns {Promise} * * @example * ``` @@ -362,39 +242,27 @@ export class TransferManager { * //- * // Download a large file in chunks utilizing parallel operations. * //- - * transferManager.downloadLargeFile(bucket.file('large-file.txt'), function(err, contents) { + * const response = await transferManager.downloadLargeFile(bucket.file('large-file.txt'); * // Your local directory now contains: * // - "large-file.txt" (with the contents from my-bucket.large-file.txt) - * }); - * - * //- - * // If the callback is omitted, we will return a promise. - * //- - * const response = await transferManager.downloadLargeFile(bucket.file('large-file.txt'); + * ``` * @experimental */ async downloadFileInChunks( file: File, - optionsOrCallback?: DownloadFileInChunksOptions | DownloadCallback, - callback?: DownloadCallback + options: DownloadFileInChunksOptions = {} ): Promise { - let options: DownloadFileInChunksOptions = {}; - if (typeof optionsOrCallback === 'function') { - callback = optionsOrCallback; - } else if (optionsOrCallback) { - options = optionsOrCallback; - } - - let chunkSize = options.chunkSizeBytes || LARGE_FILE_DEFAULT_CHUNK_SIZE; + let chunkSize = + options.chunkSizeBytes || DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE; let limit = pLimit( - options.concurrencyLimit || DEFAULT_PARALLEL_LARGE_FILE_DOWNLOAD_LIMIT + options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT ); const promises = []; const fileInfo = await file.get(); const size = parseInt(fileInfo[0].metadata.size); // If the file size does not meet the threshold download it as a single chunk. - if (size < LARGE_FILE_SIZE_THRESHOLD) { + if (size < DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD) { limit = pLimit(1); chunkSize = size; } @@ -417,17 +285,6 @@ export class TransferManager { start += chunkSize; } - if (callback) { - try { - const results = await Promise.all(promises); - callback(null, Buffer.concat(results.map(result => result.buffer))); - } catch (e) { - callback(e as Error, Buffer.alloc(0)); - } - await fileToWrite.close(); - return; - } - return Promise.all(promises) .then(results => { return results.map(result => result.buffer) as DownloadResponse; diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 43d09c5c9..159c965ce 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -15,12 +15,7 @@ */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - Metadata, - ServiceObject, - ServiceObjectConfig, - util, -} from '../src/nodejs-common'; +import {ServiceObject, ServiceObjectConfig, util} from '../src/nodejs-common'; import * as pLimit from 'p-limit'; import * as proxyquire from 'proxyquire'; import { @@ -28,7 +23,6 @@ import { CRC32C, CreateWriteStreamOptions, DownloadOptions, - File, FileOptions, IdempotencyStrategy, UploadOptions, @@ -220,21 +214,6 @@ describe('Transfer Manager', () => { await transferManager.uploadManyFiles(paths, {prefix: 'hello/world'}); }); - it('invokes the callback if one is provided', done => { - const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; - - transferManager.uploadManyFiles( - paths, - (err: Error | null, files?: File[], metadata?: Metadata[]) => { - assert.ifError(err); - assert(files); - assert(metadata); - assert.strictEqual(files[0].name, paths[0]); - done(); - } - ); - }); - it('returns a promise with the uploaded file if there is no callback', async () => { const paths = [path.join(__dirname, '../../test/testdata/testfile.json')]; const result = await transferManager.uploadManyFiles(paths); @@ -283,21 +262,6 @@ describe('Transfer Manager', () => { file.download = download; await transferManager.downloadManyFiles([file], {stripPrefix}); }); - - it('invokes the callback if one if provided', done => { - const file = new File(bucket, 'first.txt'); - file.download = () => { - return Promise.resolve(Buffer.alloc(100)); - }; - transferManager.downloadManyFiles( - [file], - (err: Error | null, contents: Buffer[]) => { - assert.strictEqual(err, null); - assert.strictEqual(contents.length, 100); - done(); - } - ); - }); }); describe('downloadFileInChunks', () => { @@ -326,20 +290,5 @@ describe('Transfer Manager', () => { await transferManager.downloadFileInChunks(file); assert.strictEqual(downloadCallCount, 1); }); - - it('invokes the callback if one is provided', done => { - file.download = () => { - return Promise.resolve([Buffer.alloc(100)]); - }; - - transferManager.downloadFileInChunks( - file, - (err: Error, contents: Buffer) => { - assert.equal(err, null); - assert.strictEqual(contents.length, 100); - done(); - } - ); - }); }); }); From 0780e50a0a72598af483437ee903583fb9e1b7bf Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Tue, 29 Nov 2022 21:03:40 +0000 Subject: [PATCH 22/25] add more experimental tags, update comments --- src/transfer-manager.ts | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 874531aa9..ba7bb3df3 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -21,10 +21,30 @@ import * as path from 'path'; import * as extend from 'extend'; import {promises as fsp} from 'fs'; +/** + * Default number of concurrently executing promises to use when calling uploadManyFiles. + * @experimental + */ const DEFAULT_PARALLEL_UPLOAD_LIMIT = 2; +/** + * Default number of concurrently executing promises to use when calling downloadManyFiles. + * @experimental + */ const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 2; +/** + * Default number of concurrently executing promises to use when calling downloadFileInChunks. + * @experimental + */ const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 2; +/** + * The minimum size threshold in bytes at which to apply a chunked download strategy when calling downloadFileInChunks. + * @experimental + */ const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 256 * 1024 * 1024; +/** + * The chunk size in bytes to use when calling downloadFileInChunks. + * @experimental + */ const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024; const EMPTY_REGEX = '(?:)'; export interface UploadManyFilesOptions { @@ -94,8 +114,8 @@ export class TransferManager { * //- * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']); * // Your bucket now contains: - * // - "file1.txt" (with the contents of '/local/path/file1.txt') - * // - "file2.txt" (with the contents of '/local/path/file2.txt') + * // - "local/path/file1.txt" (with the contents of '/local/path/file1.txt') + * // - "local/path/file2.txt" (with the contents of '/local/path/file2.txt') * ``` * @experimental */ From c47130b2a8319378bd7d00346743fd96f431e949 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Wed, 30 Nov 2022 16:51:35 +0000 Subject: [PATCH 23/25] change signature of downloadManyFiles to accept array of strings or a folder name --- .../performTransferManagerTest.ts | 3 +- src/transfer-manager.ts | 28 +++++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/internal-tooling/performTransferManagerTest.ts b/internal-tooling/performTransferManagerTest.ts index c1e432fa7..64082eca0 100644 --- a/internal-tooling/performTransferManagerTest.ts +++ b/internal-tooling/performTransferManagerTest.ts @@ -176,9 +176,8 @@ async function performDownloadManyFilesTest(): Promise { validation: checkType, }, }); - const getFilesResult = await bucket.getFiles(); const start = performance.now(); - await transferManager.downloadManyFiles(getFilesResult[0], { + await transferManager.downloadManyFiles(TEST_NAME_STRING, { prefix: path.join(__dirname, '..', '..'), concurrencyLimit: argv.numpromises, passthroughOptions: { diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index ba7bb3df3..86551de05 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -180,7 +180,8 @@ export class TransferManager { * Download multiple files in parallel to the local filesystem. This is a convenience method * that utilizes {@link File#download} to perform the download. * - * @param {array} [files] An array of file objects to be downloaded. + * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If + * a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded. * @param {DownloadManyFilesOptions} [options] Configuration options. * @returns {Promise} * @@ -194,21 +195,42 @@ export class TransferManager { * //- * // Download multiple files in parallel. * //- - * const response = await transferManager.downloadManyFiles(bucket.File('file1.txt'), bucket.File('file2.txt')]); + * const response = await transferManager.downloadManyFiles(['file1.txt', 'file2.txt']); * // The following files have been downloaded: * // - "file1.txt" (with the contents from my-bucket.file1.txt) * // - "file2.txt" (with the contents from my-bucket.file2.txt) + * const response = await transferManager.downloadManyFiles([bucket.File('file1.txt'), bucket.File('file2.txt')]); + * // The following files have been downloaded: + * // - "file1.txt" (with the contents from my-bucket.file1.txt) + * // - "file2.txt" (with the contents from my-bucket.file2.txt) + * const response = await transferManager.downloadManyFiles('test-folder'); + * // All files with GCS prefix of 'test-folder' have been downloaded. * ``` * @experimental */ async downloadManyFiles( - files: File[], + filesOrFolder: File[] | string[] | string, options: DownloadManyFilesOptions = {} ): Promise { const limit = pLimit( options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT ); const promises = []; + let files: File[] = []; + + if (!Array.isArray(filesOrFolder)) { + const directoryFiles = await this.bucket.getFiles({ + prefix: filesOrFolder, + }); + files = directoryFiles[0]; + } else { + files = filesOrFolder.map(curFile => { + if (typeof curFile === 'string') { + return this.bucket.file(curFile); + } + return curFile; + }); + } const stripRegexString = options.stripPrefix ? `^${options.stripPrefix}` From 4c2dda4685348ec497acaa23740b8475616c18ae Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Wed, 30 Nov 2022 16:55:53 +0000 Subject: [PATCH 24/25] linter fix --- src/transfer-manager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 86551de05..124753889 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -180,7 +180,7 @@ export class TransferManager { * Download multiple files in parallel to the local filesystem. This is a convenience method * that utilizes {@link File#download} to perform the download. * - * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If + * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If * a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded. * @param {DownloadManyFilesOptions} [options] Configuration options. * @returns {Promise} @@ -204,7 +204,7 @@ export class TransferManager { * // - "file1.txt" (with the contents from my-bucket.file1.txt) * // - "file2.txt" (with the contents from my-bucket.file2.txt) * const response = await transferManager.downloadManyFiles('test-folder'); - * // All files with GCS prefix of 'test-folder' have been downloaded. + * // All files with GCS prefix of 'test-folder' have been downloaded. * ``` * @experimental */ From aad8f2bfd724ac3aa4ea7588b7e5cbaa15309aa2 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Thu, 1 Dec 2022 20:47:24 +0000 Subject: [PATCH 25/25] add transfer manager samples and samples tests --- ...downloadFileInChunksWithTransferManager.js | 77 +++++++++++++++++ .../downloadManyFilesWithTransferManager.js | 67 +++++++++++++++ samples/downloaded.txt | 1 + samples/resources/test2.txt | 1 + samples/system-test/transfer-manager.test.js | 85 +++++++++++++++++++ samples/uploadManyFilesWithTransferManager.js | 67 +++++++++++++++ src/transfer-manager.ts | 8 +- 7 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 samples/downloadFileInChunksWithTransferManager.js create mode 100644 samples/downloadManyFilesWithTransferManager.js create mode 100644 samples/downloaded.txt create mode 100644 samples/resources/test2.txt create mode 100644 samples/system-test/transfer-manager.test.js create mode 100644 samples/uploadManyFilesWithTransferManager.js diff --git a/samples/downloadFileInChunksWithTransferManager.js b/samples/downloadFileInChunksWithTransferManager.js new file mode 100644 index 000000000..891f8cd7b --- /dev/null +++ b/samples/downloadFileInChunksWithTransferManager.js @@ -0,0 +1,77 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * @experimental + */ + +const path = require('path'); +const cwd = path.join(__dirname, '..'); + +// sample-metadata: +// title: Download a File in Chunks Utilzing Transfer Manager +// description: Downloads a single file in in chunks in parallel utilizing transfer manager. +// usage: node downloadFileInChunksWithTransferManager.js + +function main( + bucketName = 'my-bucket', + fileName = 'file1.txt', + destFileName = path.join(cwd, fileName), + chunkSize = 1024 +) { + // [START storage_download_many_files_transfer_manager] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The ID of the GCS file to download + // const fileName = 'your-file-name'; + + // The path to which the file should be downloaded + // const destFileName = '/local/path/to/file.txt'; + + // The size of each chunk to be downloaded + // const chunkSize = 1024; + + // Imports the Google Cloud client library + const {Storage, TransferManager} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + // Creates a transfer manager instance + const transferManager = new TransferManager(storage.bucket(bucketName)); + + async function downloadFileInChunksWithTransferManager() { + // Downloads the files + await transferManager.downloadFileInChunks(fileName, { + destination: destFileName, + chunkSizeBytes: chunkSize, + }); + + console.log( + `gs://${bucketName}/${fileName} downloaded to ${destFileName}.` + ); + } + + downloadFileInChunksWithTransferManager().catch(console.error); + // [END storage_download_many_files_transfer_manager] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/downloadManyFilesWithTransferManager.js b/samples/downloadManyFilesWithTransferManager.js new file mode 100644 index 000000000..3ddfb0980 --- /dev/null +++ b/samples/downloadManyFilesWithTransferManager.js @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * @experimental + */ + +// sample-metadata: +// title: Download Many Files With Transfer Manager +// description: Downloads many files in parallel utilizing transfer manager. +// usage: node downloadManyFilesWithTransferManager.js + +function main( + bucketName = 'my-bucket', + firstFileName = 'file1.txt', + secondFileName = 'file2.txt' +) { + // [START storage_download_many_files_transfer_manager] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The ID of the first GCS file to download + // const firstFileName = 'your-first-file-name'; + + // The ID of the second GCS file to download + // const secondFileName = 'your-second-file-name; + + // Imports the Google Cloud client library + const {Storage, TransferManager} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + // Creates a transfer manager instance + const transferManager = new TransferManager(storage.bucket(bucketName)); + + async function downloadManyFilesWithTransferManager() { + // Downloads the files + await transferManager.downloadManyFiles([firstFileName, secondFileName]); + + for (const fileName of [firstFileName, secondFileName]) { + console.log(`gs://${bucketName}/${fileName} downloaded to ${fileName}.`); + } + } + + downloadManyFilesWithTransferManager().catch(console.error); + // [END storage_download_many_files_transfer_manager] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/downloaded.txt b/samples/downloaded.txt new file mode 100644 index 000000000..c57eff55e --- /dev/null +++ b/samples/downloaded.txt @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/samples/resources/test2.txt b/samples/resources/test2.txt new file mode 100644 index 000000000..010302410 --- /dev/null +++ b/samples/resources/test2.txt @@ -0,0 +1 @@ +Hello World 2! \ No newline at end of file diff --git a/samples/system-test/transfer-manager.test.js b/samples/system-test/transfer-manager.test.js new file mode 100644 index 000000000..983d9cd32 --- /dev/null +++ b/samples/system-test/transfer-manager.test.js @@ -0,0 +1,85 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const path = require('path'); +const {Storage} = require('@google-cloud/storage'); +const {before, after, it, describe} = require('mocha'); +const uuid = require('uuid'); +const cp = require('child_process'); +const {assert} = require('chai'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); +const storage = new Storage(); +const cwd = path.join(__dirname, '..'); +const bucketName = generateName(); +const bucket = storage.bucket(bucketName); +const firstFileName = 'test.txt'; +const secondFileName = 'test2.txt'; +const firstFilePath = path.join(cwd, 'resources', firstFileName); +const secondFilePath = path.join(cwd, 'resources', secondFileName); +const downloadFilePath = path.join(cwd, 'downloaded.txt'); +const chunkSize = 1024; + +describe('transfer manager', () => { + before(async () => { + await bucket.create(); + }); + + after(async () => { + await bucket.deleteFiles({force: true}).catch(console.error); + await bucket.delete().catch(console.error); + }); + + it('should upload multiple files', async () => { + const output = execSync( + `node uploadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}` + ); + assert.match( + output, + new RegExp( + `${firstFilePath} uploaded to ${bucketName}.\n${secondFilePath} uploaded to ${bucketName}` + ) + ); + }); + + it('should download mulitple files', async () => { + const output = execSync( + `node downloadManyFilesWithTransferManager.js ${bucketName} ${firstFilePath} ${secondFilePath}` + ); + assert.match( + output, + new RegExp( + `gs://${bucketName}/${firstFilePath} downloaded to ${firstFilePath}.\ngs://${bucketName}/${secondFilePath} downloaded to ${secondFilePath}.` + ) + ); + }); + + it('should download a file utilizing chunked download', async () => { + const output = execSync( + `node downloadFileInChunksWithTransferManager.js ${bucketName} ${firstFilePath} ${downloadFilePath} ${chunkSize}` + ); + assert.match( + output, + new RegExp( + `gs://${bucketName}/${firstFilePath} downloaded to ${downloadFilePath}.` + ) + ); + }); +}); + +function generateName() { + return `nodejs-storage-samples-${uuid.v4()}`; +} diff --git a/samples/uploadManyFilesWithTransferManager.js b/samples/uploadManyFilesWithTransferManager.js new file mode 100644 index 000000000..98795029f --- /dev/null +++ b/samples/uploadManyFilesWithTransferManager.js @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * @experimental + */ + +// sample-metadata: +// title: Upload Many Files With Transfer Manager +// description: Uploads many files in parallel utilizing transfer manager. +// usage: node uploadManyFilesWithTransferManager.js + +function main( + bucketName = 'my-bucket', + firstFilePath = './local/path/to/file1.txt', + secondFilePath = './local/path/to/file2.txt' +) { + // [START storage_upload_many_files_transfer_manager] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The ID of the first GCS file to download + // const firstFileName = 'your-first-file-name'; + + // The ID of the second GCS file to download + // const secondFileName = 'your-second-file-name; + + // Imports the Google Cloud client library + const {Storage, TransferManager} = require('@google-cloud/storage'); + + // Creates a client + const storage = new Storage(); + + // Creates a transfer manager instance + const transferManager = new TransferManager(storage.bucket(bucketName)); + + async function uploadManyFilesWithTransferManager() { + // Uploads the files + await transferManager.uploadManyFiles([firstFilePath, secondFilePath]); + + for (const filePath of [firstFilePath, secondFilePath]) { + console.log(`${filePath} uploaded to ${bucketName}.`); + } + } + + uploadManyFilesWithTransferManager().catch(console.error); + // [END storage_upload_many_files_transfer_manager] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 124753889..49beabb22 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -270,7 +270,7 @@ export class TransferManager { * Download a large file in chunks utilizing parallel download operations. This is a convenience method * that utilizes {@link File#download} to perform the download. * - * @param {object} [file] {@link File} to download. + * @param {object} [file | string] {@link File} to download. * @param {DownloadFileInChunksOptions} [options] Configuration options. * @returns {Promise} * @@ -291,7 +291,7 @@ export class TransferManager { * @experimental */ async downloadFileInChunks( - file: File, + fileOrName: File | string, options: DownloadFileInChunksOptions = {} ): Promise { let chunkSize = @@ -300,6 +300,10 @@ export class TransferManager { options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT ); const promises = []; + const file: File = + typeof fileOrName === 'string' + ? this.bucket.file(fileOrName) + : fileOrName; const fileInfo = await file.get(); const size = parseInt(fileInfo[0].metadata.size);