From 4c49a3d9e36b1bafb12d17ab78a1d09286cdff42 Mon Sep 17 00:00:00 2001 From: Zoran Regvart Date: Thu, 1 Apr 2021 17:38:13 +0200 Subject: [PATCH] Support upload from named pipes Named pipes report file size as 0, which leads to reading the whole content into memory (0 is less than 64K). This adds additional check to make sure that the passed in path is not a named pipe, and in that case opts for the create-temp-file-to-gzip code path. When running on GitHub Actions infrastructure on `windows` node, named pipes can be created using `mkfifo` from MSYS2. In that case `fs.Stats`s `isFIFO()` returns `false`, and not `true` as expected. This case is detected by `process.platform` being `win32` and the passed file having length of 0. As a side note, when MSYS2's `mkfifo` is run, a pipe file is created: ``` prw-rw-rw- 1 User None 0 Mar 31 12:58 pipe ``` If `fs.stat` is invoked at this point `ENOENT` error will be thrown. As soon as the pipe is written to, this pipe file is replaced by two same- named files: ``` -rw-r--r-- 1 User None 0 Mar 31 13:00 pipe -rw-r--r-- 1 User None 0 Mar 31 13:00 pipe ``` And at this point `fs.stat` `isFIFO()` returns `false`. Even though the file acts as a named pipe. --- packages/artifact/__tests__/upload.test.ts | 54 +++++++++++++++++++ .../src/internal/upload-http-client.ts | 15 ++++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/packages/artifact/__tests__/upload.test.ts b/packages/artifact/__tests__/upload.test.ts index d21c78abdf..9f3ce1a38f 100644 --- a/packages/artifact/__tests__/upload.test.ts +++ b/packages/artifact/__tests__/upload.test.ts @@ -2,6 +2,10 @@ import * as http from 'http' import * as io from '../../io/src/io' import * as net from 'net' import * as path from 'path' +import {mocked} from 'ts-jest/utils' +import {exec, execSync} from 'child_process' +import {createGunzip} from 'zlib' +import {promisify} from 'util' import {UploadHttpClient} from '../src/internal/upload-http-client' import * as core from '@actions/core' import {promises as fs} from 'fs' @@ -174,6 +178,56 @@ describe('Upload Tests', () => { expect(uploadResult.uploadSize).toEqual(expectedTotalSize) }) + function hasMkfifo(): boolean { + try { + // make sure we drain the stdout + return execSync('which mkfifo').toString().length > 0 + } catch (e) { + return false + } + } + const withMkfifoIt = hasMkfifo() ? it : it.skip + withMkfifoIt( + 'Upload Artifact with content from named pipe - Success', + async () => { + // create a named pipe 'pipe' with content 'hello pipe' + const content = Buffer.from('hello pipe') + const pipeFilePath = path.join(root, 'pipe') + await promisify(exec)('mkfifo pipe', {cwd: root}) + // don't want to await here as that would block until read + fs.writeFile(pipeFilePath, content) + + const artifactName = 'successful-artifact' + const uploadSpecification: UploadSpecification[] = [ + { + absoluteFilePath: pipeFilePath, + uploadFilePath: `${artifactName}/pipe` + } + ] + + const uploadUrl = `${getRuntimeUrl()}_apis/resources/Containers/13` + const uploadHttpClient = new UploadHttpClient() + const uploadResult = await uploadHttpClient.uploadArtifactToFileContainer( + uploadUrl, + uploadSpecification + ) + + // accesses the ReadableStream that was passed into sendStream + // eslint-disable-next-line @typescript-eslint/unbound-method + const stream = mocked(HttpClient.prototype.sendStream).mock.calls[0][2] + expect(stream).not.toBeNull() + // decompresses the passed stream + const data: Buffer[] = [] + for await (const chunk of stream.pipe(createGunzip())) { + data.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string)) + } + const uploaded = Buffer.concat(data) + + expect(uploadResult.failedItems.length).toEqual(0) + expect(uploaded).toEqual(content) + } + ) + it('Upload Artifact - Failed Single File Upload', async () => { const uploadSpecification: UploadSpecification[] = [ { diff --git a/packages/artifact/src/internal/upload-http-client.ts b/packages/artifact/src/internal/upload-http-client.ts index e632c8706c..c9aa36502b 100644 --- a/packages/artifact/src/internal/upload-http-client.ts +++ b/packages/artifact/src/internal/upload-http-client.ts @@ -219,16 +219,22 @@ export class UploadHttpClient { httpClientIndex: number, parameters: UploadFileParameters ): Promise { - const totalFileSize: number = (await stat(parameters.file)).size + const fileStat: fs.Stats = await stat(parameters.file) + const totalFileSize = fileStat.size + // on Windows with mkfifo from MSYS2 stats.isFIFO returns false, so we check if running on Windows node and + // if the file has size of 0 to compensate + const isFIFO = + fileStat.isFIFO() || (process.platform === 'win32' && totalFileSize === 0) let offset = 0 let isUploadSuccessful = true let failedChunkSizes = 0 let uploadFileSize = 0 let isGzip = true - // the file that is being uploaded is less than 64k in size, to increase throughput and to minimize disk I/O + // the file that is being uploaded is less than 64k in size to increase throughput and to minimize disk I/O // for creating a new GZip file, an in-memory buffer is used for compression - if (totalFileSize < 65536) { + // with named pipes the file size is reported as zero in that case don't read the file in memory + if (!isFIFO && totalFileSize < 65536) { const buffer = await createGZipFileInBuffer(parameters.file) //An open stream is needed in the event of a failure and we need to retry. If a NodeJS.ReadableStream is directly passed in, @@ -287,7 +293,8 @@ export class UploadHttpClient { let uploadFilePath = tempFile.path // compression did not help with size reduction, use the original file for upload and delete the temp GZip file - if (totalFileSize < uploadFileSize) { + // for named pipes totalFileSize is zero, this assumes compression did help + if (!isFIFO && totalFileSize < uploadFileSize) { uploadFileSize = totalFileSize uploadFilePath = parameters.file isGzip = false