Skip to content

Commit

Permalink
Stream directly to disk
Browse files Browse the repository at this point in the history
The current implementation saves everything to memory and extracts
the zip files to memory, before copying to the filesystem. This can
consume a huge amount of memory if artifacts are large. This change
streams the zip file directly to disk and extracts it without loading
the entire zip in memory first.
  • Loading branch information
gabriel-samfira committed Aug 30, 2022
1 parent 5bbf14f commit df393ed
Show file tree
Hide file tree
Showing 229 changed files with 45,713 additions and 38,109 deletions.
114 changes: 98 additions & 16 deletions main.js
@@ -1,9 +1,11 @@
const core = require('@actions/core')
const github = require('@actions/github')
const AdmZip = require('adm-zip')
const filesize = require('filesize')
const pathname = require('path')
const fs = require('fs')
const github = require('@actions/github')
const https = require('follow-redirects').https;
const pathname = require('path')
const url = require('url')
const yauzl = require("yauzl");

function inform(key, val) {
core.info(`==> ${key}: ${val}`)
Expand Down Expand Up @@ -184,14 +186,48 @@ async function main() {
core.info(`==> Downloading: ${artifact.name}.zip (${size})`)

let saveTo = `${pathname.join(path, artifact.name)}.zip`
fs.mkdirSync(path, { recursive: true })
if (!fs.existsSync(path)) {
fs.mkdirSync(path, { recursive: true })
}

const zip = await client.rest.actions.downloadArtifact({
let request = client.rest.actions.downloadArtifact.endpoint({
owner: owner,
repo: repo,
artifact_id: artifact.id,
archive_format: "zip",
})
});

const sendGetRequest = async () => {
return new Promise(resolve => {
const options = {
hostname: url.parse(request.url).hostname,
path: url.parse(request.url).pathname,
headers: {
...request.headers,
Authorization: `token ${token}`,
}
}
const file = fs.createWriteStream(saveTo);
https.get(options, (response) => {
response.on('error', function(err) {
core.info(`error downloading: ${err}`);
resolve()
})
response.pipe(file);
file.on("finish", () => {
file.close();
core.info("Download Completed");
resolve()
});
file.on("error", () => {
core.info(`error saving file: ${err}`);
resolve()
})
});
})
}

await sendGetRequest();

fs.writeFileSync(saveTo, Buffer.from(zip.data), 'binary')

Expand All @@ -200,21 +236,67 @@ async function main() {
}

const dir = name ? path : pathname.join(path, artifact.name)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}

fs.mkdirSync(dir, { recursive: true })
core.startGroup(`==> Extracting: ${artifact.name}.zip`)
yauzl.open(saveTo, {lazyEntries: true}, function(err, zipfile) {
if (err) throw err;
zipfile.readEntry();
zipfile.on("entry", function(entry) {
const filepath = pathname.resolve(pathname.join(dir, entry.fileName))

const adm = new AdmZip(saveTo)
// Make sure the zip is properly crafted.
const relative = pathname.relative(dir, filepath);
const isInPath = relative && !relative.startsWith('..') && !pathname.isAbsolute(relative);
if (!isInPath) {
core.info(` ==> Path ${filepath} resolves outside of ${dir} skipping`)
zipfile.readEntry();
}

core.startGroup(`==> Extracting: ${artifact.name}.zip`)
adm.getEntries().forEach((entry) => {
const action = entry.isDirectory ? "creating" : "inflating"
const filepath = pathname.join(dir, entry.entryName)
// The zip may contain the directory names for newly created files.
if (/\/$/.test(entry.fileName)) {
// Directory file names end with '/'.
// Note that entries for directories themselves are optional.
// An entry's fileName implicitly requires its parent directories to exist.
if (!fs.existsSync(filepath)) {
core.info(` ==> Creating: ${filepath}`)
fs.mkdirSync(filepath, { recursive: true })
}
zipfile.readEntry();
} else {
// This is a file entry. Attempt to extract it.
core.info(` ==> Extracting: ${entry.fileName}`)

core.info(` ${action}: ${filepath}`)
adm.extractEntryTo(entry.entryName, dir)
})
// Ensure the parent folder exists
let dirName = pathname.dirname(filepath)
if (!fs.existsSync(dirName)) {
core.info(` ==> Creating: ${dirName}`)
fs.mkdirSync(dirName, { recursive: true })
}
zipfile.openReadStream(entry, (err, readStream) => {
if (err) throw err;

// adm.extractAllTo(dir, true)
readStream.on("end", () => {
zipfile.readEntry();
});
readStream.on("error", (err) => {
throw new Error(`Failed to extract ${entry.fileName}: ${err}`)
});

const file = fs.createWriteStream(filepath);
readStream.pipe(file);
file.on("finish", () => {
file.close();
});
file.on("error", (err) => {
throw new Error(`Failed to extract ${entry.fileName}: ${err}`)
});
});
}
});
});
core.endGroup()
}
} catch (error) {
Expand Down

0 comments on commit df393ed

Please sign in to comment.