Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Decryption of Large Files/Streams - Update documentation and Guard Test case #1462

Merged
merged 9 commits into from Jun 2, 2022
Merged

Decryption of Large Files/Streams - Update documentation and Guard Test case #1462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
68c7c29
Illustration of Memory Explosion and Process Crash
Jan 7, 2022
29c0c84
Updated documentation for allowUnauthenticatedStream
Jan 28, 2022
42e402e
allowUnauthenticatedStream implemented on demonstration test
Jan 28, 2022
358b07b
Apply suggestions from code review
larabr Jun 1, 2022
8b17f12
Use password encryption and disable compression to reduce noise
larabr Jun 2, 2022
e8319f9
Trigger stream decryption in all streaming tests
larabr Jun 2, 2022
f3f0403
Uniform input stream generation across all tests
larabr Jun 2, 2022
3b48d7d
Use 1MB instead of 1MiB to allow comparison with previous benchmarks
larabr Jun 2, 2022
db05571
Reduce size and number of chunks to reduce variability below warning …
larabr Jun 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/config/config.js
View file
Open in desktop
Expand Up @@ -92,7 +92,10 @@ export default {
*/
allowUnauthenticatedMessages: false,
/**
* Allow streaming unauthenticated data before its integrity has been checked.
* Allow streaming unauthenticated data before its integrity has been checked. This would allow the application to
* process large streams while limiting memory usage by releasing the decrypted chunks as soon as possible
* and deferring checking their integrity until the decrypted stream has been read in full.
*
* This setting is **insecure** if the partially decrypted message is processed further or displayed to the user.
* @memberof module:config
* @property {Boolean} allowUnauthenticatedStream
Expand Down
159 changes: 141 additions & 18 deletions test/benchmarks/memory_usage.js
View file
Open in desktop
@@ -1,6 +1,5 @@
/* eslint-disable no-console */
const assert = require('assert');
const stream = require('@openpgp/web-stream-tools');
const path = require('path');
const { writeFileSync, unlinkSync } = require('fs');
const { fork } = require('child_process');
Expand All @@ -17,7 +16,6 @@ const benchmark = async function(fn) {
// the code to execute must be written to a file
writeFileSync(tmpFileName, `
const assert = require('assert');
const stream = require('@openpgp/web-stream-tools');
const openpgp = require('../..');
let maxMemoryComsumption;
let activeSampling = false;
Expand Down Expand Up @@ -111,9 +109,10 @@ class MemoryBenchamrkSuite {
suite.add('empty test (baseline)', () => {});

suite.add('openpgp.encrypt/decrypt (CFB, binary)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
Expand All @@ -122,9 +121,10 @@ class MemoryBenchamrkSuite {
});

suite.add('openpgp.encrypt/decrypt (CFB, text)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
Expand All @@ -133,9 +133,10 @@ class MemoryBenchamrkSuite {
});

suite.add('openpgp.encrypt/decrypt (AEAD, binary)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(1000000).fill(1) });
const plaintextMessage = await openpgp.createMessage({ binary: new Uint8Array(ONE_MEGABYTE) });

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
Expand All @@ -144,9 +145,10 @@ class MemoryBenchamrkSuite {
});

suite.add('openpgp.encrypt/decrypt (AEAD, text)', async () => {
const ONE_MEGABYTE = 1000000;
const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(10000000 / 2) }); // two bytes per character
const plaintextMessage = await openpgp.createMessage({ text: 'a'.repeat(ONE_MEGABYTE / 2) }); // two bytes per character

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
Expand All @@ -156,61 +158,182 @@ class MemoryBenchamrkSuite {

// streaming tests
suite.add('openpgp.encrypt/decrypt (CFB, binary, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ binary: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
await stream.readToEnd(decryptedData);
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (CFB, text, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
await stream.readToEnd(decryptedData);
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (AEAD, binary, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ binary: stream.toStream(new Uint8Array(1000000).fill(1)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: new Uint8Array(ONE_MEGABYTE), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ binary:inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
await openpgp.decrypt({ message: encryptedMessage, passwords, config });
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (AEAD, text, with streaming)', async () => {
await stream.loadStreamsPonyfill();
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const plaintextMessage = await openpgp.createMessage({ text: stream.toStream('a'.repeat(10000000 / 2)) });
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 1 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
await openpgp.decrypt({ message: encryptedMessage, passwords, config });
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (CFB, text @ 10MB, with unauthenticated streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: false, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.SymEncryptedIntegrityProtectedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({
message: encryptedMessage,
passwords,
config: { ...config, allowUnauthenticatedStream: true }
});
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

suite.add('openpgp.encrypt/decrypt (AEAD, text @ 10MB, with streaming)', async () => {
const ONE_MEGABYTE = 1000000;
function* largeDataGenerator({ chunk, numberOfChunks }) {
for (let chunkNumber = 0; chunkNumber < numberOfChunks; chunkNumber++) {
yield chunk;
}
}

const passwords = 'password';
const config = { aeadProtect: true, preferredCompressionAlgorithm: openpgp.enums.compression.uncompressed };
const inputStream = require('stream').Readable.from(largeDataGenerator({ chunk: 'a'.repeat(ONE_MEGABYTE / 2), numberOfChunks: 20 }));
const plaintextMessage = await openpgp.createMessage({ text: inputStream });
assert(plaintextMessage.fromStream);

const armoredEncryptedMessage = await openpgp.encrypt({ message: plaintextMessage, passwords, config });
const encryptedMessage = await openpgp.readMessage({ armoredMessage: armoredEncryptedMessage });
assert.ok(encryptedMessage.packets[1] instanceof openpgp.AEADEncryptedDataPacket);
const { data: decryptedData } = await openpgp.decrypt({ message: encryptedMessage, passwords, config });
// read out output stream to trigger decryption
await new Promise(resolve => {
decryptedData.pipe(require('fs').createWriteStream('/dev/null'));
decryptedData.on('end', resolve);
});
});

const stats = await suite.run();
Expand Down