From 0a9332ff70f5121c1221d5c32839a759c60cb879 Mon Sep 17 00:00:00 2001 From: Andrei Alecu Date: Thu, 6 Aug 2020 17:41:45 +0300 Subject: [PATCH 1/4] feat: add option to allow line breaks in values This adds an option `{ multiline: "line-breaks" }` that allow values to contain new lines. Closes #458 --- README.md | 46 +++++++++++++++++++--- lib/cli-options.js | 2 +- lib/env-options.js | 4 ++ lib/main.js | 47 +++++++++++++++++++---- tests/.env-multiline | 32 ++++++++++++++++ tests/test-cli-options.js | 7 +++- tests/test-config.js | 9 ++++- tests/test-env-options.js | 8 +++- tests/test-parse-multiline.js | 72 +++++++++++++++++++++++++++++++++++ types/index.d.ts | 20 ++++++++++ types/test.ts | 12 +++++- 11 files changed, 241 insertions(+), 18 deletions(-) create mode 100644 tests/.env-multiline create mode 100644 tests/test-parse-multiline.js diff --git a/README.md b/README.md index 0dd184b7..b7f462db 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,29 @@ You may turn on logging to help debug why certain keys or values are not being s require('dotenv').config({ debug: process.env.DEBUG }) ``` +#### Multiline + +Default: `default` + +You may specify the value `line-breaks` to switch the parser into a mode in which line breaks +inside quoted values are allowed. + +```js +require('dotenv').config({ multiline: 'line-breaks' }) +``` + +This allows specifying multiline values in this format: + +``` +PRIVATE_KEY="-----BEGIN PRIVATE KEY----- +MIGT... +7ure... +-----END PRIVATE KEY-----" +``` + +Ensure that the value begins with a single or double quote character, and it ends with the same character. + + ## Parse The engine which parses the contents of your file containing environment @@ -166,12 +189,23 @@ The parsing engine currently supports the following rules: - whitespace is removed from both ends of unquoted values (see more on [`trim`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/Trim)) (`FOO= some value ` becomes `{FOO: 'some value'}`) - single and double quoted values are escaped (`SINGLE_QUOTE='quoted'` becomes `{SINGLE_QUOTE: "quoted"}`) - single and double quoted values maintain whitespace from both ends (`FOO=" some value "` becomes `{FOO: ' some value '}`) -- double quoted values expand new lines (`MULTILINE="new\nline"` becomes - -``` -{MULTILINE: 'new -line'} -``` +- double quoted values expand new lines. Example: `MULTILINE="new\nline"` becomes + + ``` + {MULTILINE: 'new + line'} + ``` +- multi-line values with line breaks are supported for quoted values if using the `{ multiline: "line-break" }` option. + In this mode you do not need to use `\n` to separate lines. Example: + + ``` + PRIVATE_KEY="-----BEGIN PRIVATE KEY----- + MIGT... + 7ure... + -----END PRIVATE KEY-----" + ``` + + Note that when using this option, all values that start with quotes must end in quotes. ## FAQ diff --git a/lib/cli-options.js b/lib/cli-options.js index ad69fabd..9d9cf355 100644 --- a/lib/cli-options.js +++ b/lib/cli-options.js @@ -1,6 +1,6 @@ /* @flow */ -const re = /^dotenv_config_(encoding|path|debug)=(.+)$/ +const re = /^dotenv_config_(encoding|path|debug|multiline)=(.+)$/ module.exports = function optionMatcher (args /*: Array */) { return args.reduce(function (acc, cur) { diff --git a/lib/env-options.js b/lib/env-options.js index b4c4a606..a866991d 100644 --- a/lib/env-options.js +++ b/lib/env-options.js @@ -15,4 +15,8 @@ if (process.env.DOTENV_CONFIG_DEBUG != null) { options.debug = process.env.DOTENV_CONFIG_DEBUG } +if (process.env.DOTENV_CONFIG_MULTILINE != null) { + options.multiline = process.env.DOTENV_CONFIG_MULTILINE +} + module.exports = options diff --git a/lib/main.js b/lib/main.js index e6c591c6..08d2404f 100644 --- a/lib/main.js +++ b/lib/main.js @@ -36,10 +36,15 @@ const NEWLINES_MATCH = /\n|\r|\r\n/ // Parses src into an Object function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) /*: DotenvParseOutput */ { const debug = Boolean(options && options.debug) + const multilineLineBreaks = Boolean(options && options.multiline === 'line-breaks') const obj = {} + const lines = src.toString().split(NEWLINES_MATCH) // convert Buffers before splitting into lines and processing - src.toString().split(NEWLINES_MATCH).forEach(function (line, idx) { + let idx = 0 + while (idx < lines.length) { + let line = lines[idx] + // matching "KEY' and 'VAL' in 'KEY=VAL' const keyValueArr = line.match(RE_INI_KEY_VAL) // matched? @@ -47,12 +52,30 @@ function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) const key = keyValueArr[1] // default undefined or missing values to empty string let val = (keyValueArr[2] || '') - const end = val.length - 1 + let end = val.length - 1 const isDoubleQuoted = val[0] === '"' && val[end] === '"' const isSingleQuoted = val[0] === "'" && val[end] === "'" + const isMultilineDoubleQuoted = val[0] === '"' && val[end] !== '"' + const isMultilineSingleQuoted = val[0] === "'" && val[end] !== "'" + + // if parsing line breaks and the value starts with a quote + if (multilineLineBreaks && (isMultilineDoubleQuoted || isMultilineSingleQuoted)) { + const quoteChar = isMultilineDoubleQuoted ? '"' : "'" + + val = val.substring(1) + + while (idx++ < lines.length - 1) { + line = lines[idx] + end = line.length - 1 + if (line[end] === quoteChar) { + val += NEWLINE + line.substring(0, end) + break + } + val += NEWLINE + line + } // if single or double quoted, remove quotes - if (isSingleQuoted || isDoubleQuoted) { + } else if (isSingleQuoted || isDoubleQuoted) { val = val.substring(1, end) // if double quoted, expand newlines @@ -68,7 +91,9 @@ function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) } else if (debug) { log(`did not match key and value when parsing line ${idx + 1}: ${line}`) } - }) + + idx++ + } return obj } @@ -78,6 +103,7 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */ let dotenvPath = path.resolve(process.cwd(), '.env') let encoding /*: string */ = 'utf8' let debug = false + let multiline /*: string */ = 'default' if (options) { if (options.path != null) { @@ -89,11 +115,14 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */ if (options.debug != null) { debug = true } + if (options.multiline != null) { + multiline = options.multiline + } } try { // specifying an encoding returns a string instead of a buffer - const parsed = parse(fs.readFileSync(dotenvPath, { encoding }), { debug }) + const parsed = DotEnvModule.parse(fs.readFileSync(dotenvPath, { encoding }), { debug, multiline }) Object.keys(parsed).forEach(function (key) { if (!Object.prototype.hasOwnProperty.call(process.env, key)) { @@ -109,5 +138,9 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */ } } -module.exports.config = config -module.exports.parse = parse +const DotEnvModule = { + config, + parse +} + +module.exports = DotEnvModule diff --git a/tests/.env-multiline b/tests/.env-multiline new file mode 100644 index 00000000..6c28794d --- /dev/null +++ b/tests/.env-multiline @@ -0,0 +1,32 @@ +BASIC=basic + +# previous line intentionally left blank +AFTER_LINE=after_line +EMPTY= +SINGLE_QUOTES='single_quotes' +SINGLE_QUOTES_SPACED=' single quotes ' +DOUBLE_QUOTES="double_quotes" +DOUBLE_QUOTES_SPACED=" double quotes " +EXPAND_NEWLINES="expand\nnew\nlines" +DONT_EXPAND_UNQUOTED=dontexpand\nnewlines +DONT_EXPAND_SQUOTED='dontexpand\nnewlines' +# COMMENTS=work +EQUAL_SIGNS=equals== +RETAIN_INNER_QUOTES={"foo": "bar"} + +RETAIN_INNER_QUOTES_AS_STRING='{"foo": "bar"}' +TRIM_SPACE_FROM_UNQUOTED= some spaced out string +USERNAME=therealnerdybeast@example.tld + SPACED_KEY = parsed + +MULTI_DOUBLE_QUOTED="THIS +IS +A +MULTILINE +STRING" + +MULTI_SINGLE_QUOTED='THIS +IS +A +MULTILINE +STRING' \ No newline at end of file diff --git a/tests/test-cli-options.js b/tests/test-cli-options.js index 5ecf02ff..6ace0fde 100644 --- a/tests/test-cli-options.js +++ b/tests/test-cli-options.js @@ -4,7 +4,7 @@ const t = require('tap') const options = require('../lib/cli-options') -t.plan(5) +t.plan(6) // matches encoding option t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_encoding=utf8']), { @@ -21,6 +21,11 @@ t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_debug=tru debug: 'true' }) +// matches multiline option +t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_multiline=line-breaks']), { + multiline: 'line-breaks' +}) + // ignores empty values t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_path=']), {}) diff --git a/tests/test-config.js b/tests/test-config.js index b6085b81..2e87d54d 100644 --- a/tests/test-config.js +++ b/tests/test-config.js @@ -11,7 +11,7 @@ const mockParseResponse = { test: 'foo' } let readFileSyncStub let parseStub -t.plan(8) +t.plan(9) t.beforeEach(done => { readFileSyncStub = sinon.stub(fs, 'readFileSync').returns('test=foo') @@ -53,6 +53,13 @@ t.test('takes option for debug', ct => { logStub.restore() }) +t.test('takes option for multiline', ct => { + ct.plan(1) + const testMultiline = 'line-breaks' + dotenv.config({ multiline: testMultiline }) + ct.equal(parseStub.args[0][1].multiline, testMultiline) +}) + t.test('reads path with encoding, parsing output to process.env', ct => { ct.plan(2) diff --git a/tests/test-env-options.js b/tests/test-env-options.js index 0870f895..e393c823 100644 --- a/tests/test-env-options.js +++ b/tests/test-env-options.js @@ -10,6 +10,7 @@ require('../lib/env-options') const e = process.env.DOTENV_CONFIG_ENCODING const p = process.env.DOTENV_CONFIG_PATH const d = process.env.DOTENV_CONFIG_DEBUG +const m = process.env.DOTENV_CONFIG_MULTILINE // get fresh object for each test function options () { @@ -26,12 +27,13 @@ function testOption (envVar, tmpVal, expect) { delete process.env[envVar] } -t.plan(4) +t.plan(5) // returns empty object when no options set in process.env delete process.env.DOTENV_CONFIG_ENCODING delete process.env.DOTENV_CONFIG_PATH delete process.env.DOTENV_CONFIG_DEBUG +delete process.env.DOTENV_CONFIG_MULTILINE t.same(options(), {}) @@ -44,7 +46,11 @@ testOption('DOTENV_CONFIG_PATH', '~/.env.test', { path: '~/.env.test' }) // sets debug option testOption('DOTENV_CONFIG_DEBUG', 'true', { debug: 'true' }) +// sets multiline option +testOption('DOTENV_CONFIG_MULTILINE', 'line-breaks', { multiline: 'line-breaks' }) + // restore existing env process.env.DOTENV_CONFIG_ENCODING = e process.env.DOTENV_CONFIG_PATH = p process.env.DOTENV_CONFIG_DEBUG = d +process.env.DOTENV_CONFIG_MULTILINE = m diff --git a/tests/test-parse-multiline.js b/tests/test-parse-multiline.js new file mode 100644 index 00000000..d9586c6d --- /dev/null +++ b/tests/test-parse-multiline.js @@ -0,0 +1,72 @@ +/* @flow */ + +const fs = require('fs') + +const sinon = require('sinon') +const t = require('tap') + +const dotenv = require('../lib/main') + +const parsed = dotenv.parse(fs.readFileSync('tests/.env-multiline', { encoding: 'utf8' }), { multiline: 'line-breaks' }) + +t.plan(25) + +t.type(parsed, Object, 'should return an object') + +t.equal(parsed.BASIC, 'basic', 'sets basic environment variable') + +t.equal(parsed.AFTER_LINE, 'after_line', 'reads after a skipped line') + +t.equal(parsed.EMPTY, '', 'defaults empty values to empty string') + +t.equal(parsed.SINGLE_QUOTES, 'single_quotes', 'escapes single quoted values') + +t.equal(parsed.SINGLE_QUOTES_SPACED, ' single quotes ', 'respects surrounding spaces in single quotes') + +t.equal(parsed.DOUBLE_QUOTES, 'double_quotes', 'escapes double quoted values') + +t.equal(parsed.DOUBLE_QUOTES_SPACED, ' double quotes ', 'respects surrounding spaces in double quotes') + +t.equal(parsed.EXPAND_NEWLINES, 'expand\nnew\nlines', 'expands newlines but only if double quoted') + +t.equal(parsed.DONT_EXPAND_UNQUOTED, 'dontexpand\\nnewlines', 'expands newlines but only if double quoted') + +t.equal(parsed.DONT_EXPAND_SQUOTED, 'dontexpand\\nnewlines', 'expands newlines but only if double quoted') + +t.notOk(parsed.COMMENTS, 'ignores commented lines') + +t.equal(parsed.EQUAL_SIGNS, 'equals==', 'respects equals signs in values') + +t.equal(parsed.RETAIN_INNER_QUOTES, '{"foo": "bar"}', 'retains inner quotes') + +t.equal(parsed.RETAIN_INNER_QUOTES_AS_STRING, '{"foo": "bar"}', 'retains inner quotes') + +t.equal(parsed.TRIM_SPACE_FROM_UNQUOTED, 'some spaced out string', 'retains spaces in string') + +t.equal(parsed.USERNAME, 'therealnerdybeast@example.tld', 'parses email addresses completely') + +t.equal(parsed.SPACED_KEY, 'parsed', 'parses keys and values surrounded by spaces') + +t.equal(parsed.MULTI_DOUBLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses multi-line strings when using double quotes') + +t.equal(parsed.MULTI_SINGLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses multi-line strings when using single quotes') + +const payload = dotenv.parse(Buffer.from('BUFFER=true')) +t.equal(payload.BUFFER, 'true', 'should parse a buffer into an object') + +const expectedPayload = { SERVER: 'localhost', PASSWORD: 'password', DB: 'tests' } + +const RPayload = dotenv.parse(Buffer.from('SERVER=localhost\rPASSWORD=password\rDB=tests\r')) +t.same(RPayload, expectedPayload, 'can parse (\\r) line endings') + +const NPayload = dotenv.parse(Buffer.from('SERVER=localhost\nPASSWORD=password\nDB=tests\n')) +t.same(NPayload, expectedPayload, 'can parse (\\n) line endings') + +const RNPayload = dotenv.parse(Buffer.from('SERVER=localhost\r\nPASSWORD=password\r\nDB=tests\r\n')) +t.same(RNPayload, expectedPayload, 'can parse (\\r\\n) line endings') + +// test debug path +const logStub = sinon.stub(console, 'log') +dotenv.parse(Buffer.from('what is this'), { debug: true }) +t.ok(logStub.called) +logStub.restore() diff --git a/types/index.d.ts b/types/index.d.ts index 23f05f69..e2a8fb81 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -6,6 +6,16 @@ export interface DotenvParseOptions { * You may turn on logging to help debug why certain keys or values are not being set as you expect. */ debug?: boolean; + + /** + * Use `line-breaks` to allow multi-line variables with line breaks. Example: + * MY_VAR="this + * is + * a + * multiline + * string" + */ + multiline?: 'line-breaks' | 'default'; } export interface DotenvParseOutput { @@ -39,6 +49,16 @@ export interface DotenvConfigOptions { * You may turn on logging to help debug why certain keys or values are not being set as you expect. */ debug?: boolean; + + /** + * Use `line-breaks` to allow multi-line variables with line breaks. Example: + * MY_VAR="this + * is + * a + * multiline + * string" + */ + multiline?: 'line-breaks' | 'default'; } export interface DotenvConfigOutput { diff --git a/types/test.ts b/types/test.ts index 84b17601..36f44a33 100644 --- a/types/test.ts +++ b/types/test.ts @@ -12,9 +12,19 @@ const dbUrl: string | null = config({ path: ".env-example", encoding: "utf8", - debug: true + debug: true, +}); + +config({ + multiline: "line-breaks" }); +config({ + multiline: "default" +}); + +parse("test", { multiline: "line-breaks"}); + const parsed = parse("NODE_ENV=production\nDB_HOST=a.b.c"); const dbHost: string = parsed["DB_HOST"]; From f652c447f07b66546bef6303306038c74b9cd690 Mon Sep 17 00:00:00 2001 From: Andrei Alecu Date: Thu, 6 Aug 2020 18:07:13 +0300 Subject: [PATCH 2/4] add test for unended multiline values --- tests/.env-multiline | 6 +++++- tests/test-parse-multiline.js | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/.env-multiline b/tests/.env-multiline index 6c28794d..2dffdcde 100644 --- a/tests/.env-multiline +++ b/tests/.env-multiline @@ -29,4 +29,8 @@ MULTI_SINGLE_QUOTED='THIS IS A MULTILINE -STRING' \ No newline at end of file +STRING' + +MULTI_UNENDED="THIS +LINE HAS +NO END QUOTE \ No newline at end of file diff --git a/tests/test-parse-multiline.js b/tests/test-parse-multiline.js index d9586c6d..da7de962 100644 --- a/tests/test-parse-multiline.js +++ b/tests/test-parse-multiline.js @@ -9,7 +9,7 @@ const dotenv = require('../lib/main') const parsed = dotenv.parse(fs.readFileSync('tests/.env-multiline', { encoding: 'utf8' }), { multiline: 'line-breaks' }) -t.plan(25) +t.plan(26) t.type(parsed, Object, 'should return an object') @@ -51,6 +51,8 @@ t.equal(parsed.MULTI_DOUBLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses mu t.equal(parsed.MULTI_SINGLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses multi-line strings when using single quotes') +t.equal(parsed.MULTI_UNENDED, 'THIS\nLINE HAS\nNO END QUOTE', 'parses multi-line strings when using single quotes') + const payload = dotenv.parse(Buffer.from('BUFFER=true')) t.equal(payload.BUFFER, 'true', 'should parse a buffer into an object') From 5b3aaa4111ab2ca96b335a30b004a46bbd8b5de5 Mon Sep 17 00:00:00 2001 From: Andrei Alecu Date: Thu, 6 Aug 2020 18:14:36 +0300 Subject: [PATCH 3/4] use plain for --- lib/main.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/main.js b/lib/main.js index 08d2404f..d21f411f 100644 --- a/lib/main.js +++ b/lib/main.js @@ -38,11 +38,11 @@ function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) const debug = Boolean(options && options.debug) const multilineLineBreaks = Boolean(options && options.multiline === 'line-breaks') const obj = {} - const lines = src.toString().split(NEWLINES_MATCH) // convert Buffers before splitting into lines and processing - let idx = 0 - while (idx < lines.length) { + const lines = src.toString().split(NEWLINES_MATCH) + + for (let idx = 0; idx < lines.length; idx++) { let line = lines[idx] // matching "KEY' and 'VAL' in 'KEY=VAL' @@ -91,8 +91,6 @@ function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) } else if (debug) { log(`did not match key and value when parsing line ${idx + 1}: ${line}`) } - - idx++ } return obj From ac20ff84241d380a736812a6c37ccdcc677d0c12 Mon Sep 17 00:00:00 2001 From: Andrei Alecu Date: Thu, 6 Aug 2020 18:39:35 +0300 Subject: [PATCH 4/4] fix tests on node 12 --- tests/test-config-cli.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test-config-cli.js b/tests/test-config-cli.js index c7ad3d02..835e2986 100644 --- a/tests/test-config-cli.js +++ b/tests/test-config-cli.js @@ -5,6 +5,8 @@ const path = require('path') const t = require('tap') +const configPath = path.resolve(__dirname, '../config.js') + function spawn (cmd, options = {}) { const { stdout } = cp.spawnSync( process.argv[0], // node binary @@ -29,7 +31,7 @@ t.plan(3) t.equal( spawn([ '-r', - '../config', + configPath, '-e', 'console.log(process.env.BASIC)', 'dotenv_config_encoding=utf8', @@ -40,7 +42,7 @@ t.equal( // dotenv/config supports configuration via environment variables t.equal( - spawn(['-r', '../config', '-e', 'console.log(process.env.BASIC)'], { + spawn(['-r', configPath, '-e', 'console.log(process.env.BASIC)'], { env: { DOTENV_CONFIG_PATH: './tests/.env' } @@ -53,7 +55,7 @@ t.equal( spawn( [ '-r', - '../config', + configPath, '-e', 'console.log(process.env.BASIC)', 'dotenv_config_path=./tests/.env'