feat: add option to allow line breaks in values

This adds an option `{ multiline: "line-breaks" }` that allow values to contain new lines.
Closes motdotla#458

README.md

@@ -125,6 +125,29 @@ You may turn on logging to help debug why certain keys or values are not being s
 require('dotenv').config({ debug: process.env.DEBUG })
 ```
 
+#### Multiline
+
+Default: `default`
+
+You may specify the value `line-breaks` to switch the parser into a mode in which line breaks 
+inside quoted values are allowed.
+
+```js
+require('dotenv').config({ multiline: 'line-breaks' })
+```
+
+This allows specifying multiline values in this format:
+
+```
+PRIVATE_KEY="-----BEGIN PRIVATE KEY-----
+MIGT...
+7ure...
+-----END PRIVATE KEY-----"
+```
+
+Ensure that the value begins with a single or double quote character, and it ends with the same character.
+
+
 ## Parse
 
 The engine which parses the contents of your file containing environment
@@ -166,12 +189,23 @@ The parsing engine currently supports the following rules:
 - whitespace is removed from both ends of unquoted values (see more on [`trim`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/Trim)) (`FOO=  some value  ` becomes `{FOO: 'some value'}`)
 - single and double quoted values are escaped (`SINGLE_QUOTE='quoted'` becomes `{SINGLE_QUOTE: "quoted"}`)
 - single and double quoted values maintain whitespace from both ends (`FOO="  some value  "` becomes `{FOO: '  some value  '}`)
-- double quoted values expand new lines (`MULTILINE="new\nline"` becomes
-
-```
-{MULTILINE: 'new
-line'}
-```
+- double quoted values expand new lines. Example: `MULTILINE="new\nline"` becomes
+
+  ```
+  {MULTILINE: 'new
+  line'}
+  ```
+- multi-line values with line breaks are supported for quoted values if using the `{ multiline: "line-break" }` option.
+  In this mode you do not need to use `\n` to separate lines. Example:
+
+  ```
+  PRIVATE_KEY="-----BEGIN PRIVATE KEY-----
+  MIGT...
+  7ure...
+  -----END PRIVATE KEY-----"
+  ```
+
+  Note that when using this option, all values that start with quotes must end in quotes.
 
 ## FAQ
 

lib/cli-options.js

@@ -1,6 +1,6 @@
 /* @flow */
 
-const re = /^dotenv_config_(encoding|path|debug)=(.+)$/
+const re = /^dotenv_config_(encoding|path|debug|multiline)=(.+)$/
 
 module.exports = function optionMatcher (args /*: Array<string> */) {
   return args.reduce(function (acc, cur) {

lib/env-options.js

@@ -15,4 +15,8 @@ if (process.env.DOTENV_CONFIG_DEBUG != null) {
   options.debug = process.env.DOTENV_CONFIG_DEBUG
 }
 
+if (process.env.DOTENV_CONFIG_MULTILINE != null) {
+  options.multiline = process.env.DOTENV_CONFIG_MULTILINE
+}
+
 module.exports = options

lib/main.js

@@ -37,23 +37,46 @@ const NEWLINES_MATCH = /\n|\r|\r\n/
 // Parses src into an Object
 function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */) /*: DotenvParseOutput */ {
   const debug = Boolean(options && options.debug)
+  const multilineLineBreaks = Boolean(options && options.multiline === 'line-breaks')
   const obj = {}
+  const lines = src.toString().split(NEWLINES_MATCH)
 
   // convert Buffers before splitting into lines and processing
-  src.toString().split(NEWLINES_MATCH).forEach(function (line, idx) {
+  let idx = 0
+  while (idx < lines.length) {
+    let line = lines[idx]
+
     // matching "KEY' and 'VAL' in 'KEY=VAL'
     const keyValueArr = line.match(RE_INI_KEY_VAL)
     // matched?
     if (keyValueArr != null) {
       const key = keyValueArr[1]
       // default undefined or missing values to empty string
       let val = (keyValueArr[2] || '')
-      const end = val.length - 1
+      let end = val.length - 1
       const isDoubleQuoted = val[0] === '"' && val[end] === '"'
       const isSingleQuoted = val[0] === "'" && val[end] === "'"
 
+      const isMultilineDoubleQuoted = val[0] === '"' && val[end] !== '"'
+      const isMultilineSingleQuoted = val[0] === "'" && val[end] !== "'"
+
+      // if parsing line breaks and the value starts with a quote
+      if (multilineLineBreaks && (isMultilineDoubleQuoted || isMultilineSingleQuoted)) {
+        const quoteChar = isMultilineDoubleQuoted ? '"' : "'"
+
+        val = val.substring(1)
+
+        while (idx++ < lines.length - 1) {
+          line = lines[idx]
+          end = line.length - 1
+          if (line[end] === quoteChar) {
+            val += NEWLINE + line.substring(0, end)
+            break
+          }
+          val += NEWLINE + line
+        }
       // if single or double quoted, remove quotes
-      if (isSingleQuoted || isDoubleQuoted) {
+      } else if (isSingleQuoted || isDoubleQuoted) {
         val = val.substring(1, end)
 
         // if double quoted, expand newlines
@@ -69,7 +92,9 @@ function parse (src /*: string | Buffer */, options /*: ?DotenvParseOptions */)
     } else if (debug) {
       log(`did not match key and value when parsing line ${idx + 1}: ${line}`)
     }
-  })
+
+    idx++
+  }
 
   return obj
 }
@@ -80,6 +105,7 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */
   let override = false
   let encoding /*: string */ = 'utf8'
   let debug = false
+  let multiline /*: string */ = 'default'
 
   if (options) {
     if (options.path != null) {
@@ -94,11 +120,14 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */
     if (options.debug != null) {
       debug = true
     }
+    if (options.multiline != null) {
+      multiline = options.multiline
+    }
   }
 
   try {
     // specifying an encoding returns a string instead of a buffer
-    const parsed = parse(fs.readFileSync(dotenvPath, { encoding }), { debug })
+    const parsed = DotEnvModule.parse(fs.readFileSync(dotenvPath, { encoding }), { debug, multiline })
 
     Object.keys(parsed).forEach(function (key) {
       if (!Object.prototype.hasOwnProperty.call(process.env, key) || override) {
@@ -114,5 +143,9 @@ function config (options /*: ?DotenvConfigOptions */) /*: DotenvConfigOutput */
   }
 }
 
-module.exports.config = config
-module.exports.parse = parse
+const DotEnvModule = {
+  config,
+  parse
+}
+
+module.exports = DotEnvModule

tests/.env-multiline

@@ -0,0 +1,32 @@
+BASIC=basic
+
+# previous line intentionally left blank
+AFTER_LINE=after_line
+EMPTY=
+SINGLE_QUOTES='single_quotes'
+SINGLE_QUOTES_SPACED='    single quotes    '
+DOUBLE_QUOTES="double_quotes"
+DOUBLE_QUOTES_SPACED="    double quotes    "
+EXPAND_NEWLINES="expand\nnew\nlines"
+DONT_EXPAND_UNQUOTED=dontexpand\nnewlines
+DONT_EXPAND_SQUOTED='dontexpand\nnewlines'
+# COMMENTS=work
+EQUAL_SIGNS=equals==
+RETAIN_INNER_QUOTES={"foo": "bar"}
+
+RETAIN_INNER_QUOTES_AS_STRING='{"foo": "bar"}'
+TRIM_SPACE_FROM_UNQUOTED=    some spaced out string
+USERNAME=therealnerdybeast@example.tld
+    SPACED_KEY = parsed
+
+MULTI_DOUBLE_QUOTED="THIS
+IS
+A
+MULTILINE
+STRING"
+
+MULTI_SINGLE_QUOTED='THIS
+IS
+A
+MULTILINE
+STRING'

tests/test-cli-options.js

@@ -4,7 +4,7 @@ const t = require('tap')
 
 const options = require('../lib/cli-options')
 
-t.plan(5)
+t.plan(6)
 
 // matches encoding option
 t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_encoding=utf8']), {
@@ -21,6 +21,11 @@ t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_debug=tru
   debug: 'true'
 })
 
+// matches multiline option
+t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_multiline=line-breaks']), {
+  multiline: 'line-breaks'
+})
+
 // ignores empty values
 t.same(options(['node', '-e', "'console.log(testing)'", 'dotenv_config_path=']), {})
 

tests/test-config.js

@@ -11,7 +11,7 @@ const mockParseResponse = { test: 'foo' }
 let readFileSyncStub
 let parseStub
 
-t.plan(8)
+t.plan(9)
 
 t.beforeEach(done => {
   readFileSyncStub = sinon.stub(fs, 'readFileSync').returns('test=foo')
@@ -53,6 +53,13 @@ t.test('takes option for debug', ct => {
   logStub.restore()
 })
 
+t.test('takes option for multiline', ct => {
+  ct.plan(1)
+  const testMultiline = 'line-breaks'
+  dotenv.config({ multiline: testMultiline })
+  ct.equal(parseStub.args[0][1].multiline, testMultiline)
+})
+
 t.test('reads path with encoding, parsing output to process.env', ct => {
   ct.plan(2)
 

tests/test-env-options.js

@@ -10,6 +10,7 @@ require('../lib/env-options')
 const e = process.env.DOTENV_CONFIG_ENCODING
 const p = process.env.DOTENV_CONFIG_PATH
 const d = process.env.DOTENV_CONFIG_DEBUG
+const m = process.env.DOTENV_CONFIG_MULTILINE
 
 // get fresh object for each test
 function options () {
@@ -26,12 +27,13 @@ function testOption (envVar, tmpVal, expect) {
   delete process.env[envVar]
 }
 
-t.plan(4)
+t.plan(5)
 
 // returns empty object when no options set in process.env
 delete process.env.DOTENV_CONFIG_ENCODING
 delete process.env.DOTENV_CONFIG_PATH
 delete process.env.DOTENV_CONFIG_DEBUG
+delete process.env.DOTENV_CONFIG_MULTILINE
 
 t.same(options(), {})
 
@@ -44,7 +46,11 @@ testOption('DOTENV_CONFIG_PATH', '~/.env.test', { path: '~/.env.test' })
 // sets debug option
 testOption('DOTENV_CONFIG_DEBUG', 'true', { debug: 'true' })
 
+// sets multiline option
+testOption('DOTENV_CONFIG_MULTILINE', 'line-breaks', { multiline: 'line-breaks' })
+
 // restore existing env
 process.env.DOTENV_CONFIG_ENCODING = e
 process.env.DOTENV_CONFIG_PATH = p
 process.env.DOTENV_CONFIG_DEBUG = d
+process.env.DOTENV_CONFIG_MULTILINE = m

tests/test-parse-multiline.js

@@ -0,0 +1,72 @@
+/* @flow */
+
+const fs = require('fs')
+
+const sinon = require('sinon')
+const t = require('tap')
+
+const dotenv = require('../lib/main')
+
+const parsed = dotenv.parse(fs.readFileSync('tests/.env-multiline', { encoding: 'utf8' }), { multiline: 'line-breaks' })
+
+t.plan(25)
+
+t.type(parsed, Object, 'should return an object')
+
+t.equal(parsed.BASIC, 'basic', 'sets basic environment variable')
+
+t.equal(parsed.AFTER_LINE, 'after_line', 'reads after a skipped line')
+
+t.equal(parsed.EMPTY, '', 'defaults empty values to empty string')
+
+t.equal(parsed.SINGLE_QUOTES, 'single_quotes', 'escapes single quoted values')
+
+t.equal(parsed.SINGLE_QUOTES_SPACED, '    single quotes    ', 'respects surrounding spaces in single quotes')
+
+t.equal(parsed.DOUBLE_QUOTES, 'double_quotes', 'escapes double quoted values')
+
+t.equal(parsed.DOUBLE_QUOTES_SPACED, '    double quotes    ', 'respects surrounding spaces in double quotes')
+
+t.equal(parsed.EXPAND_NEWLINES, 'expand\nnew\nlines', 'expands newlines but only if double quoted')
+
+t.equal(parsed.DONT_EXPAND_UNQUOTED, 'dontexpand\\nnewlines', 'expands newlines but only if double quoted')
+
+t.equal(parsed.DONT_EXPAND_SQUOTED, 'dontexpand\\nnewlines', 'expands newlines but only if double quoted')
+
+t.notOk(parsed.COMMENTS, 'ignores commented lines')
+
+t.equal(parsed.EQUAL_SIGNS, 'equals==', 'respects equals signs in values')
+
+t.equal(parsed.RETAIN_INNER_QUOTES, '{"foo": "bar"}', 'retains inner quotes')
+
+t.equal(parsed.RETAIN_INNER_QUOTES_AS_STRING, '{"foo": "bar"}', 'retains inner quotes')
+
+t.equal(parsed.TRIM_SPACE_FROM_UNQUOTED, 'some spaced out string', 'retains spaces in string')
+
+t.equal(parsed.USERNAME, 'therealnerdybeast@example.tld', 'parses email addresses completely')
+
+t.equal(parsed.SPACED_KEY, 'parsed', 'parses keys and values surrounded by spaces')
+
+t.equal(parsed.MULTI_DOUBLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses multi-line strings when using double quotes')
+
+t.equal(parsed.MULTI_SINGLE_QUOTED, 'THIS\nIS\nA\nMULTILINE\nSTRING', 'parses multi-line strings when using single quotes')
+
+const payload = dotenv.parse(Buffer.from('BUFFER=true'))
+t.equal(payload.BUFFER, 'true', 'should parse a buffer into an object')
+
+const expectedPayload = { SERVER: 'localhost', PASSWORD: 'password', DB: 'tests' }
+
+const RPayload = dotenv.parse(Buffer.from('SERVER=localhost\rPASSWORD=password\rDB=tests\r'))
+t.same(RPayload, expectedPayload, 'can parse (\\r) line endings')
+
+const NPayload = dotenv.parse(Buffer.from('SERVER=localhost\nPASSWORD=password\nDB=tests\n'))
+t.same(NPayload, expectedPayload, 'can parse (\\n) line endings')
+
+const RNPayload = dotenv.parse(Buffer.from('SERVER=localhost\r\nPASSWORD=password\r\nDB=tests\r\n'))
+t.same(RNPayload, expectedPayload, 'can parse (\\r\\n) line endings')
+
+// test debug path
+const logStub = sinon.stub(console, 'log')
+dotenv.parse(Buffer.from('what is this'), { debug: true })
+t.ok(logStub.called)
+logStub.restore()

types/index.d.ts

@@ -6,6 +6,16 @@ export interface DotenvParseOptions {
    * You may turn on logging to help debug why certain keys or values are not being set as you expect.
    */
   debug?: boolean;
+
+  /**
+   * Use `line-breaks` to allow multi-line variables with line breaks. Example:
+   * MY_VAR="this
+   * is
+   * a
+   * multiline
+   * string"
+   */
+  multiline?: 'line-breaks' | 'default';
 }
 
 export interface DotenvParseOutput {
@@ -44,6 +54,15 @@ export interface DotenvConfigOptions {
    * Override existing environment variable values, by default it's false
    */
   override?: boolean;
+  /**
+   * Use `line-breaks` to allow multi-line variables with line breaks. Example:
+   * MY_VAR="this
+   * is
+   * a
+   * multiline
+   * string"
+   */
+  multiline?: 'line-breaks' | 'default';
 }
 
 export interface DotenvConfigOutput {