diff --git a/dotenv/cut.go b/dotenv/cut.go new file mode 100644 index 00000000..2bb4e554 --- /dev/null +++ b/dotenv/cut.go @@ -0,0 +1,9 @@ +//go:build go1.18 + +package dotenv + +import "bytes" + +func bytesCut(s, sep []byte) (before, after []byte, found bool) { + return bytes.Cut(s, sep) +} diff --git a/dotenv/cut_go117.go b/dotenv/cut_go117.go new file mode 100644 index 00000000..139c74db --- /dev/null +++ b/dotenv/cut_go117.go @@ -0,0 +1,13 @@ +//go:build !go1.18 +// +build !go1.18 + +package dotenv + +import "bytes" + +func bytesCut(s, sep []byte) (before, after []byte, found bool) { + if i := bytes.Index(s, sep); i >= 0 { + return s[:i], s[i+len(sep):], true + } + return s, nil, false +} diff --git a/dotenv/godotenv.go b/dotenv/godotenv.go index 69ac6ece..d48a30c1 100644 --- a/dotenv/godotenv.go +++ b/dotenv/godotenv.go @@ -4,34 +4,29 @@ // // The TL;DR is that you make a .env file that looks something like // -// SOME_ENV_VAR=somevalue +// SOME_ENV_VAR=somevalue // // and then in your go code you can call // -// godotenv.Load() +// godotenv.Load() // // and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") package dotenv import ( "bytes" - "errors" - "fmt" "io" "os" - "os/exec" "regexp" - "sort" - "strconv" "strings" "github.com/compose-spec/compose-go/template" ) -const doubleQuoteSpecialChars = "\\\n\r\"!$`" - var utf8BOM = []byte("\uFEFF") +var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored + // LookupFn represents a lookup function to resolve variables from type LookupFn func(string) (string, bool) @@ -60,13 +55,13 @@ func ParseWithLookup(r io.Reader, lookupFn LookupFn) (map[string]string, error) // Load will read your env file(s) and load them into ENV for this process. // -// Call this function as close as possible to the start of your program (ideally in main) +// Call this function as close as possible to the start of your program (ideally in main). // -// If you call Load without any args it will default to loading .env in the current path +// If you call Load without any args it will default to loading .env in the current path. // -// You can otherwise tell it which files to load (there can be more than one) like +// You can otherwise tell it which files to load (there can be more than one) like: // -// godotenv.Load("fileone", "filetwo") +// godotenv.Load("fileone", "filetwo") // // It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults func Load(filenames ...string) error { @@ -75,13 +70,13 @@ func Load(filenames ...string) error { // Overload will read your env file(s) and load them into ENV for this process. // -// Call this function as close as possible to the start of your program (ideally in main) +// Call this function as close as possible to the start of your program (ideally in main). // -// If you call Overload without any args it will default to loading .env in the current path +// If you call Overload without any args it will default to loading .env in the current path. // -// You can otherwise tell it which files to load (there can be more than one) like +// You can otherwise tell it which files to load (there can be more than one) like: // -// godotenv.Overload("fileone", "filetwo") +// godotenv.Overload("fileone", "filetwo") // // It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars. func Overload(filenames ...string) error { @@ -99,8 +94,6 @@ func load(overload bool, filenames ...string) error { return nil } -var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored - // ReadWithLookup gets all env vars from the files and/or lookup function and return values as // a map rather than automatically writing values into env func ReadWithLookup(lookupFn LookupFn, filenames ...string) (map[string]string, error) { @@ -131,16 +124,6 @@ func Read(filenames ...string) (map[string]string, error) { return ReadWithLookup(nil, filenames...) } -// Unmarshal reads an env file from a string, returning a map of keys and values. -func Unmarshal(str string) (map[string]string, error) { - return UnmarshalBytes([]byte(str)) -} - -// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. -func UnmarshalBytes(src []byte) (map[string]string, error) { - return UnmarshalBytesWithLookup(src, nil) -} - // UnmarshalBytesWithLookup parses env file from byte slice of chars, returning a map of keys and values. func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string, error) { out := make(map[string]string) @@ -148,58 +131,6 @@ func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string, return out, err } -// Exec loads env vars from the specified filenames (empty map falls back to default) -// then executes the cmd specified. -// -// Simply hooks up os.Stdin/err/out to the command and calls Run() -// -// If you want more fine grained control over your command it's recommended -// that you use `Load()` or `Read()` and the `os/exec` package yourself. -func Exec(filenames []string, cmd string, cmdArgs []string) error { - if err := Load(filenames...); err != nil { - return err - } - - command := exec.Command(cmd, cmdArgs...) - command.Stdin = os.Stdin - command.Stdout = os.Stdout - command.Stderr = os.Stderr - return command.Run() -} - -// Write serializes the given environment and writes it to a file -func Write(envMap map[string]string, filename string) error { - content, err := Marshal(envMap) - if err != nil { - return err - } - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - _, err = file.WriteString(content + "\n") - if err != nil { - return err - } - return file.Sync() -} - -// Marshal outputs the given environment as a dotenv-formatted environment file. -// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. -func Marshal(envMap map[string]string) (string, error) { - lines := make([]string, 0, len(envMap)) - for k, v := range envMap { - if d, err := strconv.Atoi(v); err == nil { - lines = append(lines, fmt.Sprintf(`%s=%d`, k, d)) - } else { - lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) // nolint // Cannot use %q here - } - } - sort.Strings(lines) - return strings.Join(lines, "\n"), nil -} - func filenamesOrDefault(filenames []string) []string { if len(filenames) == 0 { return []string{".env"} @@ -239,104 +170,6 @@ func readFile(filename string, lookupFn LookupFn) (map[string]string, error) { return ParseWithLookup(file, lookupFn) } -var exportRegex = regexp.MustCompile(`^\s*(?:export\s+)?(.*?)\s*$`) - -func parseLine(line string, envMap map[string]string) (string, string, error) { - return parseLineWithLookup(line, envMap, nil) -} -func parseLineWithLookup(line string, envMap map[string]string, lookupFn LookupFn) (string, string, error) { - if line == "" { - return "", "", errors.New("zero length string") - } - - // ditch the comments (but keep quoted hashes) - if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.Contains(line, " #") { - segmentsBetweenHashes := strings.Split(line, "#") - quotesAreOpen := false - var segmentsToKeep []string - for _, segment := range segmentsBetweenHashes { - if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 { - if quotesAreOpen { - segmentsToKeep = append(segmentsToKeep, segment) - } - quotesAreOpen = !quotesAreOpen - } - - if len(segmentsToKeep) == 0 || quotesAreOpen { - segmentsToKeep = append(segmentsToKeep, segment) - } - } - - line = strings.Join(segmentsToKeep, "#") - } - - firstEquals := strings.Index(line, "=") - firstColon := strings.Index(line, ":") - splitString := strings.SplitN(line, "=", 2) - if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) { - // This is a yaml-style line - splitString = strings.SplitN(line, ":", 2) - } - - if len(splitString) != 2 { - return "", "", errors.New("can't separate key from value") - } - key := exportRegex.ReplaceAllString(splitString[0], "$1") - - // Parse the value - value := parseValue(splitString[1], envMap, lookupFn) - - return key, value, nil -} - -var ( - singleQuotesRegex = regexp.MustCompile(`\A'(.*)'\z`) - doubleQuotesRegex = regexp.MustCompile(`\A"(.*)"\z`) - escapeRegex = regexp.MustCompile(`\\.`) - unescapeCharsRegex = regexp.MustCompile(`\\([^$])`) -) - -func parseValue(value string, envMap map[string]string, lookupFn LookupFn) string { - - // trim - value = strings.Trim(value, " ") - - // check if we've got quoted values or possible escapes - if len(value) > 1 { - singleQuotes := singleQuotesRegex.FindStringSubmatch(value) - - doubleQuotes := doubleQuotesRegex.FindStringSubmatch(value) - - if singleQuotes != nil || doubleQuotes != nil { - // pull the quotes off the edges - value = value[1 : len(value)-1] - } - - if doubleQuotes != nil { - // expand newlines - value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string { - c := strings.TrimPrefix(match, `\`) - switch c { - case "n": - return "\n" - case "r": - return "\r" - default: - return match - } - }) - // unescape characters - value = unescapeCharsRegex.ReplaceAllString(value, "$1") - } - - if singleQuotes == nil { - value, _ = expandVariables(value, envMap, lookupFn) - } - } - - return value -} - func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) (string, error) { retVal, err := template.Substitute(value, func(k string) (string, bool) { if v, ok := envMap[k]; ok { @@ -349,17 +182,3 @@ func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) } return retVal, nil } - -func doubleQuoteEscape(line string) string { - for _, c := range doubleQuoteSpecialChars { - toReplace := "\\" + string(c) - if c == '\n' { - toReplace = `\n` - } - if c == '\r' { - toReplace = `\r` - } - line = strings.ReplaceAll(line, string(c), toReplace) - } - return line -} diff --git a/dotenv/godotenv_test.go b/dotenv/godotenv_test.go index c95bcfc5..f9ff5826 100644 --- a/dotenv/godotenv_test.go +++ b/dotenv/godotenv_test.go @@ -2,21 +2,27 @@ package dotenv import ( "bytes" - "fmt" "os" - "reflect" "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var noopPresets = make(map[string]string) func parseAndCompare(t *testing.T, rawEnvLine string, expectedKey string, expectedValue string) { - key, value, _ := parseLine(rawEnvLine, noopPresets) - if key != expectedKey || value != expectedValue { - t.Errorf("Expected '%v' to parse as '%v' => '%v', got '%v' => '%v' instead", rawEnvLine, expectedKey, expectedValue, key, value) + t.Helper() + env, err := Parse(strings.NewReader(rawEnvLine)) + if !assert.NoError(t, err) { + return + } + actualValue, ok := env[expectedKey] + if !ok { + t.Errorf("Key %q was not found in env: %v", expectedKey, env) + } else if actualValue != expectedValue { + t.Errorf("Expected '%v' to parse as '%v' => '%v', got '%v' => '%v' instead", rawEnvLine, expectedKey, expectedValue, expectedKey, actualValue) } } @@ -345,6 +351,7 @@ func TestParsing(t *testing.T) { // parses escaped double quotes parseAndCompare(t, `FOO="escaped\"bar"`, "FOO", `escaped"bar`) + parseAndCompare(t, `FOO="\"quoted\""`, "FOO", `"quoted"`) // parses single quotes inside double quotes parseAndCompare(t, `FOO="'d'"`, "FOO", `'d'`) @@ -368,16 +375,19 @@ func TestParsing(t *testing.T) { parseAndCompare(t, "export\tOPTION_A=2", "OPTION_A", "2") parseAndCompare(t, " export OPTION_A=2", "OPTION_A", "2") parseAndCompare(t, "\texport OPTION_A=2", "OPTION_A", "2") + parseAndCompare(t, `export OPTION_A="export A"`, "OPTION_A", "export A") // it 'expands newlines in quoted strings' do // expect(env('FOO="bar\nbaz"')).to eql('FOO' => "bar\nbaz") parseAndCompare(t, `FOO="bar\nbaz"`, "FOO", "bar\nbaz") + parseAndCompare(t, `FOO=a\tb`, "FOO", `a\tb`) + parseAndCompare(t, `FOO="a\tb"`, "FOO", "a\tb") - // it 'parses varibales with "." in the name' do + // it 'parses variables with "." in the name' do // expect(env('FOO.BAR=foobar')).to eql('FOO.BAR' => 'foobar') parseAndCompare(t, "FOO.BAR=foobar", "FOO.BAR", "foobar") - // it 'parses varibales with several "=" in the value' do + // it 'parses variables with several "=" in the value' do // expect(env('FOO=foobar=')).to eql('FOO' => 'foobar=') parseAndCompare(t, "FOO=foobar=", "FOO", "foobar=") @@ -388,10 +398,14 @@ func TestParsing(t *testing.T) { // it 'ignores inline comments' do // expect(env("foo=bar # this is foo")).to eql('foo' => 'bar') parseAndCompare(t, "FOO=bar # this is foo", "FOO", "bar") + parseAndCompare(t, "FOO=bar #this is foo", "FOO", "bar") + parseAndCompare(t, "FOO=bar #", "FOO", "bar") parseAndCompare(t, "FOO=123#not-an-inline-comment", "FOO", "123#not-an-inline-comment") // it 'allows # in quoted value' do // expect(env('foo="bar#baz" # comment')).to eql('foo' => 'bar#baz') + parseAndCompare(t, `FOO="bar#baz"`, "FOO", "bar#baz") + parseAndCompare(t, `FOO="bar#baz"#`, "FOO", "bar#baz") parseAndCompare(t, `FOO="bar#baz" # comment`, "FOO", "bar#baz") parseAndCompare(t, "FOO='bar#baz' # comment", "FOO", "bar#baz") parseAndCompare(t, `FOO="bar#baz#bang" # comment`, "FOO", "bar#baz#bang") @@ -403,28 +417,51 @@ func TestParsing(t *testing.T) { parseAndCompare(t, "FOO='ba#r'", "FOO", "ba#r") // newlines and backslashes should be escaped - parseAndCompare(t, `FOO="bar\n\ b\az"`, "FOO", "bar\n baz") - parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n baz") - parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r baz") + parseAndCompare(t, `FOO="bar\n\ b\az"`, "FOO", "bar\n\\ b\az") + parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n\\ b\az") + parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r\\ b\az") parseAndCompare(t, `="value"`, "", "value") - parseAndCompare(t, `KEY="`, "KEY", "\"") - parseAndCompare(t, `KEY="value`, "KEY", "\"value") // leading whitespace should be ignored parseAndCompare(t, " KEY =value", "KEY", "value") parseAndCompare(t, " KEY=value", "KEY", "value") parseAndCompare(t, "\tKEY=value", "KEY", "value") + // XSI-echo style octal escapes are expanded + parseAndCompare(t, `KEY="\0123"`, "KEY", "S") + + // non-XSI/POSIX escapes are ignored + parseAndCompare(t, `KEY="\x07"`, "KEY", `\x07`) + parseAndCompare(t, `KEY="\u12e4"`, "KEY", `\u12e4`) + parseAndCompare(t, `KEY="\U00101234"`, "KEY", `\U00101234`) + // it 'throws an error if line format is incorrect' do // expect{env('lol$wut')}.to raise_error(Dotenv::FormatError) badlyFormattedLine := "lol$wut" - _, _, err := parseLine(badlyFormattedLine, noopPresets) + _, err := Parse(strings.NewReader(badlyFormattedLine)) if err == nil { t.Errorf("Expected \"%v\" to return error, but it didn't", badlyFormattedLine) } } +func TestUnterminatedQuotes(t *testing.T) { + cases := []string{ + `KEY="`, + `KEY="value`, + `KEY="value\"`, + `KEY="value'`, + `KEY='`, + `KEY='value`, + `KEY='value\'`, + `KEY='value"`, + } + for _, tc := range cases { + _, err := Parse(strings.NewReader(tc)) + assert.ErrorContains(t, err, "unterminated quoted value", "Env data: %v", tc) + } +} + func TestLinesToIgnore(t *testing.T) { cases := map[string]struct { input string @@ -478,55 +515,6 @@ func TestErrorParsing(t *testing.T) { } } -func TestWrite(t *testing.T) { - writeAndCompare := func(env string, expected string) { - envMap, _ := Unmarshal(env) - actual, _ := Marshal(envMap) - if expected != actual { - t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) - } - } - // just test some single lines to show the general idea - // TestRoundtrip makes most of the good assertions - - // values are always double-quoted - writeAndCompare(`key=value`, `key="value"`) - // double-quotes are escaped - writeAndCompare(`key=va"lu"e`, `key="va\"lu\"e"`) - // but single quotes are left alone - writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) - // newlines, backslashes, and some other special chars are escaped - writeAndCompare(`foo="\n\r\\r!"`, `foo="\n\r\\r\!"`) - // lines should be sorted - writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"") - // integers should not be quoted - writeAndCompare(`key="10"`, `key=10`) - -} - -func TestRoundtrip(t *testing.T) { - fixtures := []string{"equals.env", "exported.env", "plain.env", "quoted.env"} - for _, fixture := range fixtures { - fixtureFilename := fmt.Sprintf("fixtures/%s", fixture) - env, err := readFile(fixtureFilename, nil) - if err != nil { - t.Errorf("Expected '%s' to read without error (%v)", fixtureFilename, err) - } - rep, err := Marshal(env) - if err != nil { - t.Errorf("Expected '%s' to Marshal (%v)", fixtureFilename, err) - } - roundtripped, err := Unmarshal(rep) - if err != nil { - t.Errorf("Expected '%s' to Mashal and Unmarshal (%v)", fixtureFilename, err) - } - if !reflect.DeepEqual(env, roundtripped) { - t.Errorf("Expected '%s' to roundtrip as '%v', got '%v' instead", fixtureFilename, env, roundtripped) - } - - } -} - func TestInheritedEnvVariableSameSize(t *testing.T) { const envKey = "VAR_TO_BE_LOADED_FROM_OS_ENV" const envVal = "SOME_RANDOM_VALUE" @@ -602,18 +590,18 @@ func TestInheritedEnvVariableNotFoundWithLookup(t *testing.T) { } } -func TestExpendingEnvironmentWithLookup(t *testing.T) { +func TestExpandingEnvironmentWithLookup(t *testing.T) { rawEnvLine := "TEST=$ME" expectedValue := "YES" - key, value, _ := parseLineWithLookup(rawEnvLine, noopPresets, func(s string) (string, bool) { + lookupFn := func(s string) (string, bool) { if s == "ME" { return expectedValue, true } return "NO", false - }) - if value != "YES" { - t.Errorf("Expected '%v' to parse as '%v' => '%v', got '%v' => '%v' instead", rawEnvLine, key, expectedValue, key, value) } + env, err := ParseWithLookup(strings.NewReader(rawEnvLine), lookupFn) + require.NoError(t, err) + require.Equal(t, expectedValue, env["TEST"]) } func TestSubstitutionsWithEnvFilePrecedence(t *testing.T) { diff --git a/dotenv/parser.go b/dotenv/parser.go index a0c862b8..035d489f 100644 --- a/dotenv/parser.go +++ b/dotenv/parser.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "regexp" + "strconv" "strings" "unicode" ) @@ -12,8 +14,11 @@ const ( charComment = '#' prefixSingleQuote = '\'' prefixDoubleQuote = '"' +) - exportPrefix = "export" +var ( + escapeSeqRegex = regexp.MustCompile(`(\\(?:[abcfnrtv$"\\]|0\d{0,3}))`) + exportRegex = regexp.MustCompile(`^export\s+`) ) func parseBytes(src []byte, out map[string]string, lookupFn LookupFn) error { @@ -85,7 +90,7 @@ func locateKeyName(src []byte) (string, []byte, bool, error) { var key string var inherited bool // trim "export" and space at beginning - src = bytes.TrimLeftFunc(bytes.TrimPrefix(src, []byte(exportPrefix)), isSpace) + src = bytes.TrimLeftFunc(exportRegex.ReplaceAll(src, nil), isSpace) // locate key name end and validate it in single loop offset := 0 @@ -131,21 +136,12 @@ func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (s quote, isQuoted := hasQuotePrefix(src) if !isQuoted { // unquoted value - read until new line - end := bytes.IndexFunc(src, isNewLine) - var rest []byte - if end < 0 { - value := strings.Split(string(src), "#")[0] // Remove inline comments on unquoted lines - value = strings.TrimRightFunc(value, unicode.IsSpace) - - retVal, err := expandVariables(value, envMap, lookupFn) - return retVal, nil, err - } - - value := strings.Split(string(src[0:end]), "#")[0] - value = strings.TrimRightFunc(value, unicode.IsSpace) - rest = src[end:] + value, rest, _ := bytesCut(src, []byte("\n")) - retVal, err := expandVariables(value, envMap, lookupFn) + // Remove inline comments on unquoted lines + value, _, _ = bytesCut(value, []byte(" #")) + value = bytes.TrimRightFunc(value, unicode.IsSpace) + retVal, err := expandVariables(string(value), envMap, lookupFn) return retVal, rest, err } @@ -161,12 +157,10 @@ func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (s } // trim quotes - trimFunc := isCharFunc(rune(quote)) - value := string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + value := string(src[1:i]) if quote == prefixDoubleQuote { - // unescape newlines for double quote (this is compat feature) - // and expand environment variables - + // expand standard shell escape sequences & then interpolate + // variables on the result retVal, err := expandVariables(expandEscapes(value), envMap, lookupFn) if err != nil { return "", nil, err @@ -187,20 +181,35 @@ func extractVarValue(src []byte, envMap map[string]string, lookupFn LookupFn) (s } func expandEscapes(str string) string { - out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { - c := strings.TrimPrefix(match, `\`) - switch c { - case "n": - return "\n" - case "r": - return "\r" - case "$": + out := escapeSeqRegex.ReplaceAllStringFunc(str, func(match string) string { + if match == `\$` { + // `\$` is not a Go escape sequence, the expansion parser uses + // the special `$$` syntax + // both `FOO=\$bar` and `FOO=$$bar` are valid in an env file and + // will result in FOO w/ literal value of "$bar" (no interpolation) return "$$" - default: + } + + if strings.HasPrefix(match, `\0`) { + // octal escape sequences in Go are not prefixed with `\0`, so + // rewrite the prefix, e.g. `\0123` -> `\123` -> literal value "S" + match = strings.Replace(match, `\0`, `\`, 1) + } + + // use Go to unquote (unescape) the literal + // see https://go.dev/ref/spec#Rune_literals + // + // NOTE: Go supports ADDITIONAL escapes like `\x` & `\u` & `\U`! + // These are NOT supported, which is why we use a regex to find + // only matches we support and then use `UnquoteChar` instead of a + // `Unquote` on the entire value + v, _, _, err := strconv.UnquoteChar(match, '"') + if err != nil { return match } + return string(v) }) - return unescapeCharsRegex.ReplaceAllString(out, "$1") + return out } func indexOfNonSpaceChar(src []byte) int { @@ -239,8 +248,3 @@ func isSpace(r rune) bool { } return false } - -// isNewLine reports whether the rune is a new line character -func isNewLine(r rune) bool { - return r == '\n' -}