Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

env: fix a bunch of escaping/parsing edge cases #308

Merged
merged 6 commits into from Sep 26, 2022
Merged

env: fix a bunch of escaping/parsing edge cases #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e3531d4
env: fix a bunch of escaping/parsing edge cases
milas Sep 23, 2022
03bca44
Merge remote-tracking branch 'upstream/master' into env-parser-escapes
milas Sep 26, 2022
ce012ee
ci: run go mod tidy for 1.18
milas Sep 26, 2022
5b35f69
remove conditional build for 1.17 compat
milas Sep 26, 2022
7d9b4f6
restore deleted functions as deprecated
milas Sep 26, 2022
300f36b
test: more escape cases
milas Sep 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
134 changes: 21 additions & 113 deletions dotenv/godotenv.go
View file
Open in desktop
Expand Up @@ -4,18 +4,17 @@
//
// The TL;DR is that you make a .env file that looks something like
//
// SOME_ENV_VAR=somevalue
// SOME_ENV_VAR=somevalue
//
// and then in your go code you can call
//
// godotenv.Load()
// godotenv.Load()
//
// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR")
package dotenv

import (
"bytes"
"errors"
"fmt"
"io"
"os"
Expand All @@ -28,10 +27,10 @@ import (
"github.com/compose-spec/compose-go/template"
)

const doubleQuoteSpecialChars = "\\\n\r\"!$`"

var utf8BOM = []byte("\uFEFF")

var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored

// LookupFn represents a lookup function to resolve variables from
type LookupFn func(string) (string, bool)

Expand Down Expand Up @@ -60,13 +59,13 @@ func ParseWithLookup(r io.Reader, lookupFn LookupFn) (map[string]string, error)

// Load will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
// Call this function as close as possible to the start of your program (ideally in main).
//
// If you call Load without any args it will default to loading .env in the current path
// If you call Load without any args it will default to loading .env in the current path.
//
// You can otherwise tell it which files to load (there can be more than one) like
// You can otherwise tell it which files to load (there can be more than one) like:
//
// godotenv.Load("fileone", "filetwo")
// godotenv.Load("fileone", "filetwo")
//
// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
func Load(filenames ...string) error {
Expand All @@ -75,13 +74,13 @@ func Load(filenames ...string) error {

// Overload will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
// Call this function as close as possible to the start of your program (ideally in main).
//
// If you call Overload without any args it will default to loading .env in the current path
// If you call Overload without any args it will default to loading .env in the current path.
//
// You can otherwise tell it which files to load (there can be more than one) like
// You can otherwise tell it which files to load (there can be more than one) like:
//
// godotenv.Overload("fileone", "filetwo")
// godotenv.Overload("fileone", "filetwo")
//
// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars.
func Overload(filenames ...string) error {
Expand All @@ -99,8 +98,6 @@ func load(overload bool, filenames ...string) error {
return nil
}

var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored

// ReadWithLookup gets all env vars from the files and/or lookup function and return values as
// a map rather than automatically writing values into env
func ReadWithLookup(lookupFn LookupFn, filenames ...string) (map[string]string, error) {
Expand Down Expand Up @@ -155,6 +152,8 @@ func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string,
//
// If you want more fine grained control over your command it's recommended
// that you use `Load()` or `Read()` and the `os/exec` package yourself.
//
// Deprecated: Use the `os/exec` package directly.
func Exec(filenames []string, cmd string, cmdArgs []string) error {
if err := Load(filenames...); err != nil {
return err
Expand All @@ -168,7 +167,10 @@ func Exec(filenames []string, cmd string, cmdArgs []string) error {
}

// Write serializes the given environment and writes it to a file
//
// Deprecated: The serialization functions are untested and unmaintained.
func Write(envMap map[string]string, filename string) error {
//goland:noinspection GoDeprecation
content, err := Marshal(envMap)
if err != nil {
return err
Expand All @@ -187,6 +189,8 @@ func Write(envMap map[string]string, filename string) error {

// Marshal outputs the given environment as a dotenv-formatted environment file.
// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped.
//
// Deprecated: The serialization functions are untested and unmaintained.
func Marshal(envMap map[string]string) (string, error) {
lines := make([]string, 0, len(envMap))
for k, v := range envMap {
Expand Down Expand Up @@ -239,104 +243,6 @@ func readFile(filename string, lookupFn LookupFn) (map[string]string, error) {
return ParseWithLookup(file, lookupFn)
}

var exportRegex = regexp.MustCompile(`^\s*(?:export\s+)?(.*?)\s*$`)

func parseLine(line string, envMap map[string]string) (string, string, error) {
return parseLineWithLookup(line, envMap, nil)
}
func parseLineWithLookup(line string, envMap map[string]string, lookupFn LookupFn) (string, string, error) {
if line == "" {
return "", "", errors.New("zero length string")
}

// ditch the comments (but keep quoted hashes)
if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.Contains(line, " #") {
segmentsBetweenHashes := strings.Split(line, "#")
quotesAreOpen := false
var segmentsToKeep []string
for _, segment := range segmentsBetweenHashes {
if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
if quotesAreOpen {
segmentsToKeep = append(segmentsToKeep, segment)
}
quotesAreOpen = !quotesAreOpen
}

if len(segmentsToKeep) == 0 || quotesAreOpen {
segmentsToKeep = append(segmentsToKeep, segment)
}
}

line = strings.Join(segmentsToKeep, "#")
}

firstEquals := strings.Index(line, "=")
firstColon := strings.Index(line, ":")
splitString := strings.SplitN(line, "=", 2)
if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) {
// This is a yaml-style line
splitString = strings.SplitN(line, ":", 2)
}

if len(splitString) != 2 {
return "", "", errors.New("can't separate key from value")
}
key := exportRegex.ReplaceAllString(splitString[0], "$1")

// Parse the value
value := parseValue(splitString[1], envMap, lookupFn)

return key, value, nil
}

var (
singleQuotesRegex = regexp.MustCompile(`\A'(.*)'\z`)
doubleQuotesRegex = regexp.MustCompile(`\A"(.*)"\z`)
escapeRegex = regexp.MustCompile(`\\.`)
unescapeCharsRegex = regexp.MustCompile(`\\([^$])`)
)

func parseValue(value string, envMap map[string]string, lookupFn LookupFn) string {

// trim
value = strings.Trim(value, " ")

// check if we've got quoted values or possible escapes
if len(value) > 1 {
singleQuotes := singleQuotesRegex.FindStringSubmatch(value)

doubleQuotes := doubleQuotesRegex.FindStringSubmatch(value)

if singleQuotes != nil || doubleQuotes != nil {
// pull the quotes off the edges
value = value[1 : len(value)-1]
}

if doubleQuotes != nil {
// expand newlines
value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string {
c := strings.TrimPrefix(match, `\`)
switch c {
case "n":
return "\n"
case "r":
return "\r"
default:
return match
}
})
// unescape characters
value = unescapeCharsRegex.ReplaceAllString(value, "$1")
}

if singleQuotes == nil {
value, _ = expandVariables(value, envMap, lookupFn)
}
}

return value
}

func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) (string, error) {
retVal, err := template.Substitute(value, func(k string) (string, bool) {
if v, ok := envMap[k]; ok {
Expand All @@ -350,7 +256,9 @@ func expandVariables(value string, envMap map[string]string, lookupFn LookupFn)
return retVal, nil
}

// Deprecated: only used by unsupported/untested code for Marshal/Write.
func doubleQuoteEscape(line string) string {
const doubleQuoteSpecialChars = "\\\n\r\"!$`"
for _, c := range doubleQuoteSpecialChars {
toReplace := "\\" + string(c)
if c == '\n' {
Expand Down