Merge pull request #9636 from ulyssessouza/dotenvfile-priority

Environment variables priority

cmd/compose/compose.go

@@ -354,10 +354,8 @@ func setEnvWithDotEnv(prjOpts *projectOptions) error {
 		return err
 	}
 	for k, v := range envFromFile {
-		if _, ok := os.LookupEnv(k); !ok {
-			if err := os.Setenv(k, v); err != nil {
-				return err
-			}
+		if err := os.Setenv(k, v); err != nil { // overwrite the process env with merged OS + env file results
+			return err
 		}
 	}
 	return nil

go.mod

@@ -151,3 +151,5 @@ replace (
 	k8s.io/apimachinery => k8s.io/apimachinery v0.22.4
 	k8s.io/client-go => k8s.io/client-go v0.22.4
 )
+
+replace github.com/compose-spec/compose-go => github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601 // TODO Remove and bump require section when PR is merged

go.sum

@@ -285,9 +285,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/compose-spec/compose-go v1.2.1/go.mod h1:pAy7Mikpeft4pxkFU565/DRHEbDfR84G6AQuiL+Hdg8=
-github.com/compose-spec/compose-go v1.2.9 h1:+7q2X8gCd16qUX+FU5EPtepK9lWZSGfc8Xe2cGmZqDc=
-github.com/compose-spec/compose-go v1.2.9/go.mod h1:rGaQw1U8uhZhmANQHIocMnQ+qu6ER2+1OwhWjRqQEPI=
+github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601 h1:dG3F1lPuUkmpxGs5qB3dFhqsXp06bpJHd0Btt6K15cU=
+github.com/compose-spec/compose-go v1.2.9-0.20220712021117-dcfe0889b601/go.mod h1:rGaQw1U8uhZhmANQHIocMnQ+qu6ER2+1OwhWjRqQEPI=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
@@ -1007,7 +1006,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
@@ -2121,7 +2119,6 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ=
 gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

pkg/compose/run.go

@@ -114,15 +114,12 @@ func applyRunOptions(project *types.Project, service *types.ServiceConfig, opts
 		service.Entrypoint = opts.Entrypoint
 	}
 	if len(opts.Environment) > 0 {
-		env := types.NewMappingWithEquals(opts.Environment)
-		projectEnv := env.Resolve(func(s string) (string, bool) {
-			if _, ok := service.Environment[s]; ok {
-				return "", false
-			}
+		cmdEnv := types.NewMappingWithEquals(opts.Environment)
+		serviceOverrideEnv := cmdEnv.Resolve(func(s string) (string, bool) {
 			v, ok := project.Environment[s]
 			return v, ok
 		}).RemoveEmpty()
-		service.Environment.OverrideBy(projectEnv)
+		service.Environment.OverrideBy(serviceOverrideEnv)
 	}
 	for k, v := range opts.Labels {
 		service.Labels = service.Labels.Add(k, v)

pkg/e2e/compose_environment_test.go

@@ -27,137 +27,196 @@ import (
 func TestEnvPriority(t *testing.T) {
 	c := NewParallelCLI(t)
 
-	projectDir := "./fixtures/environment/env-priority"
-
 	t.Run("up", func(t *testing.T) {
 		c.RunDockerOrExitError(t, "rmi", "env-compose-priority")
 		c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
-			"--project-directory", projectDir, "up", "-d", "--build")
+			"up", "-d", "--build")
 	})
 
 	// Full options activated
-	// 1. Compose file <-- Result expected
-	// 2. Shell environment variables
-	// 3. Environment file
-	// 4. Dockerfile
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment patched by --env-file)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("compose file priority", func(t *testing.T) {
 		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
-			"--project-directory", projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run",
-			"--rm", "-e", "WHEREAMI", "env-compose-priority")
+			"--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
 		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
 		res := icmd.RunCmd(cmd)
-		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Compose File")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
+	})
+
+	// Full options activated
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
+	// 5. Variable is not defined
+	t.Run("compose file priority", func(t *testing.T) {
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI=shell", "env-compose-priority")
+		res := icmd.RunCmd(cmd)
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
 	})
 
 	// No Compose file, all other options
-	// 1. Compose file
-	// 2. Shell environment variables <-- Result expected
-	// 3. Environment file
-	// 4. Dockerfile
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment patched by --env-file)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("shell priority", func(t *testing.T) {
-		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
-			projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
-			"WHEREAMI", "env-compose-priority")
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
+		res := icmd.RunCmd(cmd)
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
+	})
+
+	// No Compose file, all other options with env variable from OS environment
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
+	// 5. Variable is not defined
+	t.Run("shell priority file with default value", func(t *testing.T) {
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override.with.default",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
 		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
 		res := icmd.RunCmd(cmd)
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell")
 	})
 
-	//  No Compose file and env variable pass to the run command
-	// 1. Compose file
-	// 2. Shell environment variables <-- Result expected
-	// 3. Environment file
-	// 4. Dockerfile
+	// No Compose file, all other options with env variable from OS environment
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment default value from file in --env-file)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
+	// 5. Variable is not defined
+	t.Run("shell priority implicitly set", func(t *testing.T) {
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override.with.default",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		res := icmd.RunCmd(cmd)
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "EnvFileDefaultValue")
+	})
+
+	// No Compose file and env variable pass to the run command
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("shell priority from run command", func(t *testing.T) {
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
-			projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
-			"WHEREAMI=shell-run", "env-compose-priority")
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI=shell-run", "env-compose-priority")
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "shell-run")
 	})
 
-	//  No Compose file & no env variable but override env file
-	// 1. Compose file
-	// 2. Shell environment variables
-	// 3. Environment file <-- Result expected
-	// 4. Dockerfile
+	// No Compose file & no env variable but override env file
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment patched by .env as a default --env-file value)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
+	// 5. Variable is not defined
+	t.Run("override env file from compose", func(t *testing.T) {
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose-with-env-file.yaml",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
+	})
+
+	// No Compose file & no env variable but override by default env file
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment patched by --env-file value)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("override env file", func(t *testing.T) {
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
-			projectDir, "--env-file", "./fixtures/environment/env-priority/.env.override", "run", "--rm", "-e",
-			"WHEREAMI", "env-compose-priority")
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.override",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "override")
 	})
 
-	//  No Compose file & no env variable but override env file
-	// 1. Compose file
-	// 2. Shell environment variables
-	// 3. Environment file <-- Result expected
-	// 4. Dockerfile
+	// No Compose file & no env variable but override env file
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)  <-- Result expected (From environment patched by --env-file value)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("env file", func(t *testing.T) {
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
-			projectDir, "run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Env File")
 	})
 
-	//  No Compose file & no env variable, using an empty override env file
-	// 1. Compose file
-	// 2. Shell environment variables
-	// 3. Environment file
-	// 4. Dockerfile   <-- Result expected
+	// No Compose file & no env variable, using an empty override env file
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)
+	// 2. Compose File (service::environment section)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive <-- Result expected
 	// 5. Variable is not defined
 	t.Run("use Dockerfile", func(t *testing.T) {
-		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml", "--project-directory",
-			projectDir, "--env-file", "./fixtures/environment/env-priority/.env.empty", "run", "--rm", "-e", "WHEREAMI",
-			"env-compose-priority")
+		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-priority/compose.yaml",
+			"--env-file", "./fixtures/environment/env-priority/.env.empty",
+			"run", "--rm", "-e", "WHEREAMI", "env-compose-priority")
 		assert.Equal(t, strings.TrimSpace(res.Stdout()), "Dockerfile")
 	})
 
 	t.Run("down", func(t *testing.T) {
-		c.RunDockerComposeCmd(t, "--project-directory", projectDir, "down")
+		c.RunDockerComposeCmd(t, "--project-name", "env-priority", "down")
 	})
 }
 
 func TestEnvInterpolation(t *testing.T) {
 	c := NewParallelCLI(t)
 
-	projectDir := "./fixtures/environment/env-interpolation"
-
-	//  No variable defined in the Compose file and env variable pass to the run command
-	// 1. Compose file
-	// 2. Shell environment variables <-- Result expected
-	// 3. Environment file
-	// 4. Dockerfile
+	// No variable defined in the Compose file and nor env variable pass to the run command
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)
+	// 2. Compose File (service::environment section) <-- Result expected (From environment patched by .env as a default --env-file value)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
 	// 5. Variable is not defined
 	t.Run("shell priority from run command", func(t *testing.T) {
-		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml",
-			"--project-directory", projectDir, "config")
+		cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation/compose.yaml", "config")
 		cmd.Env = append(cmd.Env, "WHEREAMI=shell")
 		res := icmd.RunCmd(cmd)
-		res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:shell`})
+		res.Assert(t, icmd.Expected{Out: `IMAGE: default_env:EnvFile`})
+	})
+
+	// No variable defined in the Compose file and env variable pass to the run command
+	// 1. Command Line (docker compose run --env <KEY[=VAL]>)
+	// 2. Compose File (service::environment section) <-- Result expected (From environment patched by .env as a default --env-file value.
+	// This variable has a default value in case of an absent variable in the OS environment)
+	// 3. Compose File (service::env_file section file)
+	// 4. Container Image ENV directive
+	// 5. Variable is not defined
+	t.Run("shell priority from run command using default value fallback", func(t *testing.T) {
+		c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-interpolation-default-value/compose.yaml", "config").
+			Assert(t, icmd.Expected{Out: `IMAGE: default_env:EnvFileDefaultValue`})
 	})
 }
 
 func TestCommentsInEnvFile(t *testing.T) {
 	c := NewParallelCLI(t)
 
-	projectDir := "./fixtures/environment/env-file-comments"
-
 	t.Run("comments in env files", func(t *testing.T) {
 		c.RunDockerOrExitError(t, "rmi", "env-file-comments")
 
-		c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml", "--project-directory",
-			projectDir, "up", "-d", "--build")
+		c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml", "up", "-d", "--build")
 
 		res := c.RunDockerComposeCmd(t, "-f", "./fixtures/environment/env-file-comments/compose.yaml",
-			"--project-directory", projectDir, "run", "--rm", "-e", "COMMENT", "-e", "NO_COMMENT", "env-file-comments")
+			"run", "--rm", "-e", "COMMENT", "-e", "NO_COMMENT", "env-file-comments")
 
 		res.Assert(t, icmd.Expected{Out: `COMMENT=1234`})
 		res.Assert(t, icmd.Expected{Out: `NO_COMMENT=1234#5`})
 
-		c.RunDockerComposeCmd(t, "--project-directory", projectDir, "down", "--rmi", "all")
+		c.RunDockerComposeCmd(t, "--project-name", "env-file-comments", "down", "--rmi", "all")
 	})
 }

pkg/e2e/fixtures/environment/env-interpolation-default-value/.env

@@ -0,0 +1 @@
+IMAGE=default_env:${WHEREAMI:-EnvFileDefaultValue}

pkg/e2e/fixtures/environment/env-interpolation-default-value/compose.yaml

@@ -0,0 +1,6 @@
+services:
+  env-interpolation:
+    image: bash
+    environment:
+      IMAGE: ${IMAGE}
+    command: echo "$IMAGE"

pkg/e2e/fixtures/environment/env-interpolation/.env

@@ -1,2 +1,2 @@
-WHEREAMI=Env File
-IMAGE=default_env:${WHEREAMI}
+WHEREAMI=EnvFile
+IMAGE=default_env:${WHEREAMI}

pkg/e2e/fixtures/environment/env-priority/.env.override.with.default

@@ -0,0 +1 @@
+WHEREAMI=${WHEREAMI:-EnvFileDefaultValue}

pkg/e2e/fixtures/environment/env-priority/compose-with-env-file.yaml

@@ -0,0 +1,7 @@
+services:
+  env-compose-priority:
+    image: env-compose-priority
+    build:
+      context: .
+    env_file:
+      - .env.override