/
framework.go
447 lines (390 loc) · 13.2 KB
/
framework.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
/*
Copyright 2020 Docker Compose CLI authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"gotest.tools/v3/assert"
"gotest.tools/v3/icmd"
"gotest.tools/v3/poll"
"github.com/docker/compose/v2/cmd/compose"
)
var (
// DockerExecutableName is the OS dependent Docker CLI binary name
DockerExecutableName = "docker"
// DockerComposeExecutableName is the OS dependent Docker CLI binary name
DockerComposeExecutableName = "docker-" + compose.PluginName
// DockerScanExecutableName is the OS dependent Docker Scan plugin binary name
DockerScanExecutableName = "docker-scan"
// DockerBuildxExecutableName is the Os dependent Buildx plugin binary name
DockerBuildxExecutableName = "docker-buildx"
// WindowsExecutableSuffix is the Windows executable suffix
WindowsExecutableSuffix = ".exe"
)
func init() {
if runtime.GOOS == "windows" {
DockerExecutableName += WindowsExecutableSuffix
DockerComposeExecutableName += WindowsExecutableSuffix
DockerScanExecutableName += WindowsExecutableSuffix
DockerBuildxExecutableName += WindowsExecutableSuffix
}
}
// CLI is used to wrap the CLI for end to end testing
type CLI struct {
// ConfigDir for Docker configuration (set as DOCKER_CONFIG)
ConfigDir string
// HomeDir for tools that look for user files (set as HOME)
HomeDir string
// env overrides to apply to every invoked command
//
// To populate, use WithEnv when creating a CLI instance.
env []string
}
// CLIOption to customize behavior for all commands for a CLI instance.
type CLIOption func(c *CLI)
// NewParallelCLI marks the parent test as parallel and returns a CLI instance
// suitable for usage across child tests.
func NewParallelCLI(t *testing.T, opts ...CLIOption) *CLI {
t.Helper()
t.Parallel()
return NewCLI(t, opts...)
}
// NewCLI creates a CLI instance for running E2E tests.
func NewCLI(t testing.TB, opts ...CLIOption) *CLI {
t.Helper()
configDir := t.TempDir()
initializePlugins(t, configDir)
c := &CLI{
ConfigDir: configDir,
HomeDir: t.TempDir(),
}
for _, opt := range opts {
opt(c)
}
t.Log(c.RunDockerComposeCmdNoCheck(t, "version").Combined())
return c
}
// WithEnv sets environment variables that will be passed to commands.
func WithEnv(env ...string) CLIOption {
return func(c *CLI) {
c.env = append(c.env, env...)
}
}
// initializePlugins copies the necessary plugin files to the temporary config
// directory for the test.
func initializePlugins(t testing.TB, configDir string) {
t.Helper()
t.Cleanup(func() {
if t.Failed() {
if conf, err := os.ReadFile(filepath.Join(configDir, "config.json")); err == nil {
t.Logf("Config: %s\n", string(conf))
}
t.Log("Contents of config dir:")
for _, p := range dirContents(configDir) {
t.Logf(" - %s", p)
}
}
})
require.NoError(t, os.MkdirAll(filepath.Join(configDir, "cli-plugins"), 0o755),
"Failed to create cli-plugins directory")
composePlugin, err := findExecutable(DockerComposeExecutableName)
if os.IsNotExist(err) {
t.Logf("WARNING: docker-compose cli-plugin not found")
}
if err == nil {
CopyFile(t, composePlugin, filepath.Join(configDir, "cli-plugins", DockerComposeExecutableName))
buildxPlugin, err := findPluginExecutable(DockerBuildxExecutableName)
if err != nil {
t.Logf("WARNING: docker-buildx cli-plugin not found, using default buildx installation.")
} else {
CopyFile(t, buildxPlugin, filepath.Join(configDir, "cli-plugins", DockerBuildxExecutableName))
}
// We don't need a functional scan plugin, but a valid plugin binary
CopyFile(t, composePlugin, filepath.Join(configDir, "cli-plugins", DockerScanExecutableName))
}
}
func dirContents(dir string) []string {
var res []string
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
res = append(res, path)
return nil
})
return res
}
func findExecutable(executableName string) (string, error) {
_, filename, _, _ := runtime.Caller(0)
root := filepath.Join(filepath.Dir(filename), "..", "..")
buildPath := filepath.Join(root, "bin", "build")
bin, err := filepath.Abs(filepath.Join(buildPath, executableName))
if err != nil {
return "", err
}
if _, err := os.Stat(bin); err == nil {
return bin, nil
}
return "", errors.Wrap(os.ErrNotExist, "executable not found")
}
func findPluginExecutable(pluginExecutableName string) (string, error) {
dockerUserDir := ".docker/cli-plugins"
userDir, err := os.UserHomeDir()
if err != nil {
return "", err
}
bin, err := filepath.Abs(filepath.Join(userDir, dockerUserDir, pluginExecutableName))
if err != nil {
return "", err
}
if _, err := os.Stat(bin); err == nil {
return bin, nil
}
return "", errors.Wrap(os.ErrNotExist, fmt.Sprintf("plugin not found %s", pluginExecutableName))
}
// CopyFile copies a file from a sourceFile to a destinationFile setting permissions to 0755
func CopyFile(t testing.TB, sourceFile string, destinationFile string) {
t.Helper()
src, err := os.Open(sourceFile)
require.NoError(t, err, "Failed to open source file: %s")
//nolint:errcheck
defer src.Close()
dst, err := os.OpenFile(destinationFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o755)
require.NoError(t, err, "Failed to open destination file: %s", destinationFile)
//nolint:errcheck
defer dst.Close()
_, err = io.Copy(dst, src)
require.NoError(t, err, "Failed to copy file: %s", sourceFile)
}
// BaseEnvironment provides the minimal environment variables used across all
// Docker / Compose commands.
func (c *CLI) BaseEnvironment() []string {
return []string{
"HOME=" + c.HomeDir,
"USER=" + os.Getenv("USER"),
"DOCKER_CONFIG=" + c.ConfigDir,
"KUBECONFIG=invalid",
}
}
// NewCmd creates a cmd object configured with the test environment set
func (c *CLI) NewCmd(command string, args ...string) icmd.Cmd {
return icmd.Cmd{
Command: append([]string{command}, args...),
Env: append(c.BaseEnvironment(), c.env...),
}
}
// NewCmdWithEnv creates a cmd object configured with the test environment set with additional env vars
func (c *CLI) NewCmdWithEnv(envvars []string, command string, args ...string) icmd.Cmd {
// base env -> CLI overrides -> cmd overrides
cmdEnv := append(c.BaseEnvironment(), c.env...)
cmdEnv = append(cmdEnv, envvars...)
return icmd.Cmd{
Command: append([]string{command}, args...),
Env: cmdEnv,
}
}
// MetricsSocket get the path where test metrics will be sent
func (c *CLI) MetricsSocket() string {
return filepath.Join(c.ConfigDir, "docker-cli.sock")
}
// NewDockerCmd creates a docker cmd without running it
func (c *CLI) NewDockerCmd(t testing.TB, args ...string) icmd.Cmd {
t.Helper()
for _, arg := range args {
if arg == compose.PluginName {
t.Fatal("This test called 'RunDockerCmd' for 'compose'. Please prefer 'RunDockerComposeCmd' to be able to test as a plugin and standalone")
}
}
return c.NewCmd(DockerExecutableName, args...)
}
// RunDockerOrExitError runs a docker command and returns a result
func (c *CLI) RunDockerOrExitError(t testing.TB, args ...string) *icmd.Result {
t.Helper()
t.Logf("\t[%s] docker %s\n", t.Name(), strings.Join(args, " "))
return icmd.RunCmd(c.NewDockerCmd(t, args...))
}
// RunCmd runs a command, expects no error and returns a result
func (c *CLI) RunCmd(t testing.TB, args ...string) *icmd.Result {
t.Helper()
t.Logf("\t[%s] %s\n", t.Name(), strings.Join(args, " "))
assert.Assert(t, len(args) >= 1, "require at least one command in parameters")
res := icmd.RunCmd(c.NewCmd(args[0], args[1:]...))
res.Assert(t, icmd.Success)
return res
}
// RunCmdInDir runs a command in a given dir, expects no error and returns a result
func (c *CLI) RunCmdInDir(t testing.TB, dir string, args ...string) *icmd.Result {
t.Helper()
t.Logf("\t[%s] %s\n", t.Name(), strings.Join(args, " "))
assert.Assert(t, len(args) >= 1, "require at least one command in parameters")
cmd := c.NewCmd(args[0], args[1:]...)
cmd.Dir = dir
res := icmd.RunCmd(cmd)
res.Assert(t, icmd.Success)
return res
}
// RunDockerCmd runs a docker command, expects no error and returns a result
func (c *CLI) RunDockerCmd(t testing.TB, args ...string) *icmd.Result {
t.Helper()
res := c.RunDockerOrExitError(t, args...)
res.Assert(t, icmd.Success)
return res
}
// RunDockerComposeCmd runs a docker compose command, expects no error and returns a result
func (c *CLI) RunDockerComposeCmd(t testing.TB, args ...string) *icmd.Result {
t.Helper()
res := c.RunDockerComposeCmdNoCheck(t, args...)
res.Assert(t, icmd.Success)
return res
}
// RunDockerComposeCmdNoCheck runs a docker compose command, don't presume of any expectation and returns a result
func (c *CLI) RunDockerComposeCmdNoCheck(t testing.TB, args ...string) *icmd.Result {
t.Helper()
return icmd.RunCmd(c.NewDockerComposeCmd(t, args...))
}
// NewDockerComposeCmd creates a command object for Compose, either in plugin
// or standalone mode (based on build tags).
func (c *CLI) NewDockerComposeCmd(t testing.TB, args ...string) icmd.Cmd {
t.Helper()
if composeStandaloneMode {
return c.NewCmd(ComposeStandalonePath(t), args...)
}
args = append([]string{"compose"}, args...)
return c.NewCmd(DockerExecutableName, args...)
}
// ComposeStandalonePath returns the path to the locally-built Compose
// standalone binary from the repo.
//
// This function will fail the test immediately if invoked when not running
// in standalone test mode.
func ComposeStandalonePath(t testing.TB) string {
t.Helper()
if !composeStandaloneMode {
require.Fail(t, "Not running in standalone mode")
}
composeBinary, err := findExecutable(DockerComposeExecutableName)
require.NoError(t, err, "Could not find standalone Compose binary (%q)",
DockerComposeExecutableName)
return composeBinary
}
// StdoutContains returns a predicate on command result expecting a string in stdout
func StdoutContains(expected string) func(*icmd.Result) bool {
return func(res *icmd.Result) bool {
return strings.Contains(res.Stdout(), expected)
}
}
func IsHealthy(service string) func(res *icmd.Result) bool {
return func(res *icmd.Result) bool {
type state struct {
Name string `json:"name"`
Health string `json:"health"`
}
ps := []state{}
err := json.Unmarshal([]byte(res.Stdout()), &ps)
if err != nil {
return false
}
for _, state := range ps {
if state.Name == service && state.Health == "healthy" {
return true
}
}
return false
}
}
// WaitForCmdResult try to execute a cmd until resulting output matches given predicate
func (c *CLI) WaitForCmdResult(
t testing.TB,
command icmd.Cmd,
predicate func(*icmd.Result) bool,
timeout time.Duration,
delay time.Duration,
) {
t.Helper()
assert.Assert(t, timeout.Nanoseconds() > delay.Nanoseconds(), "timeout must be greater than delay")
var res *icmd.Result
checkStopped := func(logt poll.LogT) poll.Result {
fmt.Printf("\t[%s] %s\n", t.Name(), strings.Join(command.Command, " "))
res = icmd.RunCmd(command)
if !predicate(res) {
return poll.Continue("Cmd output did not match requirement: %q", res.Combined())
}
return poll.Success()
}
poll.WaitOn(t, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
}
// WaitForCondition wait for predicate to execute to true
func (c *CLI) WaitForCondition(
t testing.TB,
predicate func() (bool, string),
timeout time.Duration,
delay time.Duration,
) {
t.Helper()
checkStopped := func(logt poll.LogT) poll.Result {
pass, description := predicate()
if !pass {
return poll.Continue("Condition not met: %q", description)
}
return poll.Success()
}
poll.WaitOn(t, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
}
// Lines split output into lines
func Lines(output string) []string {
return strings.Split(strings.TrimSpace(output), "\n")
}
// HTTPGetWithRetry performs an HTTP GET on an `endpoint`, using retryDelay also as a request timeout.
// In the case of an error or the response status is not the expected one, it retries the same request,
// returning the response body as a string (empty if we could not reach it)
func HTTPGetWithRetry(
t testing.TB,
endpoint string,
expectedStatus int,
retryDelay time.Duration,
timeout time.Duration,
) string {
t.Helper()
var (
r *http.Response
err error
)
client := &http.Client{
Timeout: retryDelay,
}
fmt.Printf("\t[%s] GET %s\n", t.Name(), endpoint)
checkUp := func(t poll.LogT) poll.Result {
r, err = client.Get(endpoint)
if err != nil {
return poll.Continue("reaching %q: Error %s", endpoint, err.Error())
}
if r.StatusCode == expectedStatus {
return poll.Success()
}
return poll.Continue("reaching %q: %d != %d", endpoint, r.StatusCode, expectedStatus)
}
poll.WaitOn(t, checkUp, poll.WithDelay(retryDelay), poll.WithTimeout(timeout))
if r != nil {
b, err := io.ReadAll(r.Body)
assert.NilError(t, err)
return string(b)
}
return ""
}