add a new container probe for multi container builds with docker comp…

…ose (#255)

* add flag --container-probe-compose-svc

* only include files in slimmed image if they were present in the original image

* trace for syscalls with ptrace and join fdpath or cwd when relative

* revert getStringParam

* update flag help for container.probe

* readme

* bugfix ptrace arm64 interface

README.md

@@ -409,6 +409,7 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `compose-env-file` - Load compose env vars from file (host env vars override the values loaded from this file)
 - `compose-workdir` - Set custom work directory for compose
 - `compose-project-name` - Use custom project name for compose
+- `container-probe-compose-svc` - Set container probe to compose service
 - `prestart-compose-svc` - placeholder for now
 - `poststart-compose-svc` - placeholder for now
 - `--http-probe` - Enables/disables HTTP probing (ENABLED by default; you have to disable the probe if you don't need it by setting the flag to `false`: `--http-probe=false`)
@@ -469,7 +470,7 @@ In the interactive CLI prompt mode you must specify the target image using the `
 - `--container-dns` - Add a dns server analyzing image at runtime [can use this flag multiple times]
 - `--container-dns-search` - Add a dns search domain for unqualified hostnames analyzing image at runtime [can use this flag multiple times]
 - `--image-overrides` - Save runtime overrides in generated image (values is `all` or a comma delimited list of override types: `entrypoint`, `cmd`, `workdir`, `env`, `expose`, `volume`, `label`). Use this flag if you need to set a runtime value and you want to persist it in the optimized image. If you only want to add, edit or delete an image value in the optimized image use one of the `--new-*` or `--remove-*` flags (define below).
-- `--continue-after` - Select continue mode: `enter` | `signal` | `probe` | `exec` | `timeout` or numberInSeconds (default value if http probes are disabled: `enter`). You can also select `probe` and `exec` together: `'probe&exec'` (make sure to use quotes around the two modes or the `&` will break the shell command).
+- `--continue-after` - Select continue mode: `enter` | `signal` | `probe` | `exec` | `timeout-number-in-seconds` | `container.probe` (default value if http probes are disabled: `enter`). You can also select `probe` and `exec` together: `'probe&exec'` (make sure to use quotes around the two modes or the `&` will break the shell command).
 - `--dockerfile` - The source Dockerfile name to build the fat image before it's optimized.
 - `--tag-fat` - Custom tag for the fat image built from Dockerfile.
 - `--cbo-add-host` - Add an extra host-to-IP mapping in /etc/hosts to use when building an image (Container Build Option).
@@ -965,4 +966,3 @@ Apache License v2, see [LICENSE](https://github.com/docker-slim/docker-slim/blob
 ---
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/docker-slim/docker-slim)](https://goreportcard.com/report/github.com/docker-slim/docker-slim)
-

pkg/app/master/commands/build/cli.go

@@ -44,6 +44,7 @@ var CLI = cli.Command{
 		commands.Cflag(commands.FlagComposeEnvFile),
 		commands.Cflag(commands.FlagComposeProjectName),
 		commands.Cflag(commands.FlagComposeWorkdir),
+		commands.Cflag(commands.FlagContainerProbeComposeSvc),
 
 		commands.Cflag(commands.FlagHTTPProbeOff),
 		commands.Cflag(commands.FlagHTTPProbe),
@@ -186,6 +187,7 @@ var CLI = cli.Command{
 
 		composeProjectName := ctx.String(commands.FlagComposeProjectName)
 		composeWorkdir := ctx.String(commands.FlagComposeWorkdir)
+		containerProbeComposeSvc := ctx.String(commands.FlagContainerProbeComposeSvc)
 
 		var targetRef string
 
@@ -580,6 +582,19 @@ var CLI = cli.Command{
 			}
 		}
 
+		if containerProbeComposeSvc != "" {
+			continueAfter.Mode = config.CAMContainerProbe
+			xc.Out.Info("exec",
+				ovars{
+					"message": "changing continue-after to container-probe",
+				})
+			doHTTPProbe = false
+			xc.Out.Info("exec",
+				ovars{
+					"message": "changing http-probe to false",
+				})
+		}
+
 		if continueAfter.Mode == "" {
 			continueAfter.Mode = config.CAMEnter
 			xc.Out.Info("exec",
@@ -615,6 +630,7 @@ var CLI = cli.Command{
 			composeEnvNoHost,
 			composeWorkdir,
 			composeProjectName,
+			containerProbeComposeSvc,
 			cbOpts,
 			crOpts,
 			outputTags,

pkg/app/master/commands/build/handler.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -52,6 +53,33 @@ const (
 
 type ovars = app.OutVars
 
+func NewLogWriter(name string) *chanWriter {
+	r, w := io.Pipe()
+	cw := &chanWriter{
+		Name: name,
+		r:    r,
+		w:    w,
+	}
+	go func() {
+		s := bufio.NewScanner(cw.r)
+		for s.Scan() {
+			fmt.Println(name + ": " + string(s.Bytes()))
+		}
+	}()
+	return cw
+}
+
+type chanWriter struct {
+	Name string
+	Chan chan string
+	w    *io.PipeWriter
+	r    *io.PipeReader
+}
+
+func (w *chanWriter) Write(p []byte) (n int, err error) {
+	return w.w.Write(p)
+}
+
 // OnCommand implements the 'build' docker-slim command
 func OnCommand(
 	xc *app.ExecutionContext,
@@ -75,6 +103,7 @@ func OnCommand(
 	composeEnvNoHost bool,
 	composeWorkdir string,
 	composeProjectName string,
+	containerProbeComposeSvc string,
 	cbOpts *config.ContainerBuildOptions,
 	crOpts *config.ContainerRunOptions,
 	outputTags []string,
@@ -363,6 +392,7 @@ func OnCommand(
 			composeWorkdir,
 			composeEnvVars,
 			composeEnvNoHost,
+			containerProbeComposeSvc,
 			false, //buildImages
 			doPull,
 			nil,  //pullExcludes (todo: add a flag)
@@ -961,8 +991,11 @@ func OnCommand(
 
 	logger.Info("watching container monitor...")
 
-	if strings.Contains(continueAfter.Mode, config.CAMProbe) {
-		doHTTPProbe = true
+	for _, mode := range strings.Split(continueAfter.Mode, "&") {
+		if mode == config.CAMProbe {
+			doHTTPProbe = true
+			break
+		}
 	}
 
 	var probe *http.CustomProbe
@@ -1020,8 +1053,11 @@ func OnCommand(
 		continueAfterMsg = "no input required, execution will resume after the timeout"
 	}
 
-	if strings.Contains(continueAfter.Mode, config.CAMProbe) {
-		continueAfterMsg = "no input required, execution will resume when HTTP probing is completed"
+	for _, mode := range strings.Split(continueAfter.Mode, "&") {
+		if mode == config.CAMProbe {
+			continueAfterMsg = "no input required, execution will resume when HTTP probing is completed"
+			break
+		}
 	}
 
 	xc.Out.Info("continue.after",
@@ -1038,6 +1074,50 @@ func OnCommand(
 		//when probe and signal are combined
 		//because both need channels (TODO: fix)
 		switch mode {
+		case config.CAMContainerProbe:
+
+			idsToLog := make(map[string]string)
+			idsToLog[targetRef] = containerInspector.ContainerID
+			for name, svc := range depServicesExe.RunningServices {
+				idsToLog[name] = svc.ID
+			}
+			for name, id := range idsToLog {
+				name := name
+				id := id
+				go func() {
+					err := client.Logs(dockerapi.LogsOptions{
+						Container:    id,
+						OutputStream: NewLogWriter(name + "-stdout"),
+						ErrorStream:  NewLogWriter(name + "-stderr"),
+						Follow:       true,
+						Stdout:       true,
+						Stderr:       true,
+					})
+					xc.FailOn(err)
+				}()
+			}
+
+			svc, ok := depServicesExe.RunningServices[containerProbeComposeSvc]
+			if !ok {
+				xc.Out.State("error", ovars{"message": "container-prove-compose-svc not found in running services"})
+				xc.Exit(1)
+			}
+			for {
+				c, err := client.InspectContainerWithOptions(dockerapi.InspectContainerOptions{
+					ID: svc.ID,
+				})
+				xc.FailOn(err)
+				if c.State.Running {
+					xc.Out.Info("wait for container.probe to finish")
+				} else {
+					if c.State.ExitCode != 0 {
+						xc.Out.State("exited", ovars{"container.probe exit.code": c.State.ExitCode})
+						xc.Exit(1)
+					}
+					break
+				}
+				time.Sleep(1 * time.Second)
+			}
 		case config.CAMEnter:
 			xc.Out.Prompt("USER INPUT REQUIRED, PRESS <ENTER> WHEN YOU ARE DONE USING THE CONTAINER")
 			creader := bufio.NewReader(os.Stdin)

pkg/app/master/commands/build/prompt.go

@@ -25,6 +25,7 @@ var CommandFlagSuggestions = &commands.FlagSuggestions{
 		{Text: commands.FullFlagName(commands.FlagComposeEnvFile), Description: commands.FlagComposeEnvFileUsage},
 		{Text: commands.FullFlagName(commands.FlagComposeProjectName), Description: commands.FlagComposeProjectNameUsage},
 		{Text: commands.FullFlagName(commands.FlagComposeWorkdir), Description: commands.FlagComposeWorkdirUsage},
+		{Text: commands.FullFlagName(commands.FlagContainerProbeComposeSvc), Description: commands.FlagContainerProbeComposeSvcUsage},
 		{Text: commands.FullFlagName(commands.FlagPull), Description: commands.FlagPullUsage},
 		{Text: commands.FullFlagName(commands.FlagShowPullLogs), Description: commands.FlagShowPullLogsUsage},
 		{Text: commands.FullFlagName(commands.FlagRegistryAccount), Description: commands.FlagRegistryAccountUsage},

pkg/app/master/commands/cliflags.go

@@ -73,6 +73,7 @@ const (
 	FlagPrestartComposeSvc             = "prestart-compose-svc"
 	FlagPoststartComposeSvc            = "poststart-compose-svc"
 	FlagPrestartComposeWaitExit        = "prestart-compose-wait-exit"
+	FlagContainerProbeComposeSvc       = "container-probe-compose-svc"
 
 	FlagRemoveFileArtifacts = "remove-file-artifacts"
 	FlagCopyMetaArtifacts   = "copy-meta-artifacts"
@@ -159,6 +160,7 @@ const (
 	FlagComposeEnvNoHostUsage               = "Don't include the env vars from the host to compose"
 	FlagComposeEnvFileUsage                 = "Load compose env vars from file (host env vars override the values loaded from this file)"
 	FlagComposeWorkdirUsage                 = "Set custom work directory for compose"
+	FlagContainerProbeComposeSvcUsage       = "Set container probe to compose service"
 	FlagComposeProjectNameUsage             = "Use custom project name for compose"
 	FlagPrestartComposeSvcUsage             = "Run selected compose service(s) before any other compose services or target container"
 	FlagPoststartComposeSvcUsage            = "Run selected compose service(s) after the target container is running (need a new continue after mode too)"
@@ -197,7 +199,7 @@ const (
 	FlagExcludePatternUsage  = "Exclude path pattern (Glob/Match in Go and **) from image"
 	FlagUseLocalMountsUsage  = "Mount local paths for target container artifact input and output"
 	FlagUseSensorVolumeUsage = "Sensor volume name to use"
-	FlagContinueAfterUsage   = "Select continue mode: enter | signal | probe | timeout or numberInSeconds"
+	FlagContinueAfterUsage   = "Select continue mode: enter | signal | probe | timeout-number-in-seconds | container.probe"
 
 	FlagExecUsage     = "A shell script snippet to run via Docker exec"
 	FlagExecFileUsage = "A shell script file to run via Docker exec"
@@ -407,6 +409,12 @@ var CommonFlags = map[string]cli.Flag{
 		Usage:  FlagComposeWorkdirUsage,
 		EnvVar: "DSLIM_COMPOSE_WORKDIR",
 	},
+	FlagContainerProbeComposeSvc: cli.StringFlag{
+		Name:   FlagContainerProbeComposeSvc,
+		Value:  "",
+		Usage:  FlagContainerProbeComposeSvcUsage,
+		EnvVar: "DSLIM_CONTAINER_PROBE_COMPOSE_SVC",
+	},
 	FlagPrestartComposeSvc: cli.StringSliceFlag{
 		Name:   FlagPrestartComposeSvc,
 		Value:  &cli.StringSlice{},

pkg/app/master/commands/cliprompt.go

@@ -240,6 +240,7 @@ var continueAfterValues = []prompt.Suggest{
 	{Text: config.CAMEnter, Description: "Use the <enter> key to indicate you that you are done using the container"},
 	{Text: config.CAMSignal, Description: "Use SIGUSR1 to signal that you are done using the container"},
 	{Text: config.CAMTimeout, Description: "Automatically continue after the default timeout (60 seconds)"},
+	{Text: config.CAMContainerProbe, Description: "Automatically continue after the probed container exits"},
 	{Text: "<seconds>", Description: "Enter the number of seconds to wait instead of <seconds>"},
 }
 

pkg/app/master/compose/execution.go

@@ -77,19 +77,20 @@ type ExecutionOptions struct {
 
 type Execution struct {
 	*ConfigInfo
-	State           ExecutionState
-	Selectors       *ServiceSelectors
-	BuildImages     bool
-	PullImages      bool
-	OwnAllResources bool
-	AllServiceNames map[string]struct{}
-	AllServices     map[string]*ServiceInfo
-	AllNetworks     map[string]*NetworkInfo
-	PendingServices map[string]struct{}
-	RunningServices map[string]*RunningService
-	ActiveVolumes   map[string]*ActiveVolume
-	ActiveNetworks  map[string]*ActiveNetwork
-	StopTimeout     uint
+	State             ExecutionState
+	Selectors         *ServiceSelectors
+	BuildImages       bool
+	PullImages        bool
+	OwnAllResources   bool
+	AllServiceNames   map[string]struct{}
+	AllServices       map[string]*ServiceInfo
+	AllNetworks       map[string]*NetworkInfo
+	PendingServices   map[string]struct{}
+	RunningServices   map[string]*RunningService
+	ActiveVolumes     map[string]*ActiveVolume
+	ActiveNetworks    map[string]*ActiveNetwork
+	StopTimeout       uint
+	ContainerProbeSvc string
 
 	options    *ExecutionOptions
 	eventCh    chan *ExecutionEvenInfo
@@ -278,6 +279,7 @@ func NewExecution(
 	workingDir string,
 	envVars []string,
 	environmentNoHost bool,
+	containerProbeComposeSvc string,
 	buildImages bool,
 	pullImages bool,
 	pullExcludes []string,
@@ -303,26 +305,27 @@ func NewExecution(
 
 	//not supporting compose profiles for now
 	exe := &Execution{
-		ConfigInfo:      configInfo,
-		State:           XSNone,
-		Selectors:       selectors,
-		OwnAllResources: ownAllResources,
-		BuildImages:     buildImages,
-		PullImages:      pullImages,
-		AllServiceNames: map[string]struct{}{},
-		AllServices:     map[string]*ServiceInfo{},
-		AllNetworks:     map[string]*NetworkInfo{},
-		PendingServices: map[string]struct{}{},
-		RunningServices: map[string]*RunningService{},
-		ActiveVolumes:   map[string]*ActiveVolume{},
-		ActiveNetworks:  map[string]*ActiveNetwork{},
-		StopTimeout:     defaultStopTimeout,
-		apiClient:       apiClient,
-		options:         options,
-		eventCh:         eventCh,
-		printState:      printState,
-		xc:              xc,
-		logger:          logger,
+		ConfigInfo:        configInfo,
+		State:             XSNone,
+		Selectors:         selectors,
+		OwnAllResources:   ownAllResources,
+		BuildImages:       buildImages,
+		PullImages:        pullImages,
+		AllServiceNames:   map[string]struct{}{},
+		AllServices:       map[string]*ServiceInfo{},
+		AllNetworks:       map[string]*NetworkInfo{},
+		PendingServices:   map[string]struct{}{},
+		RunningServices:   map[string]*RunningService{},
+		ActiveVolumes:     map[string]*ActiveVolume{},
+		ActiveNetworks:    map[string]*ActiveNetwork{},
+		ContainerProbeSvc: containerProbeComposeSvc,
+		StopTimeout:       defaultStopTimeout,
+		apiClient:         apiClient,
+		options:           options,
+		eventCh:           eventCh,
+		printState:        printState,
+		xc:                xc,
+		logger:            logger,
 	}
 
 	exe.initVersion()
@@ -475,6 +478,11 @@ func (ref *Execution) initServices() error {
 		}
 	}
 
+	if ref.ContainerProbeSvc != "" {
+		ref.AllServices[ref.ContainerProbeSvc].Selected = true
+		ref.Selectors.Includes[ref.ContainerProbeSvc] = struct{}{}
+	}
+
 	ref.logger.Debug("Execution.initServices: checking ref.AllServices[x].Selected")
 	for shortName, svc := range ref.AllServices {
 		ref.logger.Debugf("sname=%s/%s name=%s SELECTED?=%v\n",
@@ -767,7 +775,7 @@ func (ref *Execution) StopServices() error {
 
 	for key := range ref.RunningServices {
 		err := ref.StopService(key)
-		if err != nil {
+		if err != nil && key != ref.ContainerProbeSvc {
 			return err
 		}
 	}
@@ -779,7 +787,7 @@ func (ref *Execution) CleanupServices() error {
 
 	for key := range ref.RunningServices {
 		err := ref.CleanupService(key)
-		if err != nil {
+		if err != nil && key != ref.ContainerProbeSvc {
 			return err
 		}
 	}
@@ -1548,7 +1556,7 @@ func (ref *Execution) DeleteVolumes() error {
 		ref.logger.Debugf("DeleteVolumes: key/name=%s ID=%s", key, volume.ID)
 
 		err := deleteVolume(ref.apiClient, volume.ID)
-		if err != nil {
+		if err != nil && key != ref.ContainerProbeSvc {
 			return err
 		}
 
@@ -1702,7 +1710,7 @@ func (ref *Execution) DeleteNetworks() error {
 		}
 
 		err := deleteNetwork(ref.apiClient, network.ID)
-		if err != nil {
+		if err != nil && key != ref.ContainerProbeSvc {
 			return err
 		}