Merge pull request #517 from mdelapenya/copy-dir-to-container

feat: copy directory to container

container.go

@@ -57,6 +57,7 @@ type Container interface {
 	Exec(ctx context.Context, cmd []string) (int, io.Reader, error)
 	ContainerIP(context.Context) (string, error) // get container ip
 	CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error
+	CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error
 	CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error
 	CopyFileFromContainer(ctx context.Context, filePath string) (io.ReadCloser, error)
 }

docker.go

@@ -481,7 +481,40 @@ func (c *DockerContainer) CopyFileFromContainer(ctx context.Context, filePath st
 	return ret, nil
 }
 
+// CopyDirToContainer copies the contents of a directory to a parent path in the container. This parent path must exist in the container first
+// as we cannot create it
+func (c *DockerContainer) CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error {
+	dir, err := isDir(hostDirPath)
+	if err != nil {
+		return err
+	}
+
+	if !dir {
+		// it's not a dir: let the consumer to handle an error
+		return fmt.Errorf("path %s is not a directory", hostDirPath)
+	}
+
+	buff, err := tarDir(hostDirPath, fileMode)
+	if err != nil {
+		return err
+	}
+
+	// create the directory under its parent
+	parent := filepath.Dir(containerParentPath)
+
+	return c.provider.client.CopyToContainer(ctx, c.ID, parent, buff, types.CopyToContainerOptions{})
+}
+
 func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error {
+	dir, err := isDir(hostFilePath)
+	if err != nil {
+		return err
+	}
+
+	if dir {
+		return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode)
+	}
+
 	fileContent, err := ioutil.ReadFile(hostFilePath)
 	if err != nil {
 		return err
@@ -491,20 +524,8 @@ func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath
 
 // CopyToContainer copies fileContent data to a file in container
 func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error {
-	buffer := &bytes.Buffer{}
-
-	tw := tar.NewWriter(buffer)
-	defer tw.Close()
-
-	hdr := &tar.Header{
-		Name: filepath.Base(containerFilePath),
-		Mode: fileMode,
-		Size: int64(len(fileContent)),
-	}
-	if err := tw.WriteHeader(hdr); err != nil {
-		return err
-	}
-	if _, err := tw.Write(fileContent); err != nil {
+	buffer, err := tarFile(fileContent, containerFilePath, fileMode)
+	if err != nil {
 		return err
 	}
 

docker_test.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
@@ -1855,6 +1856,33 @@ func TestDockerContainerCopyFileToContainer(t *testing.T) {
 	}
 }
 
+func TestDockerContainerCopyDirToContainer(t *testing.T) {
+	ctx := context.Background()
+
+	nginxC, err := GenericContainer(ctx, GenericContainerRequest{
+		ProviderType: providerType,
+		ContainerRequest: ContainerRequest{
+			Image:        nginxImage,
+			ExposedPorts: []string{nginxDefaultPort},
+			WaitingFor:   wait.ForListeningPort(nginxDefaultPort),
+		},
+		Started: true,
+	})
+
+	require.NoError(t, err)
+	terminateContainerOnEnd(t, ctx, nginxC)
+
+	err = nginxC.CopyDirToContainer(ctx, "./testresources/Dockerfile", "/tmp/testresources/Dockerfile", 700)
+	require.Error(t, err) // copying a file using the directory method will raise an error
+
+	err = nginxC.CopyDirToContainer(ctx, "./testresources", "/tmp/testresources", 700)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertExtractedFiles(t, ctx, nginxC, "./testresources", "/tmp/testresources/")
+}
+
 func TestDockerCreateContainerWithFiles(t *testing.T) {
 	ctx := context.Background()
 	hostFileName := "./testresources/hello.sh"
@@ -1902,7 +1930,9 @@ func TestDockerCreateContainerWithFiles(t *testing.T) {
 				Started: false,
 			})
 
-			if tc.errMsg == "" {
+			if err != nil {
+				require.Contains(t, err.Error(), tc.errMsg)
+			} else {
 				for _, f := range tc.files {
 					require.NoError(t, err)
 
@@ -1916,13 +1946,72 @@ func TestDockerCreateContainerWithFiles(t *testing.T) {
 					require.NoError(t, err)
 
 					require.Equal(t, hostFileData, containerFileData)
-
 				}
-			} else {
-				require.Error(t, err)
-				require.Equal(t, tc.errMsg, err.Error())
 			}
+		})
+	}
+}
+
+func TestDockerCreateContainerWithDirs(t *testing.T) {
+	ctx := context.Background()
+	hostDirName := "testresources"
 
+	tests := []struct {
+		name   string
+		dir    ContainerFile
+		errMsg string
+	}{
+		{
+			name: "success copy directory",
+			dir: ContainerFile{
+				HostFilePath:      "./" + hostDirName,
+				ContainerFilePath: "/tmp/" + hostDirName, // the parent dir must exist
+				FileMode:          700,
+			},
+		},
+		{
+			name: "host dir not found",
+			dir: ContainerFile{
+				HostFilePath:      "./testresources123",  // does not exist
+				ContainerFilePath: "/tmp/" + hostDirName, // the parent dir must exist
+				FileMode:          700,
+			},
+			errMsg: "can't copy " +
+				"./testresources123 to container: open " +
+				"./testresources123: no such file or directory: " +
+				"failed to create container",
+		},
+		{
+			name: "container dir not found",
+			dir: ContainerFile{
+				HostFilePath:      "./" + hostDirName,
+				ContainerFilePath: "/parent-does-not-exist/testresources123", // does not exist
+				FileMode:          700,
+			},
+			errMsg: "can't copy ./testresources to container: Error: No such container:path",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			nginxC, err := GenericContainer(ctx, GenericContainerRequest{
+				ContainerRequest: ContainerRequest{
+					Image:        "nginx:1.17.6",
+					ExposedPorts: []string{"80/tcp"},
+					WaitingFor:   wait.ForListeningPort("80/tcp"),
+					Files:        []ContainerFile{tc.dir},
+				},
+				Started: false,
+			})
+
+			if err != nil {
+				require.NotEmpty(t, tc.errMsg)
+				require.Contains(t, err.Error(), tc.errMsg)
+			} else {
+				dir := tc.dir
+
+				assertExtractedFiles(t, ctx, nginxC, dir.HostFilePath, dir.ContainerFilePath)
+			}
 		})
 	}
 }
@@ -2245,6 +2334,47 @@ func TestProviderHasConfig(t *testing.T) {
 	assert.NotNil(t, provider.Config(), "expecting DockerProvider to provide the configuration")
 }
 
+// creates a temporary dir in which the files will be extracted. Then it will compare the bytes of each file in the source with the bytes from the copied-from-container file
+func assertExtractedFiles(t *testing.T, ctx context.Context, container Container, hostFilePath string, containerFilePath string) {
+	// create all copied files into a temporary dir
+	tmpDir := filepath.Join(t.TempDir())
+
+	// compare the bytes of each file in the source with the bytes from the copied-from-container file
+	srcFiles, err := ioutil.ReadDir(hostFilePath)
+	require.NoError(t, err)
+
+	for _, srcFile := range srcFiles {
+		srcBytes, err := ioutil.ReadFile(filepath.Join(hostFilePath, srcFile.Name()))
+		if err != nil {
+			require.NoError(t, err)
+		}
+
+		// copy file by file, as there is a limitation in the Docker client to copy an entiry directory from the container
+		// paths for the container files are using Linux path separators
+		fd, err := container.CopyFileFromContainer(ctx, containerFilePath+"/"+srcFile.Name())
+		require.NoError(t, err, "Path not found in container: %s", containerFilePath+"/"+srcFile.Name())
+		defer fd.Close()
+
+		targetPath := filepath.Join(tmpDir, srcFile.Name())
+		dst, err := os.Create(targetPath)
+		if err != nil {
+			require.NoError(t, err)
+		}
+		defer dst.Close()
+
+		_, err = io.Copy(dst, fd)
+		if err != nil {
+			require.NoError(t, err)
+		}
+
+		untarBytes, err := ioutil.ReadFile(targetPath)
+		if err != nil {
+			require.NoError(t, err)
+		}
+		assert.Equal(t, srcBytes, untarBytes)
+	}
+}
+
 func terminateContainerOnEnd(tb testing.TB, ctx context.Context, ctr Container) {
 	tb.Helper()
 	if ctr == nil {

docs/features/copy_file.md

@@ -14,7 +14,7 @@ nginxC, err := GenericContainer(ctx, GenericContainerRequest{
 		Started: true,
 	})
 
-nginxC.CopyFileToContainer(ctx, "./testresources/hello.sh", "/hello_copy.sh", fileContent, 700)
+nginxC.CopyFileToContainer(ctx, "./testresources/hello.sh", "/hello_copy.sh", 700)
 ```
 
 Or you can add a list of files in ContainerRequest's struct, which can be copied before the container started:
@@ -39,3 +39,75 @@ nginxC, err := GenericContainer(ctx, GenericContainerRequest{
 	})
 ```
 
+## Copy Directories To Container
+
+It's also possible to copy an entire directory to a container, and that can happen before and/or after the container gets into the "Running" state. As an example, you could need to bulk-copy a set of files, such as a configuration directory that does not exist in the underlying Docker image.
+
+It's important to notice that, when copying the directory to the container, the container path must exist in the Docker image. And this is a strong requirement for files to be copied _before_ the container is started, as we cannot create the full path at that time.
+
+Once we understood that, there are two ways to copy directories to a container. The first one is using the existing `CopyFileToContainer` method, which will internally check if the host path is a directory, internally calling the new `CopyDirToContainer` method if needed:
+
+```go
+ctx := context.Background()
+
+// copy a directory before the container is started, using Files field
+nginxC, err := GenericContainer(ctx, GenericContainerRequest{
+		ContainerRequest: ContainerRequest{
+			Image:        "nginx:1.17.6",
+			ExposedPorts: []string{"80/tcp"},
+			WaitingFor:   wait.ForListeningPort("80/tcp"),
+			Files: []ContainerFile{
+				{
+					HostFilePath:      "./testresources",    // a directory
+					ContainerFilePath: "/tmp/testresources", // important! its parent already exists
+					FileMode:          700,
+				},
+			},
+		},
+		Started: true,
+	})
+if err != nil {
+	// handle error
+}
+
+// as the container is started, we can create the directory first
+_, _, err = nginxC.Exec(ctx, []string{"mkdir", "-p", "/usr/lib/my-software/config"})
+// because the container path is a directory, it will use the copy dir method as fallback
+err = nginxC.CopyFileToContainer(ctx, "./files", "/usr/lib/my-software/config/files", 700)
+if err != nil {
+	// handle error
+}
+```
+
+And the second way is using the `CopyDirToContainer` method which, as you probably know, needs the existence of the parent directory where to copy the directory:
+
+```go
+ctx := context.Background()
+
+// copy a directory before the container is started, using Files field
+nginxC, err := GenericContainer(ctx, GenericContainerRequest{
+		ContainerRequest: ContainerRequest{
+			Image:        "nginx:1.17.6",
+			ExposedPorts: []string{"80/tcp"},
+			WaitingFor:   wait.ForListeningPort("80/tcp"),
+			Files: []ContainerFile{
+				{
+					HostFilePath:      "./testresources",    // a directory
+					ContainerFilePath: "/tmp/testresources", // important! its parent already exists
+					FileMode:          700,
+				},
+			},
+		},
+		Started: true,
+	})
+if err != nil {
+	// handle error
+}
+
+// as the container is started, we can create the directory first
+_, _, err = nginxC.Exec(ctx, []string{"mkdir", "-p", "/usr/lib/my-software/config"})
+err = nginxC.CopyDirToContainer(ctx, "./plugins", "/usr/lib/my-software/config/plugins", 700)
+if err != nil {
+	// handle error
+}
+```