From 70153ad3861efc36a41a4ade76f8ad6fd7e753a8 Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 14 Oct 2021 15:20:53 +0100 Subject: [PATCH 01/11] Basic K3S module Fixes #4485 --- modules/k3s/build.gradle | 9 +++ .../org/testcontainers/k3s/K3sContainer.java | 66 +++++++++++++++++++ .../testcontainers/k3s/K3sContainerTest.java | 38 +++++++++++ .../k3s/src/test/resources/logback-test.xml | 16 +++++ 4 files changed, 129 insertions(+) create mode 100644 modules/k3s/build.gradle create mode 100644 modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java create mode 100644 modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java create mode 100644 modules/k3s/src/test/resources/logback-test.xml diff --git a/modules/k3s/build.gradle b/modules/k3s/build.gradle new file mode 100644 index 00000000000..259b880ab4a --- /dev/null +++ b/modules/k3s/build.gradle @@ -0,0 +1,9 @@ +description = "Testcontainers :: K3S" + +dependencies { + api project(":testcontainers") + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0' + + testImplementation 'io.fabric8:kubernetes-client:5.8.0' + testImplementation 'org.assertj:assertj-core:3.15.0' +} diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java new file mode 100644 index 00000000000..5845b2952bd --- /dev/null +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -0,0 +1,66 @@ +package org.testcontainers.k3s; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.github.dockerjava.api.command.InspectContainerResponse; +import lombok.SneakyThrows; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.utility.Base58; +import org.testcontainers.utility.DockerImageName; + +import java.util.HashMap; +import java.util.Map; + +public class K3sContainer extends GenericContainer { + + private String kubeConfigYaml; + + public K3sContainer(DockerImageName dockerImageName) { + super(dockerImageName); + dockerImageName.assertCompatibleWith(DockerImageName.parse("rancher/k3s")); + + addExposedPorts(6443, 8443); + setPrivilegedMode(true); + + Map tmpFsMapping = new HashMap<>(); + tmpFsMapping.put("/run", ""); + tmpFsMapping.put("/var/run", ""); + setTmpFsMapping(tmpFsMapping); + + final String randomToken = Base58.randomString(16); + + setCommand( + "server", + "--no-deploy=traefik", + "--token=" + randomToken, + "--tls-san=" + this.getHost() + ); + setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*Node controller sync successful.*")); + } + + @SneakyThrows + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + + JsonNode rawKubeConfig = copyFileFromContainer( + "/etc/rancher/k3s/k3s.yaml", + is -> objectMapper.readValue(is, JsonNode.class) + ); + + ObjectNode clusterConfig = (ObjectNode) rawKubeConfig.get("clusters").get(0).get("cluster"); + clusterConfig.replace("server", new TextNode("https://" + this.getHost() + ":" + this.getMappedPort(6443))); + + ((ObjectNode) rawKubeConfig).set("current-context", new TextNode("default")); + + kubeConfigYaml = objectMapper.writeValueAsString(rawKubeConfig); + } + + public String getKubeConfigYaml() { + return kubeConfigYaml; + } +} diff --git a/modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java b/modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java new file mode 100644 index 00000000000..959c3038cd0 --- /dev/null +++ b/modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java @@ -0,0 +1,38 @@ +package org.testcontainers.k3s; + +import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; + +import java.io.IOException; +import java.util.List; + +@Slf4j +public class K3sContainerTest { + + @Test + public void shouldStartAndHaveListableNode() throws IOException { + try ( + K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1")) + .withLogConsumer(new Slf4jLogConsumer(log)) + ) { + k3s.start(); + + String kubeConfigYaml = k3s.getKubeConfigYaml(); + + Config config = Config.fromKubeconfig(kubeConfigYaml); + // workaround for undiagnosed issue; fabric8 seems to not identify the client key algorithm correctly, + // and k3s uses ECDSA keys + config.setClientKeyAlgo("EC"); + + DefaultKubernetesClient client = new DefaultKubernetesClient(config); + List nodes = client.nodes().list().getItems(); + Assertions.assertThat(nodes).hasSize(1); + } + } +} diff --git a/modules/k3s/src/test/resources/logback-test.xml b/modules/k3s/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..535e406fc13 --- /dev/null +++ b/modules/k3s/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %d{HH:mm:ss.SSS} %-5level %logger - %msg%n + + + + + + + + + From 246c45290ded77e62a9f1200b7df376f3f03b945 Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 15 Oct 2021 17:11:51 +0100 Subject: [PATCH 02/11] Use shaded Jackson dependencies --- modules/k3s/build.gradle | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/k3s/build.gradle b/modules/k3s/build.gradle index 259b880ab4a..f0f28d1bad6 100644 --- a/modules/k3s/build.gradle +++ b/modules/k3s/build.gradle @@ -1,9 +1,11 @@ description = "Testcontainers :: K3S" dependencies { - api project(":testcontainers") - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0' + api project(":testcontainers") + + shaded 'com.fasterxml.jackson.core:jackson-databind:2.13.0' + shaded 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0' testImplementation 'io.fabric8:kubernetes-client:5.8.0' - testImplementation 'org.assertj:assertj-core:3.15.0' + testImplementation 'org.assertj:assertj-core:3.21.0' } From f55af61e01c1de8b37f1c51a6320134e67d83567 Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 18 Oct 2021 11:55:11 +0100 Subject: [PATCH 03/11] Add K3sContainer docs --- docs/modules/k3s.md | 49 +++++++++++++++++++ mkdocs.yml | 1 + modules/k3s/build.gradle | 1 + ...Test.java => Fabric8K3sContainerTest.java} | 14 ++++-- .../k3s/OfficialClientK3sContainerTest.java | 43 ++++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 docs/modules/k3s.md rename modules/k3s/src/test/java/org/testcontainers/k3s/{K3sContainerTest.java => Fabric8K3sContainerTest.java} (77%) create mode 100644 modules/k3s/src/test/java/org/testcontainers/k3s/OfficialClientK3sContainerTest.java diff --git a/docs/modules/k3s.md b/docs/modules/k3s.md new file mode 100644 index 00000000000..a5f62cfbe04 --- /dev/null +++ b/docs/modules/k3s.md @@ -0,0 +1,49 @@ +# K3s Module + +!!! note + This module is INCUBATING. While it is ready for use and operational in the current version of Testcontainers, it is possible that it may receive breaking changes in the future. See [our contributing guidelines](/contributing/#incubating-modules) for more information on our incubating modules policy. + +Testcontainers module for Rancher's [K3s](https://rancher.com/products/k3s/) lightweight Kubernetes. +This module is intended to be used for testing components that interact with Kubernetes APIs - for example, operators. + +## Usage example + +!!! warning + K3sContainer runs as a privileged container and needs to be able to spawn its own containers. For these reasons, + K3sContainer will not work in certain rootless Docker, Docker-in-Docker, or other environments where privileged + containers are disallowed. + +Start a K3s server as follows: + + +[Starting a K3S server](../../modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java) inside_block:starting_k3s + + +### Connecting to the server + +`K3sContainer` exposes a working Kubernetes client configuration, as a YAML String, via the `getKubeConfigYaml()` method. + +This may be used with Kubernetes clients - e.g. for the [official Java client](connecting_with_k8sio) and +[the Fabric8 Kubernetes client](https://github.com/fabric8io/kubernetes-client): + + +[Official Java client](../../modules/k3s/src/test/java/org/testcontainers/k3s/OfficialClientK3sContainerTest.java) inside_block:connecting_with_k8sio +[Fabric8 Kubernetes client](../../modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java) inside_block:connecting_with_fabric8 + + +## Adding this module to your project dependencies + +Add the following dependency to your `pom.xml`/`build.gradle` file: + +```groovy tab='Gradle' +testImplementation "org.testcontainers:k3s:{{latest_version}}" +``` + +```xml tab='Maven' + + org.testcontainers + k3s + {{latest_version}} + test + +``` diff --git a/mkdocs.yml b/mkdocs.yml index 5d182b79590..973ee0b25cb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,6 +65,7 @@ nav: - modules/docker_compose.md - modules/elasticsearch.md - modules/gcloud.md + - modules/k3s.md - modules/kafka.md - modules/localstack.md - modules/mockserver.md diff --git a/modules/k3s/build.gradle b/modules/k3s/build.gradle index f0f28d1bad6..1ee668b0393 100644 --- a/modules/k3s/build.gradle +++ b/modules/k3s/build.gradle @@ -7,5 +7,6 @@ dependencies { shaded 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0' testImplementation 'io.fabric8:kubernetes-client:5.8.0' + testImplementation 'io.kubernetes:client-java:13.0.0' testImplementation 'org.assertj:assertj-core:3.21.0' } diff --git a/modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java similarity index 77% rename from modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java rename to modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java index 959c3038cd0..ed5c6489c0b 100644 --- a/modules/k3s/src/test/java/org/testcontainers/k3s/K3sContainerTest.java +++ b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java @@ -13,25 +13,33 @@ import java.util.List; @Slf4j -public class K3sContainerTest { +public class Fabric8K3sContainerTest { @Test public void shouldStartAndHaveListableNode() throws IOException { try ( + // starting_k3s { K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1")) .withLogConsumer(new Slf4jLogConsumer(log)) + // } ) { k3s.start(); + // connecting_with_fabric8 { String kubeConfigYaml = k3s.getKubeConfigYaml(); Config config = Config.fromKubeconfig(kubeConfigYaml); - // workaround for undiagnosed issue; fabric8 seems to not identify the client key algorithm correctly, - // and k3s uses ECDSA keys + // workaround for undiagnosed issue; fabric8 seems to not identify + // the client key algorithm correctly, so fails to work with K3s + // ECDSA keys unless configured explicitly config.setClientKeyAlgo("EC"); DefaultKubernetesClient client = new DefaultKubernetesClient(config); + + // interact with the running K3s server, e.g.: List nodes = client.nodes().list().getItems(); + // } + Assertions.assertThat(nodes).hasSize(1); } } diff --git a/modules/k3s/src/test/java/org/testcontainers/k3s/OfficialClientK3sContainerTest.java b/modules/k3s/src/test/java/org/testcontainers/k3s/OfficialClientK3sContainerTest.java new file mode 100644 index 00000000000..970e2390457 --- /dev/null +++ b/modules/k3s/src/test/java/org/testcontainers/k3s/OfficialClientK3sContainerTest.java @@ -0,0 +1,43 @@ +package org.testcontainers.k3s; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1NodeList; +import io.kubernetes.client.util.Config; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; + +import java.io.IOException; +import java.io.StringReader; + +@Slf4j +public class OfficialClientK3sContainerTest { + + @Test + public void shouldStartAndHaveListableNode() throws IOException, ApiException { + try ( + // starting_k3s { + K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1")) + .withLogConsumer(new Slf4jLogConsumer(log)) + // } + ) { + k3s.start(); + + // connecting_with_k8sio { + String kubeConfigYaml = k3s.getKubeConfigYaml(); + + ApiClient client = Config.fromConfig(new StringReader(kubeConfigYaml)); + CoreV1Api api = new CoreV1Api(client); + + // interact with the running K3s server, e.g.: + V1NodeList nodes = api.listNode(null, null, null, null, null, null, null, null, null, null); + // } + + Assertions.assertThat(nodes.getItems()).hasSize(1); + } + } +} From 7e1fb03009e4dc25ca65a88f8973e630426b69cb Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 23 Dec 2021 08:38:40 +0000 Subject: [PATCH 04/11] Update based on latest available libraries --- modules/k3s/build.gradle | 8 ++++---- .../org/testcontainers/k3s/Fabric8K3sContainerTest.java | 9 ++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/modules/k3s/build.gradle b/modules/k3s/build.gradle index 1ee668b0393..df090ec2661 100644 --- a/modules/k3s/build.gradle +++ b/modules/k3s/build.gradle @@ -3,10 +3,10 @@ description = "Testcontainers :: K3S" dependencies { api project(":testcontainers") - shaded 'com.fasterxml.jackson.core:jackson-databind:2.13.0' - shaded 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.0' + shaded 'com.fasterxml.jackson.core:jackson-databind:2.13.1' + shaded 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.1' - testImplementation 'io.fabric8:kubernetes-client:5.8.0' - testImplementation 'io.kubernetes:client-java:13.0.0' + testImplementation 'io.fabric8:kubernetes-client:5.11.0' + testImplementation 'io.kubernetes:client-java:14.0.0' testImplementation 'org.assertj:assertj-core:3.21.0' } diff --git a/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java index ed5c6489c0b..570bc7c8247 100644 --- a/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java +++ b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java @@ -9,14 +9,13 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; -import java.io.IOException; import java.util.List; @Slf4j public class Fabric8K3sContainerTest { @Test - public void shouldStartAndHaveListableNode() throws IOException { + public void shouldStartAndHaveListableNode() { try ( // starting_k3s { K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1")) @@ -26,13 +25,9 @@ public void shouldStartAndHaveListableNode() throws IOException { k3s.start(); // connecting_with_fabric8 { + // obtain a kubeconfig file which allows us to connect to k3s String kubeConfigYaml = k3s.getKubeConfigYaml(); - Config config = Config.fromKubeconfig(kubeConfigYaml); - // workaround for undiagnosed issue; fabric8 seems to not identify - // the client key algorithm correctly, so fails to work with K3s - // ECDSA keys unless configured explicitly - config.setClientKeyAlgo("EC"); DefaultKubernetesClient client = new DefaultKubernetesClient(config); From 56c769e7cd144c27fd1be36df01920450f934701 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 28 Dec 2021 19:43:15 +0000 Subject: [PATCH 05/11] Update based on PR comments --- .../src/main/java/org/testcontainers/k3s/K3sContainer.java | 7 +++---- .../org/testcontainers/k3s/Fabric8K3sContainerTest.java | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java index 5845b2952bd..3f6f3e798dc 100644 --- a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -9,7 +9,6 @@ import lombok.SneakyThrows; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.utility.Base58; import org.testcontainers.utility.DockerImageName; import java.util.HashMap; @@ -19,6 +18,8 @@ public class K3sContainer extends GenericContainer { private String kubeConfigYaml; + private static final String TOKEN = "deadbeef"; + public K3sContainer(DockerImageName dockerImageName) { super(dockerImageName); dockerImageName.assertCompatibleWith(DockerImageName.parse("rancher/k3s")); @@ -31,12 +32,10 @@ public K3sContainer(DockerImageName dockerImageName) { tmpFsMapping.put("/var/run", ""); setTmpFsMapping(tmpFsMapping); - final String randomToken = Base58.randomString(16); - setCommand( "server", "--no-deploy=traefik", - "--token=" + randomToken, + "--token=" + TOKEN, "--tls-san=" + this.getHost() ); setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*Node controller sync successful.*")); diff --git a/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java index 570bc7c8247..8e7b601c5b0 100644 --- a/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java +++ b/modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java @@ -27,6 +27,8 @@ public void shouldStartAndHaveListableNode() { // connecting_with_fabric8 { // obtain a kubeconfig file which allows us to connect to k3s String kubeConfigYaml = k3s.getKubeConfigYaml(); + + // requires io.fabric8:kubernetes-client:5.11.0 or higher Config config = Config.fromKubeconfig(kubeConfigYaml); DefaultKubernetesClient client = new DefaultKubernetesClient(config); From 4ba4262950dabb1f4c53457c030029cdb2a2c7c6 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 28 Dec 2021 20:21:09 +0000 Subject: [PATCH 06/11] Change fixed token value --- .../k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java index 3f6f3e798dc..d1089994d1d 100644 --- a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -18,7 +18,7 @@ public class K3sContainer extends GenericContainer { private String kubeConfigYaml; - private static final String TOKEN = "deadbeef"; + private static final String TOKEN = "fixedtoken"; public K3sContainer(DockerImageName dockerImageName) { super(dockerImageName); From 25406af4793b0f7ba388b8af26f4b948df07f346 Mon Sep 17 00:00:00 2001 From: Richard North Date: Wed, 29 Dec 2021 15:39:24 +0000 Subject: [PATCH 07/11] Remove token --- .../k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java index d1089994d1d..fd351d61ea9 100644 --- a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -18,8 +18,6 @@ public class K3sContainer extends GenericContainer { private String kubeConfigYaml; - private static final String TOKEN = "fixedtoken"; - public K3sContainer(DockerImageName dockerImageName) { super(dockerImageName); dockerImageName.assertCompatibleWith(DockerImageName.parse("rancher/k3s")); @@ -35,7 +33,6 @@ public K3sContainer(DockerImageName dockerImageName) { setCommand( "server", "--no-deploy=traefik", - "--token=" + TOKEN, "--tls-san=" + this.getHost() ); setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*Node controller sync successful.*")); From 1fb190e42e73b68c0e5768b0d58de83a5eb0e20a Mon Sep 17 00:00:00 2001 From: Richard North Date: Wed, 29 Dec 2021 16:16:51 +0000 Subject: [PATCH 08/11] Remove direct jackson-databind dependency --- modules/k3s/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/k3s/build.gradle b/modules/k3s/build.gradle index df090ec2661..85ee36652b3 100644 --- a/modules/k3s/build.gradle +++ b/modules/k3s/build.gradle @@ -3,7 +3,6 @@ description = "Testcontainers :: K3S" dependencies { api project(":testcontainers") - shaded 'com.fasterxml.jackson.core:jackson-databind:2.13.1' shaded 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.1' testImplementation 'io.fabric8:kubernetes-client:5.11.0' From 0eb201c7ef6c7ab27628e61e203f4027efe65778 Mon Sep 17 00:00:00 2001 From: Richard North Date: Wed, 29 Dec 2021 16:25:24 +0000 Subject: [PATCH 09/11] Improve YAML handling --- .../main/java/org/testcontainers/k3s/K3sContainer.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java index fd351d61ea9..b78a33a224b 100644 --- a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -43,17 +43,17 @@ public K3sContainer(DockerImageName dockerImageName) { protected void containerIsStarted(InspectContainerResponse containerInfo) { ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); - JsonNode rawKubeConfig = copyFileFromContainer( + ObjectNode rawKubeConfig = copyFileFromContainer( "/etc/rancher/k3s/k3s.yaml", - is -> objectMapper.readValue(is, JsonNode.class) + is -> objectMapper.readValue(is, ObjectNode.class) ); - ObjectNode clusterConfig = (ObjectNode) rawKubeConfig.get("clusters").get(0).get("cluster"); + ObjectNode clusterConfig = rawKubeConfig.at("/clusters/0/cluster").require(); clusterConfig.replace("server", new TextNode("https://" + this.getHost() + ":" + this.getMappedPort(6443))); - ((ObjectNode) rawKubeConfig).set("current-context", new TextNode("default")); + rawKubeConfig.set("current-context", new TextNode("default")); - kubeConfigYaml = objectMapper.writeValueAsString(rawKubeConfig); + kubeConfigYaml = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rawKubeConfig); } public String getKubeConfigYaml() { From 83691820c88e8eb4e71dc1bef88b00f92c126b3f Mon Sep 17 00:00:00 2001 From: Richard North Date: Wed, 5 Jan 2022 11:56:09 +0000 Subject: [PATCH 10/11] Add note re btrfs problem --- docs/modules/k3s.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/modules/k3s.md b/docs/modules/k3s.md index a5f62cfbe04..a3c2c0fa754 100644 --- a/docs/modules/k3s.md +++ b/docs/modules/k3s.md @@ -8,11 +8,6 @@ This module is intended to be used for testing components that interact with Kub ## Usage example -!!! warning - K3sContainer runs as a privileged container and needs to be able to spawn its own containers. For these reasons, - K3sContainer will not work in certain rootless Docker, Docker-in-Docker, or other environments where privileged - containers are disallowed. - Start a K3s server as follows: @@ -31,6 +26,15 @@ This may be used with Kubernetes clients - e.g. for the [official Java client](c [Fabric8 Kubernetes client](../../modules/k3s/src/test/java/org/testcontainers/k3s/Fabric8K3sContainerTest.java) inside_block:connecting_with_fabric8 +## Known limitations + +!!! warning + * K3sContainer runs as a privileged container and needs to be able to spawn its own containers. For these reasons, + K3sContainer will not work in certain rootless Docker, Docker-in-Docker, or other environments where privileged + containers are disallowed. + + * k3s containers may be unable to run on host machines where `/var/lib/docker` is on a BTRFS filesystem. See [k3s-io/k3s#4863](https://github.com/k3s-io/k3s/issues/4863) for an example. + ## Adding this module to your project dependencies Add the following dependency to your `pom.xml`/`build.gradle` file: From 3e1b6ee0c6d2616c586cd8280d0317b87e6b38da Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 11 Jan 2022 17:07:37 +0000 Subject: [PATCH 11/11] Minimal adjustment to work with cgroupv2 on Docker host --- .../k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java index b78a33a224b..1080563226a 100644 --- a/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java +++ b/modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.github.dockerjava.api.command.InspectContainerResponse; import lombok.SneakyThrows; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerImageName; @@ -24,6 +25,7 @@ public K3sContainer(DockerImageName dockerImageName) { addExposedPorts(6443, 8443); setPrivilegedMode(true); + addFileSystemBind("/sys/fs/cgroup", "/sys/fs/cgroup", BindMode.READ_WRITE); Map tmpFsMapping = new HashMap<>(); tmpFsMapping.put("/run", "");