diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e81047f7..4229a999d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: pull_request: jobs: - build: + pkg: runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -29,3 +29,47 @@ jobs: name: Build run: | make ${{ matrix.target }} + + static: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + # TODO(thaJeztah) re-enable once moby cross-compile also builds containerd + # and runc, or once we build containerd and runc using containerd-packaging + # - linux/arm/v6 + # - linux/arm/v7 + # - linux/arm64 + - darwin/amd64 + - darwin/arm64 + - windows/amd64 + steps: + - + name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Build + run: | + make TARGETPLATFORM=${{ matrix.platform }} static + - + name: List files + run: | + tree -nh ./static/build + - + name: Upload static bundle + uses: actions/upload-artifact@v2 + with: + name: static-${{ env.PLATFORM_PAIR }} + path: static/build/*.tar.gz + if-no-files-found: error + retention-days: 7 diff --git a/Jenkinsfile b/Jenkinsfile index e96a994427..0e5522e7a8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,5 @@ #!groovy -def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME - def pkgs = [ [target: "centos-7", image: "centos:7", arches: ["amd64", "aarch64"]], // (EOL: June 30, 2024) [target: "centos-8", image: "quay.io/centos/centos:stream8", arches: ["amd64", "aarch64"]], @@ -19,15 +17,16 @@ def pkgs = [ [target: "ubuntu-jammy", image: "ubuntu:jammy", arches: ["amd64", "aarch64", "armhf"]], // Ubuntu 22.04 LTS (End of support: April, 2027. EOL: April, 2032) ] -def genBuildStep(LinkedHashMap pkg, String arch) { +def statics = [ + [os: "linux", arches: ["amd64", "armv6", "armv7", "aarch64"]], + [os: "darwin", arches: ["amd64", "aarch64"]], + [os: "windows", arches: ["amd64"]], +] + +def genPkgStep(LinkedHashMap pkg, String arch) { def nodeLabel = "linux&&${arch}" - def platform = "" def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME - if (arch == 'armhf') { - // Running armhf builds on EC2 requires --platform parameter - // Otherwise it accidentally pulls armel images which then breaks the verify step - platform = "--platform=linux/${arch}" nodeLabel = "${nodeLabel}&&ubuntu" } else { nodeLabel = "${nodeLabel}&&ubuntu-2004" @@ -42,6 +41,7 @@ def genBuildStep(LinkedHashMap pkg, String arch) { stage("info") { sh 'docker version' sh 'docker info' + sh 'env|sort' } stage("build") { checkout scm @@ -55,31 +55,29 @@ def genBuildStep(LinkedHashMap pkg, String arch) { } } -def build_package_steps = [ - 'static-linux': { -> - wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) { - stage("static-linux") { - // This is just a "dummy" stage to make the distro/arch visible - // in Jenkins' BlueOcean view, which truncates names.... - sh 'echo starting...' - } - stage("info") { - sh 'docker version' - sh 'docker info' - } - stage("build") { - try { - checkout scm - sh "make REF=$branch DOCKER_BUILD_PKGS='static-linux' static" - } finally { - sh "make clean" - } - } - } - }, - 'cross-mac': { -> - wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) { - stage("cross-mac") { +def genPkgSteps(opts) { + return opts.arches.collectEntries { + ["${opts.image}-${it}": genPkgStep(opts, it)] + } +} + +def genStaticStep(LinkedHashMap pkg, String arch) { + def config = [ + amd64: [label: "x86_64", targetarch: "amd64"], + aarch64: [label: "aarch64", targetarch: "arm64"], + armv6: [label: "aarch64", targetarch: "arm/v6"], + armv7: [label: "aarch64", targetarch: "arm/v7"], + ppc64le: [label: "ppc64le", targetarch: "ppc64le"], + s390x : [label: "s390x", targetarch: "s390x"], + ][arch] + def nodeLabel = "linux&&${config.label}" + if (config.label == 'x86_64') { + nodeLabel = "${nodeLabel}&&ubuntu" + } + def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME + return { -> + wrappedNode(label: nodeLabel, cleanWorkspace: true) { + stage("static-${pkg.os}-${arch}") { // This is just a "dummy" stage to make the distro/arch visible // in Jenkins' BlueOcean view, which truncates names.... sh 'echo starting...' @@ -87,46 +85,29 @@ def build_package_steps = [ stage("info") { sh 'docker version' sh 'docker info' + sh 'env|sort' } stage("build") { - try { - checkout scm - sh "make REF=$branch DOCKER_BUILD_PKGS='cross-mac' static" - } finally { - sh "make clean" - } - } - } - }, - 'cross-win': { -> - wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) { - stage("cross-win") { - // This is just a "dummy" stage to make the distro/arch visible - // in Jenkins' BlueOcean view, which truncates names.... - sh 'echo starting...' + checkout scm + sh "make REF=$branch TARGETPLATFORM=${pkg.os}/${config.targetarch} static" } - stage("info") { - sh 'docker version' - sh 'docker info' + stage("upload artifacts") { + archiveArtifacts artifacts: 'static/build/bundles-*.tar.gz', onlyIfSuccessful: true, fingerprint: true } - stage("build") { - try { - checkout scm - sh "make REF=$branch DOCKER_BUILD_PKGS='cross-win' static" - } finally { - sh "make clean" - } + stage("clean") { + sh "make clean" } } - }, -] + } +} -def genPackageSteps(opts) { +def genStaticSteps(opts) { return opts.arches.collectEntries { - ["${opts.image}-${it}": genBuildStep(opts, it)] + ["static-${opts.os}-${it}": genStaticStep(opts, it)] } } -build_package_steps << pkgs.collectEntries { genPackageSteps(it) } +def parallelStages = pkgs.collectEntries { genPkgSteps(it) } +parallelStages << statics.collectEntries { genStaticSteps(it) } -parallel(build_package_steps) +parallel(parallelStages) diff --git a/Makefile b/Makefile index 80cdae1386..0fa231f5e7 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,5 @@ include common.mk -STATIC_VERSION=$(shell static/gen-static-ver $(realpath $(CURDIR)/src/github.com/docker/docker) $(VERSION)) - # Taken from: https://www.cmcrossroads.com/article/printing-value-makefile-variable print-% : ; @echo $($*) @@ -11,63 +9,75 @@ help: ## show make targets .PHONY: clean-src clean-src: - [ ! -d src ] || $(CHOWN) -R $(shell id -u):$(shell id -g) src - $(RM) -r src + @[ ! -d src ] || $(CHOWN) -R $(shell id -u):$(shell id -g) src + @$(RM) -r src .PHONY: src src: src/github.com/docker/cli src/github.com/docker/docker src/github.com/docker/buildx src/github.com/docker/compose src/github.com/docker/scan-cli-plugin ## clone source ifdef CLI_DIR src/github.com/docker/cli: + $(call title,Copying $(CLI_DIR)) mkdir -p "$(@D)" cp -r "$(CLI_DIR)" $@ else src/github.com/docker/cli: + $(call title,Init $(DOCKER_CLI_REPO)) git init $@ git -C $@ remote add origin "$(DOCKER_CLI_REPO)" endif ifdef ENGINE_DIR src/github.com/docker/docker: + $(call title,Copying $(ENGINE_DIR)) mkdir -p "$(@D)" cp -r "$(ENGINE_DIR)" $@ else src/github.com/docker/docker: + $(call title,Init $(DOCKER_ENGINE_REPO)) git init $@ git -C $@ remote add origin "$(DOCKER_ENGINE_REPO)" endif src/github.com/docker/buildx: + $(call title,Init $(DOCKER_BUILDX_REPO)) git init $@ git -C $@ remote add origin "$(DOCKER_BUILDX_REPO)" src/github.com/docker/compose: + $(call title,Init $(DOCKER_COMPOSE_REPO)) git init $@ git -C $@ remote add origin "$(DOCKER_COMPOSE_REPO)" src/github.com/docker/scan-cli-plugin: + $(call title,Init $(DOCKER_SCAN_REPO)) git init $@ git -C $@ remote add origin "$(DOCKER_SCAN_REPO)" .PHONY: checkout-cli checkout-cli: src/github.com/docker/cli + $(call title,Checkout $(DOCKER_CLI_REPO)#$(DOCKER_CLI_REF)) ./scripts/checkout.sh src/github.com/docker/cli "$(DOCKER_CLI_REF)" .PHONY: checkout-docker checkout-docker: src/github.com/docker/docker + $(call title,Checkout $(DOCKER_ENGINE_REPO)#$(DOCKER_ENGINE_REF)) ./scripts/checkout.sh src/github.com/docker/docker "$(DOCKER_ENGINE_REF)" .PHONY: checkout-buildx checkout-buildx: src/github.com/docker/buildx + $(call title,Checkout $(DOCKER_BUILDX_REPO)#$(DOCKER_BUILDX_REF)) ./scripts/checkout.sh src/github.com/docker/buildx "$(DOCKER_BUILDX_REF)" .PHONY: checkout-compose checkout-compose: src/github.com/docker/compose + $(call title,Checkout $(DOCKER_COMPOSE_REPO)#$(DOCKER_COMPOSE_REF)) ./scripts/checkout.sh src/github.com/docker/compose "$(DOCKER_COMPOSE_REF)" .PHONY: checkout-scan-cli-plugin checkout-scan-cli-plugin: src/github.com/docker/scan-cli-plugin + $(call title,Checkout $(DOCKER_SCAN_REPO)#$(DOCKER_SCAN_REF)) ./scripts/checkout.sh src/github.com/docker/scan-cli-plugin "$(DOCKER_SCAN_REF)" .PHONY: checkout @@ -81,22 +91,20 @@ clean: clean-src ## remove build artifacts .PHONY: deb rpm deb rpm: checkout ## build rpm/deb packages - $(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@ + $(MAKE) -C $@ $@ .PHONY: centos-% fedora-% rhel-% centos-% fedora-% rhel-%: checkout ## build rpm packages for the specified distro - $(MAKE) -C rpm VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@ + $(MAKE) -C rpm $@ .PHONY: debian-% raspbian-% ubuntu-% debian-% raspbian-% ubuntu-%: checkout ## build deb packages for the specified distro - $(MAKE) -C deb VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@ + $(MAKE) -C deb $@ + .PHONY: static -static: DOCKER_BUILD_PKGS:=static-linux cross-mac cross-win cross-arm -static: checkout ## build static-compiled packages - for p in $(DOCKER_BUILD_PKGS); do \ - $(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) TARGETPLATFORM=$(TARGETPLATFORM) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \ - done +static: checkout ## build static package + $(MAKE) -C static build .PHONY: verify verify: ## verify installation of packages diff --git a/common.mk b/common.mk index fff44d2085..17f507d2d4 100644 --- a/common.mk +++ b/common.mk @@ -50,7 +50,40 @@ VERIFY_PACKAGE_REPO ?= staging # Optional flags like --platform=linux/armhf VERIFY_PLATFORM ?= +# Export vars as envs export BUILDTIME export DEFAULT_PRODUCT_LICENSE export PACKAGER_NAME export PLATFORM +export VERSION +export GO_VERSION + +export DOCKER_CLI_REPO +export DOCKER_ENGINE_REPO +export DOCKER_SCAN_REPO +export DOCKER_COMPOSE_REPO +export DOCKER_BUILDX_REPO + +export REF +export DOCKER_CLI_REF +export DOCKER_ENGINE_REF +export DOCKER_SCAN_REF +export DOCKER_COMPOSE_REF +export DOCKER_BUILDX_REF + +# utilities +BOLD := $(shell tput -T linux bold) +RED := $(shell tput -T linux setaf 1) +GREEN := $(shell tput -T linux setaf 2) +YELLOW := $(shell tput -T linux setaf 3) +BLUE := $(shell tput -T linux setaf 4) +PURPLE := $(shell tput -T linux setaf 5) +CYAN := $(shell tput -T linux setaf 6) + +RESET := $(shell tput -T linux sgr0) +TITLE := $(BOLD)$(YELLOW) +SUCCESS := $(BOLD)$(GREEN) + +define title + @printf '$(TITLE)$(1)$(RESET)\n' +endef diff --git a/scripts/checkout.sh b/scripts/checkout.sh index 6bc18479dd..c5a6ef7ae8 100755 --- a/scripts/checkout.sh +++ b/scripts/checkout.sh @@ -15,7 +15,7 @@ # limitations under the License. checkout() ( - set -ex + set -e SRC="$1" REF="$2" REF_FETCH="$REF" @@ -27,11 +27,13 @@ checkout() ( else REF="FETCH_HEAD" fi - git -C "$SRC" fetch --update-head-ok --depth 1 origin "$REF_FETCH" - git -C "$SRC" checkout -q "$REF" + ( + set -x + git -C "$SRC" fetch --update-head-ok --depth 1 origin "$REF_FETCH" + git -C "$SRC" checkout -q "$REF" + ) ) - # Only execute checkout function above if this file is executed, not sourced from another script prog=checkout.sh # needs to be in sync with this file's name if [ "$(basename -- $0)" = "$prog" ]; then diff --git a/scripts/target-platform b/scripts/target-platform new file mode 100644 index 0000000000..68a1afcf12 --- /dev/null +++ b/scripts/target-platform @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +: "${TARGETPLATFORM=}" +: "${TARGETOS=}" +: "${TARGETARCH=}" +: "${TARGETVARIANT=}" + +# get TARGETOS/TARGETARCH/TARGETVARIANT from TARGETPLATFORM +if [ -n "$TARGETPLATFORM" ]; then + os="$(cut -d"/" -f1 <<<"$TARGETPLATFORM")" + arch="$(cut -d"/" -f2 <<<"$TARGETPLATFORM")" + if [ -n "$os" ] && [ -n "$arch" ]; then + TARGETOS="$os" + TARGETARCH="$arch" + case "$arch" in + "arm") + TARGETVARIANT="$(cut -d"/" -f3 <<<"$TARGETPLATFORM")" + : "${TARGETVARIANT:=v7}" + ;; + esac + fi +fi + +# current arch/variant +CUROS="linux" +case "$(uname -m)" in + "x86_64" | "amd64") + CURARCH="amd64" + ;; + "i386") + CURARCH="386" + ;; + "aarch64" | "arm64") + CURARCH="arm64" + ;; + "armv8l") + CURARCH="arm" + CURVARIANT="v8" + ;; + "armv7l") + CURARCH="arm" + CURVARIANT="v7" + ;; + "armv6l") + CURARCH="arm" + CURVARIANT="v6" + ;; + "armv5l") + CURARCH="arm" + CURVARIANT="v5" + ;; + "riscv64") + CURARCH="riscv64" + ;; + "ppc64le") + CURARCH="ppc64le" + ;; + "s390x") + CURARCH="s390x" + ;; +esac +CURPLATFORM="$CUROS/$CURARCH" +if [ -n "$CURVARIANT" ]; then + CURPLATFORM="$CURPLATFORM/$CURVARIANT" +fi + +# use current arch if empty +if [ -z "$TARGETARCH" ]; then + TARGETOS="linux" + TARGETARCH="$CURARCH" + TARGETPLATFORM="$TARGETOS/$TARGETARCH" + if [ -n "$CURVARIANT" ]; then + TARGETVARIANT="$CURVARIANT" + TARGETPLATFORM="$TARGETPLATFORM/$TARGETVARIANT" + fi +fi + +# bundle arch +case "$TARGETARCH$TARGETVARIANT" in + "amd64") + BUNDLEARCH="x86_64" + ;; + "386") + BUNDLEARCH="i386" + ;; + "arm64") + BUNDLEARCH="aarch64" + ;; + "armv7") + BUNDLEARCH="armhf" + ;; + "armv6") + BUNDLEARCH="armel" + ;; + "riscv64") + BUNDLEARCH="riscv64" + ;; + "ppc64le") + BUNDLEARCH="ppc64le" + ;; + "s390x") + BUNDLEARCH="s390x" + ;; +esac diff --git a/static/Makefile b/static/Makefile index 4850d61263..02a51406af 100644 --- a/static/Makefile +++ b/static/Makefile @@ -1,29 +1,29 @@ include ../common.mk -CLI_DIR=$(realpath $(CURDIR)/../src/github.com/docker/cli) -ENGINE_DIR=$(realpath $(CURDIR)/../src/github.com/docker/docker) -BUILDX_DIR=$(realpath $(CURDIR)/../src/github.com/docker/buildx) +CLI_DIR = $(realpath $(CURDIR)/../src/github.com/docker/cli) +ENGINE_DIR = $(realpath $(CURDIR)/../src/github.com/docker/docker) +BUILDX_DIR = $(realpath $(CURDIR)/../src/github.com/docker/buildx) +COMPOSE_DIR = $(realpath $(CURDIR)/../src/github.com/docker/compose) +SCAN_DIR = $(realpath $(CURDIR)/../src/github.com/docker/scan-cli-plugin) + +CLI_VERSION = $(shell ./gen-static-ver $(CLI_DIR) $(VERSION)) +ENGINE_VERSION = $(shell ./gen-static-ver $(ENGINE_DIR) $(VERSION)) +CONTAINERD_PKG_VER = $(shell ./gen-containerd-ver $(ENGINE_DIR) $(CONTAINERD_VERSION)) -GEN_STATIC_VER=$(shell ./gen-static-ver $(CLI_DIR) $(VERSION)) HASH_CMD=docker run -v $(CURDIR):/sum -w /sum debian:jessie bash hash_files DIR_TO_HASH:=build/linux -DOCKER_CLI_GOLANG_IMG=golang:$(GO_VERSION) - -DOCKER_BUILD_OPTS= -ifneq ($(strip $(CONTAINERD_VERSION)),) -# Set custom build-args to override the containerd version to build for static -# packages. The Dockerfile for 20.10 and earlier used CONTAINERD_COMMIT, later -# versions use CONTAINERD_VERSION. We can remove CONTAINERD_VERSION once 20.10.x -# reaches EOL. -DOCKER_BUILD_OPTS +=--build-arg=CONTAINERD_VERSION=$(CONTAINERD_VERSION) -DOCKER_BUILD_OPTS +=--build-arg=CONTAINERD_COMMIT=$(CONTAINERD_VERSION) -endif +export CLI_DIR +export ENGINE_DIR +export BUILDX_DIR +export COMPOSE_DIR +export SCAN_DIR -ifneq ($(strip $(RUNC_VERSION)),) -# Set custom build-args to override the runc version to build for static packages. -DOCKER_BUILD_OPTS +=--build-arg=RUNC_VERSION=$(RUNC_VERSION) -endif +export CLI_VERSION +export ENGINE_VERSION +export CONTAINERD_VERSION +export CONTAINERD_PKG_VER +export RUNC_VERSION .PHONY: help help: ## show make targets @@ -31,85 +31,16 @@ help: ## show make targets .PHONY: clean clean: ## remove build artifacts - [ ! -d build ] || $(CHOWN) -R $(shell id -u):$(shell id -g) build - $(RM) -r build - -docker builder prune -f --filter until=24h - -.PHONY: static -static: static-linux cross-mac cross-win cross-arm ## create all static packages - -.PHONY: static-linux -static-linux: static-cli static-engine static-buildx-plugin ## create tgz - mkdir -p build/linux/docker - cp $(CLI_DIR)/build/docker build/linux/docker/ - for f in dockerd containerd ctr containerd-shim containerd-shim-runc-v2 docker-init docker-proxy runc; do \ - cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f build/linux/docker/$$f; \ - done - tar -C build/linux -c -z -f build/linux/docker-$(GEN_STATIC_VER).tgz docker + $(call title,Removing build artifacts) + @[ ! -d build ] || $(CHOWN) -R $(shell id -u):$(shell id -g) build + @$(RM) -r build - # extra binaries for running rootless - mkdir -p build/linux/docker-rootless-extras - for f in rootlesskit rootlesskit-docker-proxy dockerd-rootless.sh dockerd-rootless-setuptool.sh vpnkit; do \ - if [ -f $(ENGINE_DIR)/bundles/binary-daemon/$$f ]; then \ - cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f build/linux/docker-rootless-extras/$$f; \ - fi \ - done - tar -C build/linux -c -z -f build/linux/docker-rootless-extras-$(GEN_STATIC_VER).tgz docker-rootless-extras - - # buildx - tar -C $(BUILDX_DIR)/bin -c -z -f build/linux/docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).tgz docker-buildx +.PHONY: build +build: ## build static package + $(call title,Building static package for $(TARGETPLATFORM)) + ./build-static "$(CURDIR)" "$(TARGETPLATFORM)" .PHONY: hash_files -hash_files: - @echo "Hashing directory $(DIR_TO_HASH)" +hash_files: ## hash files + $(call title,Hashing directory $(DIR_TO_HASH)) $(HASH_CMD) "$(DIR_TO_HASH)" - -.PHONY: buildx -buildx: - docker buildx inspect | grep -q 'Driver: docker-container' || docker buildx create --use - -.PHONY: cross-mac -cross-mac: buildx - cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=darwin/amd64,darwin/arm64 binary - dest=$$PWD/build/mac; cd $(CLI_DIR)/build && for platform in *; do \ - arch=$$(echo $$platform | cut -d_ -f2); \ - mkdir -p $$dest/$$arch/docker; \ - cp $$platform/docker-darwin-* $$dest/$$arch/docker/docker && \ - tar -C $$dest/$$arch -c -z -f $$dest/$$arch/docker-$(GEN_STATIC_VER).tgz docker; \ - done - -.PHONY: cross-win -cross-win: cross-win-engine - cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=windows/amd64 binary - mkdir -p build/win/amd64/docker - cp $(CLI_DIR)/build/docker-windows-amd64.exe build/win/amd64/docker/docker.exe - cp $(ENGINE_DIR)/bundles/cross/windows/amd64-daemon/dockerd.exe build/win/amd64/docker/dockerd.exe - cp $(ENGINE_DIR)/bundles/cross/windows/amd64-daemon/docker-proxy.exe build/win/amd64/docker/docker-proxy.exe - docker run --rm -v $(CURDIR)/build/win/amd64:/v -w /v alpine sh -c 'apk update&&apk add zip&&zip -r docker-$(GEN_STATIC_VER).zip docker' - $(CHOWN) -R $(shell id -u):$(shell id -g) build - -.PHONY: cross-arm -cross-arm: cross-all-cli ## create tgz with linux armhf client only - mkdir -p build/arm/docker - cp $(CLI_DIR)/build/docker-linux-arm build/arm/docker/docker - tar -C build/arm -c -z -f build/arm/docker-$(GEN_STATIC_VER).tgz docker - -.PHONY: static-cli -static-cli: - cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=$(TARGETPLATFORM) --set binary.args.CGO_ENABLED=$(CGO_ENABLED) binary - -.PHONY: static-engine -static-engine: - $(MAKE) -C $(ENGINE_DIR) VERSION=$(GEN_STATIC_VER) DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" binary - -.PHONY: static-buildx-plugin -static-buildx-plugin: - cd $(BUILDX_DIR) && docker buildx bake --set binaries.platform=$(TARGETPLATFORM) binaries && mv ./bin/buildx ./bin/docker-buildx - -.PHONY: cross-all-cli -cross-all-cli: - $(MAKE) -C $(CLI_DIR) -f docker.Makefile VERSION=$(GEN_STATIC_VER) cross - -.PHONY: cross-win-engine -cross-win-engine: - $(MAKE) -C $(ENGINE_DIR) VERSION=$(GEN_STATIC_VER) DOCKER_CROSSPLATFORMS=windows/amd64 DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" cross diff --git a/static/README.md b/static/README.md new file mode 100644 index 0000000000..1f25007cd9 --- /dev/null +++ b/static/README.md @@ -0,0 +1,55 @@ +# Building your own Docker static package + +Static packages can be built from root directory with the following syntax + +```console +make TARGETPLATFORM=${TARGETOS}/${TARGETARCH}/${TARGETVARIANT} static +``` + +Format of `TARGETOS`, `TARGETARCH`, `TARGETVARIANT` is the same as the [platform ARGs in the global scope](https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope) +for a Dockerfile like `linux/arm/v7`. + +Artifacts will be located in `build` under the following directory structure: +`build/$os/$arch/$variant/` or `build/$os/$arch/` if there is no variant being +used. + +### Building from local source + +Specify the location of the source repositories for the engine and cli when +building packages + +* `ENGINE_DIR` -> Specifies the directory where the engine code is located, eg: `$GOPATH/src/github.com/docker/docker` +* `CLI_DIR` -> Specifies the directory where the cli code is located, eg: `$GOPATH/src/github.com/docker/cli` + +```shell +make ENGINE_DIR=/path/to/engine CLI_DIR=/path/to/cli TARGETPLATFORM=linux/amd64 static +``` + +## Supported platforms + +Here is a list of platforms that are currently supported: + +```console +make TARGETPLATFORM=linux/amd64 static +make TARGETPLATFORM=linux/arm/v6 static +make TARGETPLATFORM=linux/arm/v7 static +make TARGETPLATFORM=linux/arm64 static +make TARGETPLATFORM=darwin/amd64 static +make TARGETPLATFORM=darwin/arm64 static +make TARGETPLATFORM=windows/amd64 static +``` + +> note: `darwin` only packages the docker cli and plugins. + +But you can test building against whatever platform you want like: + +```console +make TARGETPLATFORM=linux/riscv64 static +make TARGETPLATFORM=linux/s390x static +``` + +Or the current one matching your host if not defined: + +```console +make static +``` diff --git a/static/build-static b/static/build-static new file mode 100755 index 0000000000..2c0846de9d --- /dev/null +++ b/static/build-static @@ -0,0 +1,399 @@ +#!/usr/bin/env bash +set -e + +for varName in CONTAINERD_VERSION DEFAULT_PRODUCT_LICENSE PACKAGER_NAME PLATFORM PRODUCT RUNC_VERSION; do + # Prevent empty environment variables from overriding build-arg defaults. + # Jenkins may set the variable, but as an empty string. + if [ -z "${!varName}" ]; then + unset "$varName" + fi +done + +source "../scripts/target-platform" + +CURDIR="$1" +TARGETPLATFORM="${2:-$CURPLATFORM}" + +if [ -z "$CURDIR" ] || [ -z "$TARGETPLATFORM" ]; then + # shellcheck disable=SC2016 + echo 'usage: ./build-static ${CURDIR} ${TARGETPLATFORM}' + exit 1 +fi + +source "../scripts/target-platform" + +build_cli() { + [ -d "${CLI_DIR:?}/build" ] && rm -r "${CLI_DIR:?}/build" + ( + set -x + cd "${CLI_DIR}" + docker buildx build \ + --build-arg BUILDKIT_MULTI_PLATFORM=true \ + --build-arg CGO_ENABLED="${cgo_enabled}" \ + --build-arg DEFAULT_PRODUCT_LICENSE \ + --build-arg PACKAGER_NAME \ + --build-arg PLATFORM \ + --build-arg PRODUCT \ + --build-arg VERSION="${CLI_VERSION}" \ + --output ./build \ + --platform "${TARGETPLATFORM}" \ + --target binary . + ) +} + +build_engine() { + [ -d "${ENGINE_DIR:?}/bundles" ] && rm -r "${ENGINE_DIR:?}/bundles" + ( + set -x + cd "${ENGINE_DIR}" + docker buildx build \ + --build-arg CGO_ENABLED="${cgo_enabled}" \ + --build-arg CONTAINERD_VERSION \ + --build-arg DEFAULT_PRODUCT_LICENSE \ + --build-arg PACKAGER_NAME \ + --build-arg PLATFORM \ + --build-arg PRODUCT \ + --build-arg RUNC_VERSION \ + --build-arg VERSION="${ENGINE_VERSION}" \ + --output ./bundles \ + --platform "${TARGETPLATFORM}" \ + --target binary . + mkdir -p ./bundles/"${TARGETPLATFORM}" + cp -r ./bundles/binary-daemon/* "./bundles/${TARGETPLATFORM}/" + ) +} + +build_engine_cross() { + [ -d "${ENGINE_DIR:?}/bundles" ] && rm -r "${ENGINE_DIR:?}/bundles" + ( + set -x + cd "${ENGINE_DIR}" + docker buildx build \ + --build-arg CGO_ENABLED="${cgo_enabled}" \ + --build-arg CONTAINERD_VERSION \ + --build-arg CROSS=true \ + --build-arg DEFAULT_PRODUCT_LICENSE \ + --build-arg DOCKER_CROSSPLATFORMS="${TARGETPLATFORM}" \ + --build-arg PACKAGER_NAME \ + --build-arg PLATFORM \ + --build-arg PRODUCT \ + --build-arg RUNC_VERSION \ + --build-arg VERSION="${ENGINE_VERSION}" \ + --output ./bundles \ + --target cross . + mkdir -p "./bundles/${TARGETPLATFORM}" + cp ./bundles/cross/"${TARGETPLATFORM}"-daemon/* "./bundles/${TARGETPLATFORM}/" + ) +} + +build_buildx() { + [ -d "${BUILDX_DIR:?}/bin" ] && rm -r "${BUILDX_DIR:?}/bin" + ( + set -x + cd "${BUILDX_DIR}" + docker buildx build \ + --platform "${TARGETPLATFORM}" \ + --build-arg BUILDKIT_MULTI_PLATFORM=true \ + --output "./bin" \ + --target binaries . + ) +} + +build_compose() { + [ -d "${COMPOSE_DIR:?}/bin" ] && rm -r "${COMPOSE_DIR:?}/bin" + ( + cd "${COMPOSE_DIR}" + set -x + # TODO: Add TARGETPLATFORM support on compose repo to build efficiently with buildx + make GIT_TAG="${DOCKER_COMPOSE_REF}" cross + ) +} + +build_scan() { + [ -d "${SCAN_DIR:?}/bin" ] && rm -r "${SCAN_DIR:?}/bin" + ( + cd "${SCAN_DIR}" + set -x + # TODO: Add TARGETPLATFORM support on scan-cli-plugin repo to build efficiently with --platform + make GIT_TAG_NAME="${DOCKER_SCAN_REF}" cross + ) +} + +CROSS=true +if [ "$TARGETOS" = "linux" ] && [ "$CURARCH$CURVARIANT" = "$TARGETARCH$TARGETVARIANT" ]; then + CROSS=false +fi + +echo "UNAME=$(uname -m)" +echo "TARGETPLATFORM=${TARGETPLATFORM}" +echo "CURPLATFORM=${CURPLATFORM}" +echo "CROSS=${CROSS}" + +cgo_enabled="" +if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then + cgo_enabled=0 +fi + +targetPair="${TARGETOS}_${TARGETARCH}" +if [ -n "$TARGETVARIANT" ]; then + targetPair="${targetPair}_${TARGETVARIANT}" +fi + +buildDir="${CURDIR}/build/${TARGETPLATFORM}" + +dockerCLIBuildDir="${buildDir}/docker-cli" +dockerBuildDir="${buildDir}/docker-engine" +containerdBuildDir="${buildDir}/containerd" +rootlessExtrasBuildDir="${buildDir}/docker-rootless-extras" +buildxBuildDir="${buildDir}/docker-buildx" +composeBuildDir="${buildDir}/docker-compose" +scanBuildDir="${buildDir}/docker-scan" + +# create docker-container builder +docker buildx inspect | grep -q 'Driver: docker-container' || docker buildx create --use + +case ${TARGETOS} in + linux) + build_cli + if [ "$CROSS" = "false" ]; then + build_engine + else + build_engine_cross + fi + build_buildx + build_compose + # TODO change once we support scan-plugin on other architectures + if [ "${TARGETARCH}" = "amd64" ]; then + build_scan + fi + ;; + darwin) + build_cli + build_buildx + build_compose + build_scan + ;; + windows) + build_cli + build_engine_cross + build_buildx + build_compose + # TODO change once we support scan-plugin on other architectures + if [ "${TARGETARCH}" = "amd64" ]; then + build_scan + fi + ;; +esac + +# cleanup +[ -d "${buildDir}" ] && rm -r "${buildDir}" + +# docker CLI +mkdir -p "${dockerCLIBuildDir}" +case ${TARGETOS} in + linux | darwin) + cp "${CLI_DIR}"/build/"${targetPair}"/docker-"${TARGETOS}"-* "${dockerCLIBuildDir}/docker" + ;; + windows) + cp "${CLI_DIR}"/build/"${targetPair}"/docker-"${TARGETOS}"-*.exe "${dockerCLIBuildDir}/docker.exe" + ;; +esac +# package docker CLI +case ${TARGETOS} in + linux | darwin) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-cli-${CLI_VERSION}.tgz" docker-cli + ) + ;; + windows) + ( + cd "${buildDir}" + set -x + zip -r "docker-cli-${CLI_VERSION}.zip" docker-cli + ) + ;; +esac + +# docker, containerd, and runc +mkdir -p "${dockerBuildDir}" +case ${TARGETOS} in + linux) + for f in dockerd docker-init docker-proxy; do + if [ -f "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" ]; then + cp -L "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" "${dockerBuildDir}/$f" + fi + done + # TODO containerd binaries should be built as part of containerd-packaging, not as part of docker/docker-ce-packaging + mkdir -p "${containerdBuildDir}" + for f in containerd ctr containerd-shim containerd-shim-runc-v2 runc; do + if [ -f "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" ]; then + cp -L "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" "${containerdBuildDir}/$f" + fi + done + ;; + windows) + cp "${ENGINE_DIR}"/bundles/"${TARGETPLATFORM}"/dockerd.exe "${dockerBuildDir}/dockerd.exe" + cp "${ENGINE_DIR}"/bundles/"${TARGETPLATFORM}"/docker-proxy.exe "${dockerBuildDir}/docker-proxy.exe" + ;; +esac +# package docker, containerd, and runc +case ${TARGETOS} in + darwin) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-engine-${ENGINE_VERSION}.tgz" docker-engine + ) + ;; + linux) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-engine-${ENGINE_VERSION}.tgz" docker-engine + if [ "$(ls -A "${containerdBuildDir}")" ]; then + # FIXME(thaJeztah) moby cross-compile currently doesn't include containerd binaries + tar -C "${buildDir}" -c -z -f "${buildDir}/containerd-${CONTAINERD_PKG_VER#v}.tgz" containerd + fi + ) + ;; + windows) + ( + cd "${buildDir}" + set -x + zip -r "docker-engine-${ENGINE_VERSION}.zip" docker-engine + ) + ;; +esac + +# rootless extras +case ${TARGETOS} in + linux) + for f in rootlesskit rootlesskit-docker-proxy dockerd-rootless.sh dockerd-rootless-setuptool.sh vpnkit; do + if [ -f "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" ]; then + mkdir -p "${rootlessExtrasBuildDir}" + cp -L "${ENGINE_DIR}/bundles/${TARGETPLATFORM}/$f" "${rootlessExtrasBuildDir}/$f" + fi + done + ;; +esac +# package rootless extras +if [ -d "${rootlessExtrasBuildDir}" ]; then + case ${TARGETOS} in + linux) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-rootless-extras-${ENGINE_VERSION}.tgz" docker-rootless-extras + ) + ;; + esac +fi + +# buildx +if [ -d "${BUILDX_DIR}/bin" ]; then + mkdir -p "${buildxBuildDir}" + case ${TARGETOS} in + linux | darwin) + cp "${BUILDX_DIR}/bin/${targetPair}/buildx" "${buildxBuildDir}/docker-buildx" + ;; + windows) + cp "${BUILDX_DIR}/bin/${targetPair}/buildx.exe" "${buildxBuildDir}/docker-buildx.exe" + ;; + esac + # package buildx + case ${TARGETOS} in + linux | darwin) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-buildx-plugin-${DOCKER_BUILDX_REF#v}.tgz" docker-buildx + ) + ;; + windows) + ( + cd "${buildDir}" + set -x + zip -r "docker-buildx-plugin-${DOCKER_BUILDX_REF#v}.zip" docker-buildx + ) + ;; + esac +fi + +# compose +if [ -d "${COMPOSE_DIR}/bin" ]; then + mkdir -p "${composeBuildDir}" + composeTargetPair="${TARGETOS}" + case ${TARGETARCH} in + amd64) + composeTargetPair="${composeTargetPair}-x86_64" + ;; + arm64) + composeTargetPair="${composeTargetPair}-aarch64" + ;; + *) + composeTargetPair="${composeTargetPair}-${TARGETARCH}" + ;; + esac + if [ -n "$TARGETVARIANT" ]; then + composeTargetPair="${composeTargetPair}${TARGETVARIANT}" + fi + case ${TARGETOS} in + linux | darwin) + cp "${COMPOSE_DIR}/bin/docker-compose-${composeTargetPair}" "${composeBuildDir}/docker-compose" + ;; + windows) + cp "${COMPOSE_DIR}/bin/docker-compose-${composeTargetPair}.exe" "${composeBuildDir}/docker-compose.exe" + ;; + esac + # package compose + case ${TARGETOS} in + linux | darwin) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-compose-plugin-${DOCKER_COMPOSE_REF#v}.tgz" docker-compose + ) + ;; + windows) + ( + cd "${buildDir}" + set -x + zip -r "docker-compose-plugin-${DOCKER_COMPOSE_REF#v}.zip" docker-compose + ) + ;; + esac +fi + +# scan +if [ -d "${SCAN_DIR}/dist" ]; then + mkdir -p "${scanBuildDir}" + case ${TARGETOS} in + linux | darwin) + cp "${SCAN_DIR}/dist/docker-scan_${TARGETOS}_${TARGETARCH}" "${scanBuildDir}/docker-scan" + ;; + windows) + cp "${SCAN_DIR}/dist/docker-scan_${TARGETOS}_${TARGETARCH}.exe" "${scanBuildDir}/docker-scan.exe" + ;; + esac + # package compose + case ${TARGETOS} in + linux | darwin) + ( + set -x + tar -C "${buildDir}" -c -z -f "${buildDir}/docker-scan-plugin-${DOCKER_SCAN_REF#v}.tgz" docker-scan + ) + ;; + windows) + ( + cd "${buildDir}" + set -x + zip -r "docker-scan-plugin-${DOCKER_SCAN_REF#v}.zip" docker-scan + ) + ;; + esac +fi + +# create bundle +( + set -x + cd "${buildDir}" + rm -r */ + # bundle is expected to have a tar.gz extension, unlike the other archives, which use .tgz + tar -zvcf "${CURDIR}/build/bundles-ce-static-${TARGETOS}-${BUNDLEARCH}.tar.gz" . +) diff --git a/static/gen-containerd-ver b/static/gen-containerd-ver new file mode 100755 index 0000000000..c2f6273def --- /dev/null +++ b/static/gen-containerd-ver @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +engine_dir="$1" +# containerd_pkg_version is used to name the containerd-.tgz file. +containerd_pkg_version="$2" + +if [ -z "${containerd_pkg_version}" ]; then + # If no custom version is set for containerd, we select the default version of + # containerd based on the docker engine source. + # which is needed for naming the produced .tgz file. + # TODO containerd binaries should be built as part of containerd-packaging, not as part of docker/docker-ce-packaging + awk -F'=' '$1 == "ARG CONTAINERD_VERSION" {{sub("v","")}; print $2 }' "${engine_dir}/Dockerfile.windows" +else + echo "${containerd_pkg_version#v}" +fi diff --git a/static/hash_files b/static/hash_files index 6218d977a5..e2feda7911 100644 --- a/static/hash_files +++ b/static/hash_files @@ -6,6 +6,6 @@ DIR_TO_LOOK_IN=${1:-build/linux} for f in $(find "$DIR_TO_LOOK_IN" -type f); do for hash_algo in md5 sha256; do - "${hash_algo}sum" "$f" >"$f.$hash_algo" + "${hash_algo}sum" "$f" > "$f.$hash_algo" done done