[master] fix and refactor static packages (carry)

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ on:
   pull_request:
 
 jobs:
-  build:
+  pkg:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -29,3 +29,47 @@ jobs:
         name: Build
         run: |
           make ${{ matrix.target }}
+
+  static:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          # TODO(thaJeztah) re-enable once moby cross-compile also builds containerd
+          # and runc, or once we build containerd and runc using containerd-packaging
+          # - linux/arm/v6
+          # - linux/arm/v7
+          # - linux/arm64
+          - darwin/amd64
+          - darwin/arm64
+          - windows/amd64
+    steps:
+      -
+        name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build
+        run: |
+          make TARGETPLATFORM=${{ matrix.platform }} static
+      -
+        name: List files
+        run: |
+          tree -nh ./static/build
+      -
+        name: Upload static bundle
+        uses: actions/upload-artifact@v2
+        with:
+          name: static-${{ env.PLATFORM_PAIR }}
+          path: static/build/*.tar.gz
+          if-no-files-found: error
+          retention-days: 7

Jenkinsfile

@@ -1,7 +1,5 @@
 #!groovy
 
-def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME
-
 def pkgs = [
     [target: "centos-7",                 image: "centos:7",                               arches: ["amd64", "aarch64"]],          // (EOL: June 30, 2024)
     [target: "centos-8",                 image: "quay.io/centos/centos:stream8",          arches: ["amd64", "aarch64"]],
@@ -19,15 +17,16 @@ def pkgs = [
     [target: "ubuntu-jammy",             image: "ubuntu:jammy",                           arches: ["amd64", "aarch64", "armhf"]], // Ubuntu 22.04 LTS (End of support: April, 2027. EOL: April, 2032)
 ]
 
-def genBuildStep(LinkedHashMap pkg, String arch) {
+def statics = [
+    [os: "linux",       arches: ["amd64", "armv6", "armv7", "aarch64"]],
+    [os: "darwin",      arches: ["amd64", "aarch64"]],
+    [os: "windows",     arches: ["amd64"]],
+]
+
+def genPkgStep(LinkedHashMap pkg, String arch) {
     def nodeLabel = "linux&&${arch}"
-    def platform = ""
     def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME
-
     if (arch == 'armhf') {
-        // Running armhf builds on EC2 requires --platform parameter
-        // Otherwise it accidentally pulls armel images which then breaks the verify step
-        platform = "--platform=linux/${arch}"
         nodeLabel = "${nodeLabel}&&ubuntu"
     } else {
         nodeLabel = "${nodeLabel}&&ubuntu-2004"
@@ -42,6 +41,7 @@ def genBuildStep(LinkedHashMap pkg, String arch) {
             stage("info") {
                 sh 'docker version'
                 sh 'docker info'
+                sh 'env|sort'
             }
             stage("build") {
                 checkout scm
@@ -55,78 +55,59 @@ def genBuildStep(LinkedHashMap pkg, String arch) {
     }
 }
 
-def build_package_steps = [
-    'static-linux': { ->
-        wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) {
-            stage("static-linux") {
-                // This is just a "dummy" stage to make the distro/arch visible
-                // in Jenkins' BlueOcean view, which truncates names....
-                sh 'echo starting...'
-            }
-            stage("info") {
-                sh 'docker version'
-                sh 'docker info'
-            }
-            stage("build") {
-                try {
-                    checkout scm
-                    sh "make REF=$branch DOCKER_BUILD_PKGS='static-linux' static"
-                } finally {
-                    sh "make clean"
-                }
-            }
-        }
-    },
-    'cross-mac': { ->
-        wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) {
-            stage("cross-mac") {
+def genPkgSteps(opts) {
+    return opts.arches.collectEntries {
+        ["${opts.image}-${it}": genPkgStep(opts, it)]
+    }
+}
+
+def genStaticStep(LinkedHashMap pkg, String arch) {
+    def config = [
+        amd64:   [label: "x86_64",      targetarch: "amd64"],
+        aarch64: [label: "aarch64",     targetarch: "arm64"],
+        armv6:   [label: "aarch64",     targetarch: "arm/v6"],
+        armv7:   [label: "aarch64",     targetarch: "arm/v7"],
+        ppc64le: [label: "ppc64le",     targetarch: "ppc64le"],
+        s390x  : [label: "s390x",       targetarch: "s390x"],
+    ][arch]
+    def nodeLabel = "linux&&${config.label}"
+    if (config.label == 'x86_64') {
+        nodeLabel = "${nodeLabel}&&ubuntu"
+    }
+    def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME
+    return { ->
+        wrappedNode(label: nodeLabel, cleanWorkspace: true) {
+            stage("static-${pkg.os}-${arch}") {
                 // This is just a "dummy" stage to make the distro/arch visible
                 // in Jenkins' BlueOcean view, which truncates names....
                 sh 'echo starting...'
             }
             stage("info") {
                 sh 'docker version'
                 sh 'docker info'
+                sh 'env|sort'
             }
             stage("build") {
-                try {
-                    checkout scm
-                    sh "make REF=$branch DOCKER_BUILD_PKGS='cross-mac' static"
-                } finally {
-                    sh "make clean"
-                }
-            }
-        }
-    },
-    'cross-win': { ->
-        wrappedNode(label: 'ubuntu-2004 && x86_64', cleanWorkspace: true) {
-            stage("cross-win") {
-                // This is just a "dummy" stage to make the distro/arch visible
-                // in Jenkins' BlueOcean view, which truncates names....
-                sh 'echo starting...'
+                checkout scm
+                sh "make REF=$branch TARGETPLATFORM=${pkg.os}/${config.targetarch} static"
             }
-            stage("info") {
-                sh 'docker version'
-                sh 'docker info'
+            stage("upload artifacts") {
+                archiveArtifacts artifacts: 'static/build/bundles-*.tar.gz', onlyIfSuccessful: true, fingerprint: true
             }
-            stage("build") {
-                try {
-                    checkout scm
-                    sh "make REF=$branch DOCKER_BUILD_PKGS='cross-win' static"
-                } finally {
-                    sh "make clean"
-                }
+            stage("clean") {
+                sh "make clean"
             }
         }
-    },
-]
+    }
+}
 
-def genPackageSteps(opts) {
+def genStaticSteps(opts) {
     return opts.arches.collectEntries {
-        ["${opts.image}-${it}": genBuildStep(opts, it)]
+        ["static-${opts.os}-${it}": genStaticStep(opts, it)]
     }
 }
 
-build_package_steps << pkgs.collectEntries { genPackageSteps(it) }
+def parallelStages = pkgs.collectEntries { genPkgSteps(it) }
+parallelStages << statics.collectEntries { genStaticSteps(it) }
 
-parallel(build_package_steps)
+parallel(parallelStages)

Makefile

@@ -1,7 +1,5 @@
 include common.mk
 
-STATIC_VERSION=$(shell static/gen-static-ver $(realpath $(CURDIR)/src/github.com/docker/docker) $(VERSION))
-
 # Taken from: https://www.cmcrossroads.com/article/printing-value-makefile-variable
 print-%  : ; @echo $($*)
 
@@ -11,63 +9,75 @@ help: ## show make targets
 
 .PHONY: clean-src
 clean-src:
-	[ ! -d src ] || $(CHOWN) -R $(shell id -u):$(shell id -g) src
-	$(RM) -r src
+	@[ ! -d src ] || $(CHOWN) -R $(shell id -u):$(shell id -g) src
+	@$(RM) -r src
 
 .PHONY: src
 src: src/github.com/docker/cli src/github.com/docker/docker src/github.com/docker/buildx src/github.com/docker/compose src/github.com/docker/scan-cli-plugin ## clone source
 
 ifdef CLI_DIR
 src/github.com/docker/cli:
+	$(call title,Copying $(CLI_DIR))
 	mkdir -p "$(@D)"
 	cp -r "$(CLI_DIR)" $@
 else
 src/github.com/docker/cli:
+	$(call title,Init $(DOCKER_CLI_REPO))
 	git init $@
 	git -C $@ remote add origin "$(DOCKER_CLI_REPO)"
 endif
 
 ifdef ENGINE_DIR
 src/github.com/docker/docker:
+	$(call title,Copying $(ENGINE_DIR))
 	mkdir -p "$(@D)"
 	cp -r "$(ENGINE_DIR)" $@
 else
 src/github.com/docker/docker:
+	$(call title,Init $(DOCKER_ENGINE_REPO))
 	git init $@
 	git -C $@ remote add origin "$(DOCKER_ENGINE_REPO)"
 endif
 
 src/github.com/docker/buildx:
+	$(call title,Init $(DOCKER_BUILDX_REPO))
 	git init $@
 	git -C $@ remote add origin "$(DOCKER_BUILDX_REPO)"
 
 src/github.com/docker/compose:
+	$(call title,Init $(DOCKER_COMPOSE_REPO))
 	git init $@
 	git -C $@ remote add origin "$(DOCKER_COMPOSE_REPO)"
 
 src/github.com/docker/scan-cli-plugin:
+	$(call title,Init $(DOCKER_SCAN_REPO))
 	git init $@
 	git -C $@ remote add origin "$(DOCKER_SCAN_REPO)"
 
 
 .PHONY: checkout-cli
 checkout-cli: src/github.com/docker/cli
+	$(call title,Checkout $(DOCKER_CLI_REPO)#$(DOCKER_CLI_REF))
 	./scripts/checkout.sh src/github.com/docker/cli "$(DOCKER_CLI_REF)"
 
 .PHONY: checkout-docker
 checkout-docker: src/github.com/docker/docker
+	$(call title,Checkout $(DOCKER_ENGINE_REPO)#$(DOCKER_ENGINE_REF))
 	./scripts/checkout.sh src/github.com/docker/docker "$(DOCKER_ENGINE_REF)"
 
 .PHONY: checkout-buildx
 checkout-buildx: src/github.com/docker/buildx
+	$(call title,Checkout $(DOCKER_BUILDX_REPO)#$(DOCKER_BUILDX_REF))
 	./scripts/checkout.sh src/github.com/docker/buildx "$(DOCKER_BUILDX_REF)"
 
 .PHONY: checkout-compose
 checkout-compose: src/github.com/docker/compose
+	$(call title,Checkout $(DOCKER_COMPOSE_REPO)#$(DOCKER_COMPOSE_REF))
 	./scripts/checkout.sh src/github.com/docker/compose "$(DOCKER_COMPOSE_REF)"
 
 .PHONY: checkout-scan-cli-plugin
 checkout-scan-cli-plugin: src/github.com/docker/scan-cli-plugin
+	$(call title,Checkout $(DOCKER_SCAN_REPO)#$(DOCKER_SCAN_REF))
 	./scripts/checkout.sh src/github.com/docker/scan-cli-plugin "$(DOCKER_SCAN_REF)"
 
 .PHONY: checkout
@@ -81,22 +91,20 @@ clean: clean-src ## remove build artifacts
 
 .PHONY: deb rpm
 deb rpm: checkout ## build rpm/deb packages
-	$(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@
+	$(MAKE) -C $@ $@
 
 .PHONY: centos-% fedora-% rhel-%
 centos-% fedora-% rhel-%: checkout ## build rpm packages for the specified distro
-	$(MAKE) -C rpm VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@
+	$(MAKE) -C rpm $@
 
 .PHONY: debian-% raspbian-% ubuntu-%
 debian-% raspbian-% ubuntu-%: checkout ## build deb packages for the specified distro
-	$(MAKE) -C deb VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) $@
+	$(MAKE) -C deb $@
+
 
 .PHONY: static
-static: DOCKER_BUILD_PKGS:=static-linux cross-mac cross-win cross-arm
-static: checkout ## build static-compiled packages
-	for p in $(DOCKER_BUILD_PKGS); do \
-		$(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) TARGETPLATFORM=$(TARGETPLATFORM) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \
-	done
+static: checkout ## build static package
+	$(MAKE) -C static build
 
 .PHONY: verify
 verify: ## verify installation of packages

common.mk

@@ -50,7 +50,40 @@ VERIFY_PACKAGE_REPO ?= staging
 # Optional flags like --platform=linux/armhf
 VERIFY_PLATFORM ?=
 
+# Export vars as envs
 export BUILDTIME
 export DEFAULT_PRODUCT_LICENSE
 export PACKAGER_NAME
 export PLATFORM
+export VERSION
+export GO_VERSION
+
+export DOCKER_CLI_REPO
+export DOCKER_ENGINE_REPO
+export DOCKER_SCAN_REPO
+export DOCKER_COMPOSE_REPO
+export DOCKER_BUILDX_REPO
+
+export REF
+export DOCKER_CLI_REF
+export DOCKER_ENGINE_REF
+export DOCKER_SCAN_REF
+export DOCKER_COMPOSE_REF
+export DOCKER_BUILDX_REF
+
+# utilities
+BOLD := $(shell tput -T linux bold)
+RED := $(shell tput -T linux setaf 1)
+GREEN := $(shell tput -T linux setaf 2)
+YELLOW := $(shell tput -T linux setaf 3)
+BLUE := $(shell tput -T linux setaf 4)
+PURPLE := $(shell tput -T linux setaf 5)
+CYAN := $(shell tput -T linux setaf 6)
+
+RESET := $(shell tput -T linux sgr0)
+TITLE := $(BOLD)$(YELLOW)
+SUCCESS := $(BOLD)$(GREEN)
+
+define title
+    @printf '$(TITLE)$(1)$(RESET)\n'
+endef

scripts/checkout.sh

@@ -15,7 +15,7 @@
 #   limitations under the License.
 
 checkout() (
-	set -ex
+	set -e
 	SRC="$1"
 	REF="$2"
 	REF_FETCH="$REF"
@@ -27,11 +27,13 @@ checkout() (
 	else
 		REF="FETCH_HEAD"
 	fi
-	git -C "$SRC" fetch --update-head-ok --depth 1 origin "$REF_FETCH"
-	git -C "$SRC" checkout -q "$REF"
+	(
+		set -x
+		git -C "$SRC" fetch --update-head-ok --depth 1 origin "$REF_FETCH"
+		git -C "$SRC" checkout -q "$REF"
+	)
 )
 
-
 # Only execute checkout function above if this file is executed, not sourced from another script
 prog=checkout.sh # needs to be in sync with this file's name
 if [ "$(basename -- $0)" = "$prog" ]; then