Only upload builder artifacts from workflow runs triggered after merge

.github/scripts/check-patterns.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Enable globstar for recursive globbing
+shopt -s globstar
+
+ALL_CHANGED_FILES="$1"
+IFS=' ' read -r -a changed <<< $ALL_CHANGED_FILES
+
+TOOL_CONFIG_PATTERNS=("docker/tool-config.Dockerfile .github/workflows/tool-config.yml docker/conf/**")
+BUILDER_2004_PATTERNS=("docker/ubuntu-2004-builder.Dockerfile .github/workflows/builder-ubuntu.yml")
+BUILDER_2204_PATTERNS=("docker/ubuntu-2204-builder.Dockerfile .github/workflows/builder-ubuntu.yml")
+CONTRIB_PATTERNS=("docker/psibase-contributor.Dockerfile .github/workflows/contributor.yml")
+
+matches_pattern() {
+    local pattern="$1"
+    for file in ${changed[@]}; do
+        if [[ $file == $pattern ]]; then
+            return 0 # Success
+        fi
+    done
+    return 1
+}
+
+
+run_tc="0"
+for pattern in ${TOOL_CONFIG_PATTERNS[@]}; do
+    if matches_pattern $pattern; then
+        run_tc="1"
+        break
+    fi
+done
+
+run_2004="0"
+for pattern in ${BUILDER_2004_PATTERNS[@]}; do
+    if matches_pattern $pattern; then
+        run_2004="1"
+        break
+    fi
+done
+
+run_2204="0"
+for pattern in ${BUILDER_2204_PATTERNS[@]}; do
+    if matches_pattern $pattern; then
+        run_2204="1"
+        break
+    fi
+done
+
+run_contrib="0"
+if [[ "$run_tc" == "1" ]] || [[ "$run_2204" == "1" ]]; then
+    run_contrib="1"
+else
+    for pattern in ${CONTRIB_PATTERNS[@]}; do
+        if matches_pattern $pattern; then
+            run_contrib="1"
+            break
+        fi
+    done
+fi
+
+echo "${run_tc} ${run_2004} ${run_2204} ${run_contrib}"

.github/scripts/latest-tag.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# references: 
+#   https://github.community/t/how-to-check-if-a-container-image-exists-on-ghcr/154836/3
+#   https://gist.github.com/eggplants/a046346571de66656f4d4d34de69fdd0
+
+USER_IMAGE="$1"
+
+# get token ('{"token":"***"}' -> '***')
+TOKEN="$(
+  curl "https://ghcr.io/token?scope=repository:${USER_IMAGE}:pull" |
+  awk -F'"' '$0=$4'
+)"
+_curl(){ curl -s -H "Authorization: Bearer ${TOKEN}" "$1" 2>&1; }
+
+# get most recent tag
+LAST_TAG=$(_curl "https://ghcr.io/v2/${USER_IMAGE}/tags/list" | jq -r '.tags[-1]')
+echo "${LAST_TAG}"

.github/workflows/_dispatcher.yml

@@ -0,0 +1,98 @@
+# This file is the dispatcher for the rest of the workflows that have complex dependencies.
+# The `cli.yml` doesn't need to be dispatched via this dispatcher because it is only run manually.
+
+name: Dispatcher
+
+env:
+  GITHUB_OUTPUT: ""
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docker/**"
+      - ".github/**"
+      - "!.github/workflows/cli.yml"
+      - "!docker/psibase.Dockerfile"
+      - "!docker/psinode.Dockerfile"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docker/**"
+      - ".github/**"
+      - "!.github/workflows/cli.yml"
+      - "!docker/psibase.Dockerfile"
+      - "!docker/psinode.Dockerfile"
+
+jobs:
+  determine-actions:
+    name: Set up the dispatching variables
+    runs-on: ubuntu-latest
+    outputs:
+      run_tc: ${{ steps.schedule-builders.outputs.run_tc }}
+      run_2004: ${{ steps.schedule-builders.outputs.run_2004 }}
+      run_2204: ${{ steps.schedule-builders.outputs.run_2204 }}
+      run_contrib: ${{ steps.schedule-builders.outputs.run_contrib }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: false
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v42
+
+      - name: (Debug) Print changed files
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          for file in $ALL_CHANGED_FILES; do
+            echo $file
+          done
+
+      - name: Determine what to dispatch
+        id: schedule-builders
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          necessary_jobs=$(./.github/scripts/check-patterns.sh "$ALL_CHANGED_FILES")  
+          read -r run_tc run_2004 run_2204 run_contrib <<< "$necessary_jobs"
+          echo "run_tc=${run_tc}" >> $GITHUB_OUTPUT
+          echo "run_2004=${run_2004}" >> $GITHUB_OUTPUT
+          echo "run_2204=${run_2204}" >> $GITHUB_OUTPUT
+          echo "run_contrib=${run_contrib}" >> $GITHUB_OUTPUT
+
+  build-tool-config:
+    name: "Build tool config"
+    needs: determine-actions
+    if: ${{ needs.determine-actions.outputs.run_tc == '1' }}
+    uses: ./.github/workflows/tool-config.yml
+
+  build-2004:
+    name: "Build 20.04 builder"
+    needs: determine-actions
+    if: ${{ needs.determine-actions.outputs.run_2004 == '1' }}
+    uses: ./.github/workflows/builder-ubuntu.yml
+    with:
+      ubuntu_version: "2004"
+
+  build-2204:
+    name: "Build 22.04 builder"
+    needs: determine-actions
+    if: ${{ needs.determine-actions.outputs.run_2204 == '1' }}
+    uses: ./.github/workflows/builder-ubuntu.yml
+    with:
+      ubuntu_version: "2204"
+
+  build-contributor:
+    name: "Build contributor"
+    needs: [determine-actions, build-tool-config, build-2204]
+    if: ${{ needs.determine-actions.outputs.run_contrib == '1' }}
+    uses: ./.github/workflows/contributor.yml
+    with:
+      new_tools: ${{ needs.build-tool-config.result == 'success'}} 
+      new_base: ${{ needs.build-2204.result == 'success'}} 

.github/workflows/builder-ubuntu.yml

@@ -0,0 +1,81 @@
+name: Generate ubuntu builder images
+# Generates an image that encapsulate an environment on which it is possible to build psibase.
+
+env:
+  GITHUB_OUTPUT: ""
+
+on:
+  workflow_call:
+    inputs:
+      ubuntu_version:
+        description: "On what version of ubuntu should the build run?"
+        type: string
+        required: true
+        default: "2204"
+
+jobs:
+  ubuntu-builder:
+    name: psibase-builder-ubuntu-${{ inputs.ubuntu_version }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: false
+          fetch-depth: 0
+
+      - name: Preparation
+        id: prep
+        run: |
+          OWNER="${{ github.repository_owner }}"
+          IMAGE="psibase-builder-ubuntu-${{ inputs.ubuntu_version }}"
+          REGISTRY="ghcr.io"
+          TAG="${{ github.sha }}"
+
+          TAGS="${REGISTRY}/${OWNER}/${IMAGE}:${TAG}"
+          echo "tags=${TAGS,,}" >> $GITHUB_OUTPUT
+
+      - name: Building ${{ steps.prep.outputs.tags }}
+        run: true
+
+      - name: Config docker buildx network
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-flags: --debug
+
+      - name: Login in to registry
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & Publish Image
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          file: docker/ubuntu-${{ inputs.ubuntu_version }}-builder.Dockerfile
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64
+          outputs: type=image,annotation-index.org.opencontainers.image.description=Psibase build environment based on Ubuntu ${{ inputs.ubuntu_version }}
+
+      - name: (PR Only) - Build image archive
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/ubuntu-${{ inputs.ubuntu_version }}-builder.Dockerfile
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64
+          outputs: type=docker,dest=builder-${{ inputs.ubuntu_version }}-image.tar
+
+      - name: (PR only) - Upload image archive as artifact
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: builder-${{ inputs.ubuntu_version }}-image
+          path: builder-${{ inputs.ubuntu_version }}-image.tar
+          retention-days: 1

.github/workflows/cli.yml

@@ -25,29 +25,35 @@ jobs:
         run: |
           REGISTRY="ghcr.io"
           IMAGE="${REGISTRY}/${{ github.repository_owner }}/psinode"
-          TAGS="${IMAGE}:${{ github.sha }},${IMAGE}:${{ github.event.inputs.version }}"
+          TAGS="${IMAGE}:${{ github.event.inputs.version }}"
           echo "tags=${TAGS,,}" >> $GITHUB_OUTPUT
-      - name: Showtag
-        id: showtag
-        run: echo ${{ steps.prep.outputs.tags }}
+      - name: Building ${{ steps.prep.outputs.tags }}
+        run: true
 
       - name: Docker Buildx setup
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           buildkitd-flags: --debug
 
       - name: Login in to registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set TOOL_CONFIG version
+        id: tool_cfg_img
+        run: |
+          latest_tag=$(./.github/scripts/latest-tag.sh "gofractally/http-tool-config")
+          echo "TOOL_CONFIG_IMAGE=ghcr.io/gofractally/http-tool-config:${latest_tag}" >> $GITHUB_OUTPUT
+
       - name: Build & Publish Image
         uses: docker/build-push-action@v4
         with:
           build-args: |
             RELEASE_TAG=${{ github.event.inputs.version }}
+            TOOL_CONFIG_IMAGE=${{ steps.tool_cfg_img.outputs.TOOL_CONFIG_IMAGE }}
           context: .
           push: true
           no-cache: true
@@ -69,19 +75,19 @@ jobs:
         run: |
           REGISTRY="ghcr.io"
           IMAGE="${REGISTRY}/${{ github.repository_owner }}/psibase"
-          TAGS="${IMAGE}:${{ github.sha }},${IMAGE}:${{ github.event.inputs.version }}"
+          TAGS="${IMAGE}:${{ github.event.inputs.version }}"
           echo "tags=${TAGS,,}" >> $GITHUB_OUTPUT
       - name: Showtag
         id: showtag
         run: echo ${{ steps.prep.outputs.tags }}
 
       - name: Docker Buildx setup
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           buildkitd-flags: --debug
 
       - name: Login in to registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}