Atomic Build and Release #252
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Atomic Build and Release | ||
on: | ||
schedule: | ||
# Run every Monday at 12:00 | ||
- cron: 0 12 * * 1 | ||
workflow_dispatch: | ||
push: | ||
tags: | ||
- "*" | ||
env: | ||
PYTHON_VERSION: "3.11" | ||
MATURIN_VERSION: "1.4.0" | ||
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | ||
MATURIN_USERNAME: "seanglaredb" | ||
PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | ||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | ||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | ||
concurrency: | ||
group: release-${{ github.ref }} | ||
cancel-in-progress: true | ||
jobs: | ||
atomic-release: | ||
# This creates a node in the DAG so that all of the "push" tasks | ||
# can depend on all build tasks. | ||
name: Atomic Release | ||
runs-on: ubuntu-latest | ||
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | ||
needs: | ||
- python-linux | ||
- python-windows | ||
- python-macos | ||
- kernel-dist | ||
- node-build | ||
steps: | ||
- run: sleep 1 | ||
######################################################################## | ||
## | ||
## Kernel Release | ||
## | ||
######################################################################## | ||
kernel-dist: | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
- os: windows-latest-8-cores | ||
Check failure on line 51 in .github/workflows/release.yaml GitHub Actions / Atomic Build and ReleaseInvalid workflow file
|
||
target: x86_64-pc-windows-msvc | ||
- os: ubuntu-2004-8-cores | ||
target: x86_64-unknown-linux-gnu | ||
- os: macos-12-large | ||
target: x86_64-apple-darwin | ||
- os: macos-13-xlarge | ||
target: aarch64-apple-darwin | ||
name: Build Kernel Release (${{ matrix.target }}) | ||
runs-on: ${{ matrix.os }} | ||
container: ${{ matrix.container }} | ||
env: | ||
DIST_TARGET_TRIPLE: ${{ matrix.target }} | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: ilammy/setup-nasm@v1 | ||
if: matrix.target == 'x86_64-pc-windows-msvc' | ||
- uses: egor-tensin/setup-gcc@v1 | ||
if: matrix.target == 'x86_64-unknown-linux-gnu' | ||
with: | ||
version: 11 | ||
- uses: dtolnay/rust-toolchain@1.75.0 | ||
with: | ||
targets: ${{ matrix.target }} | ||
- uses: extractions/setup-just@v2 | ||
with: | ||
just-version: "1.23.0" | ||
- run: just dist | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: dist-${{ matrix.target }} | ||
path: ./target/dist/ | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Build Kernel Binary", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "target", "value": "${{ matrix.target }}", "short": true }, | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "${{ github.event.pull_request.html_url || github.event.head_commit.url }}", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
docker-push: | ||
name: Build and Push Docker Image | ||
runs-on: ubuntu-latest-8-cores | ||
needs: [atomic-release] | ||
permissions: | ||
id-token: write # required to use OIDC auth | ||
contents: write # required to checkout code | ||
env: | ||
GCP_PROJECT_ID: glaredb-artifacts | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: google-github-actions/auth@v2 | ||
with: | ||
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }} | ||
- uses: google-github-actions/setup-gcloud@v2 | ||
- run: gcloud auth configure-docker -q | ||
- run: ./scripts/build-and-push-images.sh | ||
# TODO: this should pull the binary that was built in kernel-dist. | ||
kernel-push: | ||
name: Push Kernel Release | ||
runs-on: ubuntu-latest | ||
needs: [atomic-release] | ||
steps: | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: dist-aarch64-apple-darwin | ||
path: dist | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: dist-x86_64-apple-darwin | ||
path: dist | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: dist-x86_64-unknown-linux-gnu | ||
path: dist | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: dist-x86_64-pc-windows-msvc | ||
path: dist | ||
- run: ls -al ./dist | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: releases | ||
path: ./dist | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Push Kernel Release", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
######################################################################## | ||
## | ||
## Python Bindings | ||
## | ||
######################################################################## | ||
python-linux: | ||
name: Build Python Bindings (linux ${{ matrix.target }}) | ||
runs-on: ubuntu-2004-8-cores | ||
permissions: | ||
contents: read | ||
strategy: | ||
fail-fast: true | ||
matrix: | ||
target: [x86_64] | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: egor-tensin/setup-gcc@v1 | ||
with: | ||
version: 11 | ||
- uses: extractions/setup-just@v2 | ||
with: | ||
just-version: "1.23.0" | ||
- run: just protoc | ||
- uses: actions/setup-python@v5 | ||
with: | ||
python-version: ${{ env.PYTHON_VERSION }} | ||
- name: maturin build | ||
uses: PyO3/maturin-action@v1 | ||
with: | ||
manylinux: "auto" | ||
docker-options: "--env PROTOC=${{ env.PROTOC }}" | ||
target: ${{ matrix.target }} | ||
args: --release --out dist -m bindings/python/Cargo.toml | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: wheels | ||
path: dist | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Build Linux Python Bindings", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "target", "value": "${{ matrix.target }}", "short": true }, | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
python-windows: | ||
name: Build Python Bindings (windows ${{ matrix.target }}) | ||
runs-on: windows-latest-8-cores | ||
permissions: | ||
contents: read | ||
strategy: | ||
fail-fast: true | ||
matrix: | ||
target: [x64] | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: ilammy/setup-nasm@v1 | ||
- uses: extractions/setup-just@v2 | ||
with: | ||
just-version: "1.23.0" | ||
- run: just protoc | ||
- uses: actions/setup-python@v5 | ||
with: | ||
python-version: ${{ env.PYTHON_VERSION }} | ||
architecture: ${{ matrix.target }} | ||
- name: maturin build | ||
uses: PyO3/maturin-action@v1 | ||
with: | ||
target: ${{ matrix.target }} | ||
args: --release --out dist | ||
working-directory: bindings/python | ||
container: "off" | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: wheels | ||
path: bindings/python/dist | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Build Windows Python Bindings", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "target", "value": "Windows ${{ matrix.target }}", "short": true }, | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
python-macos: | ||
name: Build Python Bindings (macos ${{ matrix.target }}) | ||
runs-on: ${{ matrix.runner }} | ||
permissions: | ||
contents: read | ||
strategy: | ||
fail-fast: true | ||
matrix: | ||
- target: x86_64-apple-darwin | ||
runner: macos-13-large | ||
- target: aarch64-apple-darwin | ||
runner: macos-13-xlarge | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: extractions/setup-just@v2 | ||
with: | ||
just-version: "1.23.0" | ||
- run: just protoc | ||
- uses: actions/setup-python@v5 | ||
with: | ||
python-version: ${{ env.PYTHON_VERSION }} | ||
- name: maturin build | ||
uses: PyO3/maturin-action@v1 | ||
with: | ||
target: ${{ matrix.target }} | ||
args: --release --out dist | ||
working-directory: bindings/python | ||
container: "off" | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: wheels | ||
path: bindings/python/dist | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Build macOS Python Bindings", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "target", "value": "${{ matrix.target }}", "short": true }, | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
python-push: | ||
name: Push Python Bindings | ||
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | ||
runs-on: ubuntu-latest | ||
needs: [atomic-release] | ||
permissions: | ||
contents: read | ||
steps: | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
name: wheels | ||
- name: maturin publish | ||
uses: PyO3/maturin-action@v1 | ||
env: | ||
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | ||
with: | ||
command: upload | ||
args: --skip-existing * | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Push Python Bindings", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
######################################################################## | ||
## | ||
## Node.js Release Process | ||
## | ||
######################################################################## | ||
node-build: | ||
defaults: | ||
run: | ||
working-directory: bindings/nodejs | ||
permissions: | ||
contents: write | ||
id-token: write | ||
strategy: | ||
fail-fast: true | ||
matrix: | ||
- host: macos-12-xl | ||
target: x86_64-apple-darwin | ||
build: | | ||
yarn build | ||
strip -x *.node | ||
- host: windows-latest-8-cores | ||
build: yarn build | ||
target: x86_64-pc-windows-msvc | ||
- host: ubuntu-2004-8-cores | ||
target: x86_64-unknown-linux-gnu | ||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian | ||
build: |- | ||
set -eux && | ||
curl -L https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -o protoc.zip | ||
mkdir -p deps && | ||
unzip -o protoc.zip -d deps/protoc && | ||
rm protoc.zip | ||
export PROTOC="$PWD/deps/protoc/bin/protoc" && | ||
cd bindings/nodejs && | ||
yarn build --target x86_64-unknown-linux-gnu && | ||
strip *.node | ||
- host: macos-12-xl | ||
target: aarch64-apple-darwin | ||
build: | | ||
rustup target add aarch64-apple-darwin | ||
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*; | ||
export CC=$(xcrun -f clang); | ||
export CXX=$(xcrun -f clang++); | ||
SYSROOT=$(xcrun --sdk macosx --show-sdk-path); | ||
export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT"; | ||
yarn build --target aarch64-apple-darwin | ||
strip -x *.node | ||
name: Build Node.JS Bindings (stable - ${{ matrix.target }} - node@18) | ||
runs-on: ${{ matrix.host }} | ||
env: | ||
DEBUG: napi:* | ||
APP_NAME: "glaredb" | ||
MACOSX_DEPLOYMENT_TARGET: "10.13" | ||
PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: ilammy/setup-nasm@v1 | ||
if: matrix.target == 'x86_64-pc-windows-msvc' | ||
- uses: egor-tensin/setup-gcc@v1 | ||
if: matrix.target == 'x86_64-unknown-linux-gnu' | ||
with: | ||
version: 11 | ||
- uses: extractions/setup-just@v2 | ||
with: | ||
just-version: "1.23.0" | ||
- run: just protoc | ||
- uses: actions/setup-node@v4 | ||
if: ${{ !matrix.docker }} | ||
with: | ||
node-version: 20 | ||
check-latest: true | ||
- uses: dtolnay/rust-toolchain@stable | ||
if: ${{ !matrix.docker }} | ||
with: | ||
toolchain: stable | ||
targets: ${{ matrix.target }} | ||
- uses: actions/cache@v4 | ||
with: | ||
path: | | ||
~/.cargo/registry/index/ | ||
~/.cargo/registry/cache/ | ||
~/.cargo/git/db/ | ||
.cargo-cache | ||
target/ | ||
key: ${{ matrix.target }}-cargo-${{ matrix.host }} | ||
- uses: goto-bus-stop/setup-zig@v2 | ||
if: ${{ matrix.target == 'armv7-unknown-linux-gnueabihf' }} | ||
with: | ||
version: 0.10.1 | ||
- run: ${{ matrix.setup }} | ||
if: ${{ matrix.setup }} | ||
shell: bash | ||
- if: matrix.target == 'i686-pc-windows-msvc' | ||
run: yarn config set supportedArchitectures.cpu "ia32" | ||
shell: bash | ||
- run: yarn install --frozen-lockfile | ||
- uses: actions/setup-node@v4 | ||
if: matrix.target == 'i686-pc-windows-msvc' | ||
with: | ||
node-version: 20 | ||
check-latest: true | ||
architecture: x86 | ||
- uses: addnab/docker-run-action@v3 | ||
if: ${{ matrix.docker }} | ||
with: | ||
image: ${{ matrix.docker }} | ||
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build" | ||
run: ${{ matrix.build }} | ||
- run: ${{ matrix.build }} | ||
if: ${{ !matrix.docker }} | ||
shell: bash | ||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: bindings-${{ matrix.target }} | ||
path: bindings/nodejs/${{ env.APP_NAME }}.*.node | ||
if-no-files-found: error | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Build Node Bindings (stable - node@18)", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "target", "value": "${{ matrix.target }}", "short": true }, | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} | ||
node-push: | ||
name: Push Node.js Bindings | ||
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | ||
defaults: | ||
run: | ||
working-directory: bindings/nodejs | ||
runs-on: ubuntu-2004-8-cores | ||
needs: [atomic-release] | ||
env: | ||
DEBUG: napi:* | ||
APP_NAME: "glaredb" | ||
MACOSX_DEPLOYMENT_TARGET: "10.13" | ||
PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | ||
permissions: | ||
contents: write | ||
id-token: write | ||
steps: | ||
- uses: actions/checkout@v4 | ||
- uses: extractions/setup-just@v2 | ||
- run: just protoc | ||
- uses: actions/setup-node@v4 | ||
with: | ||
node-version: 20 | ||
check-latest: true | ||
registry-url: "https://registry.npmjs.org" | ||
- run: yarn install --frozen-lockfile | ||
- uses: actions/download-artifact@v3 | ||
with: | ||
path: bindings/nodejs/artifacts | ||
- run: yarn artifacts | ||
- run: ls -R ./npm | ||
shell: bash | ||
- name: publish | ||
run: | | ||
echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc | ||
npm publish | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | ||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | ||
- uses: slackapi/slack-github-action@v1.26.0 | ||
if: ${{ always() }} | ||
with: | ||
payload: | | ||
{ | ||
"text": "Push Node Bidnings", | ||
"attachments": [ | ||
{ | ||
"fields": [ | ||
{ "title": "status", "value": "${{ job.status }}", "short": true }, | ||
{ "title": "link", "value": "[commit](${{ github.event.pull_request.html_url || github.event.head_commit.url }})", "short": true } | ||
] | ||
} | ||
] | ||
} |