[RFC-0003] Add commands for managing OCI artifacts

.github/workflows/e2e.yaml

@@ -4,11 +4,16 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, oci ]
 
 jobs:
   kind:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -168,6 +173,36 @@ jobs:
       - name: flux delete source git
         run: |
           /tmp/flux delete source git podinfo --silent
+      - name: flux oci artifacts
+        run: |
+          /tmp/flux push artifact oci://localhost:5000/fluxcd/flux:${{ github.sha }} \
+            --path="./manifests" \
+            --source="${{ github.repositoryUrl }}" \
+            --revision="${{ github.ref }}/${{ github.sha }}"
+          /tmp/flux tag artifact oci://localhost:5000/fluxcd/flux:${{ github.sha }} \
+            --tag latest
+          /tmp/flux list artifacts oci://localhost:5000/fluxcd/flux
+      - name: flux oci repositories
+        run: |
+          /tmp/flux create source oci podinfo-oci \
+            --url oci://ghcr.io/stefanprodan/manifests/podinfo \
+            --tag-semver 6.1.x \
+            --interval 10m
+          /tmp/flux create kustomization podinfo-oci \
+            --source=OCIRepository/podinfo-oci \
+            --path="./kustomize" \
+            --prune=true \
+            --interval=5m \
+            --target-namespace=default \
+            --wait=true \
+            --health-check-timeout=3m
+          /tmp/flux reconcile source oci podinfo-oci
+          /tmp/flux suspend source oci podinfo-oci
+          /tmp/flux get sources oci
+          /tmp/flux resume source oci podinfo-oci
+          /tmp/flux export source oci podinfo-oci
+          /tmp/flux delete ks podinfo-oci --silent
+          /tmp/flux delete source oci podinfo-oci --silent
       - name: flux create tenant
         run: |
           /tmp/flux create tenant dev-team --with-namespace=apps

Makefile

@@ -1,4 +1,5 @@
 VERSION?=$(shell grep 'VERSION' cmd/flux/main.go | awk '{ print $$4 }' | head -n 1 | tr -d '"')
+DEV_VERSION?=0.0.0-$(shell git rev-parse --abbrev-ref HEAD)-$(shell git rev-parse --short HEAD)-$(shell date +%s)
 EMBEDDED_MANIFESTS_TARGET=cmd/flux/.manifests.done
 TEST_KUBECONFIG?=/tmp/flux-e2e-test-kubeconfig
 # Architecture to use envtest with
@@ -55,6 +56,9 @@ $(EMBEDDED_MANIFESTS_TARGET): $(call rwildcard,manifests/,*.yaml *.json)
 build: $(EMBEDDED_MANIFESTS_TARGET)
 	CGO_ENABLED=0 go build -ldflags="-s -w -X main.VERSION=$(VERSION)" -o ./bin/flux ./cmd/flux
 
+build-dev: $(EMBEDDED_MANIFESTS_TARGET)
+	CGO_ENABLED=0 go build -ldflags="-s -w -X main.VERSION=$(DEV_VERSION)" -o ./bin/flux ./cmd/flux
+
 .PHONY: install
 install:
 	CGO_ENABLED=0 go install ./cmd/flux

action/README.md

@@ -32,7 +32,7 @@ You can download a specific version with:
       - name: Setup Flux CLI
         uses: fluxcd/flux2/action@main
         with:
-          version: 0.8.0
+          version: 0.32.0
 ```
 
 ### Automate Flux updates
@@ -74,6 +74,92 @@ jobs:
               ${{ steps.update.outputs.flux_version }}
 ```
 
+### Push Kubernetes manifests to container registries
+
+Example workflow for publishing Kubernetes manifests bundled as OCI artifacts to GitHub Container Registry:
+
+```yaml
+name: push-artifact-staging
+
+on:
+  push:
+    branches:
+      - 'main'
+
+permissions:
+  packages: write # needed for ghcr.io access
+
+env:
+  OCI_REPO: "oci://ghcr.io/my-org/manifests/${{ github.event.repository.name }}"
+
+jobs:
+  kubernetes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Flux CLI
+        uses: fluxcd/flux2/action@main
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate manifests
+        run: |
+          kustomize build ./manifests/staging > ./deploy/app.yaml
+      - name: Push manifests
+        run: |
+          flux push artifact $OCI_REPO:$(git rev-parse --short HEAD) \
+            --path="./deploy" \
+            --source="$(git config --get remote.origin.url)" \
+            --revision="$(git branch --show-current)/$(git rev-parse HEAD)"
+      - name: Deploy manifests to staging
+        run: |
+          flux tag artifact $OCI_REPO:$(git rev-parse --short HEAD) --tag staging
+```
+
+Example workflow for publishing Kubernetes manifests bundled as OCI artifacts to Docker Hub:
+
+```yaml
+name: push-artifact-production
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  OCI_REPO: "oci://docker.io/my-org/app-config"
+
+jobs:
+  kubernetes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Flux CLI
+        uses: fluxcd/flux2/action@main
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Generate manifests
+        run: |
+          kustomize build ./manifests/production > ./deploy/app.yaml
+      - name: Push manifests
+        run: |
+          flux push artifact $OCI_REPO:$(git tag --points-at HEAD) \
+            --path="./deploy" \
+            --source="$(git config --get remote.origin.url)" \
+            --revision="$(git tag --points-at HEAD)/$(git rev-parse HEAD)"
+      - name: Deploy manifests to production
+        run: |
+          flux tag artifact $OCI_REPO:$(git tag --points-at HEAD) --tag production
+```
+
 ### End-to-end testing
 
 Example workflow for running Flux in Kubernetes Kind:

cmd/flux/build_artifact.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	oci "github.com/fluxcd/pkg/oci/client"
+)
+
+var buildArtifactCmd = &cobra.Command{
+	Use:   "artifact",
+	Short: "Build artifact",
+	Long:  `The build artifact command creates a tgz file with the manifests from the given directory.`,
+	Example: `  # Build the given manifests directory into an artifact
+  flux build artifact --path ./path/to/local/manifests --output ./path/to/artifact.tgz
+
+  # List the files bundled in the artifact
+  tar -ztvf ./path/to/artifact.tgz
+`,
+	RunE: buildArtifactCmdRun,
+}
+
+type buildArtifactFlags struct {
+	output string
+	path   string
+}
+
+var buildArtifactArgs buildArtifactFlags
+
+func init() {
+	buildArtifactCmd.Flags().StringVar(&buildArtifactArgs.path, "path", "", "Path to the directory where the Kubernetes manifests are located.")
+	buildArtifactCmd.Flags().StringVarP(&buildArtifactArgs.output, "output", "o", "artifact.tgz", "Path to where the artifact tgz file should be written.")
+	buildCmd.AddCommand(buildArtifactCmd)
+}
+
+func buildArtifactCmdRun(cmd *cobra.Command, args []string) error {
+	if buildArtifactArgs.path == "" {
+		return fmt.Errorf("invalid path %q", buildArtifactArgs.path)
+	}
+
+	if fs, err := os.Stat(buildArtifactArgs.path); err != nil || !fs.IsDir() {
+		return fmt.Errorf("invalid path '%s', must point to an existing directory", buildArtifactArgs.path)
+	}
+
+	logger.Actionf("building artifact from %s", buildArtifactArgs.path)
+
+	ociClient := oci.NewLocalClient()
+	if err := ociClient.Build(buildArtifactArgs.output, buildArtifactArgs.path); err != nil {
+		return fmt.Errorf("bulding artifact failed, error: %w", err)
+	}
+
+	logger.Successf("artifact created at %s", buildArtifactArgs.output)
+
+	return nil
+}

cmd/flux/create_kustomization.go

@@ -68,6 +68,13 @@ var createKsCmd = &cobra.Command{
     --prune=true \
     --interval=5m
 
+  # Create a Kustomization resource that references an OCIRepository
+  flux create kustomization podinfo \
+    --source=OCIRepository/podinfo \
+    --target-namespace=default \
+    --prune=true \
+    --interval=5m
+
   # Create a Kustomization resource that references a Bucket
   flux create kustomization secrets \
     --source=Bucket/secrets \

cmd/flux/create_secret_helm_test.go

@@ -1,3 +1,19 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package main
 
 import (