From 4710346ae3fb69bab7f8c3cf7433e917c508b89c Mon Sep 17 00:00:00 2001 From: Michael Gruener Date: Thu, 21 Jan 2021 13:57:21 +0100 Subject: [PATCH] Add support to render project to ArgoCD Applicaiton --- cmd/render.go | 36 +++-- go.sum | 5 + pkg/wrapper/argocd/argocd.go | 86 +++++++++++ pkg/wrapper/argocd/types.go | 169 +++++++++++++++++++++ pkg/{helmutil => wrapper/helm}/repo.go | 2 +- pkg/{helmutil => wrapper/helm}/template.go | 6 +- 6 files changed, 283 insertions(+), 21 deletions(-) create mode 100644 pkg/wrapper/argocd/argocd.go create mode 100644 pkg/wrapper/argocd/types.go rename pkg/{helmutil => wrapper/helm}/repo.go (99%) rename pkg/{helmutil => wrapper/helm}/template.go (95%) diff --git a/cmd/render.go b/cmd/render.go index 439d6f0..d7c2170 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -19,11 +19,12 @@ package cmd import ( "fmt" - "github.com/bedag/kusible/pkg/helmutil" "github.com/bedag/kusible/pkg/inventory" "github.com/bedag/kusible/pkg/playbook" "github.com/bedag/kusible/pkg/target" + argocdutil "github.com/bedag/kusible/pkg/wrapper/argocd" "github.com/bedag/kusible/pkg/wrapper/ejson" + helmutil "github.com/bedag/kusible/pkg/wrapper/helm" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -45,7 +46,7 @@ var renderHelmCmd = &cobra.Command{ groupVarsDir := viper.GetString("group-vars-dir") inventoryPath := viper.GetString("inventory") skipEval := viper.GetBool("skip-eval") - skipClusterInv := viper.GetBool("render-skip-cluster-inventory") + skipClusterInv := viper.GetBool("render-helm-skip-cluster-inventory") skipDecrypt := viper.GetBool("skip-decrypt") ejsonPrivKey := viper.GetString("ejson-privkey") ejsonKeyDir := viper.GetString("ejson-key-dir") @@ -126,7 +127,9 @@ var renderArgoCDCmd = &cobra.Command{ groupVarsDir := viper.GetString("group-vars-dir") inventoryPath := viper.GetString("inventory") skipEval := viper.GetBool("skip-eval") - skipClusterInv := viper.GetBool("render-skip-cluster-inventory") + skipClusterInv := viper.GetBool("render-argocd-skip-cluster-inventory") + namespace := viper.GetString("argocd-namespace") + project := viper.GetString("argocd-project") skipDecrypt := viper.GetBool("skip-decrypt") ejsonPrivKey := viper.GetString("ejson-privkey") ejsonKeyDir := viper.GetString("ejson-key-dir") @@ -169,28 +172,15 @@ var renderArgoCDCmd = &cobra.Command{ }).Fatal("Failed to compile playbooks.") } - settings := helmcli.New() - - // https://github.com/argoproj/argo-cd/blob/master/pkg/apis/application/v1alpha1/types.go for name, playbook := range playbookSet { for _, play := range playbook.Config.Plays { - for _, repo := range play.Repos { - if err := helmutil.RepoAdd(repo.Name, repo.URL, settings); err != nil { - log.WithFields(log.Fields{ - "play": play.Name, - "repo": repo.Name, - "entry": name, - "error": err.Error(), - }).Fatal("Failed to add helm repo for play.") - } - } - manifests, err := helmutil.TemplatePlay(play, settings) + manifests, err := argocdutil.ApplicationFromPlay(play, project, namespace, name) if err != nil { log.WithFields(log.Fields{ "play": play.Name, "entry": name, "error": err.Error(), - }).Fatal("Failed to render play manifests with helm.") + }).Fatal("Failed to render ArgoCD application manifests.") } fmt.Printf(manifests) } @@ -200,8 +190,16 @@ var renderArgoCDCmd = &cobra.Command{ func init() { renderHelmCmd.Flags().BoolP("skip-cluster-inventory", "", false, "Skip downloading the cluster-inventory ConfigMap") - viper.BindPFlag("render-skip-cluster-inventory", renderHelmCmd.Flags().Lookup("skip-cluster-inventory")) + viper.BindPFlag("render-helm-skip-cluster-inventory", renderHelmCmd.Flags().Lookup("skip-cluster-inventory")) + + renderArgoCDCmd.Flags().BoolP("skip-cluster-inventory", "", false, "Skip downloading the cluster-inventory ConfigMap") + renderArgoCDCmd.Flags().StringP("namespace", "", "argocd", "Namespace where ArgoCD is looking for ArgoCD applications") + renderArgoCDCmd.Flags().StringP("project", "", "default", "The ArgoCD project to which the applications should be assigned") + viper.BindPFlag("render-argocd-skip-cluster-inventory", renderArgoCDCmd.Flags().Lookup("skip-cluster-inventory")) + viper.BindPFlag("argocd-namespace", renderArgoCDCmd.Flags().Lookup("namespace")) + viper.BindPFlag("argocd-project", renderArgoCDCmd.Flags().Lookup("project")) renderCmd.AddCommand(renderHelmCmd) + renderCmd.AddCommand(renderArgoCDCmd) rootCmd.AddCommand(renderCmd) } diff --git a/go.sum b/go.sum index 7c1242b..7c074d3 100644 --- a/go.sum +++ b/go.sum @@ -445,6 +445,7 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -521,6 +522,7 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= @@ -681,6 +683,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -1085,6 +1088,7 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1101,6 +1105,7 @@ gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJ gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= helm.sh/helm/v3 v3.5.0 h1:uqIT3Bh4hVEyZRThyTPik8FkiABj3VJIY+POvDFT3a4= helm.sh/helm/v3 v3.5.0/go.mod h1:bjwXfmGAF+SEuJZ2AtN1xmTuz4FqaNYOJrXP+vtj6Tw= diff --git a/pkg/wrapper/argocd/argocd.go b/pkg/wrapper/argocd/argocd.go new file mode 100644 index 0000000..005b2ad --- /dev/null +++ b/pkg/wrapper/argocd/argocd.go @@ -0,0 +1,86 @@ +/* +Copyright © 2021 Michael Gruener + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package argocd + +import ( + // "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" + "fmt" + "strings" + + "github.com/bedag/kusible/pkg/playbook/config" + "sigs.k8s.io/yaml" +) + +// ApplicationFromPlay renders a a set of ArgoCD Application resources (see https://argoproj.github.io/argo-cd/operator-manual/declarative-setup/) +// for a given play. Each chart of the play results in a separate Application resource containing +// the details of the helm release (release name, chart name, repo of the chart, chart version, values). +// +// The project parameter is the argocd project the application should belong to +// The namespace parameter is the namespace where ArgoCD is expection Application resources +// The server parameter is the server name(!) as configured in ArgoCD where ArgoCD should deploy the rendered resources +func ApplicationFromPlay(play *config.Play, project string, namespace string, server string) (string, error) { + // https://github.com/argoproj/argo-cd/blob/master/pkg/apis/application/v1alpha1/types.go + result := "" + for _, chart := range play.Charts { + app := Application{} + // global Application resource settings + app.APIVersion = "argoproj.io/v1alpha1" + app.Kind = "Application" + app.ObjectMeta.Namespace = namespace + // TODO: Implement a proper approach to avoid argocd application name collisions. + // All argocd application resources for all clusters exist in the same namespace + // on the cluster hosting argocd. Aside from a proper name generation this requires + // a collision detection because the config structure of kusible allows for a setup + // that leads to non-preventable name collisions. + app.ObjectMeta.Name = fmt.Sprintf("%s.%s.%s", chart.Name, project, server) + app.Spec.Project = project + + // helm chart settings + for _, repo := range play.Repos { + if repo.Name == chart.Repo { + app.Spec.Source.RepoURL = repo.URL + } + } + + if app.Spec.Source.RepoURL == "" { + return result, fmt.Errorf("no repo '%s' for chart '%s' configured in play", chart.Repo, chart.Name) + } + + app.Spec.Source.Chart = chart.Chart + app.Spec.Source.TargetRevision = chart.Version + + // design decision: only support helm 3 + app.Spec.Source.Helm = &ApplicationSourceHelm{} + app.Spec.Source.Helm.Version = "v3" + values, err := yaml.Marshal(chart.Values) + if err != nil { + return result, fmt.Errorf("failed to convert values of chart '%s' to yaml: %s", chart.Name, err) + } + app.Spec.Source.Helm.Values = string(values) + + // target cluster + namespace settings + app.Spec.Destination.Namespace = chart.Namespace + app.Spec.Destination.Name = server + + manifest, err := yaml.Marshal(app) + if err != nil { + return result, err + } + result = fmt.Sprintf("---\n%s%s\n", result, strings.TrimSpace(string(manifest))) + } + return result, nil +} diff --git a/pkg/wrapper/argocd/types.go b/pkg/wrapper/argocd/types.go new file mode 100644 index 0000000..402d26b --- /dev/null +++ b/pkg/wrapper/argocd/types.go @@ -0,0 +1,169 @@ +/* +Copyright © 2021 Michael Gruener & The ArgoCD Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package argocd + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +/* +Because of https://github.com/argoproj/argo-cd/issues/4055 it is very cumbersome +(if at all possible in a sensible way) to import the ArgoCD go packages. According +to some comments in the issue it is easier to define the required datastructures +yourself, which we do here. + +Prepare for lots of copy & paste from https://github.com/argoproj/argo-cd/blob/master/pkg/apis/application/v1alpha1/types.go + +The definitions of the data structures are stripped down to what the kusible code +actually needs to keep the amount of required code maintenance down. +*/ + +// Application is a definition of an ArgoCD Application resource. +type Application struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + Spec ApplicationSpec `json:"spec"` +} + +// ApplicationSpec represents desired application state. Contains link to repository with application definition and additional parameters link definition revision. +type ApplicationSpec struct { + // Source is a reference to the location ksonnet application definition + Source ApplicationSource `json:"source"` + // Destination overrides the kubernetes server and namespace defined in the environment ksonnet app.yaml + Destination ApplicationDestination `json:"destination"` + // Project is a application project name. Empty name means that application belongs to 'default' project. + Project string `json:"project"` + // SyncPolicy controls when a sync will be performed + SyncPolicy *SyncPolicy `json:"syncPolicy,omitempty"` + // IgnoreDifferences controls resources fields which should be ignored during comparison + IgnoreDifferences []ResourceIgnoreDifferences `json:"ignoreDifferences,omitempty"` + // Infos contains a list of useful information (URLs, email addresses, and plain text) that relates to the application + Info []Info `json:"info,omitempty"` + // This limits this number of items kept in the apps revision history. + // This should only be changed in exceptional circumstances. + // Setting to zero will store no history. This will reduce storage used. + // Increasing will increase the space used to store the history, so we do not recommend increasing it. + // Default is 10. + RevisionHistoryLimit *int64 `json:"revisionHistoryLimit,omitempty"` +} + +// ApplicationSource contains information about github repository, path within repository and target application environment. +type ApplicationSource struct { + // RepoURL is the repository URL of the application manifests + RepoURL string `json:"repoURL"` + // TargetRevision defines the commit, tag, or branch in which to sync the application to. + // If omitted, will sync to HEAD + TargetRevision string `json:"targetRevision,omitempty"` + // Helm holds helm specific options + Helm *ApplicationSourceHelm `json:"helm,omitempty"` + // Chart is a Helm chart name + Chart string `json:"chart,omitempty"` +} + +// ApplicationDestination contains deployment destination information +type ApplicationDestination struct { + // Namespace overrides the environment namespace value in the ksonnet app.yaml + Namespace string `json:"namespace,omitempty"` + // Name of the destination cluster which can be used instead of server (url) field + Name string `json:"name,omitempty"` +} + +// ResourceIgnoreDifferences contains resource filter and list of json paths which should be ignored during comparison with live state. +type ResourceIgnoreDifferences struct { + Group string `json:"group,omitempty"` + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + JSONPointers []string `json:"jsonPointers"` +} + +// SyncPolicy controls when a sync will be performed in response to updates in git +type SyncPolicy struct { + // Automated will keep an application synced to the target revision + Automated *SyncPolicyAutomated `json:"automated,omitempty"` + // Options allow you to specify whole app sync-options + SyncOptions SyncOptions `json:"syncOptions,omitempty"` + // Retry controls failed sync retry behavior + Retry *RetryStrategy `json:"retry,omitempty"` +} + +type Info struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// ApplicationSourceHelm holds helm specific options +type ApplicationSourceHelm struct { + // ValuesFiles is a list of Helm value files to use when generating a template + ValueFiles []string `json:"valueFiles,omitempty"` + // Parameters are parameters to the helm template + Parameters []HelmParameter `json:"parameters,omitempty"` + // The Helm release name. If omitted it will use the application name + ReleaseName string `json:"releaseName,omitempty"` + // Values is Helm values, typically defined as a block + Values string `json:"values,omitempty"` + // FileParameters are file parameters to the helm template + FileParameters []HelmFileParameter `json:"fileParameters,omitempty"` + // Version is the Helm version to use for templating with + Version string `json:"version,omitempty"` +} + +// HelmParameter is a parameter to a helm template +type HelmParameter struct { + // Name is the name of the helm parameter + Name string `json:"name,omitempty"` + // Value is the value for the helm parameter + Value string `json:"value,omitempty"` + // ForceString determines whether to tell Helm to interpret booleans and numbers as strings + ForceString bool `json:"forceString,omitempty"` +} + +// HelmFileParameter is a file parameter to a helm template +type HelmFileParameter struct { + // Name is the name of the helm parameter + Name string `json:"name,omitempty"` + // Path is the path value for the helm parameter + Path string `json:"path,omitempty"` +} + +// SyncPolicyAutomated controls the behavior of an automated sync +type SyncPolicyAutomated struct { + // Prune will prune resources automatically as part of automated sync (default: false) + Prune bool `json:"prune,omitempty"` + // SelfHeal enables auto-syncing if (default: false) + SelfHeal bool `json:"selfHeal,omitempty"` + // AllowEmpty allows apps have zero live resources (default: false) + AllowEmpty bool `json:"allowEmpty,omitempty"` +} + +type SyncOptions []string + +type RetryStrategy struct { + // Limit is the maximum number of attempts when retrying a container + Limit int64 `json:"limit,omitempty"` + + // Backoff is a backoff strategy + Backoff *Backoff `json:"backoff,omitempty"` +} + +// Backoff is a backoff strategy to use within retryStrategy +type Backoff struct { + // Duration is the amount to back off. Default unit is seconds, but could also be a duration (e.g. "2m", "1h") + Duration string `json:"duration,omitempty"` + // Factor is a factor to multiply the base duration after each failed retry + Factor *int64 `json:"factor,omitempty"` + // MaxDuration is the maximum amount of time allowed for the backoff strategy + MaxDuration string `json:"maxDuration,omitempty"` +} diff --git a/pkg/helmutil/repo.go b/pkg/wrapper/helm/repo.go similarity index 99% rename from pkg/helmutil/repo.go rename to pkg/wrapper/helm/repo.go index 5c4f235..3827cf5 100644 --- a/pkg/helmutil/repo.go +++ b/pkg/wrapper/helm/repo.go @@ -16,7 +16,7 @@ limitations under the License. /* Lots of code straight from github.com/helm/helm and adapted to be used here */ -package helmutil +package helm import ( "context" diff --git a/pkg/helmutil/template.go b/pkg/wrapper/helm/template.go similarity index 95% rename from pkg/helmutil/template.go rename to pkg/wrapper/helm/template.go index 4872db8..00aa304 100644 --- a/pkg/helmutil/template.go +++ b/pkg/wrapper/helm/template.go @@ -16,7 +16,7 @@ limitations under the License. /* Lots of code straight from github.com/helm/helm and adapted to be used here */ -package helmutil +package helm import ( "fmt" @@ -50,6 +50,10 @@ func TemplatePlay(play *config.Play, settings *helmcli.EnvSettings) (string, err } } + if client.ChartPathOptions.RepoURL == "" { + return result, fmt.Errorf("no repo '%s' for chart '%s' configured in play", chart.Repo, chart.Name) + } + client.ReleaseName = chart.Name client.Version = chart.Version client.Namespace = chart.Namespace