diff --git a/cli/cmd/plugin_publish.go b/cli/cmd/plugin_publish.go index 4124301d819db5..499ccaeffed977 100644 --- a/cli/cmd/plugin_publish.go +++ b/cli/cmd/plugin_publish.go @@ -7,7 +7,6 @@ import ( "net/http" "os" "os/signal" - "path" "path/filepath" "strings" "syscall" @@ -83,6 +82,7 @@ func runPluginPublish(ctx context.Context, cmd *cobra.Command, args []string) er return errors.New("invalid plugin name. Must be in format /") } teamName, pluginName = parts[0], parts[1] + pkgJSON.Team, pkgJSON.Name = teamName, pluginName } name := fmt.Sprintf("%s/%s@%s", teamName, pluginName, pkgJSON.Version) @@ -121,13 +121,8 @@ func runPluginPublish(ctx context.Context, cmd *cobra.Command, args []string) er } // upload binaries - fmt.Println("Uploading binaries...") - for _, t := range pkgJSON.SupportedTargets { - fmt.Printf("- Uploading %s_%s...\n", t.OS, t.Arch) - err = publish.UploadPluginBinary(ctx, c, teamName, pluginName, t.OS, t.Arch, path.Join(distDir, t.Path), pkgJSON) - if err != nil { - return fmt.Errorf("failed to upload binary: %w", err) - } + if err := publishPluginAssets(ctx, c, token.String(), distDir, pkgJSON); err != nil { + return fmt.Errorf("failed to upload binaries: %w", err) } // optional: mark plugin as draft=false @@ -158,3 +153,11 @@ func runPluginPublish(ctx context.Context, cmd *cobra.Command, args []string) er return nil } + +func publishPluginAssets(ctx context.Context, c *cloudquery_api.ClientWithResponses, token, distDir string, pkgJSON publish.PackageJSONV1) error { + if pkgJSON.PackageType == string(cloudquery_api.PluginVersionPackageTypeDocker) { + return publish.PublishToDockerRegistry(ctx, token, distDir, pkgJSON) + } + + return publish.PublishNativeBinaries(ctx, c, distDir, pkgJSON) +} diff --git a/cli/go.mod b/cli/go.mod index bee5d63b5643a9..18dc03493e95e1 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -9,10 +9,14 @@ require ( github.com/cloudquery/cloudquery-api-go v1.6.3 github.com/cloudquery/plugin-pb-go v1.15.0 github.com/cloudquery/plugin-sdk/v4 v4.23.0 + github.com/distribution/reference v0.5.0 + github.com/docker/distribution v2.8.3+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/getsentry/sentry-go v0.24.1 github.com/ghodss/yaml v1.0.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 + github.com/opencontainers/go-digest v1.0.0 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/rs/zerolog v1.31.0 github.com/schollz/progressbar/v3 v3.13.1 @@ -38,17 +42,17 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/avast/retry-go/v4 v4.5.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.16.2 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect @@ -64,6 +68,7 @@ require ( github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/gorilla/css v1.0.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -84,16 +89,20 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/microcosm-cc/bluemonday v1.0.26 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.19 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.1.0 // indirect + github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect + github.com/prometheus/common v0.6.0 // indirect + github.com/prometheus/procfs v0.0.3 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect diff --git a/cli/go.sum b/cli/go.sum index 3fbb9eb8ab501d..58131a2f6d1523 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -19,6 +19,8 @@ github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/arrow/go/v15 v15.0.0-20231227193016-bcaeaa8c2d97 h1:VYsDh6GHlOrB+nN70yhjIV51FFSqByQdjeFvGnl7LAo= @@ -29,6 +31,10 @@ github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= @@ -67,8 +73,12 @@ github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -87,6 +97,9 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -95,13 +108,18 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -111,6 +129,7 @@ github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd h1:PppHBegd3uP github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -121,6 +140,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= @@ -134,9 +155,12 @@ github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvP github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kataras/blocks v0.0.8 h1:MrpVhoFTCR2v1iOOfGng5VJSILKeZZI+7NGfxEh3SUM= github.com/kataras/blocks v0.0.8/go.mod h1:9Jm5zx6BB+06NwA+OhTbHW1xkMOYxahnqTN5DveZ2Yg= @@ -158,6 +182,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -182,6 +208,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= @@ -193,10 +221,13 @@ github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -211,10 +242,25 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -231,6 +277,7 @@ github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iH github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -298,6 +345,7 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -310,22 +358,29 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -376,11 +431,13 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/cli/internal/publish/plugins.go b/cli/internal/publish/plugins.go index fa318cf8533769..d38056882d7990 100644 --- a/cli/internal/publish/plugins.go +++ b/cli/internal/publish/plugins.go @@ -1,17 +1,34 @@ package publish import ( + "bytes" "context" + "crypto/tls" + "encoding/base64" "encoding/json" "errors" "fmt" + "io" "net/http" "os" + "path" "path/filepath" "strings" + "time" cloudquery_api "github.com/cloudquery/cloudquery-api-go" "github.com/cloudquery/cloudquery/cli/internal/hub" + "github.com/distribution/reference" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema2" + distributionclient "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/client" + "github.com/opencontainers/go-digest" + "github.com/schollz/progressbar/v3" ) type PackageJSONV1 struct { @@ -26,10 +43,115 @@ type PackageJSONV1 struct { } type TargetBuild struct { - OS string `json:"os"` - Arch string `json:"arch"` - Path string `json:"path"` - Checksum string `json:"checksum"` + OS string `json:"os"` + Arch string `json:"arch"` + Path string `json:"path"` + Checksum string `json:"checksum"` + DockerImageTag string `json:"docker_image_tag"` +} + +type LoadResponse struct { + Stream string `json:"stream"` +} + +type dockerProgressReader struct { + decoder *json.Decoder + bar *progressbar.ProgressBar + layerPushedByID map[string]int64 + totalBytes int64 +} + +type dockerProgressInfo struct { + Status string `json:"status"` + Progress string `json:"progress"` + ProgressData struct { + Current int64 `json:"current"` + Total int64 `json:"total"` + } `json:"progressDetail"` + LayerID string `json:"id"` + ErrorDetail struct { + Message string `json:"message"` + } `json:"errorDetail"` +} + +type transportWithRegistryAuth struct { + http.RoundTripper + baseTransport *http.Transport + registryAuth string +} + +func newTransportWithRegistryAuth(insecureSkipVerify bool, registryAuth string) *transportWithRegistryAuth { + baseTransport := http.DefaultTransport.(*http.Transport).Clone() + baseTransport.TLSClientConfig.InsecureSkipVerify = insecureSkipVerify + return &transportWithRegistryAuth{ + baseTransport: baseTransport, + registryAuth: registryAuth, + } +} + +func (t *transportWithRegistryAuth) RoundTrip(req *http.Request) (resp *http.Response, err error) { + req.Header.Set("Authorization", "Bearer "+t.registryAuth) + return t.baseTransport.RoundTrip(req) +} + +func pushProgressBar(maxBytes int64, description ...string) *progressbar.ProgressBar { + desc := "" + if len(description) > 0 { + desc = description[0] + } + return progressbar.NewOptions64( + maxBytes, + progressbar.OptionSetDescription(desc), + progressbar.OptionSetWriter(os.Stdout), + progressbar.OptionShowBytes(true), + progressbar.OptionSetWidth(10), + progressbar.OptionThrottle(65*time.Millisecond), + progressbar.OptionShowCount(), + progressbar.OptionOnCompletion(func() { + fmt.Fprint(os.Stdout, "\n") + }), + progressbar.OptionSpinnerType(14), + progressbar.OptionFullWidth(), + progressbar.OptionSetRenderBlankState(true), + ) +} + +func (pr *dockerProgressReader) Read(_ []byte) (n int, err error) { + var progress dockerProgressInfo + err = pr.decoder.Decode(&progress) + if err != nil { + if err == io.EOF { + return 0, io.EOF + } + return 0, fmt.Errorf("failed to parse docker push response: %v", err) + } + if progress.ErrorDetail.Message != "" { + return 0, fmt.Errorf("failed to push image: %s", progress.ErrorDetail.Message) + } + if progress.Status == "Pushing" { + if pr.bar == nil { + pr.bar = pushProgressBar(1, "Pushing") + _ = pr.bar.RenderBlank() + } + if _, seen := pr.layerPushedByID[progress.LayerID]; !seen { + pr.layerPushedByID[progress.LayerID] = 0 + pr.totalBytes += progress.ProgressData.Total + pr.bar.ChangeMax64(pr.totalBytes) + } + pr.layerPushedByID[progress.LayerID] = progress.ProgressData.Current + total := int64(0) + for _, v := range pr.layerPushedByID { + total += v + } + if total < pr.totalBytes { + // progressbar stops responding if it reaches 100%, so as a workaround we don't update + // the bar if we're at 100%, because there may be more layers of the image + // coming that we don't know about. + _ = pr.bar.Set64(total) + } + } + + return 0, nil } func ReadPackageJSON(distDir string) (PackageJSONV1, error) { @@ -158,9 +280,9 @@ func UploadTableSchemas(ctx context.Context, c *cloudquery_api.ClientWithRespons return nil } -func UploadPluginBinary(ctx context.Context, c *cloudquery_api.ClientWithResponses, teamName, pluginName, goos, goarch, localPath string, pkgJSON PackageJSONV1) error { +func UploadPluginBinary(ctx context.Context, c *cloudquery_api.ClientWithResponses, goos, goarch, localPath string, pkgJSON PackageJSONV1) error { target := goos + "_" + goarch - resp, err := c.UploadPluginAssetWithResponse(ctx, teamName, pkgJSON.Kind, pluginName, pkgJSON.Version, target) + resp, err := c.UploadPluginAssetWithResponse(ctx, pkgJSON.Team, pkgJSON.Kind, pkgJSON.Name, pkgJSON.Version, target) if err != nil { return fmt.Errorf("failed to upload binary: %w", err) } @@ -184,3 +306,309 @@ func UploadPluginBinary(ctx context.Context, c *cloudquery_api.ClientWithRespons } return nil } + +func PublishNativeBinaries(ctx context.Context, c *cloudquery_api.ClientWithResponses, distDir string, pkgJSON PackageJSONV1) error { + fmt.Println("Uploading binaries CloudQuery Hub...") + for _, t := range pkgJSON.SupportedTargets { + fmt.Printf("- Uploading %s_%s...\n", t.OS, t.Arch) + err := UploadPluginBinary(ctx, c, t.OS, t.Arch, path.Join(distDir, t.Path), pkgJSON) + if err != nil { + return fmt.Errorf("failed to upload binary: %w", err) + } + } + return nil +} + +func getResponseAsString(body io.ReadCloser) string { + defer body.Close() + buf := new(bytes.Buffer) + buf.ReadFrom(body) + return buf.String() +} + +func loadDockerImage(ctx context.Context, cli *client.Client, imagePath string) error { + f, err := os.Open(imagePath) + if err != nil { + return fmt.Errorf("failed to open image file: %v", err) + } + defer f.Close() + resp, err := cli.ImageLoad(ctx, f, true) + if err != nil { + return fmt.Errorf("failed to load image: %v", err) + } + if resp.Body == nil { + return fmt.Errorf("failed to load image: response body is nil") + } + + respString := getResponseAsString(resp.Body) + loadResponse := LoadResponse{} + if err := json.Unmarshal([]byte(respString), &loadResponse); err != nil { + return fmt.Errorf("failed to parse docker load response: %v", err) + } + if loadResponse.Stream != "" { + fmt.Print(loadResponse.Stream) + } + + return nil +} + +func pushImage(ctx context.Context, dockerClient *client.Client, t TargetBuild, opts types.ImagePushOptions) error { + fmt.Printf("Pushing %s\n", t.DockerImageTag) + opts.Platform = fmt.Sprintf("%s/%s", t.OS, t.Arch) + out, err := dockerClient.ImagePush(ctx, t.DockerImageTag, opts) + if err != nil { + return fmt.Errorf("failed to push Docker image: %v", err) + } + defer out.Close() + + // Create a progress reader to display the download progress + pr := &dockerProgressReader{ + decoder: json.NewDecoder(out), + layerPushedByID: map[string]int64{}, + } + if _, err := io.Copy(io.Discard, pr); err != nil { + return err + } + if pr.bar != nil { + _ = pr.bar.Finish() + pr.bar.Close() + } + + return nil +} + +func getDockerToken(ctx context.Context, ref reference.Named, version, username, password string, insecureSkipVerify bool) (string, error) { + // https://distribution.github.io/distribution/spec/auth/token/#how-to-authenticate + domain := reference.Domain(ref) + name := reference.Path(ref) + + customTransport := http.DefaultTransport.(*http.Transport).Clone() + customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: insecureSkipVerify} + httpClient := &http.Client{Transport: customTransport} + + challengeUrl := fmt.Sprintf("https://%[1]s/v2/%[2]s/manifests/%[3]s", domain, name, version) + challengeReq, err := http.NewRequestWithContext(ctx, http.MethodPut, challengeUrl, nil) + if err != nil { + return "", fmt.Errorf("client: could not create request: %s", err) + } + challengeRes, err := httpClient.Do(challengeReq) + if err != nil { + return "", fmt.Errorf("client: could not send request: %s", err) + } + defer challengeRes.Body.Close() + if challengeRes.StatusCode != http.StatusUnauthorized { + return "", fmt.Errorf("client: unexpected status code: %d", challengeRes.StatusCode) + } + challengeHeader := challengeRes.Header.Get("WWW-Authenticate") + if challengeHeader == "" { + return "", fmt.Errorf("client: missing WWW-Authenticate header") + } + challenges := challenge.ResponseChallenges(challengeRes) + if len(challenges) != 1 { + return "", fmt.Errorf("client: expected 1 challenge header, got %d. Header value %q", len(challenges), challengeHeader) + } + realm := challenges[0].Parameters["realm"] + service := challenges[0].Parameters["service"] + scope := challenges[0].Parameters["scope"] + if realm == "" || service == "" || scope == "" { + return "", fmt.Errorf("client: could not parse challenge header %q", challengeHeader) + } + + url := fmt.Sprintf("%[1]s?account=cli&service=%[2]s&scope=%[3]s", realm, service, scope) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + basicAuth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + req.Header.Add("X-Meta-Plugin-Version", version) + req.Header.Add("Authorization", "Basic "+basicAuth) + if err != nil { + return "", fmt.Errorf("client: could not create request: %s", err) + } + + res, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("client: could not send request: %s", err) + } + defer res.Body.Close() + + tokenResponse := map[string]string{} + if err := json.NewDecoder(res.Body).Decode(&tokenResponse); err != nil { + return "", fmt.Errorf("client: could not decode response: %s", err) + } + return tokenResponse["token"], nil +} + +func getManifestParams(target string) (imageName reference.Named, repository string, err error) { + ref, err := reference.ParseNamed(target) + if err != nil { + return nil, "", fmt.Errorf("failed to parse Docker image tag: %v", err) + } + repository = reference.Domain(ref) + refPath := reference.Path(ref) + imageName, err = reference.WithName(refPath) + if err != nil { + return nil, "", fmt.Errorf("failed to create Docker repository: %v", err) + } + return imageName, repository, nil +} + +func getManifestService(ctx context.Context, imageName reference.Named, repository string, registryAuth string, insecureSkipVerify bool) (distribution.ManifestService, error) { + schema2Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + m := new(schema2.DeserializedManifest) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: schema2.MediaTypeManifest}, err + } + err := distribution.RegisterManifestSchema(schema2.MediaTypeManifest, schema2Func) + if err != nil { + return nil, fmt.Errorf("failed to create Docker manifest: %v", err) + } + + repo, err := distributionclient.NewRepository(imageName, "https://"+repository, newTransportWithRegistryAuth(insecureSkipVerify, registryAuth)) + if err != nil { + return nil, fmt.Errorf("failed to create Docker repository: %v", err) + } + manifestService, err := repo.Manifests(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create Docker manifest: %v", err) + } + return manifestService, nil +} + +func getTagsFromTargets(targets []TargetBuild) ([]reference.NamedTagged, error) { + namedTags := make([]reference.NamedTagged, len(targets)) + for i, t := range targets { + namedRef, err := reference.ParseNormalizedNamed(t.DockerImageTag) + if err != nil { + return nil, fmt.Errorf("failed to parse Docker image tag: %v", err) + } + nameWithTag, ok := namedRef.(reference.NamedTagged) + if !ok { + return nil, fmt.Errorf("failed to parse Docker image tag: %v", err) + } + namedTags[i] = nameWithTag + } + return namedTags, nil +} + +func getManifestList(ctx context.Context, manifestService distribution.ManifestService, pkgJSON PackageJSONV1) (*manifestlist.DeserializedManifestList, error) { + namedTags, err := getTagsFromTargets(pkgJSON.SupportedTargets) + if err != nil { + return nil, err + } + + descriptors := make([]manifestlist.ManifestDescriptor, 0) + for i, namedTag := range namedTags { + buildTarget := pkgJSON.SupportedTargets[i] + manifest, err := manifestService.Get(ctx, "", distribution.WithTag(namedTag.Tag())) + if err != nil { + return nil, fmt.Errorf("failed to create Docker manifest: %v", err) + } + mediaType, canonical, err := manifest.Payload() + if err != nil { + return nil, fmt.Errorf("failed to create Docker manifest: %v", err) + } + manifestDesc := manifestlist.ManifestDescriptor{ + Descriptor: distribution.Descriptor{ + Digest: digest.FromBytes(canonical), + Size: int64(len(canonical)), + MediaType: mediaType}, + Platform: manifestlist.PlatformSpec{OS: buildTarget.OS, Architecture: buildTarget.Arch}, + } + descriptors = append(descriptors, manifestDesc) + } + list, err := manifestlist.FromDescriptors(descriptors) + if err != nil { + return nil, fmt.Errorf("failed to create Docker manifest: %v", err) + } + return list, nil +} + +func pushManifest(ctx context.Context, pkgJSON PackageJSONV1, dockerToken string, insecureSkipVerify bool) error { + imageName, repository, err := getManifestParams(pkgJSON.SupportedTargets[0].DockerImageTag) + if err != nil { + return err + } + + manifestService, err := getManifestService(ctx, imageName, repository, dockerToken, insecureSkipVerify) + if err != nil { + return err + } + + manifestList, err := getManifestList(ctx, manifestService, pkgJSON) + if err != nil { + return err + } + + digest, err := manifestService.Put(ctx, manifestList, distribution.WithTag(pkgJSON.Version)) + if err != nil { + return fmt.Errorf("failed to create Docker manifest: %v", err) + } + fmt.Println("Created manifest:", digest.String()) + return nil +} + +func PublishToDockerRegistry(ctx context.Context, token, distDir string, pkgJSON PackageJSONV1) error { + // We use a mix of the Docker Go SDK that implements the Docker Engine API https://docs.docker.com/engine/api/latest/ + // and talking to the registry directly since the Docker Engine API doesn't support the manifest API yet + + // Loading and pushing the images is done via the Docker Go SDK + additionalHeaders := map[string]string{"X-Meta-Plugin-Version": pkgJSON.Version} + dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithHTTPHeaders(additionalHeaders)) + if err != nil { + return fmt.Errorf("failed to create Docker client: %v", err) + } + if len(pkgJSON.SupportedTargets) == 0 { + return fmt.Errorf("no supported targets found") + } + fmt.Println("Pushing images to CloudQuery Docker Registry...") + for _, t := range pkgJSON.SupportedTargets { + fmt.Printf("Loading %s...\n", t.Path) + imagePath := path.Join(distDir, t.Path) + err := loadDockerImage(ctx, dockerClient, imagePath) + if err != nil { + return err + } + } + username, password := "cli", token + authConfig := registry.AuthConfig{ + Username: username, + Password: password, + } + encodedAuth, err := registry.EncodeAuthConfig(authConfig) + if err != nil { + return fmt.Errorf("failed to encode Docker auth config: %v", err) + } + opts := types.ImagePushOptions{ + RegistryAuth: encodedAuth, + } + for _, t := range pkgJSON.SupportedTargets { + if err := pushImage(ctx, dockerClient, t, opts); err != nil { + return err + } + } + + // Pushing the manifest is done by talking to the registry directly, so we need to get a bearer token + ref, err := reference.ParseNamed(pkgJSON.SupportedTargets[0].DockerImageTag) + if err != nil { + return fmt.Errorf("failed to parse Docker image tag: %v", err) + } + + insecureSkipVerify := false + if strings.HasPrefix(reference.Domain(ref), "localhost") { + insecureSkipVerify = true + } + + dockerToken, err := getDockerToken(ctx, ref, pkgJSON.Version, username, password, insecureSkipVerify) + if err != nil { + return fmt.Errorf("failed to get bearer token: %v", err) + } + + if err := pushManifest(ctx, pkgJSON, dockerToken, insecureSkipVerify); err != nil { + return fmt.Errorf("failed to tag image: %v", err) + } + + return nil +}