From bde62813499765902f18f6264303b57b8d1c5e97 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Fri, 18 Mar 2022 18:50:29 -0700 Subject: [PATCH] feat: Upload projects (#1436) `sqlc upload` will package up schema, queries and output code and upload it to sqlc Cloud. Uploaded projects are used to verify future sqlc releases; all uploaded code is tested for backwards-incompaitble changes. * bundler: Add flag to dump request data Add the `--dry-run` flag to see the exact HTTP request that sqlc will make to the the upload endpoint. * docs: Add documentation for the upload subcommand --- docs/guides/privacy.md | 2 +- docs/howto/upload.md | 57 +++++++++++++++++++ docs/index.rst | 2 + docs/reference/cli.md | 6 +- internal/bundler/metadata.go | 16 ++++++ internal/bundler/multipart.go | 80 ++++++++++++++++++++++++++ internal/bundler/upload.go | 101 +++++++++++++++++++++++++++++++++ internal/cmd/cmd.go | 28 +++++++-- internal/cmd/generate.go | 20 +++++-- internal/cmd/package.go | 29 ++++++++++ internal/config/config.go | 13 +++-- internal/config/config_test.go | 2 +- internal/config/v_one.go | 2 + internal/info/facts.go | 5 ++ 14 files changed, 347 insertions(+), 16 deletions(-) create mode 100644 docs/howto/upload.md create mode 100644 internal/bundler/metadata.go create mode 100644 internal/bundler/multipart.go create mode 100644 internal/bundler/upload.go create mode 100644 internal/cmd/package.go create mode 100644 internal/info/facts.go diff --git a/docs/guides/privacy.md b/docs/guides/privacy.md index 1422fea999..da29fd8853 100644 --- a/docs/guides/privacy.md +++ b/docs/guides/privacy.md @@ -49,7 +49,7 @@ We provide a few hosted services in addition to the sqlc command line tool. * Playground data stored in [Google Cloud Storage](https://cloud.google.com/storage) * Automatically deleted after 30 days -### api.sqlc.dev +### app.sqlc.dev / api.sqlc.dev * Hosted on [Heroku](https://heroku.com) * Error tracking and tracing with [Sentry](https://sentry.io) diff --git a/docs/howto/upload.md b/docs/howto/upload.md new file mode 100644 index 0000000000..0add35c942 --- /dev/null +++ b/docs/howto/upload.md @@ -0,0 +1,57 @@ +# Uploading projects + +*This feature requires signing up for [sqlc Cloud](https://app.sqlc.dev), which is currently in beta.* + +Uploading your project ensures that future releases of sqlc do not break your +existing code. Similar to Rust's [crater](https://github.com/rust-lang/crater) +project, uploaded projects are tested against development releases of sqlc to +verify correctness. + +## Add configuration + +After creating a project, add the project ID to your sqlc configuration file. + +```yaml +version: "1" +project: + id: "" +packages: [] +``` + +```json +{ + "version": "1", + "project": { + "id": "" + }, + "packages": [ + ] +} +``` + +You'll also need to create an API token and make it available via the +`SQLC_AUTH_TOKEN` environment variable. + +```shell +export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx +``` + +## Dry run + +You can see what's included when uploading your project by using using the `--dry-run` flag: + +```shell +sqlc upload --dry-run +``` + +The output will be the exact HTTP request sent by `sqlc`. + +## Upload + +Once you're ready to upload, remove the `--dry-run` flag. + +```shell +sqlc upload +``` + +By uploading your project, you're making sqlc more stable and reliable. Thanks! diff --git a/docs/index.rst b/docs/index.rst index e0f39532fe..91b68c02f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,6 +53,8 @@ code ever again. howto/ddl.md howto/structs.md + howto/upload.md + .. toctree:: :maxdepth: 2 :caption: Reference diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ec55f3b9e1..a6033f2205 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -6,13 +6,17 @@ Usage: Available Commands: compile Statically check SQL for syntax and type errors + completion Generate the autocompletion script for the specified shell generate Generate Go code from SQL help Help about any command init Create an empty sqlc.yaml settings file + upload Upload the schema, queries, and configuration for this project version Print the sqlc version number Flags: - -h, --help help for sqlc + -x, --experimental enable experimental features (default: false) + -f, --file string specify an alternate config file (default: sqlc.yaml) + -h, --help help for sqlc Use "sqlc [command] --help" for more information about a command. ``` diff --git a/internal/bundler/metadata.go b/internal/bundler/metadata.go new file mode 100644 index 0000000000..415b131ef0 --- /dev/null +++ b/internal/bundler/metadata.go @@ -0,0 +1,16 @@ +package bundler + +import ( + "runtime" + + "github.com/kyleconroy/sqlc/internal/info" +) + +func projectMetadata() ([][2]string, error) { + return [][2]string{ + {"sqlc_version", info.Version}, + {"go_version", runtime.Version()}, + {"goos", runtime.GOOS}, + {"goarch", runtime.GOARCH}, + }, nil +} diff --git a/internal/bundler/multipart.go b/internal/bundler/multipart.go new file mode 100644 index 0000000000..a755bbfcfb --- /dev/null +++ b/internal/bundler/multipart.go @@ -0,0 +1,80 @@ +package bundler + +import ( + "io" + "mime/multipart" + "os" + "path/filepath" + + "github.com/kyleconroy/sqlc/internal/config" + "github.com/kyleconroy/sqlc/internal/sql/sqlpath" +) + +func writeInputs(w *multipart.Writer, file string, conf *config.Config) error { + refs := map[string]struct{}{} + refs[filepath.Base(file)] = struct{}{} + + for _, pkg := range conf.SQL { + for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} { + files, err := sqlpath.Glob(paths) + if err != nil { + return err + } + for _, file := range files { + refs[file] = struct{}{} + } + } + } + + for file, _ := range refs { + if err := addPart(w, file); err != nil { + return err + } + } + + params, err := projectMetadata() + if err != nil { + return err + } + params = append(params, [2]string{"project_id", conf.Project.ID}) + for _, val := range params { + if err = w.WriteField(val[0], val[1]); err != nil { + return err + } + } + return nil +} + +func addPart(w *multipart.Writer, file string) error { + h, err := os.Open(file) + if err != nil { + return err + } + defer h.Close() + part, err := w.CreateFormFile("inputs", file) + if err != nil { + return err + } + _, err = io.Copy(part, h) + if err != nil { + return err + } + return nil +} + +func writeOutputs(w *multipart.Writer, dir string, output map[string]string) error { + for filename, contents := range output { + rel, err := filepath.Rel(dir, filename) + if err != nil { + return err + } + part, err := w.CreateFormFile("outputs", rel) + if err != nil { + return err + } + if _, err := io.WriteString(part, contents); err != nil { + return err + } + } + return nil +} diff --git a/internal/bundler/upload.go b/internal/bundler/upload.go new file mode 100644 index 0000000000..8f09f8562c --- /dev/null +++ b/internal/bundler/upload.go @@ -0,0 +1,101 @@ +package bundler + +import ( + "bytes" + "context" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/httputil" + "os" + + "github.com/kyleconroy/sqlc/internal/config" +) + +type Uploader struct { + token string + configPath string + config *config.Config + dir string +} + +func NewUploader(configPath, dir string, conf *config.Config) *Uploader { + return &Uploader{ + token: os.Getenv("SQLC_AUTH_TOKEN"), + configPath: configPath, + config: conf, + dir: dir, + } +} + +func (up *Uploader) Validate() error { + if up.config.Project.ID == "" { + return fmt.Errorf("project.id is not set") + } + if up.token == "" { + return fmt.Errorf("SQLC_AUTH_TOKEN environment variable is not set") + } + return nil +} + +func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*http.Request, error) { + body := bytes.NewBuffer([]byte{}) + + w := multipart.NewWriter(body) + defer w.Close() + if err := writeInputs(w, up.configPath, up.config); err != nil { + return nil, err + } + if err := writeOutputs(w, up.dir, result); err != nil { + return nil, err + } + w.Close() + + req, err := http.NewRequest("POST", "https://api.sqlc.dev/upload", body) + if err != nil { + return nil, err + } + + // Set sqlc-version header + req.Header.Set("Content-Type", w.FormDataContentType()) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", up.token)) + return req.WithContext(ctx), nil +} + +func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error { + req, err := up.buildRequest(ctx, result) + if err != nil { + return err + } + dump, err := httputil.DumpRequest(req, true) + if err != nil { + return err + } + os.Stdout.Write(dump) + return nil +} + +func (up *Uploader) Upload(ctx context.Context, result map[string]string) error { + if err := up.Validate(); err != nil { + return err + } + req, err := up.buildRequest(ctx, result) + if err != nil { + return err + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + if resp.StatusCode >= 400 { + body, err := io.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return fmt.Errorf("upload error: endpoint returned non-200 status code: %d", resp.StatusCode) + } + return fmt.Errorf("upload error: %d: %s", resp.StatusCode, string(body)) + } + return nil +} diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 5cefe6a645..4c36e96b3e 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -15,6 +15,7 @@ import ( "github.com/kyleconroy/sqlc/internal/config" "github.com/kyleconroy/sqlc/internal/debug" + "github.com/kyleconroy/sqlc/internal/info" "github.com/kyleconroy/sqlc/internal/tracer" ) @@ -28,6 +29,8 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int rootCmd.AddCommand(genCmd) rootCmd.AddCommand(initCmd) rootCmd.AddCommand(versionCmd) + uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)") + rootCmd.AddCommand(uploadCmd) rootCmd.SetArgs(args) rootCmd.SetIn(stdin) @@ -60,9 +63,7 @@ var versionCmd = &cobra.Command{ defer trace.StartRegion(cmd.Context(), "version").End() } if version == "" { - // When no version is set, return the next bug fix version - // after the most recent tag - fmt.Printf("%s\n", "v1.12.0") + fmt.Printf("%s\n", info.Version) } else { fmt.Printf("%s\n", version) } @@ -96,11 +97,16 @@ var initCmd = &cobra.Command{ type Env struct { ExperimentalFeatures bool + DryRun bool } func ParseEnv(c *cobra.Command) Env { x := c.Flag("experimental") - return Env{ExperimentalFeatures: x != nil && x.Changed} + dr := c.Flag("dry-run") + return Env{ + ExperimentalFeatures: x != nil && x.Changed, + DryRun: dr != nil && dr.Changed, + } } func getConfigPath(stderr io.Writer, f *pflag.Flag) (string, string) { @@ -152,6 +158,20 @@ var genCmd = &cobra.Command{ }, } +var uploadCmd = &cobra.Command{ + Use: "upload", + Short: "Upload the schema, queries, and configuration for this project", + RunE: func(cmd *cobra.Command, args []string) error { + stderr := cmd.ErrOrStderr() + dir, name := getConfigPath(stderr, cmd.Flag("file")) + if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil { + fmt.Fprintf(stderr, "error uploading: %s\n", err) + os.Exit(1) + } + return nil + }, +} + var checkCmd = &cobra.Command{ Use: "compile", Short: "Statically check SQL for syntax and type errors", diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index c25d37bc0b..9d53cd4f99 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -47,7 +47,7 @@ type outPair struct { config.SQL } -func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) { +func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config, error) { configPath := "" if filename != "" { configPath = filepath.Join(dir, filename) @@ -65,12 +65,12 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer if yamlMissing && jsonMissing { fmt.Fprintln(stderr, "error parsing configuration files. sqlc.yaml or sqlc.json: file does not exist") - return nil, errors.New("config file missing") + return "", nil, errors.New("config file missing") } if !yamlMissing && !jsonMissing { fmt.Fprintln(stderr, "error: both sqlc.json and sqlc.yaml files present") - return nil, errors.New("sqlc.json and sqlc.yaml present") + return "", nil, errors.New("sqlc.json and sqlc.yaml present") } configPath = yamlPath @@ -83,7 +83,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer blob, err := os.ReadFile(configPath) if err != nil { fmt.Fprintf(stderr, "error parsing %s: file does not exist\n", base) - return nil, err + return "", nil, err } conf, err := config.ParseConfig(bytes.NewReader(blob)) @@ -97,9 +97,19 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer fmt.Fprintf(stderr, errMessageNoPackages) } fmt.Fprintf(stderr, "error parsing %s: %s\n", base, err) + return "", nil, err + } + + return configPath, &conf, nil +} + +func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) { + configPath, conf, err := readConfig(stderr, dir, filename) + if err != nil { return nil, err } + base := filepath.Base(configPath) if err := config.Validate(conf); err != nil { fmt.Fprintf(stderr, "error validating %s: %s\n", base, err) return nil, err @@ -135,7 +145,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer } for _, sql := range pairs { - combo := config.Combine(conf, sql.SQL) + combo := config.Combine(*conf, sql.SQL) // TODO: This feels like a hack that will bite us later joined := make([]string, 0, len(sql.Schema)) diff --git a/internal/cmd/package.go b/internal/cmd/package.go new file mode 100644 index 0000000000..63a4e00501 --- /dev/null +++ b/internal/cmd/package.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "context" + "io" + "os" + + "github.com/kyleconroy/sqlc/internal/bundler" +) + +func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error { + configPath, conf, err := readConfig(stderr, dir, filename) + if err != nil { + return err + } + up := bundler.NewUploader(configPath, dir, conf) + if err := up.Validate(); err != nil { + return err + } + output, err := Generate(ctx, e, dir, filename, stderr) + if err != nil { + os.Exit(1) + } + if e.DryRun { + return up.DumpRequestOut(ctx, output) + } else { + return up.Upload(ctx, output) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 66ba3f261d..0cc641925d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -78,9 +78,14 @@ const ( ) type Config struct { - Version string `json:"version" yaml:"version"` - SQL []SQL `json:"sql" yaml:"sql"` - Gen Gen `json:"overrides,omitempty" yaml:"overrides"` + Version string `json:"version" yaml:"version"` + Project Project `json:"project" yaml:"project"` + SQL []SQL `json:"sql" yaml:"sql"` + Gen Gen `json:"overrides,omitempty" yaml:"overrides"` +} + +type Project struct { + ID string `json:"id" yaml:"id"` } type Gen struct { @@ -327,7 +332,7 @@ func ParseConfig(rd io.Reader) (Config, error) { } } -func Validate(c Config) error { +func Validate(c *Config) error { for _, sql := range c.SQL { sqlGo := sql.Gen.Go if sqlGo == nil { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9250629c2e..183549ea51 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -65,7 +65,7 @@ func TestBadConfigs(t *testing.T) { } func TestInvalidConfig(t *testing.T) { - err := Validate(Config{ + err := Validate(&Config{ SQL: []SQL{{ Gen: SQLGen{ Go: &SQLGo{ diff --git a/internal/config/v_one.go b/internal/config/v_one.go index d1ad4fc117..5b357255bc 100644 --- a/internal/config/v_one.go +++ b/internal/config/v_one.go @@ -10,6 +10,7 @@ import ( type V1GenerateSettings struct { Version string `json:"version" yaml:"version"` + Project Project `json:"project" yaml:"project"` Packages []v1PackageSettings `json:"packages" yaml:"packages"` Overrides []Override `json:"overrides,omitempty" yaml:"overrides,omitempty"` Rename map[string]string `json:"rename,omitempty" yaml:"rename,omitempty"` @@ -105,6 +106,7 @@ func (c *V1GenerateSettings) ValidateGlobalOverrides() error { func (c *V1GenerateSettings) Translate() Config { conf := Config{ Version: c.Version, + Project: c.Project, } for _, pkg := range c.Packages { diff --git a/internal/info/facts.go b/internal/info/facts.go new file mode 100644 index 0000000000..3c4b478493 --- /dev/null +++ b/internal/info/facts.go @@ -0,0 +1,5 @@ +package info + +// When no version is set, return the next bug fix version +// after the most recent tag +const Version = "v1.12.0"