feat: Upload projects (#1436)

`sqlc upload` will package up schema, queries and output code and upload it to sqlc Cloud. Uploaded projects are used to verify future sqlc releases; all uploaded code is tested for backwards-incompaitble changes.

* bundler: Add flag to dump request data

Add the `--dry-run` flag to see the exact HTTP request that sqlc will
make to the the upload endpoint.

* docs: Add documentation for the upload subcommand

docs/guides/privacy.md

@@ -49,7 +49,7 @@ We provide a few hosted services in addition to the sqlc command line tool.
 * Playground data stored in [Google Cloud Storage](https://cloud.google.com/storage)
   * Automatically deleted after 30 days
 
-### api.sqlc.dev
+### app.sqlc.dev / api.sqlc.dev
 
 * Hosted on [Heroku](https://heroku.com)
 * Error tracking and tracing with [Sentry](https://sentry.io)

docs/howto/upload.md

@@ -0,0 +1,57 @@
+# Uploading projects
+
+*This feature requires signing up for [sqlc Cloud](https://app.sqlc.dev), which is currently in beta.*
+
+Uploading your project ensures that future releases of sqlc do not break your
+existing code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
+project, uploaded projects are tested against development releases of sqlc to
+verify correctness.
+
+## Add configuration
+
+After creating a project, add the project ID to your sqlc configuration file.
+
+```yaml
+version: "1"
+project:
+  id: "<PROJECT-ID>"
+packages: []
+```
+
+```json
+{
+  "version": "1",
+  "project": {
+    "id": "<PROJECT-ID>"
+  },
+  "packages": [
+  ]
+}
+```
+
+You'll also need to create an API token and make it available via the
+`SQLC_AUTH_TOKEN` environment variable.
+
+```shell
+export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx
+```
+
+## Dry run
+
+You can see what's included when uploading your project by using using the `--dry-run` flag:
+
+```shell
+sqlc upload --dry-run
+```
+
+The output will be the exact HTTP request sent by `sqlc`.
+
+## Upload
+
+Once you're ready to upload, remove the `--dry-run` flag.
+
+```shell
+sqlc upload
+```
+
+By uploading your project, you're making sqlc more stable and reliable. Thanks!

docs/index.rst

@@ -53,6 +53,8 @@ code ever again.
    howto/ddl.md
    howto/structs.md
 
+   howto/upload.md
+
 .. toctree::
    :maxdepth: 2
    :caption: Reference

docs/reference/cli.md

@@ -6,13 +6,17 @@ Usage:
 
 Available Commands:
   compile     Statically check SQL for syntax and type errors
+  completion  Generate the autocompletion script for the specified shell
   generate    Generate Go code from SQL
   help        Help about any command
   init        Create an empty sqlc.yaml settings file
+  upload      Upload the schema, queries, and configuration for this project
   version     Print the sqlc version number
 
 Flags:
-  -h, --help   help for sqlc
+  -x, --experimental   enable experimental features (default: false)
+  -f, --file string    specify an alternate config file (default: sqlc.yaml)
+  -h, --help           help for sqlc
 
 Use "sqlc [command] --help" for more information about a command.
 ```

internal/bundler/metadata.go

@@ -0,0 +1,16 @@
+package bundler
+
+import (
+	"runtime"
+
+	"github.com/kyleconroy/sqlc/internal/info"
+)
+
+func projectMetadata() ([][2]string, error) {
+	return [][2]string{
+		{"sqlc_version", info.Version},
+		{"go_version", runtime.Version()},
+		{"goos", runtime.GOOS},
+		{"goarch", runtime.GOARCH},
+	}, nil
+}

internal/bundler/multipart.go

@@ -0,0 +1,80 @@
+package bundler
+
+import (
+	"io"
+	"mime/multipart"
+	"os"
+	"path/filepath"
+
+	"github.com/kyleconroy/sqlc/internal/config"
+	"github.com/kyleconroy/sqlc/internal/sql/sqlpath"
+)
+
+func writeInputs(w *multipart.Writer, file string, conf *config.Config) error {
+	refs := map[string]struct{}{}
+	refs[filepath.Base(file)] = struct{}{}
+
+	for _, pkg := range conf.SQL {
+		for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
+			files, err := sqlpath.Glob(paths)
+			if err != nil {
+				return err
+			}
+			for _, file := range files {
+				refs[file] = struct{}{}
+			}
+		}
+	}
+
+	for file, _ := range refs {
+		if err := addPart(w, file); err != nil {
+			return err
+		}
+	}
+
+	params, err := projectMetadata()
+	if err != nil {
+		return err
+	}
+	params = append(params, [2]string{"project_id", conf.Project.ID})
+	for _, val := range params {
+		if err = w.WriteField(val[0], val[1]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func addPart(w *multipart.Writer, file string) error {
+	h, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer h.Close()
+	part, err := w.CreateFormFile("inputs", file)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(part, h)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func writeOutputs(w *multipart.Writer, dir string, output map[string]string) error {
+	for filename, contents := range output {
+		rel, err := filepath.Rel(dir, filename)
+		if err != nil {
+			return err
+		}
+		part, err := w.CreateFormFile("outputs", rel)
+		if err != nil {
+			return err
+		}
+		if _, err := io.WriteString(part, contents); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/bundler/upload.go

@@ -0,0 +1,101 @@
+package bundler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httputil"
+	"os"
+
+	"github.com/kyleconroy/sqlc/internal/config"
+)
+
+type Uploader struct {
+	token      string
+	configPath string
+	config     *config.Config
+	dir        string
+}
+
+func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
+	return &Uploader{
+		token:      os.Getenv("SQLC_AUTH_TOKEN"),
+		configPath: configPath,
+		config:     conf,
+		dir:        dir,
+	}
+}
+
+func (up *Uploader) Validate() error {
+	if up.config.Project.ID == "" {
+		return fmt.Errorf("project.id is not set")
+	}
+	if up.token == "" {
+		return fmt.Errorf("SQLC_AUTH_TOKEN environment variable is not set")
+	}
+	return nil
+}
+
+func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*http.Request, error) {
+	body := bytes.NewBuffer([]byte{})
+
+	w := multipart.NewWriter(body)
+	defer w.Close()
+	if err := writeInputs(w, up.configPath, up.config); err != nil {
+		return nil, err
+	}
+	if err := writeOutputs(w, up.dir, result); err != nil {
+		return nil, err
+	}
+	w.Close()
+
+	req, err := http.NewRequest("POST", "https://api.sqlc.dev/upload", body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set sqlc-version header
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", up.token))
+	return req.WithContext(ctx), nil
+}
+
+func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error {
+	req, err := up.buildRequest(ctx, result)
+	if err != nil {
+		return err
+	}
+	dump, err := httputil.DumpRequest(req, true)
+	if err != nil {
+		return err
+	}
+	os.Stdout.Write(dump)
+	return nil
+}
+
+func (up *Uploader) Upload(ctx context.Context, result map[string]string) error {
+	if err := up.Validate(); err != nil {
+		return err
+	}
+	req, err := up.buildRequest(ctx, result)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		body, err := io.ReadAll(resp.Body)
+		defer resp.Body.Close()
+		if err != nil {
+			return fmt.Errorf("upload error: endpoint returned non-200 status code: %d", resp.StatusCode)
+		}
+		return fmt.Errorf("upload error: %d: %s", resp.StatusCode, string(body))
+	}
+	return nil
+}

internal/cmd/cmd.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/kyleconroy/sqlc/internal/config"
 	"github.com/kyleconroy/sqlc/internal/debug"
+	"github.com/kyleconroy/sqlc/internal/info"
 	"github.com/kyleconroy/sqlc/internal/tracer"
 )
 
@@ -28,6 +29,8 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
 	rootCmd.AddCommand(genCmd)
 	rootCmd.AddCommand(initCmd)
 	rootCmd.AddCommand(versionCmd)
+	uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)")
+	rootCmd.AddCommand(uploadCmd)
 
 	rootCmd.SetArgs(args)
 	rootCmd.SetIn(stdin)
@@ -60,9 +63,7 @@ var versionCmd = &cobra.Command{
 			defer trace.StartRegion(cmd.Context(), "version").End()
 		}
 		if version == "" {
-			// When no version is set, return the next bug fix version
-			// after the most recent tag
-			fmt.Printf("%s\n", "v1.12.0")
+			fmt.Printf("%s\n", info.Version)
 		} else {
 			fmt.Printf("%s\n", version)
 		}
@@ -96,11 +97,16 @@ var initCmd = &cobra.Command{
 
 type Env struct {
 	ExperimentalFeatures bool
+	DryRun               bool
 }
 
 func ParseEnv(c *cobra.Command) Env {
 	x := c.Flag("experimental")
-	return Env{ExperimentalFeatures: x != nil && x.Changed}
+	dr := c.Flag("dry-run")
+	return Env{
+		ExperimentalFeatures: x != nil && x.Changed,
+		DryRun:               dr != nil && dr.Changed,
+	}
 }
 
 func getConfigPath(stderr io.Writer, f *pflag.Flag) (string, string) {
@@ -152,6 +158,20 @@ var genCmd = &cobra.Command{
 	},
 }
 
+var uploadCmd = &cobra.Command{
+	Use:   "upload",
+	Short: "Upload the schema, queries, and configuration for this project",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		stderr := cmd.ErrOrStderr()
+		dir, name := getConfigPath(stderr, cmd.Flag("file"))
+		if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+			fmt.Fprintf(stderr, "error uploading: %s\n", err)
+			os.Exit(1)
+		}
+		return nil
+	},
+}
+
 var checkCmd = &cobra.Command{
 	Use:   "compile",
 	Short: "Statically check SQL for syntax and type errors",