feat(upload): Point upload command at new endpoint (#2772)

* feat(upload): Point upload command at new endpoint

* fix(config): Remove unused project field

---------

Co-authored-by: Andrew Benton <andrew@sqlc.dev>

Author: kyleconroy

docs/howto/upload.md

@@ -1,35 +1,26 @@
 # Uploading projects
 
-*This feature requires signing up for [sqlc Cloud](https://app.sqlc.dev), which is currently in beta.*
+*Added in v1.22.0*
 
-Uploading your project ensures that future releases of sqlc do not break your
-existing code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
-project, uploaded projects are tested against development releases of sqlc to
+Uploading an archive of your project ensures that future releases of sqlc do not
+break your code. Similar to Rust's [crater](https://github.com/rust-lang/crater)
+project, uploaded archives are tested against development releases of sqlc to
 verify correctness.
 
+Interested in uploading projects? Sign up [here](https://docs.google.com/forms/d/e/1FAIpQLSdxoMzJ7rKkBpuez-KyBcPNyckYV-5iMR--FRB7WnhvAmEvKg/viewform) or send us an email
+at [hello@sqlc.dev](mailto:hello@sqlc.dev).
+
 ## Add configuration
 
 After creating a project, add the project ID to your sqlc configuration file.
 
 ```yaml
-version: "1"
-project:
-  id: "<PROJECT-ID>"
-packages: []
-```
-
-```json
-{
-  "version": "1",
-  "project": {
-    "id": "<PROJECT-ID>"
-  },
-  "packages": [
-  ]
-}
+version: "2"
+cloud:
+  project: "<PROJECT-ID>"
 ```
 
-You'll also need to create an API token and make it available via the
+You'll also need to create an auth token and make it available via the
 `SQLC_AUTH_TOKEN` environment variable.
 
 ```shell
@@ -38,13 +29,14 @@ export SQLC_AUTH_TOKEN=sqlc_xxxxxxxx
 
 ## Dry run
 
-You can see what's included when uploading your project by using using the `--dry-run` flag:
+You can see what's included when uploading your project by using using the
+`--dry-run` flag:
 
 ```shell
 sqlc upload --dry-run
 ```
 
-The output will be the exact HTTP request sent by `sqlc`.
+The output is the request `sqlc` would have sent without the `--dry-run` flag.
 
 ## Upload
 
@@ -54,4 +46,4 @@ Once you're ready to upload, remove the `--dry-run` flag.
 sqlc upload
 ```
 
-By uploading your project, you're making sqlc more stable and reliable. Thanks!
+By uploading your project, you're making sqlc more stable and reliable. Thanks!

internal/bundler/metadata.go

@@ -1,80 +1,55 @@
 package bundler
 
 import (
-	"io"
-	"mime/multipart"
 	"os"
 	"path/filepath"
 
 	"github.com/sqlc-dev/sqlc/internal/config"
+	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 )
 
-func writeInputs(w *multipart.Writer, file string, conf *config.Config) error {
+func readInputs(file string, conf *config.Config) ([]*pb.File, error) {
 	refs := map[string]struct{}{}
 	refs[filepath.Base(file)] = struct{}{}
 
 	for _, pkg := range conf.SQL {
 		for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
 			files, err := sqlpath.Glob(paths)
 			if err != nil {
-				return err
+				return nil, err
 			}
 			for _, file := range files {
 				refs[file] = struct{}{}
 			}
 		}
 	}
 
+	var files []*pb.File
 	for file, _ := range refs {
-		if err := addPart(w, file); err != nil {
-			return err
-		}
-	}
-
-	params, err := projectMetadata()
-	if err != nil {
-		return err
-	}
-	params = append(params, [2]string{"project_id", conf.Project.ID})
-	for _, val := range params {
-		if err = w.WriteField(val[0], val[1]); err != nil {
-			return err
+		contents, err := os.ReadFile(file)
+		if err != nil {
+			return nil, err
 		}
+		files = append(files, &pb.File{
+			Name:     file,
+			Contents: contents,
+		})
 	}
-	return nil
+	return files, nil
 }
 
-func addPart(w *multipart.Writer, file string) error {
-	h, err := os.Open(file)
-	if err != nil {
-		return err
-	}
-	defer h.Close()
-	part, err := w.CreateFormFile("inputs", file)
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(part, h)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func writeOutputs(w *multipart.Writer, dir string, output map[string]string) error {
+func readOutputs(dir string, output map[string]string) ([]*pb.File, error) {
+	var files []*pb.File
 	for filename, contents := range output {
 		rel, err := filepath.Rel(dir, filename)
 		if err != nil {
-			return err
-		}
-		part, err := w.CreateFormFile("outputs", rel)
-		if err != nil {
-			return err
-		}
-		if _, err := io.WriteString(part, contents); err != nil {
-			return err
+			return nil, err
 		}
+		files = append(files, &pb.File{
+			Name:     rel,
+			Contents: []byte(contents),
+		})
 	}
-	return nil
+	return files, nil
 }

internal/bundler/multipart.go

@@ -1,23 +1,24 @@
 package bundler
 
 import (
-	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/http/httputil"
 	"os"
 
+	"google.golang.org/protobuf/encoding/protojson"
+
 	"github.com/sqlc-dev/sqlc/internal/config"
+	"github.com/sqlc-dev/sqlc/internal/info"
+	"github.com/sqlc-dev/sqlc/internal/quickdb"
+	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 )
 
 type Uploader struct {
 	token      string
 	configPath string
 	config     *config.Config
 	dir        string
+	client     pb.QuickClient
 }
 
 func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
@@ -30,47 +31,44 @@ func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
 }
 
 func (up *Uploader) Validate() error {
-	if up.config.Project.ID == "" {
-		return fmt.Errorf("project.id is not set")
+	if up.config.Cloud.Project == "" {
+		return fmt.Errorf("cloud.project is not set")
 	}
 	if up.token == "" {
 		return fmt.Errorf("SQLC_AUTH_TOKEN environment variable is not set")
 	}
+	if up.client == nil {
+		client, err := quickdb.NewClientFromConfig(up.config.Cloud)
+		if err != nil {
+			return fmt.Errorf("client init failed: %w", err)
+		}
+		up.client = client
+	}
 	return nil
 }
 
-func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*http.Request, error) {
-	body := bytes.NewBuffer([]byte{})
-	w := multipart.NewWriter(body)
-	if err := writeInputs(w, up.configPath, up.config); err != nil {
-		return nil, err
-	}
-	if err := writeOutputs(w, up.dir, result); err != nil {
-		return nil, err
-	}
-	if err := w.Close(); err != nil {
+func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*pb.UploadArchiveRequest, error) {
+	ins, err := readInputs(up.configPath, up.config)
+	if err != nil {
 		return nil, err
 	}
-	req, err := http.NewRequest("POST", "https://api.sqlc.dev/upload", body)
+	outs, err := readOutputs(up.dir, result)
 	if err != nil {
 		return nil, err
 	}
-	// Set sqlc-version header
-	req.Header.Set("Content-Type", w.FormDataContentType())
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", up.token))
-	return req.WithContext(ctx), nil
+	return &pb.UploadArchiveRequest{
+		SqlcVersion: info.Version,
+		Inputs:      ins,
+		Outputs:     outs,
+	}, nil
 }
 
 func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error {
 	req, err := up.buildRequest(ctx, result)
 	if err != nil {
 		return err
 	}
-	dump, err := httputil.DumpRequest(req, true)
-	if err != nil {
-		return err
-	}
-	os.Stdout.Write(dump)
+	fmt.Println(protojson.Format(req))
 	return nil
 }
 
@@ -82,18 +80,8 @@ func (up *Uploader) Upload(ctx context.Context, result map[string]string) error
 	if err != nil {
 		return err
 	}
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode >= 400 {
-		body, err := io.ReadAll(resp.Body)
-		defer resp.Body.Close()
-		if err != nil {
-			return fmt.Errorf("upload error: endpoint returned non-200 status code: %d", resp.StatusCode)
-		}
-		return fmt.Errorf("upload error: %d: %s", resp.StatusCode, string(body))
+	if _, err := up.client.UploadArchive(ctx, req); err != nil {
+		return fmt.Errorf("upload error: %w", err)
 	}
 	return nil
 }

internal/bundler/upload.go

@@ -56,18 +56,13 @@ const (
 
 type Config struct {
 	Version string   `json:"version" yaml:"version"`
-	Project Project  `json:"project" yaml:"project"`
 	Cloud   Cloud    `json:"cloud" yaml:"cloud"`
 	SQL     []SQL    `json:"sql" yaml:"sql"`
 	Gen     Gen      `json:"overrides,omitempty" yaml:"overrides"`
 	Plugins []Plugin `json:"plugins" yaml:"plugins"`
 	Rules   []Rule   `json:"rules" yaml:"rules"`
 }
 
-type Project struct {
-	ID string `json:"id" yaml:"id"`
-}
-
 type Database struct {
 	URI     string `json:"uri" yaml:"uri"`
 	Managed bool   `json:"managed" yaml:"managed"`

internal/config/config.go

@@ -11,7 +11,6 @@ import (
 type V1GenerateSettings struct {
 	Version   string              `json:"version" yaml:"version"`
 	Cloud     Cloud               `json:"cloud" yaml:"cloud"`
-	Project   Project             `json:"project" yaml:"project"`
 	Packages  []v1PackageSettings `json:"packages" yaml:"packages"`
 	Overrides []Override          `json:"overrides,omitempty" yaml:"overrides,omitempty"`
 	Rename    map[string]string   `json:"rename,omitempty" yaml:"rename,omitempty"`
@@ -132,7 +131,6 @@ func (c *V1GenerateSettings) ValidateGlobalOverrides() error {
 func (c *V1GenerateSettings) Translate() Config {
 	conf := Config{
 		Version: c.Version,
-		Project: c.Project,
 		Cloud:   c.Cloud,
 		Rules:   c.Rules,
 	}

internal/config/v_one.go

@@ -8,14 +8,6 @@
         "version": {
             "const": "1"
         },
-        "project": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "type": "string"
-                }
-            }
-        },
         "cloud": {
             "type": "object",
             "properties": {

internal/config/v_one.json

@@ -8,14 +8,6 @@
         "version": {
             "const": "2"
         },
-        "project": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "type": "string"
-                }
-            }
-        },
         "cloud": {
             "type": "object",
             "properties": {