feat(verify): Add new command to verify queries and migrations (#2986)

We've renamed the upload sub-command to push. Using upload is supported for now, but please switch over to using push. We changed the data sent along in a push request. Upload used to include the configuration file, migrations, queries, and all generated code. Push drops the generated code in favor of including the [plugin.GenerateRequest], which is the protobuf message we pass to codegen plugins.

We now pass along annotations for each push. By default, we include these environment variables if they are present:

GITHUB_REPOSITORY
GITHUB_REF
GITHUB_REF_NAME
GITHUB_REF_TYPE
GITHUB_SHA

Like upload, push should be run when you tag a release of your application. We run it on every push to main, as we continuously deploy those commits.
verify

The verify command, building on top of the push command, ensures migrations are safe to deploy. Verify sends your current schema and queries to sqlc cloud. There, we run the queries for your latest push against your new schema changes. This check catches backwards incompatible schema changes for existing queries.

Author: kyleconroy

buf.gen.yaml

@@ -1,6 +1,8 @@
 version: v1
 managed:
   enabled: true
+  go_package_prefix:
+    default: "github.com/sqlc-dev/sqlc/internal"
 plugins:
   - plugin: buf.build/protocolbuffers/go:v1.30.0
     out: internal

internal/bundler/multipart.go

@@ -4,52 +4,37 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/sqlc-dev/sqlc/internal/config"
 	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
 )
 
-func readInputs(file string, conf *config.Config) ([]*pb.File, error) {
-	refs := map[string]struct{}{}
-	refs[filepath.Base(file)] = struct{}{}
-
-	for _, pkg := range conf.SQL {
-		for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
-			files, err := sqlpath.Glob(paths)
-			if err != nil {
-				return nil, err
-			}
-			for _, file := range files {
-				refs[file] = struct{}{}
-			}
-		}
+func readFiles(dir string, paths []string) ([]*pb.File, error) {
+	files, err := sqlpath.Glob(paths)
+	if err != nil {
+		return nil, err
 	}
-
-	var files []*pb.File
-	for file, _ := range refs {
-		contents, err := os.ReadFile(file)
+	var out []*pb.File
+	for _, file := range files {
+		f, err := readFile(dir, file)
 		if err != nil {
 			return nil, err
 		}
-		files = append(files, &pb.File{
-			Name:     file,
-			Contents: contents,
-		})
+		out = append(out, f)
 	}
-	return files, nil
+	return out, nil
 }
 
-func readOutputs(dir string, output map[string]string) ([]*pb.File, error) {
-	var files []*pb.File
-	for filename, contents := range output {
-		rel, err := filepath.Rel(dir, filename)
-		if err != nil {
-			return nil, err
-		}
-		files = append(files, &pb.File{
-			Name:     rel,
-			Contents: []byte(contents),
-		})
+func readFile(dir string, path string) (*pb.File, error) {
+	rel, err := filepath.Rel(dir, path)
+	if err != nil {
+		return nil, err
+	}
+	blob, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
 	}
-	return files, nil
+	return &pb.File{
+		Name:     rel,
+		Contents: blob,
+	}, nil
 }

internal/bundler/upload.go

@@ -4,11 +4,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log/slog"
+	"os"
+	"strings"
 
-	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/sqlc-dev/sqlc/internal/config"
 	"github.com/sqlc-dev/sqlc/internal/info"
+	"github.com/sqlc-dev/sqlc/internal/plugin"
 	"github.com/sqlc-dev/sqlc/internal/quickdb"
 	pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
 )
@@ -33,6 +37,13 @@ type Uploader struct {
 	client     pb.QuickClient
 }
 
+type QuerySetArchive struct {
+	Name    string
+	Queries []string
+	Schema  []string
+	Request *plugin.GenerateRequest
+}
+
 func NewUploader(configPath, dir string, conf *config.Config) *Uploader {
 	return &Uploader{
 		configPath: configPath,
@@ -58,32 +69,86 @@ func (up *Uploader) Validate() error {
 	return nil
 }
 
-func (up *Uploader) buildRequest(ctx context.Context, result map[string]string) (*pb.UploadArchiveRequest, error) {
-	ins, err := readInputs(up.configPath, up.config)
-	if err != nil {
-		return nil, err
+var envvars = []string{
+	"GITHUB_REPOSITORY",
+	"GITHUB_REF",
+	"GITHUB_REF_NAME",
+	"GITHUB_REF_TYPE",
+	"GITHUB_SHA",
+}
+
+func annotate() map[string]string {
+	labels := map[string]string{}
+	for _, ev := range envvars {
+		key := strings.ReplaceAll(strings.ToLower(ev), "_", ".")
+		labels[key] = os.Getenv(ev)
 	}
-	outs, err := readOutputs(up.dir, result)
+	return labels
+}
+
+func BuildRequest(ctx context.Context, dir, configPath string, results []*QuerySetArchive) (*pb.UploadArchiveRequest, error) {
+	conf, err := readFile(dir, configPath)
 	if err != nil {
 		return nil, err
 	}
-	return &pb.UploadArchiveRequest{
+	res := &pb.UploadArchiveRequest{
 		SqlcVersion: info.Version,
-		Inputs:      ins,
-		Outputs:     outs,
-	}, nil
+		Config:      conf,
+		Annotations: annotate(),
+	}
+	for i, result := range results {
+		schema, err := readFiles(dir, result.Schema)
+		if err != nil {
+			return nil, err
+		}
+		queries, err := readFiles(dir, result.Queries)
+		if err != nil {
+			return nil, err
+		}
+		name := result.Name
+		if name == "" {
+			name = fmt.Sprintf("queryset_%d", i)
+		}
+		genreq, err := proto.Marshal(result.Request)
+		if err != nil {
+			return nil, err
+		}
+		res.QuerySets = append(res.QuerySets, &pb.QuerySet{
+			Name:    name,
+			Schema:  schema,
+			Queries: queries,
+			CodegenRequest: &pb.File{
+				Name:     "codegen_request.pb",
+				Contents: genreq,
+			},
+		})
+	}
+	return res, nil
 }
 
-func (up *Uploader) DumpRequestOut(ctx context.Context, result map[string]string) error {
+func (up *Uploader) buildRequest(ctx context.Context, results []*QuerySetArchive) (*pb.UploadArchiveRequest, error) {
+	return BuildRequest(ctx, up.dir, up.configPath, results)
+}
+
+func (up *Uploader) DumpRequestOut(ctx context.Context, result []*QuerySetArchive) error {
 	req, err := up.buildRequest(ctx, result)
 	if err != nil {
 		return err
 	}
-	fmt.Println(protojson.Format(req))
+	slog.Info("config", "file", req.Config.Name, "bytes", len(req.Config.Contents))
+	for _, qs := range req.QuerySets {
+		slog.Info("codegen_request", "queryset", qs.Name, "file", "codegen_request.pb")
+		for _, file := range qs.Schema {
+			slog.Info("schema", "queryset", qs.Name, "file", file.Name, "bytes", len(file.Contents))
+		}
+		for _, file := range qs.Queries {
+			slog.Info("query", "queryset", qs.Name, "file", file.Name, "bytes", len(file.Contents))
+		}
+	}
 	return nil
 }
 
-func (up *Uploader) Upload(ctx context.Context, result map[string]string) error {
+func (up *Uploader) Upload(ctx context.Context, result []*QuerySetArchive) error {
 	if err := up.Validate(); err != nil {
 		return err
 	}

internal/cmd/cmd.go

@@ -26,7 +26,7 @@ import (
 
 func init() {
 	createDBCmd.Flags().StringP("queryset", "", "", "name of the queryset to use")
-	uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)")
+	pushCmd.Flags().BoolP("dry-run", "", false, "dump push request (default: false)")
 	initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file")
 	initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file")
 	initCmd.MarkFlagsMutuallyExclusive("v1", "v2")
@@ -46,7 +46,8 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
 	rootCmd.AddCommand(genCmd)
 	rootCmd.AddCommand(initCmd)
 	rootCmd.AddCommand(versionCmd)
-	rootCmd.AddCommand(uploadCmd)
+	rootCmd.AddCommand(verifyCmd)
+	rootCmd.AddCommand(pushCmd)
 	rootCmd.AddCommand(NewCmdVet())
 
 	rootCmd.SetArgs(args)
@@ -214,24 +215,6 @@ var genCmd = &cobra.Command{
 	},
 }
 
-var uploadCmd = &cobra.Command{
-	Use:   "upload",
-	Short: "Upload the schema, queries, and configuration for this project",
-	RunE: func(cmd *cobra.Command, args []string) error {
-		stderr := cmd.ErrOrStderr()
-		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		opts := &Options{
-			Env:    ParseEnv(cmd),
-			Stderr: stderr,
-		}
-		if err := createPkg(cmd.Context(), dir, name, opts); err != nil {
-			fmt.Fprintf(stderr, "error uploading: %s\n", err)
-			os.Exit(1)
-		}
-		return nil
-	},
-}
-
 var checkCmd = &cobra.Command{
 	Use:   "compile",
 	Short: "Statically check SQL for syntax and type errors",