diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index 51bf492..5c94ce7 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -14,10 +14,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: '1.20' - run: sudo apt-get install jq if: matrix.os == 'ubuntu-latest' @@ -37,8 +33,43 @@ jobs: - name: Run tests run: | mkdir /tmp/tigris_cli_coverdata - GOCOVERDIR=/tmp/tigris_cli_coverdata/ BUILD_PARAM=-cover TIGRIS_CLI_TEST_FAST=1 make test + SUDO=sudo GOCOVERDIR=/tmp/tigris_cli_coverdata/ BUILD_PARAM=-cover TIGRIS_CLI_TEST_FAST=1 make test go tool covdata textfmt -i=/tmp/tigris_cli_coverdata/ -o coverage.out - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 + + # FIXME: same as above, with TIGRIS_CLI_TEST_FAST=1 removed. + # Work on unifying. + test-all: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, macOS-latest ] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - run: sudo apt-get install jq + if: matrix.os == 'ubuntu-latest' + + - if: matrix.os == 'macOS-latest' + run: | + brew install jq colima docker docker-compose tree + colima start + mkdir -p ~/.docker/cli-plugins + ln -sfn $(brew --prefix)/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose + + - run: chocolatey install jq + if: matrix.os == 'windows-latest' + + - run: npm install -g @go-task/cli + + - name: Run tests + run: | + mkdir /tmp/tigris_cli_coverdata_all + SUDO=sudo GOCOVERDIR=/tmp/tigris_cli_coverdata_all/ BUILD_PARAM=-cover make test + go tool covdata textfmt -i=/tmp/tigris_cli_coverdata_all/ -o coverage.out + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 diff --git a/.github/workflows/test-pkg-install.yaml b/.github/workflows/test-pkg-install.yaml index e2ddde9..1dbd4f6 100644 --- a/.github/workflows/test-pkg-install.yaml +++ b/.github/workflows/test-pkg-install.yaml @@ -25,4 +25,4 @@ jobs: - name: Windows run: sh scripts/test_pkg_install.sh - if: false || matrix.os == 'windows-latest' + if: matrix.os == 'windows-latest' diff --git a/cmd/local.go b/cmd/local.go index 8b20882..4c2fd4f 100644 --- a/cmd/local.go +++ b/cmd/local.go @@ -16,14 +16,15 @@ package cmd import ( "context" - "fmt" "io" "net" "os" + "strings" "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/stdcopy" @@ -37,7 +38,8 @@ import ( ) const ( - ImagePath = "tigrisdata/tigris-local" + ImagePath = "tigrisdata/tigris-local" + ContainerName = "tigris-local-server" ) @@ -48,7 +50,12 @@ var ( loginParam bool waitUpTimeout = 30 * time.Second - pingSleepTimeout = 10 * time.Millisecond + pingSleepTimeout = 20 * time.Millisecond + + skipAuth bool + tokenAdminAuth bool = util.IsWindows() + + defaultDataDir = "/var/lib/tigris/" ) func getClient(ctx context.Context) *client.Client { @@ -126,12 +133,14 @@ func pullImage(cli *client.Client, image string, port string) { _ = reader.Close() } -func startContainer(cli *client.Client, cname string, image string, port string, env []string) string { +func startContainer(cli *client.Client, cname string, image string, port string, dataDir string, env []string) string { ctx := context.Background() log.Debug().Msg("starting local instance") - pullImage(cli, image, port) + if strings.Contains(image, "/") { + pullImage(cli, image, port) + } pm := nat.PortMap{} @@ -148,6 +157,18 @@ func startContainer(cli *client.Client, cname string, image string, port string, log.Debug().Msg("creating container") + var mounts []mount.Mount + + if dataDir != "" { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeBind, + Source: dataDir, + Target: "/var/lib/tigris", + }) + + log.Debug().Str("host_dir", dataDir).Str("docker_dir", "/var/lib/tigris").Msg("adding bind mount") + } + resp, err := cli.ContainerCreate(ctx, &container.Config{ Hostname: cname, @@ -157,6 +178,7 @@ func startContainer(cli *client.Client, cname string, image string, port string, }, &container.HostConfig{ PortBindings: pm, + Mounts: mounts, }, nil, nil, cname) if err != nil { util.Fatal(err, "error creating container docker image: %s", image) @@ -173,19 +195,22 @@ func startContainer(cli *client.Client, cname string, image string, port string, return resp.ID } -func waitServerUp(port string) { - log.Debug().Msg("waiting local instance to start") +func waitServerUp(url string, waitAuth bool) { + log.Debug().Str("url", url).Bool("waitAuth", waitAuth).Msg("waiting local instance to start") cfg := config.DefaultConfig - cfg.URL = fmt.Sprintf("localhost:%s", port) - cfg.Token = "" - cfg.ClientSecret = "" - cfg.ClientID = "" + cfg.URL = url + + if !waitAuth { + cfg.Token = "" + cfg.ClientSecret = "" + cfg.ClientID = "" + } err := tclient.Init(&cfg) util.Fatal(err, "init tigris client") - if err = pingLow(context.Background(), waitUpTimeout, pingSleepTimeout, true, true, + if err = pingLow(context.Background(), waitUpTimeout, pingSleepTimeout, true, waitAuth, util.IsTTY(os.Stdout) && !util.Quiet); err != nil { util.Fatal(err, "tigris initialization failed") } @@ -193,54 +218,190 @@ func waitServerUp(port string) { log.Debug().Msg("wait finished successfully") } -var serverUpCmd = &cobra.Command{ - Use: "start [port] [version]", - Aliases: []string{"up"}, - Short: "Starts an instance of Tigris for local development", - Run: func(cmd *cobra.Command, args []string) { - cli := getClient(cmd.Context()) +func configureEnv(initialized bool) []string { + var env []string + + if !initialized { + env = append(env, "TIGRIS_LOCAL_PERSISTENCE=1") + + cfg := &config.DefaultConfig - port := "8081" - if len(args) > 0 { - port = args[0] + if cfg.DataDir == "" { + cfg.DataDir = defaultDataDir } - rport := port + ":8081" - - if len(args) > 1 { - t := args[1] - if t[0] == 'v' { - t = t[1:] - } - ImageTag = t + + err := os.MkdirAll(cfg.DataDir, 0o700) + + util.Fatal(err, "ensuring data directory") + } + + if skipAuth { + env = append(env, "TIGRIS_SKIP_LOCAL_AUTH=1") + } else if !initialized { + env = append(env, "TIGRIS_BOOTSTRAP_LOCAL_AUTH=1") + if tokenAdminAuth { + env = append(env, "TIGRIS_LOCAL_GENERATE_ADMIN_TOKEN=1") } + } - stopContainer(cli, ContainerName) - _ = startContainer(cli, ContainerName, ImagePath+":"+ImageTag, rport, nil) + return env +} + +func configureLocal(local bool) ([]string, bool) { + var ( + env []string + initialized bool + ) + + dataDir := config.DefaultConfig.DataDir + + if dataDir != "" { + _, err := os.Stat(dataDir + "/initialized") + initialized = err == nil + + if initialized { + log.Debug().Msg("data directory not empty, skip bootstrap") + } + } - waitServerUp(port) + if local { + env = configureEnv(initialized) + } else { + if initialized { + util.Stdoutf("Warning: Local Tigris instance is initialized in provided data directory: %s"+ + ", but start of ephemeral instance requested.\n Did you mean `tigris local up ...` instead?\n", dataDir) + util.Stdoutf("Remove --data-dir parameter to avoid this warning when starting ephemeral instance.\n") + } - util.Stdoutf("Tigris is running at localhost:%s\n", port) + config.DefaultConfig.DataDir = "" + } - ctx, cancel := util.GetContext(cmd.Context()) - defer cancel() + return env, initialized +} - if config.DefaultConfig.Project != "" { - util.Infof("Creating project: %s", config.DefaultConfig.Project) - _, err := tclient.Get().CreateProject(ctx, config.DefaultConfig.Project) - util.Fatal(err, "creating project on start") +func getPortAndTag(args []string) string { + port := "8081" + if len(args) > 0 { + port = args[0] + } - if config.DefaultConfig.Branch != "" && config.DefaultConfig.Branch != DefaultBranch { - util.Infof("Creating branch: %s", config.DefaultConfig.Branch) - _, err := tclient.Get().UseDatabase(config.DefaultConfig.Project).CreateBranch(ctx, config.DefaultConfig.Branch) - util.Fatal(err, "creating branch on start") - } + if len(args) > 1 { + t := args[1] + if t[0] == 'v' { + t = t[1:] } - if loginParam { - login.LocalLogin(net.JoinHostPort("localhost", port), "") - } else if port != "8081" { - util.Stdoutf("run 'export TIGRIS_URL=localhost:%s' for tigris cli to connect to the local instance\n", port) + ImageTag = t + } + + return port +} + +func createProjectAndBranch(ctx context.Context) { + cfg := config.DefaultConfig + + if cfg.Project != "" { + util.Infof("Creating project: %s", cfg.Project) + _, err := tclient.Get().CreateProject(ctx, cfg.Project) + util.Fatal(err, "creating project on start") + + if cfg.Branch != "" && cfg.Branch != DefaultBranch { + util.Infof("Creating branch: %s", cfg.Branch) + _, err := tclient.Get().UseDatabase(cfg.Project).CreateBranch(ctx, cfg.Branch) + util.Fatal(err, "creating branch on start") } + } +} + +func setupToken() { + cfg := &config.DefaultConfig + + tokenFile := cfg.DataDir + "/user_admin_token.txt" + + for { + _, err := os.Stat(tokenFile) + if err == nil { + break + } + } + + token, err := os.ReadFile(tokenFile) + util.Fatal(err, "reading token file") + + cfg.Token = strings.Trim(string(token), "\n\r\t ") + cfg.SkipLocalTLS = true + cfg.Protocol = "http" +} + +func getLocalURL(local bool, port string, dataDir string) string { + if local && !tokenAdminAuth && config.DefaultConfig.Token == "" { + return dataDir + "/server/unix.sock" + } + + return net.JoinHostPort("localhost", port) +} + +func localUp(cmd *cobra.Command, args []string, local bool) { + cli := getClient(cmd.Context()) + + env, initialized := configureLocal(local) + + dataDir := config.DefaultConfig.DataDir + + port := getPortAndTag(args) + + stopContainer(cli, ContainerName) + _ = startContainer(cli, ContainerName, ImagePath+":"+ImageTag, port+":8081", dataDir, env) + + cfg := &config.DefaultConfig + + cfg.URL = getLocalURL(local, port, dataDir) + + waitServerUp(cfg.URL, false) + + if local && !skipAuth { + if tokenAdminAuth { + setupToken() + } + + waitServerUp(cfg.URL, true) + } + + util.Stdoutf("\nTigris is listening at %s\n", cfg.URL) + + if dataDir != "" { + util.Stdoutf("Data directory: %s\n", dataDir) + } + + ctx, cancel := util.GetContext(cmd.Context()) + defer cancel() + + createProjectAndBranch(ctx) + + if loginParam || (local && !initialized && !skipAuth) { + login.LocalLogin(cfg.URL, cfg.Token) + } else if cfg.URL != "localhost:8081" { + util.Stdoutf("run 'export TIGRIS_URL=%s' for tigris cli to connect to the local instance\n", cfg.URL) + } +} + +var localUpCmd = &cobra.Command{ + Use: "start [port] [version]", + Aliases: []string{"up"}, + Short: "Starts an instance of Tigris", + Long: "Start local instance of Tigris. The data is persisted between runs. Authentication is enabled by default", + Run: func(cmd *cobra.Command, args []string) { + localUp(cmd, args, true) + }, +} + +var devUpCmd = &cobra.Command{ + Use: "start [port] [version]", + Aliases: []string{"up"}, + Short: "Starts an instance of Tigris for local development", + Long: "Start and instance of Tigris for local development. The data is not persisted between runs", + Run: func(cmd *cobra.Command, args []string) { + localUp(cmd, args, false) }, } @@ -277,22 +438,39 @@ var serverLogsCmd = &cobra.Command{ } var localCmd = &cobra.Command{ - Use: "dev", - Aliases: []string{"local"}, - Short: "Starts and stops local development Tigris server", + Use: "local", + Short: "Starts and stops local persistent and authenticated Tigris server", +} + +var devCmd = &cobra.Command{ + Use: "dev", + Short: "Starts and stops local development Tigris server", } func init() { serverLogsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "follow logs output") localCmd.AddCommand(serverLogsCmd) - serverUpCmd.Flags().BoolVarP(&loginParam, "login", "l", false, "login to the local instance after starting it") - serverUpCmd.Flags().StringVarP(&config.DefaultConfig.Project, "create-project", "p", config.DefaultConfig.Project, + devUpCmd.Flags().BoolVarP(&loginParam, "login", "l", false, "login to the local instance after starting it") + + devUpCmd.Flags().StringVarP(&config.DefaultConfig.Project, "create-project", "p", config.DefaultConfig.Project, "create project after start") - serverUpCmd.Flags().StringVarP(&config.DefaultConfig.Branch, "create-branch", "b", config.DefaultConfig.Branch, + devUpCmd.Flags().StringVarP(&config.DefaultConfig.Branch, "create-branch", "b", config.DefaultConfig.Branch, "create database branch after start") - localCmd.AddCommand(serverUpCmd) + localUpCmd.Flags().StringVarP(&config.DefaultConfig.DataDir, "data-dir", "d", "", + "Directory for data persistence") + localUpCmd.Flags().BoolVar(&skipAuth, "skip-auth", false, + "Start unauthenticated local instance") + localUpCmd.Flags().BoolVar(&tokenAdminAuth, "token-admin-auth", tokenAdminAuth, + "Use token instead of unix socket peer for instance administrator authentication") + + localCmd.AddCommand(localUpCmd) localCmd.AddCommand(serverDownCmd) + + devCmd.AddCommand(devUpCmd) + devCmd.AddCommand(serverDownCmd) + rootCmd.AddCommand(localCmd) + rootCmd.AddCommand(devCmd) } diff --git a/config/config.go b/config/config.go index cb0c3b3..6b71543 100644 --- a/config/config.go +++ b/config/config.go @@ -51,7 +51,7 @@ type Config struct { Protocol string `json:"protocol" yaml:"protocol,omitempty"` Project string `json:"project" yaml:"project,omitempty"` Branch string `json:"branch" yaml:"branch,omitempty"` - DataDir string `json:"data_dir" yaml:"data_dir,omitempty"` + DataDir string `json:"data_dir" mapstructure:"data_dir" yaml:"data_dir,omitempty"` Log Log `json:"log" yaml:"log,omitempty"` Timeout time.Duration `json:"timeout" yaml:"timeout,omitempty"` diff --git a/login/login.go b/login/login.go index 3f55539..8eff7b1 100644 --- a/login/login.go +++ b/login/login.go @@ -275,7 +275,10 @@ func isUnixSock(url string) bool { } func isLocalConn(host string) bool { - return host == "local" || host == "dev" || strings.HasPrefix(host, "localhost") || + return host == "local" || host == "dev" || host == "localhost" || + strings.HasPrefix(host, "localhost:") || + strings.HasPrefix(host, "127.0.0.1:") || + strings.HasPrefix(host, "[::1]:") || isUnixSock(host) } diff --git a/scaffold/scaffold.go b/scaffold/scaffold.go index 925d6e0..981ffcb 100644 --- a/scaffold/scaffold.go +++ b/scaffold/scaffold.go @@ -110,8 +110,8 @@ func ensureLocalTemplates(base string, lang string, envVar string, repoURL strin if os.Getenv(envVar) != "" { // Do not allow remote path substitution _, err := url.ParseRequestURI(os.Getenv(envVar)) - if err == nil && !os.IsPathSeparator(repoURL[0]) { - util.Fatal(ErrTemplatesInvalidPath, "get examples path from env") + if err == nil && !os.IsPathSeparator(os.Getenv(envVar)[0]) { + util.Fatal(ErrTemplatesInvalidPath, "get examples path from env: %s", os.Getenv(envVar)) } repoURL = os.Getenv(envVar) diff --git a/tests/main.sh b/tests/main.sh index a9a731f..5e11acd 100644 --- a/tests/main.sh +++ b/tests/main.sh @@ -261,7 +261,7 @@ EOF {"Key1": "vK300", "Field1": 30}' diff -w -u <(echo "$exp_out") <(echo "$out") - db_branch_tests + db_branch_tests db_negative_tests db_errors_tests @@ -272,7 +272,7 @@ EOF } db_branch_tests() { - $cli drop collection --project=db1 coll_br1 || true + $cli drop collection --project=db1 coll_br1 || true echo '[{ "title" : "coll_br1", "properties": { "Key1": { "type": "string" }, "Field1": { "type": "integer" } }, "primary_key": ["Key1"] }]' | $cli create collection --project=db1 - @@ -282,7 +282,7 @@ db_branch_tests() { $cli branch list --project=db1 | grep br1 && exit 1 - $cli branch --project=db1 create br1 + $cli branch --project=db1 create br1 $cli branch list --project=db1 | grep br1 @@ -424,6 +424,8 @@ source "$BASEDIR/backup.sh" source "$BASEDIR/scaffold.sh" # shellcheck disable=SC1091,SC1090 source "$BASEDIR/search/import.sh" +# shellcheck disable=SC1091,SC1090 +source "$BASEDIR/persistence.sh" main() { test_config @@ -471,6 +473,10 @@ main() { $cli config show | grep "protocol: http" $cli config show | grep "url: grpc://localhost:$TIGRIS_TEST_PORT" db_tests + + if [ -z "$TIGRIS_CLI_TEST_FAST" ]; then + test_persistence + fi } main diff --git a/tests/persistence.sh b/tests/persistence.sh new file mode 100644 index 0000000..732dbfc --- /dev/null +++ b/tests/persistence.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +if [ -z "$cli" ]; then + cli="./tigris" +fi + +if [ -z "$TIGRIS_TEST_PORT" ]; then + TIGRIS_TEST_PORT=8090 +fi + +DATA_DIR=/tmp/tigris-cli-local-test + +test_persistence_low() { + $SUDO rm -rf $DATA_DIR + rm -f "$HOME/.tigris/tigris-cli.yaml" + + #shellcheck disable=SC2086 + $cli local up --data-dir=$DATA_DIR $1 $2 + [ -f $DATA_DIR/initialized ] || exit 1 + +env|grep TIGRIS + $cli config show + $cli create project persistence_test + $cli list projects|grep persistence_test + + $cli local down + + #shellcheck disable=SC2086 + $cli local up $2 + $cli list projects | grep persistence_test + + [ -f $DATA_DIR/initialized ] || exit 1 + + $cli local down + + #shellcheck disable=SC2086 + $cli local up --skip-auth $TIGRIS_TEST_PORT + TIGRIS_TOKEN="" TIGRIS_URL=localhost:$TIGRIS_TEST_PORT $cli list projects || grep persistence_test + + $cli local down +} + +test_persistence() { + TIGRIS_URL="" HOME=/tmp/ cli="$SUDO $cli" test_persistence_low + TIGRIS_URL="" HOME=/tmp/ cli="$SUDO $cli" TIGRIS_URL=localhost:$TIGRIS_TEST_PORT test_persistence_low --token-admin-auth "$TIGRIS_TEST_PORT" +} diff --git a/tests/scaffold.sh b/tests/scaffold.sh index faab2da..bd67101 100644 --- a/tests/scaffold.sh +++ b/tests/scaffold.sh @@ -16,7 +16,7 @@ $cli config show env|grep TIGRIS #if [ -z "$noup" ]; then -# $cli local up +# $cli dev start #fi # first parameter is path @@ -84,7 +84,7 @@ clean() { scaffold() { if [ -z "$noup" ]; then - $cli local up "$TIGRIS_TEST_PORT" + $cli dev start "$TIGRIS_TEST_PORT" fi clean @@ -99,7 +99,7 @@ scaffold() { --output-directory="$outdir" if [ -z "$noup" ]; then - $cli local down + $cli dev stop fi } @@ -121,7 +121,7 @@ test_gin_go() { # instance was stopped by the 'task' target, bring it back if [ -z "$noup" ]; then - $cli local up "$TIGRIS_TEST_PORT" + $cli dev start "$TIGRIS_TEST_PORT" fi clean @@ -210,6 +210,6 @@ test_scaffold() { test_nextjs_typescript if [ -z "$noup" ]; then - $cli local up "$TIGRIS_TEST_PORT" + $cli dev start "$TIGRIS_TEST_PORT" fi } diff --git a/util/util.go b/util/util.go index c25218d..6d5e036 100644 --- a/util/util.go +++ b/util/util.go @@ -19,9 +19,11 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" + "runtime" "strings" "text/template" "time" @@ -189,3 +191,18 @@ func ListDir(root string) map[string]bool { return list } + +func EmptyDir(dir string) bool { + f, err := os.Open(dir) + if err != nil { + return false + } + + _, err = f.ReadDir(1) + + return errors.Is(err, io.EOF) +} + +func IsWindows() bool { + return runtime.GOOS == "windows" +}