diff --git a/README.md b/README.md index 82db114..f999c15 100644 --- a/README.md +++ b/README.md @@ -65,11 +65,21 @@ Aliases: apply, a Flags: - --config string set the location of the config file (YAML or JSON) -c, --cluster string override the cluster id defined in the FDL file + --default override the cluster id defined in config file + --env-file string load environment variables and secrets from a dotenv file -h, --help help for apply + -n, --name string override the OSCAR service and primary bucket names during deployment + +Global Flags: + --config string set the location of the config file (YAML or JSON) ``` +When `--env-file` is provided, matching keys override values declared under +`environment.variables` and `environment.secrets` in the FDL. Keys that are not +declared in the FDL are ignored, so the same dotenv file can be reused across +different service deployments. + ### cluster Manages the configuration of clusters. @@ -228,18 +238,23 @@ Usage: oscar-cli hub deploy SERVICE-SLUG [flags] Flags: - -c, --cluster string set the cluster + -c, --cluster string set the cluster + --env-file string load environment variables and secrets from a dotenv file --local-path string use a local directory containing the RO-Crate metadata instead of fetching it from GitHub - --owner string GitHub owner that hosts the curated services (default "grycap") - --path string subdirectory inside the repository that contains the services - --ref string Git reference (branch, tag, or commit) to query (default "main") - -n, --name string override the OSCAR service and primary bucket names during deployment - --repo string GitHub repository that hosts the curated services (default "oscar-hub") + --owner string GitHub owner that hosts the curated services (default "grycap") + --path string subdirectory inside the repository that contains the services + --ref string Git reference (branch, tag, or commit) to query (default "main") + -n, --name string override the OSCAR service and primary bucket names during deployment + --repo string GitHub repository that hosts the curated services (default "oscar-hub") Global Flags: --config string set the location of the config file (YAML or JSON) ``` +When `--env-file` is provided, matching keys override values declared under +`environment.variables` and `environment.secrets` in the service FDL. Keys that +are not declared by the service are ignored. + Default curated source: https://github.com/grycap/oscar-hub/tree/main ##### validate diff --git a/cmd/apply.go b/cmd/apply.go index 4913692..8460e3b 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -37,6 +37,7 @@ var ( successString = color.New(color.FgGreen).Sprint("✓ ") destinationClusterID string serviceNameOverride string + serviceEnvFile string ) func applyFunc(cmd *cobra.Command, args []string) error { @@ -52,6 +53,14 @@ func applyFunc(cmd *cobra.Command, args []string) error { return err } + envFileValues := map[string]string{} + if strings.TrimSpace(serviceEnvFile) != "" { + envFileValues, err = readEnvFile(serviceEnvFile) + if err != nil { + return err + } + } + if destinationClusterID != "" { if err := conf.CheckCluster(destinationClusterID); err != nil { return err @@ -110,6 +119,8 @@ func applyFunc(cmd *cobra.Command, args []string) error { svc.ClusterID = targetCluster + applyEnvFileValuesToService(svc, envFileValues) + if trimmed := strings.TrimSpace(serviceNameOverride); trimmed != "" { overrideServiceName(svc, trimmed) } @@ -182,6 +193,7 @@ func makeApplyCmd() *cobra.Command { applyCmd.Flags().StringVarP(&destinationClusterID, "cluster", "c", "", "override the cluster id defined in the FDL file") applyCmd.Flags().Bool("default", false, "override the cluster id defined in config file") applyCmd.Flags().StringVarP(&serviceNameOverride, "name", "n", "", "override the OSCAR service and primary bucket names during deployment") + applyCmd.Flags().StringVar(&serviceEnvFile, "env-file", "", "load environment variables and secrets from a dotenv file") return applyCmd } diff --git a/cmd/apply_test.go b/cmd/apply_test.go index 6601c1e..5e302d2 100644 --- a/cmd/apply_test.go +++ b/cmd/apply_test.go @@ -1,6 +1,13 @@ package cmd import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" "github.com/grycap/oscar/v4/pkg/types" @@ -90,3 +97,98 @@ func TestReplacePathBucket(t *testing.T) { }) } } + +func TestApplyCommandUsesEnvFile(t *testing.T) { + var applied types.Service + clusterServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/system/config": + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"name":"oscar","namespace":"oscar","services_namespace":"oscar-svc"}`) + case r.Method == http.MethodGet && strings.EqualFold(r.URL.Path, "/system/services/demo"): + http.NotFound(w, r) + case r.Method == http.MethodPost && r.URL.Path == "/system/services": + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&applied); err != nil { + t.Fatalf("decoding service apply payload: %v", err) + } + w.WriteHeader(http.StatusCreated) + default: + t.Fatalf("unexpected cluster request: %s %s", r.Method, r.URL.Path) + } + })) + defer clusterServer.Close() + + tmpDir := t.TempDir() + configFile := filepath.Join(tmpDir, "config.yaml") + fdlFile := filepath.Join(tmpDir, "fdl.yml") + scriptFile := filepath.Join(tmpDir, "script.sh") + envFile := filepath.Join(tmpDir, ".env") + + configContent := fmt.Sprintf(`oscar: + test: + endpoint: "%s" + auth_user: "" + auth_password: "" + ssl_verify: false + memory: 256Mi + log_level: INFO +default: test +`, clusterServer.URL) + if err := os.WriteFile(configFile, []byte(configContent), 0o600); err != nil { + t.Fatalf("writing config file: %v", err) + } + if err := os.WriteFile(scriptFile, []byte("#!/bin/bash\necho ok\n"), 0o600); err != nil { + t.Fatalf("writing script file: %v", err) + } + + fdlContent := ` +functions: + oscar: + - test: + name: demo + image: ghcr.io/demo/demo:latest + script: script.sh + environment: + variables: + OPENAI_BASE_URL: old-url + OPENAI_MODEL: old-model + secrets: + OPENAI_API_KEY: old-secret +` + if err := os.WriteFile(fdlFile, []byte(fdlContent), 0o600); err != nil { + t.Fatalf("writing fdl file: %v", err) + } + + envContent := `OPENAI_API_KEY=new-secret +OPENAI_BASE_URL=https://example.com/v1 +OPENAI_MODEL=new-model +UNDECLARED=ignored +` + if err := os.WriteFile(envFile, []byte(envContent), 0o600); err != nil { + t.Fatalf("writing env file: %v", err) + } + + cmd := makeApplyCmd() + cmd.SetArgs([]string{fdlFile, "--config", configFile, "--cluster", "test", "--env-file", envFile}) + + if err := cmd.Execute(); err != nil { + t.Fatalf("apply command returned error: %v", err) + } + + if got := applied.Environment.Secrets["OPENAI_API_KEY"]; got != "new-secret" { + t.Fatalf("expected OPENAI_API_KEY override, got %q", got) + } + if got := applied.Environment.Vars["OPENAI_BASE_URL"]; got != "https://example.com/v1" { + t.Fatalf("expected OPENAI_BASE_URL override, got %q", got) + } + if got := applied.Environment.Vars["OPENAI_MODEL"]; got != "new-model" { + t.Fatalf("expected OPENAI_MODEL override, got %q", got) + } + if _, ok := applied.Environment.Vars["UNDECLARED"]; ok { + t.Fatal("expected undeclared env key to be ignored") + } + if _, ok := applied.Environment.Secrets["UNDECLARED"]; ok { + t.Fatal("expected undeclared secret key to be ignored") + } +} diff --git a/cmd/env_file.go b/cmd/env_file.go new file mode 100644 index 0000000..b14c1ef --- /dev/null +++ b/cmd/env_file.go @@ -0,0 +1,86 @@ +package cmd + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + + "github.com/grycap/oscar/v4/pkg/types" +) + +func readEnvFile(path string) (map[string]string, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("reading env file %s: %w", path, err) + } + defer file.Close() + + values := map[string]string{} + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + lineNumber := 0 + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + + key, value, ok := strings.Cut(line, "=") + if !ok { + return nil, fmt.Errorf("invalid env file %s at line %d: expected KEY=value", path, lineNumber) + } + + key = strings.TrimSpace(key) + if key == "" { + return nil, fmt.Errorf("invalid env file %s at line %d: empty key", path, lineNumber) + } + + values[key] = cleanEnvValue(strings.TrimSpace(value)) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading env file %s: %w", path, err) + } + + return values, nil +} + +func cleanEnvValue(value string) string { + if len(value) < 2 { + return value + } + + if strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'") { + return strings.TrimSuffix(strings.TrimPrefix(value, "'"), "'") + } + + if strings.HasPrefix(value, "\"") && strings.HasSuffix(value, "\"") { + if unquoted, err := strconv.Unquote(value); err == nil { + return unquoted + } + } + + return value +} + +func applyEnvFileValuesToService(svc *types.Service, values map[string]string) { + if svc == nil || len(values) == 0 { + return + } + + for key, value := range values { + if svc.Environment.Vars != nil { + if _, ok := svc.Environment.Vars[key]; ok { + svc.Environment.Vars[key] = value + } + } + if svc.Environment.Secrets != nil { + if _, ok := svc.Environment.Secrets[key]; ok { + svc.Environment.Secrets[key] = value + } + } + } +} diff --git a/cmd/env_file_test.go b/cmd/env_file_test.go new file mode 100644 index 0000000..e4145e4 --- /dev/null +++ b/cmd/env_file_test.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/grycap/oscar/v4/pkg/types" +) + +func TestReadEnvFile(t *testing.T) { + dir := t.TempDir() + envPath := filepath.Join(dir, ".env") + content := ` +# comment +OPENAI_API_KEY=secret +OPENAI_MODEL="agentic" +export OPENAI_BASE_URL=https://llm.ai.egi.eu/v1 +IGNORED='value with spaces' +` + if err := os.WriteFile(envPath, []byte(content), 0o600); err != nil { + t.Fatalf("writing env file: %v", err) + } + + values, err := readEnvFile(envPath) + if err != nil { + t.Fatalf("readEnvFile returned error: %v", err) + } + + if got := values["OPENAI_API_KEY"]; got != "secret" { + t.Fatalf("expected OPENAI_API_KEY secret, got %q", got) + } + if got := values["OPENAI_MODEL"]; got != "agentic" { + t.Fatalf("expected OPENAI_MODEL agentic, got %q", got) + } + if got := values["OPENAI_BASE_URL"]; got != "https://llm.ai.egi.eu/v1" { + t.Fatalf("expected OPENAI_BASE_URL URL, got %q", got) + } + if got := values["IGNORED"]; got != "value with spaces" { + t.Fatalf("expected quoted value with spaces, got %q", got) + } +} + +func TestApplyEnvFileValuesToServiceOverridesDeclaredKeysOnly(t *testing.T) { + svc := &types.Service{} + svc.Environment.Vars = map[string]string{ + "OPENAI_BASE_URL": "old-url", + "OPENAI_MODEL": "old-model", + } + svc.Environment.Secrets = map[string]string{ + "OPENAI_API_KEY": "old-secret", + } + + applyEnvFileValuesToService(svc, map[string]string{ + "OPENAI_API_KEY": "new-secret", + "OPENAI_BASE_URL": "new-url", + "OPENAI_MODEL": "new-model", + "OTHER": "ignored", + }) + + if got := svc.Environment.Secrets["OPENAI_API_KEY"]; got != "new-secret" { + t.Fatalf("expected OPENAI_API_KEY override, got %q", got) + } + if got := svc.Environment.Vars["OPENAI_BASE_URL"]; got != "new-url" { + t.Fatalf("expected OPENAI_BASE_URL override, got %q", got) + } + if got := svc.Environment.Vars["OPENAI_MODEL"]; got != "new-model" { + t.Fatalf("expected OPENAI_MODEL override, got %q", got) + } + if _, ok := svc.Environment.Vars["OTHER"]; ok { + t.Fatal("expected undeclared env key to be ignored") + } + if _, ok := svc.Environment.Secrets["OTHER"]; ok { + t.Fatal("expected undeclared secret key to be ignored") + } +} diff --git a/cmd/hub_deploy.go b/cmd/hub_deploy.go index 8d11c7a..4fd9bc9 100644 --- a/cmd/hub_deploy.go +++ b/cmd/hub_deploy.go @@ -23,6 +23,7 @@ type hubDeployOptions struct { apiBase string name string localPath string + envFile string } func (o *hubDeployOptions) applyToClient() []hub.Option { @@ -81,6 +82,14 @@ func hubDeployFunc(cmd *cobra.Command, args []string, opts *hubDeployOptions) er return err } + if strings.TrimSpace(opts.envFile) != "" { + envFileValues, err := readEnvFile(opts.envFile) + if err != nil { + return err + } + applyEnvFileValuesToService(serviceDef, envFileValues) + } + if opts.name != "" { overrideServiceName(serviceDef, opts.name) } @@ -130,6 +139,7 @@ func makeHubDeployCmd() *cobra.Command { cmd.Flags().StringVar(&opts.apiBase, "api-base", "", "override the GitHub API base URL") cmd.Flags().StringVarP(&opts.name, "name", "n", "", "override the OSCAR service and primary bucket names during deployment") cmd.Flags().StringVar(&opts.localPath, "local-path", "", "use a local directory containing the RO-Crate metadata instead of fetching it from GitHub") + cmd.Flags().StringVar(&opts.envFile, "env-file", "", "load environment variables and secrets from a dotenv file") cmd.Flags().StringP("cluster", "c", "", "set the cluster") if flag := cmd.Flags().Lookup("api-base"); flag != nil { diff --git a/cmd/hub_deploy_test.go b/cmd/hub_deploy_test.go index 53c2acb..6e8785b 100644 --- a/cmd/hub_deploy_test.go +++ b/cmd/hub_deploy_test.go @@ -25,6 +25,12 @@ functions: name: Cowsay image: ghcr.io/demo/cowsay:latest script: script.sh + environment: + variables: + OPENAI_BASE_URL: old-url + OPENAI_MODEL: old-model + secrets: + OPENAI_API_KEY: old-secret input: - storage_provider: minio.default path: cowsay/in @@ -75,6 +81,7 @@ functions: tmpDir := t.TempDir() configFile := filepath.Join(tmpDir, "config.yaml") + envFile := filepath.Join(tmpDir, ".env") configContent := fmt.Sprintf(`oscar: test: endpoint: "%s" @@ -88,6 +95,14 @@ default: test if err := os.WriteFile(configFile, []byte(configContent), 0o600); err != nil { t.Fatalf("writing config file: %v", err) } + envContent := `OPENAI_API_KEY=new-secret +OPENAI_BASE_URL=https://example.com/v1 +OPENAI_MODEL=new-model +UNDECLARED=ignored +` + if err := os.WriteFile(envFile, []byte(envContent), 0o600); err != nil { + t.Fatalf("writing env file: %v", err) + } originalConfigPath := configPath configPath = configFile @@ -98,7 +113,7 @@ default: test stderr := &bytes.Buffer{} cmd.SetOut(stdout) cmd.SetErr(stderr) - cmd.SetArgs([]string{slug, "--api-base", gitServer.URL, "--cluster", "test", "--name", override}) + cmd.SetArgs([]string{slug, "--api-base", gitServer.URL, "--cluster", "test", "--name", override, "--env-file", envFile}) if err := cmd.Execute(); err != nil { t.Fatalf("hub deploy command returned error: %v", err) @@ -125,6 +140,21 @@ default: test if applied.Script != scriptContent { t.Fatalf("expected script content %q, got %q", scriptContent, applied.Script) } + if got := applied.Environment.Secrets["OPENAI_API_KEY"]; got != "new-secret" { + t.Fatalf("expected OPENAI_API_KEY override, got %q", got) + } + if got := applied.Environment.Vars["OPENAI_BASE_URL"]; got != "https://example.com/v1" { + t.Fatalf("expected OPENAI_BASE_URL override, got %q", got) + } + if got := applied.Environment.Vars["OPENAI_MODEL"]; got != "new-model" { + t.Fatalf("expected OPENAI_MODEL override, got %q", got) + } + if _, ok := applied.Environment.Vars["UNDECLARED"]; ok { + t.Fatal("expected undeclared env key to be ignored") + } + if _, ok := applied.Environment.Secrets["UNDECLARED"]; ok { + t.Fatal("expected undeclared secret key to be ignored") + } if applied.ClusterID != "test" { t.Fatalf("expected cluster id test, got %s", applied.ClusterID) }