Support env-file service overrides

README.md

@@ -65,11 +65,21 @@ Aliases:
   apply, a
 
 Flags:
-      --config string   set the location of the config file (YAML or JSON)
   -c, --cluster string  override the cluster id defined in the FDL file
+      --default         override the cluster id defined in config file
+      --env-file string load environment variables and secrets from a dotenv file
   -h, --help            help for apply
+  -n, --name string     override the OSCAR service and primary bucket names during deployment
+
+Global Flags:
+      --config string   set the location of the config file (YAML or JSON)
 ```
 
+When `--env-file` is provided, matching keys override values declared under
+`environment.variables` and `environment.secrets` in the FDL. Keys that are not
+declared in the FDL are ignored, so the same dotenv file can be reused across
+different service deployments.
+
 ### cluster
 
 Manages the configuration of clusters.
@@ -228,18 +238,23 @@ Usage:
   oscar-cli hub deploy SERVICE-SLUG [flags]
 
 Flags:
-  -c, --cluster string  set the cluster
+  -c, --cluster string    set the cluster
+      --env-file string   load environment variables and secrets from a dotenv file
       --local-path string  use a local directory containing the RO-Crate metadata instead of fetching it from GitHub
-      --owner string    GitHub owner that hosts the curated services (default "grycap")
-      --path string     subdirectory inside the repository that contains the services
-      --ref string      Git reference (branch, tag, or commit) to query (default "main")
-  -n, --name string     override the OSCAR service and primary bucket names during deployment
-      --repo string     GitHub repository that hosts the curated services (default "oscar-hub")
+      --owner string      GitHub owner that hosts the curated services (default "grycap")
+      --path string       subdirectory inside the repository that contains the services
+      --ref string        Git reference (branch, tag, or commit) to query (default "main")
+  -n, --name string       override the OSCAR service and primary bucket names during deployment
+      --repo string       GitHub repository that hosts the curated services (default "oscar-hub")
 
 Global Flags:
       --config string   set the location of the config file (YAML or JSON)
 ```
 
+When `--env-file` is provided, matching keys override values declared under
+`environment.variables` and `environment.secrets` in the service FDL. Keys that
+are not declared by the service are ignored.
+
 Default curated source: https://github.com/grycap/oscar-hub/tree/main
 
 ##### validate

cmd/apply.go

@@ -37,6 +37,7 @@ var (
 	successString        = color.New(color.FgGreen).Sprint("✓ ")
 	destinationClusterID string
 	serviceNameOverride  string
+	serviceEnvFile       string
 )
 
 func applyFunc(cmd *cobra.Command, args []string) error {
@@ -52,6 +53,14 @@ func applyFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	envFileValues := map[string]string{}
+	if strings.TrimSpace(serviceEnvFile) != "" {
+		envFileValues, err = readEnvFile(serviceEnvFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	if destinationClusterID != "" {
 		if err := conf.CheckCluster(destinationClusterID); err != nil {
 			return err
@@ -110,6 +119,8 @@ func applyFunc(cmd *cobra.Command, args []string) error {
 
 			svc.ClusterID = targetCluster
 
+			applyEnvFileValuesToService(svc, envFileValues)
+
 			if trimmed := strings.TrimSpace(serviceNameOverride); trimmed != "" {
 				overrideServiceName(svc, trimmed)
 			}
@@ -182,6 +193,7 @@ func makeApplyCmd() *cobra.Command {
 	applyCmd.Flags().StringVarP(&destinationClusterID, "cluster", "c", "", "override the cluster id defined in the FDL file")
 	applyCmd.Flags().Bool("default", false, "override the cluster id defined in config file")
 	applyCmd.Flags().StringVarP(&serviceNameOverride, "name", "n", "", "override the OSCAR service and primary bucket names during deployment")
+	applyCmd.Flags().StringVar(&serviceEnvFile, "env-file", "", "load environment variables and secrets from a dotenv file")
 
 	return applyCmd
 }

cmd/apply_test.go

@@ -1,6 +1,13 @@
 package cmd
 
 import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/grycap/oscar/v4/pkg/types"
@@ -90,3 +97,98 @@ func TestReplacePathBucket(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyCommandUsesEnvFile(t *testing.T) {
+	var applied types.Service
+	clusterServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/system/config":
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprint(w, `{"name":"oscar","namespace":"oscar","services_namespace":"oscar-svc"}`)
+		case r.Method == http.MethodGet && strings.EqualFold(r.URL.Path, "/system/services/demo"):
+			http.NotFound(w, r)
+		case r.Method == http.MethodPost && r.URL.Path == "/system/services":
+			defer r.Body.Close()
+			if err := json.NewDecoder(r.Body).Decode(&applied); err != nil {
+				t.Fatalf("decoding service apply payload: %v", err)
+			}
+			w.WriteHeader(http.StatusCreated)
+		default:
+			t.Fatalf("unexpected cluster request: %s %s", r.Method, r.URL.Path)
+		}
+	}))
+	defer clusterServer.Close()
+
+	tmpDir := t.TempDir()
+	configFile := filepath.Join(tmpDir, "config.yaml")
+	fdlFile := filepath.Join(tmpDir, "fdl.yml")
+	scriptFile := filepath.Join(tmpDir, "script.sh")
+	envFile := filepath.Join(tmpDir, ".env")
+
+	configContent := fmt.Sprintf(`oscar:
+  test:
+    endpoint: "%s"
+    auth_user: ""
+    auth_password: ""
+    ssl_verify: false
+    memory: 256Mi
+    log_level: INFO
+default: test
+`, clusterServer.URL)
+	if err := os.WriteFile(configFile, []byte(configContent), 0o600); err != nil {
+		t.Fatalf("writing config file: %v", err)
+	}
+	if err := os.WriteFile(scriptFile, []byte("#!/bin/bash\necho ok\n"), 0o600); err != nil {
+		t.Fatalf("writing script file: %v", err)
+	}
+
+	fdlContent := `
+functions:
+  oscar:
+  - test:
+      name: demo
+      image: ghcr.io/demo/demo:latest
+      script: script.sh
+      environment:
+        variables:
+          OPENAI_BASE_URL: old-url
+          OPENAI_MODEL: old-model
+        secrets:
+          OPENAI_API_KEY: old-secret
+`
+	if err := os.WriteFile(fdlFile, []byte(fdlContent), 0o600); err != nil {
+		t.Fatalf("writing fdl file: %v", err)
+	}
+
+	envContent := `OPENAI_API_KEY=new-secret
+OPENAI_BASE_URL=https://example.com/v1
+OPENAI_MODEL=new-model
+UNDECLARED=ignored
+`
+	if err := os.WriteFile(envFile, []byte(envContent), 0o600); err != nil {
+		t.Fatalf("writing env file: %v", err)
+	}
+
+	cmd := makeApplyCmd()
+	cmd.SetArgs([]string{fdlFile, "--config", configFile, "--cluster", "test", "--env-file", envFile})
+
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("apply command returned error: %v", err)
+	}
+
+	if got := applied.Environment.Secrets["OPENAI_API_KEY"]; got != "new-secret" {
+		t.Fatalf("expected OPENAI_API_KEY override, got %q", got)
+	}
+	if got := applied.Environment.Vars["OPENAI_BASE_URL"]; got != "https://example.com/v1" {
+		t.Fatalf("expected OPENAI_BASE_URL override, got %q", got)
+	}
+	if got := applied.Environment.Vars["OPENAI_MODEL"]; got != "new-model" {
+		t.Fatalf("expected OPENAI_MODEL override, got %q", got)
+	}
+	if _, ok := applied.Environment.Vars["UNDECLARED"]; ok {
+		t.Fatal("expected undeclared env key to be ignored")
+	}
+	if _, ok := applied.Environment.Secrets["UNDECLARED"]; ok {
+		t.Fatal("expected undeclared secret key to be ignored")
+	}
+}

cmd/env_file.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/grycap/oscar/v4/pkg/types"
+)
+
+func readEnvFile(path string) (map[string]string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("reading env file %s: %w", path, err)
+	}
+	defer file.Close()
+
+	values := map[string]string{}
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+	lineNumber := 0
+	for scanner.Scan() {
+		lineNumber++
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		line = strings.TrimSpace(strings.TrimPrefix(line, "export "))
+
+		key, value, ok := strings.Cut(line, "=")
+		if !ok {
+			return nil, fmt.Errorf("invalid env file %s at line %d: expected KEY=value", path, lineNumber)
+		}
+
+		key = strings.TrimSpace(key)
+		if key == "" {
+			return nil, fmt.Errorf("invalid env file %s at line %d: empty key", path, lineNumber)
+		}
+
+		values[key] = cleanEnvValue(strings.TrimSpace(value))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("reading env file %s: %w", path, err)
+	}
+
+	return values, nil
+}
+
+func cleanEnvValue(value string) string {
+	if len(value) < 2 {
+		return value
+	}
+
+	if strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'") {
+		return strings.TrimSuffix(strings.TrimPrefix(value, "'"), "'")
+	}
+
+	if strings.HasPrefix(value, "\"") && strings.HasSuffix(value, "\"") {
+		if unquoted, err := strconv.Unquote(value); err == nil {
+			return unquoted
+		}
+	}
+
+	return value
+}
+
+func applyEnvFileValuesToService(svc *types.Service, values map[string]string) {
+	if svc == nil || len(values) == 0 {
+		return
+	}
+
+	for key, value := range values {
+		if svc.Environment.Vars != nil {
+			if _, ok := svc.Environment.Vars[key]; ok {
+				svc.Environment.Vars[key] = value
+			}
+		}
+		if svc.Environment.Secrets != nil {
+			if _, ok := svc.Environment.Secrets[key]; ok {
+				svc.Environment.Secrets[key] = value
+			}
+		}
+	}
+}

cmd/env_file_test.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/grycap/oscar/v4/pkg/types"
+)
+
+func TestReadEnvFile(t *testing.T) {
+	dir := t.TempDir()
+	envPath := filepath.Join(dir, ".env")
+	content := `
+# comment
+OPENAI_API_KEY=secret
+OPENAI_MODEL="agentic"
+export OPENAI_BASE_URL=https://llm.ai.egi.eu/v1
+IGNORED='value with spaces'
+`
+	if err := os.WriteFile(envPath, []byte(content), 0o600); err != nil {
+		t.Fatalf("writing env file: %v", err)
+	}
+
+	values, err := readEnvFile(envPath)
+	if err != nil {
+		t.Fatalf("readEnvFile returned error: %v", err)
+	}
+
+	if got := values["OPENAI_API_KEY"]; got != "secret" {
+		t.Fatalf("expected OPENAI_API_KEY secret, got %q", got)
+	}
+	if got := values["OPENAI_MODEL"]; got != "agentic" {
+		t.Fatalf("expected OPENAI_MODEL agentic, got %q", got)
+	}
+	if got := values["OPENAI_BASE_URL"]; got != "https://llm.ai.egi.eu/v1" {
+		t.Fatalf("expected OPENAI_BASE_URL URL, got %q", got)
+	}
+	if got := values["IGNORED"]; got != "value with spaces" {
+		t.Fatalf("expected quoted value with spaces, got %q", got)
+	}
+}
+
+func TestApplyEnvFileValuesToServiceOverridesDeclaredKeysOnly(t *testing.T) {
+	svc := &types.Service{}
+	svc.Environment.Vars = map[string]string{
+		"OPENAI_BASE_URL": "old-url",
+		"OPENAI_MODEL":    "old-model",
+	}
+	svc.Environment.Secrets = map[string]string{
+		"OPENAI_API_KEY": "old-secret",
+	}
+
+	applyEnvFileValuesToService(svc, map[string]string{
+		"OPENAI_API_KEY":  "new-secret",
+		"OPENAI_BASE_URL": "new-url",
+		"OPENAI_MODEL":    "new-model",
+		"OTHER":           "ignored",
+	})
+
+	if got := svc.Environment.Secrets["OPENAI_API_KEY"]; got != "new-secret" {
+		t.Fatalf("expected OPENAI_API_KEY override, got %q", got)
+	}
+	if got := svc.Environment.Vars["OPENAI_BASE_URL"]; got != "new-url" {
+		t.Fatalf("expected OPENAI_BASE_URL override, got %q", got)
+	}
+	if got := svc.Environment.Vars["OPENAI_MODEL"]; got != "new-model" {
+		t.Fatalf("expected OPENAI_MODEL override, got %q", got)
+	}
+	if _, ok := svc.Environment.Vars["OTHER"]; ok {
+		t.Fatal("expected undeclared env key to be ignored")
+	}
+	if _, ok := svc.Environment.Secrets["OTHER"]; ok {
+		t.Fatal("expected undeclared secret key to be ignored")
+	}
+}