diff --git a/core/cli/run.go b/core/cli/run.go index 7c7c92d0ed42..987b9c0e83f8 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -86,6 +86,11 @@ type RunCMD struct { AgentJobRetentionDays int `env:"LOCALAI_AGENT_JOB_RETENTION_DAYS,AGENT_JOB_RETENTION_DAYS" default:"30" help:"Number of days to keep agent job history (default: 30)" group:"api"` OpenResponsesStoreTTL string `env:"LOCALAI_OPEN_RESPONSES_STORE_TTL,OPEN_RESPONSES_STORE_TTL" default:"0" help:"TTL for Open Responses store (e.g., 1h, 30m, 0 = no expiration)" group:"api"` + BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG" default:"latest" help:"Release tag for backend images (e.g., 'latest')" group:"backends"` + BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG" default:"master" help:"Branch tag for backend images (e.g., 'master')" group:"backends"` + BackendDevSuffix string `env:"LOCALAI_BACKEND_DEV_SUFFIX" default:"development" help:"Development suffix for backend images (e.g., 'development')" group:"backends"` + + Version bool } @@ -98,10 +103,25 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { os.MkdirAll(r.BackendsPath, 0750) os.MkdirAll(r.ModelsPath, 0750) + systemStateOpts := []system.SystemStateOptions{} + + // Pass backend image fallback tags via system state + if r.BackendImagesReleaseTag != "" { + systemStateOpts = append(systemStateOpts, system.WithBackendImagesReleaseTag(r.BackendImagesReleaseTag)) + } + if r.BackendImagesBranchTag != "" { + systemStateOpts = append(systemStateOpts, system.WithBackendImagesBranchTag(r.BackendImagesBranchTag)) + } + if r.BackendDevSuffix != "" { + systemStateOpts = append(systemStateOpts, system.WithBackendDevSuffix(r.BackendDevSuffix)) + } + + systemState, err := system.GetSystemState( system.WithBackendSystemPath(r.BackendsSystemPath), system.WithModelPath(r.ModelsPath), system.WithBackendPath(r.BackendsPath), + systemStateOpts..., ) if err != nil { return err @@ -140,12 +160,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { config.WithMachineTag(r.MachineTag), config.WithAPIAddress(r.Address), config.WithAgentJobRetentionDays(r.AgentJobRetentionDays), - config.WithTunnelCallback(func(tunnels []string) { - tunnelEnvVar := strings.Join(tunnels, ",") - // TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable - os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar) - xlog.Debug("setting LLAMACPP_GRPC_SERVERS", "value", tunnelEnvVar) - }), } if r.DisableMetricsEndpoint { diff --git a/core/gallery/backends.go b/core/gallery/backends.go index 833408cfcdf5..328f247b544f 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -38,13 +38,29 @@ const ( envDevSuffix = "LOCALAI_BACKEND_DEV_SUFFIX" ) -// getFallbackTagValues returns the configurable fallback tag values from environment variables -func getFallbackTagValues() (latestTag, masterTag, devSuffix string) { - latestTag = os.Getenv(envLatestTag) - masterTag = os.Getenv(envMasterTag) - devSuffix = os.Getenv(envDevSuffix) +// getFallbackTagValues returns the configurable fallback tag values from system state, falling back to environment variables +func getFallbackTagValues(systemState *system.SystemState) (latestTag, masterTag, devSuffix string) { + // First try to get values from system state + if systemState != nil && systemState.Backend.BackendImagesReleaseTag != "" { + latestTag = systemState.Backend.BackendImagesReleaseTag + } else { + // Fallback to environment variables + latestTag = os.Getenv(envLatestTag) + } + + if systemState != nil && systemState.Backend.BackendImagesBranchTag != "" { + masterTag = systemState.Backend.BackendImagesBranchTag + } else { + masterTag = os.Getenv(envMasterTag) + } + + if systemState != nil && systemState.Backend.BackendDevSuffix != "" { + devSuffix = systemState.Backend.BackendDevSuffix + } else { + devSuffix = os.Getenv(envDevSuffix) + } - // Use defaults if environment variables are not set + // Use defaults if values are not set if latestTag == "" { latestTag = defaultLatestTag } @@ -172,8 +188,8 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery, } func InstallBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error { - // Get configurable fallback tag values from environment variables - latestTag, masterTag, devSuffix := getFallbackTagValues() + // Get configurable fallback tag values from system state, falling back to environment variables + latestTag, masterTag, devSuffix := getFallbackTagValues(systemState) // Create base path if it doesn't exist err := os.MkdirAll(systemState.Backend.BackendsPath, 0750) @@ -225,7 +241,7 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL } // Try fallback: replace latestTag + "-" with masterTag + "-" in the URI - fallbackURI := strings.Replace(string(config.URI), latestTag + "-", masterTag + "-", 1) + fallbackURI := strings.Replace(string(config.URI), latestTag+"-", masterTag+"-", 1) if fallbackURI != string(config.URI) { xlog.Debug("Trying fallback URI", "original", config.URI, "fallback", fallbackURI) if err := downloader.URI(fallbackURI).DownloadFileWithContext(ctx, backendPath, "", 1, 1, downloadStatus); err == nil { @@ -234,7 +250,7 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL } else { // Try another fallback: add "-" + devSuffix suffix to the backend name // For example: master-gpu-nvidia-cuda-13-ace-step -> master-gpu-nvidia-cuda-13-ace-step-development - if !strings.Contains(fallbackURI, "-" + devSuffix) { + if !strings.Contains(fallbackURI, "-"+devSuffix) { // Extract backend name from URI and add -development parts := strings.Split(fallbackURI, "-") if len(parts) >= 2 { diff --git a/pkg/system/state.go b/pkg/system/state.go index 6e8d2a335495..e6d76b175255 100644 --- a/pkg/system/state.go +++ b/pkg/system/state.go @@ -6,8 +6,11 @@ import ( ) type Backend struct { - BackendsPath string - BackendsSystemPath string + BackendsPath string + BackendsSystemPath string + BackendImagesReleaseTag string // Release tag for backend images + BackendImagesBranchTag string // Branch tag for backend images + BackendDevSuffix string // Development suffix for backend images } type Model struct { @@ -43,6 +46,24 @@ func WithModelPath(path string) SystemStateOptions { } } +func WithBackendImagesReleaseTag(tag string) SystemStateOptions { + return func(s *SystemState) { + s.Backend.BackendImagesReleaseTag = tag + } +} + +func WithBackendImagesBranchTag(tag string) SystemStateOptions { + return func(s *SystemState) { + s.Backend.BackendImagesBranchTag = tag + } +} + +func WithBackendDevSuffix(suffix string) SystemStateOptions { + return func(s *SystemState) { + s.Backend.BackendDevSuffix = suffix + } +} + func GetSystemState(opts ...SystemStateOptions) (*SystemState, error) { state := &SystemState{} for _, opt := range opts {