values.yaml

# Default values for inference-stack.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# Override the chart name used for resource naming
nameOverride: ""
# Override the full resource names (takes precedence over nameOverride)
fullnameOverride: ""

# Onwards Gateway Configuration
# The Onwards gateway routes client requests to model groups based on model aliases
onwards:
  # Number of gateway replicas to deploy
  replicaCount: 1

  image:
    # Docker image repository for the Onwards gateway
    repository: ghcr.io/doublewordai/onwards
    # Image pull policy (Always, IfNotPresent, Never)
    pullPolicy: IfNotPresent
    # Image tag override (defaults to chart appVersion if empty)
    tag: ""

  # Secrets for pulling private container images
  imagePullSecrets: []

  serviceAccount:
    # Whether to create a service account for the gateway
    create: true
    # Additional annotations for the service account
    annotations: {}
    # Service account name (auto-generated if empty and create=true)
    name: ""

  # Additional annotations for gateway pods
  podAnnotations: {}

  # Security context for the entire pod
  podSecurityContext:
    fsGroup: 2000

  # Security context for the gateway container
  securityContext:
    capabilities:
      drop:
        - ALL
    readOnlyRootFilesystem: true
    runAsNonRoot: true
    runAsUser: 1000

  service:
    # Kubernetes service type (ClusterIP, NodePort, LoadBalancer)
    type: ClusterIP
    # External port for the service
    port: 80
    # nodePort: 30080  # Only used if type is NodePort

  # Port that the Onwards container listens on
  containerPort: 3000

  # Environment variables for the gateway container
  env:
    []
    # - name: RUST_LOG
    #   value: "info"

  # Additional volume mounts for the gateway container
  volumeMounts: []

  # Additional volumes for the gateway pod
  volumes: []

  # Resource requests and limits for the gateway
  resources:
    limits:
      cpu: 200m
      memory: 256Mi
    requests:
      cpu: 100m
      memory: 128Mi
    # We usually recommend not to specify default resources and to leave this as a conscious
    # choice for the user. This also increases chances charts run on environments with little
    # resources, such as Minikube. If you do want to specify resources, uncomment the following
    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
    # limits:
    #   cpu: 100m
    #   memory: 128Mi
    # requests:
    #   cpu: 100m
    #   memory: 128Mi

  # Node selector constraints for pod scheduling
  nodeSelector: {}

  # Tolerations for pod scheduling (e.g., to run on tainted nodes)
  tolerations: []

  # Affinity rules for pod scheduling
  affinity: {}
  
  # Metrics configuration
  metrics:
    # Enable metrics endpoint
    enabled: true
    # Metrics port for the container
    port: 9090
    # Metrics prefix for prometheus
    prefix: onwards
    # Service configuration for metrics
    service:
      # Port for the metrics service endpoint
      port: 9090
  
  # Probe configuration
  probes:
    # Health check path (used by both liveness and readiness)
    path: /v1/models
    startup:
      initialDelaySeconds: 30
      periodSeconds: 10
      failureThreshold: 60
    liveness:
      initialDelaySeconds: 5
      periodSeconds: 10
    readiness:
      initialDelaySeconds: 2
      periodSeconds: 5
  
  # LoadBalancer service configuration (for external access)
  loadBalancer:
    # Enable a separate LoadBalancer service
    enabled: false
    # Port for the LoadBalancer service
    port: 80
    # Annotations for the LoadBalancer service
    annotations: {}
      # helm.sh/resource-policy: keep
      # service.beta.kubernetes.io/azure-load-balancer-internal: "true"
  
  # Additional command line arguments for onwards
  extraArgs: []

# Model Groups Configuration
# Each model group represents a deployment of an inference engine (vLLM, SGLang, etc.)
# that serves one or more models through an OpenAI-compatible API
modelGroups:
  # Example vLLM model group for Gemma 12B (disabled by default)
  # Uncomment and modify this section to deploy your models
  vllm-gemma:
    # Whether to deploy this model group
    enabled: false

    # Docker image configuration
    image:
      repository: vllm/vllm-openai
      tag: "latest"
      pullPolicy: IfNotPresent

    # Client-facing model aliases/names that can be used in API requests
    # These names will appear in /v1/models and can be specified in the "model" field of requests
    # The Onwards gateway routes requests with these names to this model group
    modelAlias:
      - "gemma-12b"
      - "google/gemma-3-12b-it"

    # The actual model name/path used by the inference engine (e.g., HuggingFace model path)
    modelName: "google/gemma-3-12b-it"

    # Optional Downstream API key for authenticating requests to this model group once they are in the inference engine
    apiKey: ""

    service:
      # Kubernetes service type for this model group
      type: ClusterIP
      # Service port for external access
      port: 8000

    # Port that the inference engine container listens on
    containerPort: 8000

    # Additional container ports (e.g., for metrics endpoints)
    additionalPorts:
      []
      # - name: metrics
      #   containerPort: 8001
      #   servicePort: 8001

    # Number of replicas to deploy for this model group
    replicaCount: 1

    # Update strategy for Deployment
    strategy:
      type: RollingUpdate
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 0

    # Container command passed to the inference engine
    # Specify the complete command including executable and arguments
    command:
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "8000"
      - "--model"
      - "google/gemma-3-12b-it"
      - "--dtype"
      - "bfloat16"
      # - "--tensor-parallel-size"
      # - "1"
      # - "--gpu-memory-utilization"
      # - "0.9"

    # Environment variables for the inference engine container
    env:
      - name: CUDA_VISIBLE_DEVICES
        value: "0"
      # - name: HUGGING_FACE_HUB_TOKEN
      #   valueFrom:
      #     secretKeyRef:
      #       name: hf-token
      #       key: token

    # Startup probe configuration (waits for model download and initialization)
    startupProbe:
      # Whether to enable startup probes
      enabled: true
      # Probe type: 'http' for HTTP GET requests, 'tcp' for TCP socket checks
      type: http
      # HTTP path to check (only used for http type)
      path: /health
      # Delay before first probe after container starts
      initialDelaySeconds: 30
      # How often to perform the probe
      periodSeconds: 10
      # Timeout for each probe
      timeoutSeconds: 5
      # Number of failures before giving up (30 * 10 = 5 minutes for model download)
      failureThreshold: 30

    # Liveness probe configuration (checks if container is alive)
    livenessProbe:
      # Whether to enable liveness probes
      enabled: true
      # Probe type: 'http' for HTTP GET requests, 'tcp' for TCP socket checks
      type: http
      # HTTP path to check (only used for http type)
      path: /health
      # Delay before first probe after container starts (startup probe handles initial delay)
      initialDelaySeconds: 5
      # How often to perform the probe
      periodSeconds: 10
      # Timeout for each probe
      timeoutSeconds: 5
      # Number of failures before considering the container dead
      failureThreshold: 3

    # Readiness probe configuration (checks if container is ready to serve traffic)
    readinessProbe:
      # Whether to enable readiness probes
      enabled: true
      # Probe type: 'http' for HTTP GET requests, 'tcp' for TCP socket checks
      type: http
      # HTTP path to check (only used for http type)
      path: /health
      # Delay before first probe after container starts (startup probe handles initial delay)
      initialDelaySeconds: 2
      # How often to perform the probe
      periodSeconds: 5
      # Timeout for each probe
      timeoutSeconds: 3
      # Number of failures before considering the container not ready
      failureThreshold: 3

    # Resource requests and limits for this model group
    resources:
      limits:
        nvidia.com/gpu: 1
        memory: 16Gi
      requests:
        nvidia.com/gpu: 1
        memory: 8Gi

    # Persistent volumes configuration
    # Creates PVCs for shared storage across replicas
    # Leave empty by default - use init containers to copy model weights
    persistentVolumes: []
      # Example volume configuration:
      # - name: model-cache
      #   size: 100Gi
      #   mountPath: /model-cache
      #   storageClass: ""  # empty string uses default storage class
      #   accessModes:
      #     - ReadWriteMany  # For sharing across replicas
      #   # Optional PVC-specific settings:
      #   # labels: {}
      #   # annotations: {}
      #   # selector: {}
      #   # volumeMode: Filesystem

    # Security context for the entire pod
    podSecurityContext:
      runAsUser: 0 # Often needed for GPU access

    # Security context for the container
    securityContext: {}

    serviceAccount:
      # Whether to create a service account for this model group
      create: false
      # Additional annotations for the service account
      annotations: {}

    # Init containers for setup tasks (e.g., copying model weights from bit-harbor images)
    # Full Kubernetes initContainers specification supported
    initContainers: []
      # Example: Copy model weights from bit-harbor image
      # - name: copy-model-weights
      #   image: ghcr.io/doublewordai/bit-harbor:llama-3.1-8b
      #   command:
      #     - cp
      #     - -r
      #     - /models/*
      #     - /shared/models/
      #   volumeMounts:
      #     - name: model-storage
      #       mountPath: /shared/models
      #   resources:
      #     requests:
      #       cpu: 100m
      #       memory: 128Mi

    # Additional volume mounts for the container
    volumeMounts: []

    # Additional volumes for the pod
    volumes: []

    # Additional command line arguments for the model group
    args: []

    # Node selection constraints (e.g., GPU node requirements)
    nodeSelector:
      {}
      # kubernetes.io/arch: amd64
      # nvidia.com/gpu.present: "true"

    # Tolerations for scheduling on tainted nodes (e.g., GPU nodes)
    tolerations:
      []
      # - key: nvidia.com/gpu
      #   operator: Exists
      #   effect: NoSchedule

    # Affinity rules for advanced pod scheduling
    affinity: {}

    # Additional annotations for pods
    podAnnotations: {}

    # Secrets for pulling private container images
    imagePullSecrets: []

  # Example SGLang model group (disabled by default)
  # Uncomment and modify to use SGLang inference engine
  sglang-qwen:
    # Whether to deploy this model group
    enabled: false

    # Docker image configuration
    image:
      repository: lmsysorg/sglang
      tag: "latest"
      pullPolicy: IfNotPresent

    # Client-facing model aliases for API requests
    modelAlias:
      - "qwen-2.5-7b"
      - "qwen-2.5-7b-instruct"

    # Actual model path used by SGLang
    modelName: "Qwen/Qwen2.5-7B-Instruct"

    service:
      # Service type for this model group
      type: ClusterIP
      # Service port
      port: 30000

    # Container port for SGLang server
    containerPort: 30000

    # Number of replicas
    replicaCount: 1

    # Update strategy for Deployment
    strategy:
      type: RollingUpdate
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 0

    # Command for SGLang server
    command:
      - "python"
      - "-m"
      - "sglang.launch_server"
      - "--model-path"
      - "Qwen/Qwen2.5-7B-Instruct"
      - "--host"
      - "0.0.0.0"
      - "--port"
      - "30000"

    # Environment variables
    env:
      - name: CUDA_VISIBLE_DEVICES
        value: "0"

    # Startup probe (waits for model download and initialization)
    startupProbe:
      enabled: true
      type: tcp
      initialDelaySeconds: 30
      periodSeconds: 10
      timeoutSeconds: 5
      failureThreshold: 30

    # Liveness probe (using TCP since SGLang may not have HTTP health endpoint)
    livenessProbe:
      enabled: true
      type: tcp
      initialDelaySeconds: 5
      periodSeconds: 10
      timeoutSeconds: 5
      failureThreshold: 3

    # Readiness probe
    readinessProbe:
      enabled: true
      type: tcp
      initialDelaySeconds: 2
      periodSeconds: 5
      timeoutSeconds: 3
      failureThreshold: 3

    # Resource requirements
    resources:
      limits:
        nvidia.com/gpu: 1
        memory: 12Gi
      requests:
        nvidia.com/gpu: 1
        memory: 6Gi
    
    # Persistent volumes configuration
    persistentVolumes: []
    
    # Additional container ports
    additionalPorts: []
    
    # Security context for the entire pod
    podSecurityContext: {}
    
    # Security context for the container
    securityContext: {}
    
    serviceAccount:
      # Whether to create a service account
      create: false
      annotations: {}
    
    # Init containers
    initContainers: []
    
    # Additional volume mounts
    volumeMounts: []
    
    # Additional volumes
    volumes: []
    
    # Pod annotations
    podAnnotations: {}
    
    # Image pull secrets
    imagePullSecrets: []

    # Additional command line arguments for the model group
    args: []

    # Node selection, tolerations, and affinity
    nodeSelector: {}
    tolerations: []
    affinity: {}

  # Example TensorRT-LLM model group (disabled by default)
  # Uncomment and modify to use NVIDIA TensorRT-LLM
  tensorrt-llm-mistral:
    # Whether to deploy this model group
    enabled: false

    # Docker image configuration
    image:
      repository: nvcr.io/nvidia/tritonserver
      tag: "24.01-trtllm-python-py3"
      pullPolicy: IfNotPresent

    # Client-facing model aliases
    modelAlias:
      - "mistral-7b"
      - "mistral-7b-instruct"

    # Actual model used by TensorRT-LLM
    modelName: "mistralai/Mistral-7B-Instruct-v0.2"

    service:
      # Service type
      type: ClusterIP
      # Service port
      port: 8001

    # Container port
    containerPort: 8001

    # Number of replicas
    replicaCount: 1

    # Update strategy for Deployment
    strategy:
      type: RollingUpdate
      rollingUpdate:
        maxSurge: 1
        maxUnavailable: 0

    # Command for TensorRT-LLM
    command: []
    
    # Environment variables
    env: []
    
    # Startup probe (waits for model download and compilation)
    startupProbe:
      enabled: false
      type: http
      path: /v2/health/ready
      initialDelaySeconds: 30
      periodSeconds: 15
      timeoutSeconds: 10
      failureThreshold: 60

    # Liveness probe
    livenessProbe:
      enabled: false
      type: http
      path: /v2/health/live
      initialDelaySeconds: 5
      periodSeconds: 15
      timeoutSeconds: 10
      failureThreshold: 3
    
    # Readiness probe
    readinessProbe:
      enabled: false
      type: http
      path: /v2/health/ready
      initialDelaySeconds: 2
      periodSeconds: 10
      timeoutSeconds: 5
      failureThreshold: 3
    
    # Resource requirements (TensorRT typically needs significant GPU memory)
    resources:
      limits:
        nvidia.com/gpu: 1
        memory: 16Gi
      requests:
        nvidia.com/gpu: 1
        memory: 8Gi
    
    # Persistent volumes configuration
    persistentVolumes: []
    
    # Security context for the entire pod
    podSecurityContext: {}
    
    # Security context for the container
    securityContext: {}
    
    serviceAccount:
      # Whether to create a service account for this model group
      create: false
      # Additional annotations for the service account
      annotations: {}
    
    # Init containers for setup tasks
    initContainers: []
    
    # Additional volume mounts for the container
    volumeMounts: []
    
    # Additional volumes for the pod
    volumes: []

    # Scheduling constraints
    nodeSelector: {}
    tolerations: []
    affinity: {}
    
    # Additional annotations for pods
    podAnnotations: {}
    
    # Secrets for pulling private container images
    imagePullSecrets: []
    
    # Additional container ports
    additionalPorts: []
    
    # Additional command line arguments for the model group
    args: []

# Generic ConfigMaps for custom configuration files
# Create ConfigMaps with inline data that can be mounted into pods
# Each ConfigMap can contain multiple files (data keys)
configMaps: {}
  # Example: Chat template ConfigMap with Jinja2 syntax
  # chat-templates:
  #   template.jinja: |
  #     {%- for message in messages %}
  #     {{ message.role }}: {{ message.content }}
  #     {%- endfor %}
  #
  # To mount in a model group, add to volumes and volumeMounts:
  # modelGroups:
  #   vllm-gemma:
  #     volumes:
  #       - name: chat-templates
  #         configMap:
  #           name: <release-name>-chat-templates
  #     volumeMounts:
  #       - name: chat-templates
  #         mountPath: /app/templates
  #         readOnly: true

# Ingress configuration for external access to Onwards gateway
ingress:
  # Whether to create an ingress resource for external access
  enabled: false
  # Ingress class name (e.g., nginx, traefik)
  className: ""
  # Additional annotations for the ingress resource
  annotations:
    {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
    # nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    # nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
  # Host configuration for the ingress
  hosts:
    - host: inference.local
      paths:
        - path: /
          pathType: Prefix
  # TLS configuration for HTTPS
  tls: []
  #  - secretName: inference-tls
  #    hosts:
  #      - inference.local

# NetworkPolicy configuration for network segmentation and security
networkPolicy:
  # Whether to create NetworkPolicy resources
  enabled: false
  
  # Onwards gateway NetworkPolicy spec (directly passed to Kubernetes NetworkPolicy spec after podSelector)
  # This is the standard Kubernetes NetworkPolicy spec, see: https://kubernetes.io/docs/concepts/services-networking/network-policies/
  onwards: {}
    # Example: Allow ingress from specific pods and egress to model groups
    # policyTypes:
    #   - Ingress
    #   - Egress
    # ingress:
    #   - from:
    #     - podSelector:
    #         matchLabels:
    #           app: frontend
    #     ports:
    #     - protocol: TCP
    #       port: 3000
    # egress:
    #   - to:
    #     - podSelector:
    #         matchLabels:
    #           app.kubernetes.io/component: model-group
    #     ports:
    #     - protocol: TCP
    #       port: 8000
    #   # DNS resolution
    #   - to:
    #     - namespaceSelector:
    #         matchLabels:
    #           name: kube-system
    #     ports:
    #     - protocol: UDP
    #       port: 53
  
  # Model group NetworkPolicy specs (directly passed to Kubernetes NetworkPolicy spec after podSelector)
  # Define a NetworkPolicy for each model group by name
  modelGroups: {}
    # Example for vllm-gemma model group:
    # vllm-gemma:
    #   policyTypes:
    #     - Ingress
    #     - Egress
    #   ingress:
    #     - from:
    #       - podSelector:
    #           matchLabels:
    #             app.kubernetes.io/component: onwards
    #       ports:
    #       - protocol: TCP
    #         port: 8000
    #   egress:
    #     # DNS resolution
    #     - to:
    #       - namespaceSelector:
    #           matchLabels:
    #             name: kube-system
    #       ports:
    #       - protocol: UDP
    #         port: 53
    #     # Model downloads
    #     - to:
    #       - ipBlock:
    #           cidr: 0.0.0.0/0
    #           except:
    #           - 169.254.169.254/32
    #       ports:
    #       - protocol: TCP
    #         port: 443

# ServiceMonitor for Prometheus monitoring (requires Prometheus Operator)
serviceMonitor:
  # Whether to create a ServiceMonitor resource for Prometheus scraping
  enabled: false
  # Namespace where the ServiceMonitor should be created (empty = same as release)
  namespace: ""
  # Additional labels for the ServiceMonitor (useful for Prometheus discovery)
  labels: {}
    # prometheus: kube-prometheus
    # release: prometheus-operator
  # Additional annotations for the ServiceMonitor
  annotations: {}
  # How often Prometheus should scrape metrics
  interval: 30s
  # Timeout for each scrape attempt
  scrapeTimeout: 10s
  # TLS configuration for secure metrics scraping
  tlsConfig: {}
    # insecureSkipVerify: true
  # Bearer token secret for authentication
  bearerTokenSecret: {}
    # name: prometheus-bearer-token
    # key: token