llmproxy.yaml

# Environment variables: follow .env.example file
volumes:
  cli-proxy-api-auth-data:
  cli-proxy-api-config-data:

x-common-healthcheck: &common-healthcheck
  interval: 30s # Perform health check every 30 seconds
  timeout: 10s # Health check command times out after 10 seconds
  retries: 3 # Retry up to 3 times if health check fails

x-common-deploy: &common-deploy
  replicas: 1
  update_config:
    parallelism: 1
    delay: 10s
    order: start-first
    failure_action: rollback
  rollback_config:
    parallelism: 1
    delay: 10s
    order: stop-first

x-cli-proxy-api: &cli-proxy-api
  image: eceasy/cli-proxy-api:latest
  command:
    - sh
    - -c
    - |
      if [ ! -f /CLIProxyAPI/config/config.yaml ]; then
        cp /CLIProxyAPI/config_ro/config.yaml /CLIProxyAPI/config/config.yaml
      fi
      exec ./CLIProxyAPI --config /CLIProxyAPI/config/config.yaml
  environment:
    TZ: ${TZ:-UTC}
  healthcheck:
    <<: *common-healthcheck
    test: ["CMD-SHELL", "wget -q --spider http://localhost:8317/ || exit 1"]
    start_period: 10s
  networks:
    - internal
    - public
    - monitoring

services:
  cli-proxy-api:
    <<: *cli-proxy-api
    volumes:
      - cli-proxy-api-config-data:/CLIProxyAPI/config
      - cli-proxy-api-auth-data:/CLIProxyAPI/auth
    configs:
      - source: cli-proxy-api-config-yaml
        target: /CLIProxyAPI/config_ro/config.yaml
    deploy:
      <<: *common-deploy
      labels:
        - traefik.enable=true
        - traefik.http.routers.cli-proxy-api.rule=Host(`${CLI_PROXY_API_HOST}`)
        - traefik.http.routers.cli-proxy-api.entrypoints=${TRAEFIK_ROUTER_ENTRYPOINTS:-web}
        - traefik.http.routers.cli-proxy-api.tls=${TRAEFIK_ROUTER_TLS:-false}
        - traefik.http.routers.cli-proxy-api.tls.certresolver=${LETSENCRYPT_RESOLVER:-le}
        - traefik.http.services.cli-proxy-api.loadbalancer.server.port=8317
        - traefik.http.routers.cli-proxy-api.service=cli-proxy-api

  litellm:
    image: tamntlib/litellm-database:v1.83.3-stable
    command: ["--config", "/app/config.yaml", "--num_workers", "4"]
    environment:
      DATABASE_URL: "postgresql://${DB_USER:-app}:${DB_PASSWORD:-app_db_PW@2018}@db:5432/${DB_NAME:-app}"
      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-litellm_master_key}
      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:-litellm_salt_key}
      SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL:-}
      LITELLM_LICENSE: ${LITELLM_LICENSE:-}
      LITELLM_MODE: PRODUCTION
      LITELLM_LOG: ERROR
      JSON_LOGS: "True"
      USE_PRISMA_MIGRATE: "True"
      SEPARATE_HEALTH_APP: "1"
      SEPARATE_HEALTH_PORT: "4001"
      LITELLM_DROP_PARAMS: "True"
      CLAUDE_CODE_MODELS: ${CLAUDE_CODE_MODELS}
      CLAUDE_CODE_MIN_VERSION: "${CLAUDE_CODE_MIN_VERSION}"
    configs:
      - source: litellm-config-yaml
        target: /app/config.yaml
      - source: litellm-claude-code-hook-py
        target: /app/claude_code_hook.py
    healthcheck: # Defines the health check configuration for the container
      <<: *common-healthcheck
      test:
        - CMD-SHELL
        - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4001/health/liveliness')" # Command to execute for health check
      start_period: 60s # Wait 60 seconds after container start before beginning health checks
    networks:
      - internal
      - public
      - monitoring
    deploy:
      <<: *common-deploy
      labels:
        - traefik.enable=true
        - traefik.http.routers.litellm.rule=Host(`${LITELLM_HOST}`)
        - traefik.http.routers.litellm.entrypoints=${TRAEFIK_ROUTER_ENTRYPOINTS:-web}
        - traefik.http.routers.litellm.tls=${TRAEFIK_ROUTER_TLS:-false}
        - traefik.http.routers.litellm.tls.certresolver=${LETSENCRYPT_RESOLVER:-le}
        - traefik.http.services.litellm.loadbalancer.server.port=4000
        - traefik.http.routers.litellm.service=litellm

configs:
  cli-proxy-api-config-yaml:
    name: ${CLI_PROXY_API_CONFIG_NAME:-llmproxy_cli-proxy-api-config-yaml}
    external: true
  litellm-config-yaml:
    name: ${LITELLM_CONFIG_NAME:-llmproxy_litellm-config-yaml}
    external: true
  litellm-claude-code-hook-py:
    name: ${LITELLM_CLAUDE_CODE_HOOK_NAME:-llmproxy_litellm-claude-code-hook-py}
    external: true

networks:
  internal:
    name: ${PUBLIC_NETWORK:-llmproxy}
    external: true
  public:
    name: ${PUBLIC_NETWORK:-public}
    external: true
  monitoring:
    name: ${MONITORING_NETWORK:-monitoring}
    external: true