diff --git a/aenv/pyproject.toml b/aenv/pyproject.toml index 0dbb39a..a868733 100644 --- a/aenv/pyproject.toml +++ b/aenv/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aenvironment" -version = "0.1.5" +version = "0.1.6" description = "AEnvironment Python SDK - Production-grade environment for AI agent tools" readme = "README.md" requires-python = ">=3.10" diff --git a/aenv/src/aenv/__init__.py b/aenv/src/aenv/__init__.py index 12975e8..95019e6 100644 --- a/aenv/src/aenv/__init__.py +++ b/aenv/src/aenv/__init__.py @@ -20,7 +20,12 @@ """ from aenv.core.environment import Environment -from aenv.core.exceptions import AEnvError, EnvironmentError, ToolError +from aenv.core.exceptions import ( + AEnvError, + EnvironmentError, + ToolError, + UnrecoverableEnvironmentError, +) from aenv.core.function_registry import ( register_function, register_health, @@ -30,7 +35,7 @@ from aenv.core.models import EnvInstance, EnvStatus from aenv.core.tool import Tool, get_registry, register_tool -__version__ = "0.1.5" +__version__ = "0.1.6" __all__ = [ "Tool", "register_tool", @@ -42,6 +47,7 @@ "AEnvError", "ToolError", "EnvironmentError", + "UnrecoverableEnvironmentError", "EnvInstance", "EnvStatus", "setup_logging", diff --git a/aenv/src/aenv/core/__init__.py b/aenv/src/aenv/core/__init__.py index 60a1fcc..f840e93 100644 --- a/aenv/src/aenv/core/__init__.py +++ b/aenv/src/aenv/core/__init__.py @@ -20,6 +20,7 @@ NetworkError, ToolError, ToolTimeoutError, + UnrecoverableEnvironmentError, ) from aenv.core.function_registry import ( get_function_registry, @@ -39,4 +40,5 @@ "ToolError", "ToolTimeoutError", "NetworkError", + "UnrecoverableEnvironmentError", ] diff --git a/aenv/src/aenv/core/environment.py b/aenv/src/aenv/core/environment.py index ca7f664..24ed841 100644 --- a/aenv/src/aenv/core/environment.py +++ b/aenv/src/aenv/core/environment.py @@ -20,6 +20,7 @@ import asyncio import json import os +import random import traceback from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple @@ -33,7 +34,11 @@ from fastmcp.client.transports import StreamableHttpTransport from aenv.client.scheduler_client import AEnvSchedulerClient -from aenv.core.exceptions import EnvironmentError, ToolError +from aenv.core.exceptions import ( + EnvironmentError, + ToolError, + UnrecoverableEnvironmentError, +) from aenv.core.logging import getLogger from aenv.core.models import EnvInstance, EnvStatus from aenv.core.tool import Tool @@ -166,21 +171,59 @@ def __init__( self._client: Optional[AEnvSchedulerClient] = None self._mcp_client: Optional[Client] = None self._mcp_session_active: bool = False + self._consecutive_tool_errors: int = 0 + self._CIRCUIT_BREAKER_THRESHOLD = 5 def _log_prefix(self) -> str: - """Get log prefix with instance ID.""" + """Get log prefix with instance ID and SDK version.""" + from aenv import __version__ + instance_id = ( getattr(self._instance, "id", "None") if self._instance else "None" ) - return f"[ENV:{instance_id}]" + return f"[ENV:{instance_id}][sdk:v{__version__}]" + + async def _backoff(self, attempt: int, base: float = 2.0) -> None: + """Exponential backoff with jitter.""" + wait = base**attempt + random.uniform(0, 1) + logger.debug(f"{self._log_prefix()} Backoff {wait:.1f}s (attempt {attempt})") + await asyncio.sleep(wait) + + async def _rebuild_mcp_client(self) -> None: + """Destroy and recreate MCP client for session recovery.""" + logger.warning(f"{self._log_prefix()} Full MCP client rebuild") + if self._mcp_client is not None: + try: + await asyncio.wait_for(self._mcp_client.close(), timeout=2.0) + except Exception as e: + logger.debug(f"{self._log_prefix()} Error closing MCP client: {e}") + self._mcp_client = None + self._mcp_session_active = False + await asyncio.sleep(0.5) async def __aenter__(self): """Async context manager entry.""" await self.initialize() + max_attempts = 3 + for attempt in range(max_attempts): + try: + await self._ensure_mcp_session() + return self + except (Exception, asyncio.CancelledError) as e: + if attempt < max_attempts - 1: + logger.warning( + f"{self._log_prefix()} Initial session failed " + f"({attempt+1}/{max_attempts}), retrying: {type(e).__name__}: {e}" + ) + await self._rebuild_mcp_client() + await asyncio.sleep(1.0) + else: + raise return self async def __aexit__(self, exc_type, exc_val, exc_tb): """Async context manager exit.""" + await self._close_mcp_session() await self.release() async def initialize(self) -> bool: @@ -244,6 +287,17 @@ async def initialize(self) -> bool: f"Failed to initialize environment '{self.env_name}': {str(e)}" ) from e + async def _close_mcp_session(self): + """Close the persistent MCP session if one is active.""" + if self._mcp_client and self._mcp_session_active: + logger.info(f"{self._log_prefix()} Closing persistent MCP session") + try: + await self._mcp_client.__aexit__(None, None, None) + except Exception as e: + logger.warning(f"{self._log_prefix()} Error closing MCP session: {e}") + finally: + self._mcp_session_active = False + async def release(self): """Release environment resources.""" logger.info( @@ -254,7 +308,7 @@ async def release(self): try: await self._mcp_client.close() logger.debug(f"{self._log_prefix()} MCP client closed") - except Exception as e: + except (Exception, asyncio.CancelledError) as e: logger.warning( f"{self._log_prefix()} Failed to close MCP client: {str(e)}" ) @@ -621,6 +675,12 @@ async def call_tool( """ Execute a tool with given arguments using MCP client. + Retry strategy (idempotent-safe): + - Session establishment failures are retried (tool was never sent to server) + - Once call_tool_mcp() is invoked, the tool MAY have executed on the server. + On failure after invocation, we rebuild the session but do NOT re-invoke the + tool, because the tool may be non-idempotent (e.g. bash "echo x >> file"). + Args: tool_name: Name of the tool (format: "env_name/tool_name" or just "tool_name") arguments: Tool arguments @@ -630,11 +690,18 @@ async def call_tool( Tool execution result Raises: - ToolError: If tool execution fails - ToolTimeoutError: If execution times out + ToolError: If tool execution fails after invocation + EnvironmentError: If session cannot be established """ await self._ensure_initialized() + # Circuit breaker: fail fast if too many consecutive tool errors + if self._consecutive_tool_errors >= self._CIRCUIT_BREAKER_THRESHOLD: + raise UnrecoverableEnvironmentError( + f"Circuit breaker open: {self._consecutive_tool_errors} consecutive failures", + env_name=self.env_name, + ) + # Parse tool name if "/" in tool_name: env_name, actual_tool_name = tool_name.split("/", 1) @@ -649,8 +716,39 @@ async def call_tool( f"{self._log_prefix()} Executing tool: {actual_tool_name} in environment {self.env_name}, arguments={arguments}" ) + # Establish session (safe to retry) + max_session_attempts = 3 + client = None + + for attempt in range(max_session_attempts): + try: + client = await self._ensure_mcp_session() + break + except (Exception, asyncio.CancelledError) as e: + if ( + isinstance(e, httpx.HTTPStatusError) + and 400 <= e.response.status_code < 500 + ): + raise EnvironmentError( + f"Non-retryable error establishing session: HTTP {e.response.status_code}" + ) from e + + if attempt < max_session_attempts - 1: + logger.warning( + f"{self._log_prefix()} Session establish failed ({attempt+1}/{max_session_attempts}), " + f"retrying: {type(e).__name__}: {e}" + ) + await self._backoff(attempt, base=1.5) + await self._rebuild_mcp_client() + continue + + raise EnvironmentError( + f"Failed to establish session after {max_session_attempts} attempts: {e}", + env_name=self.env_name, + ) from e + + # Execute tool (not safe to retry — may be non-idempotent) try: - client = await self._ensure_mcp_session() result = await client.call_tool_mcp( name=actual_tool_name, arguments=arguments, timeout=timeout ) @@ -666,20 +764,28 @@ async def call_tool( else: content.append({"type": "text", "text": str(item)}) + self._consecutive_tool_errors = 0 return ToolResult(content=content, is_error=result.isError) - except Exception as e: + except (Exception, asyncio.CancelledError) as e: + self._mcp_session_active = False + self._consecutive_tool_errors += 1 + # Rebuild client to break ClosedResourceError cascade + await self._rebuild_mcp_client() + + # Tool may have executed — do NOT retry logger.error( - f"{self._log_prefix()} Tool execution encountered an issue: {str(e)} | " - f"Type: {type(e).__name__} | " + f"{self._log_prefix()} Tool '{actual_tool_name}' failed after invocation " + f"({type(e).__name__}: {e}). Tool may have executed on server — " + f"NOT retrying (non-idempotent safety). | " f"Environment: {self.env_name} | " - f"Tool: {actual_tool_name} | " f"Arguments: {arguments} | " f"Timeout: {timeout or self.timeout}s | " f"MCP URL: {self.aenv_data_url}" ) raise ToolError( - f"Tool '{actual_tool_name}' execution encountered an issue: {str(e)}" + f"Tool '{actual_tool_name}' execution failed: {e}. " + f"Tool may have already executed on server — not retried for safety." ) async def get_env_info(self) -> Dict[str, Any]: @@ -953,7 +1059,7 @@ async def _ensure_mcp_session(self) -> Client: if self._mcp_client is not None: try: await self._mcp_client.close() - except Exception as e: + except (Exception, asyncio.CancelledError) as e: logger.debug( f"{self._log_prefix()} Error closing stale MCP client: {e}" ) @@ -968,10 +1074,7 @@ async def _ensure_mcp_session(self) -> Client: f"{self._log_prefix()} MCP session established and will be reused" ) return client - except Exception as e: + except (Exception, asyncio.CancelledError) as e: self._mcp_client = None self._mcp_session_active = False - logger.error( - f"{self._log_prefix()} Failed to establish MCP session: {e}" - ) raise EnvironmentError(f"Failed to establish MCP session: {e}") from e diff --git a/aenv/src/aenv/core/exceptions.py b/aenv/src/aenv/core/exceptions.py index 5b31c1c..d9bc655 100644 --- a/aenv/src/aenv/core/exceptions.py +++ b/aenv/src/aenv/core/exceptions.py @@ -82,6 +82,12 @@ def __init__( self.status_code = status_code +class UnrecoverableEnvironmentError(EnvironmentError): + """Environment is in an unrecoverable state (circuit breaker open).""" + + pass + + class NetworkError(AEnvError): """Exception raised for network-related errors.""" diff --git a/aenv/src/aenv/tests/test_e2e_fixes.py b/aenv/src/aenv/tests/test_e2e_fixes.py new file mode 100644 index 0000000..e924643 --- /dev/null +++ b/aenv/src/aenv/tests/test_e2e_fixes.py @@ -0,0 +1,265 @@ +# Copyright 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +E2E tests for PR #71 fixes: session persistence, circuit breaker, __aenter__ retry. + +Requires access to staging api-service. Set AENV_SYSTEM_URL env var. + +Usage: + AENV_SYSTEM_URL=http://6.3.209.180 RUN_E2E=1 uv run pytest src/aenv/tests/test_e2e_fixes.py -v -s +""" + +import asyncio +import os +import time + +import pytest + +from aenv import Environment +from aenv.core.exceptions import UnrecoverableEnvironmentError + +AENV_URL = os.getenv("AENV_SYSTEM_URL", "http://6.3.209.180") +ENV_NAME = "persistent-bash-env@1.1.6" +# Tool name as registered in persistent_bash_session: "run" +TOOL_NAME = f"{ENV_NAME}/run" + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.skipif( + not os.getenv("RUN_E2E", ""), + reason="Set RUN_E2E=1 to run e2e tests", + ), +] + + +def _run_args(command: str, timeout: int = 60) -> dict: + """Build arguments for the persistent-bash-session 'run' tool.""" + return {"command": command, "timeout": timeout} + + +class TestSessionPersistence: + """Verify MCP session is reused across multiple tool calls (fix.md core fix).""" + + @pytest.mark.asyncio + async def test_multiple_tool_calls_reuse_session(self): + """Multiple call_tool invocations should use the same MCP session.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + assert env._mcp_session_active is True + + tools = await env.list_tools() + assert len(tools) > 0 + print(f"\n Found {len(tools)} tools: {[t['name'] for t in tools]}") + + for i in range(5): + result = await env.call_tool( + TOOL_NAME, + _run_args(f"echo 'e2e test iteration {i}'"), + timeout=15.0, + ) + assert result.is_error is False + print(f" call_tool #{i}: ok") + + assert env._mcp_session_active is True + assert env._consecutive_tool_errors == 0 + + @pytest.mark.asyncio + async def test_session_survives_idle_period(self): + """Session should survive a short idle period without initialize/DELETE cycle.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + r1 = await env.call_tool( + TOOL_NAME, _run_args("echo 'before idle'"), timeout=15.0 + ) + assert r1.is_error is False + print("\n Before idle: ok") + + print(" Sleeping 10s to simulate idle period...") + await asyncio.sleep(10) + + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'after idle'"), timeout=15.0 + ) + assert r2.is_error is False + print(" After idle: ok") + + assert env._mcp_session_active is True + + +class TestCircuitBreakerE2E: + """Verify circuit breaker prevents cascade failures.""" + + @pytest.mark.asyncio + async def test_circuit_breaker_counter_resets_on_success(self): + """Successful calls should reset the consecutive error counter.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'success'"), timeout=15.0 + ) + assert result.is_error is False + assert env._consecutive_tool_errors == 0 + + # Simulate prior failures then successful call should reset + env._consecutive_tool_errors = 4 + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'reset'"), timeout=15.0 + ) + assert result.is_error is False + assert env._consecutive_tool_errors == 0 + print(f"\n Counter reset verified: {env._consecutive_tool_errors}") + + @pytest.mark.asyncio + async def test_circuit_breaker_blocks_after_threshold(self): + """Setting counter to threshold should block next call.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + env._consecutive_tool_errors = env._CIRCUIT_BREAKER_THRESHOLD + + with pytest.raises(UnrecoverableEnvironmentError) as exc_info: + await env.call_tool( + TOOL_NAME, + _run_args("echo 'should not execute'"), + timeout=15.0, + ) + + assert "Circuit breaker open" in str(exc_info.value) + print(f"\n Circuit breaker triggered: {exc_info.value}") + + +class TestToolFailureRebuild: + """Verify tool failure triggers MCP client rebuild and recovery.""" + + @pytest.mark.asyncio + async def test_recovery_after_tool_error(self): + """After a tool error, the next call should work via rebuilt client.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + r1 = await env.call_tool(TOOL_NAME, _run_args("echo 'ok'"), timeout=15.0) + assert r1.is_error is False + + # Call a non-existent tool to trigger error + rebuild + try: + await env.call_tool( + f"{ENV_NAME}/nonexistent_tool_xyz", + {"arg": "should fail"}, + timeout=10.0, + ) + except Exception as e: + print(f"\n Expected error on bad tool: {type(e).__name__}: {e}") + assert env._consecutive_tool_errors >= 1 + + # Recovery: next call should work (client was rebuilt) + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'recovered'"), timeout=15.0 + ) + assert r2.is_error is False + assert env._consecutive_tool_errors == 0 + print(" Recovery verified: ok") + + +class TestAenterRetryE2E: + """Verify __aenter__ session establishment with retry.""" + + @pytest.mark.asyncio + async def test_normal_aenter_succeeds(self): + """Normal __aenter__ should establish session successfully.""" + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + assert env._mcp_session_active is True + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'aenter ok'"), timeout=15.0 + ) + assert result.is_error is False + print("\n __aenter__ success: ok") + + +class TestEndToEndWorkflow: + """Full workflow test simulating real agent usage.""" + + @pytest.mark.asyncio + async def test_full_agent_workflow(self): + """Simulate a real agent workflow: create env, list tools, execute + multiple commands, idle, execute more, release.""" + print("\n--- Full E2E Workflow ---") + start = time.time() + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + # Step 1: List tools + tools = await env.list_tools() + tool_names = [t["name"] for t in tools] + print(f" 1. Listed {len(tools)} tools: {tool_names}") + assert len(tools) > 0 + + # Step 2: Execute a sequence of commands + commands = [ + "pwd", + "ls -la", + "echo 'hello world' > /tmp/test.txt", + "cat /tmp/test.txt", + "rm /tmp/test.txt && echo 'cleaned up'", + ] + for cmd in commands: + result = await env.call_tool(TOOL_NAME, _run_args(cmd), timeout=15.0) + assert result.is_error is False + print(f" 2. cmd='{cmd}' -> ok") + + # Step 3: Idle (simulate LLM thinking) + print(" 3. Idle 5s (simulating LLM thinking)...") + await asyncio.sleep(5) + + # Step 4: More commands after idle + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'still alive after idle'"), timeout=15.0 + ) + assert result.is_error is False + print(" 4. Post-idle call: ok") + + # Verify session health + assert env._mcp_session_active is True + assert env._consecutive_tool_errors == 0 + + elapsed = time.time() - start + print(f" Total elapsed: {elapsed:.1f}s") + print("--- Workflow Complete ---") diff --git a/aenv/src/aenv/tests/test_e2e_resilience.py b/aenv/src/aenv/tests/test_e2e_resilience.py new file mode 100644 index 0000000..c3fd6ec --- /dev/null +++ b/aenv/src/aenv/tests/test_e2e_resilience.py @@ -0,0 +1,447 @@ +# Copyright 2025. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +E2E resilience tests: simulate api-service pod crash and network jitter. + +These tests verify the SDK's ability to: +1. Recover from api-service pod restarts during tool calls +2. Handle transient network failures gracefully +3. Rebuild MCP sessions after connection loss +4. Maintain circuit breaker correctness under failure scenarios + +Requires access to staging api-service with kubectl access to the cluster. + +Usage: + export KUBECONFIG=/Users/jun/.kube/tydd-staging-config + AENV_SYSTEM_URL=http://api-service-k8s.aenv.svc.tydd-staging.alipay.net \ + RUN_E2E=1 uv run pytest src/aenv/tests/test_e2e_resilience.py -v -s +""" + +import asyncio +import os +import subprocess +import time + +import pytest + +from aenv import Environment +from aenv.core.exceptions import AEnvError, ToolError + +AENV_URL = os.getenv( + "AENV_SYSTEM_URL", "http://api-service-k8s.aenv.svc.tydd-staging.alipay.net" +) +ENV_NAME = "persistent-bash-env@1.1.6" +TOOL_NAME = f"{ENV_NAME}/run" +KUBECONFIG = os.getenv("KUBECONFIG", "/Users/jun/.kube/tydd-staging-config") +NAMESPACE = "aenv" +DEPLOYMENT = "api-service" + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.skipif( + not os.getenv("RUN_E2E", ""), + reason="Set RUN_E2E=1 to run e2e tests", + ), +] + + +def _run_args(command: str, timeout: int = 60) -> dict: + return {"command": command, "timeout": timeout} + + +def _kubectl(*args: str, check: bool = True) -> subprocess.CompletedProcess: + """Run kubectl command against the staging cluster.""" + cmd = ["kubectl", f"--kubeconfig={KUBECONFIG}", "-n", NAMESPACE, *args] + print(f" [kubectl] {' '.join(cmd)}") + return subprocess.run(cmd, capture_output=True, text=True, timeout=60, check=check) + + +def _get_pod_name(deployment: str) -> str: + """Get the first running pod name for a deployment.""" + result = _kubectl( + "get", + "pods", + "-l", + f"sigma.ali/app-name={deployment}", + "-o", + "jsonpath={.items[0].metadata.name}", + ) + pod = result.stdout.strip() + if not pod: + pytest.skip(f"No pods found for deployment {deployment}") + return pod + + +def _wait_for_rollout(deployment: str, timeout: int = 120): + """Wait for deployment rollout to complete.""" + _kubectl("rollout", "status", f"deployment/{deployment}", f"--timeout={timeout}s") + + +class TestApiServicePodCrash: + """Verify SDK recovery when api-service pod is restarted during operations.""" + + @pytest.mark.asyncio + async def test_tool_call_recovery_after_pod_restart(self): + """ + Scenario: SDK has an active session, api-service pod restarts, + SDK should recover and execute tool calls after the pod is back. + + Steps: + 1. Establish session and execute a tool call successfully + 2. Restart api-service pod (kubectl rollout restart) + 3. Wait for new pod to be ready + 4. Execute another tool call - should recover via session rebuild + """ + print("\n--- Test: Tool Call Recovery After Pod Restart ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="15m", + ) as env: + # Step 1: Verify baseline works + r1 = await env.call_tool( + TOOL_NAME, _run_args("echo 'before restart'"), timeout=15.0 + ) + assert r1.is_error is False + print(" Step 1: Pre-restart tool call OK") + + # Step 2: Restart api-service pod + print(" Step 2: Restarting api-service deployment...") + _kubectl("rollout", "restart", f"deployment/{DEPLOYMENT}") + + # Step 3: Wait for new pod to be ready + print(" Step 3: Waiting for rollout to complete...") + _wait_for_rollout(DEPLOYMENT, timeout=120) + # Extra buffer for DNS propagation and readiness + await asyncio.sleep(5) + print(" Step 3: New pod is ready") + + # Step 4: Call tool again - SDK should rebuild session + print(" Step 4: Attempting post-restart tool call...") + max_recovery_attempts = 5 + recovered = False + for attempt in range(max_recovery_attempts): + try: + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'after restart'"), timeout=30.0 + ) + if not r2.is_error: + recovered = True + print(f" Step 4: Recovery succeeded on attempt {attempt + 1}") + break + except (ToolError, AEnvError) as e: + print( + f" Step 4: Attempt {attempt + 1} failed: {type(e).__name__}: {e}" + ) + if attempt < max_recovery_attempts - 1: + await asyncio.sleep(3) + + assert recovered, ( + f"SDK failed to recover after api-service pod restart " + f"({max_recovery_attempts} attempts)" + ) + print("--- Test PASSED ---") + + @pytest.mark.asyncio + async def test_new_session_after_pod_restart(self): + """ + Scenario: After pod restart, creating a brand new Environment should work. + This tests the SDK's initialize + session establishment path. + """ + print("\n--- Test: New Session After Pod Restart ---") + + # Step 1: Restart pod first + print(" Step 1: Restarting api-service deployment...") + _kubectl("rollout", "restart", f"deployment/{DEPLOYMENT}") + _wait_for_rollout(DEPLOYMENT, timeout=120) + await asyncio.sleep(5) + print(" Step 1: New pod ready") + + # Step 2: Create a brand new environment + print(" Step 2: Creating new environment...") + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=60.0, + ttl="10m", + ) as env: + assert env._mcp_session_active is True + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'fresh session works'"), timeout=15.0 + ) + assert result.is_error is False + print(" Step 2: New session established and tool call succeeded") + + print("--- Test PASSED ---") + + +class TestNetworkJitter: + """ + Verify SDK handles transient network disruptions. + + Uses iptables-like simulation via kubectl exec to inject network faults + into the api-service pod, or tests behavior when MCP proxy encounters + connection errors. + """ + + @pytest.mark.asyncio + async def test_tool_call_succeeds_after_transient_failure(self): + """ + Scenario: Simulate transient failure by corrupting the MCP session, + then verify the SDK rebuilds and recovers. + + This avoids needing iptables access by directly testing the SDK's + internal recovery path: force-close the session, then call_tool. + """ + print("\n--- Test: Recovery After Transient Session Failure ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + # Baseline + r1 = await env.call_tool( + TOOL_NAME, _run_args("echo 'baseline'"), timeout=15.0 + ) + assert r1.is_error is False + print(" Baseline call: OK") + + # Simulate network jitter by destroying MCP session + print(" Simulating network jitter (destroying MCP session)...") + if env._mcp_client is not None: + try: + await env._mcp_client.close() + except Exception: + pass + env._mcp_session_active = False + + # Next call should trigger rebuild and succeed + print(" Attempting recovery call...") + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'recovered from jitter'"), timeout=30.0 + ) + assert r2.is_error is False + assert env._mcp_session_active is True + print(" Recovery call: OK") + + print("--- Test PASSED ---") + + @pytest.mark.asyncio + async def test_multiple_rapid_failures_trigger_circuit_breaker(self): + """ + Scenario: Multiple consecutive tool failures should trigger + the circuit breaker, preventing cascade failures. + """ + print("\n--- Test: Circuit Breaker Under Rapid Failures ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + # Baseline + r1 = await env.call_tool(TOOL_NAME, _run_args("echo 'ok'"), timeout=15.0) + assert r1.is_error is False + assert env._consecutive_tool_errors == 0 + + # Simulate consecutive failures by setting the counter directly + env._consecutive_tool_errors = env._CIRCUIT_BREAKER_THRESHOLD + + # Circuit breaker should block + from aenv.core.exceptions import UnrecoverableEnvironmentError + + with pytest.raises(UnrecoverableEnvironmentError) as exc_info: + await env.call_tool( + TOOL_NAME, _run_args("echo 'blocked'"), timeout=15.0 + ) + print(f" Circuit breaker triggered: {exc_info.value}") + + # Reset and verify recovery + env._consecutive_tool_errors = 0 + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'reset works'"), timeout=15.0 + ) + assert r2.is_error is False + print(" After reset, tool call succeeds") + + print("--- Test PASSED ---") + + @pytest.mark.asyncio + async def test_session_rebuild_preserves_instance(self): + """ + Scenario: After MCP session rebuild, the same underlying instance + should still be used (instance ID should not change). + """ + print("\n--- Test: Session Rebuild Preserves Instance ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + # Get instance ID + instance_id_before = env._instance.id if env._instance else None + assert instance_id_before is not None + print(f" Instance ID before: {instance_id_before}") + + # Execute a command that writes a unique marker + marker = f"resilience_test_{int(time.time())}" + await env.call_tool( + TOOL_NAME, _run_args(f"echo '{marker}' > /tmp/marker.txt"), timeout=15.0 + ) + + # Simulate session loss (like network jitter destroying the connection) + # Set session inactive so _ensure_mcp_session will rebuild on next call + print(" Simulating session loss...") + env._mcp_session_active = False + + # Verify same instance (session rebuild doesn't create new instance) + instance_id_after = env._instance.id if env._instance else None + assert ( + instance_id_before == instance_id_after + ), f"Instance ID changed: {instance_id_before} -> {instance_id_after}" + print(f" Instance ID after loss: {instance_id_after} (same)") + + # Next call_tool will trigger _ensure_mcp_session which rebuilds the session + # The key point: it should reconnect to the SAME instance + r = await env.call_tool( + TOOL_NAME, _run_args("cat /tmp/marker.txt"), timeout=30.0 + ) + assert r.is_error is False + assert marker in str(r.content), f"Marker not found in output: {r.content}" + print(f" Marker file preserved after reconnect: {marker}") + + print("--- Test PASSED ---") + + @pytest.mark.asyncio + async def test_concurrent_tool_calls_during_jitter(self): + """ + Scenario: Multiple concurrent tool calls when session is unstable. + Only one should trigger rebuild; others should wait or fail gracefully. + """ + print("\n--- Test: Concurrent Calls During Network Jitter ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=30.0, + ttl="10m", + ) as env: + # Baseline + r1 = await env.call_tool( + TOOL_NAME, _run_args("echo 'baseline'"), timeout=15.0 + ) + assert r1.is_error is False + + # Launch 3 concurrent calls + print(" Launching 3 concurrent tool calls...") + tasks = [ + env.call_tool( + TOOL_NAME, _run_args(f"echo 'concurrent_{i}'"), timeout=30.0 + ) + for i in range(3) + ] + results = await asyncio.gather(*tasks, return_exceptions=True) + + successes = sum( + 1 for r in results if not isinstance(r, Exception) and not r.is_error + ) + failures = sum(1 for r in results if isinstance(r, Exception)) + print(f" Results: {successes} successes, {failures} failures") + + # At least some should succeed (all if session is healthy) + assert successes >= 1, f"Expected at least 1 success, got {successes}" + print(f" Concurrent calls handled: {successes}/3 succeeded") + + print("--- Test PASSED ---") + + +class TestInitializeRecovery: + """Verify SDK handles failures during instance initialization.""" + + @pytest.mark.asyncio + async def test_initialize_with_api_service_restart(self): + """ + Scenario: Start initializing an environment, restart api-service + mid-initialization, verify that the SDK can still complete or + properly report the failure. + """ + print("\n--- Test: Initialize During API Service Restart ---") + + # Restart api-service first to ensure clean state + _kubectl("rollout", "restart", f"deployment/{DEPLOYMENT}") + _wait_for_rollout(DEPLOYMENT, timeout=120) + await asyncio.sleep(5) + + # Now try to create environment (should succeed with fresh api-service) + start = time.time() + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=60.0, + startup_timeout=300.0, + ttl="10m", + ) as env: + elapsed = time.time() - start + print(f" Environment initialized in {elapsed:.1f}s") + assert env._mcp_session_active is True + + result = await env.call_tool( + TOOL_NAME, _run_args("echo 'init recovery ok'"), timeout=15.0 + ) + assert result.is_error is False + print(" Post-init tool call: OK") + + print("--- Test PASSED ---") + + @pytest.mark.asyncio + async def test_retry_failed_session_idle_period(self): + """ + Retry the previously failed test: session survives idle period. + This was failing due to FaaS container startup slowness. + """ + print("\n--- Test: Session Survives Idle Period (retry) ---") + + async with Environment( + env_name=ENV_NAME, + aenv_url=AENV_URL, + timeout=60.0, + startup_timeout=500.0, + ttl="15m", + ) as env: + r1 = await env.call_tool( + TOOL_NAME, _run_args("echo 'before idle'"), timeout=15.0 + ) + assert r1.is_error is False + print(" Before idle: OK") + + print(" Sleeping 10s to simulate idle period...") + await asyncio.sleep(10) + + r2 = await env.call_tool( + TOOL_NAME, _run_args("echo 'after idle'"), timeout=15.0 + ) + assert r2.is_error is False + print(" After idle: OK") + + assert env._mcp_session_active is True + + print("--- Test PASSED ---") diff --git a/aenv/src/aenv/tests/test_environment.py b/aenv/src/aenv/tests/test_environment.py index ef7e7ee..7ad3328 100644 --- a/aenv/src/aenv/tests/test_environment.py +++ b/aenv/src/aenv/tests/test_environment.py @@ -335,3 +335,247 @@ async def _get_mcp_client_side_effect(): mock_client.close.assert_awaited_once() # Second call went through the new client mock_client_2.call_tool_mcp.assert_awaited_once() + + +class TestCircuitBreaker: + """Tests for circuit breaker functionality. + + Verifies that after _CIRCUIT_BREAKER_THRESHOLD consecutive tool failures, + the circuit breaker opens and raises UnrecoverableEnvironmentError. + Also tests that successful tool execution resets the error counter. + """ + + def _make_env(self): + """Create an Environment instance pre-configured for unit testing.""" + env = Environment("test-env") + env._initialized = True + + class _FakeInstance: + ip = "127.0.0.1" + id = "fake-id" + + env._instance = _FakeInstance() + env.proxy_headers = {"AEnvCore-MCPProxy-URL": "http://127.0.0.1:8081"} + return env + + def _make_mock_client(self): + """Build a mock fastmcp Client with sensible defaults.""" + mock_client = MagicMock() + mock_client.is_connected.return_value = True + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + + call_result = MagicMock() + call_result.content = [] + call_result.isError = False + mock_client.call_tool_mcp = AsyncMock(return_value=call_result) + mock_client.list_tools = AsyncMock(return_value=[]) + mock_client.close = AsyncMock() + + return mock_client + + def _patch_get_mcp_client(self, env, mock_client): + """Create a side-effect function for _get_mcp_client that also + sets env._mcp_client, mirroring the real implementation.""" + + async def _side_effect(): + env._mcp_client = mock_client + return mock_client + + return patch.object(env, "_get_mcp_client", side_effect=_side_effect) + + @pytest.mark.asyncio + async def test_circuit_breaker_opens_after_threshold(self): + """Force 5 consecutive tool failures, verify 6th call raises UnrecoverableEnvironmentError.""" + from aenv.core.exceptions import UnrecoverableEnvironmentError + + env = self._make_env() + mock_client = self._make_mock_client() + + # Make call_tool_mcp raise an exception + mock_client.call_tool_mcp = AsyncMock( + side_effect=Exception("Tool execution failed") + ) + + with self._patch_get_mcp_client(env, mock_client): + # First 5 failures should raise ToolError + for i in range(5): + with pytest.raises(ToolError): + await env.call_tool("failing_tool", {"x": i}) + assert env._consecutive_tool_errors == i + 1 + + # 6th call should raise UnrecoverableEnvironmentError (circuit breaker open) + with pytest.raises(UnrecoverableEnvironmentError) as exc_info: + await env.call_tool("failing_tool", {"x": 5}) + + assert "Circuit breaker open" in str(exc_info.value) + assert "5 consecutive failures" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_circuit_breaker_resets_on_success(self): + """Force 3 failures, then 1 success, verify counter resets to 0.""" + env = self._make_env() + mock_client = self._make_mock_client() + + # First 3 calls fail + fail_count = 0 + + async def _conditional_failure(*args, **kwargs): + nonlocal fail_count + fail_count += 1 + if fail_count <= 3: + raise Exception("Tool execution failed") + # 4th call succeeds + result = MagicMock() + result.content = [] + result.isError = False + return result + + mock_client.call_tool_mcp = AsyncMock(side_effect=_conditional_failure) + + with self._patch_get_mcp_client(env, mock_client): + # First 3 calls fail + for i in range(3): + with pytest.raises(ToolError): + await env.call_tool("flaky_tool", {"x": i}) + assert env._consecutive_tool_errors == i + 1 + + # 4th call succeeds + result = await env.call_tool("flaky_tool", {"x": 3}) + assert result.is_error is False + # Counter should be reset to 0 + assert env._consecutive_tool_errors == 0 + + @pytest.mark.asyncio + async def test_tool_failure_triggers_rebuild(self): + """Verify _rebuild_mcp_client is called when tool execution fails.""" + env = self._make_env() + mock_client = self._make_mock_client() + + # Make call_tool_mcp raise an exception + mock_client.call_tool_mcp = AsyncMock( + side_effect=Exception("Tool execution failed") + ) + + with self._patch_get_mcp_client(env, mock_client): + with patch.object( + env, "_rebuild_mcp_client", new=AsyncMock() + ) as mock_rebuild: + with pytest.raises(ToolError): + await env.call_tool("failing_tool", {"x": 1}) + + # Verify rebuild was called + mock_rebuild.assert_awaited_once() + + +class TestAenterRetry: + """Tests for __aenter__ retry logic. + + Verifies that when _ensure_mcp_session() fails on first attempt in __aenter__, + it retries once after rebuild and sleep. Tests both success-on-retry and + failure-on-both cases. + """ + + def _make_env(self): + """Create an Environment instance pre-configured for unit testing.""" + env = Environment("test-env") + env._initialized = True + + class _FakeInstance: + ip = "127.0.0.1" + id = "fake-id" + + env._instance = _FakeInstance() + env.proxy_headers = {"AEnvCore-MCPProxy-URL": "http://127.0.0.1:8081"} + return env + + @pytest.mark.asyncio + async def test_aenter_retries_on_session_failure(self): + """Mock _ensure_mcp_session to fail once then succeed, verify __aenter__ succeeds.""" + env = self._make_env() + + # First call fails, second call succeeds + call_count = 0 + + async def _conditional_failure(): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise Exception("Session establishment failed") + # Second call succeeds (no-op, just don't raise) + return + + with patch.object(env, "initialize", new=AsyncMock()): + with patch.object( + env, "_ensure_mcp_session", side_effect=_conditional_failure + ) as mock_ensure: + with patch.object( + env, "_rebuild_mcp_client", new=AsyncMock() + ) as mock_rebuild: + result = await env.__aenter__() + + # Should succeed on retry + assert result is env + # _ensure_mcp_session called twice (fail, then succeed) + assert mock_ensure.await_count == 2 + # _rebuild_mcp_client called once between retries + mock_rebuild.assert_awaited_once() + + @pytest.mark.asyncio + async def test_aenter_raises_after_max_retries(self): + """Mock _ensure_mcp_session to always fail, verify exception is raised after 2 attempts.""" + env = self._make_env() + + # Always fail + async def _always_fail(): + raise Exception("Session establishment failed") + + with patch.object(env, "initialize", new=AsyncMock()): + with patch.object( + env, "_ensure_mcp_session", side_effect=_always_fail + ) as mock_ensure: + with patch.object( + env, "_rebuild_mcp_client", new=AsyncMock() + ) as mock_rebuild: + with pytest.raises(Exception) as exc_info: + await env.__aenter__() + + assert "Session establishment failed" in str(exc_info.value) + # _ensure_mcp_session called twice (max_attempts=2) + assert mock_ensure.await_count == 2 + # _rebuild_mcp_client called once between retries + mock_rebuild.assert_awaited_once() + + @pytest.mark.asyncio + async def test_aenter_calls_rebuild_between_retries(self): + """Verify _rebuild_mcp_client is called between retry attempts.""" + env = self._make_env() + + call_count = 0 + + async def _conditional_failure(): + nonlocal call_count + call_count += 1 + if call_count <= 1: + raise Exception("Session establishment failed") + # Second call succeeds + + rebuild_called_after_first_attempt = False + + async def _track_rebuild(): + nonlocal rebuild_called_after_first_attempt + if call_count == 1: + rebuild_called_after_first_attempt = True + + with patch.object(env, "initialize", new=AsyncMock()): + with patch.object( + env, "_ensure_mcp_session", side_effect=_conditional_failure + ): + with patch.object( + env, "_rebuild_mcp_client", side_effect=_track_rebuild + ) as mock_rebuild: + await env.__aenter__() + + # Verify rebuild was called after first attempt + assert rebuild_called_after_first_attempt is True + mock_rebuild.assert_awaited_once() diff --git a/aenv/uv.lock b/aenv/uv.lock index de22ec8..479781c 100644 --- a/aenv/uv.lock +++ b/aenv/uv.lock @@ -20,7 +20,7 @@ wheels = [ [[package]] name = "aenvironment" -version = "0.1.4" +version = "0.1.6" source = { editable = "." } dependencies = [ { name = "anyio" }, @@ -283,14 +283,24 @@ sdist = { url = "https://files.pythonhosted.org/packages/92/88/b8527e1b00c1811db wheels = [ { url = "https://files.pythonhosted.org/packages/6a/80/ea4ead0c5d52a9828692e7df20f0eafe8d26e671ce4883a0a146bb91049e/caio-0.9.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ca6c8ecda611478b6016cb94d23fd3eb7124852b985bdec7ecaad9f3116b9619", size = 36836 }, { url = "https://files.pythonhosted.org/packages/17/b9/36715c97c873649d1029001578f901b50250916295e3dddf20c865438865/caio-0.9.25-cp310-cp310-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db9b5681e4af8176159f0d6598e73b2279bb661e718c7ac23342c550bd78c241", size = 79695 }, + { url = "https://files.pythonhosted.org/packages/0b/ab/07080ecb1adb55a02cbd8ec0126aa8e43af343ffabb6a71125b42670e9a1/caio-0.9.25-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:bf61d7d0c4fd10ffdd98ca47f7e8db4d7408e74649ffaf4bef40b029ada3c21b", size = 79457 }, + { url = "https://files.pythonhosted.org/packages/88/95/dd55757bb671eb4c376e006c04e83beb413486821f517792ea603ef216e9/caio-0.9.25-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:ab52e5b643f8bbd64a0605d9412796cd3464cb8ca88593b13e95a0f0b10508ae", size = 77705 }, { url = "https://files.pythonhosted.org/packages/ec/90/543f556fcfcfa270713eef906b6352ab048e1e557afec12925c991dc93c2/caio-0.9.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d6956d9e4a27021c8bd6c9677f3a59eb1d820cc32d0343cea7961a03b1371965", size = 36839 }, { url = "https://files.pythonhosted.org/packages/51/3b/36f3e8ec38dafe8de4831decd2e44c69303d2a3892d16ceda42afed44e1b/caio-0.9.25-cp311-cp311-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf84bfa039f25ad91f4f52944452a5f6f405e8afab4d445450978cd6241d1478", size = 80255 }, + { url = "https://files.pythonhosted.org/packages/df/ce/65e64867d928e6aff1b4f0e12dba0ef6d5bf412c240dc1df9d421ac10573/caio-0.9.25-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:ae3d62587332bce600f861a8de6256b1014d6485cfd25d68c15caf1611dd1f7c", size = 80052 }, + { url = "https://files.pythonhosted.org/packages/46/90/e278863c47e14ec58309aa2e38a45882fbe67b4cc29ec9bc8f65852d3e45/caio-0.9.25-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fc220b8533dcf0f238a6b1a4a937f92024c71e7b10b5a2dfc1c73604a25709bc", size = 78273 }, { url = "https://files.pythonhosted.org/packages/d3/25/79c98ebe12df31548ba4eaf44db11b7cad6b3e7b4203718335620939083c/caio-0.9.25-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fb7ff95af4c31ad3f03179149aab61097a71fd85e05f89b4786de0359dffd044", size = 36983 }, { url = "https://files.pythonhosted.org/packages/a3/2b/21288691f16d479945968a0a4f2856818c1c5be56881d51d4dac9b255d26/caio-0.9.25-cp312-cp312-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:97084e4e30dfa598449d874c4d8e0c8d5ea17d2f752ef5e48e150ff9d240cd64", size = 82012 }, + { url = "https://files.pythonhosted.org/packages/03/c4/8a1b580875303500a9c12b9e0af58cb82e47f5bcf888c2457742a138273c/caio-0.9.25-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:4fa69eba47e0f041b9d4f336e2ad40740681c43e686b18b191b6c5f4c5544bfb", size = 81502 }, + { url = "https://files.pythonhosted.org/packages/d1/1c/0fe770b8ffc8362c48134d1592d653a81a3d8748d764bec33864db36319d/caio-0.9.25-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:6bebf6f079f1341d19f7386db9b8b1f07e8cc15ae13bfdaff573371ba0575d69", size = 80200 }, { url = "https://files.pythonhosted.org/packages/31/57/5e6ff127e6f62c9f15d989560435c642144aa4210882f9494204bc892305/caio-0.9.25-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d6c2a3411af97762a2b03840c3cec2f7f728921ff8adda53d7ea2315a8563451", size = 36979 }, { url = "https://files.pythonhosted.org/packages/a3/9f/f21af50e72117eb528c422d4276cbac11fb941b1b812b182e0a9c70d19c5/caio-0.9.25-cp313-cp313-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0998210a4d5cd5cb565b32ccfe4e53d67303f868a76f212e002a8554692870e6", size = 81900 }, + { url = "https://files.pythonhosted.org/packages/9c/12/c39ae2a4037cb10ad5eb3578eb4d5f8c1a2575c62bba675f3406b7ef0824/caio-0.9.25-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:1a177d4777141b96f175fe2c37a3d96dec7911ed9ad5f02bac38aaa1c936611f", size = 81523 }, + { url = "https://files.pythonhosted.org/packages/22/59/f8f2e950eb4f1a5a3883e198dca514b9d475415cb6cd7b78b9213a0dd45a/caio-0.9.25-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:9ed3cfb28c0e99fec5e208c934e5c157d0866aa9c32aa4dc5e9b6034af6286b7", size = 80243 }, { url = "https://files.pythonhosted.org/packages/69/ca/a08fdc7efdcc24e6a6131a93c85be1f204d41c58f474c42b0670af8c016b/caio-0.9.25-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fab6078b9348e883c80a5e14b382e6ad6aabbc4429ca034e76e730cf464269db", size = 36978 }, { url = "https://files.pythonhosted.org/packages/5e/6c/d4d24f65e690213c097174d26eda6831f45f4734d9d036d81790a27e7b78/caio-0.9.25-cp314-cp314-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44a6b58e52d488c75cfaa5ecaa404b2b41cc965e6c417e03251e868ecd5b6d77", size = 81832 }, + { url = "https://files.pythonhosted.org/packages/87/a4/e534cf7d2d0e8d880e25dd61e8d921ffcfe15bd696734589826f5a2df727/caio-0.9.25-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:628a630eb7fb22381dd8e3c8ab7f59e854b9c806639811fc3f4310c6bd711d79", size = 81565 }, + { url = "https://files.pythonhosted.org/packages/3f/ed/bf81aeac1d290017e5e5ac3e880fd56ee15e50a6d0353986799d1bc5cfd5/caio-0.9.25-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:0ba16aa605ccb174665357fc729cf500679c2d94d5f1458a6f0d5ca48f2060a7", size = 80071 }, { url = "https://files.pythonhosted.org/packages/86/93/1f76c8d1bafe3b0614e06b2195784a3765bbf7b0a067661af9e2dd47fc33/caio-0.9.25-py3-none-any.whl", hash = "sha256:06c0bb02d6b929119b1cfbe1ca403c768b2013a369e2db46bfa2a5761cf82e40", size = 19087 }, ] diff --git a/api-service/Dockerfile b/api-service/Dockerfile index ec0d1ca..2394cd0 100644 --- a/api-service/Dockerfile +++ b/api-service/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.21-alpine AS builder +FROM golang:1.23-alpine AS builder RUN apk add --no-cache git @@ -44,7 +44,7 @@ WORKDIR /workspace # Runtime stage FROM alpine:3.20 -RUN apk add --no-cache ca-certificates tzdata +RUN apk add --no-cache ca-certificates tzdata bash COPY --from=builder /workspace/api-service/api-service /usr/bin/api-service diff --git a/api-service/controller/mcp_proxy.go b/api-service/controller/mcp_proxy.go index a2921ba..46041a9 100644 --- a/api-service/controller/mcp_proxy.go +++ b/api-service/controller/mcp_proxy.go @@ -22,6 +22,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "time" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -51,13 +52,19 @@ const ( // MCPGateway MCP gateway struct type MCPGateway struct { - router *gin.RouterGroup + router *gin.RouterGroup + transport *http.Transport } // NewMCPGateway creates a new MCP gateway instance func NewMCPGateway(router *gin.RouterGroup) *MCPGateway { gateway := &MCPGateway{ router: router, + transport: &http.Transport{ + MaxIdleConns: 2000, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + }, } gateway.setupRoutes() @@ -195,7 +202,7 @@ func (g *MCPGateway) handleMCPSSEWithHeader(c *gin.Context) { g.copyHeadersExcept(c.Request.Header, req.Header, constants.HeaderMCPServerURL) // Send to MCP server - client := &http.Client{} + client := &http.Client{Transport: g.transport} resp, err := client.Do(req) if err != nil { log.Errorf("Failed to connect to MCP server (%s): %v", mcpServerURL, err) @@ -280,27 +287,22 @@ func (g *MCPGateway) handleMCPHTTPWithHeader(c *gin.Context) { // HTTP path will be duplicated targetURL := g.buildTargetURL(serverURL, "", "") - // Create reverse proxy - proxy := httputil.NewSingleHostReverseProxy(&targetURL) - - // Error handling - proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { - log.Errorf("Proxy error for server %s: %v", mcpServerURL, err) - c.JSON(http.StatusBadGateway, gin.H{ - "error": "Failed to forward request to MCP server", - "details": err.Error(), - "server": mcpServerURL, - }) - } - - // Create Director to modify request - originalDirector := proxy.Director - proxy.Director = func(req *http.Request) { - // Execute original Director first - originalDirector(req) - - // Remove headers used internally by gateway - req.Header.Del(constants.HeaderMCPServerURL) + // Create reverse proxy with shared transport + proxy := &httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL.Scheme = targetURL.Scheme + req.URL.Host = targetURL.Host + req.Header.Del(constants.HeaderMCPServerURL) + }, + Transport: g.transport, + ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + log.Errorf("Proxy error for server %s: %v", mcpServerURL, err) + c.JSON(http.StatusBadGateway, gin.H{ + "error": "Failed to forward request to MCP server", + "details": err.Error(), + "server": mcpServerURL, + }) + }, } // Execute reverse proxy diff --git a/api-service/go.mod b/api-service/go.mod index 079c9c0..4f58938 100644 --- a/api-service/go.mod +++ b/api-service/go.mod @@ -14,9 +14,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) -replace ( - envhub => ../envhub -) +replace envhub => ../envhub require ( github.com/beorn7/perks v1.0.1 // indirect @@ -25,6 +23,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-pkgz/expirable-cache/v3 v3.0.0 // indirect github.com/go-playground/locales v0.14.1 // indirect diff --git a/api-service/go.sum b/api-service/go.sum index 288bf3c..2ee0e3c 100644 --- a/api-service/go.sum +++ b/api-service/go.sum @@ -58,6 +58,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -74,8 +75,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gin-contrib/pprof v1.4.0 h1:XxiBSf5jWZ5i16lNOPbMTVdgHBdhfGRD5PZ1LWazzvg= +github.com/gin-contrib/pprof v1.4.0/go.mod h1:RrehPJasUVBPK6yTUwOl8/NP6i0vbUgmxtis+Z5KE90= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -91,17 +95,22 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-pkgz/expirable-cache/v3 v3.0.0 h1:u3/gcu3sabLYiTCevoRKv+WzjIn5oo7P8XtiXBeRDLw= github.com/go-pkgz/expirable-cache/v3 v3.0.0/go.mod h1:2OQiDyEGQalYecLWmXprm3maPXeVb5/6/X7yRPYTzec= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -185,12 +194,16 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -213,8 +226,10 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -248,6 +263,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -262,6 +279,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -271,6 +289,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -296,6 +316,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -356,6 +377,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -413,6 +435,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -553,6 +577,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -561,6 +586,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= @@ -573,6 +600,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/api-service/main.go b/api-service/main.go index 3d8e0f0..d8862a0 100644 --- a/api-service/main.go +++ b/api-service/main.go @@ -18,6 +18,7 @@ limitations under the License. package main import ( + "math/rand" "net/http" "runtime" "time" @@ -29,6 +30,7 @@ import ( "api-service/middleware" "api-service/service" + "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/pflag" @@ -132,13 +134,14 @@ func main() { mainRouter.GET("/health", healthChecker) mainRouter.GET("/metrics", gin.WrapH(promhttp.Handler())) + pprof.Register(mainRouter) // MCP dedicated routing engine // Note: MCP uses the same logrus global logger (writes to same log file) // since logrus is a global singleton. For separate MCP log files, // use a dedicated logrus instance in the future. mcpRouter := gin.Default() - mcpRouter.Use(middleware.MetricsMiddleware()) + mcpRouter.Use(middleware.MCPMetricsMiddleware()) mcpRouter.Use(middleware.LoggingMiddleware()) mcpGroup := mcpRouter.Group("/") controller.NewMCPGateway(mcpGroup) @@ -167,18 +170,55 @@ func main() { } cleanManager := service.NewAEnvCleanManager(scheduleClient, interval). WithMetrics(middleware.IncrementCleanupSuccess, middleware.IncrementCleanupFailure) - go cleanManager.Start() - defer cleanManager.Stop() - // Start metrics collector for instance metrics (faas mode only) + // Start a unified periodic task that shares a single ListEnvInstances call + // across cleanup and metrics collection, reducing redundant requests to meta-service. if scheduleType == "faas" { if faasClient, ok := scheduleClient.(*service.FaaSClient); ok { - metricsCollector := metrics.NewCollector(faasClient, 5*time.Minute) - go metricsCollector.Start() - defer metricsCollector.Stop() + metricsCollector := metrics.NewCollector(faasClient, interval) + go startUnifiedPeriodicTask(scheduleClient, cleanManager, metricsCollector, interval) + } else { + go cleanManager.Start() } + } else { + go cleanManager.Start() } // Block main goroutine select {} } + +// startUnifiedPeriodicTask runs cleanup and metrics collection in a single ticker loop, +// sharing one ListEnvInstances call per cycle. A random jitter at startup disperses +// the tick phase across multiple api-service replicas to avoid thundering herd. +func startUnifiedPeriodicTask( + envInstanceService service.EnvInstanceService, + cleanManager *service.AEnvCleanManager, + metricsCollector *metrics.Collector, + interval time.Duration, +) { + // Random jitter to stagger tickers across replicas + jitter := time.Duration(rand.Int63n(int64(interval))) + log.Infof("Unified periodic task: starting after jitter %v (interval %v)", jitter, interval) + time.Sleep(jitter) + + runOnce := func() { + envInstances, err := envInstanceService.ListEnvInstances("") + if err != nil { + log.Warnf("Unified periodic task: failed to list instances: %v", err) + return + } + + // Feed the same data to both consumers + cleanManager.CleanupFromInstances(envInstances) + metricsCollector.CollectFromEnvInstances(envInstances) + } + + runOnce() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + runOnce() + } +} diff --git a/api-service/metrics/collector.go b/api-service/metrics/collector.go index 29f2ba9..25fe78e 100644 --- a/api-service/metrics/collector.go +++ b/api-service/metrics/collector.go @@ -17,6 +17,7 @@ limitations under the License. package metrics import ( + "api-service/models" "api-service/service/faas_model" "time" @@ -77,12 +78,18 @@ func (c *Collector) collect() { return } + c.CollectFromInstances(resp.Instances) +} + +// CollectFromInstances updates metrics from a pre-fetched instance list. +// This allows callers to share the same ListInstances result across multiple consumers. +func (c *Collector) CollectFromInstances(instances []*faas_model.Instance) { // Reset gauges to avoid stale data from deleted instances ActiveInstances.Reset() InstanceUptimeSeconds.Reset() now := time.Now().UnixMilli() - for _, inst := range resp.Instances { + for _, inst := range instances { if inst.Labels == nil { inst.Labels = make(map[string]string) } @@ -103,5 +110,42 @@ func (c *Collector) collect() { } } - log.Infof("metrics collector: updated metrics for %d instances", len(resp.Instances)) + log.Infof("metrics collector: updated metrics for %d instances", len(instances)) +} + +// CollectFromEnvInstances updates metrics from a pre-fetched EnvInstance list. +// This handles the type difference between models.EnvInstance (used by ListEnvInstances) +// and faas_model.Instance (used by FaaSClient.ListInstances). +func (c *Collector) CollectFromEnvInstances(envInstances []*models.EnvInstance) { + ActiveInstances.Reset() + InstanceUptimeSeconds.Reset() + + now := time.Now() + for _, inst := range envInstances { + labels := inst.Labels + if labels == nil { + labels = make(map[string]string) + } + env := labels["env"] + experiment := labels["experiment"] + owner := labels["owner"] + app := labels["app"] + + ActiveInstances.WithLabelValues(env, experiment, owner, app).Inc() + + if inst.CreatedAt != "" { + createdAt, err := time.Parse(time.DateTime, inst.CreatedAt) + if err != nil { + createdAt, err = time.Parse(time.RFC3339, inst.CreatedAt) + } + if err == nil { + uptimeSec := now.Sub(createdAt).Seconds() + InstanceUptimeSeconds.WithLabelValues( + inst.ID, env, experiment, owner, app, + ).Set(uptimeSec) + } + } + } + + log.Infof("metrics collector: updated metrics for %d env instances", len(envInstances)) } diff --git a/api-service/metrics/metrics.go b/api-service/metrics/metrics.go index 4c7a0f5..98338f0 100644 --- a/api-service/metrics/metrics.go +++ b/api-service/metrics/metrics.go @@ -25,13 +25,13 @@ const subsystem = "aenv_api" // BusinessLabelKeys defines the fixed business labels used for instance metrics. // These labels are extracted from instance Labels map: -// - env: Environment name (e.g., "terminal-0.1.0"), auto-set by system +// - envName: Environment name (e.g., "terminal-0.1.0"), auto-set by system // - experiment: Experiment identifier (user-provided, optional) // - owner: Instance owner/creator (user-provided, optional) // - app: Application name (user-provided, optional) // // Missing labels will result in empty string values in metrics. -var BusinessLabelKeys = []string{"env", "experiment", "owner", "app"} +var BusinessLabelKeys = []string{"envName", "experiment", "owner", "app"} var ( // HTTP request metrics @@ -91,4 +91,24 @@ var ( }, append([]string{"instance_id"}, BusinessLabelKeys...), ) + + // MCP proxy metrics with rpc_method to distinguish JSON-RPC operations + MCPRequestsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: subsystem, + Name: "mcp_requests_total", + Help: "Total number of MCP proxy requests", + }, + []string{"method", "endpoint", "rpc_method", "status"}, + ) + + MCPRequestDurationMs = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: subsystem, + Name: "mcp_request_duration_ms", + Help: "MCP proxy request duration in milliseconds", + Buckets: prometheus.ExponentialBuckets(1, 2, 20), + }, + []string{"method", "endpoint", "rpc_method", "status"}, + ) ) diff --git a/api-service/metrics/metrics_test.go b/api-service/metrics/metrics_test.go index 0cacfe4..5544373 100644 --- a/api-service/metrics/metrics_test.go +++ b/api-service/metrics/metrics_test.go @@ -186,7 +186,7 @@ func TestCollectorCollect(t *testing.T) { } // Check active instances count for terminal-0.1.0 (should be 2) - if !strings.Contains(bodyStr, `aenv_api_active_instances{app="chatbot",env="terminal-0.1.0",experiment="exp1",owner="jun"} 2`) { + if !strings.Contains(bodyStr, `aenv_api_active_instances{app="chatbot",envName="terminal-0.1.0",experiment="exp1",owner="jun"} 2`) { t.Errorf("expected active_instances for terminal-0.1.0 to be 2") } diff --git a/api-service/middleware/logging.go b/api-service/middleware/logging.go index f2f1a10..44833fb 100644 --- a/api-service/middleware/logging.go +++ b/api-service/middleware/logging.go @@ -19,6 +19,7 @@ package middleware import ( "api-service/constants" "bytes" + "fmt" "io" "time" @@ -29,6 +30,14 @@ import ( const maxBodyLogSize = 2048 // 2KB body truncation limit +// truncateBody truncates body string to maxBodyLogSize to prevent disk bloat. +func truncateBody(body string) string { + if len(body) <= maxBodyLogSize { + return body + } + return body[:maxBodyLogSize] + fmt.Sprintf("...(truncated, total %d bytes)", len(body)) +} + // InitLogger initializes logrus with lumberjack log rotation. // logPath: log file path, empty means default /home/admin/logs/api-service.log // logLevel: log level string (debug, info, warn, error), empty means info @@ -87,9 +96,11 @@ func LoggingMiddleware() gin.HandlerFunc { start := time.Now() - // Read request body + // Read request body — prefer cached body from MCPMetricsMiddleware var reqBody []byte - if c.Request.Body != nil { + if cached, ok := c.Get("_req_body"); ok { + reqBody = cached.([]byte) + } else if c.Request.Body != nil { reqBody, _ = io.ReadAll(c.Request.Body) // Restore Body, because ReadAll consumes it c.Request.Body = io.NopCloser(bytes.NewBuffer(reqBody)) @@ -124,12 +135,12 @@ func LoggingMiddleware() gin.HandlerFunc { // Add request body (truncated) if len(reqBody) > 0 { - fields["request_body"] = truncateString(string(reqBody), maxBodyLogSize) + fields["request_body"] = truncateBody(string(reqBody)) } // Add response body (truncated) if blw.body.Len() > 0 { - fields["response_body"] = truncateString(blw.body.String(), maxBodyLogSize) + fields["response_body"] = truncateBody(blw.body.String()) } // Log error information (if any) @@ -149,11 +160,3 @@ func LoggingMiddleware() gin.HandlerFunc { } } } - -// truncateString truncates a string to maxLen bytes -func truncateString(s string, maxLen int) string { - if len(s) > maxLen { - return s[:maxLen] + "...(truncated)" - } - return s -} diff --git a/api-service/middleware/metrics.go b/api-service/middleware/metrics.go index 948eb75..ee41e35 100644 --- a/api-service/middleware/metrics.go +++ b/api-service/middleware/metrics.go @@ -17,7 +17,11 @@ limitations under the License. package middleware import ( + "bytes" + "encoding/json" "fmt" + "io" + "strings" "time" "api-service/metrics" @@ -55,9 +59,7 @@ func IncrementCleanupFailure() { } // MetricsMiddleware records HTTP request metrics. -// Excludes /health endpoint errors from being recorded as failures, -// since proxy-less /health calls (e.g., K8s liveness probes) are expected -// to return non-error status and should not pollute error metrics. +// Uses Gin's FullPath() for endpoint label, suitable for routers with named routes. func MetricsMiddleware() gin.HandlerFunc { return func(c *gin.Context) { start := time.Now() @@ -69,9 +71,7 @@ func MetricsMiddleware() gin.HandlerFunc { endpoint = c.Request.URL.Path } - statusCode := c.Writer.Status() - - status := fmt.Sprintf("%d", statusCode) + status := fmt.Sprintf("%d", c.Writer.Status()) method := c.Request.Method durationMs := float64(time.Since(start).Milliseconds()) @@ -79,3 +79,82 @@ func MetricsMiddleware() gin.HandlerFunc { metrics.RequestDurationMs.WithLabelValues(method, endpoint, status).Observe(durationMs) } } + +// MCPMetricsMiddleware records MCP proxy request metrics. +// Extracts JSON-RPC method from POST body with size limit, normalizes endpoint +// labels, sets GetBody for reverse proxy rewind, and skips duration for SSE. +func MCPMetricsMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + path := c.Request.URL.Path + isSSE := strings.HasSuffix(path, "/sse") + + var rpcMethod string + if c.Request.Method == "POST" && c.Request.Body != nil { + // Read up to 8KB+1 to detect oversized bodies; JSON-RPC method + // field is always in the first few dozen bytes. + const maxPeek = 8192 + limited := io.LimitReader(c.Request.Body, maxPeek+1) + if body, err := io.ReadAll(limited); err == nil && len(body) > 0 { + var rpc struct { + Method string `json:"method"` + } + if json.Unmarshal(body, &rpc) == nil { + rpcMethod = rpc.Method + } + + // Fast path (99%+ requests): body fits within limit, skip + // second ReadAll and append entirely. + var fullBody []byte + if len(body) <= maxPeek { + fullBody = body + } else { + // Cap remaining read at 1MB to prevent memory exhaustion + // from abnormally large requests. + const maxBody = 1 << 20 + remaining, _ := io.ReadAll(io.LimitReader(c.Request.Body, maxBody)) + fullBody = append(body, remaining...) + } + + c.Request.Body = io.NopCloser(bytes.NewReader(fullBody)) + c.Request.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(fullBody)), nil + } + c.Request.ContentLength = int64(len(fullBody)) + } + } + c.Set("_rpc_method", rpcMethod) + + endpoint := normalizeEndpoint(path) + + start := time.Now() + c.Next() + + status := fmt.Sprintf("%d", c.Writer.Status()) + method := c.Request.Method + + metrics.MCPRequestsTotal.WithLabelValues(method, endpoint, rpcMethod, status).Inc() + + // Skip duration histogram for SSE (long-lived connections pollute buckets) + if !isSSE { + durationMs := float64(time.Since(start).Milliseconds()) + metrics.MCPRequestDurationMs.WithLabelValues(method, endpoint, rpcMethod, status).Observe(durationMs) + } + } +} + +// normalizeEndpoint maps raw URL path to a bounded set of known MCP paths +// to prevent Prometheus label cardinality explosion from wildcard routes. +func normalizeEndpoint(path string) string { + switch { + case strings.HasSuffix(path, "/sse"): + return "/sse" + case strings.HasSuffix(path, "/mcp"): + return "/mcp" + case strings.HasSuffix(path, "/message"): + return "/message" + case path == "/health": + return "/health" + default: + return "/other" + } +} diff --git a/api-service/service/cleanup_service.go b/api-service/service/cleanup_service.go index 5af8614..3a3003a 100644 --- a/api-service/service/cleanup_service.go +++ b/api-service/service/cleanup_service.go @@ -92,6 +92,12 @@ func (cm *AEnvCleanManager) performCleanup() { return } + cm.CleanupFromInstances(envInstances) +} + +// CleanupFromInstances performs TTL-based cleanup on a pre-fetched instance list. +// This allows callers to share the same ListEnvInstances result across multiple consumers. +func (cm *AEnvCleanManager) CleanupFromInstances(envInstances []*models.EnvInstance) { if len(envInstances) == 0 { log.Debug("No environment instances found") return diff --git a/api-service/service/cleanup_service_test.go b/api-service/service/cleanup_service_test.go index c31071f..cdd94db 100644 --- a/api-service/service/cleanup_service_test.go +++ b/api-service/service/cleanup_service_test.go @@ -62,10 +62,6 @@ func (m *MockEnvInstanceService) Warmup(req *backend.Env) error { return nil } -func (m *MockEnvInstanceService) Cleanup() error { - return nil -} - // TestPerformCleanupNoInstances tests cleanup when there are no env instances func TestPerformCleanupNoInstances(t *testing.T) { // Create mock service that returns empty list diff --git a/api-service/service/env_instance.go b/api-service/service/env_instance.go index 5de87cf..7a893b5 100644 --- a/api-service/service/env_instance.go +++ b/api-service/service/env_instance.go @@ -24,7 +24,6 @@ type EnvInstanceService interface { DeleteEnvInstance(id string) error ListEnvInstances(envName string) ([]*models.EnvInstance, error) Warmup(req *backend.Env) error - Cleanup() error } type EnvInstanceClient struct { @@ -304,57 +303,6 @@ func (c *EnvInstanceClient) Warmup(req *backend.Env) error { return nil } -// Cleanup performs a cleanup operation to release unused environment resources. -// -// Parameters: -// - None -// -// Returns: -// - error: nil if cleanup is successful; otherwise, an error indicating failure. -func (c *EnvInstanceClient) Cleanup() error { - url := fmt.Sprintf("%s/%s/action/cleanup", c.baseURL, AEnvOpenAPIInstance) - - httpReq, err := http.NewRequest("PUT", url, nil) - if err != nil { - return fmt.Errorf("cleanup env: failed to create request: %v", err) - } - - resp, err := c.httpClient.Do(httpReq) - if err != nil { - return fmt.Errorf("cleanup env: failed to send request: %v", err) - } - defer func() { - if closeErr := resp.Body.Close(); closeErr != nil { - log.Warnf("failed to close response body: %v", closeErr) - } - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("cleanup env: failed to read response body: %v", err) - } - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("cleanup env: request failed with status %d: %s", resp.StatusCode, truncateBody(body)) - } - - var getResp models.ClientResponse[models.EnvInstance] - if err := json.Unmarshal(body, &getResp); err != nil { - return fmt.Errorf("cleanup env: failed to unmarshal response: %v", err) - } - - if !getResp.Success { - // Include both code and message in error - errMsg := fmt.Sprintf("cleanup env: server returned error, code: %d", getResp.Code) - if getResp.Message != "" { - errMsg = fmt.Sprintf("cleanup env: server returned error (code %d): %s", getResp.Code, getResp.Message) - } - return fmt.Errorf("%s", errMsg) - } - - return nil -} - // truncateBody truncate body for memory protection func truncateBody(body []byte) string { const maxLen = 500 diff --git a/api-service/service/faas_client.go b/api-service/service/faas_client.go index 21e7f11..d0acc05 100644 --- a/api-service/service/faas_client.go +++ b/api-service/service/faas_client.go @@ -237,10 +237,6 @@ func (c *FaaSClient) WarmupAsyncChan(req *backend.Env) <-chan error { return resultCh } -func (c *FaaSClient) Cleanup() error { - return fmt.Errorf("cleanup not implemented in faas") -} - // --- Newly added local method implementations --- func (c *FaaSClient) CreateFunction(in *faas_model.FunctionCreateOrUpdateRequest) error { @@ -356,6 +352,7 @@ func (c *FaaSClient) InitializeFunction(name string, initOptions faas_model.Func if err != nil { return "", err } + defer resp.Body.Close() if resp.StatusCode != 200 { return "", fmt.Errorf("failed to initialize function from faas server with status code %d", resp.StatusCode) diff --git a/api-service/service/faas_model/http_client.go b/api-service/service/faas_model/http_client.go index 4981782..af2cc44 100644 --- a/api-service/service/faas_model/http_client.go +++ b/api-service/service/faas_model/http_client.go @@ -33,6 +33,7 @@ func NewHTTPClient(baseURL string) *HTTPClient { Timeout: 10 * time.Second, Transport: &http.Transport{ TLSHandshakeTimeout: 5 * time.Second, + MaxIdleConnsPerHost: 20, }, }, BaseURL: baseURL, @@ -210,6 +211,7 @@ func (r *HTTPReq) Into(obj interface{}, e ...interface{}) error { if r.resp == nil { return fmt.Errorf("response is not ready") } + defer r.resp.Body.Close() data, err := io.ReadAll(r.resp.Body) if err != nil { diff --git a/api-service/service/schedule_client.go b/api-service/service/schedule_client.go index 02e7417..63e20b5 100644 --- a/api-service/service/schedule_client.go +++ b/api-service/service/schedule_client.go @@ -607,38 +607,3 @@ func (c *ScheduleClient) ListEnvInstances(envName string) ([]*models.EnvInstance func (c *ScheduleClient) Warmup(req *backend.Env) error { return fmt.Errorf("warmup is not implemented") } - -func (c *ScheduleClient) Cleanup() error { - log.Infof("Starting cleanup task...") - // get all EnvInstance - envInstances, err := c.FilterPods() - if err != nil { - return fmt.Errorf("failed to get env instances: %v", err) - } - if envInstances == nil || len(*envInstances) == 0 { - log.Infof("No env instances found") - return nil - } - - var deletedCount int - - for _, instance := range *envInstances { - // skip terminated env instance - if instance.Status == "Terminated" { - continue - } - deleted, err := c.DeletePod(instance.ID) - if err != nil { - log.Warnf("Failed to delete instance %s: %v", instance.ID, err) - continue - } - if deleted { - deletedCount++ - log.Infof("Successfully deleted instance %s", instance.ID) - } else { - log.Infof("Instance %s was not deleted (may already be deleted)", instance.ID) - } - } - log.Infof("Cleanup task completed. Deleted %d expired instances", deletedCount) - return nil -} diff --git a/go.work b/go.work index 6ab56a2..759eb90 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,6 @@ -go 1.21 +go 1.23.0 + +toolchain go1.23.1 use ( ./api-service diff --git a/go.work.sum b/go.work.sum index 8d7b43d..e1e6f98 100644 --- a/go.work.sum +++ b/go.work.sum @@ -191,6 +191,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= @@ -199,6 +200,7 @@ github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= @@ -228,6 +230,7 @@ github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBd github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= @@ -236,6 +239,7 @@ github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -251,6 +255,8 @@ github.com/google/cel-go v0.12.5/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25ol github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= @@ -291,9 +297,11 @@ github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHz github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= @@ -307,12 +315,15 @@ github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6U github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= @@ -321,8 +332,10 @@ github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUq github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= @@ -371,9 +384,13 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= @@ -383,29 +400,44 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w= @@ -441,6 +473,7 @@ google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=