From fff74314832750b4d0ad3e2674c30ae65d5a3448 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Wed, 14 Jan 2026 19:39:18 +0100 Subject: [PATCH 01/39] Fix Docker image tag to use lowercase --- .github/workflows/docker-publish.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9a12081..d57e48b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -8,10 +8,6 @@ on: required: true type: string -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - jobs: build-and-push: runs-on: ubuntu-latest @@ -27,19 +23,18 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to Container Registry - if: github.event_name != 'pull_request' uses: docker/login-action@v3.6.0 with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} + registry: ghcr.io + username: serversidehannes password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image uses: docker/build-push-action@v6 with: context: . - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.tag }} + push: true + tags: ghcr.io/serversidehannes/s3proxy-python:${{ inputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64,linux/arm64 From 7d4dffeacea6e457cf4fb480c2512f5e9350e7ca Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Thu, 15 Jan 2026 12:22:23 +0100 Subject: [PATCH 02/39] fix: wip --- benchmarks/bench.py | 4 +- benchmarks/docker-compose.yml | 10 +-- e2e/docker-compose.e2e.yml | 24 ++---- manifests/templates/configmap.yaml | 5 +- manifests/values.yaml | 118 +++++++---------------------- s3proxy/config.py | 24 ++---- s3proxy/handlers/objects.py | 15 ++-- s3proxy/main.py | 17 ++--- s3proxy/multipart.py | 13 +--- tests/conftest.py | 2 +- tests/test_handlers.py | 3 +- tests/test_multipart.py | 19 ----- 12 files changed, 68 insertions(+), 186 deletions(-) diff --git a/benchmarks/bench.py b/benchmarks/bench.py index afdb7ec..2332b19 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -25,14 +25,12 @@ import aioboto3 -# Object sizes +# Object sizes for realistic single-part uploads SIZES = { "tiny": 1024, # 1 KB "small": 64 * 1024, # 64 KB "medium": 1024 * 1024, # 1 MB "large": 10 * 1024 * 1024, # 10 MB - "xlarge": 100 * 1024 * 1024, # 100 MB - "huge": 1024 * 1024 * 1024, # 1 GiB } # Endpoints diff --git a/benchmarks/docker-compose.yml b/benchmarks/docker-compose.yml index 398f332..ae06dd4 100644 --- a/benchmarks/docker-compose.yml +++ b/benchmarks/docker-compose.yml @@ -68,9 +68,9 @@ services: S3PROXY_ENCRYPT_KEY: benchmark-test-key-32-bytes-!! AWS_ACCESS_KEY_ID: benchmarkadminuser AWS_SECRET_ACCESS_KEY: benchmarkadminpassword - # Higher limits for benchmarking - S3PROXY_MAX_CONCURRENT_UPLOADS: "50" - S3PROXY_MAX_CONCURRENT_DOWNLOADS: "50" + # Throttle for realistic single-part uploads (10MB files) + # Memory: 10MB + 64MB = 74MB per upload, 10 concurrent = 740MB < 1GB + S3PROXY_THROTTLING_REQUESTS_MAX: "10" S3PROXY_NO_TLS: "true" S3PROXY_LOG_LEVEL: WARNING # Reduce log noise during benchmarks S3PROXY_REDIS_URL: redis://redis:6379/0 @@ -82,8 +82,8 @@ services: # Required for py-spy profiling cap_add: - SYS_PTRACE - mem_limit: 512m - memswap_limit: 512m + mem_limit: 1g + memswap_limit: 1g healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4433/readyz"] interval: 2s diff --git a/e2e/docker-compose.e2e.yml b/e2e/docker-compose.e2e.yml index e3e8d02..9bada3d 100644 --- a/e2e/docker-compose.e2e.yml +++ b/e2e/docker-compose.e2e.yml @@ -37,35 +37,23 @@ services: ports: - "8080:4433" environment: - # S3 backend configuration S3PROXY_HOST: http://minio:9000 S3PROXY_REGION: us-east-1 - - # Encryption key (32 bytes for AES-256) S3PROXY_ENCRYPT_KEY: test-key-for-e2e-testing-32bytes - - # AWS credentials for MinIO access - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - - # Concurrency limits - S3PROXY_MAX_CONCURRENT_UPLOADS: "5" - S3PROXY_MAX_CONCURRENT_DOWNLOADS: "5" - - # Server configuration S3PROXY_NO_TLS: "true" S3PROXY_LOG_LEVEL: INFO - - # Redis configuration S3PROXY_REDIS_URL: redis://redis:6379/0 + S3PROXY_THROTTLING_REQUESTS_MAX: "10" + S3PROXY_MAX_UPLOAD_SIZE_MB: "45" + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin depends_on: minio: condition: service_healthy redis: condition: service_healthy - # Hard memory limit (same as Kubernetes) - mem_limit: 512m - memswap_limit: 512m # Same as mem_limit = no swap = hard OOM kill + mem_limit: 1280m + memswap_limit: 1280m healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4433/readyz"] interval: 5s diff --git a/manifests/templates/configmap.yaml b/manifests/templates/configmap.yaml index 5e16e81..3a607a1 100644 --- a/manifests/templates/configmap.yaml +++ b/manifests/templates/configmap.yaml @@ -14,9 +14,8 @@ data: S3PROXY_IP: "0.0.0.0" S3PROXY_PORT: {{ .Values.server.port | quote }} S3PROXY_NO_TLS: {{ .Values.server.noTls | quote }} - S3PROXY_MAX_CONCURRENT_UPLOADS: {{ .Values.performance.maxConcurrentUploads | quote }} - S3PROXY_MAX_CONCURRENT_DOWNLOADS: {{ .Values.performance.maxConcurrentDownloads | quote }} - S3PROXY_AUTO_MULTIPART_MB: {{ .Values.performance.autoMultipartMb | quote }} + S3PROXY_THROTTLING_REQUESTS_MAX: {{ .Values.performance.throttlingRequestsMax | quote }} + S3PROXY_MAX_UPLOAD_SIZE_MB: {{ .Values.performance.maxUploadSizeMb | quote }} {{- if index .Values "redis-ha" "enabled" }} S3PROXY_REDIS_URL: "redis://{{ .Release.Name }}-redis-ha-haproxy:6379/0" {{- else }} diff --git a/manifests/values.yaml b/manifests/values.yaml index ef834b0..960624d 100644 --- a/manifests/values.yaml +++ b/manifests/values.yaml @@ -1,52 +1,32 @@ # S3Proxy Helm Chart Values -# Deployment settings replicaCount: 3 -# Container image image: - # IMPORTANT: Change to your image registry - # Example: ghcr.io/myorg/sseproxy-python or your private registry repository: ghcr.io/YOUR_USERNAME/sseproxy-python - # IMPORTANT: Never use 'latest' in production - use specific version tags - # Example: "v0.1.0", "v0.2.0", etc. tag: latest pullPolicy: IfNotPresent -# S3 configuration (used only when minio.enabled=false) -# Ignored if MinIO is enabled - MinIO will be used as the S3 backend +# S3 configuration (used when minio.enabled=false) s3: - # S3-compatible endpoint: AWS S3, DigitalOcean Spaces, etc. - # Examples: - # - "s3.amazonaws.com" (AWS S3) - # - "s3.us-west-2.amazonaws.com" (AWS S3 specific region) - # - "nyc3.digitaloceanspaces.com" (DigitalOcean Spaces) host: "s3.amazonaws.com" - # AWS region (ignored for non-AWS S3 services) region: "us-east-1" -# Server settings server: - port: 4433 # Listen port (should match service.port) - noTls: true # TLS termination handled by Ingress or Load Balancer + port: 4433 + noTls: true -# Performance tuning settings performance: - maxConcurrentUploads: 10 # Max concurrent upload operations - maxConcurrentDownloads: 10 # Max concurrent download operations - autoMultipartMb: 16 # Chunk size in MB for multipart uploads + throttlingRequestsMax: 10 + maxUploadSizeMb: 45 -# MinIO configuration (embedded S3 backend) -# Set enabled: false to use external S3 (AWS, DigitalOcean Spaces, etc.) -# When disabled, configure s3.host and set secrets.awsAccessKeyId/awsSecretAccessKey +# Embedded MinIO (set enabled: false to use external S3) minio: - enabled: true # For production, consider external S3 service + enabled: true image: repository: minio/minio - tag: latest # Specify version for production, e.g., "RELEASE.2024-01-16T16-07-38Z" + tag: latest pullPolicy: IfNotPresent - # IMPORTANT: Change credentials in production! - # Default credentials below are for development only rootUser: "minioadmin" rootPassword: "minioadmin" resources: @@ -57,38 +37,22 @@ minio: cpu: "500m" memory: "512Mi" -# Redis cache for upload state management -# Choose one: redis-ha (for HA) or externalRedis (for managed services) +# External Redis (for managed services) externalRedis: - # Use this for managed Redis: AWS ElastiCache, Azure Cache, Redis Cloud, etc. - # Leave empty to use redis-ha instead - # Format: redis://host:port/db or redis://:password@host:port/db - url: "" # e.g., "redis://my-elasticache.abc123.cache.amazonaws.com:6379/0" - # Include password in URL if needed: redis://:mypassword@host:port/db - # TTL for upload state in hours + url: "" uploadTtlHours: 24 -# Redis HA configuration (embedded high-availability Redis) -# Uses dandydev/redis-ha chart with Sentinel for automatic failover -# Set enabled: false if using externalRedis instead +# Redis HA (embedded) redis-ha: - enabled: true # Disable if using managed Redis service - # Number of Redis replicas (1 master + N-1 replicas) + enabled: true replicas: 3 + existingSecret: "" - # Use existing secret for Redis password (RECOMMENDED for production) - # If provided, auth and authKey below are ignored - # Create with: kubectl create secret generic redis-password --from-literal=redis-password="your-password" - existingSecret: "" # Name of existing secret with key "redis-password" - - # Persistence configuration persistentVolume: enabled: true size: 10Gi - storageClass: "" # Use default storage class, or specify e.g., "gp3", "standard" + storageClass: "" - # HAProxy for single-endpoint access (recommended) - # This allows standard redis:// URLs without sentinel-aware client code haproxy: enabled: true replicas: 2 @@ -100,7 +64,6 @@ redis-ha: cpu: "200m" memory: "128Mi" - # Sentinel configuration sentinel: port: 26379 quorum: 2 @@ -109,7 +72,6 @@ redis-ha: failover-timeout: 180000 parallel-syncs: 5 - # Redis configuration redis: port: 6379 config: @@ -117,14 +79,10 @@ redis-ha: min-replicas-to-write: 1 min-replicas-max-lag: 5 - # Security - Redis password authentication (ignored if existingSecret is set above) - auth: false # Set to true to enable password protection - authKey: "" # Redis password (required if auth=true and no existingSecret, generate with: openssl rand -base64 32) - - # Pod anti-affinity for HA (spread across nodes) + auth: false + authKey: "" hardAntiAffinity: true - # Resource limits for Redis pods resources: requests: cpu: "100m" @@ -133,54 +91,30 @@ redis-ha: cpu: "500m" memory: "512Mi" -# Secret Configuration -# IMPORTANT: Never commit actual secrets to git! -# Priority: existingSecrets > create static secret - +# Secrets (use existing secret in production) secrets: - # Option 1: Use existing Secret (RECOMMENDED for production) - # Reference an existing Kubernetes secret and optionally map its keys - # If using default key names, just set the secret name: - # kubectl create secret generic my-s3-secrets \ - # --from-literal=S3PROXY_ENCRYPT_KEY="$(openssl rand -base64 32)" \ - # --from-literal=AWS_ACCESS_KEY_ID="AKIA..." \ - # --from-literal=AWS_SECRET_ACCESS_KEY="..." existingSecrets: enabled: false - name: "" # Name of existing Kubernetes secret - # Optional: Map secret keys if using different key names + name: "" keys: - encryptKey: "S3PROXY_ENCRYPT_KEY" # Secret key name for encryption key - awsAccessKeyId: "AWS_ACCESS_KEY_ID" # Secret key name for access key - awsSecretAccessKey: "AWS_SECRET_ACCESS_KEY" # Secret key name for secret key - - # Option 2: Create new secret from values (use only for development) - # For production, use existingSecrets with a pre-created secret - # Provide values via helm --set or secure values file, never hardcode here + encryptKey: "S3PROXY_ENCRYPT_KEY" + awsAccessKeyId: "AWS_ACCESS_KEY_ID" + awsSecretAccessKey: "AWS_SECRET_ACCESS_KEY" - # S3PROXY_ENCRYPT_KEY: AES-256-GCM encryption key (base64-encoded 32 bytes) - # Generate with: openssl rand -base64 32 encryptKey: "" - - # AWS/S3 credentials (ignored if minio.enabled=true and using MinIO defaults) - # Only needed when: minio.enabled=false AND using external S3 awsAccessKeyId: "" awsSecretAccessKey: "" -# Logging -logLevel: "INFO" # Options: DEBUG, INFO, WARNING, ERROR +logLevel: "INFO" -# Resource limits for s3proxy pods -# Adjust based on your workload and cluster capacity resources: requests: cpu: "100m" - memory: "256Mi" + memory: "512Mi" limits: cpu: "1000m" - memory: "512Mi" + memory: "1Gi" -# Kubernetes Service configuration service: - type: ClusterIP # Use LoadBalancer for external access, or configure Ingress - port: 4433 # Service port (container also runs on this port) + type: ClusterIP + port: 4433 diff --git a/s3proxy/config.py b/s3proxy/config.py index 16df0ed..d978ada 100644 --- a/s3proxy/config.py +++ b/s3proxy/config.py @@ -27,14 +27,11 @@ class Settings(BaseSettings): cert_path: str = Field(default="/etc/s3proxy/certs", description="TLS certificate directory") # Performance settings - throttling_requests_max: int = Field(default=0, description="Max concurrent requests") - max_single_encrypted_mb: int = Field(default=16, description="Max single-part object size (MB)") - auto_multipart_mb: int = Field(default=16, description="Auto-multipart threshold (MB)") - max_concurrent_uploads: int = Field(default=10, description="Max concurrent uploads") - max_concurrent_downloads: int = Field(default=10, description="Max concurrent downloads") - - # Feature flags - allow_multipart: bool = Field(default=False, description="Allow unencrypted multipart") + # Memory usage: file_size + ~64MB per concurrent upload + # For 1GB pod with 10MB files: ~13 concurrent safe, default 10 for margin + # Files >16MB automatically use multipart encryption + throttling_requests_max: int = Field(default=10, description="Max concurrent requests (0=unlimited)") + max_upload_size_mb: int = Field(default=45, description="Max single-request upload size (MB)") # Redis settings (for distributed state) redis_url: str = Field(default="redis://localhost:6379/0", description="Redis connection URL") @@ -55,14 +52,9 @@ def kek(self) -> bytes: return hashlib.sha256(self.encrypt_key.encode()).digest() @property - def max_single_encrypted_bytes(self) -> int: - """Max single encrypted object size in bytes.""" - return self.max_single_encrypted_mb * 1024 * 1024 - - @property - def auto_multipart_bytes(self) -> int: - """Auto-multipart threshold in bytes.""" - return self.auto_multipart_mb * 1024 * 1024 + def max_upload_size_bytes(self) -> int: + """Max upload size in bytes.""" + return self.max_upload_size_mb * 1024 * 1024 @property def s3_endpoint(self) -> str: diff --git a/s3proxy/handlers/objects.py b/s3proxy/handlers/objects.py index 8744310..fcf0a1c 100644 --- a/s3proxy/handlers/objects.py +++ b/s3proxy/handlers/objects.py @@ -264,11 +264,13 @@ async def handle_put_object(self, request: Request, creds: S3Credentials) -> Res if needs_chunked_decode: body = decode_aws_chunked(body) - if self.settings.auto_multipart_bytes > 0 and len(body) > self.settings.auto_multipart_bytes: - return await self._put_multipart(client, bucket, key, body, content_type) + # Reject if exceeds max upload size + if len(body) > self.settings.max_upload_size_bytes: + raise HTTPException(413, f"Max upload size: {self.settings.max_upload_size_mb}MB") - if len(body) > self.settings.max_single_encrypted_bytes: - raise HTTPException(413, f"Max size: {self.settings.max_single_encrypted_mb}MB") + # Auto-use multipart for files >16MB to split encryption into parts + if len(body) > crypto.PART_SIZE: + return await self._put_multipart(client, bucket, key, body, content_type) encrypted = crypto.encrypt_object(body, self.settings.kek) etag = hashlib.md5(body).hexdigest() @@ -385,10 +387,11 @@ async def upload_part(data: bytes) -> None: total_plaintext_size += len(chunk) # Upload when buffer reaches PART_SIZE + # Process immediately without intermediate variable to reduce memory while len(buffer) >= crypto.PART_SIZE: - part_data = bytes(buffer[:crypto.PART_SIZE]) + # Extract, upload, then clear - minimizes peak memory + await upload_part(bytes(buffer[:crypto.PART_SIZE])) del buffer[:crypto.PART_SIZE] - await upload_part(part_data) # Upload remaining data if buffer: diff --git a/s3proxy/main.py b/s3proxy/main.py index 4a345a3..b423a58 100644 --- a/s3proxy/main.py +++ b/s3proxy/main.py @@ -304,20 +304,20 @@ async def _handle_object_operation( # Throttling Middleware # ============================================================================ def throttle(app: FastAPI, max_requests: int): - """Wrap app with throttling middleware.""" + """Wrap app with throttling middleware. + + Limits concurrent requests to max_requests. When limit is reached, + additional requests wait in queue instead of being rejected. + This provides memory-bounded execution with graceful backpressure. + """ semaphore = asyncio.Semaphore(max_requests) async def middleware(scope, receive, send): if scope["type"] != "http": return await app(scope, receive, send) - # Atomic acquire - no TOCTOU race - try: - semaphore.acquire_nowait() - except ValueError: - from fastapi.responses import Response - response = Response("Too Many Requests", status_code=429) - return await response(scope, receive, send) + # Wait for slot to become available (queues requests) + await semaphore.acquire() try: await app(scope, receive, send) @@ -337,7 +337,6 @@ def create_app(settings: Settings | None = None) -> FastAPI: # Load credentials and initialize components credentials_store = load_credentials() multipart_manager = MultipartStateManager( - max_concurrent=settings.max_concurrent_uploads, ttl_seconds=settings.redis_upload_ttl_seconds, ) verifier = SigV4Verifier(credentials_store) diff --git a/s3proxy/multipart.py b/s3proxy/multipart.py index 82afd5a..f824bcd 100644 --- a/s3proxy/multipart.py +++ b/s3proxy/multipart.py @@ -1,6 +1,5 @@ """Multipart upload state management.""" -import asyncio import base64 import contextlib import gzip @@ -163,14 +162,12 @@ def _deserialize_upload_state(data: bytes) -> MultipartUploadState: class MultipartStateManager: """Manages multipart upload state in Redis.""" - def __init__(self, max_concurrent: int = 10, ttl_seconds: int = 86400): + def __init__(self, ttl_seconds: int = 86400): """Initialize state manager. Args: - max_concurrent: Max concurrent uploads (per-pod limit) ttl_seconds: TTL for upload state in Redis (default 24h) """ - self._semaphore = asyncio.Semaphore(max_concurrent) self._ttl = ttl_seconds def _redis_key(self, bucket: str, key: str, upload_id: str) -> str: @@ -272,14 +269,6 @@ async def abort_upload(self, bucket: str, key: str, upload_id: str) -> None: rk = self._redis_key(bucket, key, upload_id) await redis_client.delete(rk) - async def acquire_slot(self) -> None: - """Acquire an upload slot (per-pod limit).""" - await self._semaphore.acquire() - - def release_slot(self) -> None: - """Release an upload slot.""" - self._semaphore.release() - def encode_multipart_metadata(meta: MultipartMetadata) -> str: """Encode metadata to base64-compressed JSON.""" diff --git a/tests/conftest.py b/tests/conftest.py index 8a2f8f3..828359a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -543,7 +543,7 @@ async def list_parts(self, *args, **kwargs): @pytest.fixture def multipart_manager(): """Create a multipart state manager.""" - return MultipartStateManager(max_concurrent=10) + return MultipartStateManager() # ============================================================================ diff --git a/tests/test_handlers.py b/tests/test_handlers.py index bccc762..f63dfc5 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -108,8 +108,7 @@ def test_s3_endpoint_without_scheme(self): def test_size_calculations(self, settings): """Test size calculations.""" - assert settings.max_single_encrypted_bytes == 16 * 1024 * 1024 - assert settings.auto_multipart_bytes == 16 * 1024 * 1024 + assert settings.max_upload_size_bytes == 45 * 1024 * 1024 class TestRangeParsing: diff --git a/tests/test_multipart.py b/tests/test_multipart.py index a98d845..13618a6 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -98,25 +98,6 @@ async def test_abort_upload(self): assert await manager.get_upload("bucket", "key", "upload-123") is None - @pytest.mark.asyncio - async def test_semaphore_limits_concurrent_uploads(self): - """Test semaphore limits concurrent uploads.""" - manager = MultipartStateManager(max_concurrent=2) - - # Acquire two slots - await manager.acquire_slot() - await manager.acquire_slot() - - # Third should timeout - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(manager.acquire_slot(), timeout=0.01) - - # Release one - manager.release_slot() - - # Now we can acquire again - await asyncio.wait_for(manager.acquire_slot(), timeout=0.1) - class TestMetadataEncoding: """Test metadata encoding/decoding.""" From e62533cface7c24df1a69fcdc7bb6b1a2b9bb6cc Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Thu, 15 Jan 2026 13:20:51 +0100 Subject: [PATCH 03/39] fix: wip --- .github/workflows/docker-publish.yml | 19 ++-- .github/workflows/helm-lint.yml | 68 ++++++------ .github/workflows/helm-publish.yml | 125 ++++++++++----------- .github/workflows/helm-test.yml | 155 --------------------------- 4 files changed, 102 insertions(+), 265 deletions(-) delete mode 100644 .github/workflows/helm-test.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d57e48b..25b12eb 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,12 +1,9 @@ name: Build and Push Docker Image on: - workflow_dispatch: - inputs: - tag: - description: 'Docker image tag' - required: true - type: string + push: + tags: + - 'v*' jobs: build-and-push: @@ -19,6 +16,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Get version from tag + id: version + run: | + VERSION=${GITHUB_REF#refs/tags/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -34,7 +37,9 @@ jobs: with: context: . push: true - tags: ghcr.io/serversidehannes/s3proxy-python:${{ inputs.tag }} + tags: | + ghcr.io/serversidehannes/s3proxy-python:${{ steps.version.outputs.version }} + ghcr.io/serversidehannes/s3proxy-python:latest cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 0fa75d5..58587c5 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -1,34 +1,34 @@ -# name: Helm Lint -# -# on: -# pull_request: -# branches: [main] -# paths: -# - 'manifests/**' -# -# jobs: -# helm-lint: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout -# uses: actions/checkout@v6.0.2 -# -# - name: Set up Helm -# uses: azure/setup-helm@v4.3.1 -# -# - name: Add Helm dependency repositories -# run: | -# helm repo add dandydev https://dandydeveloper.github.io/charts -# helm repo update -# -# - name: Update Helm dependencies -# run: | -# helm dependency update manifests/ -# -# - name: Lint Helm chart -# run: | -# helm lint manifests/ -# -# - name: Validate Helm template -# run: | -# helm template s3proxy manifests/ --debug > /dev/null +name: Helm Lint + +on: + pull_request: + branches: [main] + paths: + - 'manifests/**' + +jobs: + helm-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Helm + uses: azure/setup-helm@v4.3.1 + + - name: Add Helm dependency repositories + run: | + helm repo add dandydev https://dandydeveloper.github.io/charts + helm repo update + + - name: Update Helm dependencies + run: | + helm dependency update manifests/ + + - name: Lint Helm chart + run: | + helm lint manifests/ + + - name: Validate Helm template + run: | + helm template s3proxy manifests/ --debug > /dev/null diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml index fb56592..a4cad99 100644 --- a/.github/workflows/helm-publish.yml +++ b/.github/workflows/helm-publish.yml @@ -1,69 +1,56 @@ -# name: Package and Push Helm Chart -# -# on: -# push: -# tags: ['v*'] -# workflow_dispatch: -# -# env: -# REGISTRY: ghcr.io -# CHART_NAME: s3proxy-python -# -# jobs: -# helm-publish: -# runs-on: ubuntu-latest -# permissions: -# contents: read -# packages: write -# -# steps: -# - name: Checkout repository -# uses: actions/checkout@v6.0.2 -# -# - name: Set up Helm -# uses: azure/setup-helm@v4.3.1 -# -# - name: Log in to Container Registry -# run: | -# echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin -# -# - name: Add Helm dependency repositories -# run: | -# helm repo add dandydev https://dandydeveloper.github.io/charts -# helm repo update -# -# - name: Update Helm dependencies -# run: | -# helm dependency update manifests/ -# -# - name: Get chart version -# id: chart -# run: | -# VERSION=$(grep '^version:' manifests/Chart.yaml | awk '{print $2}') -# echo "version=$VERSION" >> $GITHUB_OUTPUT -# # Use tag version if this is a tag push -# if [[ "${{ github.ref }}" == refs/tags/v* ]]; then -# TAG_VERSION="${{ github.ref_name }}" -# TAG_VERSION="${TAG_VERSION#v}" -# echo "version=$TAG_VERSION" >> $GITHUB_OUTPUT -# fi -# -# - name: Update chart version for tags -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# TAG_VERSION="${{ github.ref_name }}" -# TAG_VERSION="${TAG_VERSION#v}" -# sed -i "s/^version:.*/version: $TAG_VERSION/" manifests/Chart.yaml -# sed -i "s/^appVersion:.*/appVersion: \"$TAG_VERSION\"/" manifests/Chart.yaml -# -# - name: Lint Helm chart -# run: | -# helm lint manifests/ -# -# - name: Package Helm chart -# run: | -# helm package manifests/ --destination . -# -# - name: Push Helm chart to OCI registry -# run: | -# helm push ${{ env.CHART_NAME }}-${{ steps.chart.outputs.version }}.tgz oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts +name: Package and Push Helm Chart + +on: + push: + tags: + - 'v*' + +jobs: + helm-publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Helm + uses: azure/setup-helm@v4.3.1 + + - name: Log in to Container Registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Add Helm dependency repositories + run: | + helm repo add dandydev https://dandydeveloper.github.io/charts + helm repo update + + - name: Update Helm dependencies + run: | + helm dependency update manifests/ + + - name: Get version from tag + id: version + run: | + VERSION=${GITHUB_REF#refs/tags/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Update chart version + run: | + sed -i "s/^version:.*/version: ${{ steps.version.outputs.version }}/" manifests/Chart.yaml + sed -i "s/^appVersion:.*/appVersion: \"${{ steps.version.outputs.version }}\"/" manifests/Chart.yaml + + - name: Lint Helm chart + run: | + helm lint manifests/ + + - name: Package Helm chart + run: | + helm package manifests/ --destination . + + - name: Push Helm chart to OCI registry + run: | + helm push s3proxy-python-${{ steps.version.outputs.version }}.tgz oci://ghcr.io/${{ github.repository_owner }}/charts diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml deleted file mode 100644 index 8b12242..0000000 --- a/.github/workflows/helm-test.yml +++ /dev/null @@ -1,155 +0,0 @@ -# name: Helm Test -# -# on: -# push: -# branches: [main] -# paths: -# - 'manifests/**' -# - 's3proxy/**' -# - 'Dockerfile' -# - '.github/workflows/helm-test.yml' -# pull_request: -# branches: [main] -# paths: -# - 'manifests/**' -# - 's3proxy/**' -# - 'Dockerfile' -# - '.github/workflows/helm-test.yml' -# workflow_dispatch: -# -# jobs: -# helm-test: -# runs-on: ubuntu-latest -# steps: -# - name: Checkout -# uses: actions/checkout@v6.0.2 -# -# - name: Create Kind cluster -# uses: helm/kind-action@v1.13.0 -# with: -# cluster_name: s3proxy-test -# -# - name: Install Helm -# uses: azure/setup-helm@v4.3.1 -# -# - name: Add Helm repos and update dependencies -# run: | -# helm repo add dandydev https://dandydeveloper.github.io/charts -# helm repo update -# helm dependency update ./manifests -# -# - name: Build and load image -# run: | -# docker build -t s3proxy-python:latest . -# kind load docker-image s3proxy-python:latest --name s3proxy-test -# -# - name: Install Helm chart -# run: | -# helm upgrade --install s3proxy ./manifests \ -# -n s3proxy --create-namespace \ -# --set image.repository=s3proxy-python \ -# --set image.tag=latest \ -# --wait --timeout 300s -# -# - name: Wait for pods -# run: | -# kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=s3proxy-python -n s3proxy --timeout=120s -# kubectl wait --for=condition=ready pod -l release=s3proxy -n s3proxy --timeout=180s || true -# kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=s3proxy-python-minio -n s3proxy --timeout=120s -# -# - name: Show deployment status -# run: | -# kubectl get all -n s3proxy -# kubectl get pods -n s3proxy -o wide -# -# - name: Test health endpoint -# run: | -# kubectl run curl-test --image=curlimages/curl --rm -it --restart=Never -n s3proxy -- \ -# curl -sf http://s3proxy-python:4433/healthz -# echo "Health check passed!" -# -# - name: Run load test -# run: | -# kubectl run s3-load-test -n s3proxy --rm -it --restart=Never \ -# --image=amazon/aws-cli:latest \ -# --env="AWS_ACCESS_KEY_ID=minioadmin" \ -# --env="AWS_SECRET_ACCESS_KEY=minioadmin" \ -# --env="AWS_DEFAULT_REGION=us-east-1" \ -# --command -- /bin/sh -c " -# # Create test bucket -# echo 'Creating test bucket...' -# aws --endpoint-url http://s3proxy-python:4433 s3 mb s3://ci-test-bucket 2>/dev/null || true -# -# # Generate test files (smaller for CI) -# echo 'Generating 64MB test files...' -# mkdir -p /tmp/testfiles -# for i in 1 2 3; do -# dd if=/dev/urandom of=/tmp/testfiles/file-\$i.bin bs=1M count=64 2>/dev/null & -# done -# wait -# echo 'Files generated' -# ls -lh /tmp/testfiles/ -# -# # Upload concurrently -# echo '' -# echo '=== Starting concurrent uploads ===' -# START=\$(date +%s) -# -# for i in 1 2 3; do -# aws --endpoint-url http://s3proxy-python:4433 s3 cp /tmp/testfiles/file-\$i.bin s3://ci-test-bucket/file-\$i.bin & -# done -# wait -# -# END=\$(date +%s) -# DURATION=\$((END - START)) -# echo '' -# echo \"=== Upload complete in \${DURATION}s ===\" -# -# # Verify uploads -# echo '' -# echo '=== Listing uploaded files ===' -# aws --endpoint-url http://s3proxy-python:4433 s3 ls s3://ci-test-bucket/ -# -# # Download and verify -# echo '' -# echo '=== Downloading files to verify ===' -# mkdir -p /tmp/downloads -# for i in 1 2 3; do -# aws --endpoint-url http://s3proxy-python:4433 s3 cp s3://ci-test-bucket/file-\$i.bin /tmp/downloads/file-\$i.bin & -# done -# wait -# -# echo '' -# echo '=== Comparing checksums ===' -# md5sum /tmp/testfiles/*.bin > /tmp/orig.md5 -# md5sum /tmp/downloads/*.bin > /tmp/down.md5 -# -# ORIG_SUMS=\$(cat /tmp/orig.md5 | while read sum name; do echo \$sum; done | sort) -# DOWN_SUMS=\$(cat /tmp/down.md5 | while read sum name; do echo \$sum; done | sort) -# -# cat /tmp/orig.md5 -# echo '' -# if [ \"\$ORIG_SUMS\" = \"\$DOWN_SUMS\" ]; then -# echo 'All checksums match - encryption/decryption working!' -# else -# echo 'Checksum mismatch!' -# exit 1 -# fi -# " -# -# - name: Show pod logs on failure -# if: failure() -# run: | -# echo "=== S3Proxy Logs ===" -# kubectl logs -l app.kubernetes.io/name=s3proxy-python -n s3proxy --tail=100 || true -# echo "" -# echo "=== Redis HA Logs ===" -# kubectl logs -l release=s3proxy -n s3proxy --tail=50 || true -# echo "" -# echo "=== Events ===" -# kubectl get events -n s3proxy --sort-by=.lastTimestamp | tail -20 || true -# -# - name: Cleanup -# if: always() -# run: | -# kind delete cluster --name s3proxy-test From 8da969416ed463e6e59f7181172ed5ddef19862e Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Thu, 15 Jan 2026 13:22:46 +0100 Subject: [PATCH 04/39] fix: wip --- .github/workflows/helm-lint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 58587c5..b3f8c0f 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -2,7 +2,6 @@ name: Helm Lint on: pull_request: - branches: [main] paths: - 'manifests/**' From 4eb73d7dde418505ce2fdcaf5cd1580ea09e7ea5 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 20 Jan 2026 11:57:29 +0100 Subject: [PATCH 05/39] Add gateway service for request-level load balancing via ingress - Add gateway-service.yaml: ExternalName service pointing to ingress controller - Add ingress.yaml: Auto-generates host from gateway.serviceName + namespace - Update tests to use http://s3-gateway.s3proxy endpoint - Enable gateway + ingress in e2e tests for load balancing verification - Default gateway/ingress to disabled for safer defaults Internal access: gateway.enabled=true, ingress.enabled=true -> endpoint: http://s3-gateway. External access: gateway.enabled=false, ingress.enabled=true, hosts=[...] -> endpoint: http://your-domain.com --- e2e/docker-compose.helm-test.yml | 15 +++++++ e2e/test-helm.sh | 44 +++++++++++++++++--- manifests/templates/gateway-service.yaml | 12 ++++++ manifests/templates/ingress.yaml | 53 ++++++++++++++++++++++++ manifests/values.yaml | 22 ++++++++++ 5 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 manifests/templates/gateway-service.yaml create mode 100644 manifests/templates/ingress.yaml diff --git a/e2e/docker-compose.helm-test.yml b/e2e/docker-compose.helm-test.yml index ee84199..12ffa44 100644 --- a/e2e/docker-compose.helm-test.yml +++ b/e2e/docker-compose.helm-test.yml @@ -79,6 +79,19 @@ services: done kubectl get nodes + echo "=== Installing NGINX Ingress Controller ===" + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo update + + # Install NGINX Controller with settings optimized for Kind + helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx --create-namespace \ + --set controller.service.type=ClusterIP \ + --set controller.admissionWebhooks.enabled=false \ + --wait --timeout 300s + + echo "✓ Ingress Controller installed" + echo "=== Building s3proxy image ===" docker build -t s3proxy:latest /app @@ -101,6 +114,8 @@ services: --set image.repository=s3proxy \ --set image.pullPolicy=IfNotPresent \ --set secrets.encryptKey="$$ENCRYPT_KEY" \ + --set gateway.enabled=true \ + --set ingress.enabled=true \ --set redis-ha.persistentVolume.enabled=false \ --set redis-ha.hardAntiAffinity=false \ --set redis-ha.affinity=null \ diff --git a/e2e/test-helm.sh b/e2e/test-helm.sh index 4302512..b4808aa 100755 --- a/e2e/test-helm.sh +++ b/e2e/test-helm.sh @@ -38,6 +38,17 @@ case "${1:-run}" in load-test) echo "Running S3 load test (3 concurrent 512MB uploads)..." docker compose -f $COMPOSE_FILE exec helm-test sh -c ' + # Get pod names for load balancing verification + PODS=$(kubectl get pods -n s3proxy -l app=s3proxy-python -o jsonpath="{.items[*].metadata.name}") + POD_COUNT=$(echo $PODS | wc -w) + echo "Found $POD_COUNT s3proxy pods: $PODS" + + # Save current log line counts + mkdir -p /tmp/lb-test + for pod in $PODS; do + kubectl logs $pod -n s3proxy 2>/dev/null | wc -l > /tmp/lb-test/$pod.start + done + echo "=== Creating test pod with AWS CLI ===" kubectl run s3-load-test -n s3proxy --rm -it --restart=Never \ --image=amazon/aws-cli:latest \ @@ -47,7 +58,7 @@ case "${1:-run}" in --command -- /bin/sh -c " # Create test bucket echo \"Creating test bucket...\" - aws --endpoint-url http://s3proxy-python:4433 s3 mb s3://load-test-bucket 2>/dev/null || true + aws --endpoint-url http://s3-gateway.s3proxy s3 mb s3://load-test-bucket 2>/dev/null || true # Generate 3 random 512MB files echo \"Generating 512MB test files...\" @@ -65,7 +76,7 @@ case "${1:-run}" in START=\$(date +%s) for i in 1 2 3; do - aws --endpoint-url http://s3proxy-python:4433 s3 cp /tmp/testfiles/file-\$i.bin s3://load-test-bucket/file-\$i.bin & + aws --endpoint-url http://s3-gateway.s3proxy s3 cp /tmp/testfiles/file-\$i.bin s3://load-test-bucket/file-\$i.bin & done wait @@ -77,14 +88,14 @@ case "${1:-run}" in # Verify uploads echo \"\" echo \"=== Listing uploaded files ===\" - aws --endpoint-url http://s3proxy-python:4433 s3 ls s3://load-test-bucket/ + aws --endpoint-url http://s3-gateway.s3proxy s3 ls s3://load-test-bucket/ # Download and verify echo \"\" echo \"=== Downloading files to verify ===\" mkdir -p /tmp/downloads for i in 1 2 3; do - aws --endpoint-url http://s3proxy-python:4433 s3 cp s3://load-test-bucket/file-\$i.bin /tmp/downloads/file-\$i.bin & + aws --endpoint-url http://s3-gateway.s3proxy s3 cp s3://load-test-bucket/file-\$i.bin /tmp/downloads/file-\$i.bin & done wait @@ -105,6 +116,29 @@ case "${1:-run}" in exit 1 fi " + + # Verify load balancing + echo "" + echo "=== Checking load balancing ===" + sleep 2 + PODS_HIT=0 + for pod in $PODS; do + START_LINE=$(cat /tmp/lb-test/$pod.start 2>/dev/null || echo "0") + REQUEST_COUNT=$(kubectl logs $pod -n s3proxy 2>/dev/null | tail -n +$((START_LINE + 1)) | grep -c -E "GET|POST|PUT|HEAD" || echo "0") + if [ "$REQUEST_COUNT" -gt 0 ]; then + PODS_HIT=$((PODS_HIT + 1)) + echo "✓ Pod $pod: received $REQUEST_COUNT requests" + else + echo " Pod $pod: received 0 requests" + fi + done + rm -rf /tmp/lb-test + + if [ "$PODS_HIT" -ge 2 ]; then + echo "✓ Load balancing verified - traffic distributed across $PODS_HIT pods" + else + echo "⚠ Traffic went to only $PODS_HIT pod(s)" + fi ' ;; watch) @@ -175,7 +209,7 @@ case "${1:-run}" in echo " status - Show deployment status" echo " pods - Show pod details and resources" echo " logs - Stream s3proxy logs" - echo " load-test - Run 1.5GB concurrent upload test" + echo " load-test - Run 1.5GB concurrent upload test + verify load balancing" echo " redis - Inspect Redis keys and memory" echo " watch - Live pod CPU/memory (installs metrics-server)" echo " shell - Interactive kubectl shell" diff --git a/manifests/templates/gateway-service.yaml b/manifests/templates/gateway-service.yaml new file mode 100644 index 0000000..2f41919 --- /dev/null +++ b/manifests/templates/gateway-service.yaml @@ -0,0 +1,12 @@ +{{- if .Values.gateway.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.gateway.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ .Chart.Name }}-gateway +spec: + type: ExternalName + externalName: {{ .Values.gateway.ingressService }} +{{- end }} diff --git a/manifests/templates/ingress.yaml b/manifests/templates/ingress.yaml new file mode 100644 index 0000000..e589ae0 --- /dev/null +++ b/manifests/templates/ingress.yaml @@ -0,0 +1,53 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Chart.Name }} + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if and .Values.gateway.enabled (not .Values.ingress.hosts) }} + # Auto-generate host from gateway service name + namespace + - host: {{ printf "%s.%s" .Values.gateway.serviceName .Release.Namespace | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Chart.Name }} + port: + number: {{ .Values.service.port }} + {{- else }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ $.Chart.Name }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/manifests/values.yaml b/manifests/values.yaml index 960624d..b529cc7 100644 --- a/manifests/values.yaml +++ b/manifests/values.yaml @@ -118,3 +118,25 @@ resources: service: type: ClusterIP port: 4433 + +# Gateway + Ingress for request-level load balancing +# Internal: gateway.enabled=true, ingress.enabled=true -> http://s3-gateway. +# External: gateway.enabled=false, ingress.enabled=true, hosts=[...] -> http://your-domain.com +gateway: + enabled: false + serviceName: s3-gateway + ingressService: ingress-nginx-controller.ingress-nginx.svc.cluster.local + +ingress: + enabled: false + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "60" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/upstream-keepalive-connections: "100" + tls: [] + hosts: [] \ No newline at end of file From 4163dccf4342e79d8153bbf5828a4950f38dea5c Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 20 Jan 2026 12:23:11 +0100 Subject: [PATCH 06/39] Clean up repo: remove flowchart script, gitignore helm deps --- generate_flowchart.py | 203 ------------------------------------ manifests/charts/.gitignore | 2 + 2 files changed, 2 insertions(+), 203 deletions(-) delete mode 100644 generate_flowchart.py create mode 100644 manifests/charts/.gitignore diff --git a/generate_flowchart.py b/generate_flowchart.py deleted file mode 100644 index d73850b..0000000 --- a/generate_flowchart.py +++ /dev/null @@ -1,203 +0,0 @@ -import matplotlib.pyplot as plt -import matplotlib.patches as mpatches -from matplotlib.patches import FancyBboxPatch, Polygon -import numpy as np - -# Claude/Anthropic color palette -COLORS = { - 'bg': '#FAF9F7', - 'card_bg': '#FFFFFF', - 'primary': '#D97757', - 'tan': '#C9A87C', - 'sage': '#7D8B74', - 'text': '#1F1F1F', - 'text_muted': '#6B6B6B', - 'border': '#E5E0DB', - 'success': '#6B8E5E', - 'error': '#C45B4A', - 'blue': '#5B7B9A', - 'purple': '#8B7BA5', -} - -# Larger canvas, simpler layout -fig, ax = plt.subplots(1, 1, figsize=(24, 16)) -fig.patch.set_facecolor(COLORS['bg']) -ax.set_facecolor(COLORS['bg']) -ax.set_xlim(0, 24) -ax.set_ylim(0, 16) -ax.set_aspect('equal') -ax.axis('off') - -def draw_box(ax, x, y, w, h, text, color, subtext=None, fontsize=12): - shadow = FancyBboxPatch((x+0.06, y-0.06), w, h, - boxstyle="round,pad=0.02,rounding_size=0.15", - facecolor='#00000010', edgecolor='none') - ax.add_patch(shadow) - - box = FancyBboxPatch((x, y), w, h, - boxstyle="round,pad=0.02,rounding_size=0.15", - facecolor=color, edgecolor=color, linewidth=2) - ax.add_patch(box) - - text_y = y + h/2 + (0.15 if subtext else 0) - ax.text(x + w/2, text_y, text, ha='center', va='center', fontsize=fontsize, - fontweight='bold', color='white', family='sans-serif') - - if subtext: - ax.text(x + w/2, y + h/2 - 0.25, subtext, ha='center', va='center', - fontsize=fontsize-3, color='#FFFFFFCC', family='sans-serif') - -def draw_diamond(ax, x, y, w, h, text, color=COLORS['tan']): - cx, cy = x + w/2, y + h/2 - pts = [(cx, cy+h/2), (cx+w/2, cy), (cx, cy-h/2), (cx-w/2, cy)] - diamond = Polygon(pts, facecolor=color, edgecolor=color, linewidth=2) - ax.add_patch(diamond) - ax.text(cx, cy, text, ha='center', va='center', fontsize=11, - fontweight='bold', color='white', family='sans-serif') - -def draw_arrow(ax, start, end, color=COLORS['border'], lw=2.5): - ax.annotate('', xy=end, xytext=start, - arrowprops=dict(arrowstyle='->', color=color, lw=lw, - connectionstyle='arc3,rad=0')) - -def draw_line(ax, points, color=COLORS['border'], lw=2.5): - xs = [p[0] for p in points] - ys = [p[1] for p in points] - ax.plot(xs, ys, color=color, linewidth=lw, solid_capstyle='round') - -def draw_label(ax, x, y, text, color=COLORS['text_muted'], fontsize=10): - ax.text(x, y, text, ha='center', va='center', fontsize=fontsize, - fontweight='600', color=color, family='sans-serif') - -# ============ TITLE ============ -ax.text(12, 15.3, 'S3 Proxy — High-Level Flow', ha='center', va='center', - fontsize=28, fontweight='bold', color=COLORS['text'], family='sans-serif') -ax.text(12, 14.7, 'Client-Side Encryption Proxy for AWS S3', ha='center', va='center', - fontsize=14, color=COLORS['text_muted'], family='sans-serif') - -# ============ MAIN FLOW ============ - -# 1. CLIENT REQUEST -draw_box(ax, 9.5, 13, 5, 1.1, 'Client Request', COLORS['blue'], 'S3 API call') - -draw_arrow(ax, (12, 13), (12, 12.3)) - -# 2. PARSE & AUTH -draw_box(ax, 8.5, 10.8, 7, 1.3, 'Parse & Authenticate', COLORS['purple'], 'SigV4 signature verification') - -# Auth failure branch -draw_line(ax, [(8.5, 11.45), (6.5, 11.45)]) -draw_arrow(ax, (6.5, 11.45), (5.5, 11.45)) -draw_box(ax, 2.5, 10.95, 3, 0.9, '403', COLORS['error'], 'Invalid') -draw_label(ax, 7, 11.8, 'FAIL', COLORS['error'], 9) - -draw_arrow(ax, (12, 10.8), (12, 10.1)) -draw_label(ax, 12.5, 10.45, 'PASS', COLORS['success'], 9) - -# 3. ROUTING -draw_diamond(ax, 9.5, 8.5, 5, 1.4, 'Route Request', COLORS['tan']) - -# Branch lines -draw_line(ax, [(9.5, 9.2), (4, 9.2), (4, 7.8)]) # Left branch -draw_line(ax, [(14.5, 9.2), (20, 9.2), (20, 7.8)]) # Right branch -draw_arrow(ax, (12, 8.5), (12, 7.8)) # Center branch - -# 4. OPERATIONS (3 main paths) -draw_box(ax, 1.5, 6.3, 5, 1.3, 'PUT / POST', COLORS['primary'], 'Upload / Multipart') -draw_box(ax, 9.5, 6.3, 5, 1.3, 'GET', COLORS['blue'], 'Download') -draw_box(ax, 17.5, 6.3, 5, 1.3, 'LIST / HEAD / DELETE', COLORS['sage'], 'Metadata ops') - -# Arrows down to encryption -draw_arrow(ax, (4, 6.3), (4, 5.6)) -draw_arrow(ax, (12, 6.3), (12, 5.6)) -draw_arrow(ax, (20, 6.3), (20, 5.6)) - -# 5. ENCRYPTION LAYER -# Background for encryption section -enc_bg = FancyBboxPatch((1, 3.8), 22, 1.6, - boxstyle="round,pad=0.02,rounding_size=0.2", - facecolor=COLORS['tan'], edgecolor='none', alpha=0.15) -ax.add_patch(enc_bg) - -ax.text(12, 5.15, 'ENCRYPTION LAYER', ha='center', va='center', - fontsize=12, fontweight='bold', color=COLORS['tan'], family='sans-serif', - bbox=dict(boxstyle='round,pad=0.3', facecolor=COLORS['bg'], edgecolor='none')) - -draw_box(ax, 1.5, 4, 5, 1, 'Encrypt', COLORS['tan'], 'AES-256-GCM') -draw_box(ax, 9.5, 4, 5, 1, 'Decrypt', COLORS['tan'], 'Unwrap DEK') -draw_box(ax, 17.5, 4, 5, 1, 'Pass-through', COLORS['border'], fontsize=11) -ax.text(20, 4.5, 'or read metadata', ha='center', va='center', - fontsize=9, color=COLORS['text_muted']) - -# Arrows down to S3 -draw_arrow(ax, (4, 4), (4, 3.3)) -draw_arrow(ax, (12, 4), (12, 3.3)) -draw_arrow(ax, (20, 4), (20, 3.3)) - -# Converge to S3 -draw_line(ax, [(4, 3.1), (4, 2.8), (20, 2.8), (20, 3.1)]) -draw_line(ax, [(12, 3.1), (12, 2.8)]) - -# 6. S3 BACKEND -draw_box(ax, 8.5, 1.5, 7, 1.2, 'AWS S3', COLORS['primary'], 'Actual storage') -draw_arrow(ax, (12, 2.8), (12, 2.7)) - -# 7. RESPONSE -draw_arrow(ax, (12, 1.5), (12, 0.9)) -draw_box(ax, 9.5, 0.1, 5, 0.7, 'Response', COLORS['success'], fontsize=11) - -# ============ SIDE INFO BOXES ============ - -# Left side - Key info -info_bg = FancyBboxPatch((0.3, 0.3), 4.2, 2.8, - boxstyle="round,pad=0.05,rounding_size=0.15", - facecolor=COLORS['card_bg'], edgecolor=COLORS['border'], linewidth=1.5) -ax.add_patch(info_bg) - -ax.text(2.4, 2.85, 'Encryption', ha='center', fontsize=11, fontweight='bold', - color=COLORS['tan'], family='sans-serif') -ax.text(0.5, 2.4, '• AES-256-GCM', fontsize=9, color=COLORS['text_muted']) -ax.text(0.5, 2.0, '• Per-object DEK', fontsize=9, color=COLORS['text_muted']) -ax.text(0.5, 1.6, '• KEK wraps DEK', fontsize=9, color=COLORS['text_muted']) -ax.text(0.5, 1.2, '• 12-byte nonce', fontsize=9, color=COLORS['text_muted']) -ax.text(0.5, 0.8, '• 16-byte auth tag', fontsize=9, color=COLORS['text_muted']) - -# Right side - Features -feat_bg = FancyBboxPatch((19.5, 0.3), 4.2, 2.8, - boxstyle="round,pad=0.05,rounding_size=0.15", - facecolor=COLORS['card_bg'], edgecolor=COLORS['border'], linewidth=1.5) -ax.add_patch(feat_bg) - -ax.text(21.6, 2.85, 'Features', ha='center', fontsize=11, fontweight='bold', - color=COLORS['purple'], family='sans-serif') -ax.text(19.7, 2.4, '• AWS SigV4 Auth', fontsize=9, color=COLORS['text_muted']) -ax.text(19.7, 2.0, '• Streaming (64KB)', fontsize=9, color=COLORS['text_muted']) -ax.text(19.7, 1.6, '• Multipart uploads', fontsize=9, color=COLORS['text_muted']) -ax.text(19.7, 1.2, '• Range requests', fontsize=9, color=COLORS['text_muted']) -ax.text(19.7, 0.8, '• Transparent proxy', fontsize=9, color=COLORS['text_muted']) - -# ============ LEGEND ============ -legend_items = [ - (COLORS['blue'], 'Client/Download'), - (COLORS['primary'], 'Upload/Core'), - (COLORS['sage'], 'Metadata'), - (COLORS['tan'], 'Encryption'), - (COLORS['purple'], 'Auth/Routing'), -] - -legend_x = 0.5 -for i, (color, label) in enumerate(legend_items): - x_pos = legend_x + i * 4.7 - rect = FancyBboxPatch((x_pos, 14.6), 0.4, 0.35, - boxstyle="round,pad=0.02,rounding_size=0.08", - facecolor=color, edgecolor='none') - ax.add_patch(rect) - ax.text(x_pos + 0.55, 14.77, label, fontsize=9, va='center', - color=COLORS['text_muted'], family='sans-serif') - -plt.tight_layout() -plt.savefig('/Users/hgu/Desktop/sseproxy-python/s3proxy_flowchart.png', dpi=150, - bbox_inches='tight', facecolor=COLORS['bg'], edgecolor='none') -plt.close() - -print("High-level flowchart saved to: s3proxy_flowchart.png") diff --git a/manifests/charts/.gitignore b/manifests/charts/.gitignore new file mode 100644 index 0000000..cf8b471 --- /dev/null +++ b/manifests/charts/.gitignore @@ -0,0 +1,2 @@ +# Helm dependencies (downloaded via helm dependency build) +*.tgz From 5ab0a2ef4c50dd2e5840b88e3c2d8cabbb2e084c Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 20 Jan 2026 12:23:57 +0100 Subject: [PATCH 07/39] Remove tracked helm dependency (downloaded at build time) --- manifests/charts/redis-ha-4.35.5.tgz | Bin 37508 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 manifests/charts/redis-ha-4.35.5.tgz diff --git a/manifests/charts/redis-ha-4.35.5.tgz b/manifests/charts/redis-ha-4.35.5.tgz deleted file mode 100644 index d5af37e86e16cea96dabcc20ec8446bbd5aa513a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37508 zcmV(`K-0e;iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYeciT3yINZPWDR58w&Y}f?mk>wrJVM}va~8n!Btml(Gsq!;E6ipc03et{%23*;5xhhoAn^oGvs9LW zaR@SsC>W!d;TT1L;t&NerC>r*Fo8HCmna2u7{jT6-IkFzfr(E(`_72J?Mp)vJ{_Z4251qrjB;}B%XFI(Bg`uAsux*AlmA9 z-~uf!VU(ec2Y6fOkfP2U;+R1kqf`_{bBLo6fG{PoAHl^0qbU5xl)vR2?Enx%PM;9Q z;UYwrC?W|;JMyI#03AxQG(ZAig^(e+iX#F;>)RBwS#~Z6xzFM5Yq)*E8nk~Nk@LP9 zX7$RnR`{1;3{Kgt5oDF27U zCr^gkCHcQI+!;K||4;F(t$`mRat@dp%~mLU#%B>Y^A~@z8_~%G*(z)3P>C+ zL|496qt$y@P6E11)g_vEwFkEyb(lGSIxM>pbw>S>bH41?s z4h2)hK#aJ0nWSV+7z))`NgA3O9IKR3q0oW>@W9m!2Qy%Z4M+@djG3t5BOi3X+4{B% z<_N|Ve8Xl!%l($kWV_%BM-gvDVJ5+@B-|tO zIRq3XFonE35vGi@hU&1yTXvau-q~%xvj+A^7PDfveXtuvy4CWe1>wi z)@qJNQ7Q(PgdO+tq=+Ua1&H7|if9pvs4G8=z|C!K1=U}aUg7|C)W5IndTWi%NRc&% zNyq$U7MXVS0EkM>iBGR#0!|TS;25TGju=X5-UR<(nvrX#=?tfzDd;f{Vn)bC2LQ7K znw3beb$UfH^#GlrC<z#rtR zrCOQ{{c`e%`h1*0zJeJl9+QyoO~rVE-5}ue;{Gd9{$8<)2PNpf<{j0P8J%&00zAQD z4kI>0DS#R+RTR(}$)b?Yt{_Ex_H(sL{51kB&5Y*2#jL&lm%uQ>mnvVBKAeDvXRfe|&d+>K)*tsQ!qhhZq?saRkY z<0x>Y$!o23yqz4B;l6_yo=3vs@^Tj(r{r3yA?Jv3dHIUK@H-g6IN)UJ=O&cbQl8Q* z;hSn7?D8%^h6ADbCn>q+jKqji-Hh1hbPULRj^a?XqirG40`UDgS72m~Aj)_Z>Q|;h zaL)B|U+wfU>CYf(03=J5 zhT=5|@nm7DYSMEGbb$VOOj0%i&vu@FTT?AWNkkSx-!S~4VVDjGwKqS|H@B}MfLeY6Y_ zdZX@!o~C|+=3s&&U2rV9hML(*L4+?5=-%7_{{%${Qsj$W!0oN4N0&vkE9y(#)Y|{4 zwtqwm(TJG%;`Ieuh+=t}W5{NlE9X)U&9nv$*bEx9sayz9%J2jS96i6#dH&@!2f(3? zK%dQ%erc3!-H?kYsT>LM55qTcw6JheRqzBXNCvK8%m4%-3PsbQ;0%Bb`K{&C0IeSn z&PJ9*l*v) z#!|1pm6bT%IjIGn*tU#StQq!9bih3oHVvhs?f3)#o7}45tZ-~>8FD7nnl>7BWleUe zIVwEscsnI@ABB>VWwyNP$JWAE9C9zmB|FByFVu^a>Xdxk? zuqKkQ4yLY@Tbn9+yJm*MNfViX_h_E5#Xe5ob1g#sU8r(QLa>i%nh9|SzRSWXVk-=9 zn#CSWgiPQ7?BNd2`?6y>d5T5a7{5@}N}D*%eV=%QJbX-{W}Ke#`OIx+9uqS>Tod$5e1hRYKqEgKcRHN8GVvPV*W%Vo_Z0!ENWqjJ{mU;rJt~vt$Ti*jdQs)X?pwNk|E*u_P-QT1v z#qqR?rQSlErf@RBffU}YP`1e>(RRXe_Cc2fhsU%~_3)Tl)6Lauu{g-i=Ical@uEO1 z$_()vmtO z2f-9pK3ijYN0URv=uq84iw|;sz2PH?VU!Qu{2YljB1y@dH=9u~M=Zqw1r#xc<0g%1R*7?634u(|#&RY4#ivN?Wk0-PRCNWUz3>Yhx(*W zwD2=mlqlc6rI|Qyi21sbGm)yJCV*TfiB`gn&fDlb^ zjQAEf)qN7RMP;koZxWP3MpD&ONrKuVhv$f7a`)>X(l95ID(UI~w06NMNl?l#s_PFx zahMPsGYV)H%((iJB7q-{r&|h#Q`y!QU=fwdqul)Rt$=ShQo_l%;K!!za!>a$6#`@T z_z>&~TQ!MGc&Xx{EV+ApD9Yvhj$>@g ze@ID|nBStr5s6O-Vb=1xDlTR_DppwYtL$=7*Q8kLxmI(W;!7N%DLSA5jG%NU%q2KQ zK|tooaY`mQLVC?m=8i1p!>c?`2GejiW_Wip;gfAqBC2J7gR+D&sKd|@)o$D`7{hi=mBQXr zJ4^`Q8X^@g)x6PsTss)GUxGmD;G!*q_6MbBDZ_|dXg!;;`4~`X+F&I2?9kHK=v?4% zBG&_9#+5!RYk(>xF_a5@en?kn1D+G>+#640lA_S>ls6CJfLZo2Xv9}DG2dyUFh>pR zj&NLb4CTy}_k@J6}fYH80 zxxXg<^(cx^z(^|fB8BY4GRkwgCnD58*0f^YkjMaDhGZ04wESM>;mC`4DVn1If z$9Soyc?M>od_uxIwPBe~wT(iyqf4#y2T4j82}m>or+dfFO500om6$VtvEF{60(%m& zl`|iOa|BFef;7(N>>;OLiCx5TKZy|d4)P^)HwX}=ow8ZQ!MiY=Sy)7JMyPz_UtQcrnL^d)%nlao)rIxILDD{&8{kD)~faa1n67k=bz zMn+ew^^vf^YZ}R;V?jXahRt8*$4kXJheZVR36(g3k$)~ZAZ!6t*|F?1$McLX&dGI) z%9hjg(BFCD4_nl!_nzL3uHb^^o+8>UVMZte@A>$>2NM#7axC8W`FoGQ?EvY-@RCqA zUeJb7XgI|ZW#3XU@-PziYtF~P424;Q(k+1esSmo8k@TYLciJ9NIT$fNrJ83E!%2iZ z(+ur*%-@#Z>)>byUb2+Y?GTPJ*InwH)085?^xhtX%qI zIM@CwsuN%;cf{iOKrAxg{l9*B_wLGj_s;+I>-G0iDS|OqOY})7S4^Z~SnQ`&|KUBQVgLYK=2h10XiyETY2Jo}=6;9ORw88n>AVETZ`t-CF+h+0Hjl4b?W!35j|Q zh$sgYF>wq&TU?70R(XmT1>JJN1)^QP%t~K#KC>l0C&*II{?{OJC~UR~p%kCXy}2@H zDr24q=`6Tx0j@pp>8u(QJ{4X5a!=IY%~N@HKtMMpUFnCZzZ%EU;yYf>;1qM^f+E{b z?uQfx=oqD#Yo}r;#Srib1+G#ooJ}K`P7yfS{|?X?CUiy^^}*)FHxyKD_;=#FgvkHo zdsrD2Vd3dX1C`jJHXuI&&j$IN#g{1NtdykW9NBxq+&$fh2eDn2<&Sys>fVwwqyZ_) zIQEF95?@d;j6jGYxZsSV=8kkAC}5_xE~5NCFK#-sGq85zqMtS-IN?)Y>U~9sb~r`8 z*-VO*8a~Z|pBm$JBa+I>>;y5+^t??GoslT4143oNQ(Z21+VDK^J;ad|cuSy6WI+qc zQ_FP;hIo~cPj!Wcq0fQdZ|u_|at|^1C6Es?`YtqAAAT2|C_TiW7XUuUkXKP$eZZ?Q?l9a{AWeh4o2GD~{iE_du3B$9<{(iik;-!6 zf$v|v9lzZF&i-W|n(W5HCkKaxup+I+^HCAoUTsm<1EB2hVhaiRFo~nTE$}@_f!IlS z($PnP6SjF7k%nW+5DZ0RnlNpHP0Mvm_`J*s-*WW8CPfIGqljF|CD%T;=bx-lc6nws z!BfT8<)fOsZ>DETcbBJOq7|#^_bSdgPjjKG@rG3pvCfv@o4`EfTf=ybenEv2s-)Jm zsK4x@WK~!ez-9QrmOz!uzC0|8BOncgoVX~&!m7YX z3a6-!f*e;*V=l-r0$sjKkYB^|fXtIrx|o)~%usMav$=lV;?OUjGwN`%&1v8JycV9m z7qO)BnDHS``NT%`_-+5SoDCf3LT(%BQ>*PbK)41`M*Cu#RZTo%W2_6AeX^c|+k?St ztX~F6HUh)JU@l)sfR!rGMV0Ci+$(bg_mYr{I@RDKoXRbe*ZHZPT*Ol}X8FlJx<-Ni z+qO-1`{y|B(^+>5bo*Ri;!{sWtM*xnr&E-AI(3kTC!UIMqqFX>PT%yE;_@8z9X=Hm zP%T{B-zrnJ9X2A(UnPUb!&*F~h1^rkVquGv@0JIoXwT{hYJ0vx*N~TD8U{kX9NE_8Tj>8Qlk?|!m zr>`8ciM&^*s8ZZcptSVzVpP}c7WbG4Kn}YX-b{x_b zNyGO_5K$zQ2>6+M*s zZ~a$mX5))zbjep2%EY;IJ-bzl)-jwa`BE3ucvjyRm~I)aeAUR}pcS%?vUS+%q7`%< zABrApF+kUt3L7V_!~sKaffnYF{CrUw(I3&`eT7R3JR+L8GoNhneOOGGiLt{mhw|?% z4oo6J@ZSGPst#4Qm(ypZz-0}@3sMvcZB>&;o@rPiKvE@?f|Qeq@5!|?Z3f*;CR`!3 z@pC(E3jjJ{sF4_%I-Wg)T!%-clOKUc)MzL+=q#}nKN^BRFGFzID%xgPU54OCBkzM4 zd3PR-yqcs>KcbCy@E>I3-7#bR8pn8^%|UE34@k2Dx=v7#C+p(kfiDZcWhBk!Bd~1% z{3I;7_MBhEmbAl}O+YZ+t8fXs>{$oQ>K*KvnDwZh9j#5@X;BR@B}x>bNZ9twtj+tY zn=J$r6p()SID!Iryex>IYDRRS*UFMuYPivusCI%6Wlc0(%BTDvXh_^%!H`&0E3uWL3p_=XV?-URHqDP|ipb;H7>$qpi3wr8E<7imjk=_{->sB`wIFKx5{O7O!v#uzO=uFzGyfss@&RGggK_9to$<~QgWd!| z##!qDo73bI4Z@gVPrp$)adTR`2+Dz&_ynD6k1LTMYa9f5O4R9hEJ`Vl{Bi)}iSY@P zX-rp&Vl)|zsmMP#Pa;fb(!8SjU96 zE-rE4`jE^ID9=78C>3jSi)ezjLoV#+;&=Vv7x}F&hO>19XNCF?+Mxy>t_>AkTN7C! zb^5fmHI>Z8bWy$n`53etK~s-QtSf}4Gd2QG2l-nK5yb%-*{@YV&%w1m{-veQ9O8wr zS|x;)KZzvoB4fY^91(UHR}09!LEE6PhvUFAtp{-7?QXH)vMO&qO@heaVdltKC zlvkTfg`p54ERUqB(p?8qm8b)gWCU(*@{C_ylZ=%w)Lr0%+uL%bRGLUd#j?HBF3U(l zA~IbFTV!zpXan{`vf*%0Ib{OyLNbRq_POZt zi`4v`#7Oe?+FhNDb&DQNM1skHaLrFF)_nbuAIIX`UjC^ZdZY^i2?bbC>eI2pYd|m1 z6(AEixzuW>VlJ-77wF1P0DP+3xIk(v)SC=mqY*A^E@kdsbFF!TU?rI(NtD>h2(Mhml~NfaZLT~aC_*i zpu@cKWJTpXMa4QQNsP)!WalHwTg=}Uk`(Fno-a`SmxMCQDZ#j`iE;BeB4wB)6AeTSrW_<0R7wG~Qn>R? zlsHR8@g=x~DR_T!uzxr{JNTdDHz%hDCuifmlf&cF_rByEoG>Fj-YTtCCLM<0)!SWf zED^rMP@6*Ms*#L}OALhLmqg0TF_t5uz<*FXU%|xEtyR$>APGjH2&HnSFRPp0#Y?tS z9y2OHpD85HWD|ar!D{oWj7Ed^!Bc-r!IoDM`-T*;ER8v#`5+2ArP7d8Fe2YUT%ZLN zlupUjA@!y8G%CXG@iqK*4%VQhBWpfafG*YVzj1 zCGtR4ZphFc(tUIZBMwXfl%Aim5%5CU&+^80!%*mtW!y#V8zplDW{}PRjzM@X45e#A z2@u|gFp4(2m`k|Tl>h*P?m_CQF~S212bjwwpnrtIq$xa%SxhyfPLg}w&o zm$S1uV(={A!ki@}Jc~()&N!RvdO%yUNIC+4H7`>%Mb{(nuTB4}-nx0qU^+!iG%*sU z5R(RTP5G@uxi1)j^`UtW1D#?-eAoSdE%>+v738_KCc+jt&zPnRj;Dy`XM=MaWQ9N) zj|+s7az{5foz6Bxh~^}wh<(U2#eGOf_`yhq54_tSERWuv|Mf;r%kmiTdS_?9cBIbc z+NzBAc8|tN;|tH#@w_-hfEP%vfP9unzcUd%8{&yD4l#9lZ)`1$h$Jl$vJ)-@DWtQA zki_j<#Mkj35G&*oHT!^)%a1CXXDS8a&k^{_u`5^!piY*mH#T1tHa2&1dEy$ar9sy? zOswXiRDoG`aoG8|&QvviM2%xu_U^g=Al5>BkWnl{e~@8Z2K3J3cwgVCR(?+vbgN!5 z+sY>EOsOxE7NcW1(PRCX>%kjBj?qkNVDD)O=}w zpEuFQt-9<@E$~0K9I89cNEXopm0pAYEj?N*K__dH#tm-47ib}o5Dvhb(g#A3QKQil zcg=|uHI#@rG6Z&i9)s}?ASoD+Uz$5KEXN~`+1iR8yUJ1bPTtJzoR}MjsI%5;kXCbL zE>QA)uT%Cx)+5~Hf~@W-Z1O)gp2Pf)PA5Sr#gw6#NiT@9xi|l=JSsX>4?%GjDvjgf zs&X!rbR`KDjAcwA5Z5ZmDu)w2EL%F{(7JTtzwK9lGc|M^hSzmf>UOu3;4=Xe;Ub1} z9LNjnDJHSp$4O{etQlj|N{87KmSWjZoJG+Vh)Ja}VHP$XOI%H5TOgPblE+q>3zJpL zDD?r@oF>~`AK(3X%zFUiP?)A;4d;g+54Oad`BGezJZ3Ps=vkf?78)Z?20dJ>4nPsm z|90}_q&VCl}~o2P=G1YTO)BlO1_a_(?9yWjlM%p=Mlbm z!Y9X7Y~2?rQ`qI#9w~1FLYPdgsqU!36httKgP9<*bg-Mjcq;8Hf z8u*^8YRtNxQ>|^KBLU8>!YIOtd-`fzybVu+I?Zp-Q`R7Jf1WV;@fO6|*kuv*Ie_nR zgu1{hU$3_%OI)CZT|nJY#bK!e0oE*8gW&~qzV<(>}760WDzarI6G3q zYEwq_RFA+&TZr>BFkS0HU7^?!cPH9~FVLbA-=}Nsz?#gs#<; zvV#e5guteDa!|m-8DER+5wq3~-OApsvr~pAMj(KqPa;jB)DlbCEXDDX$U_+li?=N0 z|3Y^@P&)h7SBF??D74_M0T!r3A}OZAY@YaOczz}ao1}P(V;J#fg!rZPfV3ZT8B}M@ z=(Sv0in^h^n%2&oMp)77b3<$VNeQW>#Tu;|Ji+g{OUlNf-2xQ=+3C$KhJ^-#7zG-DhSb;RYkO# zTefj5c4-k`|8N{cnb1!6PUQ9A8yuEeu}0t-3*6K>8uUWggqiiYPqalZY<%o zWsO=2lQ>8SlDt=RrWZFxO0{(YqaV?tM%-$z-%#q(?{rL(?9clZL~oLk?s2>cQ=7?OmUn|mgwQvI)7Qt zH|nU=d1W7Z8pTW=NlW91g%V$AvN)D;Ab!U_W7f4_k}E(a489|A^>}f>KVdSd>40)f z!hLcTo1->#outz7M@0_FS#p*oH~%K*FZhl{Do3_w=_HGy+`F`U3tqec0|R10(h+z% zH@_q6;skx)^(9J$8*-3ham9ar%4ykAtQsLwPBUD1N>7@Egy7~z?5Wme6Fz@+d)o!K zw_;Ok_Y(k`&F3&(tX92Kw#wu|JyP&V(%`s|?MV2U9mbfMAT&8XcC?l*(h=AxZ%K8r zwB2g^2A4FptRZQeTH1;<8dRGQL1v|MOJ8cPd}#r_`MQ@+dD%#BUc_HvbYz86pZEiR z&>udtoEKSz&VE_uf?QbGxt&G6%@W_A~YAVbYb1QX{-_sgc^=;!1Am{HfAl7;6J7@ z`t)gxKq=p+&h}|>asZk2tCINBO10c7CsGYl)aJ*3OoH?WWx8*{8p$+O9+dL8H?e+9 z@U|C5j2boR|zTL0+H>A~1%*X*I%7z_r3XHTApe+Pp>`QPoQ&z^oU zd^#LHeX_mtd}sK@VEE+8VE6?XJQM;=PevI`zZl%Rt#ar7B#+vJsr*V}zM@csIo;gI zAPbR378Kl*e6B$BzyRFdiom^_8^?EWdwX*OP|P>|KtU^((x;`jw|NrV3aGqL4vlLV zc^;S6xaV0SYtb(hY3hL~xTF(~au!Td&ceb~TyfGbP9^!y56ALgU)^ao>bRqki#8*_ zd=f70ez&(iy$pN>$``tJr*m3}C=`v^aqy!~=jO(<`!1XbZ*Mz8ADj}QvSD)9upw|8)E5&g1(3DW02~ z{#W1<&qv}kX@Vof772PW=VS?HXav6M-`>gYDg`Z`I{IU0Ng5S1SjeECg5* zH5Kx~2Y}-c#SA>rV6}t+B3dm~o`&+6wNd{NB}#}n7G$~QX`z9bf&ogICtRcY`X}vM z6>SGPyWRy_EUTvcxN5L3d&=3s!Nw=!E6uu{bX~q5GC{GAmFJ>4DJfQR>^4dlk$1s*eFLkC z!u-63508+DEs+rGP8-SS8p5{H*xYE*7I67?mP$XJ7r?YaxqOjIm4wdgT2@>rB7E-C zS+@-QY>L>K9D~gsxKZC2_&Pr*#1rrf=&p|sj!qAc4qlxdzd1SW0uTKL2H@Af>I5EP_;oAK$K&}`9gSBa1y=V9iJYYoc(a}=I!xq7kqeE8J!t|A@E`_ z1RuaOMG5c@z{bDc(XW1i-tW8K|2Ocy^}S!een-Dr-zZc@+K2UVtpC(wy}8LpdvkO1 zmx1@~uOEIHdf)yk{~U^cZnkefZ2vOwo~XCmPk$MBPk-&b^LsZtx2x9baYpKWu-n}A zzK7oA*PG$iliN}6=J{>q?T1mX_rY2C+3n`2QhIY!wWct57k(JN3*Utj%fG)1dz

a2+o{FuNeyNupZb$;@jWTNZ##7cfivMRYs><`7_VP!*@2@g}Z2qVgc+m<%z3RH{#T=Ua zeA}&Es?SpdFY0Jwa0cs}oAQsZ0q*6kg(S|GI^l6*wP_O(haBn2qHIyg>m8~RIG!p8 z0_8ZeS+ivIe7{?Q0hEhDem!0rZ-U;IwLFTrIOy|EalrkVJ_!VuW3}_#RGLHjp1y8X@d2+~G>K)@MZl^B{ zinQ{&Mo)(-+NE&_Hs4wL(c%8t`px0d{=xqg$j+;R*Z(}T%Dw2mTX$8fwp(ay{iX<} z#2-mc7Nsinw$#m!{LBA)`{v~BYa26)Lf~Q09n${4%u>8@a5AC5qLdE)E9WR>-C<)DC1D9Pfi)3*3qlO$Q1u6K5v> zNJ*BNO|xeDyH3w58BR(gc!~0aCo4@?MI;V^Olqd1piNi5F7a{+h*u`F3swQtat2tT zn=5wlp}M!MYt;!~1XefO%div!1|(T1UqA={?U6*NNZQ0+$A+4`jDU?oQyY1iAZZHS zsim45krvCl#HBb3m!MO;IkraX8eN2C$(asBbb}CZ)0=apYqj(hS&4tLo$+2b=-Hct46X;dX^@MEz>x*1!gQQlm&gR zxv<7%E|QUFDJt=~junP#c5muw8WTWbRI^6Y{=fXMR7wA)bKZ=FAvg`q$BWF-{oc^ zpeq|*6f-drwC5n10CA8z=Q{+(hbuGuL zEal1rv`o;|mUk(LiM$V|$uA_=9-5T(^1*;^y?6jnwEMR>!cTIkX{l%bkth4+avM$?XZr$2$vJVT91SmdpaWo}#q|J69C zWeeaQj5nCXVY&S;2e8W+nkQ17CGrw%5j0LNd0n#~(65BO*2Rvk?HV@j;+Ab~<;~Rr zw`pUU0nfS4KRRN4WJfH+D~_{gb-er=*f@Ig!<#qz<5#;s9c+N58kS`r*U;tlbUG_z z0bY%2!5P237(Ed(2s?+Rja<%3;%A(pB(XL8`4BK~8^_r{s^ zF{V{%;2KW|drtZ_9}R+g91j-jm#o|NUd>_GGfY+8OF8#kz_Cel7Hk+H-9NRJ-V+Qf z)6mQ5yd}2AIG*}WYeRM0+~fgQM^zZi#1?)Pb!m0lwx#P*nz}ZyDQR`c0Ygt*v9Zj~ zY}t+Zre{M%6?sXDiMZxtuu-m5Ez)clw=FgWq=k8wZ<9uzIU3G4Ofub|+kUowBkEQU zSXVP4>Me$brvS79;Uofqo`n~x2u+jF+W%he9BakkL?Ji|$mV6v0OR=Y|swVy_A zdo~usrUkHQ+HuI|_K#Ny+w z5m;OM%2P!PtMrRyVYW7fmC;2l&4pibO&4gkbqA8zNRNR+{|p|=VMG_rCEg2pxd|7f zb{f6BP`PW>%nL>PL$z!vgtu^_W;b6rdn*=o0j~=YR{Z{=jVqR@VCL+L%FRML&r7$8 zj^mh}H>r2PBAIcE$kq}CSAtrT$UuQCEhsIJs|&5H%Iy}woY9m^g6DCJnLX!=D%hC8 zlo>5cOnRMD-3SXYsFo=->r@Nt0L;AX{366D-)vgk*s(BB`5*D`T<8{c66^F_-z9w| zQgdTk(AJ6zrFvJpda5t^;ZN~6zr0!&PJ40{RiE~-8Or6bKHjib8Ec)&^;Hxt-3(X; z&`xEQb8J2J7IL}3UISMoy`bj0HF+YADTriFN*L=+rQ6tQE0O}T9xuxcWCkx$VP)7@ zzp;MaZpaO~TmUxovZ>TcJTbbFUB|W*SjY4wx|KBIj?LZLmL*vV zR45XmDD+Mc3>QM^)yYnl;^`Enp5Bmpc%n=twkTCoj;n;|w}xDgz3TUJKE zSBD2jr)O`D`I7abdp8U6=V?Gjaaf}hP7yoPb2zd$w3wb5xB2317v{}8!l@x)gM69~ z21}4iDRuzxX29GQR&BL*^ouZ(+Jsx>=B=a`^>wWbn(H?e)~Uec#J3>gctVV94ZsE` z{79J#!G{m}a$^4U)Xt(hQbx#dDPzV;+6RZ=*P@h>^HAI-8Iw>f=s}jIC}z<@E$Ul* z8By~OKw}azBA0h$y`-^&(F@02Fiy3;qxpv>B&KLZo^qz*u$gO_h5P}TtRzAjc%{5^ zT9MUU`C9M5ZJIRsk-*JObtFv{y&2Z5rAJI2jK2A?3x55ztOF=9Z;HB9Fl%$IQgy%B z(WdmP8JAOTPX6*su&ycqZ_2vjjnHQ8gO7}j?}GVa@iYm#Gz7x zPqN6YZFS36HAQ!E&1z8bPu{#b(0ga8X5{70Y}8#Y9_^|ur(uxJHg77yc+`bj!wTwkq`y?@;x6YPIS4Vd=h?JPLZ0v@sZZVwnp3jY7_DH z4!l?dpoU?&|Ih;={E;|S{VB!ul0&N*Oj2^a@PoJt%HGt0k260?X*<1C7D`^9=n2otG{Z5zq@ zB#WYY+oI#hTf)+3ZlV?gn~aQE%2(Xna4Spn3ia-3(6sV1V_r9l8ICM$%n9J07~rve z-=w~yn9a~a$>}+cVJ;#>)pHnJ@F9>~fG0|98p(NfbHm#%j!-^h)F`kZ@dO0TnEJP# zN|uoqPvp=nvAyOXD95kMwPR2=#J)@4#g+0>5&|m6=j%5=T8$UDt<=Qv;n5Fm03N?N z`oRrBBloT@mD{@-+h~hsPY#Y>?d~0%X~nWME+tI<-`m4e3usF|$_J>mzXQ+$??fXc zx@czEt>tZ{OZfbc+Bc*JKv>x9R-ckwP2vy* zIMj}-dSWsHGPS{6v*zt2B{AdQC(Nbk)GVxgFPlr?d&>G+dgyzz^q~T!5eQQE%BD8r zuSpiOV<-ebO|c8KP@{GC^(9Y?fB(K)x9JtgP`(bZOs2|pfD$2hG1nD&_vn3zBQuK5 zG@qScRQafei&6x`H*vHmv(lOfE3RSB{M)^#UWJxiE^%WzSs3 zh>bTzh$A6ZUN*z=MZO3-mR=X-YiUmz4lWko!3f3yUlwvs7OxPHcRs6^Ge~D1TO=aZ zn+mm$)iJ|^tnBVX`kJ-;vs}Y*JR$FZb@4_EaMl5PtL3;|KU*!>8AV=Yb;A>I1891A zCPEK3&4(Vi1vel*r|IS9QiZPvrU-0<&q*_A6s31stHFd%;KtJ5^AfkWAjK4!W8eo- zF+Q{IEYVd~SO^-#=%Zl!QwX+#WV1)_lL@v*dGmi<-YgeCbHq{{RCY8*_oNJ(INq{l z=>b(xjmEt-P>!CLqTez^nOWFzfIyOxIbt)EQ4d}7W>6~E_=NA68c?D^3KOLERc0v{ zbO92V!Yg5Bv#BOur)ZKQIs?N2Essvk$e?SM!o9+cVQXMF3^A8GFfvy`3pvzrbifAS zubSXmM%P$LwdsOtlb-1fV!g5gJ09XWfEWqYE$Fc`}-|zA^Y@bHs zndlmra`5ZYX1fg$ka>)bi@?C;QoMq*!E)3z5;7r)6V?!Ci*IRi7R!?>qD@H;F&&#OG z^@yQDW7E?ENm_$s*nT8S40L|aDuryK0x7W_!6Dc{UhGI%Sl4`%~RUyb~Xv`#aEA54^rQ)<( zH~OkBzxuGjhq9$ewbjJwQMp0HU2J7`YwH|wss&A6;Uw0iJd%~pl;A!r9$^`@Q0e*|uwqo@%SH9IPCfk48=;*DCt2=nA;DNO zlpW(hGL*n5GfD1D+~XE8nM6;-EKp*K5}@Y-f6Xx^?~W4J89(|G#Snc;nCPpMdPpkH zKk|U|O^!}v3mI(U^?Wt=Z|r3cq(BAdCno+m{*OIfbU%1F=V%%*{OH7};q@ z>gBKAbldjS6qJ} zFZjf`{0^nz2e)k6QOhW~aTwvFV4k7Yn-35Jnma&Ufz4x>dMlFMo%N0?%V=^foVg2Y z*hv7xpnPU(T;>2iN(jDl8^J{lR`D_IG8Qwu+5Gbt(h&Al?eZBQERho+2DTGH8e%NI z7%DozG{;M5p&haD>npg>5FJH239B&ozflb;)8-qU1PNYyKf4IyzVgQLbF8bkqIbAr z%jK}6M6x2045>nRA3OizBLb<{xmx?i(~arnhc8mcKN9YLL&pu^GGPisygxOQ#bXXB z4>rzv#t~-_{&amVTSJ`m!c-G-{LYOB8v{-j(Nh91Xu#0{ANsmTFR!Qd^W_>9r8jZ z5@b#@wOt0@_YkH>lvcci`?m8h2krld_>zjHc=POyOP>|mTS>BYQyrit6<9xKg<^iQ~&7;W^zHxlRhcWPsa}bu~@)n<cem=;rxE9>W0zSff7k8;yS;OBqg$@`rhpf$_h`Lvi<`>UJrMP)Kyyv3s` zr5W?*?VcdUyuMsQ%wvbE&Kv3e^}?_X<1`G~K26R0L*84UDuu~rw(zWvgI-~^8Nq@% z7*ePhOwa@57pFk)cn@Z4Z~sx{6nnx-e8zZ*_TG}-EH|gk_h{;3Pf2Dzr=?)1Nfg|1 zQ@RD3t?T2-Sixu#0$PhTb_lBq_u{?Ey#|2JkyIaQ9C@O9gBwF1OUqD*X=Gna4gU*H z=rT=R-;F|D;)mh7M7vw+KqdXWcNwg~#;X&2tMUXrw(*QZ6mHd+9FD?9W&}IWYXl=W z3y1}2Gm~SV?k6R4;j?LqsC2Ca5$JgyfsGKT(Csx{)Rv{9Sj_#<6YXA?q05Wvi}F7c zsbJqXuIMn5WatlKxvQiL6{3F;hOy7>zGbxUNsrTj7(tPW;HJaHrbpw;-G?+%`kg_ z9%Gu;D09=Amqn^GVgqeQAc$+syk9v$(uZcjL|m z$@R?WmNIoyM(_(3rS3N_!Z|}C!0nZa$Z4E`BE9?jb${E znN4}bVE_*ri;kH%#-3J=J~R!gZqRsi`~R`Xb3lF3iQvhZ*|6=CyqLh(GY*d>EYc)} z{TojDzkrPd?!@(oel55h01j1X02YiM)aq%S-F2Wc$AH;pir}KJJ$&BZFHcH8{!tds zm~rY#sH+>x(t-FKsk-cRAWXb-)jC*UdtK>a=79IfE)H`I|tA(-(3eqtB zkA^YpL}UJiz%b_GA(7WvtfwO3M8uh5e+X-wBz0k!r4&BhWvrd@Y$=r=GOT7e6@vzp zXJmm#LdRnq8MoO!J0&#f8qJOFG5#}s@t{A8y*D)-Vv~=(sF0M=W11l8MQL8lmZ5i0 zHPC}Wv zLQ~4?k_4@TkoNIr4;o*(!{8QF3IP#cOnktdL6{<|7>w*36waU5z!2}Gl-{U=_Mi%e zm1~M1R!0tnf)dlNv3Gk`dK{|Z%b&k$lma9ll(=MUF99{!0svndOAkN(pYKOMfMj0z z9;s2{zOV^Ajuo!6nm;|Om#F!&Gx`jp-`%rEVrb(%dbR>FTRIH>#d8%hEWJB)fm5%0 z@4Qjk7mng$bqa`g3mjkG^N-a868j?WmMWszXJijLEYKY6t7^2UlUG$0&t2N&6?*CQ zyC}+)mtE4s8cbNT5q>^Y%|-iHj?|{!&oao{f}A>w&C=_=#*L#sf2jM@wm(kN(NoFa z>orv4A*kfnbJv2&9{lhAUbtHU`o2xiCW^lRs0tG-)09fi<2H76ulhvRURSd;qi0r} zD#OJC_+5x4aa^hw*6iBLz?q@uQX4-h@;`Fh5pyGS#3wkAg;xEB_hcQ00c-bK|3OK8&iC zGQluf0``*`=mk5xH2zWHf5AYJNdXICA_^&sX^dUlqNR5HtM_ydexKDm#3lF7Juj|cJWKQM$ym;7h}{o zDXG(k^=E$o$O9+V487DjAIE!V{ua%9x9N5hN56*KRDQ{o>Rk>%{4FHm-^fBKexIF%yfPX?INh2};C+b_@F=@& z9Qo%;yFvEHLBLCS7a-vE@%6uNmvUObOoEHtw7SkUn&6J=c>4>m6jCAabubeF0Y<>~ z4pSdidq+l7cMC*I)seG(RV9=hzlYAB>Qd2Vx~X}x=j)LE*WL|n2Ryo?*9*6?U9h$$yz3(_bt>t&J=qP;CO!!4{@vbcG&I^R{ zB_cHI(S4XAuMcdti#_%Z50p;H$3gIRb8$21yzwP4Mocl*Gu*%U?a%hl-n#Gf#z|Fh`Gduio= z`L9|5*(ZSVdmkUJ`rp1ei1;%={&AzqPlh1vq-50O{m*Gzqg&#^jc6M7tz+Q&H^vUu z^XJbR!4F%SKk8EQ>Bty^nHS*7|IHxYwKFaiQyLY3KW5ZgM&qe^D19-V4y{tJfnZz? zH7F0iPoD{qHZoJS@pKOLx`2#KiQmNl7#|%yO1BWe|8gO<2X;w49nT7ZXm8(n05$+6 zzg|~7Z2WwDv-NH8aCY==>u&d8#OE+1zl0n6Z6hOd(#!P41wSF}FE< z2F7G`9rtp;325|$Xn+rH|K7)Fz3Z_+jYOLoz_-}0XSPR!-p+1wNs>7ce{VW$~ zDKeYez2r~zlWLt1qDAj$z?lq$O5xu<>_iT}55c0IX-&xdH-SQNk3PgS* z$tzy|zfvDwK0NRJ74i^ht^kj@|EaVsrS-%uFIA9vz-9!Ds7&|wE+7APcHusVEq%90 z$*e|#!daf%r@UUUww_1Yj6mIKAt!yyy`g&?nO-A{WLEd>&vN;i9gg$Tkr4>|V@#5< zz?%8hN(&$Df#!>c>xX1p<+ibfcGZkx$Z14rv`NOEa*iOo&5C>J3m#&>izsvROzA(R z#+BNjZHNd`(OS=RUZ4N@Wc&tuZ{sM1l=37<9CR_-V^*sZuowg(fmXN zA44~L(hTeTl(P|iy-baF$;U9GZ4fTNKiP$dNj*IwuE3D!p#?`L`g(UUAw!LrS0`kS zbu$id9{L%WC;mM*Y4(2q1<2#W`m6d4$P@VR_f0w@;c``-%#EN45|tV-DE9Mzye&3x z?*Slq_aXdonDhJg{&!%30q`+aZ}9r_6TiFnsL)H%*W-h+3)`Nz`|{KK{;CAn>At(# z-CIiqMAUu`kQSy@m1lVRz7mSKfH-+vsb;&23zD$`Qqqc2TPDfUiY`O(#enkpTNX$u z1hmDf*E&yREO@*Iy%J9j*S7He?-X9^(=e2MiYqH$N>dL^Nv~O{}5fSecE>5;4J_xKb@fMFqiR&3tl>a<}GN z>gbnuZe$3zXngC>qFz!aQOvXmaHIIYOpaD6XH{LkDwET83(8f|bdNX1tUc{p38+S`%MY+dOB|F2>IBNEx&4d|S8nTj|z6!QMHN_n=Rwq)3;`g8@p3jN@RYMDEm!osW4B~;Fm!*P& zoOp34(SjLxhzG|aj0L%W49><&l~wRGyWAuE&%Z#zhzl!idGDk#?|~;$Y0dub|Gapoxegn)rFlCYSumCy`nm&Rg+1E4 zwI>@ZUaf5LyP+KEwUKp8=2~}o7PE3TrWerPN=$qkzw7fD4CW0woOUA(R~paY=>?7N zSj11iWEO_bQ*;YDj#)Kb0OaB@kE(Ki}X4{yjfzo-8#qomDf=vhD@451@mAiQvRjFeA}BCgS5YlOi2CASE@ zrzXGi+wz>2qpM^I1nN`QjL4})(S5A93HFDI6Kf>Bc;!~^c~dW0+r0UHVBz9mVW7Hs z=$qwgQ)Q_5Pv*8Y4erJLZV$^`|G_pOKZZQscCB>KJf;4v}cJn$RYD_3Pe*^ z`NTzO)A0D!z5WnfdrXSJL{!@+tPMA-d(bC>iF)+3hF3GxsUj|m$mi&gxYT&&K{+WG zJR2EyZ+Jm?$5bb*%XWe~*3qOtCB!D!5a}5tg;}Rm6<8saA&T(Q@zE%#;X1F18Gnb0 zq0yBhBmDRMUgBMeq5t4X^A&j8=Hd#yE4g8ozcH- zI?n=T6k&tFJkv{yPQ)dn%KgDnbggXtv}%3TKXzJE!)|3Zj!PrxCe|jmt7RUIJWhdE z+qoya!k4kY6RIV9&{QwY8ymIJq4tq+SO2Ockz@&nfv!_PMN#E?356(4|G0!mazpNh zFj(YtDlwAui|p%pe2bfHbo^ce_dm*?LLthSk@}Sf3(^0qCX=5EmdqI`6A)faICF&5 zFTsh}7EiDs+|hO9`9Vzn%=T7xJC>n4@(1}FU*cR$Ks5dPUjSK_w*0-Br{p+;#|Qva z`I=>~j8A%7fz+w=$gpQxEo{|HSwrGy>I{&jR6?5&3ajt`Ztm;d^7#Se3$%nv*8d4- zThYLf;vHOc_x>P*2-)8aq7dlu&Lg24tBgDVbS;j=SeJXd`@Gk4+Q}Hf@c%-IIT2}X z>bItWA`qKEqOD&III$s?Z| z{L+5BCaoo|QI&+!irbV*@~SkL$`FEB<66L-J-6LcUW;^XL{*Lva3sa|c6BS%*__s# zmT?5y>g7X?Ps=2Kg<4x-U2ciL3Jj4%EBvf8 zh;J&DhHsmMk{2R%I6pt8>D2yZh0y|>5qhil-m`DVMjDSTfO z{0eUI*RTaZV>ew)kFn}6vppd4z7mqha9h5fb%nE-E(d-kKkke%TXx`h{)+PfY7WC$ z{qDRjjKQ%F@JE)nX>rtSJ{O9?y{U$OkYD}_DO8+0-~|wpbGSN9HR0KFTo{1n+?o5j z&%~y_46HS+Y)DyiAB_rwer?gv)RvC@hkiDXs~~!z_B8dcc zP_`&)lg1;lR>n`zSh7wPU_`eA@1X)NZ4J%htHkrEEh@Cwc9t#d>gGH6#B&43Dp)w) z#wwa6mZr`y-CnS+xa}OuX!27ooXxKLGl>;^X{aDiipsCx-~t}NXY;w~o#e-GVn-2D7Vm2+O?O!Qc}D(>V5`|_dE z3QK(Vy4_225Cg}UhiL0DAsS(pMJysmFaJvrENMjYEO?pHsb|lwe;`&Qf4aZ%{Cdnz z{-(ZJDJ`&K@+<)V2uwRK1K*#Wat;3LPpBcvvqmm&%|7Y84gYiYHZyTCFtOozdwln{ zJmUQuO6>bc0_zV7HJeHeMDCKtQS*PnnZw@WTLc3wKt5$|e6yjGULlhPlKBSbbBrZd15zSZfv&P5DPt zG=W{!8cui)hkTHBsLe{JB@J(rC@_OrA0h#-Qk$0bp4kuGQs-rBT^EX=2B$=~V%&)l zZr;F8S&l2K6cf^=2pAPwvF*-d@uVz?Gdzie5@_3lR1+v=iwVk#NHL>Y8mn`>epne`l4ZA>wysa5Y8f6;m2z8JB)R{gC%HF@kq)AQi9$C^K!?4qq z97xrg<~XQwAAPOAY0rJ)t&i3e@ymg@NyP;Xj*$#9X_%X(4aeZsS##0>+u90PUyBO| zx%&7M+Fl1}a0>_{G{`y0VyJZb$%wK2`|nHT@<`-%>#+|q7*>e-WaCiMRAsOuva59V zDy5bi(S&*VmoUc1d^)8tMqUiZp*Uk?0Yy!3`n@~_O)4I`(yE*#|A2GtvG49ucWwXHE>*d8-wqm5YH17>Go`DL zrxR&0qd28EwXcy(2~w-_+gQSy$`$-xTxo=(EASdb|6I=m*Y~HXZsh;YQ&)_1+MdjZcIhM)I8^$PdfDGca{#r8D7&T4xcw)=l< zuHrasDo<~GTURIBiy%z%5->+>!5-SSu-XXro0hSS&s1#N@9kgQGui^YHMV6q>ebo% zSIVBR)sP|RbbZ{G9DiF>-lOUn&pI+N8RTQs-{CeRVXp41l-^V3i3m(TC+5v?)F#dH zc-6hD*Yd~_?RT`F)9YDhO7)pj*N&F>|4c<$d(fR^x45RCm^HMV3fDSUcKNhIqmPqe z*rXcqZ+~mc)uP^PRP`*3chTCv^Eq`z8Zx`>fs{k%Y_t!wcIwH2+Pm~=%Wa^iV{hmT zusMb(MhH8hK#o(M{iV%IMpR&g7V?G%jVDec`!UAHa-`3Z(I5n)3SuAH1;~dbc{pLxcm`7zDRCX60?xwr%JpR;UG7@w-W((=|x$?9{mm&cRrYod-C zNKbYUT&sB~4dUA4c@#1V8`fw5a}2 zpfLsQ)+VrV+_5LOSBkc|w&g|N)<(E%+*dyu-1Zb=ryqvisbyGi(aS}VNnIOW4zIUE z(joBcpkFzIK2?=^_sTa@>>|rt?|EcIUfnUiVuzx>K2QH5(?`+4zB}-tG@B;`-mZn! zB}ti?9AKkJqrKe}I5mGSt{>mKN!cwlU*XMT^6bZrx&ooZpTqfbn!{NR(j;)!B+Qh` z^v4kI?!Y9$OTV{y`xRr^339Yk&xiTFtsy%|G7Fi9JMY>Lc(vD}@?D^RVGdFbIIPyF z*H`P+%6FZn>&1X%FsLV-aVs~p>8L$9b_?H%7PkTxe0EuC>$z0hI(4-3ckCE=&{@-d zl(u+W?Y(*ZSXoVz?YY9n5id}UC5N`Vw&HJfXpSl|K$+=swr=M#)1My0_Ih8P7{Cbf zIIt8+AMemX&(-BTfOSc71&d|~zH1VE(>1;M3}Yi&O1&bYc!ZL}3E9HFpBxPW814zT zbMF1kVX0hPGjdod;R39X=EyTKE{vjNzaDURMW_ZOb>-O%ci$Zj61cFNx zreR;!>*t3c4v^595zq7YpYn!y{t71%T zi>r9dTd|*r{(8I`{vo^z|1HIA09N}n3FR@fTTJ(dALn^hTC4X1gBIZ%ju#<)6z0>d zQ^&rzbGC5L`u?0rv1;HE)Ij1ywke|&^l9ndaY=(AF3kRZJ#-S|0)j?$ZhnK9?(lL5 zpDICNpvG7f69{*MleuvTlT0yrpBe~w?usAIoqi^Wjg<;b`@^Nas9oI}y017|>%eH}wSA7Xu{z2n@IWe9b|_ z2>m{9P07`B{;Y`|MEv9(7Y&fv>w_5Y- zL^QtP*0i7OP$vvDL%47Mm#+=K%%4|%UjGsRf>Uidpj1=<*zTQ0<+NTTRYui@WiE5a zMCW4~#Kn06bc3-T9cA|^#byWclIZ;`{kGU^_klQ-c9Swu{d(x!Bf!DM98S$xbJmw zDxW6s-)fIxf*zN7iMi!j#1*T%{5o!uMs()`t__EqW67HyooZTzw<* zCUc2i7un7nKPP-ln@8uHd1aWd`Kc;UHDi7doxVs_mQ@=ja5>3WA(hre2S33r!q;0F_} zHTK1rhh^snMWsJc&p{G4#Sv?Bp(9AP<@1G2t3FM+x5o;_jkeSPlj0-)(6>IndVj@8 z0FUPrfPdW0=4v)AE2vf2wu;dWw6qfg77-Ltkiv|zVz2UtPY4v$xjOc z&j=n1F3x;H7KsCbG5g07V%6AHwQlwzpi@flneDLe6H@I1fC+eHOlLUwq{v=7CX%U8 zGzB<%db5(;lX*uKk>r?g=Ey}xz=xNIy+oeVp;y?gBFvO2O>-I%M{WG^S#c=FYboa6 zqXukDDnxlm^Hx^E5tTDn-0ZZpYC*EZ`*1bBD&Y0E;!IPyl!}ioZ0LXFF`(UbVjgBR^BlUj_~Urk^9BI_z(O8dA^=t9*O4Ni&Ob zWSSc&EycJIj5>QNOYjlBAA_j3)!t?Tm#;Z+tBpXa^3`gp;kGLIY<%=*5z#%Q$jz=6 zotWZ`DfW+;QHH!kTK!wfO7v`-I?=TN)%tp!eK_e$wzMss$mb;5_K~dVu#BTlOHn;< zBeEaUHjwYz0Un$4KDqv-0T`}s1pJ28pRKFZ3qbT41O5d-M3?PJKsMxL0dR?P)g)s# z0cD?VatQ$4?4#yV|NT&d3*$87sjQft)t@+<&cOC*fK2qWUe$bK#$aTT$k-Ol zjmJb{^$|M(zg3xXeETtBQXSc9O%qpFe`k;!(aKq?%u-?${{?24x(w!D6LimD9Xj=) z+n9YEsh3F5*6STtvkztJ=rm<@0A%7v}+xL)NYVy!rsZ zn47%c`}~Z5-orocGW&YPBLy=F1n`{%E0=Rrek`e8m&fhyihr$l9LvN$HClD1rw&SV zk4_Uzt91)eYLSGgg2gN2`!GcRhIBK2E!XFFl2)>4OH}}ObMEunyfOCmj4(%mc#aAH ztxM$4RhJM$Bshufp1_HZD-<|BdGjxkL=XModW|)K1KjfY>vIEE1kQVWTL66H&tPzk zDIpdhQejVDS6Qw{;BqKga!PSWGQ4EO)_g)3EYt~_$J{e$ma&#uO^xpCOSWC%oKlS? z8eqn-e?B>7v=tP^%lgyimvWM!dWa%l)&T9fzo_&O8)17@<%GEoC`+*-wHNN+%Uvco zt@wl_Z22v)KDHLPq$x3KEv3ycepY52#EuNSFn(G&%$Mpu#(q#WwAz7KNAFLFB~Y@E z7dB&u-F7!)N8g%SaV9!HvYoi?u7C1ZQJy%asdRq1dixSebLuk~a>B+K7zAU*B{466T5@Uo?RmsM1 zut0yT^nX8c8daTjqA$Q}Hk0?9EThVfs#x)U0Mq;PNib$nyK2dbZNg+jMIN3~6Xlpx z0A?}4Or)V0A_Uu|=M!8)OQD;XvSC~!4s6^L`jcZWe8}o%&>f%^TDs!h?5dAgSwZG| z$bVzO*U!kab~>h&y-57>#O>2r+ETGd(C#Yktg{oI@6^8EDx`A4P=C zLLPd|m_BY0H>eZOV<}$V?5ZWmHnhBAhmI`6tW^?J!K4w;>k+!KeF$&edR?GNAjhVr zc@k65Zk-3WC*qQGKy~2F2geo29Ty}`y!?f>uO#68e4CZsLxfyBbWkZMo@jqvbg;MV zc)`7D?a7Lw7ORShAx$BfUIomW?fB2HZKKhCdbz(>2rO`-5{I&-$Z0-r4JTwLlDU|C z0$ZIb!70STMr}*AAjR9^S7tat(hFoExoKjKIbH)BB4D0$! z?mza96ge|`620H%0+(XH#U^{dPYEx@dKw&jEL}&HnVA6`Rx;v*j|EB*KC0(AlGE$N za`PDlAtj=TEnKtcoO0<6>yyc~?NBYUaLuOYTCPjhxin_gi>C|C=jI{1`0zp$ZKeh5 z4S4NJT$#-@NrEO^6Dlt=-g}9AZH9%&aa0ysXMCZd}#%P?>9F z8JwZkB~+f`+30hF{Fl~2hUDTi#YN-6i`vW#-{7Yu$p>8TZC$WWlaN6=TEOgn8icf9 z(Nt;b1T^Qd`qEUs1sfr4NWunjUrsBY6tYV0=Qs^##xXmj6)`t@5}Y`CK~R}SiY-r7Ot^m$ye%bH9=a$+kvaO=@%Wj$m>cwsE#SlKn0ZEHk$ z-fDG1YjGmQ`F(hG{brx`VqbPKuDa_Ri%Ak9l)rev!9ZNnW0+Sh7H}WsD31aX9Ykd` zfCi+Y+|E*U>>@t7{WX0PK4!=+O37n*4Bug%-rU3ql-dF8<>7dptd zhX@Vfxulz(LfyJheyV8(^*%}t^x*&?PtYN5BE;O>O+Se&tC1Bd6dMChgvHb(O_ncT z`4LUC4`%c5BJ;=ShEE{?o6NGgfk~0g#2{qO(gGZIT3C{u10RlKy{qTun{6L@tvATC z>IX=2r2~)gpcu;}ooL+MsKWtaGaa_vGp?Cl)145%uxBmJa}&R%-dm{;<9U+?`@joW z)n^tuKVaiftPdJ&A9tZi*NXm9yBVZ&SS9osM8P3^*1Z8e6lK$OtOm@Y{CeDbip_Hb zCfmGjHL1Q}Q)e9WIhHX<=m(lox-eR-bOWr?d;9`T?jKzrfoUVZg5STpVKcT%)$_97 z?mnosi|=bm(Cf4I^JZ2*gRxQQIoHH8YU)Gy9+crD05!N3>{q|GWdnmuv&wXAE38t~ zKW-+!8T_(Aoa`zOn``IaxYO2}=NnWo6JUGDx1-V}7^Im-kCa=WJK-&yB>Mv=jE0Cd zj?23oq&qqv1EqHcOZo#v1_w$YPJJn-OAcO0UPCDbk^)|G5VUXkUO;F=5+4)WGH#I2 zl!#Ujom7^&yTtOHI-dZku7Z0_wIOoBY*G{+9He!eE>>oSUXWkh-^0dwY}bNPa~@Mx z_nLH6l-G9o6WMiQ^PI#Ljy5yjxllY8SVDzgN9gx<4$2D`<%%JQM$n0K>=7rO~X%Ua+N?5ydpF)roJXq4WDYOup`r1T3> z)QZZ?^>PG8l8Ifqc?LyzW=VM7{QWByuvD>r8Kr6~)KO!ePVaciE^)*rM^eBTt|?^$ zwo)MDd|^sd=5C3&J5y2d9#D_nw7guLP#VX{qZKhYR~u)W z#XkyE@Nu4lx8`1TEab=@uzP@+46{AwP8@>oP$_@r{xRX zE|`pDsg-%kL1t~L5tRRoFYi~_+XTZ+V|UNCKuLW&rg#i4iptS6uj1aEmlggln%+=< zB^uLq{gOhgbqS)FplrCL(+nHUo91+M_6&s!r z0HiN6l_jl0<=IefoRj(6%sQHA{40*sgpkAP``>&3O&^_BQIT#UJ8o~YYkt1}YXzTI zVBKcRRQp=!v;4^Db4q_`D7yIHUV;odD@Yos+&QC@@9jD|Na&G+Dw+Y!Cd%sH!{iPa zxv~VjH&cV5cCO1C{ysz-cNqG zuNHF{V|~=9b1^nJa!0EjO~CRCuq;ZIcti%am!p8V<_P8|YA(`&a5=R zt~^0GHmIf?TWi+a6ax}OLoM$WeGZIXeC8-CT^fbE4b&xC&_9^r*x)uy91=s9ly;! zzMwDZBT+>`SbOXCx5QR#0P7iKp&D=tIRYJq=YH&sXWgE}oltO=s!H##%}Hd{ZHclv zBo46Fx;5JOfzAc!_XA_}4~iA&r)@j8#o>5fq8w7|d)ghv<%z32H4e`@w4?w&ck>Pv z$=&C_GgCVV)9Pn_DqYvYh}UY5Sf(={3-DWTY0Zr@<>f4N}R0faWbN3&KNmsD~@4G+d*4)4p*&wfrsYqt>!Xl(r_J1Ih^)fJ_o*A%c=m zp|#hBP-1tP#%QHN>0_idoQ8B!^ZwhmRG)FFXrOQLg7ME_Y>fd z@KcIiy$SrLS{(Z)9V)UpLsZ_a1ca_BWtNJu&k)nc?MRc5wbe()bLM9qLn1~}r=(Gdq>`e3}xq>){)gzVqI%~ABlXbr_*Ej1KS`N<(cizaR) z4x)lFlyD$^&W=Kp#yLghw83%L6{FBSC84Y^ThNRRJ)D7YDpvugca)NrlK$*#8yeP&uxX zZCEz3d$8OOA=e2cFDpy;2I3kU%`pBT_HDY7;r0Lhy7azv#Bt5{jiH$VdPmy*L1W>9 zNQEhnI8jGwp^e);m_r>Q-2I*VyXPt{ivza6;&@2{742;>5+g_d$@JAY3Y)0hRrt^> z2iatV{kE^UeL-iGUhKgC-q7BJj#+)_a4t?{JCh1_2X4Ysx?WN|1k;&4jJ=PAL$U7W zZS=g0JsdlkNPU%zae!6N(V6K;bXqg31#<3nb02Kvf=TNBL04t$Wxt?(Z5FDW)~WsD z--tMQFXn;l z(px)D_iwcMAvNHjmWl7S3CmRMIzAS~-tnl}I3UX2;u02c`XLt#ce6v&vaZtwU!oFt z8;N4T_rY0yJ&ynY_xIPU|4JPJ3WqUWJlVZ>@A8`nI$Jr`uQz>!+%;V7JBLz^pW5~A zsJD+^=w7N)rORey6t=L2yj`Seoc}Hu>qQVFdgDG-LnS zj)h^)U#IJJbdb}>_U>}NPInJ(K*G-<5XX1ZyYEYjLk$XWV2$Ue+EJQ8?OamFrG>Ty zkz{Y}L{YIB(yEOi)-p|+qfJbDw1qo4jO;Jfeb7JVC5>3_o%eZiKbCJQuaLE#RG2~u zY;OlSz(9Yj9nEo5dQx$);34Rp+ac$kZw0R5pFUNZ0k*HJH-CU;Pe0|shSnOcK-Swg zT;MdNdFTb3PyAq*Z2xS5q-5W=hW$Kc8#%PiW$lN>s$+LKsrZgL$%dU2a?hl^(54_Z z9?GG7_SY@2mG3{vGv!-P1&%w?o6AQ1;_LAx(}g8NJ1?Mcwx*uZ zt4DeHC@FwTb(rfxD5FleAq(IZ(n}#TcOlCXu_0+CuKSe&bq(N(c6AU(Jo!02?p}I5 z|NDgqHzyBp_jC&C6?li_edmfU8vg?2kl!U(>*dtmdxjg;f7W+vx4Od!x&u`5kp|nx zo!D+sG3stx?1iF8b4(uT#d$R<@Kb@9RB)6S(wd@}~7&*kxH3f^>#he#y)MmoQ~*;_4-hm;{=`?+Qzh za)E~2CWY!$>ad2CJ`dm=thG9(5ZeFi0G$ehrqJy0mk3#oWLbU4eIsu5fjM%d^!(6N zo}~(ViAfy|gFu3N|87!c(Mhs@S!->!8%)86w*!}t56KX-$!Hd5SlHi*4%*tR*3oCR zWeS?NY;?0NZ-MzH)HK28Hb@Vd@Got8gt=bp?gm)rnYAiHcsb=MDN!aeU?6bVx2S2> zv9`Q5Lxc>uYp&;$Ju%_JxIOUqY8r0v1>2Wq2O+L5|B@C|N99_(G$nKMB7-?;#10+s z+9EKg1~d790PHINH3ow)EgLNLJgPipr8=s9m-Jx~v0$T7rU=e%ZG;z%%(UASi9)6C z5YtRtr?(1!rBq9){{*qa|W6*!zWAf=$6T75B zE}3)F$X<<43c5H=w!a_<{C}6*Wyc@MV%8A6xi$sYl{6|o#eUsXRNr`n`1#;HG4`$y zL@qVJ#2(azU$FhtCL0lSpwfgN+!C&BbVPesO&)`7H1?-lCqNL^Jc-gw+*J@(465+w z-DH-7jf|Z&SSyi>w&LOQ#Af61$R8O7)hW);=i}{wJsf|HAHTi=uLNkp6-%z&2Hv~u zeip=CmfVtS}{0kh0m9q=~SCwkk{`O~7pFW73aaZS|mqU13f5O@gJV%J!?(heO5)>i;vg zW*OyCLg_U`)(C+>ddFItd(DF8T){C{VzAJt%E++>kAXdVtqg`JO*Hc}2jvnrT?|_X z+{YshtILx+oD@7@X)DRfIK`mX^GIDyO0L&aTSnh?_6eQm1uYTY>Oe{t~YPdWCQ4kBCENW)o4uG6UZIYq}J8KF>%OUs$F}$V(2Xe>{o-Wv+jRf zGO@#X-ZD&=0T1wrZ7ws8)#^gGm%+#t75nH#USFUoeW2n3Q@_V;o3n(A9h}Tg_wM`^ ztm?JalO!Ule=6&^pi_>+=wi*Ys^lNaStm(fUco;p#kn6Mu&us0@EO*Tj%I`phPD zKvjJq7;I3L{bwyQv=Y)^;D8bS<)c5r{6~4HTsB7}GEpjA>@VQmlgA*wzV%!&wJ1>$n3Y(b|^T}f()4ZDV+|{b} zxFt%QE|VD|w+NT^oeKr5v1UBX>VE>R5mD|PZZ1pWoMbXWCpezLch;E%2~@RHXRn~d z3Y*UDX%pe54yLY|YlaEX2cPZK>4mO8x6RPzkJ(1a*&N2uPie=;xutMW@?^zSfH&x` zI0fH}yE&Pnkxsfbfk!qyONGn?V0<@D4Jhzk`LKM$xhVHA7=7(P0Z)wWYBBx3w?j>0 zT_|}4>MhfFS?mv-$ZPAoU9P`t4OUv6>-h5pR{2egR$=B*$Mx71;n$o>k~OGx7FjWe zhxO~(o8*mGbn%TJ+IL()3c~yRUFc!kA_mVTuiY$F+UmwvNPptS;vK`u8=PV|kJ;7t z>h0AOPR5KVqzDLIgwpS z|Bpn>ajw_VDw1MBTa%I#Osn+-9K?_*zwYfjlT{`j(WIk%Z z0+7hiq$=B}16a;NY+dtCBMvQ7{=hOt2JCU{!{Od?C&cIm-#Rio`jQKF`VBL$~uv zZLgU84zDj53)n z(Xd!7hF9r^IQ_vL*jOpVHy|%)lR{l0OyTfA{xB_=NoJFbLB@O2AovE0S9wLwm;diW2Kt**?yZ;_L6z?c$rg2xvJuX^QPSPV6` zL^@F%0$pRJRHK9>5So-AThz$*qDV*Y+Olf1IgEj~v51Io=$>}R0l-rc>~|aL2e&+^P~8BHCly}5UVBdA_3LW0fcrF={|fV{Z!1})ms^131`dj( zr^5iCy^T(XYX>X@bfMFCRuymC#@ZKkOc#1uhANji(fK>Z#505t-V``c22ha1ayyKC zwNd9%5A*4;78=9J*t~&Ghnu#}gL@9k^9*Wj0SPz9l#Wi1&;HF+-!l1~b?#C1r#~dC z_=H86v4jjuDxB?teYVkDyf-iJ)H!zQzO2It5Zg!kT+!*Uy@%RVctBg^;_&$OaKlh1oSooHIHkd9vt+V;p#zb25g_#LVnf=SDzGW1Zy8}-d;^^ zjnHxN?%dCcpnZDq;-0Pv0KNlrD$AC3K(Jv;3^XQOLI9^y``UwcSI+Zpd#@rJU6Q5R zEbT$INK^2ja3b}ccExsY4_G+Bu+EGwId>;_Of0PpgOh5l>4-Nk#yfHnRA@4ScGlqa zB(&2BYAXWK)+#-W327X_0?o<|q)%TsN!FgasF={IbX$E|jGK{xO2Ndc_fbyfHAgv$ zLjs=d!KvgA;8eG`BI6P_erEM&_pPl)a8}AfdUB)7o-q_t9xvvQ(~=6YAcl&8lRe3F z8l6ba{nhaAA=H^o{Fx~S<}~2_>S50f zcyASzT-(EgGNQ^9t)C0uyAXa>^(NFh5(-FxN3pc>^DSJ84q_=jvT?+kFeY-*O?9A|F48;@Bw0eCnkVp9wF?Hm6vj>ksq_vCSO z@J(n@NsOtaHn{3XFJ2Z!Rtsh1Y>^T#qPIwT;wcwq<+_vBiUrS!#0i$TVI!6hgldsa z+7dg`T(_c!A%F&YRYK!$^SA68v56#^@d}#LZIDeG3egoM>z+<P5x)nKzi#TF9 zer^42rm%g?=G_8DhnWGdy*EzTxNDOB9kUpUGnj&kIgRGY6NMedA>4i~ZDw8S)8IwY z>>?-6sMApk>5?qJ0YmkNJ@@Cwkt+lNv*Fj@A4O23#nfBbE9$cUE4pKqe*?f4*#EhO zkZvMusBg4dUEGy;`pxsG7dG4uScwV2jK#g`$ z&%Owlo--khb)w>K}QH9HB*)5MHkM0ckqR z$pIpCP9R6%rO?R{Tzyx3g=q>RqiImoYJD)NTD-KVAD^matyVR=&QgXGllYP{Gwebt zAl>4NodQhInoS`EAMLbS-onr+Lpzq0z37e5wdMAKd$lw)qLA9FgNele*4uKP!p8DZ z3{d+;udUbBdnU=C)#^v+oXtsjHVd(X6SbA#ER9|RSA+}7vj6(e{~*yc8nhIa0^bvK zY7=rjk8b|!KmWrgq#R!x`mqp%gFiWwKv!`lDJQ?`q3b@6dfp5uut9Mn$vO~bKS2G; zGgJ6zfS$KX&3P8xz@pN>4sPAJXF=lt07S#dX-cb|%E&2;Ar*B6l#wR3Y9uT$eJ77oX%xRiZ zt8yRodcEeATuJw_)nabIInYS;!akkoSZy&UUq0XC~9}(pIfT0w@dQ(Q;zsrT)yT$&BVW zo~f2ehOQZhx?uJ={hs&P{Co`rLs(kV)5uI@c&8iso<< zxxx9S58Fe(Na7hYDOkTnF8N+p&r5ec?|OswNe!ABv#x2g-eTSvQFCUt)dHi6i&7F^ z*;r$V!cs}svJ}Aqa;{7FVBPa<;+M)Z#N9co)S94Dl(QC&$pp@Ds20&|8nsRh<_V^$ z+5{Cer$e9)OnTzO1Ph5s2BON-e$QDF=lDkPz%q=TsCB39T4yGx1BRFOwX78DV z(mRLtJgi<)qdkOEIaB?DP379WSDXc4iD&$^No^BtEWXAZgaEHcpz1`e*037Ms4JGM zk#!a{|2VGRcZyUf(hO;%jWJ|(cx0gv(JT<2i=ul}Y8qwYJ#|tyN(3;5rc%j8@-ffx zm?4ZtLD=Z1u>N{AkAW~)S=ePCQC=f-a zjGcY5NYiE1YAMA4CeDtgCG}zKuVglw7wltRq-a>r*@m)2pni>9B<=`}Qx;!_NxQvh zU^YP*ebDwc`;iutze-^6PSm>4`a0p%%l<~(_Lf?VgV+q271&4T_y zvO>7=G+1HbYw!d2Qw^uOH9#*8UcUBjBV;wxrV_+LeQ`Gj_*wzJp#f)*eW2&Z(0j#1NGkitWpi_ICKv7GYO$Vjy8P`-QR%6wN zj9ep5IITIv7E*M6Q$%1q37@=yXDJpKQAvR*g%c84>k}op# zbd1H!I`5pI=|%;mY)WITtshEcZK}dgX1FSAwvMB5EzpUuP2zu+05jtxX48!Rs+()_ z?~j*mJw~m4a~Pi3UV^+}CeVC<BomN2r5TIMjFL^YR;bgrajv9N(-$=|QR^AtRf?F_Sd@&X$4^O>;lKh5 zhQ!{+5=i1pwQ@F|uOKOS)Oud@S)){Ihi1C=3kbE^jjwfCh1ST}B5arBbUGzmix(y= zf&L1dZ-C(JuucB%TUJ933B)4PB(28&swB7>?856T@{f(cm}p^YA=w;TD#!`Q={9ek z$rRaYJwZo0@A5k~1%mi2>OYHKqYm2LMZ+I2PTpUf9==0w&Q1=$KYjllI(mEf{=1V1 z?e4ZtmA;=WVx6S{>0}c?Z%ry^IH_QQ2aoI1Xe6zUcbeHA4Y3wk__X64a#&(bC&q+i z21^p5zhi-RcZ))hySwOuq21lu6d$|0MZIgS!cqa;E^2l3S&~WvY+N}=zT2bsBBWO! z*JHomk9w$69`1AW)SKdCl5?Wv$Q>=o8WW*f`2LI}#fw(|AD$0&RKa2_oKsm_j3$(% zi6}}=1P4~p45>zVdMFga(GCDx7G3Y(IlIDO7jb!Do07IfVOkVpE4N>* zOZigy`swMUMseS1wK`i}t;2<6MHLAcFD)ZjkEc;oKg>uv@2KG~lmpMjmN9YXATM From 70c3a35141463988201329e89933fcd1d0bc6d82 Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 20 Jan 2026 12:50:47 +0100 Subject: [PATCH 08/39] Simplify Makefile and add auto-cleanup to cluster tests - Reduce Makefile from 17 targets to 7 essential ones - Rename helm-* targets to cluster-* for clarity - Add trap for EXIT/INT/TERM in test-helm-with-load.sh - cluster-test now auto-cleans on exit or Ctrl+C --- Makefile | 53 +++++--------------------------------- e2e/test-helm-with-load.sh | 39 ++++++++++------------------ 2 files changed, 20 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index 73b1116..21e23c4 100644 --- a/Makefile +++ b/Makefile @@ -1,64 +1,23 @@ -.PHONY: test test-cov test-full e2e e2e-quick helm helm-cleanup clean bench bench-quick bench-profile +.PHONY: test e2e cluster-test cluster-up cluster-down clean bench -# Unit tests test: pytest -test-cov: - pytest --cov=s3proxy - -# Full test suite (e2e + helm) -test-full: e2e helm - -# E2E tests e2e: ./e2e/test-e2e-fast.sh -e2e-quick: - QUICK_MODE=true ./e2e/test-e2e-fast.sh - -# Helm tests -helm-test: - ./e2e/test-helm-validate.sh - -helm: - ./e2e/test-helm.sh run - -helm-status: - ./e2e/test-helm.sh status - -helm-logs: - ./e2e/test-helm.sh logs - -helm-load-test: +cluster-test: ./e2e/test-helm-with-load.sh -helm-redis: - ./e2e/test-helm.sh redis - -helm-pods: - ./e2e/test-helm.sh pods - -helm-watch: - ./e2e/test-helm.sh watch - -helm-shell: - ./e2e/test-helm.sh shell +cluster-up: + ./e2e/test-helm.sh run -helm-cleanup: +cluster-down: ./e2e/test-helm.sh cleanup -# Cleanup clean: ./e2e/test-helm.sh cleanup - docker-compose -f e2e/docker-compose.e2e.yml down -v 2>/dev/null || true + docker compose -f e2e/docker-compose.e2e.yml down -v 2>/dev/null || true -# Benchmarks (Docker only, no external deps) bench: ./benchmarks/run.sh - -bench-quick: - ./benchmarks/run.sh --quick - -bench-profile: - ./benchmarks/profile.sh diff --git a/e2e/test-helm-with-load.sh b/e2e/test-helm-with-load.sh index c6e3903..28090ea 100755 --- a/e2e/test-helm-with-load.sh +++ b/e2e/test-helm-with-load.sh @@ -1,17 +1,25 @@ #!/bin/bash +set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="e2e/docker-compose.helm-test.yml" -echo "Starting containerized Helm test with load-test..." -echo "This will start the cluster, run load tests, then cleanup" +cleanup() { + echo "" + echo "Cleaning up..." + docker compose -f $COMPOSE_FILE down -v 2>/dev/null || true + docker rm -f s3proxy-test-control-plane 2>/dev/null || true + docker network rm kind 2>/dev/null || true +} + +trap cleanup EXIT INT TERM + +echo "Starting cluster test (auto-cleanup on exit)..." echo "" -# Start cluster in detached mode docker compose -f $COMPOSE_FILE up --build -d -# Stream logs until cluster is ready -echo "Waiting for cluster to be ready..." +echo "Waiting for cluster..." ( docker compose -f $COMPOSE_FILE logs -f & ) | while read -r line; do echo "$line" if echo "$line" | grep -q "Cluster is ready"; then @@ -20,29 +28,10 @@ echo "Waiting for cluster to be ready..." done echo "" -echo "==========================================" echo "Running load test..." -echo "==========================================" echo "" -# Run load test using the shared script $SCRIPT_DIR/test-helm.sh load-test -TEST_EXIT=$? echo "" -echo "==========================================" -echo "Cleaning up..." -echo "==========================================" -echo "" - -docker compose -f $COMPOSE_FILE down -v 2>&1 | grep -v "Resource is still in use" - -if [ $TEST_EXIT -eq 0 ]; then - echo "" - echo "✓ All tests completed successfully!" - exit 0 -else - echo "" - echo "✗ Tests failed!" - exit 1 -fi +echo "✓ Tests passed!" From f2354026618e4c9ff9c412a6c17e40608d239ddb Mon Sep 17 00:00:00 2001 From: serversidehannes Date: Tue, 20 Jan 2026 14:01:01 +0100 Subject: [PATCH 09/39] Simplify e2e tests and add cluster-test CI workflow - Remove benchmarks folder (tests didn't prove claims) - Rename e2e files from helm-* to cluster-* naming - Add encryption verification to load test (validates AES-GCM format) - Add cluster-test GitHub Action with Docker/Helm caching - Simplify Makefile targets: test, e2e, cluster-test, cluster-up, cluster-load, clean --- .github/workflows/cluster-test.yml | 58 +++ .gitignore | 3 + Makefile | 19 +- benchmarks/bench.py | 355 ------------------ benchmarks/docker-compose.yml | 106 ------ benchmarks/plot.py | 55 --- benchmarks/profile.sh | 276 -------------- benchmarks/results/.gitignore | 5 - benchmarks/results/.gitkeep | 0 benchmarks/results/benchmark.png | Bin 43554 -> 0 bytes benchmarks/run.sh | 198 ---------- e2e/{test-helm.sh => cluster.sh} | 83 +++- ...lm-test.yml => docker-compose.cluster.yml} | 11 +- ...test-helm-with-load.sh => test-cluster.sh} | 4 +- manifests/charts/.gitignore | 2 - manifests/values.yaml | 4 +- 16 files changed, 154 insertions(+), 1025 deletions(-) create mode 100644 .github/workflows/cluster-test.yml delete mode 100644 benchmarks/bench.py delete mode 100644 benchmarks/docker-compose.yml delete mode 100644 benchmarks/plot.py delete mode 100755 benchmarks/profile.sh delete mode 100644 benchmarks/results/.gitignore delete mode 100644 benchmarks/results/.gitkeep delete mode 100644 benchmarks/results/benchmark.png delete mode 100755 benchmarks/run.sh rename e2e/{test-helm.sh => cluster.sh} (66%) rename e2e/{docker-compose.helm-test.yml => docker-compose.cluster.yml} (94%) rename e2e/{test-helm-with-load.sh => test-cluster.sh} (89%) delete mode 100644 manifests/charts/.gitignore diff --git a/.github/workflows/cluster-test.yml b/.github/workflows/cluster-test.yml new file mode 100644 index 0000000..adb7cf6 --- /dev/null +++ b/.github/workflows/cluster-test.yml @@ -0,0 +1,58 @@ +name: Cluster Test + +on: + pull_request: + paths: + - 'manifests/**' + - 'src/**' + - 'Dockerfile' + - 'e2e/**' + workflow_dispatch: + +jobs: + cluster-test: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Pre-build s3proxy image (cached) + uses: docker/build-push-action@v6 + with: + context: . + load: true + tags: s3proxy:latest + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Pre-pull Kind node image + run: docker pull kindest/node:v1.29.2 + + - name: Cache Helm dependencies + uses: actions/cache@v4 + with: + path: manifests/charts + key: ${{ runner.os }}-helm-deps-${{ hashFiles('manifests/Chart.lock') }} + restore-keys: | + ${{ runner.os }}-helm-deps- + + - name: Run cluster test + run: make cluster-test + + # Avoid cache growing unbounded + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.gitignore b/.gitignore index 3a64008..fa29faa 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ htmlcov/ # OS .DS_Store Thumbs.db + +# Helm dependencies (downloaded via helm dependency build) +manifests/charts/*.tgz diff --git a/Makefile b/Makefile index 21e23c4..4b8af6d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test e2e cluster-test cluster-up cluster-down clean bench +.PHONY: test e2e cluster-test cluster-up cluster-load clean test: pytest @@ -6,18 +6,19 @@ test: e2e: ./e2e/test-e2e-fast.sh +# Full cluster test (CI) - creates cluster, runs load test, cleans up cluster-test: - ./e2e/test-helm-with-load.sh + ./e2e/test-cluster.sh +# Start cluster and keep running (local dev) - use cluster-load to test cluster-up: - ./e2e/test-helm.sh run + docker build -t s3proxy:latest . + ./e2e/cluster.sh run -cluster-down: - ./e2e/test-helm.sh cleanup +# Run load test against running cluster +cluster-load: + ./e2e/cluster.sh load-test clean: - ./e2e/test-helm.sh cleanup + ./e2e/cluster.sh cleanup docker compose -f e2e/docker-compose.e2e.yml down -v 2>/dev/null || true - -bench: - ./benchmarks/run.sh diff --git a/benchmarks/bench.py b/benchmarks/bench.py deleted file mode 100644 index 2332b19..0000000 --- a/benchmarks/bench.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 -""" -S3Proxy Benchmark - -Compares direct MinIO access vs S3Proxy (with encryption). -Uses boto3 for S3 operations with async concurrency. - -Usage: - python bench.py # Default: small objects, 10 concurrent - python bench.py --size medium # 1MB objects - python bench.py --size large # 10MB objects - python bench.py --size xlarge # 100MB objects - python bench.py --size huge # 1GiB objects - python bench.py --concurrent 50 # 50 concurrent requests - python bench.py --duration 60 # Run for 60 seconds - python bench.py --runs 3 # Multiple runs for statistics -""" - -import argparse -import asyncio -import os -import time -from dataclasses import dataclass, field -from statistics import mean, stdev - -import aioboto3 - -# Object sizes for realistic single-part uploads -SIZES = { - "tiny": 1024, # 1 KB - "small": 64 * 1024, # 64 KB - "medium": 1024 * 1024, # 1 MB - "large": 10 * 1024 * 1024, # 10 MB -} - -# Endpoints -MINIO_ENDPOINT = os.environ.get("MINIO_ENDPOINT", "http://localhost:9000") -PROXY_ENDPOINT = os.environ.get("PROXY_ENDPOINT", "http://localhost:8080") - -# Credentials -AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "benchmarkadminuser") -AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "benchmarkadminpassword") -AWS_REGION = os.environ.get("AWS_REGION", "us-east-1") - -BUCKET = "bench-test" - - -@dataclass -class BenchResult: - """Results from a benchmark run.""" - name: str - total_requests: int - duration_sec: float - put_latencies_ms: list[float] - get_latencies_ms: list[float] - errors: int - - @property - def rps(self) -> float: - return self.total_requests / self.duration_sec if self.duration_sec > 0 else 0 - - @property - def put_avg_ms(self) -> float: - return mean(self.put_latencies_ms) if self.put_latencies_ms else 0 - - @property - def get_avg_ms(self) -> float: - return mean(self.get_latencies_ms) if self.get_latencies_ms else 0 - - def percentile(self, latencies: list[float], p: int) -> float: - if not latencies: - return 0 - sorted_lat = sorted(latencies) - idx = int(len(sorted_lat) * p / 100) - return sorted_lat[min(idx, len(sorted_lat) - 1)] - - @property - def put_p95_ms(self) -> float: - return self.percentile(self.put_latencies_ms, 95) - - @property - def get_p95_ms(self) -> float: - return self.percentile(self.get_latencies_ms, 95) - - -async def ensure_bucket(session, endpoint: str): - """Create bucket if it doesn't exist.""" - async with session.client( - "s3", - endpoint_url=endpoint, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION, - ) as s3: - try: - await s3.head_bucket(Bucket=BUCKET) - except Exception: - try: - await s3.create_bucket(Bucket=BUCKET) - except Exception: - pass # Bucket might already exist - - -async def run_benchmark( - endpoint: str, - name: str, - data: bytes, - duration_sec: int, - concurrency: int, -) -> BenchResult: - """Run PUT/GET benchmark against an endpoint.""" - - put_latencies: list[float] = [] - get_latencies: list[float] = [] - errors = 0 - counter = 0 - stop_event = asyncio.Event() - - session = aioboto3.Session() - await ensure_bucket(session, endpoint) - - async def worker(worker_id: int): - nonlocal counter, errors - - async with session.client( - "s3", - endpoint_url=endpoint, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION, - ) as s3: - iteration = 0 - while not stop_event.is_set(): - key = f"bench-{worker_id}-{iteration}" - iteration += 1 - - # PUT - try: - start = time.perf_counter() - await s3.put_object(Bucket=BUCKET, Key=key, Body=data) - put_latencies.append((time.perf_counter() - start) * 1000) - except Exception as e: - errors += 1 - continue - - # GET - try: - start = time.perf_counter() - resp = await s3.get_object(Bucket=BUCKET, Key=key) - await resp["Body"].read() - get_latencies.append((time.perf_counter() - start) * 1000) - counter += 1 - except Exception as e: - errors += 1 - - # Progress reporter - async def progress_reporter(): - start = time.perf_counter() - while not stop_event.is_set(): - await asyncio.sleep(5) - if not stop_event.is_set(): - elapsed = int(time.perf_counter() - start) - print(f" [{elapsed}s] {counter:,} requests, {errors} errors", flush=True) - - # Start workers - start_time = time.perf_counter() - workers = [asyncio.create_task(worker(i)) for i in range(concurrency)] - progress_task = asyncio.create_task(progress_reporter()) - - # Run for specified duration - await asyncio.sleep(duration_sec) - stop_event.set() - - # Wait for workers to finish - progress_task.cancel() - await asyncio.gather(*workers, return_exceptions=True) - total_duration = time.perf_counter() - start_time - - return BenchResult( - name=name, - total_requests=counter, - duration_sec=total_duration, - put_latencies_ms=put_latencies, - get_latencies_ms=get_latencies, - errors=errors, - ) - - -def print_results( - baseline_runs: list[BenchResult], - proxy_runs: list[BenchResult], - size_name: str, - size_bytes: int, -): - """Print comparison table with statistics from multiple runs.""" - - def avg(results: list[BenchResult], attr: str) -> float: - return mean(getattr(r, attr) for r in results) if results else 0 - - def std(results: list[BenchResult], attr: str) -> float: - if len(results) < 2: - return 0 - return stdev(getattr(r, attr) for r in results) - - def fmt_stat(results: list[BenchResult], attr: str, precision: int = 1) -> str: - """Format as 'avg ± std' or just 'avg' for single run.""" - a = avg(results, attr) - s = std(results, attr) - if s > 0: - return f"{a:.{precision}f} ± {s:.{precision}f}" - return f"{a:.{precision}f}" - - print() - print("=" * 75) - print(f" BENCHMARK RESULTS: {size_name} objects ({size_bytes:,} bytes)") - if len(baseline_runs) > 1: - print(f" ({len(baseline_runs)} runs, showing mean ± stddev)") - print("=" * 75) - print() - print(f"{'Metric':<25} {'Baseline (MinIO)':>23} {'S3Proxy':>23}") - print("-" * 75) - - # Requests - print(f"{'Requests/sec':<25} {fmt_stat(baseline_runs, 'rps'):>23} {fmt_stat(proxy_runs, 'rps'):>23}") - print(f"{'Total requests':<25} {sum(r.total_requests for r in baseline_runs):>23,} {sum(r.total_requests for r in proxy_runs):>23,}") - print(f"{'Errors':<25} {sum(r.errors for r in baseline_runs):>23} {sum(r.errors for r in proxy_runs):>23}") - print() - - # Latencies - print(f"{'PUT avg (ms)':<25} {fmt_stat(baseline_runs, 'put_avg_ms', 2):>23} {fmt_stat(proxy_runs, 'put_avg_ms', 2):>23}") - print(f"{'PUT p95 (ms)':<25} {fmt_stat(baseline_runs, 'put_p95_ms', 2):>23} {fmt_stat(proxy_runs, 'put_p95_ms', 2):>23}") - print(f"{'GET avg (ms)':<25} {fmt_stat(baseline_runs, 'get_avg_ms', 2):>23} {fmt_stat(proxy_runs, 'get_avg_ms', 2):>23}") - print(f"{'GET p95 (ms)':<25} {fmt_stat(baseline_runs, 'get_p95_ms', 2):>23} {fmt_stat(proxy_runs, 'get_p95_ms', 2):>23}") - print() - - # Calculate overhead - baseline_rps = avg(baseline_runs, 'rps') - proxy_rps = avg(proxy_runs, 'rps') - if baseline_rps > 0: - throughput_overhead = ((baseline_rps - proxy_rps) / baseline_rps) * 100 - print(f"{'Throughput overhead':<25} {throughput_overhead:>23.1f}%") - - baseline_put = avg(baseline_runs, 'put_avg_ms') - proxy_put = avg(proxy_runs, 'put_avg_ms') - if baseline_put > 0: - print(f"{'Added PUT latency':<25} {proxy_put - baseline_put:>22.2f}ms") - - baseline_get = avg(baseline_runs, 'get_avg_ms') - proxy_get = avg(proxy_runs, 'get_avg_ms') - if baseline_get > 0: - print(f"{'Added GET latency':<25} {proxy_get - baseline_get:>22.2f}ms") - - print("=" * 75) - print() - - -async def main(): - parser = argparse.ArgumentParser(description="S3Proxy Benchmark") - parser.add_argument( - "--size", - choices=list(SIZES.keys()), - default="small", - help="Object size to test (default: small)", - ) - parser.add_argument( - "--concurrent", - type=int, - default=10, - help="Number of concurrent requests (default: 10)", - ) - parser.add_argument( - "--duration", - type=int, - default=30, - help="Test duration in seconds (default: 30)", - ) - parser.add_argument( - "--runs", - type=int, - default=1, - help="Number of runs for statistical significance (default: 1)", - ) - parser.add_argument( - "--baseline-only", - action="store_true", - help="Only run baseline benchmark", - ) - parser.add_argument( - "--proxy-only", - action="store_true", - help="Only run proxy benchmark", - ) - args = parser.parse_args() - - size_bytes = SIZES[args.size] - test_data = os.urandom(size_bytes) - - print() - print("S3Proxy Benchmark") - print("-" * 40) - print(f" Object size: {args.size} ({size_bytes:,} bytes)") - print(f" Concurrency: {args.concurrent}") - print(f" Duration: {args.duration}s per run") - print(f" Runs: {args.runs}") - print(f" MinIO: {MINIO_ENDPOINT}") - print(f" S3Proxy: {PROXY_ENDPOINT}") - print() - - baseline_runs: list[BenchResult] = [] - proxy_runs: list[BenchResult] = [] - - for run_num in range(1, args.runs + 1): - if args.runs > 1: - print(f"--- Run {run_num}/{args.runs} ---") - - if not args.proxy_only: - print(f"Running baseline benchmark (direct MinIO)...") - result = await run_benchmark( - endpoint=MINIO_ENDPOINT, - name="Baseline (MinIO)", - data=test_data, - duration_sec=args.duration, - concurrency=args.concurrent, - ) - baseline_runs.append(result) - print(f" Completed: {result.total_requests:,} requests, {result.rps:.1f} req/s") - - if not args.baseline_only: - print(f"Running proxy benchmark (S3Proxy)...") - result = await run_benchmark( - endpoint=PROXY_ENDPOINT, - name="S3Proxy", - data=test_data, - duration_sec=args.duration, - concurrency=args.concurrent, - ) - proxy_runs.append(result) - print(f" Completed: {result.total_requests:,} requests, {result.rps:.1f} req/s") - - # Brief pause between runs - if run_num < args.runs: - await asyncio.sleep(1) - - if baseline_runs and proxy_runs: - print_results(baseline_runs, proxy_runs, args.size, size_bytes) - elif baseline_runs: - r = baseline_runs[0] - print(f"\nBaseline: {r.rps:.1f} req/s, PUT avg: {r.put_avg_ms:.2f}ms") - elif proxy_runs: - r = proxy_runs[0] - print(f"\nProxy: {r.rps:.1f} req/s, PUT avg: {r.put_avg_ms:.2f}ms") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/benchmarks/docker-compose.yml b/benchmarks/docker-compose.yml deleted file mode 100644 index ae06dd4..0000000 --- a/benchmarks/docker-compose.yml +++ /dev/null @@ -1,106 +0,0 @@ -services: - redis: - image: redis:7-alpine - container_name: bench-redis - ports: - - "6379:6379" - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 2s - timeout: 2s - retries: 10 - - # Separate benchmark container - no resource contention with proxy - benchmark: - image: python:3.13-slim - container_name: bench-client - working_dir: /bench - volumes: - - ./bench.py:/bench/bench.py:ro - environment: - MINIO_ENDPOINT: http://minio:9000 - PROXY_ENDPOINT: http://s3proxy:4433 - AWS_ACCESS_KEY_ID: benchmarkadminuser - AWS_SECRET_ACCESS_KEY: benchmarkadminpassword - depends_on: - s3proxy: - condition: service_healthy - minio: - condition: service_healthy - # Install deps and sleep to keep container running - command: > - bash -c "pip install -q aioboto3 && tail -f /dev/null" - healthcheck: - test: ["CMD", "python", "-c", "import aioboto3"] - interval: 5s - timeout: 10s - retries: 10 - start_period: 15s - - minio: - image: minio/minio:latest - container_name: bench-minio - ports: - - "9000:9000" # S3 API (baseline tests hit this directly) - - "9001:9001" # Console - environment: - # Credentials must be 16+ chars for AWS SDK compatibility - MINIO_ROOT_USER: benchmarkadminuser - MINIO_ROOT_PASSWORD: benchmarkadminpassword - command: server /data --console-address ":9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 2s - timeout: 2s - retries: 10 - - s3proxy: - image: s3proxy:latest - build: - context: .. - dockerfile: Dockerfile - container_name: bench-s3proxy - ports: - - "8080:4433" # Proxy tests hit this - environment: - S3PROXY_HOST: http://minio:9000 - S3PROXY_REGION: us-east-1 - S3PROXY_ENCRYPT_KEY: benchmark-test-key-32-bytes-!! - AWS_ACCESS_KEY_ID: benchmarkadminuser - AWS_SECRET_ACCESS_KEY: benchmarkadminpassword - # Throttle for realistic single-part uploads (10MB files) - # Memory: 10MB + 64MB = 74MB per upload, 10 concurrent = 740MB < 1GB - S3PROXY_THROTTLING_REQUESTS_MAX: "10" - S3PROXY_NO_TLS: "true" - S3PROXY_LOG_LEVEL: WARNING # Reduce log noise during benchmarks - S3PROXY_REDIS_URL: redis://redis:6379/0 - depends_on: - minio: - condition: service_healthy - redis: - condition: service_healthy - # Required for py-spy profiling - cap_add: - - SYS_PTRACE - mem_limit: 1g - memswap_limit: 1g - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:4433/readyz"] - interval: 2s - timeout: 5s - retries: 10 - - # Profiler container - runs py-spy against s3proxy - profiler: - image: python:3.13-slim - container_name: bench-profiler - pid: "service:s3proxy" # Share PID namespace to profile s3proxy - cap_add: - - SYS_PTRACE - volumes: - - ./results:/results - command: > - bash -c "apt-get update && apt-get install -y -qq procps && pip install -q py-spy && tail -f /dev/null" - depends_on: - s3proxy: - condition: service_healthy diff --git a/benchmarks/plot.py b/benchmarks/plot.py deleted file mode 100644 index ebf731a..0000000 --- a/benchmarks/plot.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -"""Generate benchmark comparison chart for README.""" - -import matplotlib.pyplot as plt -import numpy as np - -# Benchmark results (from latest run) -data = { - "Throughput\n(req/s)": (492.3, 199.6), - "PUT Latency\n(ms)": (14.9, 29.1), - "GET Latency\n(ms)": (5.6, 21.2), -} - -fig, ax = plt.subplots(figsize=(8, 3.5)) - -x = np.arange(len(data)) -width = 0.35 - -baseline = [v[0] for v in data.values()] -proxy = [v[1] for v in data.values()] - -bars1 = ax.bar(x - width/2, baseline, width, label='Direct (MinIO)', color='#4CAF50', alpha=0.85) -bars2 = ax.bar(x + width/2, proxy, width, label='S3Proxy', color='#2196F3', alpha=0.85) - -ax.set_ylabel('Value') -ax.set_xticks(x) -ax.set_xticklabels(data.keys()) -ax.legend(loc='upper right') -ax.set_title('S3Proxy Performance (64KB objects, 10 concurrent)', fontsize=11, fontweight='bold') - -# Add value labels on bars -for bar in bars1: - height = bar.get_height() - ax.annotate(f'{height:.0f}' if height > 10 else f'{height:.1f}', - xy=(bar.get_x() + bar.get_width() / 2, height), - xytext=(0, 3), textcoords="offset points", - ha='center', va='bottom', fontsize=9) - -for bar in bars2: - height = bar.get_height() - ax.annotate(f'{height:.0f}' if height > 10 else f'{height:.1f}', - xy=(bar.get_x() + bar.get_width() / 2, height), - xytext=(0, 3), textcoords="offset points", - ha='center', va='bottom', fontsize=9) - -# Add overhead annotation -ax.annotate('~60% overhead\n(extra network hop + encryption)', - xy=(0, 350), fontsize=9, color='#666', style='italic') - -plt.tight_layout() -plt.savefig('results/benchmark.png', dpi=150, bbox_inches='tight', - facecolor='white', edgecolor='none') -plt.savefig('results/benchmark.svg', bbox_inches='tight', - facecolor='white', edgecolor='none') -print("Saved: results/benchmark.png and results/benchmark.svg") diff --git a/benchmarks/profile.sh b/benchmarks/profile.sh deleted file mode 100755 index 2205624..0000000 --- a/benchmarks/profile.sh +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env bash -# -# S3Proxy Profiler -# -# Profiles the S3Proxy during a benchmark run using py-spy. -# Generates flame graphs showing where time is spent. -# -# Usage: -# ./benchmarks/profile.sh # Profile with default settings -# ./benchmarks/profile.sh --duration 30 # Profile for 30 seconds -# -# Requirements: -# - Docker & Docker Compose -# -# Output: -# - benchmarks/results/flamegraph.svg # Interactive flame graph -# - benchmarks/results/profile.txt # Top functions by time -# - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -NC='\033[0m' -BOLD='\033[1m' - -# Default configuration -DURATION="20" -CONCURRENT="20" -SIZE="small" - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --duration|-d) - DURATION="$2" - shift 2 - ;; - --concurrent|-c) - CONCURRENT="$2" - shift 2 - ;; - --size|-s) - SIZE="$2" - shift 2 - ;; - --help|-h) - echo "S3Proxy Profiler" - echo "" - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Options:" - echo " --duration, -d SEC Profile duration in seconds (default: 20)" - echo " --concurrent, -c NUM Concurrent requests during profile (default: 20)" - echo " --size, -s SIZE Object size: tiny, small, medium, large (default: small)" - echo " --help, -h Show this help" - echo "" - echo "Output:" - echo " benchmarks/results/flamegraph.svg - Interactive flame graph" - echo " benchmarks/results/profile.speedscope.json - Detailed profile (speedscope.app)" - echo " benchmarks/results/profile.txt - Text summary" - exit 0 - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -log() { - echo -e "${BLUE}[$(date '+%H:%M:%S')]${NC} $1" -} - -log_success() { - echo -e "${GREEN}✓${NC} $1" -} - -header() { - echo "" - echo -e "${BOLD}${CYAN}═══════════════════════════════════════════════════════════════${NC}" - echo -e "${BOLD}${CYAN} $1${NC}" - echo -e "${BOLD}${CYAN}═══════════════════════════════════════════════════════════════${NC}" - echo "" -} - -# Cleanup -cleanup() { - log "Stopping services..." - cd "$SCRIPT_DIR" - docker compose down -v 2>/dev/null || true -} - -main() { - header "S3PROXY PROFILER" - - echo "Configuration:" - echo " Duration: ${DURATION}s" - echo " Concurrency: ${CONCURRENT}" - echo " Object Size: ${SIZE}" - echo "" - - # Create results directory - mkdir -p "$SCRIPT_DIR/results" - - # Trap cleanup - trap cleanup EXIT - - log "Starting services..." - cd "$SCRIPT_DIR" - docker compose down -v 2>/dev/null || true - docker compose up -d --build --wait - - # Wait for profiler container to have py-spy and procps installed - log "Waiting for profiler to be ready (installing py-spy + procps)..." - until docker exec bench-profiler py-spy --version >/dev/null 2>&1; do - sleep 2 - done - log_success "Profiler ready" - - # Wait for benchmark client - log "Waiting for benchmark client..." - until docker exec bench-client python -c "import aioboto3" 2>/dev/null; do - sleep 1 - done - log_success "Benchmark client ready" - - # Find the Python process PID in the s3proxy container - log "Finding S3Proxy process..." - - # Wait for pgrep to be available - until docker exec bench-profiler which pgrep >/dev/null 2>&1; do - sleep 1 - done - - PID=$(docker exec bench-profiler pgrep -f "uvicorn" 2>/dev/null | head -1 || echo "") - if [[ -z "$PID" ]]; then - PID=$(docker exec bench-profiler pgrep -f "python" 2>/dev/null | head -1 || echo "") - fi - - if [[ -z "$PID" || ! "$PID" =~ ^[0-9]+$ ]]; then - echo -e "${RED}Could not find S3Proxy process${NC}" - exit 1 - fi - log_success "Found S3Proxy process: PID $PID" - - header "PROFILING" - - # Start py-spy recording in background - log "Starting py-spy profiler (recording for ${DURATION}s)..." - docker exec -d bench-profiler py-spy record \ - --pid "$PID" \ - --duration "$DURATION" \ - --format speedscope \ - --output /results/profile.speedscope.json \ - --subprocesses - - # Also record SVG flame graph - docker exec -d bench-profiler py-spy record \ - --pid "$PID" \ - --duration "$DURATION" \ - --format flamegraph \ - --output /results/flamegraph.svg \ - --subprocesses - - # Give py-spy a moment to attach - sleep 2 - - # Run benchmark (proxy only, to focus profiling) - log "Running benchmark to generate load..." - docker exec bench-client python /bench/bench.py \ - --size "$SIZE" \ - --concurrent "$CONCURRENT" \ - --duration "$((DURATION - 2))" \ - --proxy-only \ - --runs 1 - - # Wait for py-spy to finish - log "Waiting for profiler to complete..." - sleep 3 - - header "PROFILING COMPLETE" - - # Wait a bit more for files to be written - sleep 2 - - # Generate text summary from speedscope JSON - if [[ -f "$SCRIPT_DIR/results/profile.speedscope.json" ]]; then - log "Generating text summary..." - python3 << 'PYEOF' > "$SCRIPT_DIR/results/profile.txt" -import json -from collections import defaultdict - -with open('results/profile.speedscope.json') as f: - data = json.load(f) - -frames = data.get('shared', {}).get('frames', []) -profiles = data.get('profiles', []) - -# Aggregate time per function -frame_times = defaultdict(float) -total_time = 0 - -for profile in profiles: - if profile.get('type') == 'sampled': - samples = profile.get('samples', []) - weights = profile.get('weights', []) - for sample, weight in zip(samples, weights): - total_time += weight - for frame_idx in sample: - if frame_idx < len(frames): - name = frames[frame_idx].get('name', f'frame_{frame_idx}') - file = frames[frame_idx].get('file', '') - short_file = file.split('/')[-1] if file else '' - key = f"{name} ({short_file})" if short_file else name - frame_times[key] += weight - -# Sort by time -sorted_times = sorted(frame_times.items(), key=lambda x: -x[1]) - -print("=" * 70) -print(" S3PROXY PROFILE SUMMARY") -print("=" * 70) -print() -print("Top 30 functions by CPU time:") -print() -print(f"{'Function':<50} {'Time':>8} {'%':>8}") -print("-" * 70) - -for name, time_us in sorted_times[:30]: - pct = (time_us / total_time * 100) if total_time > 0 else 0 - time_ms = time_us / 1000 - short_name = name[:48] + ".." if len(name) > 50 else name - print(f"{short_name:<50} {time_ms:>7.1f}ms {pct:>7.1f}%") - -print() -print("=" * 70) -print() - -# S3Proxy specific breakdown -print("S3Proxy breakdown:") -print() -s3proxy_funcs = [(n, t) for n, t in sorted_times if any(x in n.lower() for x in ['s3proxy', 'crypto', 'encrypt', 'decrypt', 'handler', 'objects.py', 'buckets.py', 'main.py', 's3client'])] -for name, time_us in s3proxy_funcs[:20]: - pct = (time_us / total_time * 100) if total_time > 0 else 0 - print(f" {pct:5.1f}% {name}") -PYEOF - log_success "Text summary generated" - fi - - echo "" - echo "Results saved to:" - echo " - ${SCRIPT_DIR}/results/flamegraph.svg" - echo " Open in browser for interactive flame graph" - echo "" - echo " - ${SCRIPT_DIR}/results/profile.speedscope.json" - echo " Open at https://speedscope.app for detailed analysis" - echo "" - echo " - ${SCRIPT_DIR}/results/profile.txt" - echo " Text summary of top functions" - echo "" - - # Show summary - if [[ -f "$SCRIPT_DIR/results/profile.txt" ]]; then - echo "Quick summary:" - head -40 "$SCRIPT_DIR/results/profile.txt" - fi -} - -main diff --git a/benchmarks/results/.gitignore b/benchmarks/results/.gitignore deleted file mode 100644 index 84a0821..0000000 --- a/benchmarks/results/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Profile results (generated) -*.svg -*.json -*.txt -!.gitkeep diff --git a/benchmarks/results/.gitkeep b/benchmarks/results/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/benchmarks/results/benchmark.png b/benchmarks/results/benchmark.png deleted file mode 100644 index 19262774cf30c48a79156b562abe31f2b1939fb7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43554 zcmb5WcQ}>*{|BxiMbaQeR?DWcii}cZXB=c#lD${fDKbjgBQml%#wmLXl_Yx{dxY#m zIM#8T-}A2b_jCRJ{QX>4SLagczVFxT^?W|o^L}+-Ri2ibiJF3ff>z<)9SsVKgUJ*W z`*sf>g72VvkDZ6##GUWzI&0dQIlDb{d_tk}(AnPF&e_`H5xeUXM<)wATOom~R|Kx} zv70+P+dGL13flbdZwS~qJ{7FjcWHy4a>V}L11Aa!#*@geeNU6S;Qz`AQMhwU%ROOn z@Bj!NZYIH{2GHuY{sPvH3_i%z*aImH8d#pW4Yw_kH{r{+C4 zf7f)ZVb`vGa})jB?YDstPO8AFV8Nkoq(DySsaab>+zp|b8`#Z-L{55Y|D$jL@D=1& z{ij_0-{=Ec2>CiL z_EPQNry%AA&VQf2&`$h+|8d6~Ig`B}@#~is7d~mxtM%IEQ%kyu$EqbuSr-|Y;tC91 ze_m~RnM}U=_?sz#XIc-Ht4%9Sx1(QVrV?L(ooPdptgtnrwhPqkL?%fOXLsojq^gZI zieJGiM`UhqE!UA3OK?U-Gy>(Hd9<=k{HYlVe|-(s&rpocGpg}$?M3Mfe7NOjy|=rq zP(a)qLk|hH-+ZiN;NahzquCT9xuMIe`|$$37P0~su zo&34HE%NH`Rx5cHm#AJ0s>QbYj~sE2xE%Vq@1F9f(v-b*uP8y|QA){n$xVaOt&>|j z3)rDcu?CNfy$R7aBbB|!*rXDAb6@gZbDq6FRQ;@oxcR4v&CamO^=-xC9*MY=5o=T^ zYBN~=qy)>OnfWHsYqi;RI$CG&d-#Q=@p?b*(V3ZKbkdY}>i8ulN-l0np1@-)j+@KI z1u`%8i#}|4`K~?$Ev=KMdo@=hL*Pg6?ViA%%J;O~-Cb+%OOvO|)y|J6o)dUbm|j&w#82eBvBuKrhT4i~7IEi$j-=*D&aC-L z&LsCo=F>l-6$+$HK0ntoG)6yLojNk|Y$j3Oyyap{kF6{Tm9HnV-Xd6MI=7>Z(c*rT zxH{d!DE|B1DK<9-h+ZLd3)S`VBQpU+Nf!1~?tJM`6HBaqGKFe*pm(pTJO z9eMBlqBnlHLP)E^arz>z+eW^-#&a`hdY$Y;MaO#7(BUh?iI%DMWa)KSF?qh&c$dvO zc{G`{*%5x)ps6=kCu?qJxq2KL;ijrH8b<6||XX;iLToW>G|p zs>I=O&-INVyRtiJ)Vei+HPRcs@*$X5!r+EZ_`Hx-G&X5zsBGviqoHXXX=CVwfzXup zpqOQyz+ihQi-c&d&XMn(P$5jMV9OM8Yi7PA8{hgGYwTk|b?n?nIP3kj-b4eg4}wRT z&S#i#-IHC@QmwI{xII+#G?eI@a$ojgH*9;RPm4d}gf!2s_NFUDuDobzOOzP7FHOs~ z;lgfV*igW4+02MoPQ~3&&>XijrD)rpn02G4n=W zk#!uoQvF_u`=3W;pBrGW-NHg^Q4dFo~j+z!*|6H8HGoRv@?#cII$c<1fzy>vju? zH}{~1EWsJ4OU81nygFlj{*O7@r|rhfosqEvL4Mb@QTt>9yjSX}PF(8o-dR_0-S#GK zWXx@#cmFh9d_pGguBjEu(#FFzO}WDDHC`+$f-PVg_}U9jWlFF>C6_a-bk&4MyA;v- zi%r3UqRTyXX!3o(L$tS{rcpC+-T%y=OVr8P+E^O85F>3rE|dMRG2q0ekjMIL(32Rx z2PdHn-rrh1c}el5cD~+pX)f6VDNQ{~y|5-h)0wJ?@$)t;!%{+KKkpSP!|h&(saXBN z8_8E>g)8V-nP`-SdQ;1MC~HY0Q#tNF@`OE_JZRH5Zrp1POB=5gG^)PdldFThzcg54 zI9J=JAnPGeKJMEA^-30wfJ(1uxSG`VMQ41)On_pnz)~A=HZ62`+hf#kVTH6gRz<)S zR*tQ;NeBu%{r(yB(1g+0>(h|b-Qd%-i4Nb+rj(8ONX%E7>km7%OQygL_*?d5PZXr3 zJg793EMS%NC?^JA-eT<}FZX}C;0z}@(6TRY&dBLWOAM{cd=K~fkX@bsqt2BpH8X-$ zi-jFWI1|SED(;1!w+)Il@`NRH>{%c|0ZxGpCFZ;{mlZh1+r;Xb=eanTG*@C$?;F%4 z;yl}t-BLs7(?8K-=uPOItCX$@TTCVh;Mg-3ei!uT)QwzS1Bnu>Ptxz01G=FkY$V} z5mI!Fo_$^7oaEoCRI=?)-0<6bQbOJss_`r%-dq(dvK}ZD#%g+Oa>Q+W^bqfCV%s?- z#h}``$(BSCeF~qgnTcIg-^{{&+N{X*wrfw4lwJGXs?^<4XihY(Gf(DZnlJDUkg5_E zE`&aK)lR9gd%fn?knVyW)~F_EXsixh=ZP(`8S1MMEnyK(*GRr~fa=(r3dywm*g*~h z&!v)-$q71M<&dG?i7jp`K!~F2V*_ACITZ97~l?Q(fUg@>bg`)HERBdNR2IFH~er=OUX!yu|n=Tek?UVZU2le zb`60Ie&fXfYGZ>??hdk{lwvx0TJIW?w^pc&Dy=i%d`NvAq1|rP`Pe(FWsVN)hG&fU&T| zysmgbr?jYDN}>km8e_rm+*i+bM>*516-Vcw8j;q*gr7vV;tTv-%X1(Xu8#qzw z+kWiJSlv&L)aj=DRJsT)w#9+YlM*JsJonKiR zq@JPJI(d@UTZ%DHeaj~84lR)ooJNYk$HvddP?0^nXBn4GW`EgfEV8`3LzBBce3w|5 zM;xnku__$;LhP$SRP-4CIs1hMov;f;Iy@{7pF-=N9Qu>b)Cz{$ zZ=ZoHO|tJp=VIJstJv7NPj4{%n!Gi{u%AAX9_Y@EsvRPvRgWuXeWH9hTxWq-9DgB+;0Jvg+zXLiID-5Y1EYSwq}5kZr8$ z=-8!@#_!2ZikKtq7d88NM(iTTX_Wq#%32=>z=ECW^imdF5sK}kmRXt*v|+0EE!3GQ z%8T)xY71E@XX>AbIJXdeGiA88E=jqw3&s=L>wb>7jj=e`rxFhb&{5ym#i?fm7=Kjc z%a=VLL6adRMnjfU`_xol?K9!L#o7{Bz?*bgDNut}wnrsrMG0Jy&r=n_|RcA^$|n9f|7|024+;yR3FTHL(VlD^oq+ z+mv6~5jW{5?Yd9j`MBe#?%oxRGG?!_ocLBxDKzwzi|x zLv2r*CA&3Vg!h+dG5>Cg|Ni3&c3V+N37W*$9!?PTT6h6hKVv0 zwjVt-mTAbxlc!)0)MomYEm3lfdg?A@&tH|iOx_bK5pt$PQ*;oAa&LB9-K!5e_D!Uk zPg1t;*I6mb#wuud*P5>}ZM^SVEcuk~#uhPvmG^GLII3=zS#)P!?ESVkz(5$L^kYIP z&Xbl=y*aec1Ne5N+u``n35_xT$i!tan#c{+iCOjW?LNzCny01FnvO+Re0$3r7$azm z7w9Hl*}B2Hn{>iJaLG!`CU+_2fs7x@e}^^ftdKm<%4UJn5ALk=d2x!a5a4U2S%P?` z(ujD!^8`|m2t8lLih}VQ5#`H+^;#>zvAA}5bf*u?ThTo3ElNU!lEACV5OFTrJr9iF zN1vTXOnHxvY(#W0+LAD57EH8A-oER1b&b7hrhDYSR~fQ+ZRb9tKb(oOmvkl$2FwSa zSYmv{vgqlpME`AT*_eE6L%6d(hP=Z=RBJ~z%t!IhhmW-!KK7?GjGcToBok7$HuOW?~JS#-12Ua@Kx8%MU^T>EJ{cELlHBpcxz z#+dhWY1o`` z&ntabazFI2UQtW9ezo~yvAK7CNL2AlwfcEh|Fv87 zRxbA03sRcp9Qi(ZLdt@TL20E_>&cm1ocoxS(XViP^5vO4n|Ja1^F|A&(sK$( zMOE&?(K{+)BlLUdRvEy`Y!zq_6b42@vjBa@s8@R1G^)H8y%cx$qhd4uZJ+n^?5fYgo6jUv~5o)s+l5L9@X{pHs)Hj=yhThxeeDt7IIPP4H z8V#esxvPzOao3)lkNH#Tr|O3t>#?S4HJaNb;Ed&~S=Zg*iOKV&k?!f4$B3)FI(aEo z;FOJAC2+X%fl9>~K8q47oGbBkon0MCba$&nuyQ8JbGWYLZIUPcMvHR%+X7$|qL075 z5jNh;P786wGLyEG)wZ{1l1HqRi{55xn=Oq+=o>BM;XaK;TvE~~@j2_QV)5Sex76i( z#4}{#Z0V4#Q*vO5Wp5!6$-zcC=|v;`0f)@q@tIO7c|YG73$|n?fCjLQvLCob4ha%B z$@RG44TgFgKHPDs1efVCfB=uPCVuPv!m^*nfFz&!v)B1_2PpA{TB?KC$kC;^6N7el zhL#U}Hvlkj(%1KdilNk#AM~4FWsJ81X_-OV5mqQ%1Ysy*(FO=MP6a!ZB1`pN)JE@1 z>?42v`1zas>y!_S_v}Dzc`xB+8*!l3T=n|I%j_Ady7hx~hn2$5-;MC%#tHY`+Ud&s z%)ix7($u4xyGqY>pS`)bk^PXa-oPZ zxOZ~l#=ZQWn>zJ^KRzYpuIFj4wA7KG1)mn3lbo87mO*uF0v=D_sZC?Ke(1pD2a_u^ zBk1CpQjH8n?gCj4qf_t2>{K`Rz1lx-PJd<2d)jT#BI^>7O#PW*u~}11lEFuXYb$so zUV)e8j%R(om!(bW_k(j;$+pa*)*lxyZY6Lz zA09+74er&Yq79C?KORT%#79e;*VYIp`K^^L*b>9rlX`s6&2LDKdZkb0wvsiMy7zsd znmG0RWt!h2NJN$MsRKn%^~-EW&I@cFo7a#v76r^WC5)06wi_Mxp7;ZpCC#|c_}Mgr z705^!yqGq_-5iY!ja`jQzt1MUip7FTKm?o_WPJkZ*jy(fRnyc>I&(C$R>&asjX=Aw zjl8Tvot)JnP@XiGqIlGfnf1E;XH-Ae(cLlfUhGth&&r;b!X(P{;%1iLZm|?qPIUZ? zBFgYqv-c;J7IDhP%wi+_=BVfLrJRo)*i}q?=Ngb_4W;1meZF^c8zRlI8GubMd?g+> zQsZz`#0D0#YHiWU`B8n_TRJx!c+jtQoE)s9Q2e+lB2~ktX>7A<#0G7TUCH9+6wQi4Ho`j?-G%$$7?- z742jHpwv3f8vv=O`-;gzXIh|V;uhN^P&SRcTFI{U`1N4zrPLoCQw67#%$8)YO%l!w zZ+;zIelkTkS*_yjb12`qZnWz>f3jo_X)l$a+&bgz*i|Sg)KG_3vza}@zVugV+403- z-bDFKtKnrXR2WaLXI#_0ojiO$-+e7$-%j9*gLTJwKOA9Yr{qo0dXcfxJc<2ze>T#f zq#?g-lxeT*HjlsO-j5eQ&p5_vdwA_pi;$S|4=`YPE(|6I2t9F~zU9~+9e|;v+wTj% zvu_=Ev3OqhQfHZ+@n-F6K$HcSTD7_7LE}1jbSi&5>NiObo+d^wF5E&}*Yhdqwa} zP|h_Gk!!He11`aHRQl*BP|-0gL4m6zEa2LWya}ky3om?}|LCM2EDH{p+V)c+k;CcF zT|Q?ZT_Z}Uh#q#i=E-(46!%!~pt0c230@IdKR?+ssEGygVyDl1uwd=S3o7lhQ0LCK zA%d;uZbFE5eUg!dx8Sq-YMMaz*DI~9G(n40%Rnjb zojV5dq4i^Ka<|k$+HM?DKrxCRCa^6^fu4!~BcJw=IV;J-a*Ir&(qWp<@R5>_XHBj< zuN10Y!`@TWC~>B5E?&fH5!TGHala>Fo&o2g!SJf?wh|Ua_^HvK{X){?1p-;lomb;vZK9lvJL)#|ZoFi)u)P|EJ^k&$!1H8Q!<-(F<(*U15E1F=4XV zxQf+q_XQ*w_q;1d)9R@Ub&;j64Us0TYc4>c+1l}tG_T8TN*KPvW<}zK;AXfp+B_Z^7+3$vL{5b(X_2314!;Z%~-N9$%CleRJi>k0^y} zo-upW3_Me25`ThS4O?{?Z4Pm^{$fbr7rfrBRD3o}<7C1K&hii+qy_a1jHR~b>43T| zR#R$>6xNsu@*Z(y8&vcMU$*1rd_a3q$4z`?G=5X*y|b~ztcV@AvAB#a)&=(?;sI_w z3wLXv@JVDf>u!;52I0oNr8hTZ1COZ`N_pUSh1Rz19=F8sH`_)|9elcv+O}@|r)GRF zZ%Gl`Ti}_^m9vt+Btyn0kJyGrnQ1Fp5J56)tYJ;7`3)fBtdA{Wd0bh`LrvZjYkoRo z{HT7tO~gqRHhkS4Xn>CehN|3_FD|4ko%Ry6s)_Ji+Rkt*ts1v7*8FJ|fEhoUJm$S= zSlAwJ_IAx|M-5Y(@ zmZM}(&A}!Q0}sAIti{k6$ey-w$v!*WkIuF-Wj;x@sTx;y$axl;IhE)(dZdnZeFlAV zRCLu?3gjTur_?>he|~+9T&!}biobM?zpIL_c@g(`rG~|ej;c;tIqBw&W-CRsxEfZ> zZloCfX8|C}-PRwbirP?t2)= zi5BwKWaw2kw&!nBR)MXYUcz~kd^7)S*K?@uxEP}Z!JB&f%yZNwjl`38cDyI=rf#Ws zx<%i4LMZiKB~wE0-Du?<{~mHnj>qO3W-)h+nK4M z8n$z~lo$*#2XAN)%gn1clXY@tvYN8yV~f9kw^Ho2-Z`p{G7T$r%X|2K^ruVbW#x|2 zP!VT?-kW-t%a?kJ#_{i0yP6ea5NqHALqteQruA#?mi3+$dDaTYHDMI^?3{F-baF zCgGqRJ=#){pr#pTy)mUdQf0vGAsma$Z*VZf^5=r5fOQPV%Bm#hSpvK46dtcOp(`@7ZQx3y6@f-eekf4^gTl+|S?GK0D>8WnHM%1!->{+LCL zh-OS}m}jUCM?JWQ!;^$f#rs7XWZ`jYe9vaX*hfvFKih%P=KX%h6#blKGAMqiEljY5 z#4;2gWAaGF`2`%CdoDvo>+vSuUrAPk)qQ*)PU24EqmY;T4jd{s;+T)l5G)vU^(a^u zN41}{c}zQYSTv(ebzM*Wdgf+$gQVR?J&==OLc97EX1n?JjEBXiE-Ty1s;wKLbW^2C z`YF$iHai2JWq+zQ+v)N&SUzEc_BZq3t4s>W%a(sL4Da;w-BsIn zsrVV#*=|^+8aEMo-G$8sY@OWEX^DuQSVbzr9)%*+biRI>Dk}LbvQzqZ6 zxSf!mJ^s^_K1`|^pXDbCHmH~~^Jff*rRJ^6nwBOB;kYO%o%Oi11NQ!v^K|L$=kf#1 z@LhcAmh`Mg%X*$JL0NOAs;y6wO43xl@zq~nPkZPa`B`RDLHq5c&>o5J()!t3vT`*z z`h!m>my!t{-7^(Z!9DKA`(8@v)0{e`EGnRWq=q!$7XvY4&y^_>dp{{S-)rXl_~6$1$wj&jo!5L z5221?_`NTPjt2~Mb&J_o@^yNLm@D^Jyf}7ew^5Tp%e2g_?Vcn`e=8#+$0b#xH$BhA%JikIaa6hg(Kqea2_?sFKqFvAqxTV{5oJ51CK1IV*Q(7xN&0Q)bWo`T2AMpa${)T0?Roy1X%aZ)_RYo z+Ca_bs1<1GB@UC{9*kC#b!4`HTI8YYd>xS4|W@v6=QG3vRT{ zwszZ$`G(`g_5=TJ;=I@X{ROogCH$uRdz%v7S3j->uYXUTv+@&O<4FSlv7xW!_IypTMfYW~ z1l6K01rXy$K1z_C);eVA1|Ek>{*DjKO^!7C5(&0(zdxhV>>jC(Oll|izkUL6wl7=# z0q*Iq^n#VM=NT<6v?YE2rdDKBzkQUL@|dsbfuIB9c|+Jw+_s@z8A??F zlc3QTOOtoITXi=<#2)})Y&TNn=GGDgV$^SCk9@t7LJ_OJ@)Zp1z6u}W-xYQgW>#VTjrLwcBYQ+{V0dBIlR8GJ+x8B0Mkz|0q-? z;d&=V2Wt!bO6^*3l2Iro%mZ}w#X&1?X31<>UZmVoBodvCoWGJ zalz)7K=_&s05aEw{IYLo2`LrD}i6?{+pQU6pG9z3n7s5}d3dm;wUYQ8&`h*)#3d zZUdoWLy7S(iQMku!2S)RBQXc(rbk_y(nhO+?&%^FwKP^Y2J1za3;x-cX#m!5nKf|L z&z#=ha5CHSB9ZeR6bw7ZTe6h$REcPSLuX~F}%5QB&wG)cm;53aaJ9HM0^*Cg1l z>X+L;{4!n7iGk86ir(|mul1_v&DR%;WIKyIu&L2;3rHLkP#${zGfPJ2!4LdMm0>`{ zAO%m&NhvR;>-HvJ$eTd^S%Exl&UC6Ofdi@m(I%XmJcF`gZ}gr;`3U45B5RV`bhgqT zqM%K}SJ(~b`+4IER9`GTQy+^moTK=DT_roSLxGe>km@Eby?Y$;$z8F3MWJyDG7=9# zdi(?gu6)8d{*&s%9t5B>-Ok-ET7;vVh{U#~dzK%pECt3qN_ zQhQsE-L?^-)kE@c ztbfZnh0L#(@QFyKK%&xj(8Dc|Fdd|yhCm#Q;HxvK-|xm`&18gx09r_JLZ$Vq-9Ir1 z81&O_|E**cG6IK_>92T$zKtIPtZtt*f5c_<*-Ql*Y9A4bgxqu&l0jJ;DzRd8s2Br> z%RpxfwEjv8B%BFf3e!ivR^@7)gIG`#Xv|X$=T&Ov9u%3qwYm)i(7Fnuc}tOMQb&y8 z4~3!dMMv}N_tnBPUg@|=B0^-uWnb@nZ*IO;j>g1-HH7a#Gd?&3g$4l$eV-o7yVc@} zph}g3%`o`ptdN0m&YKkvsju@#q7QqhE=2d{KIgkuI89^i2^V8JFT4WunkMv7%JQi& z$x*Ab%>pM2S1qw%|W=%N8C&r*U@+b0C#QmPHILw~aT;-Zz(Q{`qiRQMzq z5fue0rcPQ7mF7be`FV`>rg@rtXvpEDbmh2fqLw{en+Kio!viQaL)U>Pcm?$p=b1z| zv+M4EBbn@%EDtS_dGCLC&H~v0SER#1+HryUva|L3VFHrZyCAk+?l3t#iQLxtHr$`e zvYI4_5Wd6d)Xna0uLZ`U9tJ0u>YIs77VMD6v{$5dPbJZwz4nEndUhK{!|{ae$N9_9 za@Jgl>vNN@Eum;W9Z^vZA^pj&h7bb<{rgG({i#qz2jAXNRJ)$(dL69r=SWCs zqVGnhdDSpLM0G&~0zmTcd*NV%Kh4NtEcC2Km4HkN#k<#F*HzIl_MW^K*92bP#BJ_6 z$kPlN`D{Na<6rmqZhP{Ymxg~g2!-zo7I;qd8SBj+Q4@IhHxxU5?~IU{a>N?+_9vk0 zCIck&EXaQ&cQ9wAVHqMyPQ4m8T9pKArW)b_c{LgQF78$>0vV&(XK#Dv;?UhuuQe53 z!^&bB4|-op$Pnq+c71Z!t8&fv#A_L(`NkvZK~D8kD(*-?wL+?o`Ukz-HY)pmJ!K@e zdiC`Pgr5q`FH0zVf9QrhEsAln7Qp0D&|In}0+@$C<^zF~YRQCO2+H+Z$n#jkI(8=_&PnLo-JS>`XLRiR}O{IatK9j64txbs@SLgJc@J6(4FP`*B&M7jqv1ZSen|raM+4*Yv`$1Y9+yCRa`2Ao#obGZzI?b_ z*>>J|n)a#Y*U-H&qS9N0BPCjnX#PTp&N;Yiv1Gce?oV9raQz-Dm^|b)~k&1pH)f8IP1||t`;c%fA{Kl4*K~ZjIoB2?AIjp-ZQbq z7PV+4W-;4hP{&^qFF&J?$}W_jcrD$bKAy^9Or1eLzr{VS9(~_FvveI+CmituX`Pf_ z#i(CNd;Xm{;&;BAqbyz{m5AeWqW-lzB-i_RzAl40BMM?}UlU_^b=ov!8<#KYZeKKd zm-Tyo;K+aB^cyd6Pr3M`Z$c;f2yj`ddE3}$J3|5p(c_*45u3rDL?6fxPOO2EcG~s> ziyC!JkClQ@PsYY49#aj5Se`0jgEnT?cj0jp!lsC2tk5PsmmWEs(@=FmAzJ+xR<7mP z|L(Zj@{Hiid$b}sA#y4uHFAA>X5*vDV#BS*FVbpNPcFw1{<0t`LCwF2KD(27 zri6l$h?;$QV$=Jx1;7DpPuJ*Hrx?jsR#YxR6uKAJ`LPu0R>huXWzxV*v9H8H36X1Pjl z0ZmtAPC~Dt7LyL8YK_VIHjPPDE zyD8enHLfcfC2bJKY@D$$yGc9g2~~Rutd+OAMhp_bhu+TmIU_QICQ3IW8n?0wIrMu3 zm)EEH=XW%6)KK=KwR!7cn^dJZ#y?yEo7PQediyW&?y{9!q1{c`Xl)b7j=pQd zM~T0eb@HW7ej58dp)!8s+F|4z`zA7pSoTBNwLy%c%;DIEFmPeN43^m~`(}f1qL(P{ zk_VCD4V*E004&Dgk76jfIWN1Ivd%Pl#Jn}|vqR1IAmM*cCB#fSqg> z`#%XB)*yu9TIK*oC^?WoAb;vdMOPXp^i&U!41YMJ@_WsGzMCi|Kt^y_6jU0gVW$uF z>M0RPj|pQQlW)aRM*{@NAZU!ZfWGqRf2Zxp?Ua}1eWI<_nLjuvC7~nR-wKI0f0~I9 zSD)=^M$e6brz1xlsKxHoL}(eN@tm{EPLxlHWJ=;aqw!(2o{Fvfv*N@%p0plpb2kFi zhA>g_*Bu^}^YDmFMB zi9<-?e{4jC2k0`9#FO<%B)pyH+3nZg!loxBo&Y~TLR4N)=;O`R+4sQIw1u&M1D4@C zG^;-WCcSwNgfz`g$+n4jZP_!LtGv5XFYYpLS}_2Ly2aZ!{r!#F!s3=o-k3602mi(K zG7X0W07;g@O8sywf{i?gKrw5doFhpg!vCGBurT53MM#YmLF#k}5HDYu2c*MBa&{b_ z$p51+6zmUkW}?NZ4V^ph#^@u^Y6CyAfcac_;lCe2@&9Yy2H95s8~k|{_~ZZE%uX{~ z-TS}vhFz)+?df*gdJtPl-_kSm{_5j^w`Z<4{BBFM0!mW)=_!sXAmA#*n@ z5Q}#O#54rw%m!F}dr#BLgwX#k`Q+!cP$^3Q!%4_83z%rQ_dvdZS0g=0Vi}oSfWRq# z^)5W|#kXRk_GGEGpF;U?PF;a~4Nf&j=0i-z6s{5BV(|G%mG}$-uG-( zl~@u+kZs8K$-8a0{+o{)7!H-*o5e&>({q_XzB>s~H%`Qop@DAves8A-X@>>~fsjcX zaMJa`S-23B2t2LCdY~)GW1hQe9+SHW3Ze>xJG@7dpCZFe$S4oUer9FEj=`V&bT_P_1AJsrz=Bx$PuacBv!v;ibmDy4=}tRHXA^{xE%ns<1~;*7B+ zy`a&sk=LKE_Qy-%Zyl9nof7zO1DQI_FJA;s-V!yWNg0i6=Pp5pnZT`dAn!rv8_vO` zA*n0K41IzO`Y0`>)_BGmf>c*rM0S@E_5woR52TcNXh{O3gl1K{at-E;3L(EcWQe1w z+^yaI%?x5dG7ChvNrFK8CscUr+oEs>PIQuZ$N& zD$iEtL+|~H;8#W&aO8PfEg|*RS!N?|+XCR9=_2Ur`R}1HTJuHuP{5yNJWj zAln>w-62)qs4QhA3m&z=iux5z5COHYjXgP_Zaz;~af6bs)Vsk3?b*oea@)yEfer-0<;BUJP=VGXqw4twaHT}7K^qrg(4jVS_^>9PC5f?leWs{y_>tm_s z1Gx551s-U|j4IX9GbX^c4ku#=c8uI7C>hV?a95(Y5^c$duTnpFfbQlvCcYRCyv{+T zi=-+(cTr+<*lFbP(YA)H1uoO>Nj`&edkib@FTa1Yh0hP#!``ZT%^||JJW!NHle9Y1 zzTA)!Yvj?45{BU*zGedbxOsUz=9Tc2Dn~>>3Zjh}JfL6yej&8n`NVx6Jz!8U$&hgL~iHyr)vnV4HId(bv?mIwh7Uoswc0y$M=laJP_T z2;)vb3cKNbhPP(qZfi6XUJM2g#DvC>9&HQ-G2eA*P#qh1;r}Y~Nf0SA45mZP-bbDl zGJDDF&e_E5xCv+f%TZ_N_dA*l5dp1Vsgo|&G!FXqYWP^b#;X^t>h}1}L+^Fti1xj% zF3s23amQ5hzrArX!R+lR5jGtWEk&)j7ok>%42fIFiK8q@u-hgctojN|B%OnLdtxT{ zwSwe(trBO*t5tlPfJ+O0X3Qj)JwOodQJr{S;Mt=tKfs`45Z)Y0J$LIrQYD&|q|&S| z$fe#tNvYJ_Y3&>lGl1_>WKe)HhZtg+PWV(-+#guZ#;Ix!pVYIY9vQ<%68-ra@Qn%z z=aF&^NN5JlD7zsY6Mho^+o?+;k&vn0*Xpq?B8Q8OlS}^efBNu#4qO=EGB-A^oSeW0<)z0`#yNSrs+|Bb0gwaWR- z4^U0@V9v{6x(6ICe82*SC;LB}?P3D9@=mfOf0jmns?0Qq7WT5eNKqCmR`!Othvk%+ zfA@V6JM9XD*Ml58<(mL~xgV&dAxa&Dyu}~9Sm$0jkmRqQUnmHew$zq#oVq7AybLp@ zw&lsbS1C>IzNU9*A;BzYL4%Ue$XgZ((bTgQd-=a-0N-FFSHEBAgZ{f&P7?MrhIyfp zE0UO`!q}TBQHMg#fi?A16$;O~rx4e%XJi=bYlSF^>8K|;jzux}4l(76L z5G`5GEf1Bo@4>^)Gh$(H;xQ=y=vnub`5#$f*KdYa3trytVgQ2dB8e z336soqfT1)2^RamOG`WO3t4g!cq5xs?2|oX+@q37zW?3ZfS&d_K1jJaTmPrf4o?NG zC`>*_z0SE<4LiD7KmsN>AkG=i0O;aJ6!&A`nV?soD!}oA$3p+2P73y!ds{1WFymDS-FpxK-Md9c01icNb zO@{FIGsLwamhJy%`%p}MUYSEOxX5g;T*x^KN^1IhqUrHc-jzhRnYP481AL;G!>=zv zYtkUgzQbAI3K9{W9Hj+AwStYI|DGf}Czuiz5Tq}dzt=b&54PB1A?VFTP~IT-KAi_p zTS36CM!k6xmHjNR-7(C_%+OnZqZ_h$pH}H}6GRVGGnoV)%01)a~(*dy)+*+kS zJO7^CtIHg`$%ur5z+~%{@%n-n?uyXGrd5Ma^D{Y}rM#cI(u_`=DqyTW_U~VutX@?> zDO)m)g6~uXZXbU*--E)7Dh&UHf zCP8+Is4`j+rwAMIhLda?(*e%4zAFN^hd_*e1cHsRHWqOUo)2qp3xnQ_OU`-Po^)cy zQ>YCr#Ci~rCiX~e5|DLkVzmNt=1Q&u2=|TyGN?lcZ=PVEgVcN>B;zY<)hdmk(L@-u zSJgmYGF?qO=qTtSq^(+-WPkfsN!rkf0A9dW%IhNgIkgp9B#><$Dqpamomf zycqD3Ci%{-aG#n`IgTi|IL3RxM7t5zeoTr%72!xHa1TD3ypW(3rA1~WAotK>$a*x{ z`FY-t7od6%DR3r6<%H6G&tGTt24`Xspr~V!!hB6e91o!uY5$RY(ze6b-%EOyTJ4Pi z%Vo^-0#zHo16z%bN{KoOOxm-IrrcUxQ~QzE7O*(|ygKnw&v}~{Tm8#+IW<_IFyUTG z$Zc2ug@Nde5im(Z5gGnL4G*&Hj*qwC^%F3}0pML?;^nE~JO^H#f;Z)PFm>w+_Rj#2 zF(gb{_BS0{V0rk6_KjjIT${gIWw97Uq@beX~HL(~A`_Y`7H}V8wk!+dq<#3CWaG-iO%cM_A>y#7zK?{qTl~^9`Ie zIFasYh|pTXI|rH_)N;(IQ9-dIPjOm7+1}oHadLm3c?>-Bt_9dA&J*3AUmP&Z)=HD3 zb*sf(bpD7*@dD#s_KOQ*7XlwF_l9}&4LAdOfXPR;!N7JzF?5yLjq$k)%d`oB9E%t| z$Quh3d^!Kn)oIA2PcvLJ1ex!I>`dFuHDyLA?w1=4oFw9xQ))-4)FK3pYlE(Mwk6!K zJv=74IUI?Xd9W$r^jlfSF#V(Q3wNkzNN#503DOGnK@kPN=3;Y)jK6HCO5~0 zqKs|NJ7rPWz>6&kFiMRy$atri^$H-2kaF6kNMQq&Q`gML2;dV`4k5*h<|j%a^FFk% zUJzoE2cR1cZ)j%B0M47h+#zYvZY-6JReERm^4bXb>ww0jc+FXDA@d2t?%#J`b#J{%#+9j*oed87GuNI=p>bTuXV;xkMxLft zV-m?t-zvqIDMVhze=>J%gVQ^RfI}k)i-b63bq1VhdoR|WhDfP{TuxM-?@ zXUT~k!*5XDafpufT+-HO=7_v$hBMsNU8R~&`r&acmFD7>6-b{D0o;wPQdttObvN_M z54C-R05q52$+?oTLzW#MUJQ{f&ZPOe*THbz0HP!`2#(+(DnH#fgJ={fQOUknMgo$a z+Bi{aQ6DTZ8vXg<{~nwbPqHaQAOzf*u0DFnT$px1&|d>n+)@8X7?P({U&X^TQ7?Ip zJCC~!*_0t-M(EaK3djBaOnZu#QR`d>wR>CPh_Ri><`V9_QVy#d(5}gqvA}!yhz8fC zy&;`tE|em-a^DpLe|_p$)Y~)#N@35;{cMCdMLJ5ZK{fY%B|5J%KFZaHGdPSjE-iI37W2&DL&RWIwQr(z`cANIlV`}RjdvD2;`69=C?Dm|<$L{^STmpBlKrYhNE^TNFe;!jhpH2N zB{m0d!7t{}!0$|=QLV^3T?Dc_ZJ=t38`Nk*+sR%?#jyR^yr2=ChH+b;$(k^sZWYek z0G{DG5!^sG(j>85J&BjCNsUPRaHlK