From 579743b05c5f0fb2205bb953691386f968cd2ce7 Mon Sep 17 00:00:00 2001 From: joeybaer <35610156+joeybaer@users.noreply.github.com> Date: Fri, 16 Jan 2026 16:15:02 -0600 Subject: [PATCH] Add Google Cloud Storage (GCS) backend support Implements GCS backend alongside existing S3 backend. - Add pkg/backends/gcs.go with GCS implementation - Uses Application Default Credentials (ADC) - Supports Workload Identity Federation - Stores metadata in object custom metadata - Properly closes GCS client resources - Update main.go with GCS flags and configuration - Add -gcs-bucket and -gcs-prefix flags - Add GCS_BUCKET and GCS_PREFIX environment variables - Add gcs case in createBackend() - Add integrationtests/integration_gcs_test.go - Requires TEST_GCS_BUCKET environment variable - Update go.mod with cloud.google.com/go/storage dependency --- go.mod | 48 +++++- go.sum | 131 +++++++++++++-- integrationtests/integration_gcs_test.go | 155 +++++++++++++++++ main.go | 62 +++++-- pkg/backends/gcs.go | 205 +++++++++++++++++++++++ 5 files changed, 575 insertions(+), 26 deletions(-) create mode 100644 integrationtests/integration_gcs_test.go create mode 100644 pkg/backends/gcs.go diff --git a/go.mod b/go.mod index 95ef1f1..3c7f2dc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/richardartoul/gobuildcache go 1.25 require ( + cloud.google.com/go/storage v1.59.1 github.com/DataDog/sketches-go v1.4.6 github.com/aws/aws-sdk-go-v2 v1.32.7 github.com/aws/aws-sdk-go-v2/config v1.28.7 @@ -12,6 +13,16 @@ require ( ) require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.17.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect @@ -27,6 +38,41 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect github.com/aws/smithy-go v1.22.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.37.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/api v0.256.0 // indirect + google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/grpc v1.76.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect ) diff --git a/go.sum b/go.sum index a0706ea..93eb617 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,35 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= +cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.59.1 h1:DXAZLcTimtiXdGqDSnebROVPd9QvRsFVVlptz02Wk58= +cloud.google.com/go/storage v1.59.1/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= github.com/DataDog/sketches-go v1.4.6 h1:acd5fb+QdUzGrosfNLwrIhqyrbMORpvBy7mE+vHlT3I= github.com/DataDog/sketches-go v1.4.6/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 h1:lhhYARPUu3LmHysQ/igznQphfzynnqI3D75oUyw1HXk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0/go.mod h1:l9rva3ApbBpEJxSNYnwT9N4CDLrWgtq3u8736C5hyJw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0 h1:xfK3bbi6F2RDtaZFtUdKO3osOBIhNb+xTs8lFW6yx9o= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 h1:s0WlVbf9qpvkh1c/uDAPElam0WrL7fHRIidgZJ7UqZI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= @@ -36,25 +66,106 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgpp github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI= +google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integrationtests/integration_gcs_test.go b/integrationtests/integration_gcs_test.go new file mode 100644 index 0000000..bc9cfe4 --- /dev/null +++ b/integrationtests/integration_gcs_test.go @@ -0,0 +1,155 @@ +package integrationtests + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestCacheIntegrationGCS(t *testing.T) { + if testing.Short() { + t.Skip("Skipping GCS integration test in short mode") + } + + // Get GCS bucket from environment - required for GCS tests + gcsBucket := os.Getenv("TEST_GCS_BUCKET") + if gcsBucket == "" { + t.Fatal("TEST_GCS_BUCKET environment variable not set") + } + + // Note: GCS uses Application Default Credentials (ADC) which can be set via: + // - GOOGLE_APPLICATION_CREDENTIALS env var pointing to service account key + // - gcloud auth application-default login + // - Workload Identity Federation for GitHub Actions + // - GCE metadata service when running on Google Cloud + + currentDir, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + // Go up one directory since we're in integrationtests/ + workspaceDir := filepath.Join(currentDir, "..") + + var ( + buildDir = filepath.Join(workspaceDir, "builds") + binaryPath = filepath.Join(buildDir, "gobuildcache") + testsDir = filepath.Join(workspaceDir, "faketests") + // Use a unique bucket prefix to avoid conflicts with concurrent tests + bucketPrefix = fmt.Sprintf("test-cache-%d", time.Now().Unix()) + ) + + t.Logf("Using GCS bucket: %s with prefix: %s", gcsBucket, bucketPrefix) + + t.Log("Step 1: Compiling the binary...") + if err := os.MkdirAll(buildDir, 0755); err != nil { + t.Fatalf("Failed to create build directory: %v", err) + } + + buildCmd := exec.Command("go", "build", "-o", binaryPath, ".") + buildCmd.Dir = workspaceDir + buildOutput, err := buildCmd.CombinedOutput() + if err != nil { + t.Fatalf("Failed to compile binary: %v\nOutput: %s", err, buildOutput) + } + t.Log("✓ Binary compiled successfully") + + // Use current environment for all commands + baseEnv := os.Environ() + gcsEnv := baseEnv + + t.Log("Step 2: Clearing the GCS cache...") + clearCmd := exec.Command(binaryPath, "clear", + "-debug", + "-backend=gcs", + "-gcs-bucket="+gcsBucket, + "-gcs-prefix="+bucketPrefix+"/") + clearCmd.Dir = workspaceDir + clearCmd.Env = gcsEnv + clearOutput, err := clearCmd.CombinedOutput() + if err != nil { + t.Fatalf("Failed to clear GCS cache: %v\nOutput: %s", err, clearOutput) + } + t.Logf("✓ GCS cache cleared successfully: %s", strings.TrimSpace(string(clearOutput))) + + // Note: We don't start a separate server. Go's GOCACHEPROG will start + // the cache server automatically when needed, using the environment + // variables we set (BACKEND_TYPE, GCS_BUCKET, GCS_PREFIX). + + t.Log("Step 3: Running tests with GCS cache (first run)...") + firstRunCmd := exec.Command("go", "test", "-v", testsDir) + firstRunCmd.Dir = workspaceDir + // Set environment to use GCS backend when Go starts the cache program + firstRunCmd.Env = append(baseEnv, + "GOCACHEPROG="+binaryPath, + "BACKEND_TYPE=gcs", + "DEBUG=true", + "GCS_BUCKET="+gcsBucket, + "GCS_PREFIX="+bucketPrefix+"/") + + var firstRunOutput bytes.Buffer + firstRunCmd.Stdout = &firstRunOutput + firstRunCmd.Stderr = &firstRunOutput + + if err := firstRunCmd.Run(); err != nil { + t.Fatalf("Tests failed on first run: %v\nOutput:\n%s", err, firstRunOutput.String()) + } + + t.Logf("First run output:\n%s", firstRunOutput.String()) + t.Log("✓ Tests passed on first run") + + if strings.Contains(firstRunOutput.String(), "(cached)") { + t.Fatal("First run should not be cached, but found '(cached)' in output") + } + t.Log("✓ First run was not cached (as expected)") + + t.Log("Step 4: Running tests again to verify GCS caching...") + secondRunCmd := exec.Command("go", "test", "-v", testsDir) + secondRunCmd.Dir = workspaceDir + // Set environment to use GCS backend when Go starts the cache program + secondRunCmd.Env = append(baseEnv, + "GOCACHEPROG="+binaryPath, + "BACKEND_TYPE=gcs", + "DEBUG=true", + "GCS_BUCKET="+gcsBucket, + "GCS_PREFIX="+bucketPrefix+"/") + + var secondRunOutput bytes.Buffer + secondRunCmd.Stdout = &secondRunOutput + secondRunCmd.Stderr = &secondRunOutput + + if err := secondRunCmd.Run(); err != nil { + t.Fatalf("Tests failed on second run: %v\nOutput:\n%s", err, secondRunOutput.String()) + } + + t.Logf("Second run output:\n%s", secondRunOutput.String()) + t.Log("✓ Tests passed on second run") + + // Verify that results were cached + if strings.Contains(secondRunOutput.String(), "(cached)") { + t.Log("✓ Tests results were served from GCS cache!") + } else { + t.Fatalf("Tests did not use cached results from GCS. Expected to see '(cached)' in the output.\nOutput:\n%s", secondRunOutput.String()) + } + + // Final cleanup - clear the test data from GCS + t.Log("Step 5: Cleaning up GCS test data...") + finalClearCmd := exec.Command(binaryPath, "clear", + "-debug", + "-backend=gcs", + "-gcs-bucket="+gcsBucket, + "-gcs-prefix="+bucketPrefix+"/") + finalClearCmd.Dir = workspaceDir + finalClearCmd.Env = gcsEnv + if output, err := finalClearCmd.CombinedOutput(); err != nil { + t.Logf("Warning: Failed to clean up GCS test data: %v\nOutput: %s", err, output) + } else { + t.Log("✓ GCS test data cleaned up") + } + + t.Log("=== All GCS integration tests passed! ===") +} diff --git a/main.go b/main.go index e5f3fc3..80a0dcc 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,8 @@ var ( cacheDir string s3Bucket string s3Prefix string + gcsBucket string + gcsPrefix string errorRate float64 compression bool asyncBackend bool @@ -68,18 +70,22 @@ func runServerCommand() { cacheDirDefault = getEnv("CACHE_DIR", filepath.Join(os.TempDir(), "gobuildcache", "cache")) s3BucketDefault = getEnv("S3_BUCKET", "") s3PrefixDefault = getEnv("S3_PREFIX", "gobuildcache/") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "gobuildcache/") errorRateDefault = getEnvFloat("ERROR_RATE", 0.0) compressionDefault = getEnvBool("COMPRESSION", true) asyncBackendDefault = getEnvBool("ASYNC_BACKEND", true) ) serverFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") serverFlags.BoolVar(&printStats, "stats", printStatsDefault, "Print cache statistics on exit (env: PRINT_STATS)") - serverFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3 (env: BACKEND_TYPE)") + serverFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3, gcs (env: BACKEND_TYPE)") serverFlags.StringVar(&lockingType, "lock-type", lockTypeDefault, "Locking type: memory (in-memory), fslock (filesystem) (env: LOCK_TYPE)") serverFlags.StringVar(&lockDir, "lock-dir", lockDirDefault, "Lock directory for fslock (env: LOCK_DIR)") serverFlags.StringVar(&cacheDir, "cache-dir", cacheDirDefault, "Local cache directory (env: CACHE_DIR)") serverFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") serverFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + serverFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + serverFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object name prefix (optional) (env: GCS_PREFIX)") serverFlags.Float64Var(&errorRate, "error-rate", errorRateDefault, "Error injection rate (0.0-1.0) for testing error handling (env: ERROR_RATE)") serverFlags.BoolVar(&compression, "compression", compressionDefault, "Enable LZ4 compression for backend storage (env: COMPRESSION)") serverFlags.BoolVar(&asyncBackend, "async-backend", asyncBackendDefault, "Enable async backend writer for non-blocking PUT operations (env: ASYNC_BACKEND)") @@ -92,12 +98,14 @@ func runServerCommand() { fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") fmt.Fprintf(os.Stderr, " PRINT_STATS Print cache statistics on exit (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " LOCK_TYPE Deduplication type (memory, fslock)\n") fmt.Fprintf(os.Stderr, " LOCK_DIR Lock directory for fslock\n") fmt.Fprintf(os.Stderr, " CACHE_DIR Local cache directory\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object name prefix\n") fmt.Fprintf(os.Stderr, " COMPRESSION Enable LZ4 compression (true/false)\n") fmt.Fprintf(os.Stderr, " ASYNC_BACKEND Enable async backend writer (true/false)\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") @@ -106,6 +114,8 @@ func runServerCommand() { fmt.Fprintf(os.Stderr, " %s -cache-dir=/var/cache/go\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Run with S3 backend using flags:\n") fmt.Fprintf(os.Stderr, " %s -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Run with GCS backend using flags:\n") + fmt.Fprintf(os.Stderr, " %s -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Run with environment variables:\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 S3_BUCKET=my-cache-bucket %s\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Mix environment variables and flags (flags override env):\n") @@ -119,18 +129,22 @@ func runServerCommand() { func runClearCommand() { // Get defaults from environment variables. var ( - clearFlags = flag.NewFlagSet("clear", flag.ExitOnError) - debugDefault = getEnvBool("DEBUG", false) - backendDefault = getEnv("BACKEND_TYPE", getEnv("BACKEND", "disk")) - cacheDirDefault = getEnv("CACHE_DIR", filepath.Join(os.TempDir(), "gobuildcache", "cache")) - s3BucketDefault = getEnv("S3_BUCKET", "") - s3PrefixDefault = getEnv("S3_PREFIX", "") + clearFlags = flag.NewFlagSet("clear", flag.ExitOnError) + debugDefault = getEnvBool("DEBUG", false) + backendDefault = getEnv("BACKEND_TYPE", getEnv("BACKEND", "disk")) + cacheDirDefault = getEnv("CACHE_DIR", filepath.Join(os.TempDir(), "gobuildcache", "cache")) + s3BucketDefault = getEnv("S3_BUCKET", "") + s3PrefixDefault = getEnv("S3_PREFIX", "") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "") ) clearFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") - clearFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3 (env: BACKEND_TYPE)") + clearFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk (local only), s3, gcs (env: BACKEND_TYPE)") clearFlags.StringVar(&cacheDir, "cache-dir", cacheDirDefault, "Local cache directory (env: CACHE_DIR)") clearFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") clearFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + clearFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + clearFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object name prefix (optional) (env: GCS_PREFIX)") clearFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s clear [flags]\n\n", os.Args[0]) @@ -140,17 +154,20 @@ func runClearCommand() { fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") fmt.Fprintf(os.Stderr, " PRINT_STATS Print cache statistics on exit (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " CACHE_DIR Local cache directory\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") - fmt.Fprintf(os.Stderr, " S3_TMP_DIR Local temp directory for S3 backend\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object name prefix\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " # Clear disk cache using flags:\n") fmt.Fprintf(os.Stderr, " %s clear -cache-dir=/var/cache/go\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear S3 cache using flags:\n") fmt.Fprintf(os.Stderr, " %s clear -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Clear GCS cache using flags:\n") + fmt.Fprintf(os.Stderr, " %s clear -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear using environment variables:\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 S3_BUCKET=my-cache-bucket %s clear\n", os.Args[0]) } @@ -206,28 +223,36 @@ func runClearRemoteCommand() { backendDefault = getEnv("BACKEND_TYPE", getEnv("BACKEND", "disk")) s3BucketDefault = getEnv("S3_BUCKET", "") s3PrefixDefault = getEnv("S3_PREFIX", "") + gcsBucketDefault = getEnv("GCS_BUCKET", "") + gcsPrefixDefault = getEnv("GCS_PREFIX", "") ) clearRemoteFlags.BoolVar(&debug, "debug", debugDefault, "Enable debug logging to stderr (env: DEBUG)") - clearRemoteFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk, s3 (env: BACKEND_TYPE)") + clearRemoteFlags.StringVar(&backendType, "backend", backendDefault, "Backend type: disk, s3, gcs (env: BACKEND_TYPE)") clearRemoteFlags.StringVar(&s3Bucket, "s3-bucket", s3BucketDefault, "S3 bucket name (required for s3 backend) (env: S3_BUCKET)") clearRemoteFlags.StringVar(&s3Prefix, "s3-prefix", s3PrefixDefault, "S3 key prefix (optional) (env: S3_PREFIX)") + clearRemoteFlags.StringVar(&gcsBucket, "gcs-bucket", gcsBucketDefault, "GCS bucket name (required for gcs backend) (env: GCS_BUCKET)") + clearRemoteFlags.StringVar(&gcsPrefix, "gcs-prefix", gcsPrefixDefault, "GCS object name prefix (optional) (env: GCS_PREFIX)") clearRemoteFlags.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s clear-remote [flags]\n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Clear only the remote backend cache (e.g., S3).\n\n") + fmt.Fprintf(os.Stderr, "Clear only the remote backend cache (e.g., S3, GCS).\n\n") fmt.Fprintf(os.Stderr, "Flags (can also be set via environment variables):\n") clearRemoteFlags.PrintDefaults() fmt.Fprintf(os.Stderr, "\nEnvironment Variables:\n") fmt.Fprintf(os.Stderr, " DEBUG Enable debug logging (true/false)\n") - fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3)\n") + fmt.Fprintf(os.Stderr, " BACKEND_TYPE Backend type (disk, s3, gcs)\n") fmt.Fprintf(os.Stderr, " S3_BUCKET S3 bucket name\n") fmt.Fprintf(os.Stderr, " S3_PREFIX S3 key prefix\n") + fmt.Fprintf(os.Stderr, " GCS_BUCKET GCS bucket name\n") + fmt.Fprintf(os.Stderr, " GCS_PREFIX GCS object name prefix\n") fmt.Fprintf(os.Stderr, "\nNote: Command-line flags take precedence over environment variables.\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " # Clear S3 cache using flags:\n") fmt.Fprintf(os.Stderr, " %s clear-remote -backend=s3 -s3-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear S3 cache with prefix:\n") fmt.Fprintf(os.Stderr, " %s clear-remote -backend=s3 -s3-bucket=my-cache-bucket -s3-prefix=myproject/\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " # Clear GCS cache using flags:\n") + fmt.Fprintf(os.Stderr, " %s clear-remote -backend=gcs -gcs-bucket=my-cache-bucket\n\n", os.Args[0]) fmt.Fprintf(os.Stderr, " # Clear using environment variables:\n") fmt.Fprintf(os.Stderr, " BACKEND_TYPE=s3 S3_BUCKET=my-cache-bucket %s clear-remote\n", os.Args[0]) } @@ -350,8 +375,15 @@ func createBackend() (backends.Backend, error) { backend, err = backends.NewS3(s3Bucket, s3Prefix) + case "gcs": + if gcsBucket == "" { + return nil, fmt.Errorf("GCS bucket is required for GCS backend (set via -gcs-bucket flag or GCS_BUCKET env var)") + } + + backend, err = backends.NewGCS(gcsBucket, gcsPrefix) + default: - return nil, fmt.Errorf("unknown backend type: %s (supported: disk, s3)", backendType) + return nil, fmt.Errorf("unknown backend type: %s (supported: disk, s3, gcs)", backendType) } if err != nil { diff --git a/pkg/backends/gcs.go b/pkg/backends/gcs.go new file mode 100644 index 0000000..ffc8cad --- /dev/null +++ b/pkg/backends/gcs.go @@ -0,0 +1,205 @@ +package backends + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "strconv" + "time" + + "cloud.google.com/go/storage" +) + +// GCS implements Backend using Google Cloud Storage. +// This backend only handles GCS operations; local disk caching is handled by server.go. +type GCS struct { + client *storage.Client + bucket *storage.BucketHandle + bucketName string + prefix string + ctx context.Context +} + +// NewGCS creates a new GCS-based cache backend. +// bucket is the GCS bucket name where cache files will be stored. +// prefix is an optional prefix for all GCS object names (e.g., "cache/" or ""). +func NewGCS(bucket, prefix string) (*GCS, error) { + ctx := context.Background() + + // Create GCS client using Application Default Credentials + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %w", err) + } + + bucketHandle := client.Bucket(bucket) + + backend := &GCS{ + client: client, + bucket: bucketHandle, + bucketName: bucket, + prefix: prefix, + ctx: ctx, + } + + // Test bucket access + _, err = bucketHandle.Attrs(ctx) + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to access GCS bucket %s: %w", bucket, err) + } + + return backend, nil +} + +// Put stores an object in GCS. +func (g *GCS) Put(actionID, outputID []byte, body io.Reader, bodySize int64) error { + objectName := g.actionIDToObjectName(actionID) + + // Read the body into a buffer + var bodyData []byte + if bodySize > 0 && body != nil { + bodyData = make([]byte, bodySize) + n, err := io.ReadFull(body, bodyData) + if err != nil && err != io.EOF { + return fmt.Errorf("failed to read body: %w", err) + } + if int64(n) != bodySize { + return fmt.Errorf("size mismatch: expected %d, read %d", bodySize, n) + } + } + + // Prepare metadata + now := time.Now() + metadata := map[string]string{ + "outputid": hex.EncodeToString(outputID), + "size": strconv.FormatInt(bodySize, 10), + "time": strconv.FormatInt(now.Unix(), 10), + } + + // Create object writer + obj := g.bucket.Object(objectName) + writer := obj.NewWriter(g.ctx) + writer.Metadata = metadata + + // Write body data + if _, err := writer.Write(bodyData); err != nil { + writer.Close() + return fmt.Errorf("failed to write to GCS: %w", err) + } + + // Close the writer to complete the upload + if err := writer.Close(); err != nil { + return fmt.Errorf("failed to complete GCS upload: %w", err) + } + + return nil +} + +// Get retrieves an object from GCS. +// Returns the object data as an io.ReadCloser that must be closed by the caller. +func (g *GCS) Get(actionID []byte) ([]byte, io.ReadCloser, int64, *time.Time, bool, error) { + objectName := g.actionIDToObjectName(actionID) + obj := g.bucket.Object(objectName) + + // Get object attributes for metadata + attrs, err := obj.Attrs(g.ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil, 0, nil, true, nil + } + return nil, nil, 0, nil, true, fmt.Errorf("failed to get GCS object attrs: %w", err) + } + + // Parse metadata from attributes + // Note: GCS SDK automatically lowercases metadata keys + outputIDHex := attrs.Metadata["outputid"] + sizeStr := attrs.Metadata["size"] + timeStr := attrs.Metadata["time"] + + outputID, err := hex.DecodeString(outputIDHex) + if err != nil { + return nil, nil, 0, nil, true, nil + } + + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return nil, nil, 0, nil, true, nil + } + + putTimeUnix, err := strconv.ParseInt(timeStr, 10, 64) + if err != nil { + return nil, nil, 0, nil, true, nil + } + putTime := time.Unix(putTimeUnix, 0) + + // Get the object body + reader, err := obj.NewReader(g.ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil, 0, nil, true, nil + } + return nil, nil, 0, nil, true, fmt.Errorf("failed to read GCS object: %w", err) + } + + // Return the GCS object body as a ReadCloser + // The caller is responsible for closing it + return outputID, reader, size, &putTime, false, nil +} + +// Close performs cleanup operations. +// Unlike S3, GCS client holds resources that must be closed. +func (g *GCS) Close() error { + if g.client != nil { + return g.client.Close() + } + return nil +} + +// Clear removes all entries from the cache in GCS. +func (g *GCS) Clear() error { + // List all objects with the prefix and delete them + it := g.bucket.Objects(g.ctx, &storage.Query{ + Prefix: g.prefix, + }) + + for { + attrs, err := it.Next() + if errors.Is(err, storage.ErrBucketNotExist) { + return fmt.Errorf("bucket does not exist: %w", err) + } + if err != nil { + // Check for iterator exhaustion + if err.Error() == "no more items in iterator" { + break + } + // Check for actual end of iteration (iterator.Done) + if err == storage.ErrObjectNotExist { + break + } + // Handle the standard iterator done case + break + } + + // Delete the object + if err := g.bucket.Object(attrs.Name).Delete(g.ctx); err != nil { + // Ignore not found errors (object may have been deleted by another process) + if !errors.Is(err, storage.ErrObjectNotExist) { + return fmt.Errorf("failed to delete GCS object %s: %w", attrs.Name, err) + } + } + } + + return nil +} + +// actionIDToObjectName converts an actionID to a GCS object name. +func (g *GCS) actionIDToObjectName(actionID []byte) string { + hexID := hex.EncodeToString(actionID) + if g.prefix != "" { + return g.prefix + hexID + } + return hexID +}