From e3d90a3a3d82fc892152c4661183db8124861e2c Mon Sep 17 00:00:00 2001 From: Yunus Ganiyev Date: Wed, 11 Feb 2026 19:17:07 +0300 Subject: [PATCH 1/6] Refactor S3 wrapper to use `TransferManager` for uploads --- lib/cloud_storage/wrappers/s3.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 42d44f0..04473e1 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -47,9 +47,9 @@ def exist?(key) end def upload_file(key:, file:, **opts) - obj = resource.bucket(@bucket_name).object(key) + return unless upload_file_or_io(key, file, **opts) - return unless upload_file_or_io(obj, file, **opts) + obj = resource.bucket(@bucket_name).object(key) Objects::S3.new \ obj, @@ -93,11 +93,17 @@ def resource @resource ||= Aws::S3::Resource.new(@options) end - def upload_file_or_io(obj, file_or_io, **opts) + def transfer_manager + @transfer_manager ||= Aws::S3::TransferManager.new(client: client) + end + + def upload_file_or_io(key, file_or_io, **opts) if file_or_io.respond_to?(:path) - obj.upload_file(file_or_io.path, **opts) + transfer_manager.upload_file(file_or_io.path, bucket: @bucket_name, key: key, **opts) else - obj.upload_stream(**opts) { |write_stream| IO.copy_stream(file_or_io, write_stream) } + transfer_manager.upload_stream(bucket: @bucket_name, key: key, **opts) do |write_stream| + IO.copy_stream(file_or_io, write_stream) + end end end end From d5fb1c67d6b93bcccbd72f66a0877d04a006d4e4 Mon Sep 17 00:00:00 2001 From: Maxim Tretyakov Date: Thu, 12 Feb 2026 12:12:40 +0500 Subject: [PATCH 2/6] Actualize lib --- dip.yml | 3 +-- docker/Dockerfile.dip | 2 +- docker/docker-compose.yml | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dip.yml b/dip.yml index 6546cfb..a45b238 100644 --- a/dip.yml +++ b/dip.yml @@ -1,7 +1,7 @@ version: '2' environment: - DOCKER_RUBY_VERSION: 3.0 + DOCKER_RUBY_VERSION: 3.4 S3_ENDPOINT: http://s3:9000 S3_BUCKET: wallarm-devtmp-ipfeeds-presigned-urls-research GCS_ENDPOINT: http://gcs:8080/ @@ -26,7 +26,6 @@ interaction: rspec: service: app - environment: command: bundle exec rspec rubocop: diff --git a/docker/Dockerfile.dip b/docker/Dockerfile.dip index f4ad700..593b655 100644 --- a/docker/Dockerfile.dip +++ b/docker/Dockerfile.dip @@ -1,5 +1,5 @@ ARG DOCKER_RUBY_VERSION -FROM ruby:${DOCKER_RUBY_VERSION}-alpine +FROM ruby:${DOCKER_RUBY_VERSION:-3.4}-alpine RUN gem update --system RUN apk add --update --no-cache less git build-base curl mc htop diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3fa1eda..b5de9c2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,5 +40,5 @@ services: volumes: bundler-data: - external: - name: bundler_data + name: bundler_data + external: true From 33857820816eb8ef436b9ec216c6fa23ffbd8b05 Mon Sep 17 00:00:00 2001 From: Yunus Ganiyev Date: Thu, 12 Feb 2026 11:14:12 +0300 Subject: [PATCH 3/6] Fix test failures: update MinIO, handle S3 invalid bucket errors, fix GCS signed_url test --- docker/docker-compose.yml | 2 +- lib/cloud_storage/wrappers/s3.rb | 12 +++++++++--- spec/cloud_storage/objects/gcs_spec.rb | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3fa1eda..f97c757 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,7 +24,7 @@ services: - gcs s3: - image: minio/minio:edge + image: minio/minio:RELEASE.2025-04-22T22-12-26Z volumes: - ../uploads/s3:/data command: server /data --json diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 42d44f0..9281706 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -34,7 +34,7 @@ def each resource: @resource, client: @client end - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName end end @@ -44,6 +44,9 @@ def files(**opts) def exist?(key) resource.bucket(@bucket_name).object(key).exists? + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + false end def upload_file(key:, file:, **opts) @@ -56,7 +59,7 @@ def upload_file(key:, file:, **opts) bucket_name: @bucket_name, resource: resource, client: client - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName raise ObjectNotFound, @bucket_name end @@ -70,6 +73,9 @@ def find(key) bucket_name: @bucket_name, resource: resource, client: client + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + raise ObjectNotFound, key end def delete_files(keys) @@ -80,7 +86,7 @@ def delete_files(keys) objects: keys.map { |key| { key: key } }, quiet: true } - rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound + rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, Aws::S3::Errors::InvalidBucketName end private diff --git a/spec/cloud_storage/objects/gcs_spec.rb b/spec/cloud_storage/objects/gcs_spec.rb index 28d7a69..a2f66be 100644 --- a/spec/cloud_storage/objects/gcs_spec.rb +++ b/spec/cloud_storage/objects/gcs_spec.rb @@ -22,7 +22,7 @@ context 'when with some internal options' do subject(:url) { obj.signed_url(expires_in: 30, issuer: 'max@tretyakov-ma.ru', signing_key: key, version: :v2) } - it { is_expected.to match(%r{\Ahttps://storage.googleapis.com/#{ENV.fetch('GCS_BUCKET')}/test_1.txt}) } + it { is_expected.to match(/GoogleAccessId=/) } end end From 413801a277cc65e9e0a6feb0e768511bb9d07e93 Mon Sep 17 00:00:00 2001 From: Yunus Ganiyev Date: Thu, 12 Feb 2026 11:34:14 +0300 Subject: [PATCH 4/6] Add custom host assertion to GCS signed_url V2 test --- spec/cloud_storage/objects/gcs_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/cloud_storage/objects/gcs_spec.rb b/spec/cloud_storage/objects/gcs_spec.rb index a2f66be..b0a78c3 100644 --- a/spec/cloud_storage/objects/gcs_spec.rb +++ b/spec/cloud_storage/objects/gcs_spec.rb @@ -22,6 +22,7 @@ context 'when with some internal options' do subject(:url) { obj.signed_url(expires_in: 30, issuer: 'max@tretyakov-ma.ru', signing_key: key, version: :v2) } + it { is_expected.to match(%r{\A#{ENV.fetch('GCS_ENDPOINT')}#{ENV.fetch('GCS_BUCKET')}/test_1.txt}) } it { is_expected.to match(/GoogleAccessId=/) } end end From 90fea2e9c02f08d100b2bb8376886991a0b221e6 Mon Sep 17 00:00:00 2001 From: Maxim Tretyakov Date: Thu, 12 Feb 2026 13:42:05 +0500 Subject: [PATCH 5/6] Actualize dev env --- .pryrc | 4 ++-- .rubocop.yml | 2 +- Gemfile | 6 ++++++ cloud_storage.gemspec | 6 ------ lib/cloud_storage.rb | 3 ++- lib/cloud_storage/objects/gcs.rb | 2 +- lib/cloud_storage/objects/s3.rb | 2 +- lib/cloud_storage/wrappers/s3.rb | 4 ++-- 8 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.pryrc b/.pryrc index 222e0ff..593ecf6 100644 --- a/.pryrc +++ b/.pryrc @@ -6,8 +6,8 @@ require 'cloud_storage' require 'cloud_storage/wrappers/gcs' require 'cloud_storage/wrappers/s3' -require_relative './spec/rspec_helpers/gcs' -require_relative './spec/rspec_helpers/s3' +require_relative 'spec/rspec_helpers/gcs' +require_relative 'spec/rspec_helpers/s3' # rubocop:disable Style/MixinUsage include RSpecHelpers::Gcs diff --git a/.rubocop.yml b/.rubocop.yml index b907bcf..1050887 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,7 +1,7 @@ inherit_from: - .rubocop_todo.yml -require: +plugins: - rubocop-rspec - rubocop-performance diff --git a/Gemfile b/Gemfile index a500a47..69fadb2 100644 --- a/Gemfile +++ b/Gemfile @@ -11,3 +11,9 @@ gem 'rubocop-rspec', require: false gem 'aws-sdk-s3', require: false gem 'google-cloud-storage', require: false + +gem 'pry-byebug' +gem 'rake' +gem 'rspec' +gem 'simplecov' +gem 'webmock' diff --git a/cloud_storage.gemspec b/cloud_storage.gemspec index 770cafa..7280025 100644 --- a/cloud_storage.gemspec +++ b/cloud_storage.gemspec @@ -16,12 +16,6 @@ Gem::Specification.new do |spec| spec.metadata['homepage_uri'] = spec.homepage spec.metadata['source_code_uri'] = 'https://github.com/wallarm/cloud_storage' - spec.add_development_dependency 'pry-byebug' - spec.add_development_dependency 'rake' - spec.add_development_dependency 'rspec' - spec.add_development_dependency 'simplecov' - spec.add_development_dependency 'webmock' - spec.files = Dir.chdir(File.expand_path(__dir__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) } end diff --git a/lib/cloud_storage.rb b/lib/cloud_storage.rb index e36ce4a..007b603 100644 --- a/lib/cloud_storage.rb +++ b/lib/cloud_storage.rb @@ -6,7 +6,8 @@ require_relative 'cloud_storage/objects/base' module CloudStorage - ObjectNotFound = Class.new(StandardError) + class ObjectNotFound < StandardError + end class << self def register_wrapper(klass) diff --git a/lib/cloud_storage/objects/gcs.rb b/lib/cloud_storage/objects/gcs.rb index 866fe8b..e48f8b9 100644 --- a/lib/cloud_storage/objects/gcs.rb +++ b/lib/cloud_storage/objects/gcs.rb @@ -4,7 +4,7 @@ module CloudStorage module Objects class Gcs < Base def initialize(internal, uri:) - super internal + super(internal) @uri = uri end diff --git a/lib/cloud_storage/objects/s3.rb b/lib/cloud_storage/objects/s3.rb index af77471..1fa0ac8 100644 --- a/lib/cloud_storage/objects/s3.rb +++ b/lib/cloud_storage/objects/s3.rb @@ -6,7 +6,7 @@ class S3 < Base attr_reader :bucket_name def initialize(internal, resource:, client:, bucket_name:) - super internal + super(internal) @bucket_name = bucket_name @resource = resource diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 7ddba4f..34841a8 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -45,7 +45,7 @@ def files(**opts) def exist?(key) resource.bucket(@bucket_name).object(key).exists? rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, - Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest false end @@ -74,7 +74,7 @@ def find(key) resource: resource, client: client rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NotFound, - Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest + Aws::S3::Errors::InvalidBucketName, Aws::S3::Errors::BadRequest raise ObjectNotFound, key end From 20674f4141cadfc7a759a85a8bec19e20d634c99 Mon Sep 17 00:00:00 2001 From: Maxim Tretyakov Date: Thu, 12 Feb 2026 13:56:39 +0500 Subject: [PATCH 6/6] Add CI --- .github/workflows/ci.yaml | 58 ++++++++++++++++++++++++++++++++ .rubocop.yml | 1 + lib/cloud_storage/wrappers/s3.rb | 8 ++--- 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..74482a8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,58 @@ +name: CI + +on: + pull_request: + branches: ['*'] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby-version: ['3.0', '3.1', '3.4', '4.0'] + + env: + S3_ENDPOINT: http://localhost:9000 + S3_BUCKET: test + GCS_ENDPOINT: http://localhost:8080/ + GCS_BUCKET: some-bucket + + steps: + - uses: actions/checkout@v4 + + - name: Start MinIO + run: | + docker run -d --name minio \ + -p 9000:9000 \ + --entrypoint sh \ + minio/minio:RELEASE.2025-04-22T22-12-26Z \ + -c 'mkdir -p /data/test && minio server /data --json' + + - name: Start fake-gcs-server + run: | + docker run -d --name gcs \ + -p 8080:8080 \ + --entrypoint sh \ + fsouza/fake-gcs-server \ + -c 'mkdir -p /data/some-bucket && /bin/fake-gcs-server -port 8080 -scheme http -external-url=http://localhost:8080 -public-host=localhost:8080 -filesystem-root /data' + + - name: Wait for services + run: | + for i in $(seq 1 30); do + curl -sf http://localhost:9000/minio/health/live && curl -sf http://localhost:8080/ && break + sleep 1 + done + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - name: Rubocop + run: bundle exec rubocop --display-style-guide --extra-details + + - name: RSpec + run: bundle exec rspec diff --git a/.rubocop.yml b/.rubocop.yml index 1050887..4c3ae4a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -11,6 +11,7 @@ AllCops: Exclude: - 'bin/**/*' - 'uploads/**/*' + - 'vendor/**/*' Layout/LineLength: Max: 120 diff --git a/lib/cloud_storage/wrappers/s3.rb b/lib/cloud_storage/wrappers/s3.rb index 34841a8..73967ca 100644 --- a/lib/cloud_storage/wrappers/s3.rb +++ b/lib/cloud_storage/wrappers/s3.rb @@ -99,10 +99,6 @@ def resource @resource ||= Aws::S3::Resource.new(@options) end - def transfer_manager - @transfer_manager ||= Aws::S3::TransferManager.new(client: client) - end - def upload_file_or_io(key, file_or_io, **opts) if file_or_io.respond_to?(:path) transfer_manager.upload_file(file_or_io.path, bucket: @bucket_name, key: key, **opts) @@ -112,6 +108,10 @@ def upload_file_or_io(key, file_or_io, **opts) end end end + + def transfer_manager + @transfer_manager ||= Aws::S3::TransferManager.new(client: client) + end end end end