Add HTTP proxy support

- Add proxy option to Twingly::HTTP::Client initialization
- Configure Faraday to route requests through specified proxy if provided

Author: cequele

Gemfile

@@ -6,11 +6,15 @@ gemspec
 
 group :development, :test do
   gem "climate_control", "~> 0.1"
+  gem "rack", "~> 3.1"
+  gem "rack-proxy", "~> 0.7.7"
+  gem "rackup", "~> 2.2"
   gem "rake", "~> 12"
   gem "rspec", "~> 3"
   gem "rubocop", "~> 1.64.1"
   gem "rubocop-rspec", "~> 2.31.0"
   gem "toxiproxy", "~> 1.0"
   gem "vcr", "~> 5.0"
   gem "webmock", "~> 3.7"
+  gem "webrick", "~> 1.9"
 end

lib/twingly/http.rb

@@ -50,10 +50,11 @@ class Client # rubocop:disable Metrics/ClassLength
       attr_accessor :logger
       attr_accessor :retryable_exceptions
 
-      def initialize(base_user_agent:, logger: default_logger, user_agent: nil)
+      def initialize(base_user_agent:, logger: default_logger, user_agent: nil, proxy: nil)
         @base_user_agent = base_user_agent
         @logger          = logger
         @user_agent      = user_agent
+        @proxy           = proxy
 
         initialize_defaults
       end
@@ -212,6 +213,7 @@ def create_http_client # rubocop:disable Metrics/MethodLength
                            max_size_bytes: @max_response_body_size_bytes
           faraday.adapter Faraday.default_adapter
           faraday.headers[:user_agent] = user_agent
+          faraday.proxy = @proxy if @proxy
         end
       end
 

spec/lib/twingly/http_spec.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../../spec_help/http_test_server"
+
 class CustomError < StandardError; end
 
 # rubocop:disable RSpec/MultipleMemoizedHelpers
@@ -533,6 +535,31 @@ class CustomError < StandardError; end
     end
   end
 
+  RSpec.shared_examples "verifies proxy functionality" do
+    context "when a proxy is provided" do
+      let(:proxy_server)  { HttpTestServer.spawn("proxy_server") }
+      let(:target_server) { HttpTestServer.spawn("echoed_headers_in_body") }
+      let(:client) do
+        described_class.new(
+          base_user_agent: base_user_agent,
+          proxy: proxy_server.url
+        )
+      end
+      let(:url) { target_server.url }
+
+      after do
+        HttpTestServer.stop(proxy_server.pid)
+        HttpTestServer.stop(target_server.pid)
+      end
+
+      it "routes requests through the proxy", vcr: false do
+        with_real_http_connections do
+          expect(response.fetch(:body)).to include("HTTP_X_PROXIED_BY")
+        end
+      end
+    end
+  end
+
   describe "#initialize" do
     context "when no logger is given" do
       subject(:default_logger) do
@@ -552,6 +579,7 @@ class CustomError < StandardError; end
 
   describe "#post", vcr: Fixture.post_example_org do
     include_examples "common HTTP behaviour for", :post, "example.org"
+    include_examples "verifies proxy functionality"
 
     let(:post_body)    { nil }
     let(:post_headers) { {} }
@@ -600,6 +628,7 @@ class CustomError < StandardError; end
 
   describe "#get", vcr: Fixture.example_org do
     include_examples "common HTTP behaviour for", :get, "example.org"
+    include_examples "verifies proxy functionality"
 
     let(:request_response) do
       client.get(url)
@@ -692,6 +721,7 @@ class CustomError < StandardError; end
 
   describe "#put", vcr: Fixture.put_httpbin_org do
     include_examples "common HTTP behaviour for", :put, "https://httpbin.org/put"
+    include_examples "verifies proxy functionality"
 
     let(:url) { "https://httpbin.org/put" }
 
@@ -742,6 +772,7 @@ class CustomError < StandardError; end
 
   describe "#patch", vcr: Fixture.patch_httpbin_org do
     include_examples "common HTTP behaviour for", :patch, "https://httpbin.org/patch"
+    include_examples "verifies proxy functionality"
 
     let(:url) { "https://httpbin.org/patch" }
 
@@ -792,6 +823,7 @@ class CustomError < StandardError; end
 
   describe "#delete", vcr: Fixture.delete_httpbin_org do
     include_examples "common HTTP behaviour for", :delete, "https://httpbin.org/delete"
+    include_examples "verifies proxy functionality"
 
     let(:url) { "https://httpbin.org/delete" }
 

spec/rack_servers/echoed_headers_in_body.ru

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "json"
+
+run lambda { |env|
+  request_headers = env.select { |k, _v| k.start_with? "HTTP_" }
+
+  [200, { "content-type" => "application/json" }, [request_headers.to_json]]
+}

spec/rack_servers/proxy_server.ru

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rack/proxy"
+
+class TestProxy < Rack::Proxy
+  def rewrite_env(env)
+    env["HTTP_X_PROXIED_BY"] = "test-proxy"
+    env
+  end
+end
+
+run TestProxy.new

spec/spec_help/http_helpers.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module HttpHelpers
+  def with_real_http_connections
+    original_cassette = VCR.eject_cassette
+    VCR.turn_off!
+    WebMock.allow_net_connect!
+
+    yield
+  ensure
+    WebMock.disable_net_connect!
+    VCR.turn_on!
+    VCR.insert_cassette(original_cassette.name) if original_cassette
+  end
+end

spec/spec_help/http_test_server.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module HttpTestServer
+  module_function
+
+  TestServer = Struct.new(:pid, :url)
+
+  def spawn(server_name, env: {}) # rubocop:disable Metrics/MethodLength
+    ip_address = PortProber.localhost
+    port = PortProber.random(ip_address)
+    url = "http://#{ip_address}:#{port}"
+    server = "spec/rack_servers/#{server_name}.ru"
+    command = "bundle exec rackup --quiet --port #{port} #{server}"
+
+    puts "starting HTTP test server: #{command}"
+    pid = fork do
+      $stdout.reopen File::NULL
+      $stderr.reopen File::NULL
+      exec env, command
+    end
+
+    Timeout.timeout(10.0) do
+      sleep 0.05 until started?(pid) && PortProber.port_open?(ip_address, port)
+    end
+
+    TestServer.new(pid, url)
+  end
+
+  def stop(pid)
+    Process.kill(:TERM, pid)
+    Process.wait(pid)
+  end
+
+  def started?(pid)
+    Process.getpgid(pid)
+    true
+  rescue Errno::ESRCH
+    false
+  end
+end

spec/spec_help/port_prober.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "socket"
+require "timeout"
+
+module PortProber
+  module_function
+
+  def random(host)
+    server = TCPServer.new(host, 0)
+    port   = server.addr[1]
+
+    port
+  ensure
+    server&.close
+  end
+
+  def port_open?(ip_address, port)
+    Timeout.timeout(0.5) do
+      TCPSocket.new(ip_address, port).close
+      true
+    end
+  rescue StandardError
+    false
+  end
+
+  def localhost
+    info = Socket.getaddrinfo("localhost",
+                              80,
+                              Socket::AF_INET,
+                              Socket::SOCK_STREAM)
+
+    raise "unable to translate 'localhost' for TCP + IPv4" if info.empty?
+
+    info[0][3]
+  end
+end

spec/spec_helper.rb

@@ -14,8 +14,10 @@
 
 require_relative "spec_help/env_helper"
 require_relative "spec_help/fixture"
+require_relative "spec_help/http_helpers"
 require_relative "spec_help/test_logger"
 require_relative "spec_help/toxiproxy_config"
+require_relative "spec_help/port_prober"
 
 # Start with a clean slate, destroy all proxies if any
 Toxiproxy.all.destroy
@@ -33,6 +35,7 @@
 
 RSpec.configure do |conf|
   conf.include EnvHelper
+  conf.include HttpHelpers
 
   conf.after(:suite) do
     Toxiproxy.all.destroy # Be nice, end with a clean slate