From 066ab7657a86012226ddc02d79dcc90584a9d3d5 Mon Sep 17 00:00:00 2001 From: David Sabeti and Ketan Deshpande Date: Mon, 5 Jan 2015 11:20:30 -0800 Subject: [PATCH 01/76] Rename ServiceInstanceUnbinder#unbind => ServiceInstanceUnbinder#delayed_unbind [#83710826] --- lib/services/service_brokers/v2/client.rb | 2 +- .../service_brokers/v2/service_instance_unbinder.rb | 2 +- spec/unit/lib/services/service_brokers/v2/client_spec.rb | 4 ++-- .../service_brokers/v2/service_instance_unbinder_spec.rb | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 9f6cc2da4ee..0e41de0bd41 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -127,7 +127,7 @@ def bind(binding) end rescue ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e - VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder.unbind(@attrs, binding) + VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder.delayed_unbind(@attrs, binding) raise e end diff --git a/lib/services/service_brokers/v2/service_instance_unbinder.rb b/lib/services/service_brokers/v2/service_instance_unbinder.rb index 611666fa512..f371da9decb 100644 --- a/lib/services/service_brokers/v2/service_instance_unbinder.rb +++ b/lib/services/service_brokers/v2/service_instance_unbinder.rb @@ -4,7 +4,7 @@ module VCAP::CloudController module ServiceBrokers module V2 class ServiceInstanceUnbinder - def self.unbind(client_attrs, binding) + def self.delayed_unbind(client_attrs, binding) unbind_job = VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind.new( 'service-instance-unbind', client_attrs, diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 499d66b70db..09c6cd34c2d 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -716,7 +716,7 @@ module VCAP::Services::ServiceBrokers::V2 end before do - allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder).to receive(:unbind) + allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder).to receive(:delayed_unbind) end context 'when http_client make request fails with ServiceBrokerApiTimeout' do @@ -732,7 +732,7 @@ module VCAP::Services::ServiceBrokers::V2 }.to raise_error(ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:unbind). + to have_received(:delayed_unbind). with(client_attrs, binding) end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb index 228c95fea38..f087008eb6c 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb @@ -11,9 +11,9 @@ module ServiceBrokers::V2 end let(:name) { 'fake-name' } - describe 'unbind' do + describe 'delayed_unbind' do it 'creates a ServiceInstanceUnbind Job' do - job = ServiceInstanceUnbinder.unbind(client_attrs, binding) + job = ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) expect(job).to be_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind) expect(job.client_attrs).to be(client_attrs) expect(job.binding_guid).to be(binding.guid) @@ -24,7 +24,7 @@ module ServiceBrokers::V2 it 'enqueues a ServiceInstanceUnbind Job' do expect(Delayed::Job).to receive(:enqueue).with(an_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind), hash_including(queue: 'cc-generic')) - ServiceInstanceUnbinder.unbind(client_attrs, binding) + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) end end end From 17ea372a3fa9243b1e65d10d3ea5aa6599d89568 Mon Sep 17 00:00:00 2001 From: David Sabeti and Ketan Deshpande Date: Mon, 5 Jan 2015 14:35:59 -0800 Subject: [PATCH 02/76] Modify test to ensure that error is propagated in service_binding_spec [#83710826] --- spec/unit/models/services/service_binding_spec.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/unit/models/services/service_binding_spec.rb b/spec/unit/models/services/service_binding_spec.rb index cc2d3a47a07..5c2b83e3d1e 100644 --- a/spec/unit/models/services/service_binding_spec.rb +++ b/spec/unit/models/services/service_binding_spec.rb @@ -74,12 +74,13 @@ module VCAP::CloudController end context 'when unbind fails' do - before { allow(binding.client).to receive(:unbind).and_raise } + let(:error) { RuntimeError.new('Some error') } + before { allow(binding.client).to receive(:unbind).and_raise(error) } - it 'raises an error and rolls back' do + it 'propagates the error and rolls back' do expect { binding.destroy - }.to raise_error + }.to raise_error(error) expect(binding).to be_exists end From 98d0abab5e83198c40fd03c51abac5bd6166efef Mon Sep 17 00:00:00 2001 From: David Sabeti and Ketan Deshpande Date: Mon, 5 Jan 2015 15:19:50 -0800 Subject: [PATCH 03/76] Add standard error handling tests to update_service_plan method --- .../service_brokers/v2/client_spec.rb | 68 +++++++++++-------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 09c6cd34c2d..0055ebaa242 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -554,9 +554,15 @@ module VCAP::Services::ServiceBrokers::V2 let(:service_plan_guid) { new_plan.guid } let(:path) { "/v2/service_instances/#{instance.guid}/" } + let(:code) { 200 } + let(:message) { 'OK' } + let(:response_data) { '{}' } + + before do + allow(http_client).to receive(:patch).and_return(double('response', code: code, body: response_data, message: message)) + end it 'makes a patch request with the new service plan' do - allow(http_client).to receive(:patch).and_return(double('response', code: 200, body: '{}')) client.update_service_plan(instance, new_plan) expect(http_client).to have_received(:patch).with( @@ -574,46 +580,50 @@ module VCAP::Services::ServiceBrokers::V2 end it 'makes a patch request to the correct path' do - allow(http_client).to receive(:patch).and_return(double('response', code: 200, body: '{}')) - client.update_service_plan(instance, new_plan) expect(http_client).to have_received(:patch).with(path, anything) end describe 'error handling' do - before do - fake_response = double('response', code: status_code, body: body) - allow(http_client).to receive(:patch).and_return(fake_response) + it_behaves_like 'handles standard error conditions' do + let(:operation) { client.update_service_plan(instance, new_plan) } end - context 'when the broker returns a 400' do - let(:status_code) { '400' } - let(:body) { { description: 'the request was malformed' }.to_json } - it 'raises a ServiceBrokerBadResponse error' do - expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /the request was malformed/ - ) + describe 'non-standard errors' do + before do + fake_response = double('response', code: status_code, body: body) + allow(http_client).to receive(:patch).and_return(fake_response) end - end - context 'when the broker returns a 404' do - let(:status_code) { '404' } - let(:body) { { description: 'service instance not found' }.to_json } - it 'raises a ServiceBrokerBadRequest error' do - expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /service instance not found/ - ) + context 'when the broker returns a 400' do + let(:status_code) { '400' } + let(:body) { { description: 'the request was malformed' }.to_json } + it 'raises a ServiceBrokerBadResponse error' do + expect { client.update_service_plan(instance, new_plan) }.to raise_error( + ServiceBrokerBadResponse, /the request was malformed/ + ) + end end - end - context 'when the broker returns a 422' do - let(:status_code) { '422' } - let(:body) { { description: 'cannot update to this plan' }.to_json } - it 'raises a ServiceBrokerBadResponse error' do - expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /cannot update to this plan/ - ) + context 'when the broker returns a 404' do + let(:status_code) { '404' } + let(:body) { { description: 'service instance not found' }.to_json } + it 'raises a ServiceBrokerBadRequest error' do + expect { client.update_service_plan(instance, new_plan) }.to raise_error( + ServiceBrokerBadResponse, /service instance not found/ + ) + end + end + + context 'when the broker returns a 422' do + let(:status_code) { '422' } + let(:body) { { description: 'cannot update to this plan' }.to_json } + it 'raises a ServiceBrokerBadResponse error' do + expect { client.update_service_plan(instance, new_plan) }.to raise_error( + ServiceBrokerBadResponse, /cannot update to this plan/ + ) + end end end end From 60ab3a7d0ece33bd538de6a616f980df7d4a0954 Mon Sep 17 00:00:00 2001 From: David Sabeti and Ketan Deshpande Date: Mon, 5 Jan 2015 18:05:00 -0800 Subject: [PATCH 04/76] Move response parsing logic to ResponseParser object We also removed tests from client_spec that tested error handling in favor of testing that logic in the response_parser_spec --- lib/services/service_brokers/v2.rb | 1 + lib/services/service_brokers/v2/client.rb | 58 +-- .../service_brokers/v2/response_parser.rb | 62 +++ .../service_brokers/v2/client_spec.rb | 381 ++++-------------- .../v2/response_parser_spec.rb | 116 ++++++ 5 files changed, 265 insertions(+), 353 deletions(-) create mode 100644 lib/services/service_brokers/v2/response_parser.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb diff --git a/lib/services/service_brokers/v2.rb b/lib/services/service_brokers/v2.rb index b756d515b9f..5bc13214bca 100644 --- a/lib/services/service_brokers/v2.rb +++ b/lib/services/service_brokers/v2.rb @@ -8,3 +8,4 @@ module VCAP::Services::ServiceBrokers::V2 end require 'services/service_brokers/v2/client' require 'services/service_brokers/v2/service_instance_deprovisioner' require 'services/service_brokers/v2/service_instance_unbinder' +require 'services/service_brokers/v2/response_parser' diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 0e41de0bd41..70799dfae42 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -81,12 +81,13 @@ class Client def initialize(attrs) @http_client = VCAP::Services::ServiceBrokers::V2::HttpClient.new(attrs) + @response_parser = VCAP::Services::ServiceBrokers::V2::ResponseParser.new(@http_client.url) @attrs = attrs end def catalog response = @http_client.get(CATALOG_PATH) - parse_response(:get, CATALOG_PATH, response) + @response_parser.parse(:get, CATALOG_PATH, response) end # The broker is expected to guarantee uniqueness of instance_id. @@ -101,7 +102,7 @@ def provision(instance) space_guid: instance.space.guid, }) - parsed_response = parse_response(:put, path, response) + parsed_response = @response_parser.parse(:put, path, response) instance.dashboard_url = parsed_response['dashboard_url'] # DEPRECATED, but needed because of not null constraint @@ -119,7 +120,7 @@ def bind(binding) plan_id: binding.service_plan.broker_provided_id, app_guid: binding.app_guid }) - parsed_response = parse_response(:put, path, response) + parsed_response = @response_parser.parse(:put, path, response) binding.credentials = parsed_response['credentials'] if parsed_response.key?('syslog_drain_url') @@ -139,7 +140,7 @@ def unbind(binding) plan_id: binding.service_plan.broker_provided_id, }) - parse_response(:delete, path, response) + @response_parser.parse(:delete, path, response) end def deprovision(instance) @@ -150,7 +151,7 @@ def deprovision(instance) plan_id: instance.service_plan.broker_provided_id, }) - parse_response(:delete, path, response) + @response_parser.parse(:delete, path, response) rescue VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict => e raise VCAP::Errors::ApiError.new_from_details('ServiceInstanceDeprovisionFailed', e.message) @@ -169,56 +170,11 @@ def update_service_plan(instance, plan) } }) - parse_response(:put, path, response) + @response_parser.parse(:put, path, response) end private - def uri_for(path) - URI(@http_client.url + path) - end - - def parse_response(method, path, response) - uri = uri_for(path) - code = response.code.to_i - - case code - when 204 - return nil # no body - - when 200..299 - - begin - response_hash = MultiJson.load(response.body) - rescue MultiJson::ParseError - logger.warn("MultiJson parse error `#{response.try(:body).inspect}'") - end - - unless response_hash.is_a?(Hash) - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) - end - - return response_hash - - when HTTP::Status::UNAUTHORIZED - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) - - when 408 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new(uri.to_s, method, response) - - when 409 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) - - when 410 - if method == :delete - logger.warn("Already deleted: #{uri}") - return nil - end - end - - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri.to_s, method, response) - end - def logger @logger ||= Steno.logger('cc.service_broker.v2.client') end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb new file mode 100644 index 00000000000..7bc795c2f8d --- /dev/null +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -0,0 +1,62 @@ +module VCAP::Services + module ServiceBrokers + module V2 + class ResponseParser + def initialize(url) + @url = url + end + + def parse(method, path, response) + uri = uri_for(path) + code = response.code.to_i + + case code + when 204 + return nil # no body + + when 200..299 + + begin + response_hash = MultiJson.load(response.body) + rescue MultiJson::ParseError + logger.warn("MultiJson parse error `#{response.try(:body).inspect}'") + end + + unless response_hash.is_a?(Hash) + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) + end + + return response_hash + + when HTTP::Status::UNAUTHORIZED + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) + + when 408 + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new(uri.to_s, method, response) + + when 409 + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) + + when 410 + if method == :delete + logger.warn("Already deleted: #{uri.to_s}") + return nil + end + end + + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri.to_s, method, response) + end + + private + + def uri_for(path) + URI(@url + path) + end + + def logger + @logger ||= Steno.logger('cc.service_broker.v2.client') + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 0055ebaa242..43c9573d19a 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -165,76 +165,6 @@ module VCAP::Services::ServiceBrokers::V2 allow(http_client).to receive(:url).and_return(service_broker.broker_url) end - shared_examples 'handles standard error conditions' do - context 'when the API returns an error code' do - let(:response_data) { { 'foo' => 'bar' } } - let(:code) { '500' } - let(:message) { 'Internal Server Error' } - - it 'should raise a ServiceBrokerBadResponse' do - expect { - operation - }.to raise_error { |e| - expect(e).to be_a(ServiceBrokerBadResponse) - error_hash = e.to_h - expect(error_hash.fetch('description')).to eq("The service broker API returned an error from #{service_broker.broker_url}#{path}: 500 Internal Server Error") - expect(error_hash.fetch('source')).to include({ 'foo' => 'bar' }) - } - end - end - - context 'when the API returns an invalid response' do - context 'because of an unexpected status code' do - let(:code) { '404' } - let(:message) { 'Not Found' } - - it 'should raise an invalid response error' do - expect { - operation - }.to raise_error( - ServiceBrokerBadResponse, - "The service broker API returned an error from #{service_broker.broker_url}#{path}: 404 Not Found" - ) - end - end - - context 'because of a response that does not return a valid hash' do - let(:response_data) { [] } - - it 'should raise an invalid response error' do - expect { - operation - }.to raise_error(ServiceBrokerResponseMalformed) - end - end - - context 'because of an invalid JSON body' do - let(:response_data) { 'invalid' } - - it 'should raise an invalid response error' do - expect { - operation - }.to raise_error(ServiceBrokerResponseMalformed) - end - end - end - - context 'when the API cannot authenticate the client' do - let(:code) { '401' } - - it 'should raise an authentication error' do - expect { - operation - }.to raise_error { |e| - expect(e).to be_a(ServiceBrokerApiAuthenticationFailed) - error_hash = e.to_h - expect(error_hash.fetch('description')). - to eq("Authentication failed for the service broker API. Double-check that the username and password are correct: #{service_broker.broker_url}#{path}") - } - end - end - end - describe '#catalog' do let(:service_id) { Sham.guid } let(:service_name) { Sham.name } @@ -279,12 +209,6 @@ module VCAP::Services::ServiceBrokers::V2 it 'returns a catalog' do expect(client.catalog).to eq(response_data) end - - describe 'handling errors' do - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.catalog } - end - end end describe '#provision' do @@ -351,81 +275,38 @@ module VCAP::Services::ServiceBrokers::V2 expect(instance.credentials).to eq({}) end - describe 'error handling' do - context 'the instance_id is already in use' do - let(:code) { '409' } - - it 'raises ServiceBrokerConflict' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerConflict) - end - end - - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.provision(instance) } - end - end - context 'when provision fails' do + let(:uri) { 'some-uri.com/v2/service_instances/some-guid' } + let(:response) { double(:response, body: nil, message: nil)} + before do allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner).to receive(:deprovision) end - context 'and http client response is 408' do - before do - allow(response).to receive(:code).and_return('408', '200') - end - - it 'raises ServiceBrokerApiTimeout and deprovisions' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerApiTimeout) - - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision).with(client_attrs, instance) - end - end - - context 'and http client response is 5xx' do - context 'and http error code is 500' do - before do - allow(response).to receive(:code).and_return('500', '200') - end - - it 'raises ServiceBrokerBadResponse and deprovisions' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + context 'due to an http client error' do + let(:http_client) { double(:http_client) } - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). - with(client_attrs, instance) - end + before do + allow(http_client).to receive(:put).and_raise(error) end - context 'and http error code is 501' do - before do - allow(response).to receive(:code).and_return('501', '200') - end + context 'ServiceBrokerApiTimeout error' do + let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } - it 'raises ServiceBrokerBadResponse and deprovisions' do + it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). - with(client_attrs, instance) + to have_received(:deprovision).with(client_attrs, instance) end end - context 'and http error code is 502' do - before do - allow(response).to receive(:code).and_return('502', '200') - end + context 'ServiceBrokerBadResponse error' do + let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } - it 'raises ServiceBrokerBadResponse and deprovisions' do + it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) }.to raise_error(ServiceBrokerBadResponse) @@ -435,32 +316,23 @@ module VCAP::Services::ServiceBrokers::V2 with(client_attrs, instance) end end + end - context 'and http error code is 503' do - before do - allow(response).to receive(:code).and_return('503', '200') - end - - it 'raises ServiceBrokerBadResponse and deprovisions' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + context 'due to a response parser error' do + let(:response_parser) { double(:response_parser) } - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). - with(client_attrs, instance) - end + before do + allow(response_parser).to receive(:parse).and_raise(error) + allow(VCAP::Services::ServiceBrokers::V2::ResponseParser).to receive(:new).and_return(response_parser) end - context 'and http error code is 504' do - before do - allow(response).to receive(:code).and_return('504', '200') - end + context 'ServiceBrokerApiTimeout error' do + let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } - it 'raises ServiceBrokerBadResponse and deprovisions' do + it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). to have_received(:deprovision). @@ -468,12 +340,10 @@ module VCAP::Services::ServiceBrokers::V2 end end - context 'and http error code is 505' do - before do - allow(response).to receive(:code).and_return('505', '200') - end + context 'ServiceBrokerBadResponse error' do + let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } - it 'raises ServiceBrokerBadResponse and deprovisions' do + it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) }.to raise_error(ServiceBrokerBadResponse) @@ -485,58 +355,6 @@ module VCAP::Services::ServiceBrokers::V2 end end end - - context 'when provision takes longer than broker configured timeout' do - before do - allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner).to receive(:deprovision) - end - - context 'when http_client make request fails with ServiceBrokerApiTimeout' do - before do - allow(http_client).to receive(:put) do |path, message| - raise ServiceBrokerApiTimeout.new(path, :put, Timeout::Error.new(message)) - end - end - - it 'deprovisions the instance' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerApiTimeout) - - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). - with(client_attrs, instance) - end - end - - context 'when http_client make request fails with ServiceBrokerApiUnreachable' do - before do - allow(http_client).to receive(:put) do |path, message| - raise ServiceBrokerApiUnreachable.new(path, :put, Errno::ECONNREFUSED) - end - end - - it 'fails' do - expect { - client.provision(instance) - }.to raise_error(ServiceBrokerApiUnreachable) - end - end - - context 'when http_client make request fails with HttpRequestError' do - before do - allow(http_client).to receive(:put) do |path, message| - raise HttpRequestError.new(message, path, :put, Exception.new(message)) - end - end - - it 'fails' do - expect { - client.provision(instance) - }.to raise_error(HttpRequestError) - end - end - end end describe '#update_service_plan' do @@ -586,36 +404,12 @@ module VCAP::Services::ServiceBrokers::V2 end describe 'error handling' do - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.update_service_plan(instance, new_plan) } - end - describe 'non-standard errors' do before do fake_response = double('response', code: status_code, body: body) allow(http_client).to receive(:patch).and_return(fake_response) end - context 'when the broker returns a 400' do - let(:status_code) { '400' } - let(:body) { { description: 'the request was malformed' }.to_json } - it 'raises a ServiceBrokerBadResponse error' do - expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /the request was malformed/ - ) - end - end - - context 'when the broker returns a 404' do - let(:status_code) { '404' } - let(:body) { { description: 'service instance not found' }.to_json } - it 'raises a ServiceBrokerBadRequest error' do - expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /service instance not found/ - ) - end - end - context 'when the broker returns a 422' do let(:status_code) { '422' } let(:body) { { description: 'cannot update to this plan' }.to_json } @@ -718,77 +512,88 @@ module VCAP::Services::ServiceBrokers::V2 end end - context 'when bind takes longer than broker configured timeout' do + context 'when binding fails' do let(:binding) do VCAP::CloudController::ServiceBinding.make( binding_options: { 'this' => 'that' } ) end + let(:uri) { 'some-uri.com/v2/service_instances/instance-guid/service_bindings/binding-guid'} + let(:response) { double(:response, body: nil, message: nil)} before do allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder).to receive(:delayed_unbind) end - context 'when http_client make request fails with ServiceBrokerApiTimeout' do + context 'due to an http client error' do + let(:http_client) { double(:http_client) } + before do - allow(http_client).to receive(:put) do |path, message| - raise ServiceBrokerApiTimeout.new(path, :put, Timeout::Error.new(message)) - end + allow(http_client).to receive(:put).and_raise(error) end - it 'unbinds the binding' do - expect { - client.bind(binding) - }.to raise_error(ServiceBrokerApiTimeout) + context 'ServiceBrokerApiTimeout error' do + let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } + + it 'propagates the error and follows up with a deprovision request' do + expect { + client.bind(binding) + }.to raise_error(ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). to have_received(:delayed_unbind). with(client_attrs, binding) - end - end - context 'when http_client make request fails with ServiceBrokerApiUnreachable' do - before do - allow(http_client).to receive(:put) do |path, message| - raise ServiceBrokerApiUnreachable.new(path, :put, Errno::ECONNREFUSED) end end - it 'fails' do - expect { - client.bind(binding) - }.to raise_error(ServiceBrokerApiUnreachable) + context 'ServiceBrokerBadResponse error' do + let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + + it 'propagates the error and follows up with a deprovision request' do + expect { + client.bind(binding) + }.to raise_error(ServiceBrokerBadResponse) + + expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). + to have_received(:delayed_unbind).with(client_attrs, binding) + end end end - context 'when http_client make request fails with HttpRequestError' do + context 'due to a response parser error' do + let(:response_parser) { double(:response_parser) } + before do - allow(http_client).to receive(:put) do |path, message| - raise HttpRequestError.new(message, path, :put, Exception.new(message)) - end + allow(response_parser).to receive(:parse).and_raise(error) + allow(VCAP::Services::ServiceBrokers::V2::ResponseParser).to receive(:new).and_return(response_parser) end - it 'fails' do - expect { - client.bind(binding) - }.to raise_error(HttpRequestError) - end - end - end + context 'ServiceBrokerApiTimeout error' do + let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } - describe 'error handling' do - context 'the binding id is already in use' do - let(:code) { '409' } + it 'propagates the error and follows up with a deprovision request' do + expect { + client.bind(binding) + }.to raise_error(ServiceBrokerApiTimeout) - it 'raises ServiceBrokerConflict' do - expect { - client.bind(binding) - }.to raise_error(ServiceBrokerConflict) + expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). + to have_received(:delayed_unbind).with(client_attrs, binding) + end end - end - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.bind(binding) } + context 'ServiceBrokerBadResponse error' do + let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + + it 'propagates the error and follows up with a deprovision request' do + expect { + client.bind(binding) + }.to raise_error(ServiceBrokerBadResponse) + + expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). + to have_received(:delayed_unbind).with(client_attrs, binding) + end + end end end end @@ -843,20 +648,6 @@ module VCAP::Services::ServiceBrokers::V2 expect { client.unbind(binding) }.to_not raise_error end end - - describe 'handling errors' do - context 'when the API returns 410' do - let(:code) { '410' } - - it 'should swallow the error' do - expect(client.unbind(binding)).to be_nil - end - end - - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.unbind(binding) } - end - end end describe '#deprovision' do @@ -896,20 +687,6 @@ module VCAP::Services::ServiceBrokers::V2 } ) end - - describe 'handling errors' do - context 'when the API returns 410' do - let(:code) { '410' } - - it 'should swallow the error' do - expect(client.deprovision(instance)).to be_nil - end - end - - it_behaves_like 'handles standard error conditions' do - let(:operation) { client.deprovision(instance) } - end - end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb new file mode 100644 index 00000000000..ef3e488f13a --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -0,0 +1,116 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + describe ResponseParser do + let(:url) { 'my.service-broker.com' } + subject(:parser) { ResponseParser.new(url) } + + let(:logger) { double(:logger, warn: nil) } + before do + allow(Steno).to receive(:logger).and_return(logger) + end + + describe '#parse' do + let(:response) { double(:response, body: body, code: code) } + let(:body) { '{"foo": "bar"}' } + let(:code) { 200 } + + it 'returns the response body hash' do + response_hash = parser.parse(:get, '/v2/catalog', response) + expect(response_hash).to eq({'foo' => 'bar'}) + end + + context 'when the status code is 204' do + let(:code) { 204 } + it 'returns a nil response' do + response_hash = parser.parse(:get, '/v2/catalog', response) + expect(response_hash).to be_nil + end + end + + context 'when the body cannot be json parsed' do + let(:body) { '{ "not json" }' } + + it 'raises a MalformedResponse error' do + expect { + parser.parse(:get, '/v2/catalog', response) + }.to raise_error(ServiceBrokerResponseMalformed) + expect(logger).to have_received(:warn).with(/MultiJson parse error/) + end + end + + context 'when the body is JSON, but not a hash' do + let(:body) { '["just", "a", "list"]' } + + it 'raises a MalformedResponse error' do + expect { + parser.parse(:get, '/v2/catalog', response) + }.to raise_error(ServiceBrokerResponseMalformed) + expect(logger).not_to have_received(:warn) + end + end + + context 'when the status code is HTTP Unauthorized (401)' do + let(:code) { 401 } + it 'raises a ServiceBrokerApiAuthenticationFailed error' do + expect { + parser.parse(:get, '/v2/catalog', response) + }.to raise_error(ServiceBrokerApiAuthenticationFailed) + end + end + + context 'when the status code is HTTP Request Timeout (408)' do + let(:code) { 408 } + it 'raises a ServiceBrokerApiTimeout error' do + expect { + parser.parse(:get, '/v2/catalog', response) + }.to raise_error(ServiceBrokerApiTimeout) + end + end + + context 'when the status code is HTTP Conflict (409)' do + let(:code) { 409 } + it 'raises a ServiceBrokerConflict error' do + expect { + parser.parse(:get, '/v2/catalog', response) + }.to raise_error(ServiceBrokerConflict) + end + end + + context 'when the status code is HTTP Gone (410)' do + let(:code) { 410 } + let(:method) { :get } + let(:body) { '{"description": "there was an error"}' } + it 'raises ServiceBrokerBadResponse' do + expect { + parser.parse(method, '/v2/catalog', response) + }.to raise_error(ServiceBrokerBadResponse, /there was an error/) + end + + context 'and the http method is delete' do + let(:method) { :delete } + it 'does not raise an error and logs a warning' do + response_hash = parser.parse(method, '/v2/catalog', response) + expect(response_hash).to be_nil + expect(logger).to have_received(:warn).with(/Already deleted/) + end + end + end + + context 'when the status code is any other error (unhandled 4xx or 5xx)' do + let(:code) { 500 } + let(:method) { :get } + let(:body) { '{"description": "there was an error"}' } + it 'raises ServiceBrokerBadResponse' do + expect { + parser.parse(method, '/v2/catalog', response) + }.to raise_error(ServiceBrokerBadResponse, /there was an error/) + end + end + end + end + end + end +end From 27d8f316b3f523ad05c0a7136810a0ab395369a8 Mon Sep 17 00:00:00 2001 From: David Sabeti and Ketan Deshpande Date: Tue, 6 Jan 2015 15:05:23 -0800 Subject: [PATCH 05/76] Add retryable jobs [#83710826] --- app/jobs/retryable_job.rb | 25 ++++++++ lib/cloud_controller/jobs.rb | 1 + spec/unit/jobs/retryable_job_spec.rb | 85 ++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 app/jobs/retryable_job.rb create mode 100644 spec/unit/jobs/retryable_job_spec.rb diff --git a/app/jobs/retryable_job.rb b/app/jobs/retryable_job.rb new file mode 100644 index 00000000000..5b3d446fa4d --- /dev/null +++ b/app/jobs/retryable_job.rb @@ -0,0 +1,25 @@ +module VCAP::CloudController + module Jobs + class RetryableJob + attr_reader :job, :num_attempts + + def initialize(job, num_attempts = 0) + @job = job + @num_attempts = num_attempts + end + + def perform + job.perform + rescue VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse => e + raise e if num_attempts >= 10 + Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2 ** num_attempts).minutes) + end + + def max_attempts + # We don't want DelayedJob to handle the retry logic, because we only want to perform + # retries for specific failures. We'll handle retry and num_attempts separately. + 1 + end + end + end +end diff --git a/lib/cloud_controller/jobs.rb b/lib/cloud_controller/jobs.rb index 77ca951b7a1..64a5f8fc42f 100644 --- a/lib/cloud_controller/jobs.rb +++ b/lib/cloud_controller/jobs.rb @@ -17,3 +17,4 @@ require 'jobs/request_job' require 'jobs/timeout_job' require 'jobs/local_queue' +require 'jobs/retryable_job' diff --git a/spec/unit/jobs/retryable_job_spec.rb b/spec/unit/jobs/retryable_job_spec.rb new file mode 100644 index 00000000000..eb92803ef60 --- /dev/null +++ b/spec/unit/jobs/retryable_job_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +module VCAP::CloudController + module Jobs + describe "RetryableJob" do + describe '#perform' do + let(:job) { double(:job, perform: nil) } + let(:retryable_job) { RetryableJob.new(job) } + + it 'performs the job' do + retryable_job.perform + expect(job).to have_received(:perform) + end + + context 'when the inner job fails' do + let(:mock_response) { double(:response, body: nil, message: nil, code: 500) } + before do + allow(job).to receive(:perform).and_raise(error) + allow(Delayed::Job).to receive(:enqueue) + end + + context 'when the exception is ServiceBrokerApiTimeout' do + let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + + it 'enqueues another retryable job' do + retryable_job.perform + + expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| + expect(enqueued_job).to be_a RetryableJob + expect(enqueued_job.num_attempts).to eq 1 + expect(opts).to include(queue: 'cc-generic', run_at: anything) + end + end + end + + context 'when the exception is ServiceBrokerBadResponse' do + let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new('uri.com', :delete, mock_response) } + + it 'enqueues another retryable job' do + retryable_job.perform + + expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| + expect(enqueued_job).to be_a RetryableJob + expect(enqueued_job.num_attempts).to eq 1 + expect(opts).to include(queue: 'cc-generic', run_at: anything) + end + end + end + + describe 'exponential backoff' do + let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + let(:retryable_job) { RetryableJob.new(job, num_attempts) } + let(:num_attempts) { 5 } + + it 'runs the subsequent job at 2^(num_attempts) minutes from now' do + now = Time.now + Timecop.freeze now do + retryable_job.perform + + expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| + expect(enqueued_job.num_attempts).to eq(num_attempts + 1) + run_at = opts[:run_at] + expect(run_at).to be_within(0.01).of(now + (2 ** num_attempts).minutes) + end + end + end + + context 'when the max attempts have reached' do + let(:retryable_job) { RetryableJob.new(job, 10) } + let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + + it 'progagates an error' do + expect { + retryable_job.perform + }.to raise_error(VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout) + end + end + end + end + end + end + end +end + + From b58ce03520341a0a2cad46ac80e1dd9f2097cdc8 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 11:01:51 -0800 Subject: [PATCH 06/76] Reduce max_attempts for Deprovision and Unbind jobs --- app/jobs/services/service_instance_deprovision.rb | 2 +- app/jobs/services/service_instance_unbind.rb | 2 +- spec/unit/jobs/services/service_instance_deprovision_spec.rb | 4 ++-- spec/unit/jobs/services/service_instance_unbind_spec.rb | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/jobs/services/service_instance_deprovision.rb b/app/jobs/services/service_instance_deprovision.rb index bacf5b1343c..00bf98213b3 100644 --- a/app/jobs/services/service_instance_deprovision.rb +++ b/app/jobs/services/service_instance_deprovision.rb @@ -14,7 +14,7 @@ def job_name_in_configuration end def max_attempts - 3 + 1 end end end diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index 55f20c87410..aa0e987fe0c 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -17,7 +17,7 @@ def job_name_in_configuration end def max_attempts - 3 + 1 end end end diff --git a/spec/unit/jobs/services/service_instance_deprovision_spec.rb b/spec/unit/jobs/services/service_instance_deprovision_spec.rb index 7285c580d88..bd9d4e223a8 100644 --- a/spec/unit/jobs/services/service_instance_deprovision_spec.rb +++ b/spec/unit/jobs/services/service_instance_deprovision_spec.rb @@ -39,8 +39,8 @@ module Jobs::Services end describe '#max_attempts' do - it 'returns 3' do - expect(job.max_attempts).to eq 3 + it 'returns 1' do + expect(job.max_attempts).to eq 1 end end end diff --git a/spec/unit/jobs/services/service_instance_unbind_spec.rb b/spec/unit/jobs/services/service_instance_unbind_spec.rb index 5893eefa6e4..752b3a1d0fe 100644 --- a/spec/unit/jobs/services/service_instance_unbind_spec.rb +++ b/spec/unit/jobs/services/service_instance_unbind_spec.rb @@ -36,8 +36,8 @@ module Jobs::Services end describe '#max_attempts' do - it 'returns 3' do - expect(job.max_attempts).to eq 3 + it 'returns 1' do + expect(job.max_attempts).to eq 1 end end end From a63bdbf3aa7bb378ec490397bccd03ba26ac8a0d Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 11:45:18 -0800 Subject: [PATCH 07/76] Fix rubocop errors --- app/jobs/retryable_job.rb | 4 +- .../service_brokers/v2/response_parser.rb | 46 +++++++++---------- spec/unit/jobs/retryable_job_spec.rb | 6 +-- .../service_brokers/v2/client_spec.rb | 13 +++--- .../v2/response_parser_spec.rb | 2 +- 5 files changed, 34 insertions(+), 37 deletions(-) diff --git a/app/jobs/retryable_job.rb b/app/jobs/retryable_job.rb index 5b3d446fa4d..69b6559b8b6 100644 --- a/app/jobs/retryable_job.rb +++ b/app/jobs/retryable_job.rb @@ -3,7 +3,7 @@ module Jobs class RetryableJob attr_reader :job, :num_attempts - def initialize(job, num_attempts = 0) + def initialize(job, num_attempts=0) @job = job @num_attempts = num_attempts end @@ -12,7 +12,7 @@ def perform job.perform rescue VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse => e raise e if num_attempts >= 10 - Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2 ** num_attempts).minutes) + Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2**num_attempts).minutes) end def max_attempts diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 7bc795c2f8d..2d7861c063e 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -11,37 +11,37 @@ def parse(method, path, response) code = response.code.to_i case code - when 204 - return nil # no body + when 204 + return nil # no body - when 200..299 + when 200..299 - begin - response_hash = MultiJson.load(response.body) - rescue MultiJson::ParseError - logger.warn("MultiJson parse error `#{response.try(:body).inspect}'") - end + begin + response_hash = MultiJson.load(response.body) + rescue MultiJson::ParseError + logger.warn("MultiJson parse error `#{response.try(:body).inspect}'") + end - unless response_hash.is_a?(Hash) - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) - end + unless response_hash.is_a?(Hash) + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) + end - return response_hash + return response_hash - when HTTP::Status::UNAUTHORIZED - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) + when HTTP::Status::UNAUTHORIZED + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) - when 408 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new(uri.to_s, method, response) + when 408 + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new(uri.to_s, method, response) - when 409 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) + when 409 + raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) - when 410 - if method == :delete - logger.warn("Already deleted: #{uri.to_s}") - return nil - end + when 410 + if method == :delete + logger.warn("Already deleted: #{uri}") + return nil + end end raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri.to_s, method, response) diff --git a/spec/unit/jobs/retryable_job_spec.rb b/spec/unit/jobs/retryable_job_spec.rb index eb92803ef60..f7ce54fdd86 100644 --- a/spec/unit/jobs/retryable_job_spec.rb +++ b/spec/unit/jobs/retryable_job_spec.rb @@ -2,7 +2,7 @@ module VCAP::CloudController module Jobs - describe "RetryableJob" do + describe 'RetryableJob' do describe '#perform' do let(:job) { double(:job, perform: nil) } let(:retryable_job) { RetryableJob.new(job) } @@ -60,7 +60,7 @@ module Jobs expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| expect(enqueued_job.num_attempts).to eq(num_attempts + 1) run_at = opts[:run_at] - expect(run_at).to be_within(0.01).of(now + (2 ** num_attempts).minutes) + expect(run_at).to be_within(0.01).of(now + (2**num_attempts).minutes) end end end @@ -81,5 +81,3 @@ module Jobs end end end - - diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 43c9573d19a..0e0b0e7082a 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -277,7 +277,7 @@ module VCAP::Services::ServiceBrokers::V2 context 'when provision fails' do let(:uri) { 'some-uri.com/v2/service_instances/some-guid' } - let(:response) { double(:response, body: nil, message: nil)} + let(:response) { double(:response, body: nil, message: nil) } before do allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner).to receive(:deprovision) @@ -518,8 +518,8 @@ module VCAP::Services::ServiceBrokers::V2 binding_options: { 'this' => 'that' } ) end - let(:uri) { 'some-uri.com/v2/service_instances/instance-guid/service_bindings/binding-guid'} - let(:response) { double(:response, body: nil, message: nil)} + let(:uri) { 'some-uri.com/v2/service_instances/instance-guid/service_bindings/binding-guid' } + let(:response) { double(:response, body: nil, message: nil) } before do allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder).to receive(:delayed_unbind) @@ -540,10 +540,9 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:delayed_unbind). - with(client_attrs, binding) - + expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). + to have_received(:delayed_unbind). + with(client_attrs, binding) end end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index ef3e488f13a..73c89588ecf 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -19,7 +19,7 @@ module V2 it 'returns the response body hash' do response_hash = parser.parse(:get, '/v2/catalog', response) - expect(response_hash).to eq({'foo' => 'bar'}) + expect(response_hash).to eq({ 'foo' => 'bar' }) end context 'when the status code is 204' do From 80871eac33e7bcc2a008d2ecada03ebcec8fa725 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 13:54:38 -0800 Subject: [PATCH 08/76] Retry ServiceInstanceUnbind job --- .../v2/service_instance_unbinder.rb | 5 +-- .../v2/service_instance_unbinder_spec.rb | 33 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/lib/services/service_brokers/v2/service_instance_unbinder.rb b/lib/services/service_brokers/v2/service_instance_unbinder.rb index f371da9decb..6dc78137b6a 100644 --- a/lib/services/service_brokers/v2/service_instance_unbinder.rb +++ b/lib/services/service_brokers/v2/service_instance_unbinder.rb @@ -12,8 +12,9 @@ def self.delayed_unbind(client_attrs, binding) binding.service_instance.guid, binding.app.guid ) - Delayed::Job.enqueue(unbind_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) - unbind_job + + retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(unbind_job, 0) + Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb index f087008eb6c..92d9e1614cb 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb @@ -12,19 +12,28 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'delayed_unbind' do - it 'creates a ServiceInstanceUnbind Job' do - job = ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - expect(job).to be_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind) - expect(job.client_attrs).to be(client_attrs) - expect(job.binding_guid).to be(binding.guid) - expect(job.service_instance_guid).to be(binding.service_instance.guid) - expect(job.app_guid).to be(binding.app.guid) - end + it 'enques a retryable ServiceInstanceUnbind job' do + allow(Delayed::Job).to receive(:enqueue) + + Timecop.freeze do + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + + expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob + expect(job.num_attempts).to eq 0 - it 'enqueues a ServiceInstanceUnbind Job' do - expect(Delayed::Job).to receive(:enqueue).with(an_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind), - hash_including(queue: 'cc-generic')) - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) + inner_job = job.job + expect(inner_job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind + expect(inner_job.name).to eq 'service-instance-unbind' + expect(inner_job.client_attrs).to eq client_attrs + expect(inner_job.binding_guid).to be(binding.guid) + expect(inner_job.service_instance_guid).to be(binding.service_instance.guid) + expect(inner_job.app_guid).to be(binding.app.guid) + end + end end end end From 102c7ca7bf089e799a209a0a34ce90cdcccb1f29 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 15:26:01 -0800 Subject: [PATCH 09/76] Move ServiceBrokerApiTimeout and ServiceBrokerApiUnreachable into an errors folder includes corresponding namespacing VCAP::Services::ServiceBrokers::V2::Errors::* --- app/jobs/retryable_job.rb | 2 +- lib/services/service_brokers/v2.rb | 1 + lib/services/service_brokers/v2/client.rb | 4 +- lib/services/service_brokers/v2/errors.rb | 2 + .../v2/errors/service_broker_api_timeout.rb | 24 ++++++++++ .../errors/service_broker_api_unreachable.rb | 20 ++++++++ .../service_brokers/v2/http_client.rb | 30 +----------- .../service_brokers/v2/response_parser.rb | 2 +- spec/unit/jobs/retryable_job_spec.rb | 10 ++-- .../service_brokers/v2/client_spec.rb | 24 +++++----- .../errors/service_broker_api_timeout_spec.rb | 23 +++++++++ .../service_broker_api_unreachable_spec.rb | 36 ++++++++++++++ .../service_brokers/v2/http_client_spec.rb | 47 ++----------------- .../v2/response_parser_spec.rb | 4 +- 14 files changed, 134 insertions(+), 95 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors.rb create mode 100644 lib/services/service_brokers/v2/errors/service_broker_api_timeout.rb create mode 100644 lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb diff --git a/app/jobs/retryable_job.rb b/app/jobs/retryable_job.rb index 69b6559b8b6..508db1d7b51 100644 --- a/app/jobs/retryable_job.rb +++ b/app/jobs/retryable_job.rb @@ -10,7 +10,7 @@ def initialize(job, num_attempts=0) def perform job.perform - rescue VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse => e + rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse => e raise e if num_attempts >= 10 Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2**num_attempts).minutes) end diff --git a/lib/services/service_brokers/v2.rb b/lib/services/service_brokers/v2.rb index 5bc13214bca..259f89d2cc1 100644 --- a/lib/services/service_brokers/v2.rb +++ b/lib/services/service_brokers/v2.rb @@ -9,3 +9,4 @@ module VCAP::Services::ServiceBrokers::V2 end require 'services/service_brokers/v2/service_instance_deprovisioner' require 'services/service_brokers/v2/service_instance_unbinder' require 'services/service_brokers/v2/response_parser' +require 'services/service_brokers/v2/errors' diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 70799dfae42..1a6a5bae286 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -108,7 +108,7 @@ def provision(instance) # DEPRECATED, but needed because of not null constraint instance.credentials = {} - rescue ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e + rescue Errors::ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner.deprovision(@attrs, instance) raise e end @@ -127,7 +127,7 @@ def bind(binding) binding.syslog_drain_url = parsed_response['syslog_drain_url'] end - rescue ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e + rescue Errors::ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder.delayed_unbind(@attrs, binding) raise e end diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb new file mode 100644 index 00000000000..6fb8bb4122d --- /dev/null +++ b/lib/services/service_brokers/v2/errors.rb @@ -0,0 +1,2 @@ +require 'services/service_brokers/v2/errors/service_broker_api_unreachable' +require 'services/service_brokers/v2/errors/service_broker_api_timeout' diff --git a/lib/services/service_brokers/v2/errors/service_broker_api_timeout.rb b/lib/services/service_brokers/v2/errors/service_broker_api_timeout.rb new file mode 100644 index 00000000000..ddad3e8936d --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_api_timeout.rb @@ -0,0 +1,24 @@ +require 'net/http' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerApiTimeout < HttpRequestError + def initialize(uri, method, source) + super( + "The service broker API timed out: #{uri}", + uri, + method, + source + ) + end + + def response_code + 504 + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb b/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb new file mode 100644 index 00000000000..2ddc5006ea6 --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb @@ -0,0 +1,20 @@ +require 'net/http' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerApiUnreachable < HttpRequestError + def initialize(uri, method, source) + super( + "The service broker API could not be reached: #{uri}", + uri, + method, + source + ) + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/http_client.rb b/lib/services/service_brokers/v2/http_client.rb index 26301712f88..11a6b1568b1 100644 --- a/lib/services/service_brokers/v2/http_client.rb +++ b/lib/services/service_brokers/v2/http_client.rb @@ -2,32 +2,6 @@ module VCAP::Services module ServiceBrokers::V2 - class ServiceBrokerApiUnreachable < HttpRequestError - def initialize(uri, method, source) - super( - "The service broker API could not be reached: #{uri}", - uri, - method, - source - ) - end - end - - class ServiceBrokerApiTimeout < HttpRequestError - def initialize(uri, method, source) - super( - "The service broker API timed out: #{uri}", - uri, - method, - source - ) - end - - def response_code - 504 - end - end - class HttpClient attr_reader :url @@ -79,9 +53,9 @@ def make_request(method, uri, body, content_type) log_response(uri, response) return response rescue SocketError, Errno::ECONNREFUSED => error - raise ServiceBrokerApiUnreachable.new(uri.to_s, method, error) + raise Errors::ServiceBrokerApiUnreachable.new(uri.to_s, method, error) rescue Timeout::Error => error - raise ServiceBrokerApiTimeout.new(uri.to_s, method, error) + raise Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, error) rescue => error raise HttpRequestError.new(error.message, uri.to_s, method, error) end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 2d7861c063e..03f517a3d29 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -32,7 +32,7 @@ def parse(method, path, response) raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) when 408 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new(uri.to_s, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, response) when 409 raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) diff --git a/spec/unit/jobs/retryable_job_spec.rb b/spec/unit/jobs/retryable_job_spec.rb index f7ce54fdd86..67eee9aea18 100644 --- a/spec/unit/jobs/retryable_job_spec.rb +++ b/spec/unit/jobs/retryable_job_spec.rb @@ -19,8 +19,8 @@ module Jobs allow(Delayed::Job).to receive(:enqueue) end - context 'when the exception is ServiceBrokerApiTimeout' do - let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + context 'when the exception is Errors::ServiceBrokerApiTimeout' do + let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } it 'enqueues another retryable job' do retryable_job.perform @@ -48,7 +48,7 @@ module Jobs end describe 'exponential backoff' do - let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } let(:retryable_job) { RetryableJob.new(job, num_attempts) } let(:num_attempts) { 5 } @@ -67,12 +67,12 @@ module Jobs context 'when the max attempts have reached' do let(:retryable_job) { RetryableJob.new(job, 10) } - let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } + let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } it 'progagates an error' do expect { retryable_job.perform - }.to raise_error(VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiTimeout) + }.to raise_error(VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 0e0b0e7082a..453da10fea5 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -290,13 +290,13 @@ module VCAP::Services::ServiceBrokers::V2 allow(http_client).to receive(:put).and_raise(error) end - context 'ServiceBrokerApiTimeout error' do - let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } + context 'Errors::ServiceBrokerApiTimeout error' do + let(:error) { Errors::ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerApiTimeout) + }.to raise_error(Errors::ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). to have_received(:deprovision).with(client_attrs, instance) @@ -326,13 +326,13 @@ module VCAP::Services::ServiceBrokers::V2 allow(VCAP::Services::ServiceBrokers::V2::ResponseParser).to receive(:new).and_return(response_parser) end - context 'ServiceBrokerApiTimeout error' do - let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } + context 'Errors::ServiceBrokerApiTimeout error' do + let(:error) { Errors::ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerApiTimeout) + }.to raise_error(Errors::ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). to have_received(:deprovision). @@ -532,13 +532,13 @@ module VCAP::Services::ServiceBrokers::V2 allow(http_client).to receive(:put).and_raise(error) end - context 'ServiceBrokerApiTimeout error' do - let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } + context 'Errors::ServiceBrokerApiTimeout error' do + let(:error) { Errors::ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } it 'propagates the error and follows up with a deprovision request' do expect { client.bind(binding) - }.to raise_error(ServiceBrokerApiTimeout) + }.to raise_error(Errors::ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). to have_received(:delayed_unbind). @@ -568,13 +568,13 @@ module VCAP::Services::ServiceBrokers::V2 allow(VCAP::Services::ServiceBrokers::V2::ResponseParser).to receive(:new).and_return(response_parser) end - context 'ServiceBrokerApiTimeout error' do - let(:error) { ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } + context 'Errors::ServiceBrokerApiTimeout error' do + let(:error) { Errors::ServiceBrokerApiTimeout.new(uri, :put, Timeout::Error.new) } it 'propagates the error and follows up with a deprovision request' do expect { client.bind(binding) - }.to raise_error(ServiceBrokerApiTimeout) + }.to raise_error(Errors::ServiceBrokerApiTimeout) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). to have_received(:delayed_unbind).with(client_attrs, binding) diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb new file mode 100644 index 00000000000..09f655f85c3 --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe ServiceBrokerApiTimeout do + let(:uri) { 'http://uri.example.com' } + let(:method) { 'POST' } + let(:error) { StandardError.new } + + it 'initializes the base class correctly' do + exception = Errors::ServiceBrokerApiTimeout.new(uri, method, error) + expect(exception.message).to eq("The service broker API timed out: #{uri}") + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to be(error) + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb new file mode 100644 index 00000000000..78b46daa89a --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb @@ -0,0 +1,36 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe ServiceBrokerApiUnreachable do + let(:uri) { 'http://www.example.com/' } + let(:error) { SocketError.new('some message') } + + before do + error.set_backtrace(['/socketerror:1', '/backtrace:2']) + end + + it 'generates the correct hash' do + exception = ServiceBrokerApiUnreachable.new(uri, 'PUT', error) + exception.set_backtrace(['/generatedexception:3', '/backtrace:4']) + + expect(exception.to_h).to eq({ + 'description' => 'The service broker API could not be reached: http://www.example.com/', + 'backtrace' => ['/generatedexception:3', '/backtrace:4'], + 'http' => { + 'uri' => uri, + 'method' => 'PUT' + }, + 'source' => { + 'description' => error.message, + 'backtrace' => ['/socketerror:1', '/backtrace:2'] + } + }) + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb b/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb index 20e21ed29b3..3ab858f058a 100644 --- a/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb @@ -1,47 +1,6 @@ require 'spec_helper' module VCAP::Services::ServiceBrokers::V2 - describe ServiceBrokerApiUnreachable do - let(:uri) { 'http://www.example.com/' } - let(:error) { SocketError.new('some message') } - - before do - error.set_backtrace(['/socketerror:1', '/backtrace:2']) - end - - it 'generates the correct hash' do - exception = ServiceBrokerApiUnreachable.new(uri, 'PUT', error) - exception.set_backtrace(['/generatedexception:3', '/backtrace:4']) - - expect(exception.to_h).to eq({ - 'description' => 'The service broker API could not be reached: http://www.example.com/', - 'backtrace' => ['/generatedexception:3', '/backtrace:4'], - 'http' => { - 'uri' => uri, - 'method' => 'PUT' - }, - 'source' => { - 'description' => error.message, - 'backtrace' => ['/socketerror:1', '/backtrace:2'] - } - }) - end - end - - describe ServiceBrokerApiTimeout do - let(:uri) { 'http://uri.example.com' } - let(:method) { 'POST' } - let(:error) { StandardError.new } - - it 'initializes the base class correctly' do - exception = ServiceBrokerApiTimeout.new(uri, method, error) - expect(exception.message).to eq("The service broker API timed out: #{uri}") - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to be(error) - end - end - describe HttpClient do let(:auth_username) { 'me' } let(:auth_password) { 'abc123' } @@ -153,7 +112,7 @@ module VCAP::Services::ServiceBrokers::V2 it 'raises an unreachable error' do expect { request }. - to raise_error(ServiceBrokerApiUnreachable) + to raise_error(Errors::ServiceBrokerApiUnreachable) end end @@ -164,7 +123,7 @@ module VCAP::Services::ServiceBrokers::V2 it 'raises an unreachable error' do expect { request }. - to raise_error(ServiceBrokerApiUnreachable) + to raise_error(Errors::ServiceBrokerApiUnreachable) end end end @@ -177,7 +136,7 @@ module VCAP::Services::ServiceBrokers::V2 it 'raises a timeout error' do expect { request }. - to raise_error(ServiceBrokerApiTimeout) + to raise_error(Errors::ServiceBrokerApiTimeout) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index 73c89588ecf..e16452edaf8 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -63,10 +63,10 @@ module V2 context 'when the status code is HTTP Request Timeout (408)' do let(:code) { 408 } - it 'raises a ServiceBrokerApiTimeout error' do + it 'raises a Errors::ServiceBrokerApiTimeout error' do expect { parser.parse(:get, '/v2/catalog', response) - }.to raise_error(ServiceBrokerApiTimeout) + }.to raise_error(Errors::ServiceBrokerApiTimeout) end end From 89b72eb7f624d0cd6c614eb589cd09d21130f263 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 15:37:50 -0800 Subject: [PATCH 10/76] Move ServiceBrokerResponseMalformed into errors directory --- lib/services/service_brokers/v2/client.rb | 11 -------- lib/services/service_brokers/v2/errors.rb | 1 + .../service_broker_response_malformed.rb | 18 +++++++++++++ .../service_brokers/v2/response_parser.rb | 2 +- .../service_brokers/v2/client_spec.rb | 13 ---------- .../service_broker_response_malformed_spec.rb | 25 +++++++++++++++++++ .../v2/response_parser_spec.rb | 4 +-- 7 files changed, 47 insertions(+), 27 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 1a6a5bae286..ae4db6bcef5 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -20,17 +20,6 @@ def response_code end end - class ServiceBrokerResponseMalformed < HttpResponseError - def initialize(uri, method, response) - super( - 'The service broker response was not understood', - uri, - method, - response - ) - end - end - class ServiceBrokerApiAuthenticationFailed < HttpResponseError def initialize(uri, method, response) super( diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb index 6fb8bb4122d..68cad303794 100644 --- a/lib/services/service_brokers/v2/errors.rb +++ b/lib/services/service_brokers/v2/errors.rb @@ -1,2 +1,3 @@ require 'services/service_brokers/v2/errors/service_broker_api_unreachable' require 'services/service_brokers/v2/errors/service_broker_api_timeout' +require 'services/service_brokers/v2/errors/service_broker_response_malformed' diff --git a/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb b/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb new file mode 100644 index 00000000000..2d81e994c4e --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb @@ -0,0 +1,18 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerResponseMalformed < HttpResponseError + def initialize(uri, method, response) + super( + 'The service broker response was not understood', + uri, + method, + response + ) + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 03f517a3d29..acde2034a5b 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -23,7 +23,7 @@ def parse(method, path, response) end unless response_hash.is_a?(Hash) - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) end return response_hash diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 453da10fea5..9c443b6d9b3 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -59,19 +59,6 @@ module VCAP::Services::ServiceBrokers::V2 let(:method) { 'POST' } let(:error) { StandardError.new } - describe ServiceBrokerResponseMalformed do - let(:response_body) { 'foo' } - let(:response) { double(code: 200, reason: 'OK', body: response_body) } - - it 'initializes the base class correctly' do - exception = ServiceBrokerResponseMalformed.new(uri, method, response) - expect(exception.message).to eq('The service broker response was not understood') - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to be(response.body) - end - end - describe ServiceBrokerApiAuthenticationFailed do let(:response_body) { 'foo' } let(:response) { double(code: 401, reason: 'Auth Error', body: response_body) } diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb new file mode 100644 index 00000000000..6e6b6844389 --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe ServiceBrokerResponseMalformed do + let(:uri) { 'http://uri.example.com' } + let(:method) { 'POST' } + let(:error) { StandardError.new } + let(:response_body) { 'foo' } + let(:response) { double(code: 200, reason: 'OK', body: response_body) } + + it 'initializes the base class correctly' do + exception = ServiceBrokerResponseMalformed.new(uri, method, response) + expect(exception.message).to eq('The service broker response was not understood') + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to be(response.body) + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index e16452edaf8..cccaeb4ccbd 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -36,7 +36,7 @@ module V2 it 'raises a MalformedResponse error' do expect { parser.parse(:get, '/v2/catalog', response) - }.to raise_error(ServiceBrokerResponseMalformed) + }.to raise_error(Errors::ServiceBrokerResponseMalformed) expect(logger).to have_received(:warn).with(/MultiJson parse error/) end end @@ -47,7 +47,7 @@ module V2 it 'raises a MalformedResponse error' do expect { parser.parse(:get, '/v2/catalog', response) - }.to raise_error(ServiceBrokerResponseMalformed) + }.to raise_error(Errors::ServiceBrokerResponseMalformed) expect(logger).not_to have_received(:warn) end end From c53213d782efa41b93e28463ae956ebf88378956 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 15:44:38 -0800 Subject: [PATCH 11/76] Move ServiceBrokerApiAuthenticationFailed into errors directory --- lib/services/service_brokers/v2/client.rb | 15 ---------- lib/services/service_brokers/v2/errors.rb | 1 + ...ervice_broker_api_authentication_failed.rb | 22 +++++++++++++++ .../service_brokers/v2/response_parser.rb | 2 +- .../service_brokers/v2/client_spec.rb | 12 -------- ...e_broker_api_authentication_failed_spec.rb | 28 +++++++++++++++++++ .../v2/response_parser_spec.rb | 2 +- 7 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index ae4db6bcef5..c356d877532 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -20,21 +20,6 @@ def response_code end end - class ServiceBrokerApiAuthenticationFailed < HttpResponseError - def initialize(uri, method, response) - super( - "Authentication failed for the service broker API. Double-check that the username and password are correct: #{uri}", - uri, - method, - response - ) - end - - def response_code - 502 - end - end - class ServiceBrokerConflict < HttpResponseError def initialize(uri, method, response) error_message = nil diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb index 68cad303794..14cb6360790 100644 --- a/lib/services/service_brokers/v2/errors.rb +++ b/lib/services/service_brokers/v2/errors.rb @@ -1,3 +1,4 @@ require 'services/service_brokers/v2/errors/service_broker_api_unreachable' require 'services/service_brokers/v2/errors/service_broker_api_timeout' require 'services/service_brokers/v2/errors/service_broker_response_malformed' +require 'services/service_brokers/v2/errors/service_broker_api_authentication_failed' diff --git a/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed.rb b/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed.rb new file mode 100644 index 00000000000..b9d7d5ea1c0 --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed.rb @@ -0,0 +1,22 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerApiAuthenticationFailed < HttpResponseError + def initialize(uri, method, response) + super( + "Authentication failed for the service broker API. Double-check that the username and password are correct: #{uri}", + uri, + method, + response + ) + end + + def response_code + 502 + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index acde2034a5b..42f6764eea3 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -29,7 +29,7 @@ def parse(method, path, response) return response_hash when HTTP::Status::UNAUTHORIZED - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) when 408 raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, response) diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 9c443b6d9b3..36d64d5a0e6 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -59,18 +59,6 @@ module VCAP::Services::ServiceBrokers::V2 let(:method) { 'POST' } let(:error) { StandardError.new } - describe ServiceBrokerApiAuthenticationFailed do - let(:response_body) { 'foo' } - let(:response) { double(code: 401, reason: 'Auth Error', body: response_body) } - - it 'initializes the base class correctly' do - exception = ServiceBrokerApiAuthenticationFailed.new(uri, method, response) - expect(exception.message).to eq("Authentication failed for the service broker API. Double-check that the username and password are correct: #{uri}") - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to be(response.body) - end - end describe ServiceBrokerConflict do let(:response_body) { '{"message": "error message"}' } diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb new file mode 100644 index 00000000000..4932306162a --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe 'ServiceBrokerAuthenticationFailed' do + let(:uri) { 'http://uri.example.com' } + let(:method) { 'POST' } + let(:error) { StandardError.new } + + describe ServiceBrokerApiAuthenticationFailed do + let(:response_body) { 'foo' } + let(:response) { double(code: 401, reason: 'Auth Error', body: response_body) } + + it 'initializes the base class correctly' do + exception = ServiceBrokerApiAuthenticationFailed.new(uri, method, response) + expect(exception.message).to eq("Authentication failed for the service broker API. Double-check that the username and password are correct: #{uri}") + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to be(response.body) + end + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index cccaeb4ccbd..1ae51a5c828 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -57,7 +57,7 @@ module V2 it 'raises a ServiceBrokerApiAuthenticationFailed error' do expect { parser.parse(:get, '/v2/catalog', response) - }.to raise_error(ServiceBrokerApiAuthenticationFailed) + }.to raise_error(Errors::ServiceBrokerApiAuthenticationFailed) end end From 2997d5b7bc91ace8031e64da1bec20da0d9ed6cf Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 15:58:11 -0800 Subject: [PATCH 12/76] Move ServiceBrokerBadResponse into errors directory --- app/jobs/retryable_job.rb | 2 +- lib/services/service_brokers/v2/client.rb | 24 +------ lib/services/service_brokers/v2/errors.rb | 1 + .../v2/errors/service_broker_bad_response.rb | 28 ++++++++ .../service_brokers/v2/response_parser.rb | 2 +- .../service_bindings_controller_spec.rb | 2 +- .../service_instances_controller_spec.rb | 2 +- spec/unit/jobs/retryable_job_spec.rb | 2 +- .../service_broker_registration_spec.rb | 12 ++-- .../service_brokers/v2/client_spec.rb | 72 +++---------------- .../service_broker_bad_response_spec.rb | 60 ++++++++++++++++ .../v2/response_parser_spec.rb | 4 +- 12 files changed, 113 insertions(+), 98 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors/service_broker_bad_response.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb diff --git a/app/jobs/retryable_job.rb b/app/jobs/retryable_job.rb index 508db1d7b51..eaebe1bb68a 100644 --- a/app/jobs/retryable_job.rb +++ b/app/jobs/retryable_job.rb @@ -10,7 +10,7 @@ def initialize(job, num_attempts=0) def perform job.perform - rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse => e + rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse => e raise e if num_attempts >= 10 Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2**num_attempts).minutes) end diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index c356d877532..eeeacd4cba1 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -1,24 +1,4 @@ module VCAP::Services::ServiceBrokers::V2 - class ServiceBrokerBadResponse < HttpResponseError - def initialize(uri, method, response) - begin - hash = MultiJson.load(response.body) - rescue MultiJson::ParseError - end - - if hash.is_a?(Hash) && hash.key?('description') - message = "Service broker error: #{hash['description']}" - else - message = "The service broker API returned an error from #{uri}: #{response.code} #{response.message}" - end - - super(message, uri, method, response) - end - - def response_code - 502 - end - end class ServiceBrokerConflict < HttpResponseError def initialize(uri, method, response) @@ -82,7 +62,7 @@ def provision(instance) # DEPRECATED, but needed because of not null constraint instance.credentials = {} - rescue Errors::ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e + rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner.deprovision(@attrs, instance) raise e end @@ -101,7 +81,7 @@ def bind(binding) binding.syslog_drain_url = parsed_response['syslog_drain_url'] end - rescue Errors::ServiceBrokerApiTimeout, ServiceBrokerBadResponse => e + rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder.delayed_unbind(@attrs, binding) raise e end diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb index 14cb6360790..46c13941989 100644 --- a/lib/services/service_brokers/v2/errors.rb +++ b/lib/services/service_brokers/v2/errors.rb @@ -2,3 +2,4 @@ require 'services/service_brokers/v2/errors/service_broker_api_timeout' require 'services/service_brokers/v2/errors/service_broker_response_malformed' require 'services/service_brokers/v2/errors/service_broker_api_authentication_failed' +require 'services/service_brokers/v2/errors/service_broker_bad_response' diff --git a/lib/services/service_brokers/v2/errors/service_broker_bad_response.rb b/lib/services/service_brokers/v2/errors/service_broker_bad_response.rb new file mode 100644 index 00000000000..1b7c7b42abe --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_bad_response.rb @@ -0,0 +1,28 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerBadResponse < HttpResponseError + def initialize(uri, method, response) + begin + hash = MultiJson.load(response.body) + rescue MultiJson::ParseError + end + + if hash.is_a?(Hash) && hash.key?('description') + message = "Service broker error: #{hash['description']}" + else + message = "The service broker API returned an error from #{uri}: #{response.code} #{response.message}" + end + + super(message, uri, method, response) + end + + def response_code + 502 + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 42f6764eea3..5bae3db6738 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -44,7 +44,7 @@ def parse(method, path, response) end end - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri.to_s, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri.to_s, method, response) end private diff --git a/spec/unit/controllers/services/service_bindings_controller_spec.rb b/spec/unit/controllers/services/service_bindings_controller_spec.rb index 7c76807976d..1ed7f8c0417 100644 --- a/spec/unit/controllers/services/service_bindings_controller_spec.rb +++ b/spec/unit/controllers/services/service_bindings_controller_spec.rb @@ -372,7 +372,7 @@ def fake_app_staging(app) uri = 'http://broker.url.com' method = 'PUT' response = double(:response, code: 500, body: '{"description": "ERROR MESSAGE HERE"}') - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri, method, response) end end diff --git a/spec/unit/controllers/services/service_instances_controller_spec.rb b/spec/unit/controllers/services/service_instances_controller_spec.rb index a4b59b99a4f..4153f53d328 100644 --- a/spec/unit/controllers/services/service_instances_controller_spec.rb +++ b/spec/unit/controllers/services/service_instances_controller_spec.rb @@ -561,7 +561,7 @@ def self.user_sees_empty_enumerate(user_role, member_a_ivar, member_b_ivar) let(:response) { double(code: 422, reason: 'Broker rejected the upate', body: response_body) } before do - allow(client).to receive(:update_service_plan).and_raise(VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new(uri, method, response)) + allow(client).to receive(:update_service_plan).and_raise(VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri, method, response)) end it 'returns a CF-ServiceBrokerBadResponse' do diff --git a/spec/unit/jobs/retryable_job_spec.rb b/spec/unit/jobs/retryable_job_spec.rb index 67eee9aea18..5ab0bfc9f5f 100644 --- a/spec/unit/jobs/retryable_job_spec.rb +++ b/spec/unit/jobs/retryable_job_spec.rb @@ -34,7 +34,7 @@ module Jobs end context 'when the exception is ServiceBrokerBadResponse' do - let(:error) { VCAP::Services::ServiceBrokers::V2::ServiceBrokerBadResponse.new('uri.com', :delete, mock_response) } + let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new('uri.com', :delete, mock_response) } it 'enqueues another retryable job' do retryable_job.perform diff --git a/spec/unit/lib/services/service_brokers/service_broker_registration_spec.rb b/spec/unit/lib/services/service_brokers/service_broker_registration_spec.rb index 5c7b5d18897..2d3e5b2da2e 100644 --- a/spec/unit/lib/services/service_brokers/service_broker_registration_spec.rb +++ b/spec/unit/lib/services/service_brokers/service_broker_registration_spec.rb @@ -162,7 +162,7 @@ module VCAP::Services::ServiceBrokers it "raises an error, even though we'd rather it not" do expect { registration.create - }.to raise_error V2::ServiceBrokerBadResponse + }.to raise_error V2::Errors::ServiceBrokerBadResponse end it 'does not create a new service broker' do @@ -174,7 +174,7 @@ module VCAP::Services::ServiceBrokers it 'does not synchronize uaa clients' do begin registration.create - rescue V2::ServiceBrokerBadResponse + rescue V2::Errors::ServiceBrokerBadResponse end expect(client_manager).not_to have_received(:synchronize_clients_with_catalog) @@ -183,7 +183,7 @@ module VCAP::Services::ServiceBrokers it 'does not synchronize the catalog' do begin registration.create - rescue V2::ServiceBrokerBadResponse + rescue V2::Errors::ServiceBrokerBadResponse end expect(service_manager).not_to have_received(:sync_services_and_plans) @@ -433,7 +433,7 @@ module VCAP::Services::ServiceBrokers it "raises an error, even though we'd rather it not" do expect { registration.update - }.to raise_error V2::ServiceBrokerBadResponse + }.to raise_error V2::Errors::ServiceBrokerBadResponse end it 'not update the service broker' do @@ -446,7 +446,7 @@ module VCAP::Services::ServiceBrokers it 'does not synchronize uaa clients' do begin registration.update - rescue V2::ServiceBrokerBadResponse + rescue V2::Errors::ServiceBrokerBadResponse end expect(client_manager).not_to have_received(:synchronize_clients_with_catalog) @@ -455,7 +455,7 @@ module VCAP::Services::ServiceBrokers it 'does not synchronize the catalog' do begin registration.update - rescue V2::ServiceBrokerBadResponse + rescue V2::Errors::ServiceBrokerBadResponse end expect(service_manager).not_to have_received(:sync_services_and_plans) diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 36d64d5a0e6..0adda3f559d 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -1,65 +1,11 @@ require 'spec_helper' module VCAP::Services::ServiceBrokers::V2 - describe ServiceBrokerBadResponse do - let(:uri) { 'http://www.example.com/' } - let(:response) { double(code: 500, message: 'Internal Server Error', body: response_body) } - let(:method) { 'PUT' } - - context 'with a description in the body' do - let(:response_body) do - { - 'description' => 'Some error text' - }.to_json - end - - it 'generates the correct hash' do - exception = described_class.new(uri, method, response) - exception.set_backtrace(['/foo:1', '/bar:2']) - - expect(exception.to_h).to eq({ - 'description' => 'Service broker error: Some error text', - 'backtrace' => ['/foo:1', '/bar:2'], - 'http' => { - 'status' => 500, - 'uri' => uri, - 'method' => 'PUT' - }, - 'source' => { - 'description' => 'Some error text' - } - }) - end - end - - context 'without a description in the body' do - let(:response_body) do - { 'foo' => 'bar' }.to_json - end - it 'generates the correct hash' do - exception = described_class.new(uri, method, response) - exception.set_backtrace(['/foo:1', '/bar:2']) - - expect(exception.to_h).to eq({ - 'description' => 'The service broker API returned an error from http://www.example.com/: 500 Internal Server Error', - 'backtrace' => ['/foo:1', '/bar:2'], - 'http' => { - 'status' => 500, - 'uri' => uri, - 'method' => 'PUT' - }, - 'source' => { 'foo' => 'bar' } - }) - end - end - end - describe 'the remaining ServiceBrokers::V2 exceptions' do let(:uri) { 'http://uri.example.com' } let(:method) { 'POST' } let(:error) { StandardError.new } - describe ServiceBrokerConflict do let(:response_body) { '{"message": "error message"}' } let(:response) { double(code: 409, reason: 'Conflict', body: response_body) } @@ -279,12 +225,12 @@ module VCAP::Services::ServiceBrokers::V2 end context 'ServiceBrokerBadResponse error' do - let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(Errors::ServiceBrokerBadResponse) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). to have_received(:deprovision). @@ -316,12 +262,12 @@ module VCAP::Services::ServiceBrokers::V2 end context 'ServiceBrokerBadResponse error' do - let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } it 'propagates the error and follows up with a deprovision request' do expect { client.provision(instance) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(Errors::ServiceBrokerBadResponse) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). to have_received(:deprovision). @@ -390,7 +336,7 @@ module VCAP::Services::ServiceBrokers::V2 let(:body) { { description: 'cannot update to this plan' }.to_json } it 'raises a ServiceBrokerBadResponse error' do expect { client.update_service_plan(instance, new_plan) }.to raise_error( - ServiceBrokerBadResponse, /cannot update to this plan/ + Errors::ServiceBrokerBadResponse, /cannot update to this plan/ ) end end @@ -522,12 +468,12 @@ module VCAP::Services::ServiceBrokers::V2 end context 'ServiceBrokerBadResponse error' do - let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } it 'propagates the error and follows up with a deprovision request' do expect { client.bind(binding) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(Errors::ServiceBrokerBadResponse) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). to have_received(:delayed_unbind).with(client_attrs, binding) @@ -557,12 +503,12 @@ module VCAP::Services::ServiceBrokers::V2 end context 'ServiceBrokerBadResponse error' do - let(:error) { ServiceBrokerBadResponse.new(uri, :put, response) } + let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } it 'propagates the error and follows up with a deprovision request' do expect { client.bind(binding) - }.to raise_error(ServiceBrokerBadResponse) + }.to raise_error(Errors::ServiceBrokerBadResponse) expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). to have_received(:delayed_unbind).with(client_attrs, binding) diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb new file mode 100644 index 00000000000..4977dd91cc7 --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb @@ -0,0 +1,60 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe ServiceBrokerBadResponse do + let(:uri) { 'http://www.example.com/' } + let(:response) { double(code: 500, message: 'Internal Server Error', body: response_body) } + let(:method) { 'PUT' } + + context 'with a description in the body' do + let(:response_body) do + { + 'description' => 'Some error text' + }.to_json + end + + it 'generates the correct hash' do + exception = described_class.new(uri, method, response) + exception.set_backtrace(['/foo:1', '/bar:2']) + + expect(exception.to_h).to eq({ + 'description' => 'Service broker error: Some error text', + 'backtrace' => ['/foo:1', '/bar:2'], + 'http' => { + 'status' => 500, + 'uri' => uri, + 'method' => 'PUT' + }, + 'source' => { + 'description' => 'Some error text' + } + }) + end + end + + context 'without a description in the body' do + let(:response_body) do + { 'foo' => 'bar' }.to_json + end + it 'generates the correct hash' do + exception = described_class.new(uri, method, response) + exception.set_backtrace(['/foo:1', '/bar:2']) + + expect(exception.to_h).to eq({ + 'description' => 'The service broker API returned an error from http://www.example.com/: 500 Internal Server Error', + 'backtrace' => ['/foo:1', '/bar:2'], + 'http' => { + 'status' => 500, + 'uri' => uri, + 'method' => 'PUT' + }, + 'source' => { 'foo' => 'bar' } + }) + end + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index 1ae51a5c828..01b6b06d8fc 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -86,7 +86,7 @@ module V2 it 'raises ServiceBrokerBadResponse' do expect { parser.parse(method, '/v2/catalog', response) - }.to raise_error(ServiceBrokerBadResponse, /there was an error/) + }.to raise_error(Errors::ServiceBrokerBadResponse, /there was an error/) end context 'and the http method is delete' do @@ -106,7 +106,7 @@ module V2 it 'raises ServiceBrokerBadResponse' do expect { parser.parse(method, '/v2/catalog', response) - }.to raise_error(ServiceBrokerBadResponse, /there was an error/) + }.to raise_error(Errors::ServiceBrokerBadResponse, /there was an error/) end end end From d3beb2d25bfcea641ed62692f3847fb8c0efc740 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 16:12:55 -0800 Subject: [PATCH 13/76] Move ServiceBrokerConflict into errors directory --- lib/services/service_brokers/v2/client.rb | 33 +-------- lib/services/service_brokers/v2/errors.rb | 1 + .../v2/errors/service_broker_conflict.rb | 37 ++++++++++ .../service_brokers/v2/response_parser.rb | 2 +- .../service_bindings_controller_spec.rb | 2 +- .../service_brokers/v2/client_spec.rb | 62 ----------------- .../v2/errors/service_broker_conflict_spec.rb | 69 +++++++++++++++++++ .../v2/response_parser_spec.rb | 2 +- 8 files changed, 111 insertions(+), 97 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors/service_broker_conflict.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index eeeacd4cba1..bd08c722d0b 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -1,35 +1,4 @@ module VCAP::Services::ServiceBrokers::V2 - - class ServiceBrokerConflict < HttpResponseError - def initialize(uri, method, response) - error_message = nil - if parsed_json(response.body).key?('message') - error_message = parsed_json(response.body)['message'] - else - error_message = parsed_json(response.body)['description'] - end - - super( - error_message || "Resource conflict: #{uri}", - uri, - method, - response - ) - end - - def response_code - 409 - end - - private - - def parsed_json(str) - MultiJson.load(str) - rescue MultiJson::ParseError - {} - end - end - class Client CATALOG_PATH = '/v2/catalog'.freeze @@ -107,7 +76,7 @@ def deprovision(instance) @response_parser.parse(:delete, path, response) - rescue VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict => e + rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerConflict => e raise VCAP::Errors::ApiError.new_from_details('ServiceInstanceDeprovisionFailed', e.message) end diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb index 46c13941989..cb30662c780 100644 --- a/lib/services/service_brokers/v2/errors.rb +++ b/lib/services/service_brokers/v2/errors.rb @@ -3,3 +3,4 @@ require 'services/service_brokers/v2/errors/service_broker_response_malformed' require 'services/service_brokers/v2/errors/service_broker_api_authentication_failed' require 'services/service_brokers/v2/errors/service_broker_bad_response' +require 'services/service_brokers/v2/errors/service_broker_conflict' diff --git a/lib/services/service_brokers/v2/errors/service_broker_conflict.rb b/lib/services/service_brokers/v2/errors/service_broker_conflict.rb new file mode 100644 index 00000000000..b13c7d99b67 --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_conflict.rb @@ -0,0 +1,37 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerConflict < HttpResponseError + def initialize(uri, method, response) + error_message = nil + if parsed_json(response.body).key?('message') + error_message = parsed_json(response.body)['message'] + else + error_message = parsed_json(response.body)['description'] + end + + super( + error_message || "Resource conflict: #{uri}", + uri, + method, + response + ) + end + + def response_code + 409 + end + + private + + def parsed_json(str) + MultiJson.load(str) + rescue MultiJson::ParseError + {} + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 5bae3db6738..d7dbd168966 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -35,7 +35,7 @@ def parse(method, path, response) raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, response) when 409 - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri.to_s, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerConflict.new(uri.to_s, method, response) when 410 if method == :delete diff --git a/spec/unit/controllers/services/service_bindings_controller_spec.rb b/spec/unit/controllers/services/service_bindings_controller_spec.rb index 1ed7f8c0417..42cf9019f2c 100644 --- a/spec/unit/controllers/services/service_bindings_controller_spec.rb +++ b/spec/unit/controllers/services/service_bindings_controller_spec.rb @@ -350,7 +350,7 @@ def fake_app_staging(app) uri = 'http://broker.url.com' method = 'PUT' response = double(:response, code: 409, body: '{}') - raise VCAP::Services::ServiceBrokers::V2::ServiceBrokerConflict.new(uri, method, response) + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerConflict.new(uri, method, response) end end diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 0adda3f559d..8caf7daa0a0 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -1,68 +1,6 @@ require 'spec_helper' module VCAP::Services::ServiceBrokers::V2 - describe 'the remaining ServiceBrokers::V2 exceptions' do - let(:uri) { 'http://uri.example.com' } - let(:method) { 'POST' } - let(:error) { StandardError.new } - - describe ServiceBrokerConflict do - let(:response_body) { '{"message": "error message"}' } - let(:response) { double(code: 409, reason: 'Conflict', body: response_body) } - - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq('error message') - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(MultiJson.load(response.body)) - end - - it 'has a response_code of 409' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.response_code).to eq(409) - end - - context 'when the response body has no message' do - let(:response_body) { '{"description": "error description"}' } - - context 'and there is a description field' do - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq('error description') - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(MultiJson.load(response.body)) - end - end - - context 'and there is no description field' do - let(:response_body) { '{"field": "value"}' } - - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq("Resource conflict: #{uri}") - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(MultiJson.load(response.body)) - end - end - end - - context 'when the body is not JSON-parsable' do - let(:response_body) { 'foo' } - - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq("Resource conflict: #{uri}") - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(response.body) - end - end - end - end - describe Client do let(:service_broker) { VCAP::CloudController::ServiceBroker.make } diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb new file mode 100644 index 00000000000..83facbaeb0f --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb @@ -0,0 +1,69 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe 'ServiceBrokerConflict' do + let(:response_body) { '{"message": "error message"}' } + let(:response) { double(code: 409, reason: 'Conflict', body: response_body) } + + let(:uri) { 'http://uri.example.com' } + let(:method) { 'POST' } + let(:error) { StandardError.new } + + it 'initializes the base class correctly' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.message).to eq('error message') + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to eq(MultiJson.load(response.body)) + end + + it 'has a response_code of 409' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.response_code).to eq(409) + end + + context 'when the response body has no message' do + let(:response_body) { '{"description": "error description"}' } + + context 'and there is a description field' do + it 'initializes the base class correctly' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.message).to eq('error description') + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to eq(MultiJson.load(response.body)) + end + end + + context 'and there is no description field' do + let(:response_body) { '{"field": "value"}' } + + it 'initializes the base class correctly' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.message).to eq("Resource conflict: #{uri}") + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to eq(MultiJson.load(response.body)) + end + end + end + + context 'when the body is not JSON-parsable' do + let(:response_body) { 'foo' } + + it 'initializes the base class correctly' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.message).to eq("Resource conflict: #{uri}") + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to eq(response.body) + end + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index 01b6b06d8fc..bf15d12e917 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -75,7 +75,7 @@ module V2 it 'raises a ServiceBrokerConflict error' do expect { parser.parse(:get, '/v2/catalog', response) - }.to raise_error(ServiceBrokerConflict) + }.to raise_error(Errors::ServiceBrokerConflict) end end From a7cc7d5f5609d5fe13a5d5853f99beada4877b50 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 17:24:03 -0800 Subject: [PATCH 14/76] service broker http client cannot raise BadResponse error, so do not test for it --- .../service_brokers/v2/client_spec.rb | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 8caf7daa0a0..b9ff7b4791c 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -161,20 +161,6 @@ module VCAP::Services::ServiceBrokers::V2 to have_received(:deprovision).with(client_attrs, instance) end end - - context 'ServiceBrokerBadResponse error' do - let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } - - it 'propagates the error and follows up with a deprovision request' do - expect { - client.provision(instance) - }.to raise_error(Errors::ServiceBrokerBadResponse) - - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). - with(client_attrs, instance) - end - end end context 'due to a response parser error' do @@ -404,19 +390,6 @@ module VCAP::Services::ServiceBrokers::V2 with(client_attrs, binding) end end - - context 'ServiceBrokerBadResponse error' do - let(:error) { Errors::ServiceBrokerBadResponse.new(uri, :put, response) } - - it 'propagates the error and follows up with a deprovision request' do - expect { - client.bind(binding) - }.to raise_error(Errors::ServiceBrokerBadResponse) - - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:delayed_unbind).with(client_attrs, binding) - end - end end context 'due to a response parser error' do From 65c5dd52b827c3cc5c7daef7c56bd7a87e22ac95 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 17:24:34 -0800 Subject: [PATCH 15/76] Add ServiceBrokerRequestRejected error for unhandled 4xx errors [finishes #83710826] --- lib/services/service_brokers/v2/errors.rb | 1 + .../errors/service_broker_request_rejected.rb | 28 +++++++++ .../service_brokers/v2/response_parser.rb | 3 + .../service_instances_controller_spec.rb | 19 +++++- .../service_brokers/v2/client_spec.rb | 4 +- .../service_broker_request_rejected_spec.rb | 62 +++++++++++++++++++ .../v2/response_parser_spec.rb | 13 +++- 7 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 lib/services/service_brokers/v2/errors/service_broker_request_rejected.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb diff --git a/lib/services/service_brokers/v2/errors.rb b/lib/services/service_brokers/v2/errors.rb index cb30662c780..208d7197502 100644 --- a/lib/services/service_brokers/v2/errors.rb +++ b/lib/services/service_brokers/v2/errors.rb @@ -4,3 +4,4 @@ require 'services/service_brokers/v2/errors/service_broker_api_authentication_failed' require 'services/service_brokers/v2/errors/service_broker_bad_response' require 'services/service_brokers/v2/errors/service_broker_conflict' +require 'services/service_brokers/v2/errors/service_broker_request_rejected' diff --git a/lib/services/service_brokers/v2/errors/service_broker_request_rejected.rb b/lib/services/service_brokers/v2/errors/service_broker_request_rejected.rb new file mode 100644 index 00000000000..7110d6df70a --- /dev/null +++ b/lib/services/service_brokers/v2/errors/service_broker_request_rejected.rb @@ -0,0 +1,28 @@ +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + class ServiceBrokerRequestRejected < HttpResponseError + def initialize(uri, method, response) + begin + hash = MultiJson.load(response.body) + rescue MultiJson::ParseError + end + + if hash.is_a?(Hash) && hash.key?('description') + message = "Service broker error: #{hash['description']}" + else + message = "The service broker API returned an error from #{uri}: #{response.code} #{response.message}" + end + + super(message, uri, method, response) + end + + def response_code + 502 + end + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index d7dbd168966..46e804f03fb 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -42,6 +42,9 @@ def parse(method, path, response) logger.warn("Already deleted: #{uri}") return nil end + + when 400..499 + raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerRequestRejected.new(uri.to_s, method, response) end raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri.to_s, method, response) diff --git a/spec/unit/controllers/services/service_instances_controller_spec.rb b/spec/unit/controllers/services/service_instances_controller_spec.rb index 4153f53d328..dc790081403 100644 --- a/spec/unit/controllers/services/service_instances_controller_spec.rb +++ b/spec/unit/controllers/services/service_instances_controller_spec.rb @@ -558,7 +558,7 @@ def self.user_sees_empty_enumerate(user_role, member_a_ivar, member_b_ivar) let(:uri) { 'http://uri.example.com' } let(:method) { 'GET' } let(:response_body) { '{"description": "error message"}' } - let(:response) { double(code: 422, reason: 'Broker rejected the upate', body: response_body) } + let(:response) { double(code: 500, reason: 'Internal Server Error', body: response_body) } before do allow(client).to receive(:update_service_plan).and_raise(VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri, method, response)) @@ -571,6 +571,23 @@ def self.user_sees_empty_enumerate(user_role, member_a_ivar, member_b_ivar) end end + context 'when the broker client raises a ServiceBrokerRequestRejected' do + let(:uri) { 'http://uri.example.com' } + let(:method) { 'GET' } + let(:response_body) { '{"description": "error message"}' } + let(:response) { double(code: 422, reason: 'Broker rejected the upate', body: response_body) } + + before do + allow(client).to receive(:update_service_plan).and_raise(VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerRequestRejected.new(uri, method, response)) + end + + it 'returns a CF-ServiceBrokerBadResponse' do + put "/v2/service_instances/#{service_instance.guid}", body, admin_headers + expect(last_response.status).to eq 502 + expect(decoded_response['error_code']).to eq 'CF-ServiceBrokerRequestRejected' + end + end + describe 'the space_guid parameter' do let(:org) { Organization.make } let(:space) { Space.make(organization: org) } diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index b9ff7b4791c..aac77d001af 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -258,9 +258,9 @@ module VCAP::Services::ServiceBrokers::V2 context 'when the broker returns a 422' do let(:status_code) { '422' } let(:body) { { description: 'cannot update to this plan' }.to_json } - it 'raises a ServiceBrokerBadResponse error' do + it 'raises a ServiceBrokerRequestRejected error' do expect { client.update_service_plan(instance, new_plan) }.to raise_error( - Errors::ServiceBrokerBadResponse, /cannot update to this plan/ + Errors::ServiceBrokerRequestRejected, /cannot update to this plan/ ) end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb new file mode 100644 index 00000000000..52dd52dbfad --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers + module V2 + module Errors + describe ServiceBrokerRequestRejected do + let(:uri) { 'http://www.example.com/' } + let(:response) { double(code: 400, message: 'Generic bad request error', body: response_body) } + let(:method) { 'PUT' } + + context 'with a description in the body' do + let(:response_body) do + { + 'description' => 'Some error text' + }.to_json + end + + it 'generates the correct hash' do + exception = described_class.new(uri, method, response) + exception.set_backtrace(['/foo:1', '/bar:2']) + + expect(exception.to_h).to eq({ + 'description' => 'Service broker error: Some error text', + 'backtrace' => ['/foo:1', '/bar:2'], + 'http' => { + 'status' => 400, + 'uri' => uri, + 'method' => 'PUT' + }, + 'source' => { + 'description' => 'Some error text' + } + }) + end + end + + context 'without a description in the body' do + let(:response_body) do + { 'foo' => 'bar' }.to_json + end + it 'generates the correct hash' do + exception = described_class.new(uri, method, response) + exception.set_backtrace(['/foo:1', '/bar:2']) + + expect(exception.to_h).to eq({ + 'description' => 'The service broker API returned an error from http://www.example.com/: 400 Generic bad request error', + 'backtrace' => ['/foo:1', '/bar:2'], + 'http' => { + 'status' => 400, + 'uri' => uri, + 'method' => 'PUT' + }, + 'source' => { 'foo' => 'bar' } + }) + end + end + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb index bf15d12e917..894da5e23e1 100644 --- a/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/response_parser_spec.rb @@ -99,7 +99,18 @@ module V2 end end - context 'when the status code is any other error (unhandled 4xx or 5xx)' do + context 'when the status code is any other 4xx error' do + let(:code) { 400 } + let(:method) { :get } + let(:body) { '{"description": "there was an error"}' } + it 'raises ServiceBrokerRequestRejected' do + expect { + parser.parse(method, '/v2/catalog', response) + }.to raise_error(Errors::ServiceBrokerRequestRejected, /there was an error/) + end + end + + context 'when the status code is any 5xx error' do let(:code) { 500 } let(:method) { :get } let(:body) { '{"description": "there was an error"}' } From d4f16ffeac3d186145286ef1db714d386ba3c6ce Mon Sep 17 00:00:00 2001 From: Dan Lavine and Zach Robinson Date: Tue, 6 Jan 2015 17:46:05 -0800 Subject: [PATCH 16/76] Add initial paging for v3 apps. - format is not set in stone - should be added to processes so we can factor out shared code. [#79989902] --- app/controllers/v3/apps_controller.rb | 12 ++ app/handlers/apps_handler.rb | 17 ++- app/presenters/v3/app_presenter.rb | 41 +++++- .../paging/paginated_result.rb | 12 ++ .../paging/pagination_request.rb | 10 ++ .../paging/sequel_paginator.rb | 20 +++ spec/api/api_version_spec.rb | 2 +- spec/api/documentation/v3/apps_api_spec.rb | 61 ++++++++- .../controllers/v3/apps_controller_spec.rb | 48 +++++-- spec/unit/handlers/apps_handler_spec.rb | 62 ++++++++- .../paging/sequel_paginator_spec.rb | 77 +++++++++++ spec/unit/presenters/v3/app_presenter_spec.rb | 121 ++++++++++++++++++ 12 files changed, 460 insertions(+), 23 deletions(-) create mode 100644 lib/cloud_controller/paging/paginated_result.rb create mode 100644 lib/cloud_controller/paging/pagination_request.rb create mode 100644 lib/cloud_controller/paging/sequel_paginator.rb create mode 100644 spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb create mode 100644 spec/unit/presenters/v3/app_presenter_spec.rb diff --git a/app/controllers/v3/apps_controller.rb b/app/controllers/v3/apps_controller.rb index 45474b93dee..d004e1f4e7b 100644 --- a/app/controllers/v3/apps_controller.rb +++ b/app/controllers/v3/apps_controller.rb @@ -1,6 +1,7 @@ require 'presenters/v3/app_presenter' require 'handlers/processes_handler' require 'handlers/apps_handler' +require 'cloud_controller/paging/pagination_request' module VCAP::CloudController class AppsV3Controller < RestController::BaseController @@ -15,6 +16,17 @@ def inject_dependencies(dependencies) @process_presenter = dependencies[:process_presenter] end + get '/v3/apps', :list + def list + page = params['page'].to_i + per_page = params['per_page'].to_i + + pagination_request = PaginationRequest.new(page, per_page) + paginated_result = @app_handler.list(pagination_request, @access_context) + + [HTTP::OK, @app_presenter.present_json_list(paginated_result)] + end + get '/v3/apps/:guid', :show def show(guid) app = @app_handler.show(guid, @access_context) diff --git a/app/handlers/apps_handler.rb b/app/handlers/apps_handler.rb index 2bb7368a8f3..42581d2e39d 100644 --- a/app/handlers/apps_handler.rb +++ b/app/handlers/apps_handler.rb @@ -1,3 +1,6 @@ +require 'cloud_controller/paging/sequel_paginator' +require 'cloud_controller/paging/paginated_result' + module VCAP::CloudController class AppCreateMessage attr_reader :name, :space_guid @@ -46,8 +49,9 @@ class DuplicateProcessType < StandardError; end class InvalidApp < StandardError; end class IncorrectProcessSpace < StandardError; end - def initialize(process_handler) + def initialize(process_handler, paginator=SequelPaginator.new) @process_handler = process_handler + @paginator = paginator end def show(guid, access_context) @@ -56,6 +60,17 @@ def show(guid, access_context) app end + def list(pagination_request, access_context) + dataset = nil + if access_context.roles.admin? + dataset = AppModel.dataset + else + dataset = AppModel.user_visible(access_context.user) + end + + @paginator.get_page(dataset, pagination_request) + end + def create(message, access_context) app = AppModel.new app.name = message.name diff --git a/app/presenters/v3/app_presenter.rb b/app/presenters/v3/app_presenter.rb index 97c54f46d4b..22e712b8c7c 100644 --- a/app/presenters/v3/app_presenter.rb +++ b/app/presenters/v3/app_presenter.rb @@ -1,9 +1,42 @@ module VCAP::CloudController class AppPresenter def present_json(app) - app_hash = { - guid: app.guid, - name: app.name, + MultiJson.dump(app_hash(app), pretty: true) + end + + def present_json_list(paginated_result) + apps = paginated_result.records + page = paginated_result.page + per_page = paginated_result.per_page + total_results = paginated_result.total + + app_hashes = apps.collect { |app| app_hash(app) } + + last_page = (total_results.to_f / per_page.to_f).ceil + last_page = 1 if last_page < 1 + previous_page = page - 1 + next_page = page + 1 + + paginated_response = { + pagination: { + total_results: total_results, + first_url: "/v3/apps?page=1&per_page=#{per_page}", + last_url: "/v3/apps?page=#{last_page}&per_page=#{per_page}", + previous_url: previous_page > 0 ? "/v3/apps?page=#{previous_page}&per_page=#{per_page}" : nil, + next_url: next_page <= last_page ? "/v3/apps?page=#{next_page}&per_page=#{per_page}" : nil, + }, + resources: app_hashes + } + + MultiJson.dump(paginated_response, pretty: true) + end + + private + + def app_hash(app) + { + guid: app.guid, + name: app.name, _links: { self: { href: "/v3/apps/#{app.guid}" }, @@ -11,8 +44,6 @@ def present_json(app) space: { href: "/v2/spaces/#{app.space_guid}" }, } } - - MultiJson.dump(app_hash, pretty: true) end end end diff --git a/lib/cloud_controller/paging/paginated_result.rb b/lib/cloud_controller/paging/paginated_result.rb new file mode 100644 index 00000000000..1661672d8a0 --- /dev/null +++ b/lib/cloud_controller/paging/paginated_result.rb @@ -0,0 +1,12 @@ +module VCAP::CloudController + class PaginatedResult + attr_reader :records, :total, :page, :per_page + + def initialize(records, total, page, per_page) + @records = records + @total = total + @page = page + @per_page = per_page + end + end +end diff --git a/lib/cloud_controller/paging/pagination_request.rb b/lib/cloud_controller/paging/pagination_request.rb new file mode 100644 index 00000000000..62a2a336abe --- /dev/null +++ b/lib/cloud_controller/paging/pagination_request.rb @@ -0,0 +1,10 @@ +module VCAP::CloudController + class PaginationRequest + attr_reader :page, :per_page + + def initialize(page, per_page) + @page = page + @per_page = per_page + end + end +end diff --git a/lib/cloud_controller/paging/sequel_paginator.rb b/lib/cloud_controller/paging/sequel_paginator.rb new file mode 100644 index 00000000000..9628bfb46bf --- /dev/null +++ b/lib/cloud_controller/paging/sequel_paginator.rb @@ -0,0 +1,20 @@ +module VCAP::CloudController + class SequelPaginator + PAGE_DEFAULT = 1 + PER_PAGE_DEFAULT = 50 + PER_PAGE_MAX = 5000 + + def get_page(sequel_dataset, pagination_request) + page = pagination_request.page.nil? ? PAGE_DEFAULT : pagination_request.page + page = PAGE_DEFAULT if page < 1 + + per_page = pagination_request.per_page.nil? ? PER_PAGE_DEFAULT : pagination_request.per_page + per_page = PER_PAGE_DEFAULT if per_page < 1 + per_page = PER_PAGE_MAX if per_page > PER_PAGE_MAX + + query = sequel_dataset.extension(:pagination).paginate(page, per_page) + + PaginatedResult.new(query.all, query.pagination_record_count, page, per_page) + end + end +end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 34e1ca37081..7a09ec4db65 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '313d3c0d4d4186b577e08247694d34081d8bae3d' + API_FOLDER_CHECKSUM = '8feccc7f683f64682df386e2a9ece1e4c5d1df9d' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.20.0') diff --git a/spec/api/documentation/v3/apps_api_spec.rb b/spec/api/documentation/v3/apps_api_spec.rb index acd879ae1ec..df766a51f02 100644 --- a/spec/api/documentation/v3/apps_api_spec.rb +++ b/spec/api/documentation/v3/apps_api_spec.rb @@ -17,6 +17,65 @@ def do_request_with_error_handling end context 'standard endpoints' do + get '/v3/apps' do + parameter :page, 'Page to display', valid_values: '>= 1' + parameter :per_page, 'Number of results per page', valid_values: '1-5000' + + let(:name1) { 'my_app1' } + let(:name2) { 'my_app2' } + let(:name3) { 'my_app3' } + let!(:app_model1) { VCAP::CloudController::AppModel.make(name: name1, space_guid: space.guid) } + let!(:app_model2) { VCAP::CloudController::AppModel.make(name: name2, space_guid: space.guid) } + let!(:app_model3) { VCAP::CloudController::AppModel.make(name: name3, space_guid: space.guid) } + let!(:app_model4) { VCAP::CloudController::AppModel.make(space_guid: VCAP::CloudController::Space.make.guid) } + let(:space) { VCAP::CloudController::Space.make } + let(:page) { 1 } + let(:per_page) { 2 } + + before do + space.organization.add_user user + space.add_developer user + end + + example 'List all Apps' do + expected_response = { + 'pagination' => { + 'total_results' => 3, + 'first_url' => '/v3/apps?page=1&per_page=2', + 'last_url' => '/v3/apps?page=2&per_page=2', + 'next_url' => '/v3/apps?page=2&per_page=2', + 'previous_url' => nil, + }, + 'resources' => [ + { + 'name' => name1, + 'guid' => app_model1.guid, + '_links' => { + 'self' => { 'href' => "/v3/apps/#{app_model1.guid}" }, + 'processes' => { 'href' => "/v3/apps/#{app_model1.guid}/processes" }, + 'space' => { 'href' => "/v2/spaces/#{space.guid}" }, + } + }, + { + 'name' => name2, + 'guid' => app_model2.guid, + '_links' => { + 'self' => { 'href' => "/v3/apps/#{app_model2.guid}" }, + 'processes' => { 'href' => "/v3/apps/#{app_model2.guid}/processes" }, + 'space' => { 'href' => "/v2/spaces/#{space.guid}" }, + } + } + ] + } + + do_request_with_error_handling + + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(200) + expect(parsed_response).to match(expected_response) + end + end + get '/v3/apps/:guid' do let(:app_model) { VCAP::CloudController::AppModel.make(name: name) } let(:guid) { app_model.guid } @@ -68,7 +127,7 @@ def do_request_with_error_handling do_request_with_error_handling }.to change { VCAP::CloudController::AppModel.count }.by(1) - expected_guid = VCAP::CloudController::AppModel.last.guid + expected_guid = VCAP::CloudController::AppModel.last.guid expected_response = { 'name' => name, 'guid' => expected_guid, diff --git a/spec/unit/controllers/v3/apps_controller_spec.rb b/spec/unit/controllers/v3/apps_controller_spec.rb index 3b102f5162d..20bcf31bd39 100644 --- a/spec/unit/controllers/v3/apps_controller_spec.rb +++ b/spec/unit/controllers/v3/apps_controller_spec.rb @@ -5,6 +5,7 @@ module VCAP::CloudController let(:logger) { instance_double(Steno::Logger) } let(:user) { User.make } let(:req_body) { '' } + let(:params) { {} } let(:process_handler) { double(:process_handler) } let(:process_presenter) { double(:process_presenter) } let(:apps_handler) { double(:apps_handler) } @@ -12,19 +13,19 @@ module VCAP::CloudController let(:app_presenter) { double(:app_presenter) } let(:apps_controller) do AppsV3Controller.new( - {}, - logger, - {}, - {}, - req_body, - nil, - { - apps_handler: apps_handler, - app_presenter: app_presenter, - processes_handler: process_handler, - process_presenter: process_presenter - }, - ) + {}, + logger, + {}, + params, + req_body, + nil, + { + apps_handler: apps_handler, + app_presenter: app_presenter, + processes_handler: process_handler, + process_presenter: process_presenter + }, + ) end let(:app_response) { 'app_response_body' } let(:process_response) { 'process_response_body' } @@ -36,6 +37,27 @@ module VCAP::CloudController allow(process_presenter).to receive(:present_json_list).and_return(process_response) end + describe '#list' do + let(:page) { 1 } + let(:per_page) { 2 } + let(:params) { { 'page' => page, 'per_page' => per_page } } + let(:list_response) { 'list_response' } + + before do + allow(app_presenter).to receive(:present_json_list).and_return(app_response) + allow(apps_handler).to receive(:list).and_return(list_response) + end + + it 'returns 200 and lists the apps' do + response_code, response_body = apps_controller.list + + expect(apps_handler).to have_received(:list) + expect(app_presenter).to have_received(:present_json_list).with(list_response) + expect(response_code).to eq(200) + expect(response_body).to eq(app_response) + end + end + describe '#show' do context 'when the app does not exist' do let(:guid) { 'ABC123' } diff --git a/spec/unit/handlers/apps_handler_spec.rb b/spec/unit/handlers/apps_handler_spec.rb index 4b98057bc0c..67c6bb96714 100644 --- a/spec/unit/handlers/apps_handler_spec.rb +++ b/spec/unit/handlers/apps_handler_spec.rb @@ -11,6 +11,64 @@ module VCAP::CloudController allow(access_context).to receive(:cannot?).and_return(false) end + describe '#list' do + let(:space) { Space.make } + let!(:app_model1) { AppModel.make(space_guid: space.guid) } + let!(:app_model2) { AppModel.make(space_guid: space.guid) } + let(:user) { User.make } + let(:page) { 1 } + let(:per_page) { 1 } + let(:pagination_request) { PaginationRequest.new(page, per_page) } + let(:paginator) { double(:paginator) } + let(:apps_handler) { described_class.new(process_handler, paginator) } + let(:roles) { double(:roles, admin?: admin_role) } + let(:admin_role) { false } + + before do + allow(access_context).to receive(:roles).and_return(roles) + allow(access_context).to receive(:user).and_return(user) + allow(paginator).to receive(:get_page) + end + + context 'when the user is an admin' do + let(:admin_role) { true } + before do + allow(access_context).to receive(:roles).and_return(roles) + AppModel.make + end + + it 'allows viewing all apps' do + apps_handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(3) + end + end + end + + context 'when the user cannot list any apps' do + it 'applies a user visibility filter properly' do + apps_handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(0) + end + end + end + + context 'when the user can list apps' do + before do + space.organization.add_user(user) + space.add_developer(user) + end + + it 'applies a user visibility filter properly' do + apps_handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(2) + end + end + end + end + describe '#show' do let(:app_model) { AppModel.make } @@ -177,7 +235,7 @@ module VCAP::CloudController expect(result.guid).to eq(guid) expect(result.name).to eq(new_name) - updated_app = AppModel.find(guid: guid) + updated_app = AppModel.find(guid: guid) updated_process = App.find(guid: process_guid) expect(updated_app.name).to eq(new_name) @@ -190,7 +248,7 @@ module VCAP::CloudController apps_handler.update(empty_update_message, access_context) }.to raise_error - updated_app = AppModel.find(guid: guid) + updated_app = AppModel.find(guid: guid) updated_process = App.find(guid: process_guid) expect(updated_app.name).to eq(app_model.name) diff --git a/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb b/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb new file mode 100644 index 00000000000..25329845d78 --- /dev/null +++ b/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' +require 'cloud_controller/paging/sequel_paginator' +require 'cloud_controller/paging/pagination_request' + +module VCAP::CloudController + describe SequelPaginator do + let(:paginator) { SequelPaginator.new } + + describe '#get_page' do + let(:dataset) { AppModel.dataset } + let!(:app_model1) { AppModel.make } + let!(:app_model2) { AppModel.make } + let(:page) { 1 } + let(:per_page) { 1 } + + it 'defaults to the first page if page is nil' do + pagination_request = PaginationRequest.new(nil, per_page) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.page).to eq(1) + end + + it 'defaults to the first page if page is 0' do + pagination_request = PaginationRequest.new(0, per_page) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.page).to eq(1) + end + + it 'defaults to listing 50 records per page if per_page is nil' do + pagination_request = PaginationRequest.new(page, nil) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.per_page).to eq(50) + end + + it 'defaults to listing 50 records per page if per_page is 0' do + pagination_request = PaginationRequest.new(page, 0) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.per_page).to eq(50) + end + + it 'limits the listing to 5000 records per page' do + pagination_request = PaginationRequest.new(page, 5001) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.per_page).to eq(5000) + end + + it 'finds all records from the page upto the per_page limit' do + per_page = 1 + pagination_request = PaginationRequest.new(page, per_page) + + paginated_result = paginator.get_page(dataset, pagination_request) + + expect(paginated_result.records.length).to eq(1) + end + + it 'pages properly' do + pagination_request = PaginationRequest.new(1, per_page) + first_paginated_result = paginator.get_page(dataset, pagination_request) + + pagination_request = PaginationRequest.new(2, per_page) + second_paginated_result = paginator.get_page(dataset, pagination_request) + + expect(first_paginated_result.records.first.guid).to eq(app_model1.guid) + expect(second_paginated_result.records.first.guid).to eq(app_model2.guid) + end + end + end +end diff --git a/spec/unit/presenters/v3/app_presenter_spec.rb b/spec/unit/presenters/v3/app_presenter_spec.rb new file mode 100644 index 00000000000..a2874137f4c --- /dev/null +++ b/spec/unit/presenters/v3/app_presenter_spec.rb @@ -0,0 +1,121 @@ +require 'spec_helper' +require 'presenters/v3/app_presenter' + +module VCAP::CloudController + describe AppPresenter do + describe '#present_json' do + it 'presents the app as json' do + app = AppModel.make + + json_result = AppPresenter.new.present_json(app) + result = MultiJson.load(json_result) + + expect(result['guid']).to eq(app.guid) + end + end + + describe '#present_json_list' do + let(:app_model1) { AppModel.make } + let(:app_model2) { AppModel.make } + let(:apps) { [app_model1, app_model2] } + let(:presenter) { AppPresenter.new } + let(:page) { 1 } + let(:per_page) { 1 } + let(:total_results) { 2 } + let(:paginated_result) { PaginatedResult.new(apps, total_results, page, per_page) } + + it 'presents the apps as a json array under resources' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + guids = result['resources'].collect { |app_json| app_json['guid'] } + expect(guids).to eq([app_model1.guid, app_model2.guid]) + end + + it 'includes total_results' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + tr = result['pagination']['total_results'] + expect(tr).to eq(total_results) + end + + it 'includes first_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + first_url = result['pagination']['first_url'] + expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") + end + + it 'includes last_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + last_url = result['pagination']['last_url'] + expect(last_url).to eq("/v3/apps?page=2&per_page=#{per_page}") + end + + it 'sets first and last page to 1 if there is 1 page' do + paginated_result = PaginatedResult.new([], 0, page, per_page) + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + last_url = result['pagination']['last_url'] + first_url = result['pagination']['first_url'] + expect(last_url).to eq("/v3/apps?page=1&per_page=#{per_page}") + expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") + end + + context 'when on the first page' do + let(:page) { 1 } + + it 'sets previous_url to nil' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + previous_url = result['pagination']['previous_url'] + expect(previous_url).to be_nil + end + end + + context 'when NOT on the first page' do + let(:page) { 2 } + + it 'includes previous_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + previous_url = result['pagination']['previous_url'] + expect(previous_url).to eq("/v3/apps?page=1&per_page=#{per_page}") + end + end + + context 'when on the last page' do + let(:page) { apps.length } + let(:per_page) { 1 } + + it 'sets next_url to nil' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + next_url = result['pagination']['next_url'] + expect(next_url).to be_nil + end + end + + context 'when NOT on the last page' do + let(:page) { 1 } + let(:per_page) { 1 } + + it 'includes next_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + next_url = result['pagination']['next_url'] + expect(next_url).to eq("/v3/apps?page=2&per_page=#{per_page}") + end + end + end + end +end From 2c15d4f54d230516fb28f7768388d8d87a3811dd Mon Sep 17 00:00:00 2001 From: Dan Lavine and Zach Robinson Date: Wed, 7 Jan 2015 14:49:37 -0800 Subject: [PATCH 17/76] Paginate /v3/processes [#79989902] --- app/controllers/v3/apps_controller.rb | 8 +- app/controllers/v3/processes_controller.rb | 11 ++ app/handlers/processes_handler.rb | 18 ++- app/presenters/v3/app_presenter.rb | 8 +- app/presenters/v3/process_presenter.rb | 28 ++++- .../paging/sequel_paginator.rb | 2 +- spec/api/api_version_spec.rb | 2 +- spec/api/documentation/v3/apps_api_spec.rb | 32 ++++-- .../documentation/v3/processes_api_spec.rb | 49 +++++++++ .../controllers/v3/apps_controller_spec.rb | 6 + .../v3/processes_controller_spec.rb | 21 ++++ spec/unit/handlers/processes_handler_spec.rb | 89 ++++++++++++++- spec/unit/presenters/v3/app_presenter_spec.rb | 16 +-- .../presenters/v3/process_presenter_spec.rb | 103 +++++++++++++++++- 14 files changed, 351 insertions(+), 42 deletions(-) diff --git a/app/controllers/v3/apps_controller.rb b/app/controllers/v3/apps_controller.rb index d004e1f4e7b..f065adc38af 100644 --- a/app/controllers/v3/apps_controller.rb +++ b/app/controllers/v3/apps_controller.rb @@ -85,7 +85,13 @@ def list_processes(guid) app = @app_handler.show(guid, @access_context) app_not_found! if app.nil? - [HTTP::OK, @process_presenter.present_json_list(app.processes)] + page = params['page'].to_i + per_page = params['per_page'].to_i + + pagination_request = PaginationRequest.new(page, per_page) + paginated_result = @process_handler.list(pagination_request, @access_context, app_guid: app.guid) + + [HTTP::OK, @process_presenter.present_json_list(paginated_result)] end put '/v3/apps/:guid/processes', :add_process diff --git a/app/controllers/v3/processes_controller.rb b/app/controllers/v3/processes_controller.rb index 7f728ce837d..e98454405ac 100644 --- a/app/controllers/v3/processes_controller.rb +++ b/app/controllers/v3/processes_controller.rb @@ -12,6 +12,17 @@ def inject_dependencies(dependencies) @process_presenter = dependencies[:process_presenter] end + get '/v3/processes', :list + def list + page = params['page'].to_i + per_page = params['per_page'].to_i + + pagination_request = PaginationRequest.new(page, per_page) + paginated_result = @processes_handler.list(pagination_request, @access_context) + + [HTTP::OK, @process_presenter.present_json_list(paginated_result)] + end + get '/v3/processes/:guid', :show def show(guid) process = @processes_handler.show(guid, @access_context) diff --git a/app/handlers/processes_handler.rb b/app/handlers/processes_handler.rb index 13c483da66c..8b8ac2e64d3 100644 --- a/app/handlers/processes_handler.rb +++ b/app/handlers/processes_handler.rb @@ -66,9 +66,23 @@ class ProcessesHandler class InvalidProcess < StandardError; end class Unauthorized < StandardError; end - def initialize(process_repository, process_event_repository) - @process_repository = process_repository + def initialize(process_repository, process_event_repository, paginator=SequelPaginator.new) + @process_repository = process_repository @process_event_repository = process_event_repository + @paginator = paginator + end + + def list(pagination_request, access_context, filter_options={}) + dataset = nil + if access_context.roles.admin? + dataset = App.dataset + else + dataset = App.user_visible(access_context.user) + end + + dataset = dataset.where(app_guid: filter_options[:app_guid]) if filter_options[:app_guid] + + @paginator.get_page(dataset, pagination_request) end def show(guid, access_context) diff --git a/app/presenters/v3/app_presenter.rb b/app/presenters/v3/app_presenter.rb index 22e712b8c7c..a857086ad5d 100644 --- a/app/presenters/v3/app_presenter.rb +++ b/app/presenters/v3/app_presenter.rb @@ -20,10 +20,10 @@ def present_json_list(paginated_result) paginated_response = { pagination: { total_results: total_results, - first_url: "/v3/apps?page=1&per_page=#{per_page}", - last_url: "/v3/apps?page=#{last_page}&per_page=#{per_page}", - previous_url: previous_page > 0 ? "/v3/apps?page=#{previous_page}&per_page=#{per_page}" : nil, - next_url: next_page <= last_page ? "/v3/apps?page=#{next_page}&per_page=#{per_page}" : nil, + first: { href: "/v3/apps?page=1&per_page=#{per_page}" }, + last: { href: "/v3/apps?page=#{last_page}&per_page=#{per_page}" }, + previous: previous_page > 0 ? { href: "/v3/apps?page=#{previous_page}&per_page=#{per_page}" } : nil, + next: next_page <= last_page ? { href: "/v3/apps?page=#{next_page}&per_page=#{per_page}" } : nil, }, resources: app_hashes } diff --git a/app/presenters/v3/process_presenter.rb b/app/presenters/v3/process_presenter.rb index 57921e36507..85a77bd2e45 100644 --- a/app/presenters/v3/process_presenter.rb +++ b/app/presenters/v3/process_presenter.rb @@ -4,9 +4,31 @@ def present_json(process) MultiJson.dump(process_hash(process), pretty: true) end - def present_json_list(processes) - process_hashes = processes.collect { |process| process_hash(process) } - MultiJson.dump(process_hashes, pretty: true) + def present_json_list(paginated_result) + processes = paginated_result.records + page = paginated_result.page + per_page = paginated_result.per_page + total_results = paginated_result.total + + process_hashes = processes.collect { |app| process_hash(app) } + + last_page = (total_results.to_f / per_page.to_f).ceil + last_page = 1 if last_page < 1 + previous_page = page - 1 + next_page = page + 1 + + paginated_response = { + pagination: { + total_results: total_results, + first: { href: "/v3/processes?page=1&per_page=#{per_page}" }, + last: { href: "/v3/processes?page=#{last_page}&per_page=#{per_page}" }, + next: next_page <= last_page ? { href: "/v3/processes?page=#{next_page}&per_page=#{per_page}" } : nil, + previous: previous_page > 0 ? { href: "/v3/processes?page=#{previous_page}&per_page=#{per_page}" } : nil, + }, + resources: process_hashes + } + + MultiJson.dump(paginated_response, pretty: true) end private diff --git a/lib/cloud_controller/paging/sequel_paginator.rb b/lib/cloud_controller/paging/sequel_paginator.rb index 9628bfb46bf..5e7b142f116 100644 --- a/lib/cloud_controller/paging/sequel_paginator.rb +++ b/lib/cloud_controller/paging/sequel_paginator.rb @@ -12,7 +12,7 @@ def get_page(sequel_dataset, pagination_request) per_page = PER_PAGE_DEFAULT if per_page < 1 per_page = PER_PAGE_MAX if per_page > PER_PAGE_MAX - query = sequel_dataset.extension(:pagination).paginate(page, per_page) + query = sequel_dataset.extension(:pagination).paginate(page, per_page).order(:id) PaginatedResult.new(query.all, query.pagination_record_count, page, per_page) end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 7a09ec4db65..1018fdda01e 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '8feccc7f683f64682df386e2a9ece1e4c5d1df9d' + API_FOLDER_CHECKSUM = '4ca579532ca567f0e62a3ba0c57a3bfdc5dc66ed' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.20.0') diff --git a/spec/api/documentation/v3/apps_api_spec.rb b/spec/api/documentation/v3/apps_api_spec.rb index df766a51f02..bf0af2bc740 100644 --- a/spec/api/documentation/v3/apps_api_spec.rb +++ b/spec/api/documentation/v3/apps_api_spec.rb @@ -41,10 +41,10 @@ def do_request_with_error_handling expected_response = { 'pagination' => { 'total_results' => 3, - 'first_url' => '/v3/apps?page=1&per_page=2', - 'last_url' => '/v3/apps?page=2&per_page=2', - 'next_url' => '/v3/apps?page=2&per_page=2', - 'previous_url' => nil, + 'first' => { 'href' => '/v3/apps?page=1&per_page=2' }, + 'last' => { 'href' => '/v3/apps?page=2&per_page=2' }, + 'next' => { 'href' => '/v3/apps?page=2&per_page=2' }, + 'previous' => nil, }, 'resources' => [ { @@ -54,7 +54,7 @@ def do_request_with_error_handling 'self' => { 'href' => "/v3/apps/#{app_model1.guid}" }, 'processes' => { 'href' => "/v3/apps/#{app_model1.guid}/processes" }, 'space' => { 'href' => "/v2/spaces/#{space.guid}" }, - } + } }, { 'name' => name2, @@ -250,12 +250,22 @@ def do_request_with_error_handling end example 'List associated processes' do - expected_response = [ - { - 'guid' => process_guid, - 'type' => process_type, - } - ] + expected_response = { + 'pagination' => { + 'total_results' => 1, + 'first' => { 'href' => '/v3/processes?page=1&per_page=50' }, + 'last' => { 'href' => '/v3/processes?page=1&per_page=50' }, + 'next' => nil, + 'previous' => nil, + }, + 'resources' => [ + { + 'guid' => process_guid, + 'type' => process_type, + } + ] + } + do_request_with_error_handling parsed_response = MultiJson.load(response_body) diff --git a/spec/api/documentation/v3/processes_api_spec.rb b/spec/api/documentation/v3/processes_api_spec.rb index 2c3b499d4d2..9e37eb02a57 100644 --- a/spec/api/documentation/v3/processes_api_spec.rb +++ b/spec/api/documentation/v3/processes_api_spec.rb @@ -16,6 +16,55 @@ def do_request_with_error_handling end end + get '/v3/processes' do + parameter :page, 'Page to display', valid_values: '>= 1' + parameter :per_page, 'Number of results per page', valid_values: '1-5000' + + let(:name1) { 'my_process1' } + let(:name2) { 'my_process2' } + let(:name3) { 'my_process3' } + let!(:process1) { VCAP::CloudController::App.make(name: name1, space: space) } + let!(:process2) { VCAP::CloudController::App.make(name: name2, space: space) } + let!(:process3) { VCAP::CloudController::App.make(name: name3, space: space) } + let!(:process4) { VCAP::CloudController::App.make(space: VCAP::CloudController::Space.make) } + let(:space) { VCAP::CloudController::Space.make } + let(:page) { 1 } + let(:per_page) { 2 } + + before do + space.organization.add_user user + space.add_developer user + end + + example 'List all Processes' do + expected_response = { + 'pagination' => { + 'total_results' => 3, + 'first' => { 'href' => '/v3/processes?page=1&per_page=2' }, + 'last' => { 'href' => '/v3/processes?page=2&per_page=2' }, + 'next' => { 'href' => '/v3/processes?page=2&per_page=2' }, + 'previous' => nil, + }, + 'resources' => [ + { + 'guid' => process1.guid, + 'type' => process1.type, + }, + { + 'guid' => process2.guid, + 'type' => process2.type, + } + ] + } + + do_request_with_error_handling + + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(200) + expect(parsed_response).to match(expected_response) + end + end + get '/v3/processes/:guid' do let(:process) { VCAP::CloudController::AppFactory.make } let(:guid) { process.guid } diff --git a/spec/unit/controllers/v3/apps_controller_spec.rb b/spec/unit/controllers/v3/apps_controller_spec.rb index 20bcf31bd39..c622687108e 100644 --- a/spec/unit/controllers/v3/apps_controller_spec.rb +++ b/spec/unit/controllers/v3/apps_controller_spec.rb @@ -511,6 +511,12 @@ module VCAP::CloudController context 'when the app does exist' do let(:app_model) { AppModel.make } let(:guid) { app_model.guid } + let(:list_response) { 'list_response' } + + before do + allow(process_presenter).to receive(:present_json_list).and_return(process_response) + allow(process_handler).to receive(:list).and_return(list_response) + end it 'returns a 200' do response_code, _ = apps_controller.list_processes(guid) diff --git a/spec/unit/controllers/v3/processes_controller_spec.rb b/spec/unit/controllers/v3/processes_controller_spec.rb index c270c3ca664..c1595e46bf4 100644 --- a/spec/unit/controllers/v3/processes_controller_spec.rb +++ b/spec/unit/controllers/v3/processes_controller_spec.rb @@ -33,6 +33,27 @@ module VCAP::CloudController allow(process_presenter).to receive(:present_json).and_return(expected_response) end + describe '#list' do + let(:page) { 1 } + let(:per_page) { 2 } + let(:params) { { 'page' => page, 'per_page' => per_page } } + let(:list_response) { 'list_response' } + + before do + allow(process_presenter).to receive(:present_json_list).and_return(expected_response) + allow(processes_handler).to receive(:list).and_return(list_response) + end + + it 'returns 200 and lists the apps' do + response_code, response_body = process_controller.list + + expect(processes_handler).to have_received(:list) + expect(process_presenter).to have_received(:present_json_list).with(list_response) + expect(response_code).to eq(200) + expect(response_body).to eq(expected_response) + end + end + describe '#show' do before do allow(processes_handler).to receive(:show).and_return(process) diff --git a/spec/unit/handlers/processes_handler_spec.rb b/spec/unit/handlers/processes_handler_spec.rb index 48d0eba88b2..02b9e5d03ff 100644 --- a/spec/unit/handlers/processes_handler_spec.rb +++ b/spec/unit/handlers/processes_handler_spec.rb @@ -31,14 +31,87 @@ module VCAP::CloudController let(:process_repo) { double(:process_repo) } let(:process_event_repo) { double(:process_event_repo) } let(:space) { Space.make } - let!(:handler) { ProcessesHandler.new(process_repo, process_event_repo) } - let(:process_opts) { { space: space } } - let!(:process) do - process_model = AppFactory.make(process_opts) - ProcessMapper.map_model_to_domain(process_model) + let(:handler) { ProcessesHandler.new(process_repo, process_event_repo) } + let(:access_context) { double(:access_context) } + + describe '#list' do + let!(:process1) { AppFactory.make(space: space) } + let!(:process2) { AppFactory.make(space: space) } + let(:user) { User.make } + let(:page) { 1 } + let(:per_page) { 1 } + let(:pagination_request) { PaginationRequest.new(page, per_page) } + let(:paginator) { double(:paginator) } + let(:handler) { described_class.new(process_repo, process_event_repo, paginator) } + let(:roles) { double(:roles, admin?: admin_role) } + let(:admin_role) { false } + + before do + allow(access_context).to receive(:roles).and_return(roles) + allow(access_context).to receive(:user).and_return(user) + allow(paginator).to receive(:get_page) + end + + context 'when the user is an admin' do + let(:admin_role) { true } + before do + allow(access_context).to receive(:roles).and_return(roles) + AppFactory.make + end + + it 'allows viewing all processes' do + handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(3) + end + end + end + + context 'when the user cannot list any processes' do + it 'applies a user visibility filter properly' do + handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(0) + end + end + end + + context 'when the user can list processes' do + before do + space.organization.add_user(user) + space.add_developer(user) + end + + it 'applies a user visibility filter properly' do + handler.list(pagination_request, access_context) + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(2) + end + end + + it 'can filter by app_guid' do + v3app = AppModel.make + process1.app_guid = v3app.guid + process1.save + + filter_options = { app_guid: v3app.guid } + + handler.list(pagination_request, access_context, filter_options) + + expect(paginator).to have_received(:get_page) do |dataset, _| + expect(dataset.count).to eq(1) + end + end + end end context '#update' do + let(:process_opts) { { space: space } } + let(:process) do + process_model = AppFactory.make(process_opts) + ProcessMapper.map_model_to_domain(process_model) + end + context 'changing type to an invalid value' do it 'raises an InvalidProcess exception' do update_opts = { 'type' => 'worker' } @@ -113,6 +186,12 @@ module VCAP::CloudController end context '#delete' do + let(:process_opts) { { space: space } } + let(:process) do + process_model = AppFactory.make(process_opts) + ProcessMapper.map_model_to_domain(process_model) + end + it 'saves an event when deleting a process' do ac = double(:ac, user: User.make, user_email: 'jim@jim.com') diff --git a/spec/unit/presenters/v3/app_presenter_spec.rb b/spec/unit/presenters/v3/app_presenter_spec.rb index a2874137f4c..c25b5fe046e 100644 --- a/spec/unit/presenters/v3/app_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_presenter_spec.rb @@ -44,7 +44,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - first_url = result['pagination']['first_url'] + first_url = result['pagination']['first']['href'] expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") end @@ -52,7 +52,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - last_url = result['pagination']['last_url'] + last_url = result['pagination']['last']['href'] expect(last_url).to eq("/v3/apps?page=2&per_page=#{per_page}") end @@ -61,8 +61,8 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - last_url = result['pagination']['last_url'] - first_url = result['pagination']['first_url'] + last_url = result['pagination']['last']['href'] + first_url = result['pagination']['first']['href'] expect(last_url).to eq("/v3/apps?page=1&per_page=#{per_page}") expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") end @@ -74,7 +74,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - previous_url = result['pagination']['previous_url'] + previous_url = result['pagination']['previous'] expect(previous_url).to be_nil end end @@ -86,7 +86,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - previous_url = result['pagination']['previous_url'] + previous_url = result['pagination']['previous']['href'] expect(previous_url).to eq("/v3/apps?page=1&per_page=#{per_page}") end end @@ -99,7 +99,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - next_url = result['pagination']['next_url'] + next_url = result['pagination']['next'] expect(next_url).to be_nil end end @@ -112,7 +112,7 @@ module VCAP::CloudController json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - next_url = result['pagination']['next_url'] + next_url = result['pagination']['next']['href'] expect(next_url).to eq("/v3/apps?page=2&per_page=#{per_page}") end end diff --git a/spec/unit/presenters/v3/process_presenter_spec.rb b/spec/unit/presenters/v3/process_presenter_spec.rb index 07a566d2599..77e8d714451 100644 --- a/spec/unit/presenters/v3/process_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_presenter_spec.rb @@ -16,15 +16,106 @@ module VCAP::CloudController end describe '#present_json_list' do - it 'presents the processes as a json array' do - process_model1 = AppFactory.make - process_model2 = AppFactory.make + let(:process1) { AppFactory.make } + let(:process2) { AppFactory.make } + let(:processes) { [process1, process2] } + let(:presenter) { ProcessPresenter.new } + let(:page) { 1 } + let(:per_page) { 1 } + let(:total_results) { 2 } + let(:paginated_result) { PaginatedResult.new(processes, total_results, page, per_page) } - json_result = ProcessPresenter.new.present_json_list([process_model1, process_model2]) + it 'presents the processes as a json array under resources' do + json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - guids = result.collect { |process_json| process_json['guid'] } - expect(guids).to eq([process_model1.guid, process_model2.guid]) + guids = result['resources'].collect { |app_json| app_json['guid'] } + expect(guids).to eq([process1.guid, process2.guid]) + end + + it 'includes total_results' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + tr = result['pagination']['total_results'] + expect(tr).to eq(total_results) + end + + it 'includes first_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + first_url = result['pagination']['first']['href'] + expect(first_url).to eq("/v3/processes?page=1&per_page=#{per_page}") + end + + it 'includes last_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + last_url = result['pagination']['last']['href'] + expect(last_url).to eq("/v3/processes?page=2&per_page=#{per_page}") + end + + it 'sets first and last page to 1 if there is 1 page' do + paginated_result = PaginatedResult.new([], 0, page, per_page) + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + last_url = result['pagination']['last']['href'] + first_url = result['pagination']['first']['href'] + expect(last_url).to eq("/v3/processes?page=1&per_page=#{per_page}") + expect(first_url).to eq("/v3/processes?page=1&per_page=#{per_page}") + end + + context 'when on the first page' do + let(:page) { 1 } + + it 'sets previous_url to nil' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + previous_url = result['pagination']['previous'] + expect(previous_url).to be_nil + end + end + + context 'when NOT on the first page' do + let(:page) { 2 } + + it 'includes previous_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + previous_url = result['pagination']['previous']['href'] + expect(previous_url).to eq("/v3/processes?page=1&per_page=#{per_page}") + end + end + + context 'when on the last page' do + let(:page) { processes.length } + let(:per_page) { 1 } + + it 'sets next_url to nil' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + next_url = result['pagination']['next'] + expect(next_url).to be_nil + end + end + + context 'when NOT on the last page' do + let(:page) { 1 } + let(:per_page) { 1 } + + it 'includes next_url' do + json_result = presenter.present_json_list(paginated_result) + result = MultiJson.load(json_result) + + next_url = result['pagination']['next']['href'] + expect(next_url).to eq("/v3/processes?page=2&per_page=#{per_page}") + end end end end From ce1da01ca0b3950318d869771ed68103d1e8611e Mon Sep 17 00:00:00 2001 From: Luan Santos and Zach Robinson Date: Wed, 7 Jan 2015 16:23:06 -0800 Subject: [PATCH 18/76] Refactor pagination presentation logic into its own class. [#79989902] --- app/presenters/v3/app_presenter.rb | 25 ++--- app/presenters/v3/pagination_presenter.rb | 22 +++++ app/presenters/v3/process_presenter.rb | 25 ++--- spec/unit/presenters/v3/app_presenter_spec.rb | 85 +---------------- .../v3/pagination_presenter_spec.rb | 92 +++++++++++++++++++ .../presenters/v3/process_presenter_spec.rb | 85 +---------------- 6 files changed, 138 insertions(+), 196 deletions(-) create mode 100644 app/presenters/v3/pagination_presenter.rb create mode 100644 spec/unit/presenters/v3/pagination_presenter_spec.rb diff --git a/app/presenters/v3/app_presenter.rb b/app/presenters/v3/app_presenter.rb index a857086ad5d..e0f061e4c76 100644 --- a/app/presenters/v3/app_presenter.rb +++ b/app/presenters/v3/app_presenter.rb @@ -1,30 +1,21 @@ +require 'presenters/v3/pagination_presenter' + module VCAP::CloudController class AppPresenter + def initialize(pagination_presenter=PaginationPresenter.new) + @pagination_presenter = pagination_presenter + end + def present_json(app) MultiJson.dump(app_hash(app), pretty: true) end def present_json_list(paginated_result) - apps = paginated_result.records - page = paginated_result.page - per_page = paginated_result.per_page - total_results = paginated_result.total - + apps = paginated_result.records app_hashes = apps.collect { |app| app_hash(app) } - last_page = (total_results.to_f / per_page.to_f).ceil - last_page = 1 if last_page < 1 - previous_page = page - 1 - next_page = page + 1 - paginated_response = { - pagination: { - total_results: total_results, - first: { href: "/v3/apps?page=1&per_page=#{per_page}" }, - last: { href: "/v3/apps?page=#{last_page}&per_page=#{per_page}" }, - previous: previous_page > 0 ? { href: "/v3/apps?page=#{previous_page}&per_page=#{per_page}" } : nil, - next: next_page <= last_page ? { href: "/v3/apps?page=#{next_page}&per_page=#{per_page}" } : nil, - }, + pagination: @pagination_presenter.present_pagination_hash(paginated_result, '/v3/apps'), resources: app_hashes } diff --git a/app/presenters/v3/pagination_presenter.rb b/app/presenters/v3/pagination_presenter.rb new file mode 100644 index 00000000000..0dcc50e364f --- /dev/null +++ b/app/presenters/v3/pagination_presenter.rb @@ -0,0 +1,22 @@ +module VCAP::CloudController + class PaginationPresenter + def present_pagination_hash(paginated_result, base_url) + page = paginated_result.page + per_page = paginated_result.per_page + total_results = paginated_result.total + + last_page = (total_results.to_f / per_page.to_f).ceil + last_page = 1 if last_page < 1 + previous_page = page - 1 + next_page = page + 1 + + { + total_results: total_results, + first: { href: "#{base_url}?page=1&per_page=#{per_page}" }, + last: { href: "#{base_url}?page=#{last_page}&per_page=#{per_page}" }, + next: next_page <= last_page ? { href: "#{base_url}?page=#{next_page}&per_page=#{per_page}" } : nil, + previous: previous_page > 0 ? { href: "#{base_url}?page=#{previous_page}&per_page=#{per_page}" } : nil, + } + end + end +end diff --git a/app/presenters/v3/process_presenter.rb b/app/presenters/v3/process_presenter.rb index 85a77bd2e45..29f74be6739 100644 --- a/app/presenters/v3/process_presenter.rb +++ b/app/presenters/v3/process_presenter.rb @@ -1,30 +1,21 @@ +require 'presenters/v3/pagination_presenter' + module VCAP::CloudController class ProcessPresenter + def initialize(pagination_presenter=PaginationPresenter.new) + @pagination_presenter = pagination_presenter + end + def present_json(process) MultiJson.dump(process_hash(process), pretty: true) end def present_json_list(paginated_result) - processes = paginated_result.records - page = paginated_result.page - per_page = paginated_result.per_page - total_results = paginated_result.total - + processes = paginated_result.records process_hashes = processes.collect { |app| process_hash(app) } - last_page = (total_results.to_f / per_page.to_f).ceil - last_page = 1 if last_page < 1 - previous_page = page - 1 - next_page = page + 1 - paginated_response = { - pagination: { - total_results: total_results, - first: { href: "/v3/processes?page=1&per_page=#{per_page}" }, - last: { href: "/v3/processes?page=#{last_page}&per_page=#{per_page}" }, - next: next_page <= last_page ? { href: "/v3/processes?page=#{next_page}&per_page=#{per_page}" } : nil, - previous: previous_page > 0 ? { href: "/v3/processes?page=#{previous_page}&per_page=#{per_page}" } : nil, - }, + pagination: @pagination_presenter.present_pagination_hash(paginated_result, '/v3/processes'), resources: process_hashes } diff --git a/spec/unit/presenters/v3/app_presenter_spec.rb b/spec/unit/presenters/v3/app_presenter_spec.rb index c25b5fe046e..b9e398c4924 100644 --- a/spec/unit/presenters/v3/app_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_presenter_spec.rb @@ -15,10 +15,11 @@ module VCAP::CloudController end describe '#present_json_list' do + let(:pagination_presenter) { double(:pagination_presenter, present_pagination_hash: 'pagination_stuff') } let(:app_model1) { AppModel.make } let(:app_model2) { AppModel.make } let(:apps) { [app_model1, app_model2] } - let(:presenter) { AppPresenter.new } + let(:presenter) { AppPresenter.new(pagination_presenter) } let(:page) { 1 } let(:per_page) { 1 } let(:total_results) { 2 } @@ -32,89 +33,11 @@ module VCAP::CloudController expect(guids).to eq([app_model1.guid, app_model2.guid]) end - it 'includes total_results' do + it 'includes pagination section' do json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - tr = result['pagination']['total_results'] - expect(tr).to eq(total_results) - end - - it 'includes first_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - first_url = result['pagination']['first']['href'] - expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") - end - - it 'includes last_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - last_url = result['pagination']['last']['href'] - expect(last_url).to eq("/v3/apps?page=2&per_page=#{per_page}") - end - - it 'sets first and last page to 1 if there is 1 page' do - paginated_result = PaginatedResult.new([], 0, page, per_page) - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - last_url = result['pagination']['last']['href'] - first_url = result['pagination']['first']['href'] - expect(last_url).to eq("/v3/apps?page=1&per_page=#{per_page}") - expect(first_url).to eq("/v3/apps?page=1&per_page=#{per_page}") - end - - context 'when on the first page' do - let(:page) { 1 } - - it 'sets previous_url to nil' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - previous_url = result['pagination']['previous'] - expect(previous_url).to be_nil - end - end - - context 'when NOT on the first page' do - let(:page) { 2 } - - it 'includes previous_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - previous_url = result['pagination']['previous']['href'] - expect(previous_url).to eq("/v3/apps?page=1&per_page=#{per_page}") - end - end - - context 'when on the last page' do - let(:page) { apps.length } - let(:per_page) { 1 } - - it 'sets next_url to nil' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - next_url = result['pagination']['next'] - expect(next_url).to be_nil - end - end - - context 'when NOT on the last page' do - let(:page) { 1 } - let(:per_page) { 1 } - - it 'includes next_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - next_url = result['pagination']['next']['href'] - expect(next_url).to eq("/v3/apps?page=2&per_page=#{per_page}") - end + expect(result['pagination']).to eq('pagination_stuff') end end end diff --git a/spec/unit/presenters/v3/pagination_presenter_spec.rb b/spec/unit/presenters/v3/pagination_presenter_spec.rb new file mode 100644 index 00000000000..cdb9075987d --- /dev/null +++ b/spec/unit/presenters/v3/pagination_presenter_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' +require 'presenters/v3/pagination_presenter' + +module VCAP::CloudController + describe PaginationPresenter do + describe '#present_pagination_hash' do + let(:presenter) { PaginationPresenter.new } + let(:page) { 1 } + let(:per_page) { 1 } + let(:total_results) { 2 } + let(:paginated_result) { PaginatedResult.new(double(:results), total_results, page, per_page) } + let(:base_url) { '/cloudfoundry/is-great' } + + it 'includes total_results' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + tr = result[:total_results] + expect(tr).to eq(total_results) + end + + it 'includes first_url' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + first_url = result[:first][:href] + expect(first_url).to eq("/cloudfoundry/is-great?page=1&per_page=#{per_page}") + end + + it 'includes last_url' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + last_url = result[:last][:href] + expect(last_url).to eq("/cloudfoundry/is-great?page=2&per_page=#{per_page}") + end + + it 'sets first and last page to 1 if there is 1 page' do + paginated_result = PaginatedResult.new([], 0, page, per_page) + result = presenter.present_pagination_hash(paginated_result, base_url) + + last_url = result[:last][:href] + first_url = result[:first][:href] + expect(last_url).to eq("/cloudfoundry/is-great?page=1&per_page=#{per_page}") + expect(first_url).to eq("/cloudfoundry/is-great?page=1&per_page=#{per_page}") + end + + context 'when on the first page' do + let(:page) { 1 } + + it 'sets previous_url to nil' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + previous_url = result[:previous] + expect(previous_url).to be_nil + end + end + + context 'when NOT on the first page' do + let(:page) { 2 } + + it 'includes previous_url' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + previous_url = result[:previous][:href] + expect(previous_url).to eq("/cloudfoundry/is-great?page=1&per_page=#{per_page}") + end + end + + context 'when on the last page' do + let(:page) { total_results / per_page } + let(:per_page) { 1 } + + it 'sets next_url to nil' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + next_url = result[:next] + expect(next_url).to be_nil + end + end + + context 'when NOT on the last page' do + let(:page) { 1 } + let(:per_page) { 1 } + + it 'includes next_url' do + result = presenter.present_pagination_hash(paginated_result, base_url) + + next_url = result[:next][:href] + expect(next_url).to eq("/cloudfoundry/is-great?page=2&per_page=#{per_page}") + end + end + end + end +end diff --git a/spec/unit/presenters/v3/process_presenter_spec.rb b/spec/unit/presenters/v3/process_presenter_spec.rb index 77e8d714451..604e54d3a4a 100644 --- a/spec/unit/presenters/v3/process_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_presenter_spec.rb @@ -16,10 +16,11 @@ module VCAP::CloudController end describe '#present_json_list' do + let(:pagination_presenter) { double(:pagination_presenter, present_pagination_hash: 'pagination_stuff') } let(:process1) { AppFactory.make } let(:process2) { AppFactory.make } let(:processes) { [process1, process2] } - let(:presenter) { ProcessPresenter.new } + let(:presenter) { ProcessPresenter.new(pagination_presenter) } let(:page) { 1 } let(:per_page) { 1 } let(:total_results) { 2 } @@ -33,89 +34,11 @@ module VCAP::CloudController expect(guids).to eq([process1.guid, process2.guid]) end - it 'includes total_results' do + it 'includes pagination section' do json_result = presenter.present_json_list(paginated_result) result = MultiJson.load(json_result) - tr = result['pagination']['total_results'] - expect(tr).to eq(total_results) - end - - it 'includes first_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - first_url = result['pagination']['first']['href'] - expect(first_url).to eq("/v3/processes?page=1&per_page=#{per_page}") - end - - it 'includes last_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - last_url = result['pagination']['last']['href'] - expect(last_url).to eq("/v3/processes?page=2&per_page=#{per_page}") - end - - it 'sets first and last page to 1 if there is 1 page' do - paginated_result = PaginatedResult.new([], 0, page, per_page) - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - last_url = result['pagination']['last']['href'] - first_url = result['pagination']['first']['href'] - expect(last_url).to eq("/v3/processes?page=1&per_page=#{per_page}") - expect(first_url).to eq("/v3/processes?page=1&per_page=#{per_page}") - end - - context 'when on the first page' do - let(:page) { 1 } - - it 'sets previous_url to nil' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - previous_url = result['pagination']['previous'] - expect(previous_url).to be_nil - end - end - - context 'when NOT on the first page' do - let(:page) { 2 } - - it 'includes previous_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - previous_url = result['pagination']['previous']['href'] - expect(previous_url).to eq("/v3/processes?page=1&per_page=#{per_page}") - end - end - - context 'when on the last page' do - let(:page) { processes.length } - let(:per_page) { 1 } - - it 'sets next_url to nil' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - next_url = result['pagination']['next'] - expect(next_url).to be_nil - end - end - - context 'when NOT on the last page' do - let(:page) { 1 } - let(:per_page) { 1 } - - it 'includes next_url' do - json_result = presenter.present_json_list(paginated_result) - result = MultiJson.load(json_result) - - next_url = result['pagination']['next']['href'] - expect(next_url).to eq("/v3/processes?page=2&per_page=#{per_page}") - end + expect(result['pagination']).to eq('pagination_stuff') end end end From 22bf04e21f0a57af854c86bfe30a61f18cc437ca Mon Sep 17 00:00:00 2001 From: Luan Santos and Zach Robinson Date: Wed, 7 Jan 2015 16:46:44 -0800 Subject: [PATCH 19/76] Refactor PaginationOptions to be created from request params [#79989902] --- app/controllers/v3/apps_controller.rb | 16 +++---- app/controllers/v3/processes_controller.rb | 8 ++-- app/handlers/apps_handler.rb | 4 +- app/handlers/processes_handler.rb | 4 +- app/presenters/v3/pagination_presenter.rb | 4 +- .../paging/paginated_result.rb | 11 +++-- .../paging/pagination_options.rb | 16 +++++++ .../paging/pagination_request.rb | 10 ----- .../paging/sequel_paginator.rb | 8 ++-- spec/unit/handlers/apps_handler_spec.rb | 8 ++-- spec/unit/handlers/processes_handler_spec.rb | 10 ++--- .../paging/sequel_paginator_spec.rb | 44 +++++++++---------- spec/unit/presenters/v3/app_presenter_spec.rb | 2 +- .../v3/pagination_presenter_spec.rb | 4 +- .../presenters/v3/process_presenter_spec.rb | 2 +- 15 files changed, 74 insertions(+), 77 deletions(-) create mode 100644 lib/cloud_controller/paging/pagination_options.rb delete mode 100644 lib/cloud_controller/paging/pagination_request.rb diff --git a/app/controllers/v3/apps_controller.rb b/app/controllers/v3/apps_controller.rb index f065adc38af..96d908cd695 100644 --- a/app/controllers/v3/apps_controller.rb +++ b/app/controllers/v3/apps_controller.rb @@ -1,7 +1,7 @@ require 'presenters/v3/app_presenter' require 'handlers/processes_handler' require 'handlers/apps_handler' -require 'cloud_controller/paging/pagination_request' +require 'cloud_controller/paging/pagination_options' module VCAP::CloudController class AppsV3Controller < RestController::BaseController @@ -18,11 +18,8 @@ def inject_dependencies(dependencies) get '/v3/apps', :list def list - page = params['page'].to_i - per_page = params['per_page'].to_i - - pagination_request = PaginationRequest.new(page, per_page) - paginated_result = @app_handler.list(pagination_request, @access_context) + pagination_options = PaginationOptions.from_params(params) + paginated_result = @app_handler.list(pagination_options, @access_context) [HTTP::OK, @app_presenter.present_json_list(paginated_result)] end @@ -85,11 +82,8 @@ def list_processes(guid) app = @app_handler.show(guid, @access_context) app_not_found! if app.nil? - page = params['page'].to_i - per_page = params['per_page'].to_i - - pagination_request = PaginationRequest.new(page, per_page) - paginated_result = @process_handler.list(pagination_request, @access_context, app_guid: app.guid) + pagination_options = PaginationOptions.from_params(params) + paginated_result = @process_handler.list(pagination_options, @access_context, app_guid: app.guid) [HTTP::OK, @process_presenter.present_json_list(paginated_result)] end diff --git a/app/controllers/v3/processes_controller.rb b/app/controllers/v3/processes_controller.rb index e98454405ac..fc59d93c070 100644 --- a/app/controllers/v3/processes_controller.rb +++ b/app/controllers/v3/processes_controller.rb @@ -1,5 +1,6 @@ require 'presenters/v3/process_presenter' require 'handlers/processes_handler' +require 'cloud_controller/paging/pagination_options' module VCAP::CloudController # TODO: would be nice not needing to use this BaseController @@ -14,11 +15,8 @@ def inject_dependencies(dependencies) get '/v3/processes', :list def list - page = params['page'].to_i - per_page = params['per_page'].to_i - - pagination_request = PaginationRequest.new(page, per_page) - paginated_result = @processes_handler.list(pagination_request, @access_context) + pagination_options = PaginationOptions.from_params(params) + paginated_result = @processes_handler.list(pagination_options, @access_context) [HTTP::OK, @process_presenter.present_json_list(paginated_result)] end diff --git a/app/handlers/apps_handler.rb b/app/handlers/apps_handler.rb index 42581d2e39d..72a246d236b 100644 --- a/app/handlers/apps_handler.rb +++ b/app/handlers/apps_handler.rb @@ -60,7 +60,7 @@ def show(guid, access_context) app end - def list(pagination_request, access_context) + def list(pagination_options, access_context) dataset = nil if access_context.roles.admin? dataset = AppModel.dataset @@ -68,7 +68,7 @@ def list(pagination_request, access_context) dataset = AppModel.user_visible(access_context.user) end - @paginator.get_page(dataset, pagination_request) + @paginator.get_page(dataset, pagination_options) end def create(message, access_context) diff --git a/app/handlers/processes_handler.rb b/app/handlers/processes_handler.rb index 8b8ac2e64d3..475d7a325a0 100644 --- a/app/handlers/processes_handler.rb +++ b/app/handlers/processes_handler.rb @@ -72,7 +72,7 @@ def initialize(process_repository, process_event_repository, paginator=SequelPag @paginator = paginator end - def list(pagination_request, access_context, filter_options={}) + def list(pagination_options, access_context, filter_options={}) dataset = nil if access_context.roles.admin? dataset = App.dataset @@ -82,7 +82,7 @@ def list(pagination_request, access_context, filter_options={}) dataset = dataset.where(app_guid: filter_options[:app_guid]) if filter_options[:app_guid] - @paginator.get_page(dataset, pagination_request) + @paginator.get_page(dataset, pagination_options) end def show(guid, access_context) diff --git a/app/presenters/v3/pagination_presenter.rb b/app/presenters/v3/pagination_presenter.rb index 0dcc50e364f..f169babb790 100644 --- a/app/presenters/v3/pagination_presenter.rb +++ b/app/presenters/v3/pagination_presenter.rb @@ -1,8 +1,8 @@ module VCAP::CloudController class PaginationPresenter def present_pagination_hash(paginated_result, base_url) - page = paginated_result.page - per_page = paginated_result.per_page + page = paginated_result.pagination_options.page + per_page = paginated_result.pagination_options.per_page total_results = paginated_result.total last_page = (total_results.to_f / per_page.to_f).ceil diff --git a/lib/cloud_controller/paging/paginated_result.rb b/lib/cloud_controller/paging/paginated_result.rb index 1661672d8a0..dfc042d992f 100644 --- a/lib/cloud_controller/paging/paginated_result.rb +++ b/lib/cloud_controller/paging/paginated_result.rb @@ -1,12 +1,11 @@ module VCAP::CloudController class PaginatedResult - attr_reader :records, :total, :page, :per_page + attr_reader :records, :total, :pagination_options - def initialize(records, total, page, per_page) - @records = records - @total = total - @page = page - @per_page = per_page + def initialize(records, total, pagination_options) + @records = records + @total = total + @pagination_options = pagination_options end end end diff --git a/lib/cloud_controller/paging/pagination_options.rb b/lib/cloud_controller/paging/pagination_options.rb new file mode 100644 index 00000000000..81cb0afe44d --- /dev/null +++ b/lib/cloud_controller/paging/pagination_options.rb @@ -0,0 +1,16 @@ +module VCAP::CloudController + class PaginationOptions + attr_reader :page, :per_page + + def initialize(page, per_page) + @page = page + @per_page = per_page + end + + def self.from_params(params) + page = params['page'].to_i + per_page = params['per_page'].to_i + PaginationOptions.new(page, per_page) + end + end +end diff --git a/lib/cloud_controller/paging/pagination_request.rb b/lib/cloud_controller/paging/pagination_request.rb deleted file mode 100644 index 62a2a336abe..00000000000 --- a/lib/cloud_controller/paging/pagination_request.rb +++ /dev/null @@ -1,10 +0,0 @@ -module VCAP::CloudController - class PaginationRequest - attr_reader :page, :per_page - - def initialize(page, per_page) - @page = page - @per_page = per_page - end - end -end diff --git a/lib/cloud_controller/paging/sequel_paginator.rb b/lib/cloud_controller/paging/sequel_paginator.rb index 5e7b142f116..980835818b5 100644 --- a/lib/cloud_controller/paging/sequel_paginator.rb +++ b/lib/cloud_controller/paging/sequel_paginator.rb @@ -4,17 +4,17 @@ class SequelPaginator PER_PAGE_DEFAULT = 50 PER_PAGE_MAX = 5000 - def get_page(sequel_dataset, pagination_request) - page = pagination_request.page.nil? ? PAGE_DEFAULT : pagination_request.page + def get_page(sequel_dataset, pagination_options) + page = pagination_options.page.nil? ? PAGE_DEFAULT : pagination_options.page page = PAGE_DEFAULT if page < 1 - per_page = pagination_request.per_page.nil? ? PER_PAGE_DEFAULT : pagination_request.per_page + per_page = pagination_options.per_page.nil? ? PER_PAGE_DEFAULT : pagination_options.per_page per_page = PER_PAGE_DEFAULT if per_page < 1 per_page = PER_PAGE_MAX if per_page > PER_PAGE_MAX query = sequel_dataset.extension(:pagination).paginate(page, per_page).order(:id) - PaginatedResult.new(query.all, query.pagination_record_count, page, per_page) + PaginatedResult.new(query.all, query.pagination_record_count, PaginationOptions.new(page, per_page)) end end end diff --git a/spec/unit/handlers/apps_handler_spec.rb b/spec/unit/handlers/apps_handler_spec.rb index 67c6bb96714..7199ab30898 100644 --- a/spec/unit/handlers/apps_handler_spec.rb +++ b/spec/unit/handlers/apps_handler_spec.rb @@ -18,7 +18,7 @@ module VCAP::CloudController let(:user) { User.make } let(:page) { 1 } let(:per_page) { 1 } - let(:pagination_request) { PaginationRequest.new(page, per_page) } + let(:pagination_options) { PaginationOptions.new(page, per_page) } let(:paginator) { double(:paginator) } let(:apps_handler) { described_class.new(process_handler, paginator) } let(:roles) { double(:roles, admin?: admin_role) } @@ -38,7 +38,7 @@ module VCAP::CloudController end it 'allows viewing all apps' do - apps_handler.list(pagination_request, access_context) + apps_handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(3) end @@ -47,7 +47,7 @@ module VCAP::CloudController context 'when the user cannot list any apps' do it 'applies a user visibility filter properly' do - apps_handler.list(pagination_request, access_context) + apps_handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(0) end @@ -61,7 +61,7 @@ module VCAP::CloudController end it 'applies a user visibility filter properly' do - apps_handler.list(pagination_request, access_context) + apps_handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(2) end diff --git a/spec/unit/handlers/processes_handler_spec.rb b/spec/unit/handlers/processes_handler_spec.rb index 02b9e5d03ff..5e9a42b3a4d 100644 --- a/spec/unit/handlers/processes_handler_spec.rb +++ b/spec/unit/handlers/processes_handler_spec.rb @@ -40,7 +40,7 @@ module VCAP::CloudController let(:user) { User.make } let(:page) { 1 } let(:per_page) { 1 } - let(:pagination_request) { PaginationRequest.new(page, per_page) } + let(:pagination_options) { PaginationOptions.new(page, per_page) } let(:paginator) { double(:paginator) } let(:handler) { described_class.new(process_repo, process_event_repo, paginator) } let(:roles) { double(:roles, admin?: admin_role) } @@ -60,7 +60,7 @@ module VCAP::CloudController end it 'allows viewing all processes' do - handler.list(pagination_request, access_context) + handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(3) end @@ -69,7 +69,7 @@ module VCAP::CloudController context 'when the user cannot list any processes' do it 'applies a user visibility filter properly' do - handler.list(pagination_request, access_context) + handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(0) end @@ -83,7 +83,7 @@ module VCAP::CloudController end it 'applies a user visibility filter properly' do - handler.list(pagination_request, access_context) + handler.list(pagination_options, access_context) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(2) end @@ -96,7 +96,7 @@ module VCAP::CloudController filter_options = { app_guid: v3app.guid } - handler.list(pagination_request, access_context, filter_options) + handler.list(pagination_options, access_context, filter_options) expect(paginator).to have_received(:get_page) do |dataset, _| expect(dataset.count).to eq(1) diff --git a/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb b/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb index 25329845d78..88ea5e1684f 100644 --- a/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb +++ b/spec/unit/lib/cloud_controller/paging/sequel_paginator_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' require 'cloud_controller/paging/sequel_paginator' -require 'cloud_controller/paging/pagination_request' +require 'cloud_controller/paging/pagination_options' module VCAP::CloudController describe SequelPaginator do @@ -14,60 +14,60 @@ module VCAP::CloudController let(:per_page) { 1 } it 'defaults to the first page if page is nil' do - pagination_request = PaginationRequest.new(nil, per_page) + pagination_options = PaginationOptions.new(nil, per_page) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) - expect(paginated_result.page).to eq(1) + expect(paginated_result.pagination_options.page).to eq(1) end it 'defaults to the first page if page is 0' do - pagination_request = PaginationRequest.new(0, per_page) + pagination_options = PaginationOptions.new(0, per_page) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) - expect(paginated_result.page).to eq(1) + expect(paginated_result.pagination_options.page).to eq(1) end it 'defaults to listing 50 records per page if per_page is nil' do - pagination_request = PaginationRequest.new(page, nil) + pagination_options = PaginationOptions.new(page, nil) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) - expect(paginated_result.per_page).to eq(50) + expect(paginated_result.pagination_options.per_page).to eq(50) end it 'defaults to listing 50 records per page if per_page is 0' do - pagination_request = PaginationRequest.new(page, 0) + pagination_options = PaginationOptions.new(page, 0) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) - expect(paginated_result.per_page).to eq(50) + expect(paginated_result.pagination_options.per_page).to eq(50) end it 'limits the listing to 5000 records per page' do - pagination_request = PaginationRequest.new(page, 5001) + pagination_options = PaginationOptions.new(page, 5001) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) - expect(paginated_result.per_page).to eq(5000) + expect(paginated_result.pagination_options.per_page).to eq(5000) end it 'finds all records from the page upto the per_page limit' do per_page = 1 - pagination_request = PaginationRequest.new(page, per_page) + pagination_options = PaginationOptions.new(page, per_page) - paginated_result = paginator.get_page(dataset, pagination_request) + paginated_result = paginator.get_page(dataset, pagination_options) expect(paginated_result.records.length).to eq(1) end it 'pages properly' do - pagination_request = PaginationRequest.new(1, per_page) - first_paginated_result = paginator.get_page(dataset, pagination_request) + pagination_options = PaginationOptions.new(1, per_page) + first_paginated_result = paginator.get_page(dataset, pagination_options) - pagination_request = PaginationRequest.new(2, per_page) - second_paginated_result = paginator.get_page(dataset, pagination_request) + pagination_options = PaginationOptions.new(2, per_page) + second_paginated_result = paginator.get_page(dataset, pagination_options) expect(first_paginated_result.records.first.guid).to eq(app_model1.guid) expect(second_paginated_result.records.first.guid).to eq(app_model2.guid) diff --git a/spec/unit/presenters/v3/app_presenter_spec.rb b/spec/unit/presenters/v3/app_presenter_spec.rb index b9e398c4924..3db1363d7f2 100644 --- a/spec/unit/presenters/v3/app_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_presenter_spec.rb @@ -23,7 +23,7 @@ module VCAP::CloudController let(:page) { 1 } let(:per_page) { 1 } let(:total_results) { 2 } - let(:paginated_result) { PaginatedResult.new(apps, total_results, page, per_page) } + let(:paginated_result) { PaginatedResult.new(apps, total_results, PaginationOptions.new(page, per_page)) } it 'presents the apps as a json array under resources' do json_result = presenter.present_json_list(paginated_result) diff --git a/spec/unit/presenters/v3/pagination_presenter_spec.rb b/spec/unit/presenters/v3/pagination_presenter_spec.rb index cdb9075987d..4294c058f31 100644 --- a/spec/unit/presenters/v3/pagination_presenter_spec.rb +++ b/spec/unit/presenters/v3/pagination_presenter_spec.rb @@ -8,7 +8,7 @@ module VCAP::CloudController let(:page) { 1 } let(:per_page) { 1 } let(:total_results) { 2 } - let(:paginated_result) { PaginatedResult.new(double(:results), total_results, page, per_page) } + let(:paginated_result) { PaginatedResult.new(double(:results), total_results, PaginationOptions.new(page, per_page)) } let(:base_url) { '/cloudfoundry/is-great' } it 'includes total_results' do @@ -33,7 +33,7 @@ module VCAP::CloudController end it 'sets first and last page to 1 if there is 1 page' do - paginated_result = PaginatedResult.new([], 0, page, per_page) + paginated_result = PaginatedResult.new([], 0, PaginationOptions.new(page, per_page)) result = presenter.present_pagination_hash(paginated_result, base_url) last_url = result[:last][:href] diff --git a/spec/unit/presenters/v3/process_presenter_spec.rb b/spec/unit/presenters/v3/process_presenter_spec.rb index 604e54d3a4a..b83740c2794 100644 --- a/spec/unit/presenters/v3/process_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_presenter_spec.rb @@ -24,7 +24,7 @@ module VCAP::CloudController let(:page) { 1 } let(:per_page) { 1 } let(:total_results) { 2 } - let(:paginated_result) { PaginatedResult.new(processes, total_results, page, per_page) } + let(:paginated_result) { PaginatedResult.new(processes, total_results, PaginationOptions.new(page, per_page)) } it 'presents the processes as a json array under resources' do json_result = presenter.present_json_list(paginated_result) From e0a6fa47f65cb2b0fefcae48a4025dc2539f0d0e Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 7 Jan 2015 17:57:36 -0800 Subject: [PATCH 20/76] Retry deprovision job [finishes #83710822] --- .../v2/service_instance_deprovisioner.rb | 5 ++-- .../v2/service_instance_deprovisioner_spec.rb | 30 ++++++++++++------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb index 4782ecb000d..bf0ad4537e6 100644 --- a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb +++ b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb @@ -11,8 +11,9 @@ def self.deprovision(client_attrs, service_instance) service_instance.guid, service_instance.service_plan.guid ) - Delayed::Job.enqueue(deprovision_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) - deprovision_job + + retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(deprovision_job, 0) + Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb index b9444e6585b..bb2e8009b69 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb @@ -12,18 +12,26 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'deprovision' do - it 'creates a ServiceInstanceDeprovision Job' do - job = ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - expect(job).to be_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision) - expect(job.client_attrs).to be(client_attrs) - expect(job.service_instance_guid).to be(service_instance.guid) - expect(job.service_plan_guid).to be(service_instance.service_plan.guid) - end + it 'enqueues a retryable ServiceInstanceDeprovision job' do + allow(Delayed::Job).to receive(:enqueue) + + Timecop.freeze do + ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + + expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob + expect(job.num_attempts).to eq 0 - it 'enqueues a ServiceInstanceDeprovision Job' do - expect(Delayed::Job).to receive(:enqueue).with(an_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision), - hash_including(queue: 'cc-generic')) - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) + inner_job = job.job + expect(inner_job).to be_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision) + expect(inner_job.client_attrs).to be(client_attrs) + expect(inner_job.service_instance_guid).to be(service_instance.guid) + expect(inner_job.service_plan_guid).to be(service_instance.service_plan.guid) + end + end end end end From 09a0b47ab1cb1a090cee897cab15501e62550f44 Mon Sep 17 00:00:00 2001 From: Joseph Palermo and Sujoy Basu Date: Wed, 7 Jan 2015 09:31:38 -0800 Subject: [PATCH 21/76] Extract UaaClient from Services UaaClientManager [#63345104] --- lib/cloud_controller.rb | 3 + .../sso => cloud_controller}/uaa/errors.rb | 2 +- lib/cloud_controller/uaa/uaa_client.rb | 48 +++++++++++ lib/services/sso/uaa.rb | 1 - lib/services/sso/uaa/uaa_client_manager.rb | 79 ++++++++----------- .../sso/uaa/uaa_client_manager_spec.rb | 55 ++----------- spec/unit/lib/uaa/uaa_client_spec.rb | 70 ++++++++++++++++ 7 files changed, 159 insertions(+), 99 deletions(-) rename lib/{services/sso => cloud_controller}/uaa/errors.rb (95%) create mode 100644 lib/cloud_controller/uaa/uaa_client.rb create mode 100644 spec/unit/lib/uaa/uaa_client_spec.rb diff --git a/lib/cloud_controller.rb b/lib/cloud_controller.rb index 563f4de1bfe..33c2940a478 100644 --- a/lib/cloud_controller.rb +++ b/lib/cloud_controller.rb @@ -88,4 +88,7 @@ module VCAP::CloudController; end require 'cloud_controller/errors/instances_unavailable' +require 'cloud_controller/uaa/errors' +require 'cloud_controller/uaa/uaa_client' + require 'services' diff --git a/lib/services/sso/uaa/errors.rb b/lib/cloud_controller/uaa/errors.rb similarity index 95% rename from lib/services/sso/uaa/errors.rb rename to lib/cloud_controller/uaa/errors.rb index b5575468382..b30e0b4152a 100644 --- a/lib/services/sso/uaa/errors.rb +++ b/lib/cloud_controller/uaa/errors.rb @@ -1,4 +1,4 @@ -module VCAP::Services::SSO::UAA +module VCAP::CloudController class UaaError < StandardError; end class UaaResourceNotFound < UaaError diff --git a/lib/cloud_controller/uaa/uaa_client.rb b/lib/cloud_controller/uaa/uaa_client.rb new file mode 100644 index 00000000000..b3322be5d08 --- /dev/null +++ b/lib/cloud_controller/uaa/uaa_client.rb @@ -0,0 +1,48 @@ +module VCAP::CloudController + class UaaClient + attr_reader :uaa_target, :client_id, :secret, :options + def initialize(uaa_target, client_id, secret, options = {}) + @uaa_target = uaa_target + @client_id = client_id + @secret = secret + @options = options + end + + def scim + @scim ||= CF::UAA::Scim.new(uaa_target, token_info.auth_header, uaa_connection_opts) + end + + def get_clients(client_ids) + client_ids.map do |id| + begin + scim.get(:client, id) + rescue CF::UAA::NotFound + nil + end + end.compact + end + + def token_info + token_issuer.client_credentials_grant + rescue CF::UAA::NotFound => e + logger.error("UAA request for token failed: #{e.inspect}") + raise UaaUnavailable.new + end + + private + + def token_issuer + CF::UAA::TokenIssuer.new(uaa_target, client_id, secret, uaa_connection_opts) + end + + def uaa_connection_opts + { + skip_ssl_validation: !!options[:skip_ssl_validation] + } + end + + def logger + @logger ||= Steno.logger('cc.uaa_client') + end + end +end diff --git a/lib/services/sso/uaa.rb b/lib/services/sso/uaa.rb index ac2155dee2c..25b96d574b3 100644 --- a/lib/services/sso/uaa.rb +++ b/lib/services/sso/uaa.rb @@ -1,4 +1,3 @@ module VCAP::Services::SSO::UAA end -require 'services/sso/uaa/errors' require 'services/sso/uaa/uaa_client_manager' diff --git a/lib/services/sso/uaa/uaa_client_manager.rb b/lib/services/sso/uaa/uaa_client_manager.rb index 107bc3a1bda..690bc3db381 100644 --- a/lib/services/sso/uaa/uaa_client_manager.rb +++ b/lib/services/sso/uaa/uaa_client_manager.rb @@ -7,16 +7,11 @@ class UaaClientManager def initialize(opts={}) @opts = opts + @uaa_client = create_uaa_client end def get_clients(client_ids) - client_ids.map do |id| - begin - scim.get(:client, id) - rescue CF::UAA::NotFound - nil - end - end.compact + @uaa_client.get_clients(client_ids) end def modify_transaction(changeset) @@ -29,7 +24,7 @@ def modify_transaction(changeset) request = Net::HTTP::Post.new(uri.path) request.body = request_body.to_json request.content_type = 'application/json' - request['Authorization'] = token_info.auth_header + request['Authorization'] = uaa_client.token_info.auth_header http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = use_ssl @@ -45,24 +40,25 @@ def modify_transaction(changeset) return when 400 log_bad_uaa_response(response) - raise UaaResourceInvalid.new + raise VCAP::CloudController::UaaResourceInvalid.new when 404 log_bad_uaa_response(response) if response[ROUTER_404_KEY] == ROUTER_404_VALUE - raise UaaUnavailable.new + raise VCAP::CloudController::UaaUnavailable.new else - raise UaaResourceNotFound.new + raise VCAP::CloudController::UaaResourceNotFound.new end when 409 log_bad_uaa_response(response) - raise UaaResourceAlreadyExists.new + raise VCAP::CloudController::UaaResourceAlreadyExists.new else log_bad_uaa_response(response) - raise UaaUnexpectedResponse.new + raise VCAP::CloudController::UaaUnexpectedResponse.new end end private + attr_reader :uaa_client def verify_certs? !VCAP::CloudController::Config.config[:skip_cert_verify] @@ -86,34 +82,6 @@ def batch_request(changeset) end end - def scim - @opts.fetch(:scim) do - CF::UAA::Scim.new(uaa_target, token_info.auth_header, uaa_connection_opts) - end - end - - def uaa_target - VCAP::CloudController::Config.config[:uaa][:url] - end - - def uaa_connection_opts - { - skip_ssl_validation: !verify_certs? - } - end - - def issuer - uaa_client, uaa_client_secret = issuer_client_config - CF::UAA::TokenIssuer.new(uaa_target, uaa_client, uaa_client_secret, uaa_connection_opts) - end - - def token_info - issuer.client_credentials_grant - rescue CF::UAA::NotFound => e - logger.error("UAA request for token failed: #{e.inspect}") - raise UaaUnavailable.new - end - def sso_client_info(client_attrs) { client_id: client_attrs['id'], @@ -124,13 +92,6 @@ def sso_client_info(client_attrs) } end - def issuer_client_config - uaa_client = VCAP::CloudController::Config.config[:uaa_client_name] - uaa_client_secret = VCAP::CloudController::Config.config[:uaa_client_secret] - - [uaa_client, uaa_client_secret] if uaa_client && uaa_client_secret - end - def logger @logger ||= Steno.logger('cc.uaa_client_manager') end @@ -143,5 +104,27 @@ def filter_uaa_client_scope filtered_scope end + + def create_uaa_client + VCAP::CloudController::UaaClient.new(uaa_target, uaa_client_name, uaa_client_secret, uaa_connection_opts) + end + + def uaa_target + VCAP::CloudController::Config.config[:uaa][:url] + end + + def uaa_client_name + VCAP::CloudController::Config.config[:uaa_client_name] + end + + def uaa_client_secret + VCAP::CloudController::Config.config[:uaa_client_secret] + end + + def uaa_connection_opts + { + skip_ssl_validation: !verify_certs? + } + end end end diff --git a/spec/unit/lib/services/sso/uaa/uaa_client_manager_spec.rb b/spec/unit/lib/services/sso/uaa/uaa_client_manager_spec.rb index 679db4a4d6c..b202ae8dc0c 100644 --- a/spec/unit/lib/services/sso/uaa/uaa_client_manager_spec.rb +++ b/spec/unit/lib/services/sso/uaa/uaa_client_manager_spec.rb @@ -9,49 +9,6 @@ module VCAP::Services::SSO::UAA 'redirect_uri' => 'http://redirect.com' } end - let(:scim) { double('scim') } - - describe 'building a scim' do - it 'knows how to build a valid scim' do - creator = UaaClientManager.new - token_info = double('info', auth_header: 'bearer BLAH') - token_issuer = double('issuer', client_credentials_grant: token_info) - - opts = { skip_ssl_validation: false } - allow(CF::UAA::TokenIssuer).to receive(:new).with('http://localhost:8080/uaa', 'cc-service-dashboards', 'some-sekret', opts).and_return(token_issuer) - - expect(creator.send(:scim)).to be_a(CF::UAA::Scim) - expect(token_issuer).to have_received(:client_credentials_grant) - end - end - - describe '#get_clients' do - it 'returns the clients that are in uaa' do - allow(scim).to receive(:get).and_return({ 'client_id' => 'existing-id' }) - client_manager = UaaClientManager.new(scim: scim) - - result = client_manager.get_clients(['existing-id']) - - expect(scim).to have_received(:get).with(:client, 'existing-id').once - expect(result).to be_a(Array) - expect(result.length).to eq(1) - expect(result[0]).to include('client_id' => 'existing-id') - end - - it 'does not return clients that are not in uaa' do - allow(scim).to receive(:get).with(:client, 'existing-id').and_return({ 'client_id' => 'existing-id' }) - allow(scim).to receive(:get).with(:client, 'non-existing-id').and_raise(CF::UAA::NotFound.new) - client_manager = UaaClientManager.new(scim: scim) - - result = client_manager.get_clients(['existing-id', 'non-existing-id']) - - expect(scim).to have_received(:get).with(:client, 'existing-id').once - expect(scim).to have_received(:get).with(:client, 'non-existing-id').once - expect(result).to be_a(Array) - expect(result.length).to eq(1) - expect(result[0]).to include('client_id' => 'existing-id') - end - end describe '#modify_transaction' do let(:uaa_uri) { VCAP::CloudController::Config.config[:uaa][:url] } @@ -152,7 +109,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaResourceNotFound) + }.to raise_error(VCAP::CloudController::UaaResourceNotFound) end end @@ -171,7 +128,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaUnavailable) + }.to raise_error(VCAP::CloudController::UaaUnavailable) end end @@ -189,7 +146,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaResourceAlreadyExists) + }.to raise_error(VCAP::CloudController::UaaResourceAlreadyExists) end end @@ -207,7 +164,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaResourceInvalid) + }.to raise_error(VCAP::CloudController::UaaResourceInvalid) end end @@ -225,7 +182,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaUnexpectedResponse) + }.to raise_error(VCAP::CloudController::UaaUnexpectedResponse) end end @@ -243,7 +200,7 @@ module VCAP::Services::SSO::UAA expect { client_manager.modify_transaction(changeset) - }.to raise_error(UaaUnavailable) + }.to raise_error(VCAP::CloudController::UaaUnavailable) end end diff --git a/spec/unit/lib/uaa/uaa_client_spec.rb b/spec/unit/lib/uaa/uaa_client_spec.rb new file mode 100644 index 00000000000..b190a29bfe1 --- /dev/null +++ b/spec/unit/lib/uaa/uaa_client_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +module VCAP::CloudController + describe UaaClient do + let(:url) { 'http://uaa.example.com' } + let(:client_id) { 'client_id' } + let(:secret) { 'secret_key' } + let(:uaa_options) { { skip_ssl_validation: false } } + + let(:uaa_client) { UaaClient.new(url, client_id, secret) } + let(:auth_header) { 'bearer STUFF' } + let(:token_info) { double(CF::UAA::TokenInfo, auth_header: auth_header) } + let(:token_issuer) { double(CF::UAA::TokenIssuer, client_credentials_grant: token_info) } + + describe 'scim' do + before do + allow(CF::UAA::TokenIssuer).to receive(:new).with(url, client_id, secret, uaa_options).and_return(token_issuer) + end + + it 'knows how to build a valid scim' do + scim = uaa_client.scim + expect(scim).to be_a(CF::UAA::Scim) + expect(scim.instance_variable_get(:@target)).to eq(url) + expect(scim.instance_variable_get(:@auth_header)).to eq(auth_header) + end + + it 'caches the scim' do + expect(uaa_client.scim).to be(uaa_client.scim) + end + + context 'when skip_ssl_validation is true' do + let(:uaa_options) { { skip_ssl_validation: true } } + let(:uaa_client) { UaaClient.new(url, client_id, secret, uaa_options) } + + it 'skips ssl validation for TokenIssuer and Scim creation' do + scim = uaa_client.scim + expect(scim.instance_variable_get(:@skip_ssl_validation)).to eq(true) + end + end + end + + describe '#get_clients' do + let(:scim) { double('scim') } + + it 'returns the clients that are in uaa' do + allow(scim).to receive(:get).and_return({ 'client_id' => 'existing-id' }) + allow(uaa_client).to receive(:scim).and_return(scim) + result = uaa_client.get_clients(['existing-id']) + + expect(scim).to have_received(:get).with(:client, 'existing-id').once + expect(result).to be_a(Array) + expect(result.length).to eq(1) + expect(result[0]).to include('client_id' => 'existing-id') + end + + it 'does not return clients that are not in uaa' do + allow(scim).to receive(:get).with(:client, 'existing-id').and_return({ 'client_id' => 'existing-id' }) + allow(scim).to receive(:get).with(:client, 'non-existing-id').and_raise(CF::UAA::NotFound.new) + allow(uaa_client).to receive(:scim).and_return(scim) + result = uaa_client.get_clients(['existing-id', 'non-existing-id']) + + expect(scim).to have_received(:get).with(:client, 'existing-id').once + expect(scim).to have_received(:get).with(:client, 'non-existing-id').once + expect(result).to be_a(Array) + expect(result.length).to eq(1) + expect(result[0]).to include('client_id' => 'existing-id') + end + end + end +end From 182a7660733f1369ae5fc2f9efc11aaeb9895850 Mon Sep 17 00:00:00 2001 From: "Dan Lavine, Joseph Palermo and Sujoy Basu" Date: Wed, 7 Jan 2015 12:28:11 -0800 Subject: [PATCH 22/76] User collection endpoints now return usernames in the json response - Instances can modify export_attrs at runtime - PaginatedCollections can now have a transformer to modify records - UaaClient can fetch usernames for user guids [#63345104] --- .../username_populator.rb | 15 +++++ app/controllers/base/model_controller.rb | 10 +-- app/controllers/runtime/users_controller.rb | 9 +++ app/models/runtime/user.rb | 10 +++ bosh-templates/cloud_controller_api.yml.erb | 3 + config/cloud_controller.yml | 3 + lib/cloud_controller.rb | 1 + .../collection_transformers.rb | 3 + lib/cloud_controller/config.rb | 3 + lib/cloud_controller/dependency_locator.rb | 14 +++++ .../paginated_collection_renderer.rb | 30 ++++++--- lib/cloud_controller/uaa/uaa_client.rb | 15 ++++- lib/sequel_plugins/vcap_serialization.rb | 9 ++- lib/services/sso/dashboard_client_manager.rb | 4 +- lib/services/sso/uaa/uaa_client_manager.rb | 1 + spec/api/api_version_spec.rb | 2 +- .../documentation/organizations_api_spec.rb | 4 ++ spec/api/documentation/spaces_api_spec.rb | 3 + spec/api/documentation/users_api_spec.rb | 4 ++ .../username_populator_spec.rb | 26 ++++++++ .../controllers/base/model_controller_spec.rb | 9 +++ .../runtime/organizations_controller_spec.rb | 3 + .../runtime/spaces_controller_spec.rb | 3 + .../runtime/users_controller_spec.rb | 1 + .../dependency_locator_spec.rb | 35 +++++++++++ .../paginated_collection_renderer_spec.rb | 23 +++++++ .../sequel_plugins/vcap_serialization_spec.rb | 15 +++++ .../sso/dashboard_client_manager_spec.rb | 8 +-- spec/unit/lib/uaa/uaa_client_spec.rb | 62 +++++++++++++++++-- spec/unit/models/runtime/user_spec.rb | 13 ++++ 30 files changed, 314 insertions(+), 27 deletions(-) create mode 100644 app/collection_transformers/username_populator.rb create mode 100644 lib/cloud_controller/collection_transformers.rb create mode 100644 spec/unit/collection_transformers/username_populator_spec.rb diff --git a/app/collection_transformers/username_populator.rb b/app/collection_transformers/username_populator.rb new file mode 100644 index 00000000000..7a9e50b44be --- /dev/null +++ b/app/collection_transformers/username_populator.rb @@ -0,0 +1,15 @@ +module VCAP::CloudController + class UsernamePopulator + attr_reader :uaa_client + + def initialize(uaa_client) + @uaa_client = uaa_client + end + + def transform(users) + user_ids = users.collect(&:guid) + username_mapping = uaa_client.usernames_for_ids(user_ids) + users.each { |user| user.username = username_mapping[user.guid] } + end + end +end diff --git a/app/controllers/base/model_controller.rb b/app/controllers/base/model_controller.rb index 6f8722bd273..4f98667e486 100644 --- a/app/controllers/base/model_controller.rb +++ b/app/controllers/base/model_controller.rb @@ -5,6 +5,8 @@ module VCAP::CloudController::RestController class ModelController < BaseController include Routes + attr_reader :object_renderer, :collection_renderer + def inject_dependencies(dependencies) super @object_renderer = dependencies.fetch(:object_renderer) @@ -130,7 +132,9 @@ def enumerate_related(guid, name) @opts ) - collection_renderer.render_json( + associated_controller_instance = CloudController::ControllerFactory.new(@config, @logger, @env, @params, @body, @sinatra).create_controller(associated_controller) + + associated_controller_instance.collection_renderer.render_json( associated_controller, filtered_dataset, associated_path, @@ -228,10 +232,6 @@ def model self.class.model end - protected - - attr_reader :object_renderer, :collection_renderer - private def enumerate_dataset diff --git a/app/controllers/runtime/users_controller.rb b/app/controllers/runtime/users_controller.rb index 93f2772181c..c0c11a4e2b5 100644 --- a/app/controllers/runtime/users_controller.rb +++ b/app/controllers/runtime/users_controller.rb @@ -1,5 +1,9 @@ module VCAP::CloudController class UsersController < RestController::ModelController + def self.dependencies + [:username_populating_collection_renderer] + end + define_attributes do attribute :guid, String, exclude_in: :update attribute :admin, Message::Boolean, default: false @@ -33,6 +37,11 @@ def delete(guid) do_delete(find_guid_and_validate_access(:delete, guid)) end + def inject_dependencies(dependencies) + super + @collection_renderer = dependencies[:username_populating_collection_renderer] + end + define_messages define_routes end diff --git a/app/models/runtime/user.rb b/app/models/runtime/user.rb index 82af78f7560..24201aebec9 100644 --- a/app/models/runtime/user.rb +++ b/app/models/runtime/user.rb @@ -1,6 +1,7 @@ module VCAP::CloudController class User < Sequel::Model class InvalidOrganizationRelation < VCAP::Errors::InvalidRelation; end + attr_accessor :username no_auto_guid @@ -81,6 +82,15 @@ def validate_organization_roles(org) end end + def export_attrs + attrs = super + if username + attrs + [:username] + else + attrs + end + end + def admin? admin end diff --git a/bosh-templates/cloud_controller_api.yml.erb b/bosh-templates/cloud_controller_api.yml.erb index 8a8d566c6f1..352511dff45 100644 --- a/bosh-templates/cloud_controller_api.yml.erb +++ b/bosh-templates/cloud_controller_api.yml.erb @@ -238,6 +238,9 @@ uaa_client_secret: <%= p("uaa.clients.cc-service-dashboards.secret") %> uaa_client_scope: <%= p("uaa.clients.cc-service-dashboards.scope") %> <% end %> +cloud_controller_username_lookup_client_name: "cloud_controller_username_lookup" +cloud_controller_username_lookup_client_secret: <%= p("uaa.clients.cloud_controller_username_lookup.secret") %> + diego: staging: <%= p("cc.diego.staging") %> running: <%= p("cc.diego.running") %> diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index a5f16458c22..0b262a333cf 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -142,6 +142,9 @@ uaa_client_name: 'cc-service-dashboards' uaa_client_secret: 'some-sekret' uaa_client_scope: openid,cloud_controller_service_permissions.read +cloud_controller_username_lookup_client_name: 'username_lookup_client_name' +cloud_controller_username_lookup_client_secret: 'username_lookup_secret' + diego: staging: optional # disabled|optional|required running: optional # disabled|optional|required diff --git a/lib/cloud_controller.rb b/lib/cloud_controller.rb index 33c2940a478..8c69955412d 100644 --- a/lib/cloud_controller.rb +++ b/lib/cloud_controller.rb @@ -43,6 +43,7 @@ module VCAP::CloudController; end require 'cloud_controller/runner' require 'cloud_controller/app_observer' require 'cloud_controller/dea/app_stager_task' +require 'cloud_controller/collection_transformers' require 'cloud_controller/controllers' require 'cloud_controller/roles' require 'cloud_controller/encryptor' diff --git a/lib/cloud_controller/collection_transformers.rb b/lib/cloud_controller/collection_transformers.rb new file mode 100644 index 00000000000..ee0dfee0372 --- /dev/null +++ b/lib/cloud_controller/collection_transformers.rb @@ -0,0 +1,3 @@ +Dir[File.expand_path('../../../app/collection_transformers/**/*.rb', __FILE__)].each do |file| + require file +end diff --git a/lib/cloud_controller/config.rb b/lib/cloud_controller/config.rb index b027c74b210..f81b88b1553 100644 --- a/lib/cloud_controller/config.rb +++ b/lib/cloud_controller/config.rb @@ -180,6 +180,9 @@ class Config < VCAP::Config optional(:uaa_client_secret) => String, optional(:uaa_client_scope) => String, + optional(:cloud_controller_username_lookup_client_name) => String, + optional(:cloud_controller_username_lookup_client_secret) => String, + :renderer => { max_results_per_page: Integer, default_results_per_page: Integer, diff --git a/lib/cloud_controller/dependency_locator.rb b/lib/cloud_controller/dependency_locator.rb index d1b2192be1a..d2c89ef04cc 100644 --- a/lib/cloud_controller/dependency_locator.rb +++ b/lib/cloud_controller/dependency_locator.rb @@ -198,6 +198,18 @@ def entity_only_paginated_collection_renderer create_paginated_collection_renderer(serializer: VCAP::CloudController::RestController::EntityOnlyPreloadedObjectSerializer.new) end + def username_populating_collection_renderer + create_paginated_collection_renderer(collection_transformer: UsernamePopulator.new(username_lookup_uaa_client)) + end + + def username_lookup_uaa_client + client_id = @config[:cloud_controller_username_lookup_client_name] + secret = @config[:cloud_controller_username_lookup_client_secret] + target = @config[:uaa][:url] + skip_cert_verify = @config[:skip_cert_verify] + UaaClient.new(target, client_id, secret, { skip_ssl_validation: skip_cert_verify }) + end + def missing_blob_handler CloudController::BlobSender::MissingBlobHandler.new end @@ -218,11 +230,13 @@ def create_paginated_collection_renderer(opts={}) max_results_per_page = opts[:max_results_per_page] || @config[:renderer][:max_results_per_page] default_results_per_page = opts[:default_results_per_page] || @config[:renderer][:default_results_per_page] max_inline_relations_depth = opts[:max_inline_relations_depth] || @config[:renderer][:max_inline_relations_depth] + collection_transformer = opts[:collection_transformer] VCAP::CloudController::RestController::PaginatedCollectionRenderer.new(eager_loader, serializer, { max_results_per_page: max_results_per_page, default_results_per_page: default_results_per_page, max_inline_relations_depth: max_inline_relations_depth, + collection_transformer: collection_transformer }) end end diff --git a/lib/cloud_controller/rest_controller/paginated_collection_renderer.rb b/lib/cloud_controller/rest_controller/paginated_collection_renderer.rb index b29e58e3f7c..34f3c2166d4 100644 --- a/lib/cloud_controller/rest_controller/paginated_collection_renderer.rb +++ b/lib/cloud_controller/rest_controller/paginated_collection_renderer.rb @@ -3,6 +3,8 @@ module VCAP::CloudController::RestController class PaginatedCollectionRenderer + attr_reader :collection_transformer + def initialize(eager_loader, serializer, opts) @eager_loader = eager_loader @serializer = serializer @@ -12,6 +14,8 @@ def initialize(eager_loader, serializer, opts) @max_inline_relations_depth = opts.fetch(:max_inline_relations_depth) @default_inline_relations_depth = 0 + + @collection_transformer = opts[:collection_transformer] end # @param [RestController] controller Controller for the @@ -51,13 +55,6 @@ def render_json(controller, ds, path, opts, request_params) ordered_dataset = order_applicator.apply(ds) paginated_dataset = ordered_dataset.extension(:pagination).paginate(page, page_size) - dataset = @eager_loader.eager_load_dataset( - paginated_dataset, - controller, - default_visibility_filter, - opts[:additional_visibility_filters] || {}, - inline_relations_depth, - ) if paginated_dataset.prev_page prev_url = url(controller, path, paginated_dataset.prev_page, page_size, order_direction, opts, request_params) @@ -69,7 +66,8 @@ def render_json(controller, ds, path, opts, request_params) opts[:max_inline] ||= PreloadedObjectSerializer::MAX_INLINE_DEFAULT orphans = opts[:orphan_relations] == 1 ? {} : nil - resources = dataset.all.map { |obj| @serializer.serialize(controller, obj, opts, orphans) } + + resources = fetch_and_process_records(paginated_dataset, controller, inline_relations_depth, orphans, opts) result = { total_results: paginated_dataset.pagination_record_count, @@ -88,6 +86,22 @@ def render_json(controller, ds, path, opts, request_params) private + def fetch_and_process_records(paginated_dataset, controller, inline_relations_depth, orphans, opts) + dataset = @eager_loader.eager_load_dataset( + paginated_dataset, + controller, + default_visibility_filter, + opts[:additional_visibility_filters] || {}, + inline_relations_depth, + ) + + dataset_records = dataset.all + + collection_transformer.transform(dataset_records) if collection_transformer + + dataset_records.map { |obj| @serializer.serialize(controller, obj, opts, orphans) } + end + def default_visibility_filter user = VCAP::CloudController::SecurityContext.current_user admin = VCAP::CloudController::SecurityContext.admin? diff --git a/lib/cloud_controller/uaa/uaa_client.rb b/lib/cloud_controller/uaa/uaa_client.rb index b3322be5d08..2523a302a38 100644 --- a/lib/cloud_controller/uaa/uaa_client.rb +++ b/lib/cloud_controller/uaa/uaa_client.rb @@ -1,7 +1,7 @@ module VCAP::CloudController class UaaClient attr_reader :uaa_target, :client_id, :secret, :options - def initialize(uaa_target, client_id, secret, options = {}) + def initialize(uaa_target, client_id, secret, options={}) @uaa_target = uaa_target @client_id = client_id @secret = secret @@ -29,6 +29,19 @@ def token_info raise UaaUnavailable.new end + def usernames_for_ids(user_ids) + return {} unless user_ids.present? + filter_string = user_ids.map { |user_id| %(id eq "#{user_id}") }.join(' or ') + results = scim.query(:user_id, filter: filter_string) + + results['resources'].each_with_object({}) do |resource, results_hash| + results_hash[resource['id']] = resource['username'] + results_hash + end + rescue UaaUnavailable, CF::UAA::TargetError + {} + end + private def token_issuer diff --git a/lib/sequel_plugins/vcap_serialization.rb b/lib/sequel_plugins/vcap_serialization.rb index d4cc0c387cf..32284a567ef 100644 --- a/lib/sequel_plugins/vcap_serialization.rb +++ b/lib/sequel_plugins/vcap_serialization.rb @@ -17,7 +17,7 @@ module InstanceMethods def to_hash(opts={}) hash = {} redact_vals = opts[:redact] - attrs = opts[:attrs] || self.class.export_attrs || [] + attrs = opts[:attrs] || self.export_attrs attrs.each do |k| if opts[:only].nil? || opts[:only].include?(k) @@ -36,6 +36,11 @@ def to_hash(opts={}) hash end + # Returns an array of attribute names that will be used when calling to_hash + def export_attrs + self.class.export_attrs || [] + end + # Update the model instance from the supplied json string. Only update # attributes specified by import_attributes. # @@ -100,7 +105,7 @@ def create_from_hash(hash, opts={}) # @param [Array] List of attributes to include when serializing to # json or a hash. def export_attributes(*attributes) - self.export_attrs = attributes + self.export_attrs = attributes.freeze end # @param [Array] List of attributes to include when importing diff --git a/lib/services/sso/dashboard_client_manager.rb b/lib/services/sso/dashboard_client_manager.rb index 0e3257d1495..aaa1cf73076 100644 --- a/lib/services/sso/dashboard_client_manager.rb +++ b/lib/services/sso/dashboard_client_manager.rb @@ -90,7 +90,7 @@ def all_clients_can_be_claimed_in_uaa?(existing_uaa_client_ids, catalog) def fetch_clients_from_uaa(requested_client_ids) client_manager.get_clients(requested_client_ids) - rescue VCAP::Services::SSO::UAA::UaaError => e + rescue VCAP::CloudController::UaaError => e raise VCAP::Errors::ApiError.new_from_details('ServiceBrokerDashboardClientFailure', e.message) end @@ -109,7 +109,7 @@ def claim_clients_and_update_uaa(requested_clients, existing_db_clients, existin db_changeset.each(&:db_command) client_manager.modify_transaction(uaa_changeset) end - rescue VCAP::Services::SSO::UAA::UaaError => e + rescue VCAP::CloudController::UaaError => e raise VCAP::Errors::ApiError.new_from_details('ServiceBrokerDashboardClientFailure', e.message) end diff --git a/lib/services/sso/uaa/uaa_client_manager.rb b/lib/services/sso/uaa/uaa_client_manager.rb index 690bc3db381..7e8ddcda75d 100644 --- a/lib/services/sso/uaa/uaa_client_manager.rb +++ b/lib/services/sso/uaa/uaa_client_manager.rb @@ -58,6 +58,7 @@ def modify_transaction(changeset) end private + attr_reader :uaa_client def verify_certs? diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 1018fdda01e..29f72e41886 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '4ca579532ca567f0e62a3ba0c57a3bfdc5dc66ed' + API_FOLDER_CHECKSUM = '539ac3b3750923838e9ea798cda05d14509e3427' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.20.0') diff --git a/spec/api/documentation/organizations_api_spec.rb b/spec/api/documentation/organizations_api_spec.rb index f53d33da6c3..58fe3f9c668 100644 --- a/spec/api/documentation/organizations_api_spec.rb +++ b/spec/api/documentation/organizations_api_spec.rb @@ -83,6 +83,7 @@ describe 'Users' do before do organization.add_user(associated_user) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_user.guid => 'user@example.com' }) end let!(:associated_user) { VCAP::CloudController::User.make } @@ -98,6 +99,7 @@ describe 'Managers' do before do organization.add_manager(associated_manager) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_manager.guid => 'manager@example.com' }) make_manager_for_org(organization) end @@ -114,6 +116,7 @@ describe 'Billing Managers' do before do organization.add_billing_manager(associated_billing_manager) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_billing_manager.guid => 'billing_manager@example.com' }) end let!(:associated_billing_manager) { VCAP::CloudController::User.make } @@ -129,6 +132,7 @@ describe 'Auditors' do before do organization.add_auditor(associated_auditor) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_auditor.guid => 'auditor@example.com' }) end let!(:associated_auditor) { VCAP::CloudController::User.make } diff --git a/spec/api/documentation/spaces_api_spec.rb b/spec/api/documentation/spaces_api_spec.rb index ae567b794b0..509f9087c13 100644 --- a/spec/api/documentation/spaces_api_spec.rb +++ b/spec/api/documentation/spaces_api_spec.rb @@ -93,6 +93,7 @@ def after_standard_model_delete(guid) space.organization.add_user(associated_developer) space.organization.add_user(developer) space.add_developer(associated_developer) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_developer.guid => 'developer@example.com' }) end let!(:associated_developer) { VCAP::CloudController::User.make } @@ -110,6 +111,7 @@ def after_standard_model_delete(guid) space.organization.add_user(associated_manager) space.organization.add_user(manager) space.add_manager(associated_manager) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_manager.guid => 'manager@example.com' }) end let!(:associated_manager) { VCAP::CloudController::User.make } @@ -127,6 +129,7 @@ def after_standard_model_delete(guid) space.organization.add_user(associated_auditor) space.organization.add_user(auditor) space.add_auditor(associated_auditor) + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ associated_auditor.guid => 'auditor@example.com' }) end let!(:associated_auditor) { VCAP::CloudController::User.make } diff --git a/spec/api/documentation/users_api_spec.rb b/spec/api/documentation/users_api_spec.rb index b219cdc1580..ce1563aa22d 100644 --- a/spec/api/documentation/users_api_spec.rb +++ b/spec/api/documentation/users_api_spec.rb @@ -19,6 +19,10 @@ end describe 'Standard endpoints' do + before do + allow_any_instance_of(VCAP::CloudController::UaaClient).to receive(:usernames_for_ids).and_return({ guid => 'user@example.com' }) + end + standard_model_list(:user, VCAP::CloudController::UsersController) standard_model_get(:user, nested_associations: [:default_space]) standard_model_delete(:user) diff --git a/spec/unit/collection_transformers/username_populator_spec.rb b/spec/unit/collection_transformers/username_populator_spec.rb new file mode 100644 index 00000000000..cf952c214c3 --- /dev/null +++ b/spec/unit/collection_transformers/username_populator_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +module VCAP::CloudController + describe UsernamePopulator do + let(:uaa_client) { double(UaaClient) } + let(:username_populator) { UsernamePopulator.new(uaa_client) } + let(:user1) { User.new(guid: '1') } + let(:user2) { User.new(guid: '2') } + let(:users) { [user1, user2] } + + before do + allow(uaa_client).to receive(:usernames_for_ids).with(['1', '2']).and_return({ + '1' => 'Username1', + '2' => 'Username2' + }) + end + + describe 'transform' do + it 'populates users with usernames from UAA' do + username_populator.transform(users) + expect(user1.username).to eq('Username1') + expect(user2.username).to eq('Username2') + end + end + end +end diff --git a/spec/unit/controllers/base/model_controller_spec.rb b/spec/unit/controllers/base/model_controller_spec.rb index e12215868a7..50e782aca2d 100644 --- a/spec/unit/controllers/base/model_controller_spec.rb +++ b/spec/unit/controllers/base/model_controller_spec.rb @@ -687,6 +687,15 @@ def run_delayed_job expect(found_guids).to match_array([associated_model1.guid, associated_model2.guid]) end + it 'uses the collection_renderer for the associated class' do + collection_renderer = double('Collection Renderer', render_json: 'JSON!') + allow_any_instance_of(TestModelManyToManiesController).to receive(:collection_renderer).and_return(collection_renderer) + + get "/v2/test_models/#{model.guid}/test_model_many_to_manies", '', admin_headers + + expect(last_response.body).to eq('JSON!') + end + it 'fails when you do not have access to the associated model' do allow_any_instance_of(TestModelManyToOneAccess).to receive(:index?). with(TestModelManyToOne, { related_obj: instance_of(TestModel), related_model: TestModel }).and_return(false) diff --git a/spec/unit/controllers/runtime/organizations_controller_spec.rb b/spec/unit/controllers/runtime/organizations_controller_spec.rb index f3f013a67b0..4b424fd888f 100644 --- a/spec/unit/controllers/runtime/organizations_controller_spec.rb +++ b/spec/unit/controllers/runtime/organizations_controller_spec.rb @@ -445,6 +445,9 @@ def decoded_guids let(:mgr) { User.make } let(:user) { User.make } let(:org) { Organization.make(manager_guids: [mgr.guid], user_guids: [mgr.guid, user.guid]) } + before do + allow_any_instance_of(UaaClient).to receive(:usernames_for_ids).and_return({}) + end it 'allows org managers' do get "/v2/organizations/#{org.guid}/users", '', headers_for(mgr) diff --git a/spec/unit/controllers/runtime/spaces_controller_spec.rb b/spec/unit/controllers/runtime/spaces_controller_spec.rb index 744f082424d..f44fe67b16b 100644 --- a/spec/unit/controllers/runtime/spaces_controller_spec.rb +++ b/spec/unit/controllers/runtime/spaces_controller_spec.rb @@ -591,6 +591,9 @@ def decoded_guids let(:user) { User.make } let(:org) { Organization.make(manager_guids: [mgr.guid], user_guids: [mgr.guid, user.guid]) } let(:space) { Space.make(organization: org, manager_guids: [mgr.guid], developer_guids: [user.guid]) } + before do + allow_any_instance_of(UaaClient).to receive(:usernames_for_ids).and_return({}) + end it 'allows space managers' do get "/v2/spaces/#{space.guid}/developers", '', headers_for(mgr) diff --git a/spec/unit/controllers/runtime/users_controller_spec.rb b/spec/unit/controllers/runtime/users_controller_spec.rb index 33386d59fbf..809a0bb4ad7 100644 --- a/spec/unit/controllers/runtime/users_controller_spec.rb +++ b/spec/unit/controllers/runtime/users_controller_spec.rb @@ -63,6 +63,7 @@ module VCAP::CloudController include_context 'permissions' before do @obj_a = member_a + allow_any_instance_of(UaaClient).to receive(:usernames_for_ids).and_return({}) end context 'normal user' do diff --git a/spec/unit/lib/cloud_controller/dependency_locator_spec.rb b/spec/unit/lib/cloud_controller/dependency_locator_spec.rb index 0948b1e1cac..afbd2d18181 100644 --- a/spec/unit/lib/cloud_controller/dependency_locator_spec.rb +++ b/spec/unit/lib/cloud_controller/dependency_locator_spec.rb @@ -258,6 +258,7 @@ max_results_per_page: 100_000, default_results_per_page: 100_001, max_inline_relations_depth: 100_002, + collection_transformer: nil } TestConfig.override(renderer: opts) @@ -280,6 +281,7 @@ max_results_per_page: 10, default_results_per_page: 100_001, max_inline_relations_depth: 100_002, + collection_transformer: nil } TestConfig.override(renderer: opts) @@ -302,6 +304,7 @@ max_results_per_page: 100_000, default_results_per_page: 100_001, max_inline_relations_depth: 100_002, + collection_transformer: nil } TestConfig.override(renderer: opts) @@ -316,6 +319,38 @@ end end + describe '#username_populating_collection_renderer' do + it 'returns paginated collection renderer with a UsernamePopulator transformer' do + renderer = locator.username_populating_collection_renderer + expect(renderer.collection_transformer).to be_a(VCAP::CloudController::UsernamePopulator) + end + + it 'uses the username_lookup_uaa_client for the populator' do + uaa_client = double('uaa client') + expect(locator).to receive(:username_lookup_uaa_client).and_return(uaa_client) + renderer = locator.username_populating_collection_renderer + expect(renderer.collection_transformer.uaa_client).to eq(uaa_client) + end + end + + describe '#username_lookup_uaa_client' do + it 'returns a uaa client with credentials for lookuping up usernames' do + uaa_client = locator.username_lookup_uaa_client + expect(uaa_client.client_id).to eq(config[:cloud_controller_username_lookup_client_name]) + expect(uaa_client.secret).to eq(config[:cloud_controller_username_lookup_client_secret]) + expect(uaa_client.uaa_target).to eq(config[:uaa][:url]) + end + + context 'when skip_cert_verify is true in the config' do + before { TestConfig.override(skip_cert_verify: true) } + + it 'skips ssl validation to uaa' do + uaa_client = locator.username_lookup_uaa_client + expect(uaa_client.options[:skip_ssl_validation]).to be true + end + end + end + describe '#missing_blob_handler' do it 'returns the correct handler' do handler = double('a missing blob handler') diff --git a/spec/unit/lib/rest_controller/paginated_collection_renderer_spec.rb b/spec/unit/lib/rest_controller/paginated_collection_renderer_spec.rb index 3e9c565a6ec..e304bc3d338 100644 --- a/spec/unit/lib/rest_controller/paginated_collection_renderer_spec.rb +++ b/spec/unit/lib/rest_controller/paginated_collection_renderer_spec.rb @@ -14,11 +14,13 @@ module VCAP::CloudController::RestController default_results_per_page: default_results_per_page, max_results_per_page: max_results_per_page, max_inline_relations_depth: max_inline_relations_depth, + collection_transformer: collection_transformer } end let(:default_results_per_page) { 100_000 } let(:max_results_per_page) { 100_000 } let(:max_inline_relations_depth) { 100_000 } + let(:collection_transformer) { nil } describe '#render_json' do let(:opts) do @@ -290,6 +292,27 @@ module VCAP::CloudController::RestController end end end + + context 'when collection_transformer is given' do + let(:collection_transformer) { double('collection_transformer') } + let!(:test_model) { VCAP::CloudController::TestModel.make } + + it 'passes the populated dataset to the transformer' do + expect(collection_transformer).to receive(:transform) do |collection| + expect(collection).to eq([test_model]) + end + + render_json_call + end + + it 'serializes the transformed collection' do + expect(collection_transformer).to receive(:transform) do |collection| + collection.first.unique_value = 'test_value' + end + + expect(JSON.parse(render_json_call)['resources'][0]['entity']['unique_value']).to eq('test_value') + end + end end end end diff --git a/spec/unit/lib/sequel_plugins/vcap_serialization_spec.rb b/spec/unit/lib/sequel_plugins/vcap_serialization_spec.rb index ed5eab5bd26..561d87dde7e 100644 --- a/spec/unit/lib/sequel_plugins/vcap_serialization_spec.rb +++ b/spec/unit/lib/sequel_plugins/vcap_serialization_spec.rb @@ -54,4 +54,19 @@ class TestModel < Sequel::Model expect(instance.unique_value).to eq('unique') end end + + describe '#to_hash' do + context 'when the instance defines different attributes to export' do + class TestModel < Sequel::Model + def export_attrs + super + [:required_attr] + end + end + + it 'uses the attributes defined by the instance' do + hash = TestModel.new.to_hash + expect(hash.keys).to eq(['required_attr']) + end + end + end end diff --git a/spec/unit/lib/services/sso/dashboard_client_manager_spec.rb b/spec/unit/lib/services/sso/dashboard_client_manager_spec.rb index d6c7f697558..f883ad3bcaa 100644 --- a/spec/unit/lib/services/sso/dashboard_client_manager_spec.rb +++ b/spec/unit/lib/services/sso/dashboard_client_manager_spec.rb @@ -307,7 +307,7 @@ module VCAP::Services::SSO describe 'exception handling' do context 'when getting UAA clients raises an error' do before do - error = VCAP::Services::SSO::UAA::UaaError.new('my test error') + error = VCAP::CloudController::UaaError.new('my test error') expect(client_manager).to receive(:get_clients).and_raise(error) end @@ -324,7 +324,7 @@ module VCAP::Services::SSO before do allow(client_manager).to receive(:get_clients).and_return([{ 'client_id' => unused_id }]) - allow(client_manager).to receive(:modify_transaction).and_raise(VCAP::Services::SSO::UAA::UaaError.new('error message')) + allow(client_manager).to receive(:modify_transaction).and_raise(VCAP::CloudController::UaaError.new('error message')) VCAP::CloudController::ServiceDashboardClient.new( uaa_id: unused_id, @@ -491,7 +491,7 @@ module VCAP::Services::SSO context 'when deleting UAA clients fails' do before do - error = VCAP::Services::SSO::UAA::UaaError.new('error message') + error = VCAP::CloudController::UaaError.new('error message') allow(client_manager).to receive(:modify_transaction).and_raise(error) end @@ -518,7 +518,7 @@ module VCAP::Services::SSO context 'when getting UAA clients raises an error' do before do - error = VCAP::Services::SSO::UAA::UaaError.new('my test error') + error = VCAP::CloudController::UaaError.new('my test error') expect(client_manager).to receive(:get_clients).and_raise(error) end diff --git a/spec/unit/lib/uaa/uaa_client_spec.rb b/spec/unit/lib/uaa/uaa_client_spec.rb index b190a29bfe1..ea5dfe5f63c 100644 --- a/spec/unit/lib/uaa/uaa_client_spec.rb +++ b/spec/unit/lib/uaa/uaa_client_spec.rb @@ -12,11 +12,11 @@ module VCAP::CloudController let(:token_info) { double(CF::UAA::TokenInfo, auth_header: auth_header) } let(:token_issuer) { double(CF::UAA::TokenIssuer, client_credentials_grant: token_info) } - describe 'scim' do - before do - allow(CF::UAA::TokenIssuer).to receive(:new).with(url, client_id, secret, uaa_options).and_return(token_issuer) - end + before do + allow(CF::UAA::TokenIssuer).to receive(:new).with(url, client_id, secret, uaa_options).and_return(token_issuer) + end + describe '#scim' do it 'knows how to build a valid scim' do scim = uaa_client.scim expect(scim).to be_a(CF::UAA::Scim) @@ -66,5 +66,59 @@ module VCAP::CloudController expect(result[0]).to include('client_id' => 'existing-id') end end + + describe '#usernames_for_ids' do + let(:userid_1) { '111' } + let(:userid_2) { '222' } + + it 'returns a map of the given ids to the corresponding usernames from UAA' do + response_body = { + 'resources' => [ + { 'id' => '111', 'origin' => 'uaa', 'username' => 'user_1' }, + { 'id' => '222', 'origin' => 'uaa', 'username' => 'user_2' } + ], + 'schemas' => ['urn:scim:schemas:core:1.0'], + 'startindex' => 1, + 'itemsperpage' => 100, + 'totalresults' => 2 } + + WebMock::API.stub_request(:get, "#{url}/ids/Users"). + with(query: { 'filter' => 'id eq "111" or id eq "222"' }). + to_return( + status: 200, + headers: { 'content-type' => 'application/json' }, + body: response_body.to_json) + + mapping = uaa_client.usernames_for_ids([userid_1, userid_2]) + expect(mapping[userid_1]).to eq('user_1') + expect(mapping[userid_2]).to eq('user_2') + end + + it 'returns an empty hash when given no ids' do + expect(uaa_client.usernames_for_ids([])).to eq({}) + end + + context 'when UAA is unavailable' do + before do + allow(uaa_client).to receive(:token_info).and_raise(UaaUnavailable) + end + + it 'returns an empty hash' do + expect(uaa_client.usernames_for_ids([userid_1])).to eq({}) + end + end + + context 'when the endpoint returns an error' do + before do + scim = double('scim') + allow(scim).to receive(:query).and_raise(CF::UAA::TargetError) + allow(uaa_client).to receive(:scim).and_return(scim) + end + + it 'returns an empty hash' do + expect(uaa_client.usernames_for_ids([userid_1])).to eq({}) + end + end + end end end diff --git a/spec/unit/models/runtime/user_spec.rb b/spec/unit/models/runtime/user_spec.rb index 17aca037cd9..6e953e9e03c 100644 --- a/spec/unit/models/runtime/user_spec.rb +++ b/spec/unit/models/runtime/user_spec.rb @@ -230,5 +230,18 @@ module VCAP::CloudController end end end + + describe '#export_attrs' do + let(:user) { User.make } + + it 'does not include username when username has not been set' do + expect(user.export_attrs).to_not include(:username) + end + + it 'includes username when username has been set' do + user.username = 'somebody' + expect(user.export_attrs).to include(:username) + end + end end end From f723172568b58a772d8c2cd625c02cc54e284308 Mon Sep 17 00:00:00 2001 From: "Dan Lavine, Joseph Palermo and Sujoy Basu" Date: Wed, 7 Jan 2015 14:47:29 -0800 Subject: [PATCH 23/76] Bump API version to 2.21.0 [#63345104] --- lib/cloud_controller/constants.rb | 2 +- spec/api/api_version_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cloud_controller/constants.rb b/lib/cloud_controller/constants.rb index bbea378bbc2..b70fd554b57 100644 --- a/lib/cloud_controller/constants.rb +++ b/lib/cloud_controller/constants.rb @@ -1,5 +1,5 @@ module VCAP::CloudController class Constants - API_VERSION = '2.20.0'.freeze + API_VERSION = '2.21.0'.freeze end end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 29f72e41886..f2cd468b6bd 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,10 +2,10 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '539ac3b3750923838e9ea798cda05d14509e3427' + API_FOLDER_CHECKSUM = 'ef734f694dd797ef339e77780c516b07eb11ebfb' it 'double-checks the version' do - expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.20.0') + expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') end it 'tells the developer if the API specs change' do From b8b70858a94ca346f4b919adb10784d0145fe0b4 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Thu, 8 Jan 2015 17:46:03 -0800 Subject: [PATCH 24/76] Add request ID to ServiceInstanceUnbind job [finishes #84874928] --- app/jobs/services/service_instance_unbind.rb | 3 ++ .../v2/service_instance_unbinder.rb | 3 +- .../v2/service_instance_unbinder_spec.rb | 46 ++++++++++++------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index aa0e987fe0c..3b1e29d33b4 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -3,6 +3,9 @@ module Jobs module Services class ServiceInstanceUnbind < Struct.new(:name, :client_attrs, :binding_guid, :service_instance_guid, :app_guid) def perform + logger = Steno.logger('cc-background') + logger.info('There was an error during service binding creation. Attempting to delete potentially orphaned binding.') + client = VCAP::Services::ServiceBrokers::V2::Client.new(client_attrs) app = VCAP::CloudController::App.first(guid: app_guid) service_instance = VCAP::CloudController::ServiceInstance.first(guid: service_instance_guid) diff --git a/lib/services/service_brokers/v2/service_instance_unbinder.rb b/lib/services/service_brokers/v2/service_instance_unbinder.rb index 6dc78137b6a..d7232a62bec 100644 --- a/lib/services/service_brokers/v2/service_instance_unbinder.rb +++ b/lib/services/service_brokers/v2/service_instance_unbinder.rb @@ -13,7 +13,8 @@ def self.delayed_unbind(client_attrs, binding) binding.app.guid ) - retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(unbind_job, 0) + request_job = VCAP::CloudController::Jobs::RequestJob.new(unbind_job, ::VCAP::Request.current_id) + retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(request_job, 0) Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb index 92d9e1614cb..533975a7b7b 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb @@ -12,27 +12,41 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'delayed_unbind' do - it 'enques a retryable ServiceInstanceUnbind job' do + before do allow(Delayed::Job).to receive(:enqueue) + allow(VCAP::Request).to receive(:current_id).and_return('current_thread_id') + end + + it 'enqueues a job that unbinds the instance' do + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - Timecop.freeze do - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + unbind_job = job.job.job + expect(unbind_job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind + expect(unbind_job.name).to eq 'service-instance-unbind' + expect(unbind_job.client_attrs).to eq client_attrs + expect(unbind_job.binding_guid).to be(binding.guid) + expect(unbind_job.service_instance_guid).to be(binding.service_instance.guid) + expect(unbind_job.app_guid).to be(binding.app.guid) + end + end - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - expect(opts[:queue]).to eq 'cc-generic' - expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + it 'creates the job in the same context as the original request' do + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + request_job = job.job + expect(request_job).to be_a VCAP::CloudController::Jobs::RequestJob + expect(request_job.request_id).to eq 'current_thread_id' + end + end - expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob - expect(job.num_attempts).to eq 0 + it 'makes the job retryable' do + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - inner_job = job.job - expect(inner_job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind - expect(inner_job.name).to eq 'service-instance-unbind' - expect(inner_job.client_attrs).to eq client_attrs - expect(inner_job.binding_guid).to be(binding.guid) - expect(inner_job.service_instance_guid).to be(binding.service_instance.guid) - expect(inner_job.app_guid).to be(binding.app.guid) - end + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob + expect(job.num_attempts).to eq 0 end end end From 5ac302d179f086055e502e06512bf20f6c32c60f Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Fri, 9 Jan 2015 12:14:35 -0800 Subject: [PATCH 25/76] Refactor ServiceInstanceUnbinder spec Tests the result, instead of making assertions about the nested jobs --- .../v2/service_instance_unbinder_spec.rb | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb index 533975a7b7b..9a30f88a0d1 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb @@ -12,33 +12,47 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'delayed_unbind' do + let(:mock_client) { double(:client, unbind: nil) } + before do allow(Delayed::Job).to receive(:enqueue) allow(VCAP::Request).to receive(:current_id).and_return('current_thread_id') + allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(mock_client) + end + + it 'enqueues a job with the correct queue and run_at time' do + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + end end it 'enqueues a job that unbinds the instance' do + mock_binding = double(:binding, + guid: binding.guid, + app_guid: binding.app_guid, + service_instance_guid: binding.service_instance_guid) + allow(VCAP::CloudController::ServiceBinding).to receive(:new).and_return(mock_binding) + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - unbind_job = job.job.job - expect(unbind_job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind - expect(unbind_job.name).to eq 'service-instance-unbind' - expect(unbind_job.client_attrs).to eq client_attrs - expect(unbind_job.binding_guid).to be(binding.guid) - expect(unbind_job.service_instance_guid).to be(binding.service_instance.guid) - expect(unbind_job.app_guid).to be(binding.app.guid) + job.perform end + expect(mock_client).to have_received(:unbind).with(mock_binding) end it 'creates the job in the same context as the original request' do + allow(VCAP::Request).to receive(:current_id=) + ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - request_job = job.job - expect(request_job).to be_a VCAP::CloudController::Jobs::RequestJob - expect(request_job.request_id).to eq 'current_thread_id' + job.perform end + expect(VCAP::Request).to have_received(:current_id=).twice.with('current_thread_id') end it 'makes the job retryable' do From 8dfd53260f9b62f71fe1c12b20ee457666e77389 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 9 Jan 2015 12:15:10 -0800 Subject: [PATCH 26/76] Add request id to ServiceInstanceDeprovision job [finishes #84874912] --- .../v2/service_instance_deprovisioner.rb | 3 +- .../v2/service_instance_deprovisioner_spec.rb | 55 ++++++++++++++----- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb index bf0ad4537e6..c0aae56b4c1 100644 --- a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb +++ b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb @@ -12,7 +12,8 @@ def self.deprovision(client_attrs, service_instance) service_instance.service_plan.guid ) - retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(deprovision_job, 0) + request_job = VCAP::CloudController::Jobs::RequestJob.new(deprovision_job, ::VCAP::Request.current_id) + retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(request_job, 0) Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb index bb2e8009b69..813dc087a7f 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb @@ -12,25 +12,52 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'deprovision' do - it 'enqueues a retryable ServiceInstanceDeprovision job' do + let(:mock_client) { double(:client, deprovision: nil) } + + before do allow(Delayed::Job).to receive(:enqueue) + allow(VCAP::Request).to receive(:current_id).and_return('current_thread_id') + allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(mock_client) + end + + it 'enqueues a job with the correct queue and run_at time' do + ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + end + end + + it 'enqueues a job that deprovisions an instance' do + mock_instance = double(:instance, guid: service_instance.guid) + allow(VCAP::CloudController::ServiceInstance).to receive(:new).and_return(mock_instance) - Timecop.freeze do - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) + ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - expect(opts[:queue]).to eq 'cc-generic' - expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + job.perform + end + expect(mock_client).to have_received(:deprovision).with(mock_instance) + end + + it 'creates the job in the same context as the original request' do + allow(VCAP::Request).to receive(:current_id=) + + ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) + + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + job.perform + end + expect(VCAP::Request).to have_received(:current_id=).twice.with('current_thread_id') + end - expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob - expect(job.num_attempts).to eq 0 + it 'makes the job retryable' do + ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - inner_job = job.job - expect(inner_job).to be_instance_of(VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision) - expect(inner_job.client_attrs).to be(client_attrs) - expect(inner_job.service_instance_guid).to be(service_instance.guid) - expect(inner_job.service_plan_guid).to be(service_instance.service_plan.guid) - end + expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob + expect(job.num_attempts).to eq 0 end end end From 885a8f80cfc4c44cda6e75810e4d5c7a6fbc77a2 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 9 Jan 2015 15:46:27 -0800 Subject: [PATCH 27/76] Use DelayedJob reschedule_at functionality instead of RetryableJob --- .../services/service_instance_deprovision.rb | 6 ++- app/jobs/services/service_instance_unbind.rb | 6 ++- .../v2/service_instance_deprovisioner.rb | 5 +- .../v2/service_instance_unbinder.rb | 5 +- .../service_instance_deprovision_spec.rb | 14 ++++- .../services/service_instance_unbind_spec.rb | 13 ++++- .../v2/service_instance_deprovisioner_spec.rb | 51 ++++-------------- .../v2/service_instance_unbinder_spec.rb | 53 ++++--------------- 8 files changed, 59 insertions(+), 94 deletions(-) diff --git a/app/jobs/services/service_instance_deprovision.rb b/app/jobs/services/service_instance_deprovision.rb index 00bf98213b3..eaf87697988 100644 --- a/app/jobs/services/service_instance_deprovision.rb +++ b/app/jobs/services/service_instance_deprovision.rb @@ -14,7 +14,11 @@ def job_name_in_configuration end def max_attempts - 1 + 10 + end + + def reschedule_at(time, attempts) + time + (2**attempts).minutes end end end diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index 3b1e29d33b4..fa19fecd806 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -20,7 +20,11 @@ def job_name_in_configuration end def max_attempts - 1 + 10 + end + + def reschedule_at(time, attempts) + time + (2**attempts).minutes end end end diff --git a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb index c0aae56b4c1..5fa975f4579 100644 --- a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb +++ b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb @@ -12,9 +12,8 @@ def self.deprovision(client_attrs, service_instance) service_instance.service_plan.guid ) - request_job = VCAP::CloudController::Jobs::RequestJob.new(deprovision_job, ::VCAP::Request.current_id) - retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(request_job, 0) - Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(deprovision_job, opts).enqueue end end end diff --git a/lib/services/service_brokers/v2/service_instance_unbinder.rb b/lib/services/service_brokers/v2/service_instance_unbinder.rb index d7232a62bec..4e287882f5d 100644 --- a/lib/services/service_brokers/v2/service_instance_unbinder.rb +++ b/lib/services/service_brokers/v2/service_instance_unbinder.rb @@ -13,9 +13,8 @@ def self.delayed_unbind(client_attrs, binding) binding.app.guid ) - request_job = VCAP::CloudController::Jobs::RequestJob.new(unbind_job, ::VCAP::Request.current_id) - retryable_job = VCAP::CloudController::Jobs::RetryableJob.new(request_job, 0) - Delayed::Job.enqueue(retryable_job, queue: 'cc-generic', run_at: Delayed::Job.db_time_now) + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(unbind_job, opts).enqueue end end end diff --git a/spec/unit/jobs/services/service_instance_deprovision_spec.rb b/spec/unit/jobs/services/service_instance_deprovision_spec.rb index bd9d4e223a8..1ca6da718cf 100644 --- a/spec/unit/jobs/services/service_instance_deprovision_spec.rb +++ b/spec/unit/jobs/services/service_instance_deprovision_spec.rb @@ -39,8 +39,18 @@ module Jobs::Services end describe '#max_attempts' do - it 'returns 1' do - expect(job.max_attempts).to eq 1 + it 'returns 10' do + expect(job.max_attempts).to eq 10 + end + end + + describe '#reschedule_at' do + it 'uses exponential backoff' do + now = Time.now + attempts = 5 + + run_at = job.reschedule_at(now, attempts) + expect(run_at).to eq(now + (2**attempts).minutes) end end end diff --git a/spec/unit/jobs/services/service_instance_unbind_spec.rb b/spec/unit/jobs/services/service_instance_unbind_spec.rb index 752b3a1d0fe..2d0c00cd05f 100644 --- a/spec/unit/jobs/services/service_instance_unbind_spec.rb +++ b/spec/unit/jobs/services/service_instance_unbind_spec.rb @@ -36,8 +36,17 @@ module Jobs::Services end describe '#max_attempts' do - it 'returns 1' do - expect(job.max_attempts).to eq 1 + it 'returns 10' do + expect(job.max_attempts).to eq 10 + end + end + + describe '#reschedule_at' do + it 'uses exponential backoff' do + now = Time.now + + run_at = job.reschedule_at(now, 5) + expect(run_at).to eq(now + (2**5).minutes) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb index 813dc087a7f..53b6d4bc104 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb @@ -3,7 +3,7 @@ module VCAP::CloudController module ServiceBrokers::V2 describe ServiceInstanceDeprovisioner do - let(:client_attrs) { {} } + let(:client_attrs) { { uri: 'broker.com' } } let(:plan) { VCAP::CloudController::ServicePlan.make } let(:space) { VCAP::CloudController::Space.make } @@ -12,53 +12,24 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'deprovision' do - let(:mock_client) { double(:client, deprovision: nil) } + it 'enqueues a deprovison job' do + mock_enqueuer = double(:enqueuer, enqueue: nil) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - before do - allow(Delayed::Job).to receive(:enqueue) - allow(VCAP::Request).to receive(:current_id).and_return('current_thread_id') - allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(mock_client) - end - - it 'enqueues a job with the correct queue and run_at time' do ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| expect(opts[:queue]).to eq 'cc-generic' expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) - end - end - - it 'enqueues a job that deprovisions an instance' do - mock_instance = double(:instance, guid: service_instance.guid) - allow(VCAP::CloudController::ServiceInstance).to receive(:new).and_return(mock_instance) - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - job.perform + expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision + expect(job.name).to eq 'service-instance-deprovision' + expect(job.client_attrs).to eq client_attrs + expect(job.service_instance_guid).to eq service_instance.guid + expect(job.service_plan_guid).to eq service_instance.service_plan.guid end - expect(mock_client).to have_received(:deprovision).with(mock_instance) - end - - it 'creates the job in the same context as the original request' do - allow(VCAP::Request).to receive(:current_id=) - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - job.perform - end - expect(VCAP::Request).to have_received(:current_id=).twice.with('current_thread_id') - end - - it 'makes the job retryable' do - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob - expect(job.num_attempts).to eq 0 - end + expect(mock_enqueuer).to have_received(:enqueue) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb index 9a30f88a0d1..6ef8e70a926 100644 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb @@ -12,56 +12,25 @@ module ServiceBrokers::V2 let(:name) { 'fake-name' } describe 'delayed_unbind' do - let(:mock_client) { double(:client, unbind: nil) } + it 'enqueues an unbind job' do + mock_enqueuer = double(:enqueuer, enqueue: nil) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - before do - allow(Delayed::Job).to receive(:enqueue) - allow(VCAP::Request).to receive(:current_id).and_return('current_thread_id') - allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(mock_client) - end - - it 'enqueues a job with the correct queue and run_at time' do ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| expect(opts[:queue]).to eq 'cc-generic' expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) - end - end - - it 'enqueues a job that unbinds the instance' do - mock_binding = double(:binding, - guid: binding.guid, - app_guid: binding.app_guid, - service_instance_guid: binding.service_instance_guid) - allow(VCAP::CloudController::ServiceBinding).to receive(:new).and_return(mock_binding) - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - job.perform + expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind + expect(job.name).to eq 'service-instance-unbind' + expect(job.client_attrs).to eq client_attrs + expect(job.binding_guid).to eq binding.guid + expect(job.service_instance_guid).to eq binding.service_instance.guid + expect(job.app_guid).to eq binding.app.guid end - expect(mock_client).to have_received(:unbind).with(mock_binding) - end - - it 'creates the job in the same context as the original request' do - allow(VCAP::Request).to receive(:current_id=) - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - job.perform - end - expect(VCAP::Request).to have_received(:current_id=).twice.with('current_thread_id') - end - - it 'makes the job retryable' do - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - - expect(Delayed::Job).to have_received(:enqueue) do |job, opts| - expect(job).to be_a VCAP::CloudController::Jobs::RetryableJob - expect(job.num_attempts).to eq 0 - end + expect(mock_enqueuer).to have_received(:enqueue) end end end From 8bba6a38e1d6708b1d7e6a210cb90b51724f3cd6 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Mon, 12 Jan 2015 09:48:37 -0800 Subject: [PATCH 28/76] Remove Retryable Job no longer being used --- app/jobs/retryable_job.rb | 25 --------- lib/cloud_controller/jobs.rb | 1 - spec/unit/jobs/retryable_job_spec.rb | 83 ---------------------------- 3 files changed, 109 deletions(-) delete mode 100644 app/jobs/retryable_job.rb delete mode 100644 spec/unit/jobs/retryable_job_spec.rb diff --git a/app/jobs/retryable_job.rb b/app/jobs/retryable_job.rb deleted file mode 100644 index eaebe1bb68a..00000000000 --- a/app/jobs/retryable_job.rb +++ /dev/null @@ -1,25 +0,0 @@ -module VCAP::CloudController - module Jobs - class RetryableJob - attr_reader :job, :num_attempts - - def initialize(job, num_attempts=0) - @job = job - @num_attempts = num_attempts - end - - def perform - job.perform - rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout, VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse => e - raise e if num_attempts >= 10 - Delayed::Job.enqueue(RetryableJob.new(job, num_attempts + 1), queue: 'cc-generic', run_at: Delayed::Job.db_time_now + (2**num_attempts).minutes) - end - - def max_attempts - # We don't want DelayedJob to handle the retry logic, because we only want to perform - # retries for specific failures. We'll handle retry and num_attempts separately. - 1 - end - end - end -end diff --git a/lib/cloud_controller/jobs.rb b/lib/cloud_controller/jobs.rb index 64a5f8fc42f..77ca951b7a1 100644 --- a/lib/cloud_controller/jobs.rb +++ b/lib/cloud_controller/jobs.rb @@ -17,4 +17,3 @@ require 'jobs/request_job' require 'jobs/timeout_job' require 'jobs/local_queue' -require 'jobs/retryable_job' diff --git a/spec/unit/jobs/retryable_job_spec.rb b/spec/unit/jobs/retryable_job_spec.rb deleted file mode 100644 index 5ab0bfc9f5f..00000000000 --- a/spec/unit/jobs/retryable_job_spec.rb +++ /dev/null @@ -1,83 +0,0 @@ -require 'spec_helper' - -module VCAP::CloudController - module Jobs - describe 'RetryableJob' do - describe '#perform' do - let(:job) { double(:job, perform: nil) } - let(:retryable_job) { RetryableJob.new(job) } - - it 'performs the job' do - retryable_job.perform - expect(job).to have_received(:perform) - end - - context 'when the inner job fails' do - let(:mock_response) { double(:response, body: nil, message: nil, code: 500) } - before do - allow(job).to receive(:perform).and_raise(error) - allow(Delayed::Job).to receive(:enqueue) - end - - context 'when the exception is Errors::ServiceBrokerApiTimeout' do - let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } - - it 'enqueues another retryable job' do - retryable_job.perform - - expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| - expect(enqueued_job).to be_a RetryableJob - expect(enqueued_job.num_attempts).to eq 1 - expect(opts).to include(queue: 'cc-generic', run_at: anything) - end - end - end - - context 'when the exception is ServiceBrokerBadResponse' do - let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new('uri.com', :delete, mock_response) } - - it 'enqueues another retryable job' do - retryable_job.perform - - expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| - expect(enqueued_job).to be_a RetryableJob - expect(enqueued_job.num_attempts).to eq 1 - expect(opts).to include(queue: 'cc-generic', run_at: anything) - end - end - end - - describe 'exponential backoff' do - let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } - let(:retryable_job) { RetryableJob.new(job, num_attempts) } - let(:num_attempts) { 5 } - - it 'runs the subsequent job at 2^(num_attempts) minutes from now' do - now = Time.now - Timecop.freeze now do - retryable_job.perform - - expect(Delayed::Job).to have_received(:enqueue) do |enqueued_job, opts| - expect(enqueued_job.num_attempts).to eq(num_attempts + 1) - run_at = opts[:run_at] - expect(run_at).to be_within(0.01).of(now + (2**num_attempts).minutes) - end - end - end - - context 'when the max attempts have reached' do - let(:retryable_job) { RetryableJob.new(job, 10) } - let(:error) { VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new('uri.com', :delete, mock_response) } - - it 'progagates an error' do - expect { - retryable_job.perform - }.to raise_error(VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout) - end - end - end - end - end - end - end -end From 1a360b2c79e9576505049fe228bba6abcd6bfd52 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Mon, 12 Jan 2015 09:49:11 -0800 Subject: [PATCH 29/76] Add logging to deprovision job --- app/jobs/services/service_instance_deprovision.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/jobs/services/service_instance_deprovision.rb b/app/jobs/services/service_instance_deprovision.rb index eaf87697988..4586d99ed57 100644 --- a/app/jobs/services/service_instance_deprovision.rb +++ b/app/jobs/services/service_instance_deprovision.rb @@ -3,6 +3,9 @@ module Jobs module Services class ServiceInstanceDeprovision < Struct.new(:name, :client_attrs, :service_instance_guid, :service_plan_guid) def perform + logger = Steno.logger('cc-background') + logger.info('There was an error during service instance provisioning. Attempting to delete potentially orphaned instance.') + client = VCAP::Services::ServiceBrokers::V2::Client.new(client_attrs) service_plan = ServicePlan.first(guid: service_plan_guid) service_instance = ManagedServiceInstance.new(guid: service_instance_guid, service_plan: service_plan) From f2d85b4dd0306ae19fb9793a9ce6d6d013b1191f Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Mon, 12 Jan 2015 09:50:04 -0800 Subject: [PATCH 30/76] Merge Unbinder and Deprovisoner into the single OrphanMitigator --- lib/services/service_brokers/v2.rb | 3 +- lib/services/service_brokers/v2/client.rb | 4 +- .../service_brokers/v2/orphan_mitigator.rb | 35 ++++++++++ .../v2/service_instance_deprovisioner.rb | 21 ------ .../v2/service_instance_unbinder.rb | 22 ------- .../service_brokers/v2/client_spec.rb | 28 ++++---- .../v2/orphan_mitigator_spec.rb | 64 +++++++++++++++++++ .../v2/service_instance_deprovisioner_spec.rb | 37 ----------- .../v2/service_instance_unbinder_spec.rb | 38 ----------- 9 files changed, 116 insertions(+), 136 deletions(-) create mode 100644 lib/services/service_brokers/v2/orphan_mitigator.rb delete mode 100644 lib/services/service_brokers/v2/service_instance_deprovisioner.rb delete mode 100644 lib/services/service_brokers/v2/service_instance_unbinder.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb delete mode 100644 spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb delete mode 100644 spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb diff --git a/lib/services/service_brokers/v2.rb b/lib/services/service_brokers/v2.rb index 259f89d2cc1..69f31b708bb 100644 --- a/lib/services/service_brokers/v2.rb +++ b/lib/services/service_brokers/v2.rb @@ -6,7 +6,6 @@ module VCAP::Services::ServiceBrokers::V2 end require 'services/service_brokers/v2/catalog_plan' require 'services/service_brokers/v2/http_client' require 'services/service_brokers/v2/client' -require 'services/service_brokers/v2/service_instance_deprovisioner' -require 'services/service_brokers/v2/service_instance_unbinder' +require 'services/service_brokers/v2/orphan_mitigator' require 'services/service_brokers/v2/response_parser' require 'services/service_brokers/v2/errors' diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index bd08c722d0b..bcf3376aeb4 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -32,7 +32,7 @@ def provision(instance) instance.credentials = {} rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e - VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner.deprovision(@attrs, instance) + VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator.cleanup_failed_provision(@attrs, instance) raise e end @@ -51,7 +51,7 @@ def bind(binding) end rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e - VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder.delayed_unbind(@attrs, binding) + VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator.cleanup_failed_bind(@attrs, binding) raise e end diff --git a/lib/services/service_brokers/v2/orphan_mitigator.rb b/lib/services/service_brokers/v2/orphan_mitigator.rb new file mode 100644 index 00000000000..1a98bc87a98 --- /dev/null +++ b/lib/services/service_brokers/v2/orphan_mitigator.rb @@ -0,0 +1,35 @@ +require 'jobs/services/service_instance_deprovision' +require 'jobs/services/service_instance_unbind' + +module VCAP::Services + module ServiceBrokers + module V2 + class OrphanMitigator + def self.cleanup_failed_provision(client_attrs, service_instance) + deprovision_job = VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision.new( + 'service-instance-deprovision', + client_attrs, + service_instance.guid, + service_instance.service_plan.guid + ) + + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(deprovision_job, opts).enqueue + end + + def self.cleanup_failed_bind(client_attrs, binding) + unbind_job = VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind.new( + 'service-instance-unbind', + client_attrs, + binding.guid, + binding.service_instance.guid, + binding.app.guid + ) + + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(unbind_job, opts).enqueue + end + end + end + end +end diff --git a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb b/lib/services/service_brokers/v2/service_instance_deprovisioner.rb deleted file mode 100644 index 5fa975f4579..00000000000 --- a/lib/services/service_brokers/v2/service_instance_deprovisioner.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'jobs/services/service_instance_deprovision' - -module VCAP::CloudController - module ServiceBrokers - module V2 - class ServiceInstanceDeprovisioner - def self.deprovision(client_attrs, service_instance) - deprovision_job = VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision.new( - 'service-instance-deprovision', - client_attrs, - service_instance.guid, - service_instance.service_plan.guid - ) - - opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } - VCAP::CloudController::Jobs::Enqueuer.new(deprovision_job, opts).enqueue - end - end - end - end -end diff --git a/lib/services/service_brokers/v2/service_instance_unbinder.rb b/lib/services/service_brokers/v2/service_instance_unbinder.rb deleted file mode 100644 index 4e287882f5d..00000000000 --- a/lib/services/service_brokers/v2/service_instance_unbinder.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'jobs/services/service_instance_unbind' - -module VCAP::CloudController - module ServiceBrokers - module V2 - class ServiceInstanceUnbinder - def self.delayed_unbind(client_attrs, binding) - unbind_job = VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind.new( - 'service-instance-unbind', - client_attrs, - binding.guid, - binding.service_instance.guid, - binding.app.guid - ) - - opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } - VCAP::CloudController::Jobs::Enqueuer.new(unbind_job, opts).enqueue - end - end - end - end -end diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index aac77d001af..303ba53a9b8 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -139,7 +139,7 @@ module VCAP::Services::ServiceBrokers::V2 let(:response) { double(:response, body: nil, message: nil) } before do - allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner).to receive(:deprovision) + allow(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator).to receive(:cleanup_failed_provision) end context 'due to an http client error' do @@ -157,8 +157,8 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision).with(client_attrs, instance) + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_provision).with(client_attrs, instance) end end end @@ -179,8 +179,8 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_provision). with(client_attrs, instance) end end @@ -193,8 +193,8 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerBadResponse) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceDeprovisioner). - to have_received(:deprovision). + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_provision). with(client_attrs, instance) end end @@ -367,7 +367,7 @@ module VCAP::Services::ServiceBrokers::V2 let(:response) { double(:response, body: nil, message: nil) } before do - allow(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder).to receive(:delayed_unbind) + allow(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator).to receive(:cleanup_failed_bind) end context 'due to an http client error' do @@ -385,8 +385,8 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:delayed_unbind). + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_bind). with(client_attrs, binding) end end @@ -408,8 +408,8 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:delayed_unbind).with(client_attrs, binding) + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_bind).with(client_attrs, binding) end end @@ -421,8 +421,8 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerBadResponse) - expect(VCAP::CloudController::ServiceBrokers::V2::ServiceInstanceUnbinder). - to have_received(:delayed_unbind).with(client_attrs, binding) + expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). + to have_received(:cleanup_failed_bind).with(client_attrs, binding) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb new file mode 100644 index 00000000000..f0835fc51d6 --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers::V2 + describe OrphanMitigator do + let(:client_attrs) { { uri: 'broker.com' } } + + let(:plan) { VCAP::CloudController::ServicePlan.make } + let(:space) { VCAP::CloudController::Space.make } + let(:service_instance) { VCAP::CloudController::ManagedServiceInstance.new(service_plan: plan, space: space) } + let(:binding) do + VCAP::CloudController::ServiceBinding.make( + binding_options: { 'this' => 'that' } + ) + end + let(:name) { 'fake-name' } + + describe 'cleanup_failed_provision' do + it 'enqueues a deprovison job' do + mock_enqueuer = double(:enqueuer, enqueue: nil) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) + + OrphanMitigator.cleanup_failed_provision(client_attrs, service_instance) + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + + expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision + expect(job.name).to eq 'service-instance-deprovision' + expect(job.client_attrs).to eq client_attrs + expect(job.service_instance_guid).to eq service_instance.guid + expect(job.service_plan_guid).to eq service_instance.service_plan.guid + end + + expect(mock_enqueuer).to have_received(:enqueue) + end + end + + describe 'cleanup_failed_bind' do + it 'enqueues an unbind job' do + mock_enqueuer = double(:enqueuer, enqueue: nil) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) + + OrphanMitigator.cleanup_failed_bind(client_attrs, binding) + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + + expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind + expect(job.name).to eq 'service-instance-unbind' + expect(job.client_attrs).to eq client_attrs + expect(job.binding_guid).to eq binding.guid + expect(job.service_instance_guid).to eq binding.service_instance.guid + expect(job.app_guid).to eq binding.app.guid + end + + expect(mock_enqueuer).to have_received(:enqueue) + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb deleted file mode 100644 index 53b6d4bc104..00000000000 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_deprovisioner_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -require 'spec_helper' - -module VCAP::CloudController - module ServiceBrokers::V2 - describe ServiceInstanceDeprovisioner do - let(:client_attrs) { { uri: 'broker.com' } } - - let(:plan) { VCAP::CloudController::ServicePlan.make } - let(:space) { VCAP::CloudController::Space.make } - let(:service_instance) { VCAP::CloudController::ManagedServiceInstance.new(service_plan: plan, space: space) } - - let(:name) { 'fake-name' } - - describe 'deprovision' do - it 'enqueues a deprovison job' do - mock_enqueuer = double(:enqueuer, enqueue: nil) - allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - - ServiceInstanceDeprovisioner.deprovision(client_attrs, service_instance) - - expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| - expect(opts[:queue]).to eq 'cc-generic' - expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) - - expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision - expect(job.name).to eq 'service-instance-deprovision' - expect(job.client_attrs).to eq client_attrs - expect(job.service_instance_guid).to eq service_instance.guid - expect(job.service_plan_guid).to eq service_instance.service_plan.guid - end - - expect(mock_enqueuer).to have_received(:enqueue) - end - end - end - end -end diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb deleted file mode 100644 index 6ef8e70a926..00000000000 --- a/spec/unit/lib/services/service_brokers/v2/service_instance_unbinder_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -require 'spec_helper' - -module VCAP::CloudController - module ServiceBrokers::V2 - describe ServiceInstanceUnbinder do - let(:client_attrs) { {} } - let(:binding) do - VCAP::CloudController::ServiceBinding.make( - binding_options: { 'this' => 'that' } - ) - end - let(:name) { 'fake-name' } - - describe 'delayed_unbind' do - it 'enqueues an unbind job' do - mock_enqueuer = double(:enqueuer, enqueue: nil) - allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - - ServiceInstanceUnbinder.delayed_unbind(client_attrs, binding) - - expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| - expect(opts[:queue]).to eq 'cc-generic' - expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) - - expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind - expect(job.name).to eq 'service-instance-unbind' - expect(job.client_attrs).to eq client_attrs - expect(job.binding_guid).to eq binding.guid - expect(job.service_instance_guid).to eq binding.service_instance.guid - expect(job.app_guid).to eq binding.app.guid - end - - expect(mock_enqueuer).to have_received(:enqueue) - end - end - end - end -end From 146073ea4b05b429d9d41d1448b93090549fc635 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Mon, 12 Jan 2015 10:58:26 -0800 Subject: [PATCH 31/76] Make orphan mitigator class methods into instance methods --- lib/services/service_brokers/v2/client.rb | 5 +-- .../service_brokers/v2/orphan_mitigator.rb | 4 +-- .../service_brokers/v2/client_spec.rb | 31 ++++++------------- .../v2/orphan_mitigator_spec.rb | 4 +-- 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index bcf3376aeb4..57627760dc8 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -6,6 +6,7 @@ def initialize(attrs) @http_client = VCAP::Services::ServiceBrokers::V2::HttpClient.new(attrs) @response_parser = VCAP::Services::ServiceBrokers::V2::ResponseParser.new(@http_client.url) @attrs = attrs + @orphan_mitigator = VCAP::Services::ServiceBrokers::V2::OrphanMitigator.new end def catalog @@ -32,7 +33,7 @@ def provision(instance) instance.credentials = {} rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e - VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator.cleanup_failed_provision(@attrs, instance) + @orphan_mitigator.cleanup_failed_provision(@attrs, instance) raise e end @@ -51,7 +52,7 @@ def bind(binding) end rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e - VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator.cleanup_failed_bind(@attrs, binding) + @orphan_mitigator.cleanup_failed_bind(@attrs, binding) raise e end diff --git a/lib/services/service_brokers/v2/orphan_mitigator.rb b/lib/services/service_brokers/v2/orphan_mitigator.rb index 1a98bc87a98..7d3882a663b 100644 --- a/lib/services/service_brokers/v2/orphan_mitigator.rb +++ b/lib/services/service_brokers/v2/orphan_mitigator.rb @@ -5,7 +5,7 @@ module VCAP::Services module ServiceBrokers module V2 class OrphanMitigator - def self.cleanup_failed_provision(client_attrs, service_instance) + def cleanup_failed_provision(client_attrs, service_instance) deprovision_job = VCAP::CloudController::Jobs::Services::ServiceInstanceDeprovision.new( 'service-instance-deprovision', client_attrs, @@ -17,7 +17,7 @@ def self.cleanup_failed_provision(client_attrs, service_instance) VCAP::CloudController::Jobs::Enqueuer.new(deprovision_job, opts).enqueue end - def self.cleanup_failed_bind(client_attrs, binding) + def cleanup_failed_bind(client_attrs, binding) unbind_job = VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind.new( 'service-instance-unbind', client_attrs, diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 303ba53a9b8..8204d2c56d7 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -15,12 +15,16 @@ module VCAP::Services::ServiceBrokers::V2 subject(:client) { Client.new(client_attrs) } let(:http_client) { double('http_client') } + let(:orphan_mitigator) { double('orphan_mitigator', cleanup_failed_provision: nil, cleanup_failed_bind: nil) } before do allow(HttpClient).to receive(:new). with(url: service_broker.broker_url, auth_username: service_broker.auth_username, auth_password: service_broker.auth_password). and_return(http_client) + allow(VCAP::Services::ServiceBrokers::V2::OrphanMitigator).to receive(:new). + and_return(orphan_mitigator) + allow(http_client).to receive(:url).and_return(service_broker.broker_url) end @@ -138,10 +142,6 @@ module VCAP::Services::ServiceBrokers::V2 let(:uri) { 'some-uri.com/v2/service_instances/some-guid' } let(:response) { double(:response, body: nil, message: nil) } - before do - allow(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator).to receive(:cleanup_failed_provision) - end - context 'due to an http client error' do let(:http_client) { double(:http_client) } @@ -157,8 +157,7 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_provision).with(client_attrs, instance) + expect(orphan_mitigator).to have_received(:cleanup_failed_provision).with(client_attrs, instance) end end end @@ -179,8 +178,7 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_provision). + expect(orphan_mitigator).to have_received(:cleanup_failed_provision). with(client_attrs, instance) end end @@ -193,9 +191,7 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) }.to raise_error(Errors::ServiceBrokerBadResponse) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_provision). - with(client_attrs, instance) + expect(orphan_mitigator).to have_received(:cleanup_failed_provision).with(client_attrs, instance) end end end @@ -366,10 +362,6 @@ module VCAP::Services::ServiceBrokers::V2 let(:uri) { 'some-uri.com/v2/service_instances/instance-guid/service_bindings/binding-guid' } let(:response) { double(:response, body: nil, message: nil) } - before do - allow(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator).to receive(:cleanup_failed_bind) - end - context 'due to an http client error' do let(:http_client) { double(:http_client) } @@ -385,8 +377,7 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_bind). + expect(orphan_mitigator).to have_received(:cleanup_failed_bind). with(client_attrs, binding) end end @@ -408,8 +399,7 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerApiTimeout) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_bind).with(client_attrs, binding) + expect(orphan_mitigator).to have_received(:cleanup_failed_bind).with(client_attrs, binding) end end @@ -421,8 +411,7 @@ module VCAP::Services::ServiceBrokers::V2 client.bind(binding) }.to raise_error(Errors::ServiceBrokerBadResponse) - expect(VCAP::CloudController::ServiceBrokers::V2::OrphanMitigator). - to have_received(:cleanup_failed_bind).with(client_attrs, binding) + expect(orphan_mitigator).to have_received(:cleanup_failed_bind).with(client_attrs, binding) end end end diff --git a/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb index f0835fc51d6..4d307bab7ff 100644 --- a/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb @@ -20,7 +20,7 @@ module ServiceBrokers::V2 mock_enqueuer = double(:enqueuer, enqueue: nil) allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - OrphanMitigator.cleanup_failed_provision(client_attrs, service_instance) + OrphanMitigator.new.cleanup_failed_provision(client_attrs, service_instance) expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| expect(opts[:queue]).to eq 'cc-generic' @@ -42,7 +42,7 @@ module ServiceBrokers::V2 mock_enqueuer = double(:enqueuer, enqueue: nil) allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - OrphanMitigator.cleanup_failed_bind(client_attrs, binding) + OrphanMitigator.new.cleanup_failed_bind(client_attrs, binding) expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| expect(opts[:queue]).to eq 'cc-generic' From d23704b14dd62aa17b04848b92d3373d2225fd99 Mon Sep 17 00:00:00 2001 From: "Chris Piraino, Dan Lavine, James Myers and Sujoy Basu" Date: Wed, 7 Jan 2015 12:16:12 -0800 Subject: [PATCH 32/76] Implement POST /v3/apps/:guid/packages Implement GET /v3/packages/:guid * For post, multipart form request with bits as the upload argument * AccessContext is the same as v3 apps at the moment [#79388094] --- app/access/v3/package_access.rb | 37 ++++ app/controllers/v3/packages_controller.rb | 58 ++++++ app/handlers/packages_handler.rb | 72 +++++++ app/handlers/processes_handler.rb | 2 +- app/jobs/v3/package_bits.rb | 36 ++++ app/models.rb | 5 +- app/models/runtime/app_bits_package.rb | 30 +++ app/models/v3/persistence/package_model.rb | 9 + app/presenters/v3/package_presenter.rb | 24 +++ bosh-templates/nginx.conf.erb | 3 +- .../20141226222846_create_packages.rb | 14 ++ lib/cloud_controller/dependency_locator.rb | 8 + spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 141 +++++++++++++ spec/support/fakes/blueprints.rb | 4 + spec/support/matchers/sequel_validations.rb | 11 ++ spec/unit/access/v3/package_access_spec.rb | 144 ++++++++++++++ .../v3/packages_controller_spec.rb | 186 ++++++++++++++++++ .../v3/processes_controller_spec.rb | 1 + spec/unit/handlers/packages_handler_spec.rb | 176 +++++++++++++++++ spec/unit/jobs/v3/package_bits_spec.rb | 46 +++++ .../models/runtime/app_bits_package_spec.rb | 56 ++++++ .../v3/persistence/package_model_spec.rb | 8 + .../presenters/v3/package_presenter_spec.rb | 24 +++ 24 files changed, 1092 insertions(+), 5 deletions(-) create mode 100644 app/access/v3/package_access.rb create mode 100644 app/controllers/v3/packages_controller.rb create mode 100644 app/handlers/packages_handler.rb create mode 100644 app/jobs/v3/package_bits.rb create mode 100644 app/models/v3/persistence/package_model.rb create mode 100644 app/presenters/v3/package_presenter.rb create mode 100644 db/migrations/20141226222846_create_packages.rb create mode 100644 spec/api/documentation/v3/packages_api_spec.rb create mode 100644 spec/unit/access/v3/package_access_spec.rb create mode 100644 spec/unit/controllers/v3/packages_controller_spec.rb create mode 100644 spec/unit/handlers/packages_handler_spec.rb create mode 100644 spec/unit/jobs/v3/package_bits_spec.rb create mode 100644 spec/unit/models/runtime/v3/persistence/package_model_spec.rb create mode 100644 spec/unit/presenters/v3/package_presenter_spec.rb diff --git a/app/access/v3/package_access.rb b/app/access/v3/package_access.rb new file mode 100644 index 00000000000..4e8a056e913 --- /dev/null +++ b/app/access/v3/package_access.rb @@ -0,0 +1,37 @@ +module VCAP::CloudController + class PackageModelAccess + include Allowy::AccessControl + + def read?(package) + return true if context.roles.admin? + + has_read_scope = SecurityContext.scopes.include?('cloud_controller.read') + user_visible = AppModel.user_visible(context.user).where(guid: package.app_guid).count > 0 + + has_read_scope && user_visible + end + + def create?(desired_package) + return true if context.roles.admin? + + has_write_scope = SecurityContext.scopes.include?('cloud_controller.write') + + app = AppModel.find(guid: desired_package.app_guid) + space = Space.find(guid: app.space_guid) + + is_space_developer = space && space.developers.include?(context.user) + + org_active = space && space.organization.active? + + has_write_scope && is_space_developer && org_active + end + + def delete?(app) + create?(app) + end + + def update?(app) + create?(app) + end + end +end diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb new file mode 100644 index 00000000000..3af00d2258b --- /dev/null +++ b/app/controllers/v3/packages_controller.rb @@ -0,0 +1,58 @@ +require 'presenters/v3/package_presenter' +require 'handlers/packages_handler' + +module VCAP::CloudController + class PackagesController < RestController::BaseController + def self.dependencies + [ :packages_handler, :package_presenter ] + end + + def inject_dependencies(dependencies) + @packages_handler = dependencies[:packages_handler] + @package_presenter = dependencies[:package_presenter] + end + + post '/v3/apps/:guid/packages', :create + def create(app_guid) + message = PackageCreateMessage.new(app_guid, params) + valid, errors = message.validate + unprocessable!(errors.join(', ')) if !valid + + package = @packages_handler.create(message, @access_context) + package_json = @package_presenter.present_json(package) + + [HTTP::CREATED, package_json] + rescue PackagesHandler::Unauthorized + unauthorized! + end + + get '/v3/packages/:guid', :show + def show(package_guid) + package = @packages_handler.show(package_guid, @access_context) + package_not_found! if package.nil? + + package_json = @package_presenter.present_json(package) + [HTTP::OK, package_json] + rescue PackagesHandler::Unauthorized + unauthorized! + end + + private + + def package_not_found! + raise VCAP::Errors::ApiError.new_from_details('ResourceNotFound', 'Package not found') + end + + def bad_request!(message) + raise VCAP::Errors::ApiError.new_from_details('MessageParseError', message) + end + + def unauthorized! + raise VCAP::Errors::ApiError.new_from_details('NotAuthorized') + end + + def unprocessable!(message) + raise VCAP::Errors::ApiError.new_from_details('UnprocessableEntity', message) + end + end +end diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb new file mode 100644 index 00000000000..93d969b8e73 --- /dev/null +++ b/app/handlers/packages_handler.rb @@ -0,0 +1,72 @@ +module VCAP::CloudController + class PackageCreateMessage + attr_reader :app_guid, :type, :filepath + + def initialize(app_guid, opts) + @app_guid = app_guid + @type = opts['type'] + @filepath = opts['bits_path'] + @filename = opts['bits_name'] + end + + def validate + errors = [] + errors << validate_type_field + errors << validate_file if @type == 'bits' + errs = errors.compact + return errs.length == 0, errs + end + + private + + def validate_type_field + return 'The type field is required' if @type.nil? + valid_type_fields = %w(bits docker) + + if !valid_type_fields.include?(@type) + return "The type field needs to be one of '#{valid_type_fields.join(", ")}'" + end + nil + end + + def validate_file + return 'Must upload an application zip file' if @filepath.nil? + nil + end + end + + class PackagesHandler + class Unauthorized < StandardError; end + class InvalidPackage < StandardError; end + + PACKAGE_STATES = %w[PENDING READY FAILED].map(&:freeze).freeze + + def initialize(config) + @config = config + end + + def create(message, access_context) + package = PackageModel.new + package.app_guid = message.app_guid + package.type = message.type + + raise Unauthorized if access_context.cannot?(:create, package) + + package.save + + bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) + Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue() + + package + rescue Sequel::ValidationFailed => e + raise InvalidPackage.new(e.message) + end + + def show(guid, access_context) + package = PackageModel.find(guid: guid) + return nil if package.nil? + raise Unauthorized if access_context.cannot?(:read, package) + package + end + end +end diff --git a/app/handlers/processes_handler.rb b/app/handlers/processes_handler.rb index 475d7a325a0..1c671436053 100644 --- a/app/handlers/processes_handler.rb +++ b/app/handlers/processes_handler.rb @@ -41,7 +41,7 @@ def validate errors = [] errors << validate_name_field errors << validate_has_opts - errors.compact! + errors.compact end private diff --git a/app/jobs/v3/package_bits.rb b/app/jobs/v3/package_bits.rb new file mode 100644 index 00000000000..9716e451d7e --- /dev/null +++ b/app/jobs/v3/package_bits.rb @@ -0,0 +1,36 @@ +require "cloud_controller/blobstore/cdn" +require "cloud_controller/dependency_locator" + +module VCAP::CloudController + module Jobs + module Runtime + class PackageBits + + def initialize(package_guid, uploaded_compressed_path) + @package_guid = package_guid + @uploaded_compressed_path = uploaded_compressed_path + end + + def perform + logger = Steno.logger("cc.background") + logger.info("Packing the app bits for package '#{@package_guid}'") + + package_blobstore = CloudController::DependencyLocator.instance.package_blobstore + max_package_size = VCAP::CloudController::Config.config[:packages][:max_package_size] || 512 * 1024 * 1024 + + app_bits_packer = AppBitsPackage.new(package_blobstore, nil, max_package_size, VCAP::CloudController::Config.config[:directories][:tmpdir]) + app_bits_packer.create_package_in_blobstore(@package_guid, @uploaded_compressed_path) + end + + def job_name_in_configuration + :package_bits + end + + def max_attempts + 1 + end + end + end + end +end + diff --git a/app/models.rb b/app/models.rb index ceeb6bd614e..4108819c7e4 100644 --- a/app/models.rb +++ b/app/models.rb @@ -60,5 +60,6 @@ require 'models/job' -require 'models/v3/domain/app_process' -require 'models/v3/persistence/app_model' +require "models/v3/domain/app_process" +require "models/v3/persistence/app_model" +require "models/v3/persistence/package_model" diff --git a/app/models/runtime/app_bits_package.rb b/app/models/runtime/app_bits_package.rb index 6adf7853718..215908fc36b 100644 --- a/app/models/runtime/app_bits_package.rb +++ b/app/models/runtime/app_bits_package.rb @@ -2,6 +2,8 @@ require 'cloud_controller/blobstore/fingerprints_collection' class AppBitsPackage + class PackageNotFound < StandardError; end + attr_reader :package_blobstore, :global_app_bits_cache, :max_package_size, :tmp_dir def initialize(package_blobstore, global_app_bits_cache, max_package_size, tmp_dir) @@ -30,6 +32,34 @@ def create(app, uploaded_tmp_compressed_path, fingerprints_in_app_cache) FileUtils.rm_f(uploaded_tmp_compressed_path) if uploaded_tmp_compressed_path end + def create_package_in_blobstore(package_guid, uploaded_tmp_compressed_path) + return unless uploaded_tmp_compressed_path + + package = VCAP::CloudController::PackageModel.find(guid: package_guid) + raise PackageNotFound if package.nil? + + begin + zip_file = File.new(uploaded_tmp_compressed_path) + package_blobstore.cp_to_blobstore(uploaded_tmp_compressed_path, package_guid) + + package.db.transaction do + package.lock! + package.package_hash = zip_file.hexdigest + package.state = 'READY' + package.save + end + rescue => e + package.db.transaction do + package.state = 'FAILED' + package.error = e.message + package.save + end + raise e + end + ensure + FileUtils.rm_f(uploaded_tmp_compressed_path) if uploaded_tmp_compressed_path + end + private def validate_size!(fingerprints_in_app_cache, local_app_bits) diff --git a/app/models/v3/persistence/package_model.rb b/app/models/v3/persistence/package_model.rb new file mode 100644 index 00000000000..6e7bc62ac54 --- /dev/null +++ b/app/models/v3/persistence/package_model.rb @@ -0,0 +1,9 @@ +module VCAP::CloudController + class PackageModel < Sequel::Model(:packages) + PACKAGE_STATES = %w[PENDING READY FAILED].map(&:freeze).freeze + + def validate + validates_includes PACKAGE_STATES, :state, allow_missing: true + end + end +end diff --git a/app/presenters/v3/package_presenter.rb b/app/presenters/v3/package_presenter.rb new file mode 100644 index 00000000000..f500186e5fc --- /dev/null +++ b/app/presenters/v3/package_presenter.rb @@ -0,0 +1,24 @@ +module VCAP::CloudController + class PackagePresenter + def present_json(package) + package_hash = { + guid: package.guid, + type: package.type, + hash: package.package_hash, + state: package.state, + error: package.error, + created_at: package.created_at, + _links: { + self: { + href: "/v3/packages/#{package.guid}" + }, + app: { + href: "/v3/apps/#{package.app_guid}", + }, + }, + } + + MultiJson.dump(package_hash, pretty: true) + end + end +end diff --git a/bosh-templates/nginx.conf.erb b/bosh-templates/nginx.conf.erb index 13d14d9d96f..36f4cb038d2 100644 --- a/bosh-templates/nginx.conf.erb +++ b/bosh-templates/nginx.conf.erb @@ -91,7 +91,7 @@ http { } <% end %> - location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits) { + location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits|/v3/apps/.*/packages) { # Pass altered request body to this location upload_pass @cc_uploads; upload_pass_args on; @@ -112,6 +112,7 @@ http { #forward the following fields from existing body upload_pass_form_field "^resources$"; upload_pass_form_field "^_method$"; + upload_pass_form_field "^type$"; #on any error, delete uploaded files. upload_cleanup 400-505; diff --git a/db/migrations/20141226222846_create_packages.rb b/db/migrations/20141226222846_create_packages.rb new file mode 100644 index 00000000000..5c5e8f5699d --- /dev/null +++ b/db/migrations/20141226222846_create_packages.rb @@ -0,0 +1,14 @@ +Sequel.migration do + change do + create_table :packages do + VCAP::Migration.common(self) + String :app_guid + index :app_guid + String :type + index :type + String :package_hash + String :state, default: 'PENDING', null: false + String :error, text: true + end + end +end diff --git a/lib/cloud_controller/dependency_locator.rb b/lib/cloud_controller/dependency_locator.rb index d2c89ef04cc..1438379f0e1 100644 --- a/lib/cloud_controller/dependency_locator.rb +++ b/lib/cloud_controller/dependency_locator.rb @@ -177,6 +177,14 @@ def app_presenter AppPresenter.new end + def packages_handler + PackagesHandler.new(@config) + end + + def package_presenter + PackagePresenter.new + end + def object_renderer eager_loader = VCAP::CloudController::RestController::SecureEagerLoader.new serializer = VCAP::CloudController::RestController::PreloadedObjectSerializer.new diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index f2cd468b6bd..cf6a091e086 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = 'ef734f694dd797ef339e77780c516b07eb11ebfb' + API_FOLDER_CHECKSUM = '89550450a3565fda71642c8fa545f88892c7e4bf' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb new file mode 100644 index 00000000000..c2516fd621b --- /dev/null +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -0,0 +1,141 @@ +require 'spec_helper' +require 'awesome_print' +require 'rspec_api_documentation/dsl' + +resource 'Packages (Experimental)', type: :api do + let(:tmpdir) { Dir.mktmpdir } + let(:valid_zip) { + zip_name = File.join(tmpdir, 'file.zip') + TestZip.create(zip_name, 1, 1024) + File.new(zip_name) + } + + let(:user) { VCAP::CloudController::User.make } + let(:user_header) { headers_for(user)['HTTP_AUTHORIZATION'] } + header 'AUTHORIZATION', :user_header + + def do_request_with_error_handling + do_request + if response_status == 500 + error = MultiJson.load(response_body) + ap error + raise error['description'] + end + end + + context 'standard endpoints' do + get '/v3/packages/:guid' do + let(:space) { VCAP::CloudController::Space.make } + let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } + + let(:package_model) do + VCAP::CloudController::PackageModel.make(app_guid: app_model.guid) + end + + let(:guid) { package_model.guid } + let(:app_guid) { app_model.guid } + + before do + space.organization.add_user user + space.add_developer user + end + + example 'Get a Package' do + # MultiJson/Ruby Json library formats strings differently than to_s + created_at_string = MultiJson.load(package_model.created_at.to_json) + + expected_response = { + 'type' => package_model.type, + 'guid' => guid, + 'hash' => nil, + 'state' => "PENDING", + 'error' => nil, + 'created_at' => created_at_string, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{guid}" }, + 'app' => { 'href' => "/v3/apps/#{app_guid}" }, + } + } + + do_request_with_error_handling + + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(200) + expect(parsed_response).to match(expected_response) + end + end + + post '/v3/apps/:guid/packages' do + let(:space) { VCAP::CloudController::Space.make } + let(:space_guid) { space.guid } + let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } + let(:guid) { app_model.guid } + let(:type) { 'bits' } + let(:packages_params) do + { + type: 'bits', + bits_name: 'application.zip', + bits_path: "#{tmpdir}/application.zip", + } + end + + before do + space.organization.add_user(user) + space.add_developer(user) + end + + parameter :guid, 'GUID of the app that is going to use the package', required: true + parameter :type, 'Package type', required: true, valid_values: ['bits', 'docker'] + parameter :bits, 'A binary zip file containing the package bits', required: false + + let(:request_body_example) do + <<-eos.gsub(/^ */, '') + --AaB03x + Content-Disposition: form-data; name="type" + + #{type} + --AaB03x + Content-Disposition: form-data; name="bits"; filename="application.zip" + Content-Type: application/zip + Content-Length: 123 + Content-Transfer-Encoding: binary + + <<binary artifact bytes>> + --AaB03x + eos + end + + example 'Create a Package' do + expect { + do_request packages_params + }.to change{ VCAP::CloudController::PackageModel.count }.by(1) + + + package = VCAP::CloudController::PackageModel.last + expected_guid = VCAP::CloudController::AppModel.last.guid + expected_created_at = package.created_at.as_json + + job = Delayed::Job.last + expect(job.handler).to include(package.guid) + expect(job.guid).not_to be_nil + + expected_response = { + 'guid' => package.guid, + 'type' => type, + 'hash' => nil, + 'state' => 'PENDING', + 'error' => nil, + 'created_at' => expected_created_at, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package.guid}" }, + 'app' => { 'href' => "/v3/apps/#{expected_guid}" }, + } + } + + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(201) + expect(parsed_response).to match(expected_response) + end + end + end +end diff --git a/spec/support/fakes/blueprints.rb b/spec/support/fakes/blueprints.rb index 175da5e1307..4f943aec323 100644 --- a/spec/support/fakes/blueprints.rb +++ b/spec/support/fakes/blueprints.rb @@ -35,6 +35,10 @@ module VCAP::CloudController space_guid { Space.make.guid } end + PackageModel.blueprint do + guid { Sham.guid } + end + User.blueprint do guid { Sham.uaa_id } end diff --git a/spec/support/matchers/sequel_validations.rb b/spec/support/matchers/sequel_validations.rb index 3c46b882b59..bb4224c50ea 100644 --- a/spec/support/matchers/sequel_validations.rb +++ b/spec/support/matchers/sequel_validations.rb @@ -54,3 +54,14 @@ end end end + +RSpec::Matchers.define :validates_includes do |values, attribute, options = {}| + description do + "validate includes of #{attribute} with #{values}" + end + match do |instance| + allow(instance).to receive(:validates_includes) + instance.valid? + expect(instance).to have_received(:validates_includes).with(values, attribute, options) + end +end diff --git a/spec/unit/access/v3/package_access_spec.rb b/spec/unit/access/v3/package_access_spec.rb new file mode 100644 index 00000000000..e0046dab7eb --- /dev/null +++ b/spec/unit/access/v3/package_access_spec.rb @@ -0,0 +1,144 @@ +require 'spec_helper' + +module VCAP::CloudController + describe PackageModelAccess, type: :access do + let(:token) { {} } + let(:admin) { false } + let(:user) { User.make } + let(:roles) { double(:roles, admin?: admin) } + let(:package) { PackageModel.make } + let(:access_context) { double(:access_context, roles: roles, user: user) } + + before do + SecurityContext.set(nil, token) + end + + after do + SecurityContext.clear + end + + describe '#read?' do + let(:space) { Space.make } + let(:app) { AppModel.make(space_guid: space.guid) } + let(:package) { PackageModel.new(app_guid: app.guid) } + + context 'admin user' do + let(:admin) { true } + + it 'allows the user to read' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.read?(nil)).to be_truthy + end + end + + context 'non admin users' do + context 'when the user has sufficient scope and permission' do + let(:token) { { 'scope' => ['cloud_controller.read'] } } + + it 'allows the user to read' do + allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: app.guid)) + access_control = PackageModelAccess.new(access_context) + expect(access_control.read?(package)).to be_truthy + end + end + + context 'when the user has insufficient scope' do + it 'disallows the user from reading' do + allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: app.guid)) + access_control = PackageModelAccess.new(access_context) + expect(access_control.read?(package)).to be_falsey + end + end + + context 'when the package is not visible to the user' do + let(:token) { { 'scope' => ['cloud_controller.read'] } } + + it 'disallows the user from reading' do + allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: nil)) + access_control = PackageModelAccess.new(access_context) + expect(access_control.read?(package)).to be_falsey + end + end + end + end + + describe '#create?, #update?, #delete?' do + let(:space) { Space.make } + let(:app) { AppModel.make(space_guid: space.guid) } + let(:package) { PackageModel.new(app_guid: app.guid) } + + context 'admin user' do + let(:admin) { true } + + it 'allows the user to perform the action' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.create?(package)).to be_truthy + expect(access_control.update?(package)).to be_truthy + expect(access_control.delete?(package)).to be_truthy + end + end + + context 'non admin users' do + context 'when the user has sufficient scope and permissions' do + let(:token) { { 'scope' => ['cloud_controller.write'] } } + + before do + space.organization.add_user(user) + space.add_developer(user) + end + + it 'allows the user to create' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.create?(package)).to be_truthy + expect(access_control.update?(package)).to be_truthy + expect(access_control.delete?(package)).to be_truthy + end + end + + context 'when the user has insufficient scope' do + let(:token) { { 'scope' => ['cloud_controller.read'] } } + + before do + space.organization.add_user(user) + space.add_developer(user) + end + + it 'disallows the user from creating' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.create?(package)).to be_falsey + expect(access_control.update?(package)).to be_falsey + expect(access_control.delete?(package)).to be_falsey + end + end + + context 'when the user has insufficient permissions' do + let(:token) { { 'scope' => ['cloud_controller.write'] } } + + it 'disallows the user from creating' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.create?(package)).to be_falsey + expect(access_control.update?(package)).to be_falsey + expect(access_control.delete?(package)).to be_falsey + end + end + + context 'when the organization is not active' do + let(:token) { { 'scope' => ['cloud_controller.write'] } } + + before do + space.organization.add_user(user) + space.add_developer(user) + space.organization.update(status: 'suspended') + end + + it 'disallows the user from creating' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.create?(package)).to be_falsey + expect(access_control.update?(package)).to be_falsey + expect(access_control.delete?(package)).to be_falsey + end + end + end + end + end +end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb new file mode 100644 index 00000000000..b31d430fbcd --- /dev/null +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -0,0 +1,186 @@ +require 'spec_helper' + +module VCAP::CloudController + describe PackagesController do + let(:logger) { instance_double(Steno::Logger) } + let(:user) { User.make } + let(:params) { {} } + let(:packages_handler) { double(:process_handler) } + let(:package_presenter) { double(:package_presenter) } + + let(:packages_controller) do + PackagesController.new( + {}, + logger, + {}, + params.stringify_keys, + '', + nil, + { + packages_handler: packages_handler, + package_presenter: package_presenter, + }, + ) + end + + before do + allow(logger).to receive(:debug) + end + + describe '#create' do + let(:tmpdir) { Dir.mktmpdir } + let(:params) { { type: 'bits', bits_path: '/tmp/app.zip', bits_name: 'app.zip' } } + let(:app_obj) { AppModel.make } + let(:app_guid) { app_obj.guid } + let(:package) { PackageModel.make } + + let(:valid_zip) do + zip_name = File.join(tmpdir, "file.zip") + TestZip.create(zip_name, 1, 1024) + zip_file = File.new(zip_name) + Rack::Test::UploadedFile.new(zip_file) + end + + let(:package_response) do + { + type: "bits", + package_hash: "a-hash", + created_at: "a-date", + _links: { + app: { + href: "/v3/apps/#{app_guid}", + }, + }, + } + end + + before do + allow(package_presenter).to receive(:present_json).and_return(MultiJson.dump(package_response, pretty: true)) + allow(packages_handler).to receive(:create).and_return(package) + end + + after do + FileUtils.rm_rf(tmpdir) + end + + context 'when a user can create a package' do + it 'returns a 201 Created response' do + response_code, _ = packages_controller.create(app_guid) + expect(response_code).to eq 201 + end + + it 'returns the package' do + _, response = packages_controller.create(app_guid) + expect(MultiJson.load(response, symbolize_keys: true)).to eq(package_response) + end + end + + context "as an admin" do + let(:headers) { admin_headers } + + it "allows upload even if app_bits_upload flag is disabled" do + FeatureFlag.make(name: 'app_bits_upload', enabled: false) + response_code, _ = packages_controller.create(app_guid) + expect(response_code).to eq 201 + end + end + + context "as a developer" do + let(:user) { make_developer_for_space(app_obj.space) } + + context "with an invalid package" do + let(:params) { {} } + + it 'returns an UnprocessableEntity error' do + expect { + packages_controller.create(app_guid) + }.to raise_error do |error| + expect(error.name).to eq 'UnprocessableEntity' + expect(error.response_code).to eq 422 + end + end + end + + context "with an invalid type field" do + let(:params) { { type: 'ninja' } } + + it 'returns an UnprocessableEntity error' do + expect { + packages_controller.create(app_guid) + }.to raise_error do |error| + expect(error.name).to eq 'UnprocessableEntity' + expect(error.response_code).to eq 422 + end + end + end + end + + context 'when the user cannot create a package' do + before do + allow(packages_handler).to receive(:create).and_raise(PackagesHandler::Unauthorized) + end + + it 'returns a 403 NotAuthorized error' do + expect { + packages_controller.create(app_guid) + }.to raise_error do |error| + expect(error.name).to eq 'NotAuthorized' + expect(error.response_code).to eq 403 + end + end + end + end + + describe '#show' do + context 'when the package does not exist' do + before do + allow(packages_handler).to receive(:show).and_return(nil) + end + + it 'returns a 404 Not Found' do + expect { + packages_controller.show('non-existant') + }.to raise_error do |error| + expect(error.name).to eq 'ResourceNotFound' + expect(error.response_code).to eq 404 + end + end + end + + context 'when the package exists' do + let(:package) { PackageModel.make } + let(:package_guid) { package.guid } + + context 'when a user can access a package' do + let(:expected_response) { "im a response" } + + before do + allow(packages_handler).to receive(:show).and_return(package) + allow(package_presenter).to receive(:present_json).and_return(expected_response) + end + + it 'returns a 200 OK and the package' do + response_code, response = packages_controller.show(package.guid) + expect(response_code).to eq 200 + expect(response).to eq(expected_response) + end + end + + context 'when the user cannot access the package' do + before do + allow(packages_handler).to receive(:show).and_raise(PackagesHandler::Unauthorized) + end + + it 'returns a 403 NotAuthorized error' do + expect { + packages_controller.show(package_guid) + }.to raise_error do |error| + expect(error.name).to eq 'NotAuthorized' + expect(error.response_code).to eq 403 + end + end + end + end + end + end +end diff --git a/spec/unit/controllers/v3/processes_controller_spec.rb b/spec/unit/controllers/v3/processes_controller_spec.rb index c1595e46bf4..a7a1d074d57 100644 --- a/spec/unit/controllers/v3/processes_controller_spec.rb +++ b/spec/unit/controllers/v3/processes_controller_spec.rb @@ -64,6 +64,7 @@ module VCAP::CloudController before do allow(processes_handler).to receive(:show).and_return(nil) end + it 'raises an ApiError with a 404 code' do expect { process_controller.show(guid) diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb new file mode 100644 index 00000000000..a338bb27611 --- /dev/null +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -0,0 +1,176 @@ +require 'spec_helper' +require 'handlers/packages_handler' + +module VCAP::CloudController + describe PackageCreateMessage do + context 'when a type parameter that is not allowed is provided' do + let(:opts) { { 'type' => 'not-allowed' } } + let(:guid) { 'my-guid' } + + it 'is not valid' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_falsey + expect(errors).to include('The type field needs to be one of \'bits, docker\'') + end + end + + context 'when nil type is provided' do + let(:opts) { { 'type' => nil } } + let(:guid) { 'my-guid' } + + it 'is not valid' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_falsey + expect(errors).to include('The type field is required') + end + end + + context 'when type is bits' do + let(:opts) { { 'type' => 'bits', 'bits_path' => nil, 'bits_name' => nil } } + let(:guid) { 'my-guid' } + + context 'and no zip file is uploaded' do + it 'is not valid' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_falsey + expect(errors).to include('Must upload an application zip file') + end + end + + context 'and a zip file is uploaded' do + let(:opts) { { 'type' => 'bits', 'bits_path' => '/tmp', 'bits_name' => 'app.zip' } } + + it 'is valid' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_truthy + expect(errors).to be_empty + end + end + + context 'when type is docker' do + let(:opts) { { 'type' => 'docker', 'bits_path' => nil, 'bits_name' => nil } } + it 'does not care about a zip file' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_truthy + expect(errors).to be_empty + end + end + end + end + + describe PackagesHandler do + let(:tmpdir) { Dir.mktmpdir } + let(:valid_zip) { + zip_name = File.join(tmpdir, 'file.zip') + TestZip.create(zip_name, 1, 1024) + zip_file = File.new(zip_name) + Rack::Test::UploadedFile.new(zip_file) + } + + let(:config) { TestConfig.config } + let(:packages_handler) { described_class.new(config) } + let(:access_context) { double(:access_context) } + let(:app) { AppModel.make } + + before do + allow(access_context).to receive(:cannot?).and_return(false) + end + + describe '#create' do + let(:create_opts) do + { + 'type' => 'bits', + 'bits_path' => tmpdir, + 'bits_name' => 'file.zip', + } + end + let(:create_message) { PackageCreateMessage.new(app.guid, create_opts) } + + context 'when a user can create a package' do + it 'creates the package' do + result = packages_handler.create(create_message, access_context) + + created_package = PackageModel.find(guid: result.guid) + expect(created_package.app_guid).to eq(result.app_guid) + expect(created_package.type).to eq(result.type) + end + + it 'adds a delayed job to upload the package bits' do + result = nil + expect { + result = packages_handler.create(create_message, access_context) + }.to change{ Delayed::Job.count }.by(1) + + expect(result.state).to eq('PENDING') + end + end + + context 'when the user cannot create an package' do + before do + allow(access_context).to receive(:cannot?).and_return(true) + end + + it 'raises Unauthorized error' do + expect { + packages_handler.create(create_message, access_context) + }.to raise_error(PackagesHandler::Unauthorized) + expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel)) + end + end + + context 'when the package is invalid' do + before do + allow_any_instance_of(PackageModel).to receive(:save).and_raise(Sequel::ValidationFailed.new('the message')) + end + + it 'raises an PackageInvalid error' do + expect { + packages_handler.create(create_message, access_context) + }.to raise_error(PackagesHandler::InvalidPackage, 'the message') + end + end + end + + describe 'show' do + let(:package) { PackageModel.make } + let(:package_guid) { package.guid } + + context 'when the package does not exist' do + it 'returns nil' do + expect(access_context).not_to receive(:cannot?) + expect(packages_handler.show('non-existant', access_context)).to eq(nil) + end + end + + context 'when the package does exist' do + context 'when the user can access a package' do + before do + allow(access_context).to receive(:cannot?).and_return(false) + end + + it 'returns the package' do + expect(packages_handler.show(package_guid, access_context)).to eq(package) + end + end + + context 'when the user cannot access a package' do + before do + allow(access_context).to receive(:cannot?).and_return(true) + end + + it 'raises Unauthorized error' do + expect { + packages_handler.show(package_guid, access_context) + }.to raise_error(PackagesHandler::Unauthorized) + expect(access_context).to have_received(:cannot?).with(:read, kind_of(PackageModel)) + end + end + end + end + end +end diff --git a/spec/unit/jobs/v3/package_bits_spec.rb b/spec/unit/jobs/v3/package_bits_spec.rb new file mode 100644 index 00000000000..dcc838513fb --- /dev/null +++ b/spec/unit/jobs/v3/package_bits_spec.rb @@ -0,0 +1,46 @@ +require "spec_helper" + +module VCAP::CloudController + module Jobs::Runtime + describe PackageBits do + let(:uploaded_path) { "tmp/uploaded.zip" } + let(:package_guid) { SecureRandom.uuid } + + subject(:job) do + PackageBits.new(package_guid, uploaded_path) + end + + it { is_expected.to be_a_valid_job } + + describe "#perform" do + let(:package_blobstore) { double(:package_blobstore) } + let(:tmpdir) { "/tmp/special_temp" } + let(:max_package_size) { 256 } + + before do + TestConfig.override({:directories => {:tmpdir => tmpdir}, :packages => TestConfig.config[:packages].merge(:max_package_size => max_package_size)}) + + allow(AppBitsPackage).to receive(:new) { double(:packer, create_package_in_blobstore: "done") } + end + + it "creates blob stores" do + expect(CloudController::DependencyLocator.instance).to receive(:package_blobstore) + job.perform + end + + it "creates an AppBitsPackage and performs" do + expect(CloudController::DependencyLocator.instance).to receive(:package_blobstore).and_return(package_blobstore) + + packer = double + expect(AppBitsPackage).to receive(:new).with(package_blobstore, nil, max_package_size, tmpdir).and_return(packer) + expect(packer).to receive(:create_package_in_blobstore).with(package_guid, uploaded_path) + job.perform + end + + it "knows its job name" do + expect(job.job_name_in_configuration).to equal(:package_bits) + end + end + end + end +end diff --git a/spec/unit/models/runtime/app_bits_package_spec.rb b/spec/unit/models/runtime/app_bits_package_spec.rb index eaa4838775b..84812ef75fd 100644 --- a/spec/unit/models/runtime/app_bits_package_spec.rb +++ b/spec/unit/models/runtime/app_bits_package_spec.rb @@ -12,6 +12,7 @@ let(:compressed_path) { File.expand_path('../../../fixtures/good.zip', File.dirname(__FILE__)) } let(:app) { VCAP::CloudController::AppFactory.make } + let(:package) { VCAP::CloudController::PackageModel.make } let(:blobstore_dir) { Dir.mktmpdir } let(:local_tmp_dir) { Dir.mktmpdir } @@ -37,6 +38,61 @@ FileUtils.remove_entry_secure blobstore_dir end + describe "#create_package_in_blobstore" do + let(:package_guid) { package.guid } + subject(:create) { packer.create_package_in_blobstore(package_guid, compressed_path) } + + it "uploads the package zip to the package blob store" do + create + expect(package_blobstore.exists?(package_guid)).to be true + end + + it "sets the package sha to the package" do + expect { create }.to change { package.refresh.package_hash } + end + + it "sets the state of the package" do + expect { create }.to change { package.refresh.state }.to ('READY') + end + + it "removes the compressed path afterwards" do + expect(FileUtils).to receive(:rm_f).with(compressed_path) + create + end + + context "when there is no package uploaded" do + let(:compressed_path) { nil } + + it "doesn't try to remove the file" do + expect(FileUtils).not_to receive(:rm_f) + create + end + end + + context 'when the package no longer exists' do + let(:package_guid) { 'abcd' } + + it 'raises an error and removes the compressed path' do + expect(FileUtils).to receive(:rm_f).with(compressed_path) + expect { create }.to raise_error + end + end + + context 'when copying to the blobstore fails' do + it 'logs the exception on the package and reraises the exception' do + allow(package_blobstore).to receive(:cp_to_blobstore).and_raise('BOOM') + expect { create }.to raise_error + expect(package.reload.state).to eq('FAILED') + expect(package.error).to eq('BOOM') + end + + it "removes the compressed path afterwards" do + expect(FileUtils).to receive(:rm_f).with(compressed_path) + create + end + end + end + describe '#create' do subject(:create) { packer.create(app, compressed_path, fingerprints_in_app_cache) } diff --git a/spec/unit/models/runtime/v3/persistence/package_model_spec.rb b/spec/unit/models/runtime/v3/persistence/package_model_spec.rb new file mode 100644 index 00000000000..620c0c902b3 --- /dev/null +++ b/spec/unit/models/runtime/v3/persistence/package_model_spec.rb @@ -0,0 +1,8 @@ +# encoding: utf-8 +require 'spec_helper' + +module VCAP::CloudController + describe PackageModel do + it { is_expected.to validates_includes PackageModel::PACKAGE_STATES, :state, allow_missing: true } + end +end diff --git a/spec/unit/presenters/v3/package_presenter_spec.rb b/spec/unit/presenters/v3/package_presenter_spec.rb new file mode 100644 index 00000000000..f8a70d4c38a --- /dev/null +++ b/spec/unit/presenters/v3/package_presenter_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' +require 'presenters/v3/package_presenter' + +module VCAP::CloudController + describe PackagePresenter do + describe '#present_json' do + it 'presents the package as json' do + package = PackageModel.make(type: "package_type") + + json_result = PackagePresenter.new.present_json(package) + result = MultiJson.load(json_result) + + expect(result['guid']).to eq(package.guid) + expect(result['type']).to eq(package.type) + expect(result['state']).to eq(package.state) + expect(result['error']).to eq(package.error) + expect(result['hash']).to eq(package.package_hash) + expect(result['created_at']).to eq(package.created_at.as_json) + expect(result['_links']).to include('self') + expect(result['_links']).to include('app') + end + end + end +end From b486985eadce677d517b74f7bd94d3fb594beb49 Mon Sep 17 00:00:00 2001 From: James Myers and Sujoy Basu Date: Wed, 7 Jan 2015 17:29:38 -0800 Subject: [PATCH 33/76] Implement DELETE /v3/packages/:guid * Remove corresponding blob from blobstore * Lock app when checking permissions for packages [#79388094] --- app/access/v3/package_access.rb | 13 +---- app/controllers/v3/packages_controller.rb | 13 +++-- app/handlers/packages_handler.rb | 32 +++++++++- app/jobs/runtime/blobstore_delete.rb | 3 +- spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 24 ++++++++ spec/unit/access/v3/package_access_spec.rb | 27 ++++----- .../v3/packages_controller_spec.rb | 49 ++++++++++++++++ spec/unit/handlers/packages_handler_spec.rb | 58 ++++++++++++++++++- 9 files changed, 184 insertions(+), 37 deletions(-) diff --git a/app/access/v3/package_access.rb b/app/access/v3/package_access.rb index 4e8a056e913..ebe6129a17d 100644 --- a/app/access/v3/package_access.rb +++ b/app/access/v3/package_access.rb @@ -11,14 +11,11 @@ def read?(package) has_read_scope && user_visible end - def create?(desired_package) + def create?(desired_package, app, space) return true if context.roles.admin? has_write_scope = SecurityContext.scopes.include?('cloud_controller.write') - app = AppModel.find(guid: desired_package.app_guid) - space = Space.find(guid: app.space_guid) - is_space_developer = space && space.developers.include?(context.user) org_active = space && space.organization.active? @@ -26,12 +23,8 @@ def create?(desired_package) has_write_scope && is_space_developer && org_active end - def delete?(app) - create?(app) - end - - def update?(app) - create?(app) + def delete?(package, app, space) + create?(package, app, space) end end end diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index 3af00d2258b..71eac518a25 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -37,16 +37,21 @@ def show(package_guid) unauthorized! end + delete '/v3/packages/:guid', :delete + def delete(package_guid) + package = @packages_handler.delete(package_guid, @access_context) + package_not_found! unless package + [HTTP::NO_CONTENT] + rescue PackagesHandler::Unauthorized + unauthorized! + end + private def package_not_found! raise VCAP::Errors::ApiError.new_from_details('ResourceNotFound', 'Package not found') end - def bad_request!(message) - raise VCAP::Errors::ApiError.new_from_details('MessageParseError', message) - end - def unauthorized! raise VCAP::Errors::ApiError.new_from_details('NotAuthorized') end diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index 93d969b8e73..870857d739a 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -50,9 +50,16 @@ def create(message, access_context) package.app_guid = message.app_guid package.type = message.type - raise Unauthorized if access_context.cannot?(:create, package) + app = AppModel.find(guid: package.app_guid) + space = Space.find(guid: app.space_guid) - package.save + app.db.transaction do + app.lock! + + raise Unauthorized if access_context.cannot?(:create, package, app, space) + + package.save + end bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue() @@ -62,6 +69,27 @@ def create(message, access_context) raise InvalidPackage.new(e.message) end + def delete(guid, access_context) + package = PackageModel.find(guid: guid) + return nil if package.nil? + + app = AppModel.find(guid: package.app_guid) + space = Space.find(guid: app.space_guid) + + package.db.transaction do + app.lock! + + raise Unauthorized if access_context.cannot?(:delete, package, app, space) + + package.destroy + end + + blobstore_delete = Jobs::Runtime::BlobstoreDelete.new(package.guid, :package_blobstore, nil) + Jobs::Enqueuer.new(blobstore_delete, queue: 'cc-generic').enqueue() + + package + end + def show(guid, access_context) package = PackageModel.find(guid: guid) return nil if package.nil? diff --git a/app/jobs/runtime/blobstore_delete.rb b/app/jobs/runtime/blobstore_delete.rb index 0eae93da2eb..0a205acc4b5 100644 --- a/app/jobs/runtime/blobstore_delete.rb +++ b/app/jobs/runtime/blobstore_delete.rb @@ -4,11 +4,12 @@ module Runtime class BlobstoreDelete < Struct.new(:key, :blobstore_name, :attributes) def perform logger = Steno.logger('cc.background') - logger.info("Deleting '#{key}' from blobstore '#{blobstore_name}'") + logger.info("Attempting delete of '#{key}' from blobstore '#{blobstore_name}'") blobstore = CloudController::DependencyLocator.instance.public_send(blobstore_name) blob = blobstore.blob(key) if blob && same_blob(blob) + logger.info("Deleting '#{key}' from blobstore '#{blobstore_name}'") blobstore.delete_blob(blob) end end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index cf6a091e086..1a2da764d93 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '89550450a3565fda71642c8fa545f88892c7e4bf' + API_FOLDER_CHECKSUM = 'a9864824db93a74c3a20f1e88941e68baf942caa' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index c2516fd621b..4f02c48fe90 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -137,5 +137,29 @@ def do_request_with_error_handling expect(parsed_response).to match(expected_response) end end + + delete '/v3/packages/:guid' do + let(:space) { VCAP::CloudController::Space.make } + let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } + + let!(:package_model) do + VCAP::CloudController::PackageModel.make(app_guid: app_model.guid) + end + + let(:guid) { package_model.guid } + let(:app_guid) { app_model.guid } + + before do + space.organization.add_user user + space.add_developer user + end + + example 'Delete a Package' do + expect { + do_request_with_error_handling + }.to change { VCAP::CloudController::PackageModel.count }.by(-1) + expect(response_status).to eq(204) + end + end end end diff --git a/spec/unit/access/v3/package_access_spec.rb b/spec/unit/access/v3/package_access_spec.rb index e0046dab7eb..a590f8be904 100644 --- a/spec/unit/access/v3/package_access_spec.rb +++ b/spec/unit/access/v3/package_access_spec.rb @@ -62,7 +62,7 @@ module VCAP::CloudController end end - describe '#create?, #update?, #delete?' do + describe '#create?, #delete?' do let(:space) { Space.make } let(:app) { AppModel.make(space_guid: space.guid) } let(:package) { PackageModel.new(app_guid: app.guid) } @@ -72,9 +72,8 @@ module VCAP::CloudController it 'allows the user to perform the action' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package)).to be_truthy - expect(access_control.update?(package)).to be_truthy - expect(access_control.delete?(package)).to be_truthy + expect(access_control.create?(package, app, space)).to be_truthy + expect(access_control.delete?(package, app, space)).to be_truthy end end @@ -89,9 +88,8 @@ module VCAP::CloudController it 'allows the user to create' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package)).to be_truthy - expect(access_control.update?(package)).to be_truthy - expect(access_control.delete?(package)).to be_truthy + expect(access_control.create?(package, app, space)).to be_truthy + expect(access_control.delete?(package, app, space)).to be_truthy end end @@ -105,9 +103,8 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package)).to be_falsey - expect(access_control.update?(package)).to be_falsey - expect(access_control.delete?(package)).to be_falsey + expect(access_control.create?(package, app, space)).to be_falsey + expect(access_control.delete?(package, app, space)).to be_falsey end end @@ -116,9 +113,8 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package)).to be_falsey - expect(access_control.update?(package)).to be_falsey - expect(access_control.delete?(package)).to be_falsey + expect(access_control.create?(package, app, space)).to be_falsey + expect(access_control.delete?(package, app, space)).to be_falsey end end @@ -133,9 +129,8 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package)).to be_falsey - expect(access_control.update?(package)).to be_falsey - expect(access_control.delete?(package)).to be_falsey + expect(access_control.create?(package, app, space)).to be_falsey + expect(access_control.delete?(package, app, space)).to be_falsey end end end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index b31d430fbcd..3c241951b40 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -182,5 +182,54 @@ module VCAP::CloudController end end end + + describe '#delete' do + context 'when the package does not exist' do + before do + allow(packages_handler).to receive(:delete).and_return(nil) + end + + it 'returns a 404 Not Found' do + expect { + packages_controller.delete('non-existant') + }.to raise_error do |error| + expect(error.name).to eq 'ResourceNotFound' + expect(error.response_code).to eq 404 + end + end + end + + context 'when the package exists' do + let(:package) { PackageModel.make } + let(:package_guid) { package.guid } + + context 'when a user can access a package' do + before do + allow(packages_handler).to receive(:delete).and_return(package) + end + + it 'returns a 204 NO CONTENT' do + response_code, response = packages_controller.delete(package.guid) + expect(response_code).to eq 204 + expect(response).to be_nil + end + end + + context 'when the user cannot access the package' do + before do + allow(packages_handler).to receive(:delete).and_raise(PackagesHandler::Unauthorized) + end + + it 'returns a 403 NotAuthorized error' do + expect { + packages_controller.delete(package_guid) + }.to raise_error do |error| + expect(error.name).to eq 'NotAuthorized' + expect(error.response_code).to eq 403 + end + end + end + end + end end end diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index a338bb27611..939ae533264 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -75,7 +75,8 @@ module VCAP::CloudController let(:config) { TestConfig.config } let(:packages_handler) { described_class.new(config) } let(:access_context) { double(:access_context) } - let(:app) { AppModel.make } + let(:app) { AppModel.make(space_guid: space.guid) } + let(:space) { Space.make } before do allow(access_context).to receive(:cannot?).and_return(false) @@ -103,7 +104,7 @@ module VCAP::CloudController it 'adds a delayed job to upload the package bits' do result = nil expect { - result = packages_handler.create(create_message, access_context) + result = packages_handler.create(create_message, access_context) }.to change{ Delayed::Job.count }.by(1) expect(result.state).to eq('PENDING') @@ -119,7 +120,7 @@ module VCAP::CloudController expect { packages_handler.create(create_message, access_context) }.to raise_error(PackagesHandler::Unauthorized) - expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel)) + expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel), app, space) end end @@ -172,5 +173,56 @@ module VCAP::CloudController end end end + + describe 'delete' do + let!(:package) { PackageModel.make(app_guid: app.guid) } + let(:package_guid) { package.guid } + + context 'when the user can access a package' do + before do + allow(access_context).to receive(:cannot?).and_return(false) + end + + context 'and the package does not exist' do + it 'returns nil' do + expect(packages_handler.delete('non-existant', access_context)).to eq(nil) + end + end + + context 'and the package exists' do + it 'deletes the package and returns the deleted package' do + expect { + deleted_package = packages_handler.delete(package_guid, access_context) + expect(deleted_package.guid).to eq(package_guid) + }.to change{ PackageModel.count }.by(-1) + expect(PackageModel.find(guid: package_guid)).to be_nil + end + + it 'enqueues a job to delete the corresponding blob from the blobstore' do + job_opts = { queue: 'cc-generic' } + expect(Jobs::Enqueuer).to receive(:new).with(kind_of(BlobstoreDelete), job_opts). + and_call_original + + expect { + packages_handler.delete(package_guid, access_context) + }.to change{ Delayed::Job.count }.by(1) + end + end + end + + context 'when the user cannot access a package' do + before do + allow(access_context).to receive(:cannot?).and_return(true) + end + + it 'raises Unauthorized error' do + expect { + deleted_package = packages_handler.delete(package_guid, access_context) + expect(deleted_package).to be_nil + }.to raise_error(PackagesHandler::Unauthorized) + expect(access_context).to have_received(:cannot?).with(:delete, kind_of(PackageModel), app, space) + end + end + end end end From 4e5b683f09ae52e06308680630df1bf7dc713ec2 Mon Sep 17 00:00:00 2001 From: Dan Lavine and James Myers Date: Thu, 8 Jan 2015 10:08:33 -0800 Subject: [PATCH 34/76] Return a 404 on POST /v3/apps/:guid/packages if the app does not exist [#79388094] --- app/controllers/v3/packages_controller.rb | 6 ++ app/handlers/packages_handler.rb | 3 + .../v3/packages_controller_spec.rb | 91 +++++++++++-------- spec/unit/handlers/packages_handler_spec.rb | 71 +++++++++------ 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index 71eac518a25..a8bffd4f475 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -24,6 +24,8 @@ def create(app_guid) [HTTP::CREATED, package_json] rescue PackagesHandler::Unauthorized unauthorized! + rescue PackagesHandler::AppNotFound + app_not_found! end get '/v3/packages/:guid', :show @@ -52,6 +54,10 @@ def package_not_found! raise VCAP::Errors::ApiError.new_from_details('ResourceNotFound', 'Package not found') end + def app_not_found! + raise VCAP::Errors::ApiError.new_from_details('ResourceNotFound', 'App not found') + end + def unauthorized! raise VCAP::Errors::ApiError.new_from_details('NotAuthorized') end diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index 870857d739a..286c1db38d4 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -38,6 +38,7 @@ def validate_file class PackagesHandler class Unauthorized < StandardError; end class InvalidPackage < StandardError; end + class AppNotFound < StandardError; end PACKAGE_STATES = %w[PENDING READY FAILED].map(&:freeze).freeze @@ -51,6 +52,8 @@ def create(message, access_context) package.type = message.type app = AppModel.find(guid: package.app_guid) + raise AppNotFound if app.nil? + space = Space.find(guid: app.space_guid) app.db.transaction do diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index 3c241951b40..14e07370336 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -63,69 +63,86 @@ module VCAP::CloudController FileUtils.rm_rf(tmpdir) end - context 'when a user can create a package' do - it 'returns a 201 Created response' do - response_code, _ = packages_controller.create(app_guid) - expect(response_code).to eq 201 - end + context 'when the app exists' do + context 'when a user can create a package' do + it 'returns a 201 Created response' do + response_code, _ = packages_controller.create(app_guid) + expect(response_code).to eq 201 + end - it 'returns the package' do - _, response = packages_controller.create(app_guid) - expect(MultiJson.load(response, symbolize_keys: true)).to eq(package_response) + it 'returns the package' do + _, response = packages_controller.create(app_guid) + expect(MultiJson.load(response, symbolize_keys: true)).to eq(package_response) + end end - end - context "as an admin" do - let(:headers) { admin_headers } + context "as an admin" do + let(:headers) { admin_headers } - it "allows upload even if app_bits_upload flag is disabled" do - FeatureFlag.make(name: 'app_bits_upload', enabled: false) - response_code, _ = packages_controller.create(app_guid) - expect(response_code).to eq 201 + it "allows upload even if app_bits_upload flag is disabled" do + FeatureFlag.make(name: 'app_bits_upload', enabled: false) + response_code, _ = packages_controller.create(app_guid) + expect(response_code).to eq 201 + end end - end - context "as a developer" do - let(:user) { make_developer_for_space(app_obj.space) } + context "as a developer" do + let(:user) { make_developer_for_space(app_obj.space) } - context "with an invalid package" do - let(:params) { {} } + context "with an invalid package" do + let(:params) { {} } - it 'returns an UnprocessableEntity error' do - expect { - packages_controller.create(app_guid) - }.to raise_error do |error| - expect(error.name).to eq 'UnprocessableEntity' - expect(error.response_code).to eq 422 + it 'returns an UnprocessableEntity error' do + expect { + packages_controller.create(app_guid) + }.to raise_error do |error| + expect(error.name).to eq 'UnprocessableEntity' + expect(error.response_code).to eq 422 + end + end + end + + context "with an invalid type field" do + let(:params) { { type: 'ninja' } } + + it 'returns an UnprocessableEntity error' do + expect { + packages_controller.create(app_guid) + }.to raise_error do |error| + expect(error.name).to eq 'UnprocessableEntity' + expect(error.response_code).to eq 422 + end end end end - context "with an invalid type field" do - let(:params) { { type: 'ninja' } } + context 'when the user cannot create a package' do + before do + allow(packages_handler).to receive(:create).and_raise(PackagesHandler::Unauthorized) + end - it 'returns an UnprocessableEntity error' do + it 'returns a 403 NotAuthorized error' do expect { packages_controller.create(app_guid) }.to raise_error do |error| - expect(error.name).to eq 'UnprocessableEntity' - expect(error.response_code).to eq 422 + expect(error.name).to eq 'NotAuthorized' + expect(error.response_code).to eq 403 end end end end - context 'when the user cannot create a package' do + context 'when the app does not exist' do before do - allow(packages_handler).to receive(:create).and_raise(PackagesHandler::Unauthorized) + allow(packages_handler).to receive(:create).and_raise(PackagesHandler::AppNotFound) end - it 'returns a 403 NotAuthorized error' do + it 'returns a 404 ResourceNotFound error' do expect { - packages_controller.create(app_guid) + packages_controller.create('bogus') }.to raise_error do |error| - expect(error.name).to eq 'NotAuthorized' - expect(error.response_code).to eq 403 + expect(error.name).to eq 'ResourceNotFound' + expect(error.response_code).to eq 404 end end end diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index 939ae533264..9b5cbc6c69b 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -90,49 +90,62 @@ module VCAP::CloudController 'bits_name' => 'file.zip', } end - let(:create_message) { PackageCreateMessage.new(app.guid, create_opts) } - context 'when a user can create a package' do - it 'creates the package' do - result = packages_handler.create(create_message, access_context) + context 'when the app exists' do + let(:create_message) { PackageCreateMessage.new(app.guid, create_opts) } - created_package = PackageModel.find(guid: result.guid) - expect(created_package.app_guid).to eq(result.app_guid) - expect(created_package.type).to eq(result.type) - end - - it 'adds a delayed job to upload the package bits' do - result = nil - expect { + context 'when a user can create a package' do + it 'creates the package' do result = packages_handler.create(create_message, access_context) - }.to change{ Delayed::Job.count }.by(1) - expect(result.state).to eq('PENDING') + created_package = PackageModel.find(guid: result.guid) + expect(created_package.app_guid).to eq(result.app_guid) + expect(created_package.type).to eq(result.type) + end + + it 'adds a delayed job to upload the package bits' do + result = nil + expect { + result = packages_handler.create(create_message, access_context) + }.to change{ Delayed::Job.count }.by(1) + + expect(result.state).to eq('PENDING') + end end - end - context 'when the user cannot create an package' do - before do - allow(access_context).to receive(:cannot?).and_return(true) + context 'when the user cannot create an package' do + before do + allow(access_context).to receive(:cannot?).and_return(true) + end + + it 'raises Unauthorized error' do + expect { + packages_handler.create(create_message, access_context) + }.to raise_error(PackagesHandler::Unauthorized) + expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel), app, space) + end end - it 'raises Unauthorized error' do - expect { - packages_handler.create(create_message, access_context) - }.to raise_error(PackagesHandler::Unauthorized) - expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel), app, space) + context 'when the package is invalid' do + before do + allow_any_instance_of(PackageModel).to receive(:save).and_raise(Sequel::ValidationFailed.new('the message')) + end + + it 'raises an PackageInvalid error' do + expect { + packages_handler.create(create_message, access_context) + }.to raise_error(PackagesHandler::InvalidPackage, 'the message') + end end end - context 'when the package is invalid' do - before do - allow_any_instance_of(PackageModel).to receive(:save).and_raise(Sequel::ValidationFailed.new('the message')) - end + context 'when the app does not exist' do + let(:create_message) { PackageCreateMessage.new('non-existant', create_opts) } - it 'raises an PackageInvalid error' do + it 'raises AppNotFound' do expect { packages_handler.create(create_message, access_context) - }.to raise_error(PackagesHandler::InvalidPackage, 'the message') + }.to raise_error(PackagesHandler::AppNotFound) end end end From 23a3f9aaed895bb2586f1c0f223e158f54088b8a Mon Sep 17 00:00:00 2001 From: Dan Lavine and James Myers Date: Thu, 8 Jan 2015 10:35:22 -0800 Subject: [PATCH 35/76] Implement POST /v3/apps/:guid/packages with type docker [#79388094] --- app/handlers/packages_handler.rb | 7 +- spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 152 +++++++++++------- spec/unit/handlers/packages_handler_spec.rb | 27 +++- 4 files changed, 125 insertions(+), 63 deletions(-) diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index 286c1db38d4..75a88c1666f 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -50,6 +50,7 @@ def create(message, access_context) package = PackageModel.new package.app_guid = message.app_guid package.type = message.type + package.state = 'READY' if message.type == 'docker' app = AppModel.find(guid: package.app_guid) raise AppNotFound if app.nil? @@ -64,8 +65,10 @@ def create(message, access_context) package.save end - bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) - Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue() + if package.type == 'bits' + bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) + Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue() + end package rescue Sequel::ValidationFailed => e diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 1a2da764d93..9fd3c769c85 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = 'a9864824db93a74c3a20f1e88941e68baf942caa' + API_FOLDER_CHECKSUM = '9c61ee8a851c4a269af6c51f284bd4de815b99b0' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index 4f02c48fe90..2dc7418bd7f 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -41,16 +41,13 @@ def do_request_with_error_handling end example 'Get a Package' do - # MultiJson/Ruby Json library formats strings differently than to_s - created_at_string = MultiJson.load(package_model.created_at.to_json) - expected_response = { 'type' => package_model.type, 'guid' => guid, 'hash' => nil, 'state' => "PENDING", 'error' => nil, - 'created_at' => created_at_string, + 'created_at' => package_model.created_at.as_json, '_links' => { 'self' => { 'href' => "/v3/packages/#{guid}" }, 'app' => { 'href' => "/v3/apps/#{app_guid}" }, @@ -70,14 +67,6 @@ def do_request_with_error_handling let(:space_guid) { space.guid } let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } let(:guid) { app_model.guid } - let(:type) { 'bits' } - let(:packages_params) do - { - type: 'bits', - bits_name: 'application.zip', - bits_path: "#{tmpdir}/application.zip", - } - end before do space.organization.add_user(user) @@ -88,53 +77,108 @@ def do_request_with_error_handling parameter :type, 'Package type', required: true, valid_values: ['bits', 'docker'] parameter :bits, 'A binary zip file containing the package bits', required: false - let(:request_body_example) do - <<-eos.gsub(/^ */, '') - --AaB03x - Content-Disposition: form-data; name="type" - - #{type} - --AaB03x - Content-Disposition: form-data; name="bits"; filename="application.zip" - Content-Type: application/zip - Content-Length: 123 - Content-Transfer-Encoding: binary - - <<binary artifact bytes>> - --AaB03x - eos - end - - example 'Create a Package' do - expect { - do_request packages_params - }.to change{ VCAP::CloudController::PackageModel.count }.by(1) - - package = VCAP::CloudController::PackageModel.last - expected_guid = VCAP::CloudController::AppModel.last.guid - expected_created_at = package.created_at.as_json + context 'when the type is bits' do + let(:type) { 'bits' } + let(:packages_params) do + { + type: type, + bits_name: 'application.zip', + bits_path: "#{tmpdir}/application.zip", + } + end + + let(:request_body_example) do + <<-eos.gsub(/^ */, '') + Content-type: multipart/form-data, boundary=AaB03x + --AaB03x + Content-Disposition: form-data; name="type" + + #{type} + --AaB03x + Content-Disposition: form-data; name="bits"; filename="application.zip" + Content-Type: application/zip + Content-Length: 123 + Content-Transfer-Encoding: binary + + <<binary artifact bytes>> + --AaB03x + eos + end + + example 'Create a Package with application bits' do + expect { + do_request packages_params + }.to change{ VCAP::CloudController::PackageModel.count }.by(1) + + package = VCAP::CloudController::PackageModel.last + + job = Delayed::Job.last + expect(job.handler).to include(package.guid) + expect(job.guid).not_to be_nil + + expected_response = { + 'guid' => package.guid, + 'type' => type, + 'hash' => nil, + 'state' => 'PENDING', + 'error' => nil, + 'created_at' => package.created_at.as_json, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package.guid}" }, + 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, + } + } - job = Delayed::Job.last - expect(job.handler).to include(package.guid) - expect(job.guid).not_to be_nil + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(201) + expect(parsed_response).to match(expected_response) + end + end - expected_response = { - 'guid' => package.guid, - 'type' => type, - 'hash' => nil, - 'state' => 'PENDING', - 'error' => nil, - 'created_at' => expected_created_at, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{package.guid}" }, - 'app' => { 'href' => "/v3/apps/#{expected_guid}" }, + context 'when the type is docker' do + let(:type) { 'docker' } + let(:packages_params) do + { + type: type, + } + end + + let(:request_body_example) do + <<-eos.gsub(/^ */, '') + Content-type: multipart/form-data, boundary=AaB03x + --AaB03x + Content-Disposition: form-data; name="type" + + #{type} + --AaB03x + eos + end + + example 'Create a Package with docker image' do + expect { + do_request packages_params + }.to change{ VCAP::CloudController::PackageModel.count }.by(1) + + package = VCAP::CloudController::PackageModel.last + + expected_response = { + 'guid' => package.guid, + 'type' => type, + 'hash' => nil, + 'state' => 'READY', + 'error' => nil, + 'created_at' => package.created_at.as_json, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package.guid}" }, + 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, + } } - } - parsed_response = MultiJson.load(response_body) - expect(response_status).to eq(201) - expect(parsed_response).to match(expected_response) + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(201) + expect(parsed_response).to match(expected_response) + end end end diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index 9b5cbc6c69b..fe1eb2dc190 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -103,13 +103,28 @@ module VCAP::CloudController expect(created_package.type).to eq(result.type) end - it 'adds a delayed job to upload the package bits' do - result = nil - expect { - result = packages_handler.create(create_message, access_context) - }.to change{ Delayed::Job.count }.by(1) + context 'when the type is bits' do + it 'adds a delayed job to upload the package bits' do + result = nil + expect { + result = packages_handler.create(create_message, access_context) + }.to change{ Delayed::Job.count }.by(1) + + expect(result.state).to eq('PENDING') + end + end + + context 'when the type is docker' do + let(:create_opts) { { 'type' => 'docker' } } + + it 'adds a delayed job to upload the package bits' do + result = nil + expect { + result = packages_handler.create(create_message, access_context) + }.to_not change{ Delayed::Job.count } - expect(result.state).to eq('PENDING') + expect(result.state).to eq('READY') + end end end From b473a1d1cac3530e043122769e7c22a0ab5b09fb Mon Sep 17 00:00:00 2001 From: Dan Lavine and James Myers Date: Thu, 8 Jan 2015 12:07:28 -0800 Subject: [PATCH 36/76] Respect max_package_size for /v3/apps/:guid/package [#79388094] --- app/models/runtime/app_bits_package.rb | 20 +++++++++++++------ .../models/runtime/app_bits_package_spec.rb | 12 +++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/app/models/runtime/app_bits_package.rb b/app/models/runtime/app_bits_package.rb index 215908fc36b..c491967ff76 100644 --- a/app/models/runtime/app_bits_package.rb +++ b/app/models/runtime/app_bits_package.rb @@ -3,6 +3,7 @@ class AppBitsPackage class PackageNotFound < StandardError; end + class ZipSizeExceeded < StandardError; end attr_reader :package_blobstore, :global_app_bits_cache, :max_package_size, :tmp_dir @@ -32,19 +33,21 @@ def create(app, uploaded_tmp_compressed_path, fingerprints_in_app_cache) FileUtils.rm_f(uploaded_tmp_compressed_path) if uploaded_tmp_compressed_path end - def create_package_in_blobstore(package_guid, uploaded_tmp_compressed_path) - return unless uploaded_tmp_compressed_path + def create_package_in_blobstore(package_guid, package_path) + return unless package_path package = VCAP::CloudController::PackageModel.find(guid: package_guid) raise PackageNotFound if package.nil? begin - zip_file = File.new(uploaded_tmp_compressed_path) - package_blobstore.cp_to_blobstore(uploaded_tmp_compressed_path, package_guid) + package_file = File.new(package_path) + raise ZipSizeExceeded if @max_package_size && package_size(package_path) > @max_package_size + + package_blobstore.cp_to_blobstore(package_path, package_guid) package.db.transaction do package.lock! - package.package_hash = zip_file.hexdigest + package.package_hash = package_file.hexdigest package.state = 'READY' package.save end @@ -57,11 +60,16 @@ def create_package_in_blobstore(package_guid, uploaded_tmp_compressed_path) raise e end ensure - FileUtils.rm_f(uploaded_tmp_compressed_path) if uploaded_tmp_compressed_path + FileUtils.rm_f(package_path) if package_path end private + def package_size(package_path) + zip_info = `unzip -l #{package_path}` + zip_info.split("\n").last.match(/^\s*(\d+)/)[1].to_i + end + def validate_size!(fingerprints_in_app_cache, local_app_bits) return unless max_package_size diff --git a/spec/unit/models/runtime/app_bits_package_spec.rb b/spec/unit/models/runtime/app_bits_package_spec.rb index 84812ef75fd..300ba8e46d0 100644 --- a/spec/unit/models/runtime/app_bits_package_spec.rb +++ b/spec/unit/models/runtime/app_bits_package_spec.rb @@ -39,6 +39,7 @@ end describe "#create_package_in_blobstore" do + let(:max_package_size) { nil } let(:package_guid) { package.guid } subject(:create) { packer.create_package_in_blobstore(package_guid, compressed_path) } @@ -91,6 +92,17 @@ create end end + + context "when the app bits are too large" do + let(:max_package_size) { 10 } + + it "raises an exception and deletes the bits" do + expect(FileUtils).to receive(:rm_f).with(compressed_path) + expect { + create + }.to raise_error(AppBitsPackage::ZipSizeExceeded) + end + end end describe '#create' do From ea83e10558a8f5a2833f65ca5114b3f61544c46d Mon Sep 17 00:00:00 2001 From: Dan Lavine and James Myers Date: Thu, 8 Jan 2015 12:30:09 -0800 Subject: [PATCH 37/76] Fix rubocop offenses --- app/controllers/v3/packages_controller.rb | 2 +- app/handlers/packages_handler.rb | 10 +++++----- app/jobs/v3/package_bits.rb | 8 +++----- app/models.rb | 6 +++--- app/models/v3/persistence/package_model.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 7 +++---- spec/support/matchers/sequel_validations.rb | 2 +- .../v3/packages_controller_spec.rb | 20 +++++++++---------- spec/unit/handlers/packages_handler_spec.rb | 8 ++++---- spec/unit/jobs/v3/package_bits_spec.rb | 18 ++++++++--------- .../models/runtime/app_bits_package_spec.rb | 20 +++++++++---------- .../presenters/v3/package_presenter_spec.rb | 2 +- 12 files changed, 51 insertions(+), 54 deletions(-) diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index a8bffd4f475..2981fc05d93 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -4,7 +4,7 @@ module VCAP::CloudController class PackagesController < RestController::BaseController def self.dependencies - [ :packages_handler, :package_presenter ] + [:packages_handler, :package_presenter] end def inject_dependencies(dependencies) diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index 75a88c1666f..a05c310f0cf 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -14,7 +14,7 @@ def validate errors << validate_type_field errors << validate_file if @type == 'bits' errs = errors.compact - return errs.length == 0, errs + [errs.length == 0, errs] end private @@ -24,7 +24,7 @@ def validate_type_field valid_type_fields = %w(bits docker) if !valid_type_fields.include?(@type) - return "The type field needs to be one of '#{valid_type_fields.join(", ")}'" + return "The type field needs to be one of '#{valid_type_fields.join(', ')}'" end nil end @@ -40,7 +40,7 @@ class Unauthorized < StandardError; end class InvalidPackage < StandardError; end class AppNotFound < StandardError; end - PACKAGE_STATES = %w[PENDING READY FAILED].map(&:freeze).freeze + PACKAGE_STATES = %w(PENDING READY FAILED).map(&:freeze).freeze def initialize(config) @config = config @@ -67,7 +67,7 @@ def create(message, access_context) if package.type == 'bits' bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) - Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue() + Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue end package @@ -91,7 +91,7 @@ def delete(guid, access_context) end blobstore_delete = Jobs::Runtime::BlobstoreDelete.new(package.guid, :package_blobstore, nil) - Jobs::Enqueuer.new(blobstore_delete, queue: 'cc-generic').enqueue() + Jobs::Enqueuer.new(blobstore_delete, queue: 'cc-generic').enqueue package end diff --git a/app/jobs/v3/package_bits.rb b/app/jobs/v3/package_bits.rb index 9716e451d7e..4cbf494ec1b 100644 --- a/app/jobs/v3/package_bits.rb +++ b/app/jobs/v3/package_bits.rb @@ -1,18 +1,17 @@ -require "cloud_controller/blobstore/cdn" -require "cloud_controller/dependency_locator" +require 'cloud_controller/blobstore/cdn' +require 'cloud_controller/dependency_locator' module VCAP::CloudController module Jobs module Runtime class PackageBits - def initialize(package_guid, uploaded_compressed_path) @package_guid = package_guid @uploaded_compressed_path = uploaded_compressed_path end def perform - logger = Steno.logger("cc.background") + logger = Steno.logger('cc.background') logger.info("Packing the app bits for package '#{@package_guid}'") package_blobstore = CloudController::DependencyLocator.instance.package_blobstore @@ -33,4 +32,3 @@ def max_attempts end end end - diff --git a/app/models.rb b/app/models.rb index 4108819c7e4..81e1b85b855 100644 --- a/app/models.rb +++ b/app/models.rb @@ -60,6 +60,6 @@ require 'models/job' -require "models/v3/domain/app_process" -require "models/v3/persistence/app_model" -require "models/v3/persistence/package_model" +require 'models/v3/domain/app_process' +require 'models/v3/persistence/app_model' +require 'models/v3/persistence/package_model' diff --git a/app/models/v3/persistence/package_model.rb b/app/models/v3/persistence/package_model.rb index 6e7bc62ac54..06d41f3ba28 100644 --- a/app/models/v3/persistence/package_model.rb +++ b/app/models/v3/persistence/package_model.rb @@ -1,6 +1,6 @@ module VCAP::CloudController class PackageModel < Sequel::Model(:packages) - PACKAGE_STATES = %w[PENDING READY FAILED].map(&:freeze).freeze + PACKAGE_STATES = %w(PENDING READY FAILED).map(&:freeze).freeze def validate validates_includes PACKAGE_STATES, :state, allow_missing: true diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index 2dc7418bd7f..564c8fc26a5 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -45,7 +45,7 @@ def do_request_with_error_handling 'type' => package_model.type, 'guid' => guid, 'hash' => nil, - 'state' => "PENDING", + 'state' => 'PENDING', 'error' => nil, 'created_at' => package_model.created_at.as_json, '_links' => { @@ -77,7 +77,6 @@ def do_request_with_error_handling parameter :type, 'Package type', required: true, valid_values: ['bits', 'docker'] parameter :bits, 'A binary zip file containing the package bits', required: false - context 'when the type is bits' do let(:type) { 'bits' } let(:packages_params) do @@ -109,7 +108,7 @@ def do_request_with_error_handling example 'Create a Package with application bits' do expect { do_request packages_params - }.to change{ VCAP::CloudController::PackageModel.count }.by(1) + }.to change { VCAP::CloudController::PackageModel.count }.by(1) package = VCAP::CloudController::PackageModel.last @@ -158,7 +157,7 @@ def do_request_with_error_handling example 'Create a Package with docker image' do expect { do_request packages_params - }.to change{ VCAP::CloudController::PackageModel.count }.by(1) + }.to change { VCAP::CloudController::PackageModel.count }.by(1) package = VCAP::CloudController::PackageModel.last diff --git a/spec/support/matchers/sequel_validations.rb b/spec/support/matchers/sequel_validations.rb index bb4224c50ea..823dca24ccd 100644 --- a/spec/support/matchers/sequel_validations.rb +++ b/spec/support/matchers/sequel_validations.rb @@ -55,7 +55,7 @@ end end -RSpec::Matchers.define :validates_includes do |values, attribute, options = {}| +RSpec::Matchers.define :validates_includes do |values, attribute, options={}| description do "validate includes of #{attribute} with #{values}" end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index 14e07370336..97251448a8d 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -35,7 +35,7 @@ module VCAP::CloudController let(:package) { PackageModel.make } let(:valid_zip) do - zip_name = File.join(tmpdir, "file.zip") + zip_name = File.join(tmpdir, 'file.zip') TestZip.create(zip_name, 1, 1024) zip_file = File.new(zip_name) Rack::Test::UploadedFile.new(zip_file) @@ -43,9 +43,9 @@ module VCAP::CloudController let(:package_response) do { - type: "bits", - package_hash: "a-hash", - created_at: "a-date", + type: 'bits', + package_hash: 'a-hash', + created_at: 'a-date', _links: { app: { href: "/v3/apps/#{app_guid}", @@ -76,20 +76,20 @@ module VCAP::CloudController end end - context "as an admin" do + context 'as an admin' do let(:headers) { admin_headers } - it "allows upload even if app_bits_upload flag is disabled" do + it 'allows upload even if app_bits_upload flag is disabled' do FeatureFlag.make(name: 'app_bits_upload', enabled: false) response_code, _ = packages_controller.create(app_guid) expect(response_code).to eq 201 end end - context "as a developer" do + context 'as a developer' do let(:user) { make_developer_for_space(app_obj.space) } - context "with an invalid package" do + context 'with an invalid package' do let(:params) { {} } it 'returns an UnprocessableEntity error' do @@ -102,7 +102,7 @@ module VCAP::CloudController end end - context "with an invalid type field" do + context 'with an invalid type field' do let(:params) { { type: 'ninja' } } it 'returns an UnprocessableEntity error' do @@ -169,7 +169,7 @@ module VCAP::CloudController let(:package_guid) { package.guid } context 'when a user can access a package' do - let(:expected_response) { "im a response" } + let(:expected_response) { 'im a response' } before do allow(packages_handler).to receive(:show).and_return(package) diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index fe1eb2dc190..9dea81c0854 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -108,7 +108,7 @@ module VCAP::CloudController result = nil expect { result = packages_handler.create(create_message, access_context) - }.to change{ Delayed::Job.count }.by(1) + }.to change { Delayed::Job.count }.by(1) expect(result.state).to eq('PENDING') end @@ -121,7 +121,7 @@ module VCAP::CloudController result = nil expect { result = packages_handler.create(create_message, access_context) - }.to_not change{ Delayed::Job.count } + }.to_not change { Delayed::Job.count } expect(result.state).to eq('READY') end @@ -222,7 +222,7 @@ module VCAP::CloudController expect { deleted_package = packages_handler.delete(package_guid, access_context) expect(deleted_package.guid).to eq(package_guid) - }.to change{ PackageModel.count }.by(-1) + }.to change { PackageModel.count }.by(-1) expect(PackageModel.find(guid: package_guid)).to be_nil end @@ -233,7 +233,7 @@ module VCAP::CloudController expect { packages_handler.delete(package_guid, access_context) - }.to change{ Delayed::Job.count }.by(1) + }.to change { Delayed::Job.count }.by(1) end end end diff --git a/spec/unit/jobs/v3/package_bits_spec.rb b/spec/unit/jobs/v3/package_bits_spec.rb index dcc838513fb..8bc55b67fdd 100644 --- a/spec/unit/jobs/v3/package_bits_spec.rb +++ b/spec/unit/jobs/v3/package_bits_spec.rb @@ -1,9 +1,9 @@ -require "spec_helper" +require 'spec_helper' module VCAP::CloudController module Jobs::Runtime describe PackageBits do - let(:uploaded_path) { "tmp/uploaded.zip" } + let(:uploaded_path) { 'tmp/uploaded.zip' } let(:package_guid) { SecureRandom.uuid } subject(:job) do @@ -12,23 +12,23 @@ module Jobs::Runtime it { is_expected.to be_a_valid_job } - describe "#perform" do + describe '#perform' do let(:package_blobstore) { double(:package_blobstore) } - let(:tmpdir) { "/tmp/special_temp" } + let(:tmpdir) { '/tmp/special_temp' } let(:max_package_size) { 256 } before do - TestConfig.override({:directories => {:tmpdir => tmpdir}, :packages => TestConfig.config[:packages].merge(:max_package_size => max_package_size)}) + TestConfig.override({ directories: { tmpdir: tmpdir }, packages: TestConfig.config[:packages].merge(max_package_size: max_package_size) }) - allow(AppBitsPackage).to receive(:new) { double(:packer, create_package_in_blobstore: "done") } + allow(AppBitsPackage).to receive(:new) { double(:packer, create_package_in_blobstore: 'done') } end - it "creates blob stores" do + it 'creates blob stores' do expect(CloudController::DependencyLocator.instance).to receive(:package_blobstore) job.perform end - it "creates an AppBitsPackage and performs" do + it 'creates an AppBitsPackage and performs' do expect(CloudController::DependencyLocator.instance).to receive(:package_blobstore).and_return(package_blobstore) packer = double @@ -37,7 +37,7 @@ module Jobs::Runtime job.perform end - it "knows its job name" do + it 'knows its job name' do expect(job.job_name_in_configuration).to equal(:package_bits) end end diff --git a/spec/unit/models/runtime/app_bits_package_spec.rb b/spec/unit/models/runtime/app_bits_package_spec.rb index 300ba8e46d0..c345ce5f9c4 100644 --- a/spec/unit/models/runtime/app_bits_package_spec.rb +++ b/spec/unit/models/runtime/app_bits_package_spec.rb @@ -38,30 +38,30 @@ FileUtils.remove_entry_secure blobstore_dir end - describe "#create_package_in_blobstore" do + describe '#create_package_in_blobstore' do let(:max_package_size) { nil } let(:package_guid) { package.guid } subject(:create) { packer.create_package_in_blobstore(package_guid, compressed_path) } - it "uploads the package zip to the package blob store" do + it 'uploads the package zip to the package blob store' do create expect(package_blobstore.exists?(package_guid)).to be true end - it "sets the package sha to the package" do + it 'sets the package sha to the package' do expect { create }.to change { package.refresh.package_hash } end - it "sets the state of the package" do - expect { create }.to change { package.refresh.state }.to ('READY') + it 'sets the state of the package' do + expect { create }.to change { package.refresh.state }.to('READY') end - it "removes the compressed path afterwards" do + it 'removes the compressed path afterwards' do expect(FileUtils).to receive(:rm_f).with(compressed_path) create end - context "when there is no package uploaded" do + context 'when there is no package uploaded' do let(:compressed_path) { nil } it "doesn't try to remove the file" do @@ -87,16 +87,16 @@ expect(package.error).to eq('BOOM') end - it "removes the compressed path afterwards" do + it 'removes the compressed path afterwards' do expect(FileUtils).to receive(:rm_f).with(compressed_path) create end end - context "when the app bits are too large" do + context 'when the app bits are too large' do let(:max_package_size) { 10 } - it "raises an exception and deletes the bits" do + it 'raises an exception and deletes the bits' do expect(FileUtils).to receive(:rm_f).with(compressed_path) expect { create diff --git a/spec/unit/presenters/v3/package_presenter_spec.rb b/spec/unit/presenters/v3/package_presenter_spec.rb index f8a70d4c38a..abb1742f829 100644 --- a/spec/unit/presenters/v3/package_presenter_spec.rb +++ b/spec/unit/presenters/v3/package_presenter_spec.rb @@ -5,7 +5,7 @@ module VCAP::CloudController describe PackagePresenter do describe '#present_json' do it 'presents the package as json' do - package = PackageModel.make(type: "package_type") + package = PackageModel.make(type: 'package_type') json_result = PackagePresenter.new.present_json(package) result = MultiJson.load(json_result) From 4f2114ec783226f5484595d12bc450643e3c737b Mon Sep 17 00:00:00 2001 From: Dan Lavine and James Myers Date: Fri, 9 Jan 2015 10:23:21 -0800 Subject: [PATCH 38/76] Implement /v3/packages/:guid/upload * Seperate creation of packages from uploading of bits * Can only upload if type bits * Packages of type bits cannot have a url field. [#79388094] --- app/access/v3/package_access.rb | 2 +- app/controllers/v3/packages_controller.rb | 24 +++ app/handlers/packages_handler.rb | 58 +++-- app/models/v3/persistence/package_model.rb | 6 +- app/presenters/v3/package_presenter.rb | 1 + bosh-templates/nginx.conf.erb | 3 +- .../20141226222846_create_packages.rb | 1 + lib/cloud_controller/jobs.rb | 1 + spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 200 +++++++++--------- .../v3/packages_controller_spec.rb | 78 +++++++ spec/unit/handlers/packages_handler_spec.rb | 185 +++++++++++++--- .../presenters/v3/package_presenter_spec.rb | 3 +- 13 files changed, 413 insertions(+), 151 deletions(-) diff --git a/app/access/v3/package_access.rb b/app/access/v3/package_access.rb index ebe6129a17d..fa23e99f802 100644 --- a/app/access/v3/package_access.rb +++ b/app/access/v3/package_access.rb @@ -11,7 +11,7 @@ def read?(package) has_read_scope && user_visible end - def create?(desired_package, app, space) + def create?(_, _, space) return true if context.roles.admin? has_write_scope = SecurityContext.scopes.include?('cloud_controller.write') diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index 2981fc05d93..ccb57c38401 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -28,6 +28,26 @@ def create(app_guid) app_not_found! end + post '/v3/packages/:guid/upload', :upload + def upload(package_guid) + message = PackageUploadMessage.new(package_guid, params) + valid, error = message.validate + unprocessable!(error) if !valid + + package = @packages_handler.upload(message, @access_context) + package_json = @package_presenter.present_json(package) + + [HTTP::CREATED, package_json] + rescue PackagesHandler::InvalidPackageType => e + invalid_request!(e.message) + rescue PackagesHandler::AppNotFound + app_not_found! + rescue PackagesHandler::PackageNotFound + package_not_found! + rescue PackagesHandler::Unauthorized + unauthorized! + end + get '/v3/packages/:guid', :show def show(package_guid) package = @packages_handler.show(package_guid, @access_context) @@ -65,5 +85,9 @@ def unauthorized! def unprocessable!(message) raise VCAP::Errors::ApiError.new_from_details('UnprocessableEntity', message) end + + def invalid_request!(message) + raise VCAP::Errors::ApiError.new_from_details('InvalidRequest', message) + end end end diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index a05c310f0cf..3ba41f0d292 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -1,18 +1,31 @@ module VCAP::CloudController + class PackageUploadMessage + attr_reader :package_path, :package_guid + + def initialize(package_guid, opts) + @package_guid = package_guid + @package_path = opts['bits_path'] + end + + def validate + return false, 'An application zip file must be uploaded.' unless @package_path + true + end + end + class PackageCreateMessage - attr_reader :app_guid, :type, :filepath + attr_reader :app_guid, :type, :url def initialize(app_guid, opts) @app_guid = app_guid @type = opts['type'] - @filepath = opts['bits_path'] - @filename = opts['bits_name'] + @url = opts['url'] end def validate errors = [] errors << validate_type_field - errors << validate_file if @type == 'bits' + errors << validate_url errs = errors.compact [errs.length == 0, errs] end @@ -29,18 +42,19 @@ def validate_type_field nil end - def validate_file - return 'Must upload an application zip file' if @filepath.nil? + def validate_url + return 'The url field cannot be provided when type is bits.' if @type == 'bits' && !@url.nil? + return 'The url field must be provided for type docker.' if @type == 'docker' && @url.nil? nil end end class PackagesHandler class Unauthorized < StandardError; end + class InvalidPackageType < StandardError; end class InvalidPackage < StandardError; end class AppNotFound < StandardError; end - - PACKAGE_STATES = %w(PENDING READY FAILED).map(&:freeze).freeze + class PackageNotFound < StandardError; end def initialize(config) @config = config @@ -50,7 +64,8 @@ def create(message, access_context) package = PackageModel.new package.app_guid = message.app_guid package.type = message.type - package.state = 'READY' if message.type == 'docker' + package.url = message.url + package.state = message.type == 'bits' ? PackageModel::CREATED_STATE : PackageModel::READY_STATE app = AppModel.find(guid: package.app_guid) raise AppNotFound if app.nil? @@ -65,16 +80,31 @@ def create(message, access_context) package.save end - if package.type == 'bits' - bits_packer_job = Jobs::Runtime::PackageBits.new(package.guid, message.filepath) - Jobs::Enqueuer.new(bits_packer_job, queue: Jobs::LocalQueue.new(@config)).enqueue - end - package rescue Sequel::ValidationFailed => e raise InvalidPackage.new(e.message) end + def upload(message, access_context) + package = PackageModel.find(guid: message.package_guid) + raise PackageNotFound if package.nil? + + app = AppModel.find(guid: package.app_guid) + raise AppNotFound if app.nil? + + raise InvalidPackageType.new('Package type must be bits.') if package.type != 'bits' + + space = Space.find(guid: app.space_guid) + raise Unauthorized if access_context.cannot?(:create, package, app, space) + + package.update(state: PackageModel::PENDING_STATE) + + bits_upload_job = Jobs::Runtime::PackageBits.new(package.guid, message.package_path) + Jobs::Enqueuer.new(bits_upload_job, queue: Jobs::LocalQueue.new(@config)).enqueue + + package + end + def delete(guid, access_context) package = PackageModel.find(guid: guid) return nil if package.nil? diff --git a/app/models/v3/persistence/package_model.rb b/app/models/v3/persistence/package_model.rb index 06d41f3ba28..027063bc039 100644 --- a/app/models/v3/persistence/package_model.rb +++ b/app/models/v3/persistence/package_model.rb @@ -1,6 +1,10 @@ module VCAP::CloudController class PackageModel < Sequel::Model(:packages) - PACKAGE_STATES = %w(PENDING READY FAILED).map(&:freeze).freeze + PENDING_STATE = 'PENDING' + READY_STATE = 'READY' + FAILED_STATE = 'FAILED' + CREATED_STATE = 'CREATED' + PACKAGE_STATES = [CREATED_STATE, PENDING_STATE, READY_STATE, FAILED_STATE].map(&:freeze).freeze def validate validates_includes PACKAGE_STATES, :state, allow_missing: true diff --git a/app/presenters/v3/package_presenter.rb b/app/presenters/v3/package_presenter.rb index f500186e5fc..037d515b1b4 100644 --- a/app/presenters/v3/package_presenter.rb +++ b/app/presenters/v3/package_presenter.rb @@ -5,6 +5,7 @@ def present_json(package) guid: package.guid, type: package.type, hash: package.package_hash, + url: package.url, state: package.state, error: package.error, created_at: package.created_at, diff --git a/bosh-templates/nginx.conf.erb b/bosh-templates/nginx.conf.erb index 36f4cb038d2..21085c0417b 100644 --- a/bosh-templates/nginx.conf.erb +++ b/bosh-templates/nginx.conf.erb @@ -91,7 +91,7 @@ http { } <% end %> - location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits|/v3/apps/.*/packages) { + location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits|/v3/packages/.*/upload) { # Pass altered request body to this location upload_pass @cc_uploads; upload_pass_args on; @@ -112,7 +112,6 @@ http { #forward the following fields from existing body upload_pass_form_field "^resources$"; upload_pass_form_field "^_method$"; - upload_pass_form_field "^type$"; #on any error, delete uploaded files. upload_cleanup 400-505; diff --git a/db/migrations/20141226222846_create_packages.rb b/db/migrations/20141226222846_create_packages.rb index 5c5e8f5699d..80920c653db 100644 --- a/db/migrations/20141226222846_create_packages.rb +++ b/db/migrations/20141226222846_create_packages.rb @@ -9,6 +9,7 @@ String :package_hash String :state, default: 'PENDING', null: false String :error, text: true + String :url end end end diff --git a/lib/cloud_controller/jobs.rb b/lib/cloud_controller/jobs.rb index 77ca951b7a1..1074a5c42c8 100644 --- a/lib/cloud_controller/jobs.rb +++ b/lib/cloud_controller/jobs.rb @@ -12,6 +12,7 @@ require 'jobs/runtime/legacy_jobs' require 'jobs/runtime/failed_jobs_cleanup' require 'jobs/runtime/pending_packages_cleanup' +require 'jobs/v3/package_bits' require 'jobs/enqueuer' require 'jobs/exception_catching_job' require 'jobs/request_job' diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 9fd3c769c85..a2d66cc437b 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '9c61ee8a851c4a269af6c51f284bd4de815b99b0' + API_FOLDER_CHECKSUM = 'e66229d92e7d7a1d46189279c17840672b0d4fd6' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index 564c8fc26a5..1c28486c189 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -46,6 +46,7 @@ def do_request_with_error_handling 'guid' => guid, 'hash' => nil, 'state' => 'PENDING', + 'url' => nil, 'error' => nil, 'created_at' => package_model.created_at.as_json, '_links' => { @@ -62,122 +63,121 @@ def do_request_with_error_handling end end + post '/v3/packages/:guid/upload' do + let(:type) { 'bits' } + let!(:package_model) do + VCAP::CloudController::PackageModel.make(app_guid: app_model.guid, type: type) + end + let(:space) { VCAP::CloudController::Space.make } + let(:space_guid) { space.guid } + let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } + let(:guid) { package_model.guid } + + before do + space.organization.add_user(user) + space.add_developer(user) + end + + parameter :bits, 'A binary zip file containing the package bits', required: true + + let(:packages_params) do + { + bits_name: 'application.zip', + bits_path: "#{tmpdir}/application.zip", + } + end + + let(:request_body_example) do + <<-eos.gsub(/^ */, '') + Content-type: multipart/form-data, boundary=AaB03x + --AaB03x + Content-Disposition: form-data; name="type" + + #{type} + --AaB03x + Content-Disposition: form-data; name="bits"; filename="application.zip" + Content-Type: application/zip + Content-Length: 123 + Content-Transfer-Encoding: binary + + <<binary artifact bytes>> + --AaB03x + eos + end + + example 'Upload Bits for a Package of type bits' do + expect { do_request packages_params }.to change { Delayed::Job.count }.by(1) + + job = Delayed::Job.last + expect(job.handler).to include(package_model.guid) + expect(job.guid).not_to be_nil + + expected_response = { + 'guid' => guid, + 'type' => type, + 'hash' => nil, + 'state' => 'PENDING', + 'url' => nil, + 'error' => nil, + 'created_at' => package_model.created_at.as_json, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package_model.guid}" }, + 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, + } + } + + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(201) + expect(parsed_response).to match(expected_response) + end + end + post '/v3/apps/:guid/packages' do let(:space) { VCAP::CloudController::Space.make } let(:space_guid) { space.guid } let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } let(:guid) { app_model.guid } + let(:type) { 'docker' } + let(:url) { 'docker://cloudfoundry/runtime-ci' } + let(:packages_params) do + { + type: type, + url: url + } + end before do space.organization.add_user(user) space.add_developer(user) end - parameter :guid, 'GUID of the app that is going to use the package', required: true parameter :type, 'Package type', required: true, valid_values: ['bits', 'docker'] - parameter :bits, 'A binary zip file containing the package bits', required: false - - context 'when the type is bits' do - let(:type) { 'bits' } - let(:packages_params) do - { - type: type, - bits_name: 'application.zip', - bits_path: "#{tmpdir}/application.zip", - } - end - - let(:request_body_example) do - <<-eos.gsub(/^ */, '') - Content-type: multipart/form-data, boundary=AaB03x - --AaB03x - Content-Disposition: form-data; name="type" - - #{type} - --AaB03x - Content-Disposition: form-data; name="bits"; filename="application.zip" - Content-Type: application/zip - Content-Length: 123 - Content-Transfer-Encoding: binary - - <<binary artifact bytes>> - --AaB03x - eos - end - - example 'Create a Package with application bits' do - expect { - do_request packages_params - }.to change { VCAP::CloudController::PackageModel.count }.by(1) - - package = VCAP::CloudController::PackageModel.last - - job = Delayed::Job.last - expect(job.handler).to include(package.guid) - expect(job.guid).not_to be_nil - - expected_response = { - 'guid' => package.guid, - 'type' => type, - 'hash' => nil, - 'state' => 'PENDING', - 'error' => nil, - 'created_at' => package.created_at.as_json, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{package.guid}" }, - 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, - } - } + parameter :url, 'Url of docker image', required: false - parsed_response = MultiJson.load(response_body) - expect(response_status).to eq(201) - expect(parsed_response).to match(expected_response) - end - end + example 'Create a Package' do + expect { + do_request packages_params + }.to change { VCAP::CloudController::PackageModel.count }.by(1) - context 'when the type is docker' do - let(:type) { 'docker' } - let(:packages_params) do - { - type: type, - } - end - - let(:request_body_example) do - <<-eos.gsub(/^ */, '') - Content-type: multipart/form-data, boundary=AaB03x - --AaB03x - Content-Disposition: form-data; name="type" - - #{type} - --AaB03x - eos - end - - example 'Create a Package with docker image' do - expect { - do_request packages_params - }.to change { VCAP::CloudController::PackageModel.count }.by(1) - - package = VCAP::CloudController::PackageModel.last - - expected_response = { - 'guid' => package.guid, - 'type' => type, - 'hash' => nil, - 'state' => 'READY', - 'error' => nil, - 'created_at' => package.created_at.as_json, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{package.guid}" }, - 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, - } + package = VCAP::CloudController::PackageModel.last + + expected_response = { + 'guid' => package.guid, + 'type' => type, + 'hash' => nil, + 'state' => 'READY', + 'error' => nil, + 'url' => url, + 'created_at' => package.created_at.as_json, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package.guid}" }, + 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, } + } - parsed_response = MultiJson.load(response_body) - expect(response_status).to eq(201) - expect(parsed_response).to match(expected_response) - end + parsed_response = MultiJson.load(response_body) + expect(response_status).to eq(201) + expect(parsed_response).to match(expected_response) end end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index 97251448a8d..17b227507c0 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -148,6 +148,84 @@ module VCAP::CloudController end end + describe 'upload' do + let(:package) { PackageModel.make } + let(:params) { { 'bits_path' => 'path/to/bits' } } + + context 'when the package type is not bits' do + before do + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::InvalidPackageType) + end + + it 'returns a 400 InvalidRequest' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'InvalidRequest' + expect(error.response_code).to eq 400 + end + end + end + + context 'when the app does not exist' do + before do + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::AppNotFound) + end + + it 'returns a 404 ResourceNotFound error' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'ResourceNotFound' + expect(error.response_code).to eq 404 + end + end + end + + context 'when the package does not exist' do + before do + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::PackageNotFound) + end + + it 'returns a 404 ResourceNotFound error' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'ResourceNotFound' + expect(error.response_code).to eq 404 + end + end + end + + context 'when the message is not valid' do + let(:params) { {} } + + it 'returns a 422 UnprocessableEntity error' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'UnprocessableEntity' + expect(error.response_code).to eq 422 + end + end + end + + context 'when the user cannot access the package' do + before do + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::Unauthorized) + end + + it 'returns an Unauthorized error' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'NotAuthorized' + expect(error.response_code).to eq 403 + end + end + end + end + describe '#show' do context 'when the package does not exist' do before do diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index 9dea81c0854..c0a1516d718 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -2,10 +2,35 @@ require 'handlers/packages_handler' module VCAP::CloudController + describe PackageUploadMessage do + let(:guid) { 'my-guid' } + + context 'when the path is not provided' do + let(:opts) { {} } + it 'is not valid' do + create_message = PackageUploadMessage.new(guid, opts) + valid, error = create_message.validate + expect(valid).to be_falsey + expect(error).to include('An application zip file must be uploaded.') + end + end + + context 'and the path is provided' do + let(:opts) { { 'bits_path' => 'foobar' } } + it 'is valid' do + create_message = PackageUploadMessage.new(guid, opts) + valid, error = create_message.validate + expect(valid).to be_truthy + expect(error).to be_nil + end + end + end + describe PackageCreateMessage do + let(:guid) { 'my-guid' } + context 'when a type parameter that is not allowed is provided' do let(:opts) { { 'type' => 'not-allowed' } } - let(:guid) { 'my-guid' } it 'is not valid' do create_message = PackageCreateMessage.new(guid, opts) @@ -17,7 +42,6 @@ module VCAP::CloudController context 'when nil type is provided' do let(:opts) { { 'type' => nil } } - let(:guid) { 'my-guid' } it 'is not valid' do create_message = PackageCreateMessage.new(guid, opts) @@ -28,21 +52,32 @@ module VCAP::CloudController end context 'when type is bits' do - let(:opts) { { 'type' => 'bits', 'bits_path' => nil, 'bits_name' => nil } } - let(:guid) { 'my-guid' } + let(:opts) { { 'type' => 'bits' } } + + context 'no url is provided' do + it 'is valid' do + create_message = PackageCreateMessage.new(guid, opts) + valid, errors = create_message.validate + expect(valid).to be_truthy + expect(errors).to be_empty + end + end + + context 'and a url is provided' do + let(:opts) { { 'type' => 'bits', 'url' => 'foobar' } } - context 'and no zip file is uploaded' do it 'is not valid' do create_message = PackageCreateMessage.new(guid, opts) valid, errors = create_message.validate expect(valid).to be_falsey - expect(errors).to include('Must upload an application zip file') + expect(errors).to include('The url field cannot be provided when type is bits.') end end + end - context 'and a zip file is uploaded' do - let(:opts) { { 'type' => 'bits', 'bits_path' => '/tmp', 'bits_name' => 'app.zip' } } - + context 'when type is docker' do + context 'and a url is provided' do + let(:opts) { { 'type' => 'docker', 'url' => 'foobar' } } it 'is valid' do create_message = PackageCreateMessage.new(guid, opts) valid, errors = create_message.validate @@ -51,13 +86,14 @@ module VCAP::CloudController end end - context 'when type is docker' do - let(:opts) { { 'type' => 'docker', 'bits_path' => nil, 'bits_name' => nil } } - it 'does not care about a zip file' do + context 'and a url is not provided' do + let(:opts) { { 'type' => 'docker' } } + + it 'is not valid' do create_message = PackageCreateMessage.new(guid, opts) valid, errors = create_message.validate - expect(valid).to be_truthy - expect(errors).to be_empty + expect(valid).to be_falsey + expect(errors).to include('The url field must be provided for type docker.') end end end @@ -83,15 +119,15 @@ module VCAP::CloudController end describe '#create' do + let(:url) { 'docker://cloudfoundry/runtime-ci' } let(:create_opts) do { - 'type' => 'bits', - 'bits_path' => tmpdir, - 'bits_name' => 'file.zip', + 'type' => 'docker', + 'url' => url } end - context 'when the app exists' do + context 'when the app exist' do let(:create_message) { PackageCreateMessage.new(app.guid, create_opts) } context 'when a user can create a package' do @@ -99,31 +135,28 @@ module VCAP::CloudController result = packages_handler.create(create_message, access_context) created_package = PackageModel.find(guid: result.guid) - expect(created_package.app_guid).to eq(result.app_guid) - expect(created_package.type).to eq(result.type) + expect(created_package).to eq(result) end context 'when the type is bits' do + let(:create_opts) { { 'type' => 'bits' } } + it 'adds a delayed job to upload the package bits' do - result = nil - expect { - result = packages_handler.create(create_message, access_context) - }.to change { Delayed::Job.count }.by(1) + result = packages_handler.create(create_message, access_context) - expect(result.state).to eq('PENDING') + expect(result.type).to eq('bits') + expect(result.state).to eq('CREATED') + expect(result.url).to be_nil end end context 'when the type is docker' do - let(:create_opts) { { 'type' => 'docker' } } - it 'adds a delayed job to upload the package bits' do - result = nil - expect { - result = packages_handler.create(create_message, access_context) - }.to_not change { Delayed::Job.count } + result = packages_handler.create(create_message, access_context) + expect(result.type).to eq('docker') expect(result.state).to eq('READY') + expect(result.url).to eq(url) end end end @@ -165,6 +198,96 @@ module VCAP::CloudController end end + describe 'upload' do + let(:package) { PackageModel.make(app_guid: app_guid, type: 'bits', state: 'CREATED') } + let(:upload_message) { PackageUploadMessage.new(package_guid, upload_opts) } + let(:create_opts) { { 'bit_path' => 'path/to/bits' } } + let(:upload_opts) { { 'bits_path' => 'foobar' } } + let(:app_guid) { app.guid } + let(:package_guid) { package.guid } + + before do + allow(access_context).to receive(:cannot?).and_return(false) + end + + context 'when the package exists' do + context 'when the app exists' do + context 'when the user can access the package' do + context 'when the package is of type bits' do + before do + config[:name] = 'local' + config[:index] = '1' + end + + it 'enqueues a upload job' do + expect { + packages_handler.upload(upload_message, access_context) + }.to change { Delayed::Job.count }.by(1) + + job = Delayed::Job.last + expect(job.queue).to eq('cc-local-1') + expect(job.handler).to include(package_guid) + expect(job.handler).to include('PackageBits') + end + + it 'changes the state to pending' do + packages_handler.upload(upload_message, access_context) + expect(PackageModel.find(guid: package_guid).state).to eq(PackageModel::PENDING_STATE) + end + + it 'returns the package' do + resulting_package = packages_handler.upload(upload_message, access_context) + expected_package = PackageModel.find(guid: package_guid) + expect(resulting_package.guid).to eq(expected_package.guid) + end + end + + context 'when the package is not of type bits' do + let(:package) { PackageModel.make(app_guid: app_guid, type: 'docker') } + + it 'raises an InvalidPackage exception' do + expect { + packages_handler.upload(upload_message, access_context) + }.to raise_error(PackagesHandler::InvalidPackageType) + end + end + end + + context 'when the user cannot access the package' do + before do + allow(access_context).to receive(:cannot?).and_return(true) + end + + it 'raises an Unathorized exception' do + expect { + packages_handler.upload(upload_message, access_context) + }.to raise_error(PackagesHandler::Unauthorized) + end + end + end + + context 'when the app does not exist' do + let(:app_guid) { 'non-existant' } + + it 'raises an AppNotFound exception' do + expect { + packages_handler.upload(upload_message, access_context) + }.to raise_error(PackagesHandler::AppNotFound) + end + end + end + + context 'when the package does not exist' do + let(:package_guid) { 'non-existant' } + + it 'raises a PackageNotFound exception' do + expect { + packages_handler.upload(upload_message, access_context) + }.to raise_error(PackagesHandler::PackageNotFound) + end + end + end + describe 'show' do let(:package) { PackageModel.make } let(:package_guid) { package.guid } diff --git a/spec/unit/presenters/v3/package_presenter_spec.rb b/spec/unit/presenters/v3/package_presenter_spec.rb index abb1742f829..3c0a10015cc 100644 --- a/spec/unit/presenters/v3/package_presenter_spec.rb +++ b/spec/unit/presenters/v3/package_presenter_spec.rb @@ -5,7 +5,7 @@ module VCAP::CloudController describe PackagePresenter do describe '#present_json' do it 'presents the package as json' do - package = PackageModel.make(type: 'package_type') + package = PackageModel.make(type: 'package_type', url: 'foobar') json_result = PackagePresenter.new.present_json(package) result = MultiJson.load(json_result) @@ -15,6 +15,7 @@ module VCAP::CloudController expect(result['state']).to eq(package.state) expect(result['error']).to eq(package.error) expect(result['hash']).to eq(package.package_hash) + expect(result['url']).to eq(package.url) expect(result['created_at']).to eq(package.created_at.as_json) expect(result['_links']).to include('self') expect(result['_links']).to include('app') From c628109bd149f6496770f41ea3044eec2bf8315a Mon Sep 17 00:00:00 2001 From: James Myers and Zach Robinson Date: Mon, 12 Jan 2015 09:34:05 -0800 Subject: [PATCH 39/76] Factor apart app and package logic. [#79388094] --- app/access/v3/package_access.rb | 16 +++-- app/controllers/v3/packages_controller.rb | 12 ++-- app/handlers/packages_handler.rb | 58 ++++++++-------- app/models/v3/persistence/package_model.rb | 4 +- app/presenters/v3/package_presenter.rb | 4 +- .../20141226222846_create_packages.rb | 6 +- spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 25 ++++--- spec/unit/access/v3/package_access_spec.rb | 66 ++++++++++++++----- .../v3/packages_controller_spec.rb | 52 +++++---------- spec/unit/handlers/packages_handler_spec.rb | 65 ++++++++++++------ .../presenters/v3/package_presenter_spec.rb | 2 +- 12 files changed, 178 insertions(+), 134 deletions(-) diff --git a/app/access/v3/package_access.rb b/app/access/v3/package_access.rb index fa23e99f802..ee4df66e87e 100644 --- a/app/access/v3/package_access.rb +++ b/app/access/v3/package_access.rb @@ -6,25 +6,29 @@ def read?(package) return true if context.roles.admin? has_read_scope = SecurityContext.scopes.include?('cloud_controller.read') - user_visible = AppModel.user_visible(context.user).where(guid: package.app_guid).count > 0 + user_visible = Space.user_visible(context.user).where(guid: package.space_guid).count > 0 has_read_scope && user_visible end - def create?(_, _, space) + def create?(_, space) return true if context.roles.admin? has_write_scope = SecurityContext.scopes.include?('cloud_controller.write') - is_space_developer = space && space.developers.include?(context.user) - org_active = space && space.organization.active? has_write_scope && is_space_developer && org_active end - def delete?(package, app, space) - create?(package, app, space) + def delete?(package, space) + create?(package, space) + end + + def upload?(package, space) + return true if context.roles.admin? + return false unless FeatureFlag.enabled?('app_bits_upload') + create?(package, space) end end end diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index ccb57c38401..e89cd73a70c 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -4,17 +4,21 @@ module VCAP::CloudController class PackagesController < RestController::BaseController def self.dependencies - [:packages_handler, :package_presenter] + [:packages_handler, :package_presenter, :apps_handler] end def inject_dependencies(dependencies) @packages_handler = dependencies[:packages_handler] @package_presenter = dependencies[:package_presenter] + @apps_handler = dependencies[:apps_handler] end post '/v3/apps/:guid/packages', :create def create(app_guid) - message = PackageCreateMessage.new(app_guid, params) + app = @apps_handler.show(app_guid, @access_context) + app_not_found! if app.nil? + + message = PackageCreateMessage.create_from_http_request(app.space_guid, body) valid, errors = message.validate unprocessable!(errors.join(', ')) if !valid @@ -24,8 +28,6 @@ def create(app_guid) [HTTP::CREATED, package_json] rescue PackagesHandler::Unauthorized unauthorized! - rescue PackagesHandler::AppNotFound - app_not_found! end post '/v3/packages/:guid/upload', :upload @@ -40,7 +42,7 @@ def upload(package_guid) [HTTP::CREATED, package_json] rescue PackagesHandler::InvalidPackageType => e invalid_request!(e.message) - rescue PackagesHandler::AppNotFound + rescue PackagesHandler::SpaceNotFound app_not_found! rescue PackagesHandler::PackageNotFound package_not_found! diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index 3ba41f0d292..b4c348e742b 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -14,15 +14,27 @@ def validate end class PackageCreateMessage - attr_reader :app_guid, :type, :url + attr_reader :space_guid, :type, :url + attr_accessor :error + + def self.create_from_http_request(space_guid, body) + opts = body && MultiJson.load(body) + raise MultiJson::ParseError.new('invalid request body') unless opts.is_a?(Hash) + PackageCreateMessage.new(space_guid, opts) + rescue MultiJson::ParseError => e + message = PackageCreateMessage.new(space_guid, {}) + message.error = e.message + message + end - def initialize(app_guid, opts) - @app_guid = app_guid + def initialize(space_guid, opts) + @space_guid = space_guid @type = opts['type'] @url = opts['url'] end def validate + return false, [error] if error errors = [] errors << validate_type_field errors << validate_url @@ -54,6 +66,7 @@ class Unauthorized < StandardError; end class InvalidPackageType < StandardError; end class InvalidPackage < StandardError; end class AppNotFound < StandardError; end + class SpaceNotFound < StandardError; end class PackageNotFound < StandardError; end def initialize(config) @@ -62,23 +75,16 @@ def initialize(config) def create(message, access_context) package = PackageModel.new - package.app_guid = message.app_guid + package.space_guid = message.space_guid package.type = message.type package.url = message.url package.state = message.type == 'bits' ? PackageModel::CREATED_STATE : PackageModel::READY_STATE - app = AppModel.find(guid: package.app_guid) - raise AppNotFound if app.nil? - - space = Space.find(guid: app.space_guid) - - app.db.transaction do - app.lock! + space = Space.find(guid: package.space_guid) + raise SpaceNotFound if space.nil? - raise Unauthorized if access_context.cannot?(:create, package, app, space) - - package.save - end + raise Unauthorized if access_context.cannot?(:create, package, space) + package.save package rescue Sequel::ValidationFailed => e @@ -87,15 +93,14 @@ def create(message, access_context) def upload(message, access_context) package = PackageModel.find(guid: message.package_guid) - raise PackageNotFound if package.nil? - - app = AppModel.find(guid: package.app_guid) - raise AppNotFound if app.nil? + raise PackageNotFound if package.nil? raise InvalidPackageType.new('Package type must be bits.') if package.type != 'bits' - space = Space.find(guid: app.space_guid) - raise Unauthorized if access_context.cannot?(:create, package, app, space) + space = Space.find(guid: package.space_guid) + raise SpaceNotFound if space.nil? + + raise Unauthorized if access_context.cannot?(:create, package, space) package.update(state: PackageModel::PENDING_STATE) @@ -109,14 +114,11 @@ def delete(guid, access_context) package = PackageModel.find(guid: guid) return nil if package.nil? - app = AppModel.find(guid: package.app_guid) - space = Space.find(guid: app.space_guid) + space = Space.find(guid: package.space_guid) package.db.transaction do - app.lock! - - raise Unauthorized if access_context.cannot?(:delete, package, app, space) - + package.lock! + raise Unauthorized if access_context.cannot?(:delete, package, space) package.destroy end @@ -129,7 +131,7 @@ def delete(guid, access_context) def show(guid, access_context) package = PackageModel.find(guid: guid) return nil if package.nil? - raise Unauthorized if access_context.cannot?(:read, package) + raise Unauthorized if access_context.cannot?(:read, package) package end end diff --git a/app/models/v3/persistence/package_model.rb b/app/models/v3/persistence/package_model.rb index 027063bc039..d103dda88d9 100644 --- a/app/models/v3/persistence/package_model.rb +++ b/app/models/v3/persistence/package_model.rb @@ -1,9 +1,9 @@ module VCAP::CloudController class PackageModel < Sequel::Model(:packages) - PENDING_STATE = 'PENDING' + PENDING_STATE = 'PROCESSING_UPLOAD' READY_STATE = 'READY' FAILED_STATE = 'FAILED' - CREATED_STATE = 'CREATED' + CREATED_STATE = 'AWAITING_UPLOAD' PACKAGE_STATES = [CREATED_STATE, PENDING_STATE, READY_STATE, FAILED_STATE].map(&:freeze).freeze def validate diff --git a/app/presenters/v3/package_presenter.rb b/app/presenters/v3/package_presenter.rb index 037d515b1b4..2cc695b084b 100644 --- a/app/presenters/v3/package_presenter.rb +++ b/app/presenters/v3/package_presenter.rb @@ -13,8 +13,8 @@ def present_json(package) self: { href: "/v3/packages/#{package.guid}" }, - app: { - href: "/v3/apps/#{package.app_guid}", + space: { + href: "/v2/spaces/#{package.space_guid}", }, }, } diff --git a/db/migrations/20141226222846_create_packages.rb b/db/migrations/20141226222846_create_packages.rb index 80920c653db..a7f2c374435 100644 --- a/db/migrations/20141226222846_create_packages.rb +++ b/db/migrations/20141226222846_create_packages.rb @@ -2,12 +2,12 @@ change do create_table :packages do VCAP::Migration.common(self) - String :app_guid - index :app_guid + String :space_guid + index :space_guid String :type index :type String :package_hash - String :state, default: 'PENDING', null: false + String :state, null: false String :error, text: true String :url end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index a2d66cc437b..5b2d626add0 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = 'e66229d92e7d7a1d46189279c17840672b0d4fd6' + API_FOLDER_CHECKSUM = 'e8f05751187784b408333115770e96e6bf6cd748' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index 1c28486c189..74ed4f97503 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -26,14 +26,12 @@ def do_request_with_error_handling context 'standard endpoints' do get '/v3/packages/:guid' do let(:space) { VCAP::CloudController::Space.make } - let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } - let(:package_model) do - VCAP::CloudController::PackageModel.make(app_guid: app_model.guid) + VCAP::CloudController::PackageModel.make(space_guid: space_guid) end let(:guid) { package_model.guid } - let(:app_guid) { app_model.guid } + let(:space_guid) { space.guid } before do space.organization.add_user user @@ -51,7 +49,7 @@ def do_request_with_error_handling 'created_at' => package_model.created_at.as_json, '_links' => { 'self' => { 'href' => "/v3/packages/#{guid}" }, - 'app' => { 'href' => "/v3/apps/#{app_guid}" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } @@ -66,11 +64,10 @@ def do_request_with_error_handling post '/v3/packages/:guid/upload' do let(:type) { 'bits' } let!(:package_model) do - VCAP::CloudController::PackageModel.make(app_guid: app_model.guid, type: type) + VCAP::CloudController::PackageModel.make(space_guid: space_guid, type: type) end let(:space) { VCAP::CloudController::Space.make } let(:space_guid) { space.guid } - let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } let(:guid) { package_model.guid } before do @@ -122,7 +119,7 @@ def do_request_with_error_handling 'created_at' => package_model.created_at.as_json, '_links' => { 'self' => { 'href' => "/v3/packages/#{package_model.guid}" }, - 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } @@ -135,7 +132,7 @@ def do_request_with_error_handling post '/v3/apps/:guid/packages' do let(:space) { VCAP::CloudController::Space.make } let(:space_guid) { space.guid } - let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } + let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space_guid) } let(:guid) { app_model.guid } let(:type) { 'docker' } let(:url) { 'docker://cloudfoundry/runtime-ci' } @@ -151,6 +148,8 @@ def do_request_with_error_handling space.add_developer(user) end + let(:raw_post) { MultiJson.dump(packages_params, pretty: true) } + parameter :type, 'Package type', required: true, valid_values: ['bits', 'docker'] parameter :url, 'Url of docker image', required: false @@ -171,7 +170,7 @@ def do_request_with_error_handling 'created_at' => package.created_at.as_json, '_links' => { 'self' => { 'href' => "/v3/packages/#{package.guid}" }, - 'app' => { 'href' => "/v3/apps/#{app_model.guid}" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } @@ -183,14 +182,12 @@ def do_request_with_error_handling delete '/v3/packages/:guid' do let(:space) { VCAP::CloudController::Space.make } - let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } - + let(:space_guid) { space.guid } let!(:package_model) do - VCAP::CloudController::PackageModel.make(app_guid: app_model.guid) + VCAP::CloudController::PackageModel.make(space_guid: space_guid) end let(:guid) { package_model.guid } - let(:app_guid) { app_model.guid } before do space.organization.add_user user diff --git a/spec/unit/access/v3/package_access_spec.rb b/spec/unit/access/v3/package_access_spec.rb index a590f8be904..c801680cfee 100644 --- a/spec/unit/access/v3/package_access_spec.rb +++ b/spec/unit/access/v3/package_access_spec.rb @@ -19,8 +19,7 @@ module VCAP::CloudController describe '#read?' do let(:space) { Space.make } - let(:app) { AppModel.make(space_guid: space.guid) } - let(:package) { PackageModel.new(app_guid: app.guid) } + let(:package) { PackageModel.new(space_guid: space.guid) } context 'admin user' do let(:admin) { true } @@ -36,7 +35,7 @@ module VCAP::CloudController let(:token) { { 'scope' => ['cloud_controller.read'] } } it 'allows the user to read' do - allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: app.guid)) + allow(Space).to receive(:user_visible).and_return(Space.where(guid: space.guid)) access_control = PackageModelAccess.new(access_context) expect(access_control.read?(package)).to be_truthy end @@ -44,7 +43,7 @@ module VCAP::CloudController context 'when the user has insufficient scope' do it 'disallows the user from reading' do - allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: app.guid)) + allow(Space).to receive(:user_visible).and_return(Space.where(guid: space.guid)) access_control = PackageModelAccess.new(access_context) expect(access_control.read?(package)).to be_falsey end @@ -54,7 +53,7 @@ module VCAP::CloudController let(:token) { { 'scope' => ['cloud_controller.read'] } } it 'disallows the user from reading' do - allow(AppModel).to receive(:user_visible).and_return(AppModel.where(guid: nil)) + allow(Space).to receive(:user_visible).and_return(Space.where(guid: nil)) access_control = PackageModelAccess.new(access_context) expect(access_control.read?(package)).to be_falsey end @@ -62,18 +61,18 @@ module VCAP::CloudController end end - describe '#create?, #delete?' do + describe '#create?, #delete?, #upload?' do let(:space) { Space.make } - let(:app) { AppModel.make(space_guid: space.guid) } - let(:package) { PackageModel.new(app_guid: app.guid) } + let(:package) { PackageModel.new(space_guid: space.guid) } context 'admin user' do let(:admin) { true } it 'allows the user to perform the action' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package, app, space)).to be_truthy - expect(access_control.delete?(package, app, space)).to be_truthy + expect(access_control.create?(package, space)).to be_truthy + expect(access_control.delete?(package, space)).to be_truthy + expect(access_control.upload?(package, space)).to be_truthy end end @@ -88,8 +87,9 @@ module VCAP::CloudController it 'allows the user to create' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package, app, space)).to be_truthy - expect(access_control.delete?(package, app, space)).to be_truthy + expect(access_control.create?(package, space)).to be_truthy + expect(access_control.delete?(package, space)).to be_truthy + expect(access_control.upload?(package, space)).to be_truthy end end @@ -103,8 +103,9 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package, app, space)).to be_falsey - expect(access_control.delete?(package, app, space)).to be_falsey + expect(access_control.create?(package, space)).to be_falsey + expect(access_control.delete?(package, space)).to be_falsey + expect(access_control.upload?(package, space)).to be_falsey end end @@ -113,8 +114,9 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package, app, space)).to be_falsey - expect(access_control.delete?(package, app, space)).to be_falsey + expect(access_control.create?(package, space)).to be_falsey + expect(access_control.delete?(package, space)).to be_falsey + expect(access_control.upload?(package, space)).to be_falsey end end @@ -129,11 +131,39 @@ module VCAP::CloudController it 'disallows the user from creating' do access_control = PackageModelAccess.new(access_context) - expect(access_control.create?(package, app, space)).to be_falsey - expect(access_control.delete?(package, app, space)).to be_falsey + expect(access_control.create?(package, space)).to be_falsey + expect(access_control.delete?(package, space)).to be_falsey + expect(access_control.upload?(package, space)).to be_falsey end end end end + + describe '#upload? when the app_bits_upload feature flag is disabled' do + let(:space) { Space.make } + let(:package) { PackageModel.new(space_guid: space.guid) } + + before do + FeatureFlag.make(name: 'app_bits_upload', enabled: false) + end + + context 'as an admin user' do + let(:admin) { true } + + it 'allows the user to upload' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.upload?(package, space)).to be_truthy + end + end + + context 'as a non-admin user' do + let(:token) { { 'scope' => ['cloud_controller.write'] } } + + it 'disallows the user from uploading' do + access_control = PackageModelAccess.new(access_context) + expect(access_control.upload?(package, space)).to be_falsey + end + end + end end end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index 17b227507c0..6458ede58be 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -5,8 +5,10 @@ module VCAP::CloudController let(:logger) { instance_double(Steno::Logger) } let(:user) { User.make } let(:params) { {} } - let(:packages_handler) { double(:process_handler) } + let(:packages_handler) { double(:packages_handler) } + let(:apps_handler) { double(:apps_handler) } let(:package_presenter) { double(:package_presenter) } + let(:req_body) { '{}' } let(:packages_controller) do PackagesController.new( @@ -14,11 +16,12 @@ module VCAP::CloudController logger, {}, params.stringify_keys, - '', + req_body, nil, { packages_handler: packages_handler, package_presenter: package_presenter, + apps_handler: apps_handler }, ) end @@ -29,10 +32,11 @@ module VCAP::CloudController describe '#create' do let(:tmpdir) { Dir.mktmpdir } - let(:params) { { type: 'bits', bits_path: '/tmp/app.zip', bits_name: 'app.zip' } } - let(:app_obj) { AppModel.make } - let(:app_guid) { app_obj.guid } + let(:app_model) { AppModel.make } + let(:app_guid) { app_model.guid } + let(:space_guid) { app_model.space_guid } let(:package) { PackageModel.make } + let(:req_body) { '{"type":"bits"}' } let(:valid_zip) do zip_name = File.join(tmpdir, 'file.zip') @@ -41,22 +45,12 @@ module VCAP::CloudController Rack::Test::UploadedFile.new(zip_file) end - let(:package_response) do - { - type: 'bits', - package_hash: 'a-hash', - created_at: 'a-date', - _links: { - app: { - href: "/v3/apps/#{app_guid}", - }, - }, - } - end + let(:package_response) { 'foobar' } before do allow(package_presenter).to receive(:present_json).and_return(MultiJson.dump(package_response, pretty: true)) allow(packages_handler).to receive(:create).and_return(package) + allow(apps_handler).to receive(:show).and_return(app_model) end after do @@ -76,21 +70,11 @@ module VCAP::CloudController end end - context 'as an admin' do - let(:headers) { admin_headers } - - it 'allows upload even if app_bits_upload flag is disabled' do - FeatureFlag.make(name: 'app_bits_upload', enabled: false) - response_code, _ = packages_controller.create(app_guid) - expect(response_code).to eq 201 - end - end - context 'as a developer' do - let(:user) { make_developer_for_space(app_obj.space) } + let(:user) { make_developer_for_space(app_model.space) } context 'with an invalid package' do - let(:params) { {} } + let(:req_body) { 'all sorts of invalid' } it 'returns an UnprocessableEntity error' do expect { @@ -103,7 +87,7 @@ module VCAP::CloudController end context 'with an invalid type field' do - let(:params) { { type: 'ninja' } } + let(:req_body) { '{ "type": "ninja" }' } it 'returns an UnprocessableEntity error' do expect { @@ -134,7 +118,7 @@ module VCAP::CloudController context 'when the app does not exist' do before do - allow(packages_handler).to receive(:create).and_raise(PackagesHandler::AppNotFound) + allow(apps_handler).to receive(:show).and_return(nil) end it 'returns a 404 ResourceNotFound error' do @@ -148,7 +132,7 @@ module VCAP::CloudController end end - describe 'upload' do + describe '#upload' do let(:package) { PackageModel.make } let(:params) { { 'bits_path' => 'path/to/bits' } } @@ -167,9 +151,9 @@ module VCAP::CloudController end end - context 'when the app does not exist' do + context 'when the space does not exist' do before do - allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::AppNotFound) + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::SpaceNotFound) end it 'returns a 404 ResourceNotFound error' do diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index c0a1516d718..3962af83666 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -97,6 +97,32 @@ module VCAP::CloudController end end end + + describe 'create_from_http_request' do + context 'when the body is valid json' do + let(:body) { MultiJson.dump({ type: 'bits' }) } + + it 'creates a PackageCreateMessage from the json' do + pcm = PackageCreateMessage.create_from_http_request(guid, body) + valid, errors = pcm.validate + + expect(valid).to be_truthy + expect(errors).to be_empty + end + end + + context 'when the body is not valid json' do + let(:body) { '{{' } + + it 'returns a PackageCreateMessage that is not valid' do + pcm = PackageCreateMessage.create_from_http_request(guid, body) + valid, errors = pcm.validate + + expect(valid).to be_falsey + expect(errors[0]).to include('parse error') + end + end + end end describe PackagesHandler do @@ -111,7 +137,6 @@ module VCAP::CloudController let(:config) { TestConfig.config } let(:packages_handler) { described_class.new(config) } let(:access_context) { double(:access_context) } - let(:app) { AppModel.make(space_guid: space.guid) } let(:space) { Space.make } before do @@ -127,8 +152,8 @@ module VCAP::CloudController } end - context 'when the app exist' do - let(:create_message) { PackageCreateMessage.new(app.guid, create_opts) } + context 'when the space exists' do + let(:create_message) { PackageCreateMessage.new(space.guid, create_opts) } context 'when a user can create a package' do it 'creates the package' do @@ -170,7 +195,7 @@ module VCAP::CloudController expect { packages_handler.create(create_message, access_context) }.to raise_error(PackagesHandler::Unauthorized) - expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel), app, space) + expect(access_context).to have_received(:cannot?).with(:create, kind_of(PackageModel), space) end end @@ -187,31 +212,31 @@ module VCAP::CloudController end end - context 'when the app does not exist' do + context 'when the space does not exist' do let(:create_message) { PackageCreateMessage.new('non-existant', create_opts) } - it 'raises AppNotFound' do + it 'raises SpaceNotFound' do expect { packages_handler.create(create_message, access_context) - }.to raise_error(PackagesHandler::AppNotFound) + }.to raise_error(PackagesHandler::SpaceNotFound) end end end - describe 'upload' do - let(:package) { PackageModel.make(app_guid: app_guid, type: 'bits', state: 'CREATED') } + describe '#upload' do + let(:package) { PackageModel.make(space_guid: space_guid, type: 'bits', state: 'CREATED') } let(:upload_message) { PackageUploadMessage.new(package_guid, upload_opts) } let(:create_opts) { { 'bit_path' => 'path/to/bits' } } let(:upload_opts) { { 'bits_path' => 'foobar' } } - let(:app_guid) { app.guid } let(:package_guid) { package.guid } + let(:space_guid) { space.guid } before do allow(access_context).to receive(:cannot?).and_return(false) end context 'when the package exists' do - context 'when the app exists' do + context 'when the space exists' do context 'when the user can access the package' do context 'when the package is of type bits' do before do @@ -243,7 +268,7 @@ module VCAP::CloudController end context 'when the package is not of type bits' do - let(:package) { PackageModel.make(app_guid: app_guid, type: 'docker') } + let(:package) { PackageModel.make(space_guid: space_guid, type: 'docker') } it 'raises an InvalidPackage exception' do expect { @@ -266,13 +291,13 @@ module VCAP::CloudController end end - context 'when the app does not exist' do - let(:app_guid) { 'non-existant' } + context 'when the space does not exist' do + let(:space_guid) { 'non-existant' } - it 'raises an AppNotFound exception' do + it 'raises an SpaceNotFound exception' do expect { packages_handler.upload(upload_message, access_context) - }.to raise_error(PackagesHandler::AppNotFound) + }.to raise_error(PackagesHandler::SpaceNotFound) end end end @@ -288,7 +313,7 @@ module VCAP::CloudController end end - describe 'show' do + describe '#show' do let(:package) { PackageModel.make } let(:package_guid) { package.guid } @@ -325,8 +350,8 @@ module VCAP::CloudController end end - describe 'delete' do - let!(:package) { PackageModel.make(app_guid: app.guid) } + describe '#delete' do + let!(:package) { PackageModel.make(space_guid: space.guid) } let(:package_guid) { package.guid } context 'when the user can access a package' do @@ -371,7 +396,7 @@ module VCAP::CloudController deleted_package = packages_handler.delete(package_guid, access_context) expect(deleted_package).to be_nil }.to raise_error(PackagesHandler::Unauthorized) - expect(access_context).to have_received(:cannot?).with(:delete, kind_of(PackageModel), app, space) + expect(access_context).to have_received(:cannot?).with(:delete, kind_of(PackageModel), space) end end end diff --git a/spec/unit/presenters/v3/package_presenter_spec.rb b/spec/unit/presenters/v3/package_presenter_spec.rb index 3c0a10015cc..59be3f9025d 100644 --- a/spec/unit/presenters/v3/package_presenter_spec.rb +++ b/spec/unit/presenters/v3/package_presenter_spec.rb @@ -18,7 +18,7 @@ module VCAP::CloudController expect(result['url']).to eq(package.url) expect(result['created_at']).to eq(package.created_at.as_json) expect(result['_links']).to include('self') - expect(result['_links']).to include('app') + expect(result['_links']).to include('space') end end end From c75c99a229291e68b4b4cdafe05a7517e4e4f173 Mon Sep 17 00:00:00 2001 From: Serguei Filimonov and Zach Robinson Date: Mon, 12 Jan 2015 14:50:46 -0800 Subject: [PATCH 40/76] Package bits can only be uploaded once. [#79388094] --- app/handlers/packages_handler.rb | 3 +- app/models/runtime/app_bits_package.rb | 4 +- app/models/v3/persistence/package_model.rb | 8 +-- app/presenters/v3/package_presenter.rb | 3 + spec/api/api_version_spec.rb | 2 +- .../api/documentation/v3/packages_api_spec.rb | 59 ++++++++++--------- spec/support/fakes/blueprints.rb | 3 +- spec/unit/handlers/packages_handler_spec.rb | 17 +++++- .../presenters/v3/package_presenter_spec.rb | 11 ++++ 9 files changed, 71 insertions(+), 39 deletions(-) diff --git a/app/handlers/packages_handler.rb b/app/handlers/packages_handler.rb index b4c348e742b..4388c786aa2 100644 --- a/app/handlers/packages_handler.rb +++ b/app/handlers/packages_handler.rb @@ -65,9 +65,9 @@ class PackagesHandler class Unauthorized < StandardError; end class InvalidPackageType < StandardError; end class InvalidPackage < StandardError; end - class AppNotFound < StandardError; end class SpaceNotFound < StandardError; end class PackageNotFound < StandardError; end + class BitsAlreadyUploaded < StandardError; end def initialize(config) @config = config @@ -96,6 +96,7 @@ def upload(message, access_context) raise PackageNotFound if package.nil? raise InvalidPackageType.new('Package type must be bits.') if package.type != 'bits' + raise BitsAlreadyUploaded.new('Bits may be uploaded only once. Create a new package to upload different bits.') if package.state != PackageModel::CREATED_STATE space = Space.find(guid: package.space_guid) raise SpaceNotFound if space.nil? diff --git a/app/models/runtime/app_bits_package.rb b/app/models/runtime/app_bits_package.rb index c491967ff76..313736f5dd9 100644 --- a/app/models/runtime/app_bits_package.rb +++ b/app/models/runtime/app_bits_package.rb @@ -48,12 +48,12 @@ def create_package_in_blobstore(package_guid, package_path) package.db.transaction do package.lock! package.package_hash = package_file.hexdigest - package.state = 'READY' + package.state = VCAP::CloudController::PackageModel::READY_STATE package.save end rescue => e package.db.transaction do - package.state = 'FAILED' + package.state = VCAP::CloudController::PackageModel::FAILED_STATE package.error = e.message package.save end diff --git a/app/models/v3/persistence/package_model.rb b/app/models/v3/persistence/package_model.rb index d103dda88d9..b8d41e841a8 100644 --- a/app/models/v3/persistence/package_model.rb +++ b/app/models/v3/persistence/package_model.rb @@ -1,9 +1,9 @@ module VCAP::CloudController class PackageModel < Sequel::Model(:packages) - PENDING_STATE = 'PROCESSING_UPLOAD' - READY_STATE = 'READY' - FAILED_STATE = 'FAILED' - CREATED_STATE = 'AWAITING_UPLOAD' + PENDING_STATE = 'PROCESSING_UPLOAD' + READY_STATE = 'READY' + FAILED_STATE = 'FAILED' + CREATED_STATE = 'AWAITING_UPLOAD' PACKAGE_STATES = [CREATED_STATE, PENDING_STATE, READY_STATE, FAILED_STATE].map(&:freeze).freeze def validate diff --git a/app/presenters/v3/package_presenter.rb b/app/presenters/v3/package_presenter.rb index 2cc695b084b..a1d88bd47ca 100644 --- a/app/presenters/v3/package_presenter.rb +++ b/app/presenters/v3/package_presenter.rb @@ -13,6 +13,9 @@ def present_json(package) self: { href: "/v3/packages/#{package.guid}" }, + upload: { + href: "/v3/packages/#{package.guid}/upload", + }, space: { href: "/v2/spaces/#{package.space_guid}", }, diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index 5b2d626add0..c6ff4aa6adc 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = 'e8f05751187784b408333115770e96e6bf6cd748' + API_FOLDER_CHECKSUM = 'cfaef9aa6c8588cfd6a23c3d89c20ce2f9857211' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/packages_api_spec.rb b/spec/api/documentation/v3/packages_api_spec.rb index 74ed4f97503..da4410527dc 100644 --- a/spec/api/documentation/v3/packages_api_spec.rb +++ b/spec/api/documentation/v3/packages_api_spec.rb @@ -40,16 +40,17 @@ def do_request_with_error_handling example 'Get a Package' do expected_response = { - 'type' => package_model.type, - 'guid' => guid, - 'hash' => nil, - 'state' => 'PENDING', - 'url' => nil, - 'error' => nil, + 'type' => package_model.type, + 'guid' => guid, + 'hash' => nil, + 'state' => VCAP::CloudController::PackageModel::CREATED_STATE, + 'url' => nil, + 'error' => nil, 'created_at' => package_model.created_at.as_json, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{guid}" }, - 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{guid}" }, + 'upload' => { 'href' => "/v3/packages/#{guid}/upload" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } @@ -110,16 +111,17 @@ def do_request_with_error_handling expect(job.guid).not_to be_nil expected_response = { - 'guid' => guid, - 'type' => type, - 'hash' => nil, - 'state' => 'PENDING', - 'url' => nil, - 'error' => nil, + 'guid' => guid, + 'type' => type, + 'hash' => nil, + 'state' => VCAP::CloudController::PackageModel::PENDING_STATE, + 'url' => nil, + 'error' => nil, 'created_at' => package_model.created_at.as_json, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{package_model.guid}" }, - 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package_model.guid}" }, + 'upload' => { 'href' => "/v3/packages/#{package_model.guid}/upload" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } @@ -139,7 +141,7 @@ def do_request_with_error_handling let(:packages_params) do { type: type, - url: url + url: url } end @@ -161,16 +163,17 @@ def do_request_with_error_handling package = VCAP::CloudController::PackageModel.last expected_response = { - 'guid' => package.guid, - 'type' => type, - 'hash' => nil, - 'state' => 'READY', - 'error' => nil, - 'url' => url, + 'guid' => package.guid, + 'type' => type, + 'hash' => nil, + 'state' => 'READY', + 'error' => nil, + 'url' => url, 'created_at' => package.created_at.as_json, - '_links' => { - 'self' => { 'href' => "/v3/packages/#{package.guid}" }, - 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, + '_links' => { + 'self' => { 'href' => "/v3/packages/#{package.guid}" }, + 'upload' => { 'href' => "/v3/packages/#{package.guid}/upload" }, + 'space' => { 'href' => "/v2/spaces/#{space_guid}" }, } } diff --git a/spec/support/fakes/blueprints.rb b/spec/support/fakes/blueprints.rb index 4f943aec323..a7db1d0c16c 100644 --- a/spec/support/fakes/blueprints.rb +++ b/spec/support/fakes/blueprints.rb @@ -36,7 +36,8 @@ module VCAP::CloudController end PackageModel.blueprint do - guid { Sham.guid } + guid { Sham.guid } + state { VCAP::CloudController::PackageModel::CREATED_STATE } end User.blueprint do diff --git a/spec/unit/handlers/packages_handler_spec.rb b/spec/unit/handlers/packages_handler_spec.rb index 3962af83666..1984e51e094 100644 --- a/spec/unit/handlers/packages_handler_spec.rb +++ b/spec/unit/handlers/packages_handler_spec.rb @@ -170,7 +170,7 @@ module VCAP::CloudController result = packages_handler.create(create_message, access_context) expect(result.type).to eq('bits') - expect(result.state).to eq('CREATED') + expect(result.state).to eq(PackageModel::CREATED_STATE) expect(result.url).to be_nil end end @@ -224,7 +224,7 @@ module VCAP::CloudController end describe '#upload' do - let(:package) { PackageModel.make(space_guid: space_guid, type: 'bits', state: 'CREATED') } + let(:package) { PackageModel.make(space_guid: space_guid, type: 'bits', state: PackageModel::CREATED_STATE) } let(:upload_message) { PackageUploadMessage.new(package_guid, upload_opts) } let(:create_opts) { { 'bit_path' => 'path/to/bits' } } let(:upload_opts) { { 'bits_path' => 'foobar' } } @@ -265,6 +265,19 @@ module VCAP::CloudController expected_package = PackageModel.find(guid: package_guid) expect(resulting_package.guid).to eq(expected_package.guid) end + + context 'when the bits have already been uploaded' do + before do + package.state = PackageModel::PENDING_STATE + package.save + end + + it 'raises BitsAlreadyUploaded error' do + expect { + packages_handler.upload(upload_message, access_context) + }.to raise_error(PackagesHandler::BitsAlreadyUploaded) + end + end end context 'when the package is not of type bits' do diff --git a/spec/unit/presenters/v3/package_presenter_spec.rb b/spec/unit/presenters/v3/package_presenter_spec.rb index 59be3f9025d..62ec0c38002 100644 --- a/spec/unit/presenters/v3/package_presenter_spec.rb +++ b/spec/unit/presenters/v3/package_presenter_spec.rb @@ -20,6 +20,17 @@ module VCAP::CloudController expect(result['_links']).to include('self') expect(result['_links']).to include('space') end + + context 'when the package type is bits' do + let(:package) { PackageModel.make(type: 'bits', url: 'foobar') } + + it 'includes a link to upload' do + json_result = PackagePresenter.new.present_json(package) + result = MultiJson.load(json_result) + + expect(result['_links']['upload']['href']).to eq("/v3/packages/#{package.guid}/upload") + end + end end end end From eda570d48a39b23752fcaafec43c65a3af63a595 Mon Sep 17 00:00:00 2001 From: Luan Santos and Sujoy Basu Date: Mon, 12 Jan 2015 16:06:09 -0800 Subject: [PATCH 41/76] Fix time consistency, use DB time where possible - Sets time zone on the database to UTC upon startup - Makes Sequel interpret every timestamp as UTC - Use Time.now.utc where DB time is not applicable [#82077856] --- .../runtime/billing_events_controller.rb | 2 +- app/jobs/runtime/pending_packages_cleanup.rb | 3 +-- app/models/runtime/app.rb | 2 +- app/models/runtime/app_start_event.rb | 2 +- app/models/runtime/app_stop_event.rb | 2 +- .../runtime/organization_start_event.rb | 2 +- app/models/services/service_create_event.rb | 2 +- app/models/services/service_delete_event.rb | 2 +- app/presenters/api/job_presenter.rb | 4 ++-- .../runtime/app_event_repository.rb | 2 +- .../runtime/space_event_repository.rb | 6 +++--- app/repositories/services/event_repository.rb | 2 +- .../20130219194917_create_stacks_table.rb | 2 +- lib/cloud_controller.rb | 2 ++ lib/cloud_controller/blobstore/blob.rb | 2 +- lib/cloud_controller/blobstore/client.rb | 4 ++-- lib/cloud_controller/clock.rb | 4 ++-- lib/cloud_controller/db.rb | 3 +++ lib/cloud_controller/dea/client.rb | 4 ++-- lib/cloud_controller/dea/pool.rb | 6 +++--- lib/cloud_controller/dea/stager_pool.rb | 4 ++-- lib/cloud_controller/diagnostics.rb | 6 +++--- .../diego/instances_reporter.rb | 2 +- .../diego/service_registry.rb | 4 ++-- lib/cloud_controller/resource_pool.rb | 4 ++-- lib/vcap/rest_api/query.rb | 2 +- lib/vcap/uaa_token_decoder.rb | 4 ++-- spec/support/fakes/blueprints.rb | 6 +++--- spec/support/integration/http.rb | 2 +- .../controllers/base/model_controller_spec.rb | 2 +- .../internal/bulk_apps_controller_spec.rb | 2 +- .../app_bits_upload_controller_spec.rb | 4 ++-- .../app_usage_events_controller_spec.rb | 2 +- .../runtime/billing_events_controller_spec.rb | 2 +- .../runtime/events_controller_spec.rb | 6 +++--- .../services/snapshots_controller_spec.rb | 4 ++-- .../jobs/runtime/blobstore_upload_spec.rb | 2 +- spec/unit/jobs/runtime/droplet_upload_spec.rb | 2 +- .../jobs/runtime/failed_jobs_cleanup_spec.rb | 6 +++--- .../runtime/pending_packages_cleanup_spec.rb | 8 ++++---- .../cloud_controller/backends/runners_spec.rb | 4 ++-- .../cloud_controller/blobstore/blob_spec.rb | 2 +- .../lib/cloud_controller/dea/client_spec.rb | 8 ++++---- .../nats_messages/dea_advertisment_spec.rb | 8 ++++---- .../nats_messages/stager_advertisment_spec.rb | 8 ++++---- .../cloud_controller/dea/respondent_spec.rb | 4 ++-- .../lib/cloud_controller/diagnostics_spec.rb | 6 +++--- .../diego/service_registry_spec.rb | 2 +- ...multi_response_message_bus_request_spec.rb | 4 ++-- spec/unit/lib/vcap/rest_api/query_spec.rb | 4 ++-- spec/unit/lib/vcap/uaa_token_decoder_spec.rb | 20 +++++++++---------- spec/unit/models/runtime/app_spec.rb | 4 +++- .../unit/models/runtime/billing_event_spec.rb | 8 +++++--- spec/unit/models/runtime/event_spec.rb | 2 +- .../unit/presenters/api/job_presenter_spec.rb | 6 +++--- .../api/staging_job_presenter_spec.rb | 2 +- .../app_usage_event_repository_spec.rb | 2 +- 57 files changed, 117 insertions(+), 109 deletions(-) diff --git a/app/controllers/runtime/billing_events_controller.rb b/app/controllers/runtime/billing_events_controller.rb index f03efcb18b5..91b306d91e7 100644 --- a/app/controllers/runtime/billing_events_controller.rb +++ b/app/controllers/runtime/billing_events_controller.rb @@ -38,7 +38,7 @@ def end_time def parse_date_param(param) str = @params[param] - Time.parse(str).localtime if str + Time.parse(str).utc if str rescue raise Errors::ApiError.new_from_details('BillingEventQueryInvalid') end diff --git a/app/jobs/runtime/pending_packages_cleanup.rb b/app/jobs/runtime/pending_packages_cleanup.rb index e5c1acde79d..af985198494 100644 --- a/app/jobs/runtime/pending_packages_cleanup.rb +++ b/app/jobs/runtime/pending_packages_cleanup.rb @@ -3,8 +3,7 @@ module Jobs module Runtime class PendingPackagesCleanup < Struct.new(:expiration_in_seconds) def perform - cutoff_time = Time.now - expiration_in_seconds - App.where('package_pending_since < ?', cutoff_time).update( + App.where("package_pending_since < ? - INTERVAL '?' SECOND", Sequel::CURRENT_TIMESTAMP, expiration_in_seconds.to_i).update( package_state: 'FAILED', staging_failed_reason: 'StagingTimeExpired', package_pending_since: nil, diff --git a/app/models/runtime/app.rb b/app/models/runtime/app.rb index 4ffca6ce136..780f5520268 100644 --- a/app/models/runtime/app.rb +++ b/app/models/runtime/app.rb @@ -428,7 +428,7 @@ def mark_as_failed_to_stage(reason='StagingError') def mark_for_restaging self.package_state = 'PENDING' self.staging_failed_reason = nil - self.package_pending_since = Time.now + self.package_pending_since = Sequel::CURRENT_TIMESTAMP end def buildpack diff --git a/app/models/runtime/app_start_event.rb b/app/models/runtime/app_start_event.rb index b5e936c8a43..f6ffe2cbc3e 100644 --- a/app/models/runtime/app_start_event.rb +++ b/app/models/runtime/app_start_event.rb @@ -35,7 +35,7 @@ def event_type def self.create_from_app(app) return unless app.space.organization.billing_enabled? AppStartEvent.create( - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, organization_guid: app.space.organization_guid, organization_name: app.space.organization.name, space_guid: app.space.guid, diff --git a/app/models/runtime/app_stop_event.rb b/app/models/runtime/app_stop_event.rb index bd62c9f01ae..6661428cb3e 100644 --- a/app/models/runtime/app_stop_event.rb +++ b/app/models/runtime/app_stop_event.rb @@ -41,7 +41,7 @@ def create_from_app(app) end AppStopEvent.create( - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, organization_guid: app.space.organization_guid, organization_name: app.space.organization.name, space_guid: app.space.guid, diff --git a/app/models/runtime/organization_start_event.rb b/app/models/runtime/organization_start_event.rb index 79d8092e3a9..c996d80b100 100644 --- a/app/models/runtime/organization_start_event.rb +++ b/app/models/runtime/organization_start_event.rb @@ -16,7 +16,7 @@ def event_type def self.create_from_org(org) raise BillingNotEnabled unless org.billing_enabled? OrganizationStartEvent.create( - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, organization_guid: org.guid, organization_name: org.name, ) diff --git a/app/models/services/service_create_event.rb b/app/models/services/service_create_event.rb index ea2c6cccf26..36a4c19bd96 100644 --- a/app/models/services/service_create_event.rb +++ b/app/models/services/service_create_event.rb @@ -41,7 +41,7 @@ def self.create_from_service_instance(instance) return unless org.billing_enabled? ServiceCreateEvent.create( - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, organization_guid: org.guid, organization_name: org.name, space_guid: space.guid, diff --git a/app/models/services/service_delete_event.rb b/app/models/services/service_delete_event.rb index 47e7fb50101..5bdaa34181c 100644 --- a/app/models/services/service_delete_event.rb +++ b/app/models/services/service_delete_event.rb @@ -31,7 +31,7 @@ def self.create_from_service_instance(instance) return unless org.billing_enabled? ServiceDeleteEvent.create( - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, organization_guid: org.guid, organization_name: org.name, space_guid: space.guid, diff --git a/app/presenters/api/job_presenter.rb b/app/presenters/api/job_presenter.rb index 557a3d03e19..a6d84bfee02 100644 --- a/app/presenters/api/job_presenter.rb +++ b/app/presenters/api/job_presenter.rb @@ -93,11 +93,11 @@ def guid end def created_at - Time.at(0) + Time.at(0).utc end def run_at - Time.at(0) + Time.at(0).utc end def cf_api_error diff --git a/app/repositories/runtime/app_event_repository.rb b/app/repositories/runtime/app_event_repository.rb index 5cdedb354f0..49f63ea6eb7 100644 --- a/app/repositories/runtime/app_event_repository.rb +++ b/app/repositories/runtime/app_event_repository.rb @@ -73,7 +73,7 @@ def create_app_audit_event(type, app, space, actor, metadata) Event.create( space: space, type: type, - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, actee: app.guid, actee_type: 'app', actee_name: app.name, diff --git a/app/repositories/runtime/space_event_repository.rb b/app/repositories/runtime/space_event_repository.rb index 17f1d5e4c22..6cbc78928cd 100644 --- a/app/repositories/runtime/space_event_repository.rb +++ b/app/repositories/runtime/space_event_repository.rb @@ -12,7 +12,7 @@ def record_space_create(space, actor, actor_name, request_attrs) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, metadata: { request: request_attrs } @@ -29,7 +29,7 @@ def record_space_update(space, actor, actor_name, request_attrs) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, metadata: { request: request_attrs } @@ -45,7 +45,7 @@ def record_space_delete_request(space, actor, actor_name, recursive) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, space_guid: space.guid, organization_guid: space.organization.guid, metadata: { diff --git a/app/repositories/services/event_repository.rb b/app/repositories/services/event_repository.rb index a4d05034ad4..a9f0e18f9a2 100644 --- a/app/repositories/services/event_repository.rb +++ b/app/repositories/services/event_repository.rb @@ -214,7 +214,7 @@ def user_actor def create_event(type, actor_data, actee_data, metadata, space_data=nil) base_data = { type: type, - timestamp: Time.now, + timestamp: Sequel::CURRENT_TIMESTAMP, metadata: metadata } diff --git a/db/migrations/20130219194917_create_stacks_table.rb b/db/migrations/20130219194917_create_stacks_table.rb index eee64fbc9d5..5abc484329a 100644 --- a/db/migrations/20130219194917_create_stacks_table.rb +++ b/db/migrations/20130219194917_create_stacks_table.rb @@ -19,7 +19,7 @@ guid: SecureRandom.uuid, name: 'lucid64', description: 'Ubuntu 10.04 on x86-64', - created_at: Time.now, + created_at: Sequel::CURRENT_TIMESTAMP, ) alter_table :apps do diff --git a/lib/cloud_controller.rb b/lib/cloud_controller.rb index 8c69955412d..fa09b318460 100644 --- a/lib/cloud_controller.rb +++ b/lib/cloud_controller.rb @@ -21,6 +21,8 @@ require 'active_support/core_ext/object/to_query' require 'active_support/json/encoding' +Sequel.default_timezone = :utc + module VCAP::CloudController; end require 'vcap/errors/invalid_relation' diff --git a/lib/cloud_controller/blobstore/blob.rb b/lib/cloud_controller/blobstore/blob.rb index 2482604c88c..c480aaeadcc 100644 --- a/lib/cloud_controller/blobstore/blob.rb +++ b/lib/cloud_controller/blobstore/blob.rb @@ -36,7 +36,7 @@ def download_uri_for_file end if file.respond_to?(:url) - return file.url(Time.now + 3600) + return file.url(Time.now.utc + 3600) end file.public_url end diff --git a/lib/cloud_controller/blobstore/client.rb b/lib/cloud_controller/blobstore/client.rb index 801b880f4e9..366c4dfeee5 100644 --- a/lib/cloud_controller/blobstore/client.rb +++ b/lib/cloud_controller/blobstore/client.rb @@ -49,7 +49,7 @@ def cp_r_to_blobstore(source_dir) end def cp_to_blobstore(source_path, destination_key, retries=2) - start = Time.now + start = Time.now.utc logger.info('blobstore.cp-start', destination_key: destination_key, source_path: source_path, bucket: @directory_key) size = -1 log_entry = 'blobstore.cp-skip' @@ -78,7 +78,7 @@ def cp_to_blobstore(source_path, destination_key, retries=2) log_entry = 'blobstore.cp-finish' end - duration = Time.now - start + duration = Time.now.utc - start logger.info(log_entry, destination_key: destination_key, duration_seconds: duration, diff --git a/lib/cloud_controller/clock.rb b/lib/cloud_controller/clock.rb index 5d845ad3ad4..0bdeea24107 100644 --- a/lib/cloud_controller/clock.rb +++ b/lib/cloud_controller/clock.rb @@ -21,7 +21,7 @@ def start def schedule_cleanup(name, klass, at) Clockwork.every(1.day, "#{name}.cleanup.job", at: at) do |_| - @logger.info("Queueing #{klass} at #{Time.now}") + @logger.info("Queueing #{klass} at #{Time.now.utc}") cutoff_age_in_days = @config.fetch(name.to_sym).fetch(:cutoff_age_in_days) job = klass.new(cutoff_age_in_days) Jobs::Enqueuer.new(job, queue: 'cc-generic').enqueue @@ -32,7 +32,7 @@ def schedule_frequent_cleanup(name, klass) config = @config.fetch(name.to_sym) Clockwork.every(config.fetch(:frequency_in_seconds), "#{name}.cleanup.job") do |_| - @logger.info("Queueing #{klass} at #{Time.now}") + @logger.info("Queueing #{klass} at #{Time.now.utc}") expiration = config.fetch(:expiration_in_seconds) job = klass.new(expiration) Jobs::Enqueuer.new(job, queue: 'cc-generic').enqueue diff --git a/lib/cloud_controller/db.rb b/lib/cloud_controller/db.rb index fd4168c0651..d55f7f58d4c 100644 --- a/lib/cloud_controller/db.rb +++ b/lib/cloud_controller/db.rb @@ -33,6 +33,9 @@ def self.connect(opts, logger) if db.database_type == :mysql Sequel::MySQL.default_collate = 'utf8_bin' + db.run("SET time_zone = 'UTC'") + elsif db.database_type == :postgres + db.run("SET time zone 'UTC'") end db diff --git a/lib/cloud_controller/dea/client.rb b/lib/cloud_controller/dea/client.rb index 1496d52e81f..35d7028108d 100644 --- a/lib/cloud_controller/dea/client.rb +++ b/lib/cloud_controller/dea/client.rb @@ -90,7 +90,7 @@ def find_all_instances(app) unless all_instances[index] all_instances[index] = { state: 'DOWN', - since: Time.now.to_i, + since: Time.now.utc.to_i, } end end @@ -234,7 +234,7 @@ def find_stats(app) unless stats[index] stats[index] = { state: 'DOWN', - since: Time.now.to_i, + since: Time.now.utc.to_i, } end end diff --git a/lib/cloud_controller/dea/pool.rb b/lib/cloud_controller/dea/pool.rb index 3c1d4ece438..aafd1d91295 100644 --- a/lib/cloud_controller/dea/pool.rb +++ b/lib/cloud_controller/dea/pool.rb @@ -21,7 +21,7 @@ def register_subscriptions end def process_advertise_message(message) - advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.to_i + @advertise_timeout) + advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.utc.to_i + @advertise_timeout) mutex.synchronize do remove_advertisement_for_id(advertisement.dea_id) @@ -30,7 +30,7 @@ def process_advertise_message(message) end def process_shutdown_message(message) - fake_advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.to_i + @advertise_timeout) + fake_advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.utc.to_i + @advertise_timeout) mutex.synchronize do remove_advertisement_for_id(fake_advertisement.dea_id) @@ -69,7 +69,7 @@ def reserve_app_memory(dea_id, app_memory) attr_reader :message_bus def prune_stale_deas - now = Time.now.to_i + now = Time.now.utc.to_i @dea_advertisements.delete_if { |ad| ad.expired?(now) } end diff --git a/lib/cloud_controller/dea/stager_pool.rb b/lib/cloud_controller/dea/stager_pool.rb index dd5eda4926c..9aa2b50efca 100644 --- a/lib/cloud_controller/dea/stager_pool.rb +++ b/lib/cloud_controller/dea/stager_pool.rb @@ -14,7 +14,7 @@ def initialize(config, message_bus, blobstore_url_generator) end def process_advertise_message(msg) - advertisement = NatsMessages::StagerAdvertisement.new(msg, Time.now.to_i + @advertise_timeout) + advertisement = NatsMessages::StagerAdvertisement.new(msg, Time.now.utc.to_i + @advertise_timeout) publish_buildpacks unless stager_in_pool?(advertisement.stager_id) mutex.synchronize do @@ -68,7 +68,7 @@ def top_5_stagers_for(memory, disk, stack) end def prune_stale_advertisements - now = Time.now.to_i + now = Time.now.utc.to_i @stager_advertisements.delete_if { |ad| ad.expired?(now) } end diff --git a/lib/cloud_controller/diagnostics.rb b/lib/cloud_controller/diagnostics.rb index ba96194d868..835d8fcef00 100644 --- a/lib/cloud_controller/diagnostics.rb +++ b/lib/cloud_controller/diagnostics.rb @@ -2,7 +2,7 @@ module VCAP::CloudController class Diagnostics def self.collect(output_directory) data = { - time: Time.now, + time: Time.now.utc, threads: thread_data, varz: varz_data } @@ -27,7 +27,7 @@ def self.request_complete def self.request_info(request) { - start_time: Time.now.to_f, + start_time: Time.now.utc.to_f, request_id: ::VCAP::Request.current_id, request_method: request.request_method, request_uri: request_uri(request) @@ -57,7 +57,7 @@ def self.varz_data end def self.output_file_name - Time.now.strftime("diag-#{Process.pid}-%Y%m%d-%H:%M:%S.%L.json") + Time.now.utc.strftime("diag-#{Process.pid}-%Y%m%d-%H:%M:%S.%L.json") end end end diff --git a/lib/cloud_controller/diego/instances_reporter.rb b/lib/cloud_controller/diego/instances_reporter.rb index 097235dcc74..f007b6dee2e 100644 --- a/lib/cloud_controller/diego/instances_reporter.rb +++ b/lib/cloud_controller/diego/instances_reporter.rb @@ -106,7 +106,7 @@ def fill_unreported_instances_with_down_instances(reported_instances, app) unless reported_instances[i] reported_instances[i] = { state: 'DOWN', - since: Time.now.to_i, + since: Time.now.utc.to_i, } end end diff --git a/lib/cloud_controller/diego/service_registry.rb b/lib/cloud_controller/diego/service_registry.rb index b295da922ba..4ad8ed4fa1a 100644 --- a/lib/cloud_controller/diego/service_registry.rb +++ b/lib/cloud_controller/diego/service_registry.rb @@ -25,12 +25,12 @@ def tps_addrs attr_reader :message_bus def set_tps_addr(guid, addr, ttl) - expires_at = Time.now + ttl + expires_at = Time.now.utc + ttl tps_services[guid] = { addr: addr, expires_at: expires_at } end def expire_tps_addrs - now = Time.now + now = Time.now.utc tps_services.select! { |_, val| val[:expires_at] > now } end diff --git a/lib/cloud_controller/resource_pool.rb b/lib/cloud_controller/resource_pool.rb index 2787d434b10..59196652889 100644 --- a/lib/cloud_controller/resource_pool.rb +++ b/lib/cloud_controller/resource_pool.rb @@ -125,7 +125,7 @@ def overwrite_destination_with!(descriptor, destination) logger.debug 'resource_pool.download.starting', destination: destination - start = Time.now + start = Time.now.utc if @cdn && @cdn[:uri] logger.debug 'resource_pool.download.using-cdn' @@ -146,7 +146,7 @@ def overwrite_destination_with!(descriptor, destination) end end - took = Time.now - start + took = Time.now.utc - start logger.debug 'resource_pool.download.complete', took: took, destination: destination end diff --git a/lib/vcap/rest_api/query.rb b/lib/vcap/rest_api/query.rb index 7e918ba2190..7d7b8734e7d 100644 --- a/lib/vcap/rest_api/query.rb +++ b/lib/vcap/rest_api/query.rb @@ -169,7 +169,7 @@ def clean_up_boolean(q_key, q_val) end def clean_up_datetime(q_val) - q_val.empty? ? nil : Time.parse(q_val).localtime + q_val.empty? ? nil : Time.parse(q_val).utc end def clean_up_integer(q_val) diff --git a/lib/vcap/uaa_token_decoder.rb b/lib/vcap/uaa_token_decoder.rb index 2e58de27267..66c20c6f319 100644 --- a/lib/vcap/uaa_token_decoder.rb +++ b/lib/vcap/uaa_token_decoder.rb @@ -59,9 +59,9 @@ def decode_token_with_asymmetric_key(auth_token) def decode_token_with_key(auth_token, options) options = { audience_ids: config[:resource_id] }.merge(options) - token = CF::UAA::TokenCoder.new(options).decode_at_reference_time(auth_token, Time.now.to_i - @grace_period_in_seconds) + token = CF::UAA::TokenCoder.new(options).decode_at_reference_time(auth_token, Time.now.utc.to_i - @grace_period_in_seconds) expiration_time = token['exp'] || token[:exp] - if expiration_time && expiration_time < Time.now.to_i + if expiration_time && expiration_time < Time.now.utc.to_i @logger.warn("token currently expired but accepted within grace period of #{@grace_period_in_seconds} seconds") end token diff --git a/spec/support/fakes/blueprints.rb b/spec/support/fakes/blueprints.rb index a7db1d0c16c..9eeb5ca2126 100644 --- a/spec/support/fakes/blueprints.rb +++ b/spec/support/fakes/blueprints.rb @@ -201,13 +201,13 @@ module VCAP::CloudController end BillingEvent.blueprint do - timestamp { Time.now } + timestamp { Time.now.utc } organization_guid { Sham.guid } organization_name { Sham.name } end Event.blueprint do - timestamp { Time.now } + timestamp { Time.now.utc } type { Sham.name } actor { Sham.guid } actor_type { Sham.name } @@ -249,7 +249,7 @@ module VCAP::CloudController instance_index { Sham.instance_index } exit_status { Random.rand(256) } exit_description { Sham.description } - timestamp { Time.now } + timestamp { Time.now.utc } end ServiceCreateEvent.blueprint do diff --git a/spec/support/integration/http.rb b/spec/support/integration/http.rb index 0c4fd901984..f9e71c61b89 100644 --- a/spec/support/integration/http.rb +++ b/spec/support/integration/http.rb @@ -5,7 +5,7 @@ module IntegrationHttp def admin_token token = { 'aud' => 'cloud_controller', - 'exp' => Time.now.to_i + 10_000, + 'exp' => Time.now.utc.to_i + 10_000, 'client_id' => Sham.guid, 'scope' => ['cloud_controller.admin'], } diff --git a/spec/unit/controllers/base/model_controller_spec.rb b/spec/unit/controllers/base/model_controller_spec.rb index 50e782aca2d..30eb9d18257 100644 --- a/spec/unit/controllers/base/model_controller_spec.rb +++ b/spec/unit/controllers/base/model_controller_spec.rb @@ -361,7 +361,7 @@ def run_delayed_job end describe '#enumerate' do - let(:timestamp) { Time.now.change(usec: 0) } + let(:timestamp) { Time.now.utc.change(usec: 0) } let!(:model1) { TestModel.make(created_at: timestamp) } let!(:model2) { TestModel.make(created_at: timestamp + 1.second) } let!(:model3) { TestModel.make(created_at: timestamp + 2.seconds) } diff --git a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb index cf4edb5e688..0abeccb1cee 100644 --- a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb +++ b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb @@ -283,7 +283,7 @@ def make_diego_app(options={}) end it 'does not return deleted apps' do - make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.now) + make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) get '/internal/bulk/apps', { 'batch_size' => App.count, diff --git a/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb b/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb index 2e92e6922e3..ae5a2247288 100644 --- a/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb +++ b/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb @@ -172,7 +172,7 @@ def make_request it 'succeeds' do headers = headers_for(user) - Timecop.travel(Time.now + 1.week + 100.seconds) do + Timecop.travel(Time.now.utc + 1.week + 100.seconds) do put "/v2/apps/#{app_obj.guid}/bits", req_body, headers end expect(last_response.status).to eq(201) @@ -183,7 +183,7 @@ def make_request it 'fails to authorize the upload' do headers = headers_for(user) - Timecop.travel(Time.now + 1.week + 10000.seconds) do + Timecop.travel(Time.now.utc + 1.week + 10000.seconds) do put "/v2/apps/#{app_obj.guid}/bits", req_body, headers end expect(last_response.status).to eq(401) diff --git a/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb b/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb index 0e1c1d4f05f..e214bb95d63 100644 --- a/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb +++ b/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb @@ -84,7 +84,7 @@ module VCAP::CloudController expect(last_response).to be_successful expect(AppUsageEvent.count).to eq(1) expect(AppUsageEvent.last).to match_app(app) - expect(AppUsageEvent.last.created_at).to be_within(5.seconds).of(Time.now) + expect(AppUsageEvent.last.created_at).to be_within(5.seconds).of(Time.now.utc) end it 'returns 403 as a non-admin' do diff --git a/spec/unit/controllers/runtime/billing_events_controller_spec.rb b/spec/unit/controllers/runtime/billing_events_controller_spec.rb index 5cca08ab364..90dd6554076 100644 --- a/spec/unit/controllers/runtime/billing_events_controller_spec.rb +++ b/spec/unit/controllers/runtime/billing_events_controller_spec.rb @@ -17,7 +17,7 @@ module VCAP::CloudController BillingEvent.plugin :scissors BillingEvent.delete - timestamp = Time.new(2012, 01, 01, 00, 00, 01) + timestamp = Time.new(2012, 01, 01, 00, 00, 01).utc @start_time = timestamp @org_event = OrganizationStartEvent.make( diff --git a/spec/unit/controllers/runtime/events_controller_spec.rb b/spec/unit/controllers/runtime/events_controller_spec.rb index e233eeddced..08929ac017d 100644 --- a/spec/unit/controllers/runtime/events_controller_spec.rb +++ b/spec/unit/controllers/runtime/events_controller_spec.rb @@ -33,9 +33,9 @@ module VCAP::CloudController describe 'default order' do it 'sorts by timestamp' do type = SecureRandom.uuid - Event.make(timestamp: Time.new(1990, 1, 1), type: type, actor: 'earlier') - Event.make(timestamp: Time.new(2000, 1, 1), type: type, actor: 'later') - Event.make(timestamp: Time.new(1995, 1, 1), type: type, actor: 'middle') + Event.make(timestamp: Time.new(1990, 1, 1).utc, type: type, actor: 'earlier') + Event.make(timestamp: Time.new(2000, 1, 1).utc, type: type, actor: 'later') + Event.make(timestamp: Time.new(1995, 1, 1).utc, type: type, actor: 'middle') get '/v2/events', {}, admin_headers parsed_body = MultiJson.load(last_response.body) diff --git a/spec/unit/controllers/services/snapshots_controller_spec.rb b/spec/unit/controllers/services/snapshots_controller_spec.rb index 6bf07cafd7b..b79e8714df6 100644 --- a/spec/unit/controllers/services/snapshots_controller_spec.rb +++ b/spec/unit/controllers/services/snapshots_controller_spec.rb @@ -29,7 +29,7 @@ module VCAP::CloudController describe 'POST', '/v2/snapshots' do let(:new_name) { 'new name' } - let(:snapshot_created_at) { Time.now.to_s } + let(:snapshot_created_at) { Time.now.utc.to_s } let(:new_snapshot) { VCAP::Services::Api::SnapshotV2.new(snapshot_id: '1', name: 'foo', state: 'empty', size: 0, created_time: snapshot_created_at) } let(:payload) { MultiJson.dump( @@ -157,7 +157,7 @@ module VCAP::CloudController end it 'returns an list of snapshots' do - created_time = Time.now.to_s + created_time = Time.now.utc.to_s expect(service_instance).to receive(:enum_snapshots) do [VCAP::Services::Api::SnapshotV2.new( 'snapshot_id' => '1234', diff --git a/spec/unit/jobs/runtime/blobstore_upload_spec.rb b/spec/unit/jobs/runtime/blobstore_upload_spec.rb index 833d82d7708..77bb022a9b1 100644 --- a/spec/unit/jobs/runtime/blobstore_upload_spec.rb +++ b/spec/unit/jobs/runtime/blobstore_upload_spec.rb @@ -40,7 +40,7 @@ module Jobs::Runtime BlobstoreUpload.class_eval do def reschedule_at(_, _=nil) # induce the jobs to reschedule almost immediately instead of waiting around for the backoff algorithm - Time.now + Time.now.utc end end BlobstoreUpload.new(local_file.path, blobstore_key, blobstore_name) diff --git a/spec/unit/jobs/runtime/droplet_upload_spec.rb b/spec/unit/jobs/runtime/droplet_upload_spec.rb index 55aef470331..259ab00887b 100644 --- a/spec/unit/jobs/runtime/droplet_upload_spec.rb +++ b/spec/unit/jobs/runtime/droplet_upload_spec.rb @@ -74,7 +74,7 @@ module Jobs::Runtime DropletUpload.class_eval do def reschedule_at(_, _=nil) # induce the jobs to reschedule almost immediately instead of waiting around for the backoff algorithm - Time.now + Time.now.utc end end DropletUpload.new(local_file.path, app.id) diff --git a/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb b/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb index fc3f1db34e0..449d244ba7f 100644 --- a/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb @@ -42,7 +42,7 @@ def max_attempts end context 'non-failing jobs' do - let(:run_at) { Time.now + 1.day } + let(:run_at) { Time.now.utc + 1.day } let(:the_job) { SuccessJob.new } it 'the job is not removed' do @@ -53,7 +53,7 @@ def max_attempts end context 'failing jobs' do - let(:run_at) { Time.now - 1.day } + let(:run_at) { Time.now.utc - 1.day } let(:the_job) { FailingJob.new } context 'when younger than specified cut-off' do @@ -65,7 +65,7 @@ def max_attempts end context 'when older than specified cut-off' do - let(:run_at) { Time.now - 3.days } + let(:run_at) { Time.now.utc - 3.days } it 'removes the job' do expect { diff --git a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb index bcb3c9cc813..3d19e9b37ab 100644 --- a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb @@ -15,8 +15,8 @@ module Jobs::Runtime describe '#perform' do context 'with packages which have been pending for too long' do - let!(:app1) { AppFactory.make(package_pending_since: Time.now - expiration_in_seconds - 1.second) } - let!(:app2) { AppFactory.make(package_pending_since: Time.now - expiration_in_seconds - 2.second) } + let!(:app1) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 1.second) } + let!(:app2) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 2.second) } before do cleanup_job.perform @@ -41,8 +41,8 @@ module Jobs::Runtime end it "ignores apps that haven't been pending for too long" do - app1 = AppFactory.make(package_pending_since: Time.now - expiration_in_seconds + 1.second) - app2 = AppFactory.make(package_pending_since: Time.now - expiration_in_seconds + 2.second) + app1 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 1.second) + app2 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 2.second) cleanup_job.perform app1.reload diff --git a/spec/unit/lib/cloud_controller/backends/runners_spec.rb b/spec/unit/lib/cloud_controller/backends/runners_spec.rb index 026be6592fc..db414710bb7 100644 --- a/spec/unit/lib/cloud_controller/backends/runners_spec.rb +++ b/spec/unit/lib/cloud_controller/backends/runners_spec.rb @@ -245,7 +245,7 @@ def make_dea_app(options={}) end it 'does not return deleted apps' do - deleted_app = make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.current) + deleted_app = make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) batch = runners.diego_apps(100, 0) @@ -368,7 +368,7 @@ def make_dea_app(options={}) end it 'does not return deleted apps' do - deleted_app = make_dea_app(id: 6, state: 'STARTED', deleted_at: DateTime.current) + deleted_app = make_dea_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) batch = runners.dea_apps(100, 0) diff --git a/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb b/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb index d44849e4216..2395e3d589d 100644 --- a/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb @@ -36,7 +36,7 @@ module Blobstore it 'is valid for an hour' do Timecop.freeze do - now = Time.now + now = Time.now.utc expect(file).to receive(:url).with(now + 3600) blob.download_url end diff --git a/spec/unit/lib/cloud_controller/dea/client_spec.rb b/spec/unit/lib/cloud_controller/dea/client_spec.rb index 34c49bacf47..8b84f40dfed 100644 --- a/spec/unit/lib/cloud_controller/dea/client_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/client_spec.rb @@ -605,7 +605,7 @@ module VCAP::CloudController 'stats' => stats, } - allow(Time).to receive(:now) { 1 } + allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } message_bus.respond_to_synchronous_request('dea.find.droplet', [instance]) @@ -654,7 +654,7 @@ module VCAP::CloudController 'stats' => stats, } - allow(Time).to receive(:now).and_return(1) + allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } message_bus.respond_to_synchronous_request('dea.find.droplet', [instance_0, instance_1, instance_2]) @@ -781,7 +781,7 @@ module VCAP::CloudController with(app, search_options, { expected: 2 }). and_return([starting_instance, running_instance]) - allow(Time).to receive(:now) { 1 } + allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } app_instances = Dea::Client.find_all_instances(app) expect(app_instances).to eq({ @@ -811,7 +811,7 @@ module VCAP::CloudController with(app, search_options, { expected: 2 }). and_return([]) - allow(Time).to receive(:now) { 1 } + allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } app_instances = Dea::Client.find_all_instances(app) expect(app_instances).to eq({ diff --git a/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb b/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb index f0c2b459051..9a3e49ac3e7 100644 --- a/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb @@ -16,7 +16,7 @@ module Dea::NatsMessages } } end - let(:expires) { Time.now.to_i + 10 } + let(:expires) { Time.now.utc.to_i + 10 } subject(:ad) { DeaAdvertisement.new(message, expires) } @@ -37,13 +37,13 @@ module Dea::NatsMessages end describe '#expired?' do - let(:now) { Time.now } + let(:now) { Time.now.utc } context 'when the time since the advertisment is greater than or equal 10 seconds' do it 'returns true' do Timecop.freeze now do ad Timecop.travel now + 11.seconds do - expect(ad).to be_expired(Time.now) + expect(ad).to be_expired(Time.now.utc) end end end @@ -54,7 +54,7 @@ module Dea::NatsMessages Timecop.freeze now do ad Timecop.travel now + 9.seconds do - expect(ad).to_not be_expired(Time.now) + expect(ad).to_not be_expired(Time.now.utc) end end end diff --git a/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb b/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb index d5e079777dc..1563adae8ba 100644 --- a/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb @@ -11,7 +11,7 @@ module Dea::NatsMessages 'available_memory' => 1024, } end - let(:expires) { Time.now.to_i + 10 } + let(:expires) { Time.now.utc.to_i + 10 } subject(:ad) { StagerAdvertisement.new(message, expires) } @@ -28,13 +28,13 @@ module Dea::NatsMessages end describe '#expired?' do - let(:now) { Time.now } + let(:now) { Time.now.utc } context 'when the time since the advertisment is greater than or equal to 10 seconds' do it 'returns true' do Timecop.freeze now do ad Timecop.freeze now + 10.seconds do - expect(ad).to be_expired(Time.now) + expect(ad).to be_expired(Time.now.utc) end end end @@ -45,7 +45,7 @@ module Dea::NatsMessages Timecop.freeze now do ad Timecop.freeze now + 9.seconds do - expect(ad).to_not be_expired(Time.now) + expect(ad).to_not be_expired(Time.now.utc) end end end diff --git a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb index 569c15be024..f080017b1d9 100644 --- a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb @@ -51,8 +51,8 @@ module VCAP::CloudController context 'when the app crashed' do context 'the app described in the event exists' do it 'adds a record in the Events table' do - time = Time.now - Timecop.freeze(time) do + Timecop.freeze do + time = Time.now.utc respondent.process_droplet_exited_message(payload) app_event = Event.find(actee: app.guid) diff --git a/spec/unit/lib/cloud_controller/diagnostics_spec.rb b/spec/unit/lib/cloud_controller/diagnostics_spec.rb index c07c4164e7e..8640181f349 100644 --- a/spec/unit/lib/cloud_controller/diagnostics_spec.rb +++ b/spec/unit/lib/cloud_controller/diagnostics_spec.rb @@ -34,7 +34,7 @@ def current_request end it 'populates the start time to now' do - now = Time.now + now = Time.now.utc Timecop.freeze now do expect(current_request[:start_time]).to be_within(0.01).of(now.to_f) end @@ -97,9 +97,9 @@ def current_request describe 'file name' do it 'uses a file name that includes a time stamp' do - Timecop.freeze Time.now do + Timecop.freeze Time.now.utc do filename = Diagnostics.collect(output_dir) - timestamp = Time.now.strftime('%Y%m%d-%H:%M:%S.%L') + timestamp = Time.now.utc.strftime('%Y%m%d-%H:%M:%S.%L') expect(filename).to match_regex(/#{timestamp}/) end end diff --git a/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb b/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb index 94c67cfd201..d2c288e6765 100644 --- a/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb @@ -23,7 +23,7 @@ module VCAP::CloudController::Diego context 'when a broadcast message has expired' do before do - Timecop.travel(Time.now + 30) + Timecop.travel(Time.now.utc + 30) end it 'no longer contains the expired ip address' do diff --git a/spec/unit/lib/multi_response_message_bus_request_spec.rb b/spec/unit/lib/multi_response_message_bus_request_spec.rb index 2a521940f7f..149baa2a74b 100644 --- a/spec/unit/lib/multi_response_message_bus_request_spec.rb +++ b/spec/unit/lib/multi_response_message_bus_request_spec.rb @@ -172,7 +172,7 @@ end it 'cancels timeout' do - t = Time.now + t = Time.now.utc multi_response_message_bus_request.on_response(100) { |*args| raise 'Must never be called' } multi_response_message_bus_request.request({}) multi_response_message_bus_request.ignore_subsequent_responses @@ -180,7 +180,7 @@ # if timeout timer does not get cancelled # this test will take ~100s instead of less than 100s # (Use within 50s instead of 0.1s since system might be busy.) - expect(Time.now).to be_within(50).of(t) + expect(Time.now.utc).to be_within(50).of(t) end it 'raises error when request was not made' do diff --git a/spec/unit/lib/vcap/rest_api/query_spec.rb b/spec/unit/lib/vcap/rest_api/query_spec.rb index c754c1cd8c5..a7f8f765cba 100644 --- a/spec/unit/lib/vcap/rest_api/query_spec.rb +++ b/spec/unit/lib/vcap/rest_api/query_spec.rb @@ -28,13 +28,13 @@ class Subscriber < Sequel::Model a = Author.create(num_val: i + 1, str_val: "str #{i}", published: (i == 0), - published_at: (i == 0) ? nil : Time.at(0) + i) + published_at: (i == 0) ? nil : Time.at(0).utc + i) 2.times do |j| a.add_book(Book.create(num_val: j + 1, str_val: "str #{i} #{j}")) end end - @owner_nil_num = Author.create(str_val: 'no num', published: false, published_at: Time.at(0) + @num_authors) + @owner_nil_num = Author.create(str_val: 'no num', published: false, published_at: Time.at(0).utc + @num_authors) @queryable_attributes = Set.new(%w(num_val str_val author_id book_id published published_at)) end diff --git a/spec/unit/lib/vcap/uaa_token_decoder_spec.rb b/spec/unit/lib/vcap/uaa_token_decoder_spec.rb index 00e40b73caf..f1d61305e02 100644 --- a/spec/unit/lib/vcap/uaa_token_decoder_spec.rb +++ b/spec/unit/lib/vcap/uaa_token_decoder_spec.rb @@ -44,12 +44,12 @@ module VCAP end describe '#decode_token' do - before { Timecop.freeze(Time.now) } + before { Timecop.freeze(Time.now.utc) } after { Timecop.return } context 'when symmetric key is used' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } end before { config_hash[:symmetric_secret] = 'symmetric-key' } @@ -81,7 +81,7 @@ module VCAP context 'when token is valid' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } end it 'successfully decodes token and caches key' do @@ -118,7 +118,7 @@ module VCAP context 'when token has invalid audience' do let(:token_content) do - { 'aud' => 'invalid-audience', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } + { 'aud' => 'invalid-audience', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } end it 'raises an BadToken error' do @@ -131,7 +131,7 @@ module VCAP context 'when token has expired' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i } end it 'raises a BadToken error' do @@ -155,14 +155,14 @@ module VCAP subject { described_class.new(config_hash, 100) } let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i } end let(:token) { generate_token(rsa_key, token_content) } context 'and the token is currently expired but had not expired within the grace period' do it 'decodes the token and logs a warning about expiration within the grace period' do - token_content['exp'] = Time.now.to_i - 50 + token_content['exp'] = Time.now.utc.to_i - 50 expect(logger).to receive(:warn).with(/token currently expired but accepted within grace period of 100 seconds/i) expect(subject.decode_token("bearer #{token}")).to eq token_content end @@ -170,7 +170,7 @@ module VCAP context 'and the token expired outside of the grace period' do it 'raises and logs a warning about the expired token' do - token_content['exp'] = Time.now.to_i - 150 + token_content['exp'] = Time.now.utc.to_i - 150 expect(logger).to receive(:warn).with(/token expired/i) expect { subject.decode_token("bearer #{token}") @@ -182,14 +182,14 @@ module VCAP subject { described_class.new(config_hash, -10) } it 'sets the grace period to be 0 instead' do - token_content['exp'] = Time.now.to_i + token_content['exp'] = Time.now.utc.to_i expired_token = generate_token(rsa_key, token_content) allow(logger).to receive(:warn) expect { subject.decode_token("bearer #{expired_token}") }.to raise_error(VCAP::UaaTokenDecoder::BadToken) - token_content['exp'] = Time.now.to_i + 1 + token_content['exp'] = Time.now.utc.to_i + 1 valid_token = generate_token(rsa_key, token_content) expect(subject.decode_token("bearer #{valid_token}")).to eq token_content end diff --git a/spec/unit/models/runtime/app_spec.rb b/spec/unit/models/runtime/app_spec.rb index d2141a24104..9d3e173c8ff 100644 --- a/spec/unit/models/runtime/app_spec.rb +++ b/spec/unit/models/runtime/app_spec.rb @@ -1528,9 +1528,11 @@ def self.it_does_not_mark_for_re_staging it 'updates the package_pending_since date to current' do app.package_pending_since = nil + app.save expect { app.mark_for_restaging - }.to change { app.package_pending_since }.from(nil).to(kind_of(Time)) + app.save + }.to change { app.reload.package_pending_since }.from(nil).to(kind_of(Time)) end end diff --git a/spec/unit/models/runtime/billing_event_spec.rb b/spec/unit/models/runtime/billing_event_spec.rb index ab8ec151e2b..300d8461ad1 100644 --- a/spec/unit/models/runtime/billing_event_spec.rb +++ b/spec/unit/models/runtime/billing_event_spec.rb @@ -9,11 +9,13 @@ module VCAP::CloudController it { is_expected.to have_timestamp_columns } describe '.create' do - let(:values) { { - timestamp: Time.now, + let(:values) { + { + timestamp: Time.now.utc, organization_guid: 'abc', organization_name: 'def', - } } + } + } context 'when billing event writing is enabled' do before do diff --git a/spec/unit/models/runtime/event_spec.rb b/spec/unit/models/runtime/event_spec.rb index bee15a61541..a82bcabe908 100644 --- a/spec/unit/models/runtime/event_spec.rb +++ b/spec/unit/models/runtime/event_spec.rb @@ -13,7 +13,7 @@ module VCAP::CloudController actee: 'jtravolta', actee_type: 'Scientologist', actee_name: 'John Travolta', - timestamp: Time.new(1997, 6, 27), + timestamp: Time.new(1997, 6, 27).utc, metadata: { 'popcorn_price' => '$(arm + leg)' }, space: space ) diff --git a/spec/unit/presenters/api/job_presenter_spec.rb b/spec/unit/presenters/api/job_presenter_spec.rb index d26312c4a40..fb968fcd0d7 100644 --- a/spec/unit/presenters/api/job_presenter_spec.rb +++ b/spec/unit/presenters/api/job_presenter_spec.rb @@ -4,7 +4,7 @@ describe '#to_hash' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:run_at) { Time.now.months_since(1) } + allow(job).to receive(:run_at) { Time.now.utc.months_since(1) } job end @@ -30,7 +30,7 @@ context 'when the job has started' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:locked_at) { Time.now } + allow(job).to receive(:locked_at) { Time.now.utc } job end @@ -56,7 +56,7 @@ expect(JobPresenter.new(job).to_hash).to eq( metadata: { guid: '0', - created_at: Time.at(0).iso8601, + created_at: Time.at(0).utc.iso8601, url: '/v2/jobs/0' }, entity: { diff --git a/spec/unit/presenters/api/staging_job_presenter_spec.rb b/spec/unit/presenters/api/staging_job_presenter_spec.rb index e344f9969cd..0968ff6f2d5 100644 --- a/spec/unit/presenters/api/staging_job_presenter_spec.rb +++ b/spec/unit/presenters/api/staging_job_presenter_spec.rb @@ -5,7 +5,7 @@ describe '#to_hash' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:run_at) { Time.now.months_since(1) } + allow(job).to receive(:run_at) { Time.now.utc.months_since(1) } job end diff --git a/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb b/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb index 907c8e49209..47ac463ddc9 100644 --- a/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb +++ b/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb @@ -202,7 +202,7 @@ module Repositories::Runtime before do AppUsageEvent.dataset.delete - old = Time.now - 999.days + old = Time.now.utc - 999.days 3.times do event = repository.create_from_app(App.make) From b9a1b5289c3ead7698c8620f117e6f0f6b4dc11a Mon Sep 17 00:00:00 2001 From: "David Sabeti, Michael Maximilien and Whitney Schaefer" Date: Mon, 12 Jan 2015 17:17:09 -0800 Subject: [PATCH 42/76] Return state after requesting service instance provision [#62068908] --- .../services/managed_service_instance.rb | 3 +- ..._state_description_to_service_instances.rb | 6 +++ lib/services/service_brokers/v2/client.rb | 5 ++- .../service_instances_controller_spec.rb | 4 ++ .../service_brokers/v2/client_spec.rb | 37 +++++++++++++++++++ .../services/managed_service_instance_spec.rb | 2 +- 6 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 db/migrations/20150113000312_add_state_and_state_description_to_service_instances.rb diff --git a/app/models/services/managed_service_instance.rb b/app/models/services/managed_service_instance.rb index 51db1023e56..f4cb1256938 100644 --- a/app/models/services/managed_service_instance.rb +++ b/app/models/services/managed_service_instance.rb @@ -54,7 +54,8 @@ def do_request(method, payload=nil) many_to_one :service_plan export_attributes :name, :credentials, :service_plan_guid, - :space_guid, :gateway_data, :dashboard_url, :type + :space_guid, :gateway_data, :dashboard_url, :type, :state, + :state_description import_attributes :name, :service_plan_guid, :space_guid, :gateway_data diff --git a/db/migrations/20150113000312_add_state_and_state_description_to_service_instances.rb b/db/migrations/20150113000312_add_state_and_state_description_to_service_instances.rb new file mode 100644 index 00000000000..d01ac2b6c71 --- /dev/null +++ b/db/migrations/20150113000312_add_state_and_state_description_to_service_instances.rb @@ -0,0 +1,6 @@ +Sequel.migration do + change do + add_column :service_instances, :state, String + add_column :service_instances, :state_description, String, text: true + end +end diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 57627760dc8..0454420d5a1 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -3,7 +3,8 @@ class Client CATALOG_PATH = '/v2/catalog'.freeze def initialize(attrs) - @http_client = VCAP::Services::ServiceBrokers::V2::HttpClient.new(attrs) + http_client_attrs = attrs.select { |key, _| [:url, :auth_username, :auth_password].include?(key) } + @http_client = VCAP::Services::ServiceBrokers::V2::HttpClient.new(http_client_attrs) @response_parser = VCAP::Services::ServiceBrokers::V2::ResponseParser.new(@http_client.url) @attrs = attrs @orphan_mitigator = VCAP::Services::ServiceBrokers::V2::OrphanMitigator.new @@ -28,6 +29,8 @@ def provision(instance) parsed_response = @response_parser.parse(:put, path, response) instance.dashboard_url = parsed_response['dashboard_url'] + instance.state = parsed_response['state'] || 'available' + instance.state_description = parsed_response['state_description'] || '' # DEPRECATED, but needed because of not null constraint instance.credentials = {} diff --git a/spec/unit/controllers/services/service_instances_controller_spec.rb b/spec/unit/controllers/services/service_instances_controller_spec.rb index dc790081403..354f10fa58f 100644 --- a/spec/unit/controllers/services/service_instances_controller_spec.rb +++ b/spec/unit/controllers/services/service_instances_controller_spec.rb @@ -178,6 +178,8 @@ def self.user_sees_empty_enumerate(user_role, member_a_ivar, member_b_ivar) allow(client).to receive(:provision) do |instance| instance.credentials = '{}' instance.dashboard_url = 'the dashboard_url' + instance.state = 'creating' + instance.state_description = '' end allow(client).to receive(:deprovision) allow_any_instance_of(Service).to receive(:client).and_return(client) @@ -190,6 +192,8 @@ def self.user_sees_empty_enumerate(user_role, member_a_ivar, member_b_ivar) expect(instance.credentials).to eq('{}') expect(instance.dashboard_url).to eq('the dashboard_url') + expect(decoded_response['entity']['state']).to eq 'creating' + expect(decoded_response['entity']['state_description']).to eq '' end it 'creates a service audit event for creating the service instance' do diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 8204d2c56d7..41dc6733451 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -28,6 +28,14 @@ module VCAP::Services::ServiceBrokers::V2 allow(http_client).to receive(:url).and_return(service_broker.broker_url) end + describe '#initialize' do + it 'creates HttpClient with correct attrs' do + Client.new(client_attrs.merge(extra_arg: 'foo')) + + expect(HttpClient).to have_received(:new).with(client_attrs) + end + end + describe '#catalog' do let(:service_id) { Sham.guid } let(:service_name) { Sham.name } @@ -132,12 +140,41 @@ module VCAP::Services::ServiceBrokers::V2 expect(instance.dashboard_url).to eq('foo') end + it 'defaults the state to "available"' do + client.provision(instance) + + expect(instance.state).to eq('available') + end + + it 'leaves the description blank' do + client.provision(instance) + + expect(instance.state_description).to eq('') + end + it 'DEPRECATED, maintain for database not null contraint: sets the credentials on the instance' do client.provision(instance) expect(instance.credentials).to eq({}) end + context 'when the broker returns a state' do + let(:response_data) do + { + state: 'creating', + state_description: '10% done' + } + end + + it 'return immediately with the broker response' do + client = Client.new(client_attrs.merge(accept_unavailable: true)) + client.provision(instance) + + expect(instance.state).to eq('creating') + expect(instance.state_description).to eq('10% done') + end + end + context 'when provision fails' do let(:uri) { 'some-uri.com/v2/service_instances/some-guid' } let(:response) { double(:response, body: nil, message: nil) } diff --git a/spec/unit/models/services/managed_service_instance_spec.rb b/spec/unit/models/services/managed_service_instance_spec.rb index 255546f805b..9783d4143b8 100644 --- a/spec/unit/models/services/managed_service_instance_spec.rb +++ b/spec/unit/models/services/managed_service_instance_spec.rb @@ -63,7 +63,7 @@ module VCAP::CloudController end describe 'Serialization' do - it { is_expected.to export_attributes :name, :credentials, :service_plan_guid, :space_guid, :gateway_data, :dashboard_url, :type } + it { is_expected.to export_attributes :name, :credentials, :service_plan_guid, :space_guid, :gateway_data, :dashboard_url, :type, :state, :state_description } it { is_expected.to import_attributes :name, :service_plan_guid, :space_guid, :gateway_data } end From 3635820e536dc00406e40fd2d9337431d260b261 Mon Sep 17 00:00:00 2001 From: Matthew Sykes Date: Wed, 7 Jan 2015 16:08:30 -0500 Subject: [PATCH 43/76] Extract versioned_guid from app [#85459782] Signed-off-by: Michael Fraenkel --- app/models/runtime/app.rb | 4 --- lib/cloud_controller/diego/client.rb | 3 +- lib/cloud_controller/diego/common/protocol.rb | 2 +- lib/cloud_controller/diego/docker/protocol.rb | 3 +- lib/cloud_controller/diego/process_guid.rb | 21 ++++++++++++ .../diego/traditional/protocol.rb | 3 +- .../diego/common/protocol_spec.rb | 4 +-- .../diego/docker/protocol_spec.rb | 2 +- .../diego/process_guid_spec.rb | 34 +++++++++++++++++++ .../diego/traditional/protocol_spec.rb | 4 +-- spec/unit/models/runtime/app_spec.rb | 12 ------- 11 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 lib/cloud_controller/diego/process_guid.rb create mode 100644 spec/unit/lib/cloud_controller/diego/process_guid_spec.rb diff --git a/app/models/runtime/app.rb b/app/models/runtime/app.rb index 780f5520268..8d9255b9fc0 100644 --- a/app/models/runtime/app.rb +++ b/app/models/runtime/app.rb @@ -227,10 +227,6 @@ def desired_instances started? ? instances : 0 end - def versioned_guid - "#{guid}-#{version}" - end - def organization space && space.organization end diff --git a/lib/cloud_controller/diego/client.rb b/lib/cloud_controller/diego/client.rb index 6e4bda1306a..90cd1e95ca7 100644 --- a/lib/cloud_controller/diego/client.rb +++ b/lib/cloud_controller/diego/client.rb @@ -1,4 +1,5 @@ require 'cloud_controller/diego/unavailable' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController module Diego @@ -17,7 +18,7 @@ def lrp_instances(app) end address = @service_registry.tps_addrs.first - guid = app.versioned_guid + guid = ProcessGuid.from_app(app) uri = URI("#{address}/lrps/#{guid}") logger.info "Requesting lrp information for #{guid} from #{address}" diff --git a/lib/cloud_controller/diego/common/protocol.rb b/lib/cloud_controller/diego/common/protocol.rb index e8f903c4bcd..516bb8a1563 100644 --- a/lib/cloud_controller/diego/common/protocol.rb +++ b/lib/cloud_controller/diego/common/protocol.rb @@ -10,7 +10,7 @@ def stop_index_request(app, index) def stop_index_message(app, index) { - 'process_guid' => app.versioned_guid, + 'process_guid' => ProcessGuid.from_app(app), 'index' => index, } end diff --git a/lib/cloud_controller/diego/docker/protocol.rb b/lib/cloud_controller/diego/docker/protocol.rb index 7c38ac2aa8d..d56a9aae32b 100644 --- a/lib/cloud_controller/diego/docker/protocol.rb +++ b/lib/cloud_controller/diego/docker/protocol.rb @@ -1,4 +1,5 @@ require 'cloud_controller/diego/environment' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController module Diego @@ -35,7 +36,7 @@ def stop_staging_app_request(app, task_id) def desire_app_message(app) message = { - 'process_guid' => app.versioned_guid, + 'process_guid' => ProcessGuid.from_app(app), 'memory_mb' => app.memory, 'disk_mb' => app.disk_quota, 'file_descriptors' => app.file_descriptors, diff --git a/lib/cloud_controller/diego/process_guid.rb b/lib/cloud_controller/diego/process_guid.rb new file mode 100644 index 00000000000..765ded312de --- /dev/null +++ b/lib/cloud_controller/diego/process_guid.rb @@ -0,0 +1,21 @@ +module VCAP::CloudController + module Diego + class ProcessGuid + def self.from(app_guid, app_version) + "#{app_guid}-#{app_version}" + end + + def self.from_app(app) + from(app.guid, app.version) + end + + def self.app_guid(versioned_guid) + versioned_guid[0..35] + end + + def self.app_version(versioned_guid) + versioned_guid[37..-1] + end + end + end +end diff --git a/lib/cloud_controller/diego/traditional/protocol.rb b/lib/cloud_controller/diego/traditional/protocol.rb index 823d6b30002..39454300ba3 100644 --- a/lib/cloud_controller/diego/traditional/protocol.rb +++ b/lib/cloud_controller/diego/traditional/protocol.rb @@ -1,5 +1,6 @@ require 'cloud_controller/diego/traditional/buildpack_entry_generator' require 'cloud_controller/diego/environment' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController module Diego @@ -43,7 +44,7 @@ def stage_app_message(app, staging_timeout) def desire_app_message(app) message = { - 'process_guid' => app.versioned_guid, + 'process_guid' => ProcessGuid.from_app(app), 'memory_mb' => app.memory, 'disk_mb' => app.disk_quota, 'file_descriptors' => app.file_descriptors, diff --git a/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb index 5e283a18616..3ffd26e639a 100644 --- a/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb @@ -8,14 +8,14 @@ module Common subject(:protocol) { described_class.new } describe '#stop_index_request' do - let(:app) { double(:app, versioned_guid: 'versioned-guid') } + let(:app) { double(:app, { guid: 'guid', version: 'versioned' }) } it 'includes a subject and message for CfMessageBus::MessageBus#publish' do request = protocol.stop_index_request(app, 33) expect(request.size).to eq(2) expect(request.first).to eq('diego.stop.index') - expect(request.last).to match_json({ 'process_guid' => 'versioned-guid', 'index' => 33 }) + expect(request.last).to match_json({ 'process_guid' => 'guid-versioned', 'index' => 33 }) end end end diff --git a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb index 0a6df69ec99..3279b3a4b9f 100644 --- a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb @@ -71,7 +71,7 @@ module Docker it 'includes the fields needed to desire a Docker app' do expect(message).to eq({ - 'process_guid' => app.versioned_guid, + 'process_guid' => ProcessGuid.from_app(app), 'memory_mb' => app.memory, 'disk_mb' => app.disk_quota, 'file_descriptors' => app.file_descriptors, diff --git a/spec/unit/lib/cloud_controller/diego/process_guid_spec.rb b/spec/unit/lib/cloud_controller/diego/process_guid_spec.rb new file mode 100644 index 00000000000..974c4a0645c --- /dev/null +++ b/spec/unit/lib/cloud_controller/diego/process_guid_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' +require 'cloud_controller/diego/process_guid' + +module VCAP::CloudController::Diego + describe ProcessGuid do + let(:app) do + VCAP::CloudController::AppFactory.make + end + + describe 'process_guid' do + it 'returns the appropriate composed guid' do + expect(ProcessGuid.from(app.guid, app.version)).to eq("#{app.guid}-#{app.version}") + end + end + + describe 'from_app' do + it 'returns the appropriate versioned guid for the app' do + expect(ProcessGuid.from_app(app)).to eq("#{app.guid}-#{app.version}") + end + end + + describe 'app_guid' do + it 'it returns the app guid from the versioned guid' do + expect(ProcessGuid.app_guid(ProcessGuid.from_app(app))).to eq(app.guid) + end + end + + describe 'app_version' do + it 'it returns the app version from the versioned guid' do + expect(ProcessGuid.app_version(ProcessGuid.from_app(app))).to eq(app.version) + end + end + end +end diff --git a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb index 08c5b3c763b..c98217e62c1 100644 --- a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb @@ -89,7 +89,7 @@ module Traditional health_check_timeout: 444, memory: 555, stack: instance_double(Stack, name: 'fake-stack'), - versioned_guid: 'fake-versioned_guid', + version: 'version-guid', uris: ['fake-uris'], ) end @@ -114,7 +114,7 @@ module Traditional 'log_guid' => 'fake-guid', 'memory_mb' => 555, 'num_instances' => 111, - 'process_guid' => 'fake-versioned_guid', + 'process_guid' => 'fake-guid-version-guid', 'stack' => 'fake-stack', 'start_command' => 'the-custom-command', 'execution_metadata' => 'staging-metadata', diff --git a/spec/unit/models/runtime/app_spec.rb b/spec/unit/models/runtime/app_spec.rb index 9d3e173c8ff..ece11f528fd 100644 --- a/spec/unit/models/runtime/app_spec.rb +++ b/spec/unit/models/runtime/app_spec.rb @@ -1579,18 +1579,6 @@ def self.it_does_not_mark_for_re_staging end end - describe 'versioned_guid' do - before do - @app = App.new - @app.guid = 'appguid' - @app.version = 'versionuuid' - end - - it "is the app's guid qualified by its version" do - expect(@app.versioned_guid).to eq('appguid-versionuuid') - end - end - describe 'uris' do it 'should return the uris on the app' do app = AppFactory.make(space: space) From 47a642a854db6a4b961eb34e4ed8115e035bb4a6 Mon Sep 17 00:00:00 2001 From: Matthew Sykes Date: Thu, 8 Jan 2015 09:55:37 -0500 Subject: [PATCH 44/76] Add etag to desire app messages [#85459782] Signed-off-by: Michael Fraenkel --- lib/cloud_controller/diego/docker/protocol.rb | 1 + lib/cloud_controller/diego/traditional/protocol.rb | 1 + spec/unit/controllers/internal/bulk_apps_controller_spec.rb | 3 ++- spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb | 1 + spec/unit/lib/cloud_controller/diego/messenger_spec.rb | 1 + .../lib/cloud_controller/diego/traditional/protocol_spec.rb | 2 ++ 6 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/cloud_controller/diego/docker/protocol.rb b/lib/cloud_controller/diego/docker/protocol.rb index d56a9aae32b..229cb7a4f01 100644 --- a/lib/cloud_controller/diego/docker/protocol.rb +++ b/lib/cloud_controller/diego/docker/protocol.rb @@ -49,6 +49,7 @@ def desire_app_message(app) 'log_guid' => app.guid, 'docker_image' => app.docker_image, 'health_check_type' => app.health_check_type, + 'etag' => app.updated_at.to_f.to_s } message['health_check_timeout_in_seconds'] = app.health_check_timeout if app.health_check_timeout diff --git a/lib/cloud_controller/diego/traditional/protocol.rb b/lib/cloud_controller/diego/traditional/protocol.rb index 39454300ba3..c8ccc26db19 100644 --- a/lib/cloud_controller/diego/traditional/protocol.rb +++ b/lib/cloud_controller/diego/traditional/protocol.rb @@ -57,6 +57,7 @@ def desire_app_message(app) 'routes' => app.uris, 'log_guid' => app.guid, 'health_check_type' => app.health_check_type, + 'etag' => app.updated_at.to_f.to_s } message['health_check_timeout_in_seconds'] = app.health_check_timeout if app.health_check_timeout diff --git a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb index 0abeccb1cee..661caad6994 100644 --- a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb +++ b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb @@ -150,7 +150,7 @@ def make_diego_app(options={}) expect(decoded_response['apps'].size).to eq(6) last_response_app = decoded_response['apps'][5] - expect(last_response_app.except('environment')).to match_object({ + expect(last_response_app.except('environment', 'etag')).to match_object({ 'disk_mb' => 1_024, 'file_descriptors' => 16_384, 'num_instances' => 4, @@ -164,6 +164,7 @@ def make_diego_app(options={}) 'execution_metadata' => '', 'health_check_type' => 'port', }) + expect(last_response_app['etag']).to_not be_nil last_response_app_env = last_response_app['environment'] expect(last_response_app_env).to(be_any) { |e| e['name'] == 'VCAP_APPLICATION' } diff --git a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb index 3279b3a4b9f..30301eace31 100644 --- a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb @@ -84,6 +84,7 @@ module Docker 'log_guid' => app.guid, 'docker_image' => app.docker_image, 'health_check_type' => app.health_check_type, + 'etag' => app.updated_at.to_f.to_s, }) end diff --git a/spec/unit/lib/cloud_controller/diego/messenger_spec.rb b/spec/unit/lib/cloud_controller/diego/messenger_spec.rb index 597d5f293eb..6303d4a01cd 100644 --- a/spec/unit/lib/cloud_controller/diego/messenger_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/messenger_spec.rb @@ -77,6 +77,7 @@ module Diego 'health_check_type' => app.health_check_type, 'health_check_timeout_in_seconds' => 120, 'log_guid' => app.guid, + 'etag' => app.updated_at.to_f.to_s, } end diff --git a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb index c98217e62c1..4d3a1732105 100644 --- a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb @@ -90,6 +90,7 @@ module Traditional memory: 555, stack: instance_double(Stack, name: 'fake-stack'), version: 'version-guid', + updated_at: Time.at(12345.6789), uris: ['fake-uris'], ) end @@ -119,6 +120,7 @@ module Traditional 'start_command' => 'the-custom-command', 'execution_metadata' => 'staging-metadata', 'routes' => ['fake-uris'], + 'etag' => '12345.6789' ) end From 698b952f1724e800abedf3e103254fce67a24979 Mon Sep 17 00:00:00 2001 From: Matthew Sykes Date: Thu, 8 Jan 2015 09:58:42 -0500 Subject: [PATCH 45/76] Add new bulk endpoints for nsync * get with 'cache' format only returns process guid and etag * post with a list of process guids provides desire app messages for the specified process guids [#85459782] Signed-off-by: Michael Fraenkel --- .../internal/bulk_apps_controller.rb | 53 ++- lib/cloud_controller/backends/runners.rb | 31 ++ .../internal/bulk_apps_controller_spec.rb | 439 +++++++++++------- .../cloud_controller/backends/runners_spec.rb | 193 ++++++++ 4 files changed, 535 insertions(+), 181 deletions(-) diff --git a/app/controllers/internal/bulk_apps_controller.rb b/app/controllers/internal/bulk_apps_controller.rb index fbb9015893b..c6696532884 100644 --- a/app/controllers/internal/bulk_apps_controller.rb +++ b/app/controllers/internal/bulk_apps_controller.rb @@ -2,6 +2,7 @@ require 'controllers/base/base_controller' require 'cloud_controller/diego/client' require 'cloud_controller/internal_api' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController class BulkAppsController < RestController::BaseController @@ -21,21 +22,59 @@ def bulk_apps bulk_token = MultiJson.load(params.fetch('token')) last_id = Integer(bulk_token['id'] || 0) - dependency_locator = ::CloudController::DependencyLocator.instance - runners = dependency_locator.runners - - apps = runners.diego_apps(batch_size, last_id) - messages = apps.map { |app| runners.runner_for_app(app).desire_app_message } - id_for_next_token = apps.empty? ? nil : apps.last.id + if params['format'] == 'cache' + messages, id_for_token = bulk_cache_format(batch_size, last_id) + else + messages, id_for_token = bulk_desire_app_format(batch_size, last_id) + end MultiJson.dump( apps: messages, - token: { 'id' => id_for_next_token } + token: { 'id' => id_for_token } ) rescue IndexError => e raise ApiError.new_from_details('BadQueryParameter', e.message) end get '/internal/bulk/apps', :bulk_apps + + def filtered_bulk_apps + raise ApiError.new_from_details('MessageParseError', 'Missing request body') if body.length == 0 + payload = MultiJson.load(body) + + apps = runners.diego_apps_from_process_guids(payload) + messages = apps.map { |app| runners.runner_for_app(app).desire_app_message } + + MultiJson.dump(messages) + rescue MultiJson::ParseError => e + raise ApiError.new_from_details('MessageParseError', e.message) + end + + post '/internal/bulk/apps', :filtered_bulk_apps + + private + + def bulk_desire_app_format(batch_size, last_id) + apps = runners.diego_apps(batch_size, last_id) + messages = apps.map { |app| runners.runner_for_app(app).desire_app_message } + id_for_next_token = apps.empty? ? nil : apps.last.id + + [messages, id_for_next_token] + end + + def bulk_cache_format(batch_size, last_id) + id_for_next_token = nil + messages = runners.diego_apps_cache_data(batch_size, last_id).map do |id, guid, version, updated| + id_for_next_token = id + { 'process_guid' => Diego::ProcessGuid.from(guid, version), 'etag' => updated.to_f.to_s } + end + + [messages, id_for_next_token] + end + + def runners + dependency_locator = ::CloudController::DependencyLocator.instance + @runners ||= dependency_locator.runners + end end end diff --git a/lib/cloud_controller/backends/runners.rb b/lib/cloud_controller/backends/runners.rb index 882fa0f39e4..fd8b221e7b2 100644 --- a/lib/cloud_controller/backends/runners.rb +++ b/lib/cloud_controller/backends/runners.rb @@ -1,5 +1,6 @@ require 'cloud_controller/dea/runner' require 'cloud_controller/diego/runner' +require 'cloud_controller/diego/process_guid' require 'cloud_controller/diego/traditional/protocol' require 'cloud_controller/diego/docker/protocol' require 'cloud_controller/diego/common/protocol' @@ -38,6 +39,36 @@ def diego_apps(batch_size, last_id) all end + def diego_apps_from_process_guids(process_guids) + return [] if diego_running_disabled? + + process_guids = Array(process_guids).to_set + App. + eager(:current_saved_droplet, :space, :stack, :service_bindings, { routes: :domain }). + where(guid: process_guids.map { |pg| Diego::ProcessGuid.app_guid(pg) }). + where('deleted_at IS NULL'). + where(state: 'STARTED'). + where(package_state: 'STAGED'). + where(diego: true). + order(:id). + all. + select { |app| process_guids.include?(Diego::ProcessGuid.from_app(app)) } + end + + def diego_apps_cache_data(batch_size, last_id) + return [] if diego_running_disabled? + + App.select(:id, :guid, :version, :updated_at). + where('id > ?', last_id). + where(state: 'STARTED'). + where(package_state: 'STAGED'). + where('deleted_at IS NULL'). + where(diego: true). + order(:id). + limit(batch_size). + select_map([:id, :guid, :version, :updated_at]) + end + def dea_apps(batch_size, last_id) query = App. where('id > ?', last_id). diff --git a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb index 661caad6994..90a169e92d6 100644 --- a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb +++ b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb @@ -1,47 +1,54 @@ require 'spec_helper' -require 'membrane' module VCAP::CloudController describe BulkAppsController do + def make_diego_app(options={}) + AppFactory.make(options).tap do |app| + app.environment_json = (app.environment_json || {}).merge('DIEGO_RUN_BETA' => 'true') + app.package_state = 'STAGED' + app.save + end + end + + def app_table_entry(index) + App.order_by(:id).all[index - 1] + end + + let(:runners) do + ::CloudController::DependencyLocator.instance.runners + end + before do allow_any_instance_of(::CloudController::Blobstore::UrlGenerator). to receive(:perma_droplet_download_url). and_return('http://blobsto.re/droplet') - end - before do @internal_user = 'internal_user' @internal_password = 'internal_password' + + 5.times { |i| make_diego_app(state: 'STARTED') } end describe 'GET', '/internal/bulk/apps' do - def make_diego_app(options={}) - AppFactory.make(options).tap do |app| - app.environment_json = (app.environment_json || {}).merge('DIEGO_RUN_BETA' => 'true') - app.package_state = 'STAGED' - app.save + context 'without credentials' do + it 'rejects the request as unauthorized' do + get '/internal/bulk/apps' + expect(last_response.status).to eq(401) end end - before do - 5.times do |i| - make_diego_app( - id: i + 1, - state: 'STARTED', - ) + context 'with invalid credentials' do + before do + authorize 'bar', 'foo' end - end - - it 'requires authentication' do - get '/internal/bulk/apps' - expect(last_response.status).to eq(401) - authorize 'bar', 'foo' - get '/internal/bulk/apps' - expect(last_response.status).to eq(401) + it 'rejects the request as unauthorized' do + get '/internal/bulk/apps' + expect(last_response.status).to eq(401) + end end - describe 'with authentication' do + context 'with valid credentials' do before do authorize @internal_user, @internal_password end @@ -56,11 +63,7 @@ def make_diego_app(options={}) context 'when diego_running is set to disabled' do before do - allow(Config.config).to receive(:[]).with(anything).and_call_original - allow(Config.config).to receive(:[]).with(:diego).and_return( - staging: 'optional', - running: 'disabled', - ) + TestConfig.override(diego: { staging: 'optional', running: 'disabled' }) end it 'returns an empty result' do @@ -76,11 +79,7 @@ def make_diego_app(options={}) context 'when diego_running is set to optional' do before do - allow(Config.config).to receive(:[]).with(anything).and_call_original - allow(Config.config).to receive(:[]).with(:diego).and_return( - staging: 'optional', - running: 'optional', - ) + TestConfig.override(diego: { staging: 'optional', running: 'optional' }) end it 'returns a populated token for the initial request (which has an empty bulk token)' do @@ -90,161 +89,152 @@ def make_diego_app(options={}) } expect(last_response.status).to eq(200) - expect(decoded_response['token']).to eq({ 'id' => 3 }) + expect(decoded_response['token']).to eq({ 'id' => app_table_entry(3).id }) end it 'returns apps in the response body' do get '/internal/bulk/apps', { 'batch_size' => 20, - 'token' => { id: 2 }.to_json, + 'token' => { id: app_table_entry(2).id }.to_json, } expect(last_response.status).to eq(200) expect(decoded_response['apps'].size).to eq(3) end - it 'returns apps that have the desired data' do - last_app = make_diego_app( - id: 6, - state: 'STARTED', - package_state: 'STAGED', - package_hash: 'package-hash', - disk_quota: 1_024, - environment_json: { + context 'when a format parameter is not specified' do + before do + app = make_diego_app( + state: 'STARTED', + package_state: 'STAGED', + package_hash: 'package-hash', + disk_quota: 1_024, + environment_json: { 'env-key-3' => 'env-value-3', 'env-key-4' => 'env-value-4', 'DIEGO_RUN_BETA' => 'true', - }, - file_descriptors: 16_384, - instances: 4, - memory: 1_024, - guid: 'app-guid-6', - command: 'start-command-6', - stack: Stack.make(name: 'stack-6'), - ) - - route1 = Route.make( - space: last_app.space, + }, + file_descriptors: 16_384, + instances: 4, + memory: 1_024, + guid: 'app-guid-6', + command: 'start-command-6', + stack: Stack.make(name: 'stack-6'), + ) + + route1 = Route.make( + space: app.space, host: 'arsenio', domain: SharedDomain.make(name: 'lo-mein.com'), - ) - last_app.add_route(route1) + ) + app.add_route(route1) - route2 = Route.make( - space: last_app.space, + route2 = Route.make( + space: app.space, host: 'conan', domain: SharedDomain.make(name: 'doe-mane.com'), - ) - last_app.add_route(route2) + ) + app.add_route(route2) - last_app.version = 'app-version-6' - last_app.save + app.version = 'app-version-6' + app.save + end - get '/internal/bulk/apps', { - 'batch_size' => 100, - 'token' => "{\"id\": 0 }", - } + it 'uses the desire app message format' do + get '/internal/bulk/apps', { + 'batch_size' => 100, + 'token' => { id: 0 }.to_json, + } - expect(last_response.status).to eq(200) + expect(last_response.status).to eq(200) + expect(decoded_response['apps'].size).to eq(6) + + last_response_app = decoded_response['apps'].last + last_app = app_table_entry(6) - expect(decoded_response['apps'].size).to eq(6) - - last_response_app = decoded_response['apps'][5] - expect(last_response_app.except('environment', 'etag')).to match_object({ - 'disk_mb' => 1_024, - 'file_descriptors' => 16_384, - 'num_instances' => 4, - 'log_guid' => 'app-guid-6', - 'memory_mb' => 1_024, - 'process_guid' => 'app-guid-6-app-version-6', - 'routes' => ['arsenio.lo-mein.com', 'conan.doe-mane.com'], - 'droplet_uri' => 'http://blobsto.re/droplet', - 'stack' => 'stack-6', - 'start_command' => 'start-command-6', - 'execution_metadata' => '', - 'health_check_type' => 'port', - }) - expect(last_response_app['etag']).to_not be_nil - - last_response_app_env = last_response_app['environment'] - expect(last_response_app_env).to(be_any) { |e| e['name'] == 'VCAP_APPLICATION' } - expect(last_response_app_env).to(be_any) { |e| e['name'] == 'MEMORY_LIMIT' } - expect(last_response_app_env).to(be_any) { |e| e['name'] == 'VCAP_SERVICES' } - expect(last_response_app_env.find { |e| e['name'] == 'env-key-3' }['value']).to eq('env-value-3') - expect(last_response_app_env.find { |e| e['name'] == 'env-key-4' }['value']).to eq('env-value-4') - expect(last_response_app_env.find { |e| e['name'] == 'DIEGO_RUN_BETA' }['value']).to eq('true') + expect(last_response_app).to eq(runners.runner_for_app(last_app).desire_app_message) + end end - it 'respects the batch_size parameter' do - [3, 5].each { |size| + context 'when a format=cache parameter is set' do + it 'uses the cache data format' do get '/internal/bulk/apps', { - 'batch_size' => size, - 'token' => "{\"id\":0}", - } + 'batch_size' => 1, + 'format' => 'cache', + 'token' => { id: 0 }.to_json, + } expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(size) - } - end - - it 'returns non-intersecting apps when token is supplied' do - get '/internal/bulk/apps', { - 'batch_size' => 2, - 'token' => "{\"id\":0}", - } + expect(decoded_response['apps'].size).to eq(1) - expect(last_response.status).to eq(200) + app = App.order(:id).first - saved_apps = decoded_response['apps'].dup - expect(saved_apps.size).to eq(2) + message = decoded_response['apps'][0] + expect(message).to match_object({ + 'process_guid' => "#{app.guid}-#{app.version}", + 'etag' => app.updated_at.to_f.to_s + }) + end + end - get '/internal/bulk/apps', { - 'batch_size' => 2, - 'token' => MultiJson.dump(decoded_response['token']), - } + context 'when there are unstaged apps' do + before do + app = make_diego_app(state: 'STARTED') + app.package_state = 'PENDING' + app.save + end - expect(last_response.status).to eq(200) + it 'only returns staged apps' do + get '/internal/bulk/apps', { + 'batch_size' => App.count, + 'token' => '{}', + } - new_apps = decoded_response['apps'].dup - expect(new_apps.size).to eq(2) - saved_apps.each do |saved_result| - expect(new_apps).not_to include(saved_result) + expect(last_response.status).to eq(200) + expect(decoded_response['apps'].size).to eq(App.count - 1) end end - it 'should eventually return entire collection, batch after batch' do - apps = [] - total_size = App.count + context 'when apps are not in the STARTED state' do + before do + make_diego_app(state: 'STOPPED') + end - token = '{}' - while apps.size < total_size + it 'does not return apps in the STOPPED state' do get '/internal/bulk/apps', { - 'batch_size' => 2, - 'token' => MultiJson.dump(token), - } + 'batch_size' => App.count, + 'token' => '{}', + } expect(last_response.status).to eq(200) - token = decoded_response['token'] - apps += decoded_response['apps'] + expect(decoded_response['apps'].size).to eq(App.count - 1) end + end - expect(apps.size).to eq(total_size) - get '/internal/bulk/apps', { - 'batch_size' => 2, - 'token' => MultiJson.dump(token), - } + context 'when there is a mixture of diego and traditional apps' do + before do + app = AppFactory.make + expect(app.diego).to be_falsey + end - expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(0) + it 'only returns diego apps' do + get '/internal/bulk/apps', { + 'batch_size' => App.count, + 'token' => '{}', + } + + expect(last_response.status).to eq(200) + expect(decoded_response['apps'].size).to eq(App.count - 1) + end end context 'when docker is enabled' do before do - allow(Config.config).to receive(:[]).with(:diego_docker).and_return true + TestConfig.override(diego_docker: true, diego: { staging: 'optional', running: 'optional' }) end it 'does return docker apps' do - app = make_diego_app(id: 6, state: 'STARTED', docker_image: 'fake-docker-image') + app = make_diego_app(state: 'STARTED', docker_image: 'fake-docker-image') app.save get '/internal/bulk/apps', { @@ -257,58 +247,159 @@ def make_diego_app(options={}) end end - it 'does not return unstaged apps' do - app = make_diego_app(id: 6, state: 'STARTED') - app.package_state = 'PENDING' - app.save + describe 'pagination' do + it 'respects the batch_size parameter' do + [3, 5].each { |size| + get '/internal/bulk/apps', { + 'batch_size' => size, + 'token' => { id: 0 }.to_json, + } - get '/internal/bulk/apps', { - 'batch_size' => App.count, - 'token' => '{}', - } + expect(last_response.status).to eq(200) + expect(decoded_response['apps'].size).to eq(size) + } + end - expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(App.count - 1) + it 'returns non-intersecting apps when token is supplied' do + get '/internal/bulk/apps', { + 'batch_size' => 2, + 'token' => { id: 0 }.to_json, + } + + expect(last_response.status).to eq(200) + + saved_apps = decoded_response['apps'].dup + expect(saved_apps.size).to eq(2) + + get '/internal/bulk/apps', { + 'batch_size' => 2, + 'token' => MultiJson.dump(decoded_response['token']), + } + + expect(last_response.status).to eq(200) + + new_apps = decoded_response['apps'].dup + expect(new_apps.size).to eq(2) + saved_apps.each do |saved_result| + expect(new_apps).not_to include(saved_result) + end + end + + it 'should eventually return entire collection, batch after batch' do + apps = [] + total_size = App.count + + token = '{}' + while apps.size < total_size + get '/internal/bulk/apps', { + 'batch_size' => 2, + 'token' => MultiJson.dump(token), + } + + expect(last_response.status).to eq(200) + token = decoded_response['token'] + apps += decoded_response['apps'] + end + + expect(apps.size).to eq(total_size) + get '/internal/bulk/apps', { + 'batch_size' => 2, + 'token' => MultiJson.dump(token), + } + + expect(last_response.status).to eq(200) + expect(decoded_response['apps'].size).to eq(0) + end end + end + end + end - it "does not return apps which aren't expected to be started" do - make_diego_app(id: 6, state: 'STOPPED') + describe 'POST' '/internal/bulk/apps' do + context 'without credentials' do + it 'rejects the request as unauthorized' do + post '/internal/bulk/apps', {} - get '/internal/bulk/apps', { - 'batch_size' => App.count, - 'token' => '{}', - } + expect(last_response.status).to eq(401) + end + end - expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(App.count - 1) + context 'with invalid credentials' do + before do + authorize 'bar', 'foo' + end + + it 'rejects the request as unauthorized' do + post '/internal/bulk/apps' + + expect(last_response.status).to eq(401) + end + end + + context 'with valid credentials' do + before do + authorize @internal_user, @internal_password + end + + context 'without a body' do + it 'is an invalid request' do + post '/internal/bulk/apps' + + expect(last_response.status).to eq(400) end + end - it 'does not return deleted apps' do - make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) + context 'with a body' do + context 'with invalid json' do + it 'is an invalid request' do + post '/internal/bulk/apps', 'foo' - get '/internal/bulk/apps', { - 'batch_size' => App.count, - 'token' => '{}', - } + expect(last_response.status).to eq(400) + end + end - expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(App.count - 1) + context 'with an empty list' do + it 'returns an empty list' do + post '/internal/bulk/apps', [].to_json + + expect(last_response.status).to eq(200) + expect(decoded_response).to eq([]) + end end - it 'only returns apps with DIEGO_RUN_BETA' do - make_diego_app(id: 6, state: 'STARTED').tap do |a| - a.environment_json = {} # remove DIEGO_RUN_BETA - a.save + context 'with a list of process guids' do + it 'returns a list of desire app messages that match the process guids' do + diego_apps = runners.diego_apps(100, 0) + + guids = diego_apps.map { |app| Diego::ProcessGuid.from_app(app) } + post '/internal/bulk/apps', guids.to_json + + expect(last_response.status).to eq(200) + expect(decoded_response.length).to eq(5) + + diego_apps.each do |app| + expect(decoded_response).to include(runners.runner_for_app(app).desire_app_message) + end end - get '/internal/bulk/apps', { - 'batch_size' => App.count, - 'token' => '{}', - } + context 'when there is a mixture of diego and traditional apps' do + before do + 5.times { AppFactory.make } + end - expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(App.count - 1) + it 'only returns the diego apps' do + diego_apps = runners.diego_apps(100, 0) + + guids = App.all.map { |app| Diego::ProcessGuid.from_app(app) } + post '/internal/bulk/apps', guids.to_json + + expect(last_response.status).to eq(200) + expect(decoded_response.length).to eq(diego_apps.length) + end + end end + + # validate max batch size; reject requests that are too large end end end diff --git a/spec/unit/lib/cloud_controller/backends/runners_spec.rb b/spec/unit/lib/cloud_controller/backends/runners_spec.rb index db414710bb7..a34a211182f 100644 --- a/spec/unit/lib/cloud_controller/backends/runners_spec.rb +++ b/spec/unit/lib/cloud_controller/backends/runners_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController describe Runners do @@ -294,6 +295,198 @@ def make_dea_app(options={}) end end + describe '#diego_apps_from_process_guids' do + before do + 5.times do + app = make_diego_app(state: 'STARTED') + app.add_route(Route.make(space: app.space)) + end + + expect(App.all.length).to eq(5) + end + + it 'does not return unstaged apps' do + unstaged_app = make_diego_app(state: 'STARTED') + unstaged_app.package_state = 'PENDING' + unstaged_app.save + + batch = runners.diego_apps_from_process_guids(Diego::ProcessGuid.from_app(unstaged_app)) + + expect(batch).not_to include(unstaged_app) + end + + it 'does not return apps that are stopped' do + stopped_app = make_diego_app(state: 'STOPPED') + + batch = runners.diego_apps_from_process_guids(Diego::ProcessGuid.from_app(stopped_app)) + + expect(batch).not_to include(stopped_app) + end + + it 'does not return deleted apps' do + deleted_app = make_diego_app(state: 'STARTED', deleted_at: DateTime.now.utc) + + batch = runners.diego_apps_from_process_guids(Diego::ProcessGuid.from_app(deleted_app)) + + expect(batch).not_to include(deleted_app) + end + + it 'only includes diego apps' do + non_diego_app = make_diego_app(state: 'STARTED') + non_diego_app.environment_json = {} + non_diego_app.save + + batch = runners.diego_apps_from_process_guids(Diego::ProcessGuid.from_app(non_diego_app)) + + expect(batch).not_to include(non_diego_app) + end + + it 'accepts a process guid or an array of process guids' do + app = App.where(diego: true).order(:id).first + process_guid = Diego::ProcessGuid.from_app(app) + + expect(runners.diego_apps_from_process_guids(process_guid)).to eq([app]) + expect(runners.diego_apps_from_process_guids([process_guid])).to eq([app]) + end + + it 'returns diego apps for each requested process guid' do + diego_apps = App.where(diego: true).all + diego_guids = diego_apps.map { |app| Diego::ProcessGuid.from_app(app) } + + expect(runners.diego_apps_from_process_guids(diego_guids)).to match_array(diego_apps) + end + + it 'loads all of the associations eagerly' do + diego_apps = App.where(diego: true).all + diego_guids = diego_apps.map { |app| Diego::ProcessGuid.from_app(app) } + + expect { + runners.diego_apps_from_process_guids(diego_guids).each do |app| + app.current_droplet + app.space + app.stack + app.routes + app.service_bindings + app.routes.map(&:domain) + end + }.to have_queried_db_times(/SELECT/, [ + :apps, + :droplets, + :spaces, + :stacks, + :routes, + :service_bindings, + :domain + ].length) + end + + context 'when the process guid is not found' do + it 'does not return an app' do + app = App.where(diego: true).order(:id).first + process_guid = Diego::ProcessGuid.from_app(app) + + expect { + app.set_new_version + app.save + }.to change { + runners.diego_apps_from_process_guids(process_guid) + }.from([app]).to([]) + end + end + + context 'when diego running is disabled' do + before do + allow(runners).to receive(:diego_running_disabled?).and_return(true) + end + + it 'returns no apps' do + all_process_guids = App.all.map { |app| Diego::ProcessGuid.from_app(app) } + + expect(runners.diego_apps_from_process_guids(all_process_guids)).to be_empty + end + end + end + + describe '#diego_apps_cache_data' do + before do + 5.times { make_diego_app(state: 'STARTED') } + expect(App.all.length).to eq(5) + end + + it 'respects the batch_size' do + data_count = [3, 5].map do |batch_size| + runners.diego_apps_cache_data(batch_size, 0).count + end + + expect(data_count).to eq([3, 5]) + end + + it 'returns data for non-intersecting apps across subsequent batches' do + first_batch = runners.diego_apps_cache_data(3, 0) + expect(first_batch.count).to eq(3) + + last_id = first_batch.last[0] + second_batch = runners.diego_apps_cache_data(3, last_id) + expect(second_batch.count).to eq(2) + end + + it 'does not return unstaged apps' do + unstaged_app = make_diego_app(state: 'STARTED') + unstaged_app.package_state = 'PENDING' + unstaged_app.save + + batch = runners.diego_apps_cache_data(100, 0) + app_ids = batch.map { |data| data[0] } + + expect(app_ids).not_to include(unstaged_app.id) + end + + it 'does not return apps that are stopped' do + stopped_app = make_diego_app(state: 'STOPPED') + + batch = runners.diego_apps_cache_data(100, 0) + app_ids = batch.map { |data| data[0] } + + expect(app_ids).not_to include(stopped_app.id) + end + + it 'does not return deleted apps' do + deleted_app = make_diego_app(state: 'STARTED', deleted_at: DateTime.now.utc) + + batch = runners.diego_apps_cache_data(100, 0) + app_ids = batch.map { |data| data[0] } + + expect(app_ids).not_to include(deleted_app.id) + end + + it 'only includes diego apps' do + non_diego_app = make_diego_app(state: 'STARTED') + non_diego_app.environment_json = {} + non_diego_app.save + + batch = runners.diego_apps_cache_data(100, 0) + app_ids = batch.map { |data| data[0] } + + expect(app_ids).not_to include(non_diego_app.id) + end + + context 'when diego running is disabled' do + before do + allow(runners).to receive(:diego_running_disabled?).and_return(true) + end + + it 'returns no apps' do + expect(runners.diego_apps_cache_data(100, 0)).to be_empty + end + end + + it 'acquires the data in one select' do + expect { + runners.diego_apps_cache_data(100, 0) + }.to have_queried_db_times(/SELECT/, 1) + end + end + describe '#dea_apps' do let!(:diego_app) { make_diego_app(id: 99, state: 'STARTED') } From 516782414ab3fe2b1b2c7572121a1ddf2a6b280f Mon Sep 17 00:00:00 2001 From: Matthew Sykes Date: Thu, 8 Jan 2015 10:48:02 -0500 Subject: [PATCH 46/76] Rename format to 'fingerprint' [#85459782] Signed-off-by: Michael Fraenkel --- .../internal/bulk_apps_controller.rb | 23 ++++++++++--------- .../internal/bulk_apps_controller_spec.rb | 9 ++++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/app/controllers/internal/bulk_apps_controller.rb b/app/controllers/internal/bulk_apps_controller.rb index c6696532884..9c9fb66583d 100644 --- a/app/controllers/internal/bulk_apps_controller.rb +++ b/app/controllers/internal/bulk_apps_controller.rb @@ -22,16 +22,11 @@ def bulk_apps bulk_token = MultiJson.load(params.fetch('token')) last_id = Integer(bulk_token['id'] || 0) - if params['format'] == 'cache' - messages, id_for_token = bulk_cache_format(batch_size, last_id) + if params['format'] == 'fingerprint' + bulk_fingerprint_format(batch_size, last_id) else - messages, id_for_token = bulk_desire_app_format(batch_size, last_id) + bulk_desire_app_format(batch_size, last_id) end - - MultiJson.dump( - apps: messages, - token: { 'id' => id_for_token } - ) rescue IndexError => e raise ApiError.new_from_details('BadQueryParameter', e.message) end @@ -59,17 +54,23 @@ def bulk_desire_app_format(batch_size, last_id) messages = apps.map { |app| runners.runner_for_app(app).desire_app_message } id_for_next_token = apps.empty? ? nil : apps.last.id - [messages, id_for_next_token] + MultiJson.dump( + apps: messages, + token: { 'id' => id_for_next_token } + ) end - def bulk_cache_format(batch_size, last_id) + def bulk_fingerprint_format(batch_size, last_id) id_for_next_token = nil messages = runners.diego_apps_cache_data(batch_size, last_id).map do |id, guid, version, updated| id_for_next_token = id { 'process_guid' => Diego::ProcessGuid.from(guid, version), 'etag' => updated.to_f.to_s } end - [messages, id_for_next_token] + MultiJson.dump( + fingerprints: messages, + token: { 'id' => id_for_next_token } + ) end def runners diff --git a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb index 90a169e92d6..14e5dd21ad5 100644 --- a/spec/unit/controllers/internal/bulk_apps_controller_spec.rb +++ b/spec/unit/controllers/internal/bulk_apps_controller_spec.rb @@ -1,4 +1,5 @@ require 'spec_helper' +require 'cloud_controller/diego/process_guid' module VCAP::CloudController describe BulkAppsController do @@ -160,18 +161,18 @@ def app_table_entry(index) it 'uses the cache data format' do get '/internal/bulk/apps', { 'batch_size' => 1, - 'format' => 'cache', + 'format' => 'fingerprint', 'token' => { id: 0 }.to_json, } expect(last_response.status).to eq(200) - expect(decoded_response['apps'].size).to eq(1) + expect(decoded_response['fingerprints'].size).to eq(1) app = App.order(:id).first - message = decoded_response['apps'][0] + message = decoded_response['fingerprints'][0] expect(message).to match_object({ - 'process_guid' => "#{app.guid}-#{app.version}", + 'process_guid' => Diego::ProcessGuid.from_app(app), 'etag' => app.updated_at.to_f.to_s }) end From de41ab8145afc99b0ad32e67ad5dfcf145318374 Mon Sep 17 00:00:00 2001 From: JT Archie and Luan Santos Date: Tue, 13 Jan 2015 09:46:28 -0800 Subject: [PATCH 47/76] Fix flaky time dependent test [#82077856] --- .../cloud_controller/dea/respondent_spec.rb | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb index f080017b1d9..71f1d73f374 100644 --- a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb @@ -51,24 +51,23 @@ module VCAP::CloudController context 'when the app crashed' do context 'the app described in the event exists' do it 'adds a record in the Events table' do - Timecop.freeze do - time = Time.now.utc - respondent.process_droplet_exited_message(payload) + time = Time.new(1990, 07, 06) + stub_const("Sequel::CURRENT_TIMESTAMP", time) + respondent.process_droplet_exited_message(payload) - app_event = Event.find(actee: app.guid) + app_event = Event.find(actee: app.guid) - expect(app_event).to be - expect(app_event.space).to eq(app.space) - expect(app_event.type).to eq('app.crash') - expect(app_event.timestamp.to_i).to eq(time.to_i) - expect(app_event.actor_type).to eq('app') - expect(app_event.actor).to eq(app.guid) - expect(app_event.metadata['instance']).to eq(payload['instance']) - expect(app_event.metadata['index']).to eq(payload['index']) - expect(app_event.metadata['exit_status']).to eq(payload['exit_status']) - expect(app_event.metadata['exit_description']).to eq(payload['exit_description']) - expect(app_event.metadata['reason']).to eq(reason) - end + expect(app_event).to be + expect(app_event.space).to eq(app.space) + expect(app_event.type).to eq('app.crash') + expect(app_event.timestamp.to_i).to eq(time.to_i) + expect(app_event.actor_type).to eq('app') + expect(app_event.actor).to eq(app.guid) + expect(app_event.metadata['instance']).to eq(payload['instance']) + expect(app_event.metadata['index']).to eq(payload['index']) + expect(app_event.metadata['exit_status']).to eq(payload['exit_status']) + expect(app_event.metadata['exit_description']).to eq(payload['exit_description']) + expect(app_event.metadata['reason']).to eq(reason) end end end From 72a671314035dd1688f00cc6ebb6929bb2f374c2 Mon Sep 17 00:00:00 2001 From: JT Archie and Luan Santos Date: Tue, 13 Jan 2015 09:58:20 -0800 Subject: [PATCH 48/76] Fix rubocop offense --- spec/unit/lib/cloud_controller/dea/respondent_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb index 71f1d73f374..447486a9551 100644 --- a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb @@ -52,7 +52,7 @@ module VCAP::CloudController context 'the app described in the event exists' do it 'adds a record in the Events table' do time = Time.new(1990, 07, 06) - stub_const("Sequel::CURRENT_TIMESTAMP", time) + stub_const('Sequel::CURRENT_TIMESTAMP', time) respondent.process_droplet_exited_message(payload) app_event = Event.find(actee: app.guid) From 0cdb27eeaf0e7498530a9a6ca9d1cee4176d7430 Mon Sep 17 00:00:00 2001 From: JT Archie and Luan Santos Date: Tue, 13 Jan 2015 11:08:03 -0800 Subject: [PATCH 49/76] Fix incompatibility with MySQL 5.5 when setting timezone --- lib/cloud_controller/db.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cloud_controller/db.rb b/lib/cloud_controller/db.rb index d55f7f58d4c..87c8a0c8182 100644 --- a/lib/cloud_controller/db.rb +++ b/lib/cloud_controller/db.rb @@ -33,7 +33,7 @@ def self.connect(opts, logger) if db.database_type == :mysql Sequel::MySQL.default_collate = 'utf8_bin' - db.run("SET time_zone = 'UTC'") + db.run("SET time_zone = '+0:00'") elsif db.database_type == :postgres db.run("SET time zone 'UTC'") end From a73f8460dcc33d7bb124a64cd5401dea844de79c Mon Sep 17 00:00:00 2001 From: James Myers and Sujoy Basu Date: Tue, 13 Jan 2015 10:43:07 -0800 Subject: [PATCH 50/76] Upload api_docs after_success and only for the api_docs build [#81240534] --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e49e3ed329..bb0a7a1bf24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,13 +19,13 @@ before_script: script: - bundle exec rake $TASKS +after_success: + - "[ $TRAVIS_BRANCH == 'master' -a $UPLOAD_DOCUMENTATION == 'true' ] && travis-artifacts upload --path doc/api --target-path $TRAVIS_BUILD_ID && echo https://s3-us-east-1.amazonaws.com/cc-api-docs/$TRAVIS_BUILD_ID/index.html" + services: - mysql - postgresql -after_script: # and this only on success, fail the whole build if it fails - - "[ $TRAVIS_TEST_RESULT == 0 -a $TRAVIS_BRANCH == 'master' ] && travis-artifacts upload --path doc/api --target-path $TRAVIS_BUILD_ID && echo https://s3-us-east-1.amazonaws.com/cc-api-docs/$TRAVIS_BUILD_ID/index.html" - env: global: # API Docs @@ -39,5 +39,5 @@ env: - COVERAGE=true DB=postgres TASKS=spec:all - DB=mysql TASKS=spec:all - - DB=postgres TASKS=spec:api + - DB=postgres TASKS=spec:api UPLOAD_DOCUMENTATION=true - TASKS=rubocop From e3acff9fb6e84a58ab8ff591ad351c016d23bc3d Mon Sep 17 00:00:00 2001 From: James Myers and Sujoy Basu Date: Tue, 13 Jan 2015 14:37:31 -0800 Subject: [PATCH 51/76] Update created_at to not have ON UPDATE CURRENT_TIMESTAMP for mysql * This only effects deployments that have run migrations prior to bb642ffdca7926adcfafaeb53afa000cbbc63b76 [#82509564] --- .../20150113201824_fix_created_at_column.rb | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 db/migrations/20150113201824_fix_created_at_column.rb diff --git a/db/migrations/20150113201824_fix_created_at_column.rb b/db/migrations/20150113201824_fix_created_at_column.rb new file mode 100644 index 00000000000..8b40d4a2295 --- /dev/null +++ b/db/migrations/20150113201824_fix_created_at_column.rb @@ -0,0 +1,53 @@ +Sequel.migration do + TABLE_NAMES = %w( + app_events + app_usage_events + apps + apps_v3 + billing_events + buildpacks + delayed_jobs + domains + droplets + env_groups + events + feature_flags + organizations + quota_definitions + routes + security_groups + service_auth_tokens + service_bindings + service_brokers + service_dashboard_clients + service_instances + service_plan_visibilities + service_plans + service_usage_events + services + space_quota_definitions + spaces + stacks + users + ) + + up do + if self.class.name.match /mysql/i + TABLE_NAMES.each do |table| + run <<-SQL + ALTER TABLE #{table} MODIFY created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; + SQL + end + end + end + + down do + if self.class.name.match /mysql/i + TABLE_NAMES.each do |table| + run <<-SQL + ALTER TABLE #{table} MODIFY created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP; + SQL + end + end + end +end From ec493f8240fdc27b4f6bd2f7c777561822e1e034 Mon Sep 17 00:00:00 2001 From: "David Sabeti, Jeff Hui and Michael Maximilien" Date: Tue, 13 Jan 2015 10:18:13 -0800 Subject: [PATCH 52/76] Remove message field login from ServiceBrokerConflict Error We only need to support description field [finishes #80794268] --- .../v2/errors/service_broker_conflict.rb | 4 +-- .../v2/errors/service_broker_conflict_spec.rb | 32 ++++++------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/lib/services/service_brokers/v2/errors/service_broker_conflict.rb b/lib/services/service_brokers/v2/errors/service_broker_conflict.rb index b13c7d99b67..8272a397c78 100644 --- a/lib/services/service_brokers/v2/errors/service_broker_conflict.rb +++ b/lib/services/service_brokers/v2/errors/service_broker_conflict.rb @@ -5,9 +5,7 @@ module Errors class ServiceBrokerConflict < HttpResponseError def initialize(uri, method, response) error_message = nil - if parsed_json(response.body).key?('message') - error_message = parsed_json(response.body)['message'] - else + if parsed_json(response.body).key?('description') error_message = parsed_json(response.body)['description'] end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb index 83facbaeb0f..db20fab7412 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_conflict_spec.rb @@ -5,7 +5,7 @@ module ServiceBrokers module V2 module Errors describe 'ServiceBrokerConflict' do - let(:response_body) { '{"message": "error message"}' } + let(:response_body) { '{"description": "error message"}' } let(:response) { double(code: 409, reason: 'Conflict', body: response_body) } let(:uri) { 'http://uri.example.com' } @@ -25,29 +25,15 @@ module Errors expect(exception.response_code).to eq(409) end - context 'when the response body has no message' do - let(:response_body) { '{"description": "error description"}' } + context 'when the response body has no description field' do + let(:response_body) { '{"field": "value"}' } - context 'and there is a description field' do - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq('error description') - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(MultiJson.load(response.body)) - end - end - - context 'and there is no description field' do - let(:response_body) { '{"field": "value"}' } - - it 'initializes the base class correctly' do - exception = ServiceBrokerConflict.new(uri, method, response) - expect(exception.message).to eq("Resource conflict: #{uri}") - expect(exception.uri).to eq(uri) - expect(exception.method).to eq(method) - expect(exception.source).to eq(MultiJson.load(response.body)) - end + it 'initializes the base class correctly' do + exception = ServiceBrokerConflict.new(uri, method, response) + expect(exception.message).to eq("Resource conflict: #{uri}") + expect(exception.uri).to eq(uri) + expect(exception.method).to eq(method) + expect(exception.source).to eq(MultiJson.load(response.body)) end end From 5baa59bec242758449a297d1bec30c7636a07fe6 Mon Sep 17 00:00:00 2001 From: David Sabeti and Jeff Hui Date: Tue, 13 Jan 2015 16:19:21 -0800 Subject: [PATCH 53/76] Swap Net::HTTP for HTTPClient in ServiceBrokers::V2::HttpClient to avoid automatic retry on timeout [finishes #84037538] --- .../service_brokers/v2/http_client.rb | 88 +++++++++---------- spec/acceptance/orphan_mitigation_spec.rb | 4 +- .../service_brokers/v2/http_client_spec.rb | 67 +++++++++----- 3 files changed, 87 insertions(+), 72 deletions(-) diff --git a/lib/services/service_brokers/v2/http_client.rb b/lib/services/service_brokers/v2/http_client.rb index 11a6b1568b1..83bf1d12d77 100644 --- a/lib/services/service_brokers/v2/http_client.rb +++ b/lib/services/service_brokers/v2/http_client.rb @@ -1,7 +1,25 @@ -require 'net/http' +require 'httpclient' module VCAP::Services module ServiceBrokers::V2 + class HttpResponse + def initialize(http_client_response) + @http_client_response = http_client_response + end + + def code + @http_client_response.code + end + + def message + @http_client_response.reason + end + + def body + @http_client_response.body + end + end + class HttpClient attr_reader :url @@ -40,63 +58,37 @@ def uri_for(path) end def make_request(method, uri, body, content_type) - req = build_request(method, uri, body, content_type) - opts = build_options(uri) + client = HTTPClient.new(force_basic_auth: true) + client.set_auth(uri, auth_username, auth_password) + + client.default_header[VCAP::Request::HEADER_BROKER_API_VERSION] = '2.4' + client.default_header[VCAP::Request::HEADER_NAME] = VCAP::Request.current_id + client.default_header['Accept'] = 'application/json' + + client.ssl_config.verify_mode = verify_certs? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE + client.connect_timeout = broker_client_timeout + client.receive_timeout = broker_client_timeout + client.send_timeout = broker_client_timeout - response = Net::HTTP.start(uri.hostname, uri.port, opts) do |http| - http.open_timeout = broker_client_timeout - http.read_timeout = broker_client_timeout + opts = { body: body } + opts[:header] = { 'Content-Type' => content_type } if content_type - http.request(req) - end + headers = client.default_header.merge(opts[:header]) if opts[:header] + logger.debug "Sending #{method} to #{uri}, BODY: #{body.inspect}, HEADERS: #{headers}" - log_response(uri, response) - return response + response = client.request(method, uri, opts) + + logger.debug "Response from request to #{uri}: STATUS #{response.code}, BODY: #{response.body.inspect}, HEADERS: #{response.headers.inspect}" + + HttpResponse.new(response) rescue SocketError, Errno::ECONNREFUSED => error raise Errors::ServiceBrokerApiUnreachable.new(uri.to_s, method, error) - rescue Timeout::Error => error + rescue HTTPClient::TimeoutError => error raise Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, error) rescue => error raise HttpRequestError.new(error.message, uri.to_s, method, error) end - def log_request(uri, req) - logger.debug "Sending #{req.method} to #{uri}, BODY: #{req.body.inspect}, HEADERS: #{req.to_hash.inspect}" - end - - def log_response(uri, response) - logger.debug "Response from request to #{uri}: STATUS #{response.code}, BODY: #{response.body.inspect}, HEADERS: #{response.to_hash.inspect}" - end - - def build_request(method, uri, body, content_type) - req_class = method.to_s.capitalize - req = Net::HTTP.const_get(req_class).new(uri.request_uri) - - req.basic_auth(auth_username, auth_password) - - req[VCAP::Request::HEADER_NAME] = VCAP::Request.current_id - req[VCAP::Request::HEADER_BROKER_API_VERSION] = '2.4' - req['Accept'] = 'application/json' - - req.body = body - req.content_type = content_type if content_type - - log_request(uri, req) - - req - end - - def build_options(uri) - opts = {} - - use_ssl = uri.scheme.to_s.downcase == 'https' - opts.merge!(use_ssl: use_ssl) - - verify_mode = verify_certs? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE - opts.merge!(verify_mode: verify_mode) if use_ssl - opts - end - def verify_certs? !VCAP::CloudController::Config.config[:skip_cert_verify] end diff --git a/spec/acceptance/orphan_mitigation_spec.rb b/spec/acceptance/orphan_mitigation_spec.rb index 8a85f38aea5..220e63171fd 100644 --- a/spec/acceptance/orphan_mitigation_spec.rb +++ b/spec/acceptance/orphan_mitigation_spec.rb @@ -19,7 +19,7 @@ before do stub_request(:put, %r{#{broker_url}/v2/service_instances/#{guid_pattern}}).to_return { |request| - raise Timeout::Error.new('fake-timeout') + raise HTTPClient::TimeoutError.new('fake-timeout') } stub_request(:delete, %r{#{broker_url}/v2/service_instances/#{guid_pattern}}). @@ -59,7 +59,7 @@ create_app stub_request(:put, %r{/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}}).to_return { |request| - raise Timeout::Error.new('fake-timeout') + raise HTTPClient::TimeoutError.new('fake-timeout') } stub_request(:delete, %r{/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}}). diff --git a/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb b/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb index 3ab858f058a..72d8c526628 100644 --- a/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/http_client_spec.rb @@ -28,7 +28,7 @@ module VCAP::Services::ServiceBrokers::V2 describe 'returning a correct response object' do subject { make_request } - its(:code) { should eq('200') } + its(:code) { should eq(200) } its(:body) { should_not be_nil } end @@ -68,22 +68,34 @@ module VCAP::Services::ServiceBrokers::V2 end describe 'ssl cert verification' do - let(:response) { double(code: nil, body: nil, to_hash: nil) } + let(:http_client) do + double(:http_client, + :connect_timeout= => nil, + :receive_timeout= => nil, + :send_timeout= => nil, + :set_auth => nil, + :default_header => {}, + :ssl_config => ssl_config, + :request => response) + end + + let(:response) { double(:response, code: nil, body: nil, headers: nil) } + let(:ssl_config) { double(:ssl_config, :verify_mode= => nil) } before do allow(VCAP::CloudController::Config).to receive(:config).and_return(config) + allow(HTTPClient).to receive(:new).and_return(http_client) + allow(http_client).to receive(http_method) end context 'and the skip_cert_verify is set to true' do let(:config) { { skip_cert_verify: true } } it 'accepts self-signed cert from the broker' do - expect(Net::HTTP).to receive(:start) { |host, port, opts, &blk| - expect(host).to eq 'broker.example.com' - expect(port).to eq 443 - expect(opts).to eq({ use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE }) - }.and_return(response) make_request + + expect(http_client).to have_received(:ssl_config) + expect(ssl_config).to have_received(:verify_mode=).with(OpenSSL::SSL::VERIFY_NONE) end end @@ -91,12 +103,10 @@ module VCAP::Services::ServiceBrokers::V2 let(:config) { { skip_cert_verify: false } } it 'does not accept self-signed cert from the broker' do - expect(Net::HTTP).to receive(:start) { |host, port, opts, &blk| - expect(host).to eq 'broker.example.com' - expect(port).to eq 443 - expect(opts).to eq({ use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_PEER }) - }.and_return(response) make_request + + expect(http_client).to have_received(:ssl_config) + expect(ssl_config).to have_received(:verify_mode=).with(OpenSSL::SSL::VERIFY_PEER) end end end @@ -131,7 +141,7 @@ module VCAP::Services::ServiceBrokers::V2 context 'when the API times out' do context 'because the client gave up' do before do - stub_request(http_method, full_url).to_raise(Timeout::Error) + stub_request(http_method, full_url).to_raise(HTTPClient::TimeoutError) end it 'raises a timeout error' do @@ -145,23 +155,36 @@ module VCAP::Services::ServiceBrokers::V2 shared_examples 'timeout behavior' do before do allow(VCAP::CloudController::Config).to receive(:config).and_return(config) - allow(Net::HTTP).to receive(:start).and_yield(http) + allow(HTTPClient).to receive(:new).and_return(http_client) + allow(http_client).to receive(http_method) + end + + let(:http_client) do + double(:http_client, + :connect_timeout= => nil, + :receive_timeout= => nil, + :send_timeout= => nil, + :set_auth => nil, + :default_header => {}, + :ssl_config => ssl_config, + :request => response) end - let(:http) { double('http', request: response) } - let(:response) { double(:response, code: 200, body: {}.to_json, to_hash: {}) } + let(:response) { double(:response, code: 200, body: {}.to_json, headers: {}) } + let(:ssl_config) { double(:ssl_config, :verify_mode= => nil) } def expect_timeout_to_be(timeout) - expect(http).to receive(:open_timeout=).with(timeout) - expect(http).to receive(:read_timeout=).with(timeout) + expect(http_client).to have_received(:connect_timeout=).with(timeout) + expect(http_client).to have_received(:receive_timeout=).with(timeout) + expect(http_client).to have_received(:send_timeout=).with(timeout) end context 'when the broker client timeout is set' do let(:config) { { broker_client_timeout_seconds: 100 } } it 'sets HTTP timeouts on request' do - expect_timeout_to_be 100 request + expect_timeout_to_be 100 end end @@ -169,8 +192,8 @@ def expect_timeout_to_be(timeout) let(:config) { { missing_broker_client_timeout: nil } } it 'defaults to a 60 second timeout' do - expect_timeout_to_be 60 request + expect_timeout_to_be 60 end end end @@ -203,7 +226,7 @@ def expect_timeout_to_be(timeout) it 'does not have a content body' do make_request expect(a_request(:get, full_url). - with { |req| expect(req.body).to be_nil }). + with { |req| expect(req.body).to be_empty }). to have_been_made end @@ -360,7 +383,7 @@ def expect_timeout_to_be(timeout) make_request expect(a_request(:delete, full_url). with(query: message). - with { |req| expect(req.body).to be_nil }). + with { |req| expect(req.body).to be_empty }). to have_been_made end From b009554572ddfbb296d475bcbb02436433c5f4c3 Mon Sep 17 00:00:00 2001 From: James Myers and Sujoy Basu Date: Tue, 13 Jan 2015 17:22:09 -0800 Subject: [PATCH 54/76] Respect true, t, false, and f when querying boolean values * anything else will be treated as false * Do not need to special case tinyint columns as Sequel does the * conversion for us [#84942534] --- lib/vcap/rest_api/query.rb | 13 ++----------- spec/unit/lib/vcap/rest_api/query_spec.rb | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/vcap/rest_api/query.rb b/lib/vcap/rest_api/query.rb index 7d7b8734e7d..72650f51591 100644 --- a/lib/vcap/rest_api/query.rb +++ b/lib/vcap/rest_api/query.rb @@ -153,19 +153,10 @@ def clean_up_foreign_key(query_key, query_values, foreign_key_column_name) { foreign_key_column_name => foreign_key_value } end - TINYINT_TYPE = 'tinyint(1)'.freeze - TINYINT_FROM_TRUE_FALSE = { 't' => 1, 'f' => 0 }.freeze - # Sequel uses tinyint(1) to store booleans in Mysql. # Mysql does not support using 't'/'f' for querying. - def clean_up_boolean(q_key, q_val) - column = model.db_schema[q_key.to_sym] - - if column[:db_type] == TINYINT_TYPE - TINYINT_FROM_TRUE_FALSE.fetch(q_val, q_val) - else - q_val == 't' - end + def clean_up_boolean(_, q_val) + q_val == 't' || q_val == 'true' end def clean_up_datetime(q_val) diff --git a/spec/unit/lib/vcap/rest_api/query_spec.rb b/spec/unit/lib/vcap/rest_api/query_spec.rb index a7f8f765cba..8044206a806 100644 --- a/spec/unit/lib/vcap/rest_api/query_spec.rb +++ b/spec/unit/lib/vcap/rest_api/query_spec.rb @@ -230,17 +230,35 @@ class Subscriber < Sequel::Model end describe 'boolean values on boolean column' do - it 'returns correctly filtered results for true' do + it 'returns correctly filtered results for t' do ds = Query.filtered_dataset_from_query_params( Author, Author.dataset, @queryable_attributes, q: 'published:t') expect(ds.all).to eq([Author.first]) end - it 'returns correctly filtered results for false' do + it 'returns correctly filtered results for true' do + ds = Query.filtered_dataset_from_query_params( + Author, Author.dataset, @queryable_attributes, q: 'published:true') + expect(ds.all).to eq([Author.first]) + end + + it 'returns correctly filtered results for f' do ds = Query.filtered_dataset_from_query_params( Author, Author.dataset, @queryable_attributes, q: 'published:f') expect(ds.all).to eq(Author.all - [Author.first]) end + + it 'returns correctly filtered results for false' do + ds = Query.filtered_dataset_from_query_params( + Author, Author.dataset, @queryable_attributes, q: 'published:false') + expect(ds.all).to eq(Author.all - [Author.first]) + end + + it 'returns resulted filtered as false for any other value' do + ds = Query.filtered_dataset_from_query_params( + Author, Author.dataset, @queryable_attributes, q: 'published:foobar') + expect(ds.all).to eq(Author.all - [Author.first]) + end end describe 'errors' do From d19d376631155df0e3088b81e8561bd945b2a9dd Mon Sep 17 00:00:00 2001 From: David Sabeti and Jeff Hui Date: Tue, 13 Jan 2015 17:49:02 -0800 Subject: [PATCH 55/76] Factor out enqueueing logic from do_delete so that controllers can call it directly [finishes #84895424] --- app/controllers/base/model_controller.rb | 8 ++++++-- .../services/service_bindings_controller.rb | 19 ++++--------------- .../services/service_instances_controller.rb | 8 +------- .../service_plan_visibilities_controller.rb | 8 +------- ...r_provided_service_instances_controller.rb | 8 +------- 5 files changed, 13 insertions(+), 38 deletions(-) diff --git a/app/controllers/base/model_controller.rb b/app/controllers/base/model_controller.rb index 4f98667e486..bd6652619c9 100644 --- a/app/controllers/base/model_controller.rb +++ b/app/controllers/base/model_controller.rb @@ -76,11 +76,15 @@ def update(guid) def do_delete(obj) raise_if_has_associations!(obj) if v2_api? && !recursive? model_deletion_job = Jobs::Runtime::ModelDeletion.new(obj.class, obj.guid) + enqueue_deletion_job(model_deletion_job) + end + + def enqueue_deletion_job(deletion_job) if async? - job = Jobs::Enqueuer.new(model_deletion_job, queue: 'cc-generic').enqueue + job = Jobs::Enqueuer.new(deletion_job, queue: 'cc-generic').enqueue [HTTP::ACCEPTED, JobPresenter.new(job).to_json] else - model_deletion_job.perform + deletion_job.perform [HTTP::NO_CONTENT, nil] end end diff --git a/app/controllers/services/service_bindings_controller.rb b/app/controllers/services/service_bindings_controller.rb index b02bc0ae218..55fa32da47e 100644 --- a/app/controllers/services/service_bindings_controller.rb +++ b/app/controllers/services/service_bindings_controller.rb @@ -24,19 +24,14 @@ def inject_dependencies(dependencies) post path, :create def create - json_msg = self.class::CreateMessage.decode(body) - - @request_attrs = json_msg.extract(stringify_keys: true) + @request_attrs = self.class::CreateMessage.decode(body).extract(stringify_keys: true) logger.debug 'cc.create', model: self.class.model_class_name, attributes: request_attrs raise InvalidRequest unless request_attrs - instance_guid = request_attrs['service_instance_guid'] - app_guid = request_attrs['app_guid'] - - validate_service_instance(instance_guid) - validate_app(app_guid) + validate_service_instance(request_attrs['service_instance_guid']) + validate_app(request_attrs['app_guid']) service_binding = ServiceBinding.new(@request_attrs) validate_access(:create, service_binding) @@ -63,13 +58,7 @@ def delete(guid) deletion_job = Jobs::Runtime::ModelDeletion.new(ServiceBinding, guid) delete_and_audit_job = Jobs::AuditEventJob.new(deletion_job, @services_event_repository, :record_service_binding_event, :delete, service_binding) - if async? - job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue - [HTTP::ACCEPTED, JobPresenter.new(job).to_json] - else - delete_and_audit_job.perform - [HTTP::NO_CONTENT, nil] - end + enqueue_deletion_job(delete_and_audit_job) end private diff --git a/app/controllers/services/service_instances_controller.rb b/app/controllers/services/service_instances_controller.rb index 9bfaad6f836..79320049445 100644 --- a/app/controllers/services/service_instances_controller.rb +++ b/app/controllers/services/service_instances_controller.rb @@ -200,13 +200,7 @@ def delete(guid) event_method = service_instance.type == 'managed_service_instance' ? :record_service_instance_event : :record_user_provided_service_instance_event delete_and_audit_job = Jobs::AuditEventJob.new(deletion_job, @services_event_repository, event_method, :delete, service_instance, {}) - if async? - job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue - [HTTP::ACCEPTED, JobPresenter.new(job).to_json] - else - delete_and_audit_job.perform - [HTTP::NO_CONTENT, nil] - end + enqueue_deletion_job(delete_and_audit_job) end def get_filtered_dataset_for_enumeration(model, ds, qp, opts) diff --git a/app/controllers/services/service_plan_visibilities_controller.rb b/app/controllers/services/service_plan_visibilities_controller.rb index 8392fcd302e..6a5b9b1661a 100644 --- a/app/controllers/services/service_plan_visibilities_controller.rb +++ b/app/controllers/services/service_plan_visibilities_controller.rb @@ -81,13 +81,7 @@ def delete(guid) {} ) - if async? - job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue - [HTTP::ACCEPTED, JobPresenter.new(job).to_json] - else - delete_and_audit_job.perform - [HTTP::NO_CONTENT, nil] - end + enqueue_deletion_job(delete_and_audit_job) end define_messages diff --git a/app/controllers/services/user_provided_service_instances_controller.rb b/app/controllers/services/user_provided_service_instances_controller.rb index 1cb6a9e1bf1..25552119caf 100644 --- a/app/controllers/services/user_provided_service_instances_controller.rb +++ b/app/controllers/services/user_provided_service_instances_controller.rb @@ -99,13 +99,7 @@ def delete(guid) {} ) - if async? - job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue - [HTTP::ACCEPTED, JobPresenter.new(job).to_json] - else - delete_and_audit_job.perform - [HTTP::NO_CONTENT, nil] - end + enqueue_deletion_job(delete_and_audit_job) end define_messages From da665b78016e68304cadd875265bd392797f2d0f Mon Sep 17 00:00:00 2001 From: JT Archie and Luan Santos Date: Tue, 13 Jan 2015 17:20:47 -0800 Subject: [PATCH 56/76] Decrease sensitivity on time based test --- spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb index 3d19e9b37ab..c0097e69820 100644 --- a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb @@ -15,8 +15,8 @@ module Jobs::Runtime describe '#perform' do context 'with packages which have been pending for too long' do - let!(:app1) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 1.second) } - let!(:app2) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 2.second) } + let!(:app1) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 1.minute) } + let!(:app2) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 2.minutes) } before do cleanup_job.perform @@ -41,8 +41,8 @@ module Jobs::Runtime end it "ignores apps that haven't been pending for too long" do - app1 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 1.second) - app2 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 2.second) + app1 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 1.minute) + app2 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 2.minutes) cleanup_job.perform app1.reload From 315edac2ac7c3d1c8f631ca3011ceb14b96a390c Mon Sep 17 00:00:00 2001 From: JT Archie and Zach Robinson Date: Wed, 14 Jan 2015 10:12:59 -0800 Subject: [PATCH 57/76] ensure Timecop.return is called after every test --- spec/spec_helper.rb | 4 ++++ spec/unit/models/runtime/buildpack_spec.rb | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 47b1e6cf683..ef0c3425d0f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -86,6 +86,10 @@ TmpdirCleaner.clean end + rspec_config.after :each do + Timecop.return + end + rspec_config.after(:each, type: :legacy_api) { add_deprecation_warning } RspecApiDocumentation.configure do |c| diff --git a/spec/unit/models/runtime/buildpack_spec.rb b/spec/unit/models/runtime/buildpack_spec.rb index 7f2fb9ad900..4c81b1b9876 100644 --- a/spec/unit/models/runtime/buildpack_spec.rb +++ b/spec/unit/models/runtime/buildpack_spec.rb @@ -52,6 +52,10 @@ def ordered_buildpacks Buildpack.dataset.destroy end + after do + Timecop.return + end + subject(:all_buildpacks) { Buildpack.list_admin_buildpacks } context 'with prioritized buildpacks' do From 23930e6813bea99d8fec1961b68999901461ad7e Mon Sep 17 00:00:00 2001 From: James Myers Date: Tue, 13 Jan 2015 17:54:34 -0800 Subject: [PATCH 58/76] Add `rake db:dev:migrate` and `rake db:dev:rollback` to migrate dev dbs * All you need to specify is the DB env var * i.e. `DB=mysql rake db:dev:migrate` [#76726312] --- lib/cloud_controller/runner.rb | 8 -------- lib/tasks/db.rake | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/cloud_controller/runner.rb b/lib/cloud_controller/runner.rb index 6f709d9f02c..bd28a8e1354 100644 --- a/lib/cloud_controller/runner.rb +++ b/lib/cloud_controller/runner.rb @@ -140,14 +140,6 @@ def stop! end end - def merge_vcap_config - services = JSON.parse(ENV['VCAP_SERVICES']) - pg_key = services.keys.select { |svc| svc =~ /postgres/i }.first - c = services[pg_key].first['credentials'] - @config[:db][:database] = "postgres://#{c['user']}:#{c['password']}@#{c['hostname']}:#{c['port']}/#{c['name']}" - @config[:external_port] = ENV['VCAP_APP_PORT'].to_i - end - private def stop_router_registrar(&blk) diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 333ffbc1314..9d550c6a6a6 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -31,7 +31,7 @@ end task :rollback do Steno.init(Steno::Config.new(sinks: [Steno::Sink::IO.new(STDOUT)])) db_logger = Steno.logger("cc.db.migrations") - DBMigrator.from_config(config, db_logger).rollback(number_to_rollback=1) + DBMigrator.from_config(config, db_logger).rollback(1) end namespace :migrate do @@ -39,6 +39,22 @@ end task :redo => [:rollback, :migrate] end + namespace :dev do + task :migrate do + require_relative "../../spec/support/bootstrap/db_config" + + config[:db][:database] = DbConfig.connection_string + Rake::Task["db:migrate"].invoke + end + + task :rollback do + require_relative "../../spec/support/bootstrap/db_config" + + config[:db][:database] = DbConfig.connection_string + Rake::Task["db:rollback"].invoke + end + end + task :pick do unless ENV["DB_CONNECTION_STRING"] || ENV["DB_CONNECTION"] ENV["DB"] ||= %w[mysql postgres].sample From 106628d7fd20a52952f56d1dc4663abd5a36ffc0 Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 14 Jan 2015 14:17:39 -0800 Subject: [PATCH 59/76] ServiceBrokerResponseMalformed and ServiceBrokerApiUnreachable are rendered to users as a 502 We should never return 500 for an expected error [#84963628] [#85739060] --- .../v2/errors/service_broker_api_unreachable.rb | 4 ++++ .../v2/errors/service_broker_response_malformed.rb | 4 ++++ .../v2/errors/service_broker_api_unreachable_spec.rb | 5 +++++ .../v2/errors/service_broker_response_malformed_spec.rb | 4 ++++ 4 files changed, 17 insertions(+) diff --git a/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb b/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb index 2ddc5006ea6..e0dd4838a91 100644 --- a/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb +++ b/lib/services/service_brokers/v2/errors/service_broker_api_unreachable.rb @@ -13,6 +13,10 @@ def initialize(uri, method, source) source ) end + + def response_code + 502 + end end end end diff --git a/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb b/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb index 2d81e994c4e..815e696c762 100644 --- a/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb +++ b/lib/services/service_brokers/v2/errors/service_broker_response_malformed.rb @@ -11,6 +11,10 @@ def initialize(uri, method, response) response ) end + + def response_code + 502 + end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb index 78b46daa89a..b3e2159a932 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_unreachable_spec.rb @@ -29,6 +29,11 @@ module Errors } }) end + + it 'renders the correct status code to the user' do + exception = ServiceBrokerApiUnreachable.new(uri, 'PUT', error) + expect(exception.response_code).to eq 502 + end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb index 6e6b6844389..0d51bc53d25 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_response_malformed_spec.rb @@ -18,6 +18,10 @@ module Errors expect(exception.method).to eq(method) expect(exception.source).to be(response.body) end + + it 'renders a 502 to the user' do + expect(ServiceBrokerResponseMalformed.new(uri, method, response).response_code).to eq 502 + end end end end From ee7b93b4ef13860443e7a61b0521156724cc204d Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 14 Jan 2015 14:23:59 -0800 Subject: [PATCH 60/76] Backfill tests for response codes from service broker errors This is sort of testing configuration, but it has value in that having a method called response_code causes different behavior in the error presenter, and we want to make sure this code conforms to that interface [finishes #84963640] --- .../service_broker_api_authentication_failed_spec.rb | 5 +++++ .../v2/errors/service_broker_api_timeout_spec.rb | 7 ++++++- .../v2/errors/service_broker_bad_response_spec.rb | 11 +++++++++++ .../v2/errors/service_broker_request_rejected_spec.rb | 11 +++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb index 4932306162a..62f97f7cb95 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_authentication_failed_spec.rb @@ -20,6 +20,11 @@ module Errors expect(exception.method).to eq(method) expect(exception.source).to be(response.body) end + + it 'renders the correct status code to the user' do + exception = ServiceBrokerApiAuthenticationFailed.new(uri, method, response) + expect(exception.response_code).to eq 502 + end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb index 09f655f85c3..8c1d8cd0d8c 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_api_timeout_spec.rb @@ -10,12 +10,17 @@ module Errors let(:error) { StandardError.new } it 'initializes the base class correctly' do - exception = Errors::ServiceBrokerApiTimeout.new(uri, method, error) + exception = ServiceBrokerApiTimeout.new(uri, method, error) expect(exception.message).to eq("The service broker API timed out: #{uri}") expect(exception.uri).to eq(uri) expect(exception.method).to eq(method) expect(exception.source).to be(error) end + + it 'renders the correct status code to the user' do + exception = ServiceBrokerApiTimeout.new(uri, method, error) + expect(exception.response_code).to eq 504 + end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb index 4977dd91cc7..2ce744b810f 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_bad_response_spec.rb @@ -31,12 +31,18 @@ module Errors } }) end + + it 'renders the correct status code to the user' do + exception = described_class.new(uri, method, response) + expect(exception.response_code).to eq 502 + end end context 'without a description in the body' do let(:response_body) do { 'foo' => 'bar' }.to_json end + it 'generates the correct hash' do exception = described_class.new(uri, method, response) exception.set_backtrace(['/foo:1', '/bar:2']) @@ -52,6 +58,11 @@ module Errors 'source' => { 'foo' => 'bar' } }) end + + it 'renders the correct status code to the user' do + exception = described_class.new(uri, method, response) + expect(exception.response_code).to eq 502 + end end end end diff --git a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb index 52dd52dbfad..448b41a9291 100644 --- a/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/errors/service_broker_request_rejected_spec.rb @@ -33,12 +33,18 @@ module Errors } }) end + + it 'renders the correct status code to the user' do + exception = described_class.new(uri, method, response) + expect(exception.response_code).to eq 502 + end end context 'without a description in the body' do let(:response_body) do { 'foo' => 'bar' }.to_json end + it 'generates the correct hash' do exception = described_class.new(uri, method, response) exception.set_backtrace(['/foo:1', '/bar:2']) @@ -54,6 +60,11 @@ module Errors 'source' => { 'foo' => 'bar' } }) end + + it 'renders the correct status code to the user' do + exception = described_class.new(uri, method, response) + expect(exception.response_code).to eq 502 + end end end end From b9ae86ac21ad5e29468ddb20c4eae5bab9838caa Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 14 Jan 2015 14:26:00 -0800 Subject: [PATCH 61/76] Reduce unnecessary namespacing --- lib/services/service_brokers/v2/response_parser.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/services/service_brokers/v2/response_parser.rb b/lib/services/service_brokers/v2/response_parser.rb index 46e804f03fb..eba850d0131 100644 --- a/lib/services/service_brokers/v2/response_parser.rb +++ b/lib/services/service_brokers/v2/response_parser.rb @@ -23,19 +23,19 @@ def parse(method, path, response) end unless response_hash.is_a?(Hash) - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) + raise Errors::ServiceBrokerResponseMalformed.new(uri.to_s, method, response) end return response_hash when HTTP::Status::UNAUTHORIZED - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) + raise Errors::ServiceBrokerApiAuthenticationFailed.new(uri.to_s, method, response) when 408 - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, response) + raise Errors::ServiceBrokerApiTimeout.new(uri.to_s, method, response) when 409 - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerConflict.new(uri.to_s, method, response) + raise Errors::ServiceBrokerConflict.new(uri.to_s, method, response) when 410 if method == :delete @@ -44,10 +44,10 @@ def parse(method, path, response) end when 400..499 - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerRequestRejected.new(uri.to_s, method, response) + raise Errors::ServiceBrokerRequestRejected.new(uri.to_s, method, response) end - raise VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse.new(uri.to_s, method, response) + raise Errors::ServiceBrokerBadResponse.new(uri.to_s, method, response) end private From 29019f47e7f8c8252e2973436f4091eaca09b43a Mon Sep 17 00:00:00 2001 From: David Sabeti Date: Wed, 14 Jan 2015 14:26:12 -0800 Subject: [PATCH 62/76] Remove unnecessary contexts --- spec/acceptance/orphan_mitigation_spec.rb | 40 ++++++++++------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/spec/acceptance/orphan_mitigation_spec.rb b/spec/acceptance/orphan_mitigation_spec.rb index 220e63171fd..7ffbd6a5179 100644 --- a/spec/acceptance/orphan_mitigation_spec.rb +++ b/spec/acceptance/orphan_mitigation_spec.rb @@ -34,19 +34,17 @@ json_headers(admin_headers)) end - context 'when the broker times out' do - it 'makes the request to the broker and deprovisions' do - expect(a_request(:put, %r{http://username:password@broker-url/v2/service_instances/#{guid_pattern}})).to have_been_made + it 'makes the request to the broker and deprovisions' do + expect(a_request(:put, %r{http://username:password@broker-url/v2/service_instances/#{guid_pattern}})).to have_been_made - successes, failures = Delayed::Worker.new.work_off - expect([successes, failures]).to eq [1, 0] + successes, failures = Delayed::Worker.new.work_off + expect([successes, failures]).to eq [1, 0] - expect(a_request(:delete, %r{http://username:password@broker-url/v2/service_instances/#{guid_pattern}})).to have_been_made - end + expect(a_request(:delete, %r{http://username:password@broker-url/v2/service_instances/#{guid_pattern}})).to have_been_made + end - it 'responds to user with 504' do - expect(last_response.status).to eq(504) - end + it 'responds to user with 504' do + expect(last_response.status).to eq(504) end end @@ -70,21 +68,19 @@ json_headers(admin_headers)) end - context 'when the broker times out' do - it 'makes the request to the broker and deprovisions' do - expect(a_request(:put, %r{http://username:password@broker-url/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}})). - to have_been_made + it 'makes the request to the broker and deprovisions' do + expect(a_request(:put, %r{http://username:password@broker-url/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}})). + to have_been_made - successes, failures = Delayed::Worker.new.work_off - expect([successes, failures]).to eq [1, 0] + successes, failures = Delayed::Worker.new.work_off + expect([successes, failures]).to eq [1, 0] - expect(a_request(:delete, %r{http://username:password@broker-url/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}})). - to have_been_made - end + expect(a_request(:delete, %r{http://username:password@broker-url/v2/service_instances/#{service_instance_guid}/service_bindings/#{guid_pattern}})). + to have_been_made + end - it 'responds to user with 504' do - expect(last_response.status).to eq(504) - end + it 'responds to user with 504' do + expect(last_response.status).to eq(504) end end end From 452a2a7a3f50ff35ceb6b44e92b829beee0792c7 Mon Sep 17 00:00:00 2001 From: James Myers and Ketan Deshpande Date: Wed, 14 Jan 2015 16:16:19 -0800 Subject: [PATCH 63/76] Specifying the max on health check timeout if the limit is exceeded [#78074408] --- app/models/runtime/constraints/health_check_policy.rb | 6 +++--- .../models/runtime/constraints/health_check_policy_spec.rb | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/models/runtime/constraints/health_check_policy.rb b/app/models/runtime/constraints/health_check_policy.rb index 7d4ee8e2a2f..758416d21c8 100644 --- a/app/models/runtime/constraints/health_check_policy.rb +++ b/app/models/runtime/constraints/health_check_policy.rb @@ -8,9 +8,9 @@ def initialize(app, health_check_timeout) def validate return unless @health_check_timeout @errors.add(:health_check_timeout, :less_than_zero) unless @health_check_timeout >= 0 - - if @health_check_timeout > VCAP::CloudController::Config.config[:maximum_health_check_timeout] - @errors.add(:health_check_timeout, :maximum_exceeded) + max_timeout = VCAP::CloudController::Config.config[:maximum_health_check_timeout] + if @health_check_timeout > max_timeout + @errors.add(:health_check_timeout, "Maximum exceeded: max #{max_timeout}s") end end end diff --git a/spec/unit/models/runtime/constraints/health_check_policy_spec.rb b/spec/unit/models/runtime/constraints/health_check_policy_spec.rb index 333bc3a5b0f..f88ad821a36 100644 --- a/spec/unit/models/runtime/constraints/health_check_policy_spec.rb +++ b/spec/unit/models/runtime/constraints/health_check_policy_spec.rb @@ -4,17 +4,19 @@ let(:app) { VCAP::CloudController::AppFactory.make } subject(:validator) { HealthCheckPolicy.new(app, health_check_timeout) } + let(:max_health_check_timeout) { 512 } describe 'health_check_timeout' do before do - TestConfig.override({ maximum_health_check_timeout: 512 }) + TestConfig.override({ maximum_health_check_timeout: max_health_check_timeout }) end context 'when a health_check_timeout exceeds the maximum' do let(:health_check_timeout) { 1024 } it 'registers error' do - expect(validator).to validate_with_error(app, :health_check_timeout, :maximum_exceeded) + error_msg = "Maximum exceeded: max #{max_health_check_timeout}s" + expect(validator).to validate_with_error(app, :health_check_timeout, error_msg) end end From 79bf28c5cfe128b7f1dcfbac93038da5a0fc38b8 Mon Sep 17 00:00:00 2001 From: Jeff Hui and Michael Maximilien Date: Thu, 15 Jan 2015 13:51:59 -0800 Subject: [PATCH 64/76] adding async service instance provisioning * includes polling instance state * includes recovering from failed state * includes recovering from DB errors [#84254704] --- .../services/service_instance_state_fetch.rb | 37 ++++++ lib/services/service_brokers/v2.rb | 1 + lib/services/service_brokers/v2/client.rb | 24 +++- .../v2/service_instance_state_poller.rb | 21 +++ .../service_instance_state_fetch_spec.rb | 123 ++++++++++++++++++ .../service_brokers/v2/client_spec.rb | 107 ++++++++++++++- .../v2/service_instance_state_poller_spec.rb | 35 +++++ 7 files changed, 342 insertions(+), 6 deletions(-) create mode 100644 app/jobs/services/service_instance_state_fetch.rb create mode 100644 lib/services/service_brokers/v2/service_instance_state_poller.rb create mode 100644 spec/unit/jobs/services/service_instance_state_fetch_spec.rb create mode 100644 spec/unit/lib/services/service_brokers/v2/service_instance_state_poller_spec.rb diff --git a/app/jobs/services/service_instance_state_fetch.rb b/app/jobs/services/service_instance_state_fetch.rb new file mode 100644 index 00000000000..848bf43da14 --- /dev/null +++ b/app/jobs/services/service_instance_state_fetch.rb @@ -0,0 +1,37 @@ +module VCAP::CloudController + module Jobs + module Services + class ServiceInstanceStateFetch < Struct.new(:name, :client_attrs, :service_instance_guid, :service_plan_guid) + def perform + client = VCAP::Services::ServiceBrokers::V2::Client.new(client_attrs) + service_plan = ServicePlan.first(guid: service_plan_guid) + service_instance = ManagedServiceInstance.first(guid: service_instance_guid, service_plan: service_plan) + client.fetch_service_instance_state(service_instance) + + service_instance.save + + if service_instance.state != 'available' + retry_job + end + rescue HttpRequestError, Sequel::Error => e + retry_job + end + + def job_name_in_configuration + :service_instance_state_fetch + end + + def max_attempts + 1 + end + + private + + def retry_job + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(self, opts).enqueue + end + end + end + end +end diff --git a/lib/services/service_brokers/v2.rb b/lib/services/service_brokers/v2.rb index 69f31b708bb..4aa8059e2b6 100644 --- a/lib/services/service_brokers/v2.rb +++ b/lib/services/service_brokers/v2.rb @@ -7,5 +7,6 @@ module VCAP::Services::ServiceBrokers::V2 end require 'services/service_brokers/v2/http_client' require 'services/service_brokers/v2/client' require 'services/service_brokers/v2/orphan_mitigator' +require 'services/service_brokers/v2/service_instance_state_poller' require 'services/service_brokers/v2/response_parser' require 'services/service_brokers/v2/errors' diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index 0454420d5a1..f0820866d99 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -8,6 +8,7 @@ def initialize(attrs) @response_parser = VCAP::Services::ServiceBrokers::V2::ResponseParser.new(@http_client.url) @attrs = attrs @orphan_mitigator = VCAP::Services::ServiceBrokers::V2::OrphanMitigator.new + @state_poller = VCAP::Services::ServiceBrokers::V2::ServiceInstanceStatePoller.new end def catalog @@ -18,7 +19,7 @@ def catalog # The broker is expected to guarantee uniqueness of instance_id. # raises ServiceBrokerConflict if the id is already in use def provision(instance) - path = "/v2/service_instances/#{instance.guid}" + path = "/v2/service_instances/#{instance.guid}?accept_unavailable=true" response = @http_client.put(path, { service_id: instance.service.broker_provided_id, @@ -29,17 +30,34 @@ def provision(instance) parsed_response = @response_parser.parse(:put, path, response) instance.dashboard_url = parsed_response['dashboard_url'] - instance.state = parsed_response['state'] || 'available' instance.state_description = parsed_response['state_description'] || '' + if parsed_response['state'] + instance.state = parsed_response['state'] + @state_poller.poll_service_instance_state(@attrs, instance) if instance.state == 'creating' + else + instance.state = 'available' + end + # DEPRECATED, but needed because of not null constraint instance.credentials = {} - rescue Errors::ServiceBrokerApiTimeout, Errors::ServiceBrokerBadResponse => e @orphan_mitigator.cleanup_failed_provision(@attrs, instance) raise e end + def fetch_service_instance_state(instance) + path = "/v2/service_instances/#{instance.guid}" + + response = @http_client.get(path) + parsed_response = @response_parser.parse(:get, path, response) + + instance.dashboard_url = parsed_response['dashboard_url'] + instance.state = parsed_response['state'] + instance.state_description = parsed_response['state_description'] + instance + end + def bind(binding) path = "/v2/service_instances/#{binding.service_instance.guid}/service_bindings/#{binding.guid}" response = @http_client.put(path, { diff --git a/lib/services/service_brokers/v2/service_instance_state_poller.rb b/lib/services/service_brokers/v2/service_instance_state_poller.rb new file mode 100644 index 00000000000..5415c4a0a04 --- /dev/null +++ b/lib/services/service_brokers/v2/service_instance_state_poller.rb @@ -0,0 +1,21 @@ +require 'jobs/services/service_instance_state_fetch' + +module VCAP::Services + module ServiceBrokers + module V2 + class ServiceInstanceStatePoller + def poll_service_instance_state(client_attrs, service_instance) + job = VCAP::CloudController::Jobs::Services::ServiceInstanceStateFetch.new( + 'service-instance-state-fetch', + client_attrs, + service_instance.guid, + service_instance.service_plan.guid + ) + + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + VCAP::CloudController::Jobs::Enqueuer.new(job, opts).enqueue + end + end + end + end +end diff --git a/spec/unit/jobs/services/service_instance_state_fetch_spec.rb b/spec/unit/jobs/services/service_instance_state_fetch_spec.rb new file mode 100644 index 00000000000..767641ddab3 --- /dev/null +++ b/spec/unit/jobs/services/service_instance_state_fetch_spec.rb @@ -0,0 +1,123 @@ +require 'spec_helper' + +module VCAP::CloudController + module Jobs::Services + describe ServiceInstanceStateFetch do + let(:client) { instance_double('VCAP::Services::ServiceBrokers::V2::Client') } + let(:enqueuer) { instance_double('VCAP::CloudController::Jobs::Enqueuer') } + let(:service_instance) { VCAP::CloudController::ManagedServiceInstance.make } + + let(:name) { 'fake-name' } + + subject(:job) do + VCAP::CloudController::Jobs::Services::ServiceInstanceStateFetch.new(name, {}, + service_instance.guid, service_instance.service_plan.guid) + end + + describe '#perform' do + before do + allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(client) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(enqueuer) + allow(enqueuer).to receive(:enqueue) + end + + context 'when all operations succeed and the state is available' do + before do + allow(client).to receive(:fetch_service_instance_state) do |instance| + instance.state = 'available' + end + end + + it 'fetches the service instance state' do + job.perform + + expect(client).to have_received(:fetch_service_instance_state) do |instance| + expect(instance.guid).to eq service_instance.guid + expect(instance.service_plan.guid).to eq service_instance.service_plan.guid + end + + db_service_instance = ManagedServiceInstance.first(guid: service_instance.guid) + expect(db_service_instance.state).to eq('available') + end + + it 'should not enqueue another fetch job' do + job.perform + + expect(VCAP::CloudController::Jobs::Enqueuer).to_not have_received(:new) + end + end + + context 'when all operations succeed, but the state is not available' do + before do + allow(client).to receive(:fetch_service_instance_state) do |instance| + instance.state = 'creating' + end + end + + it 'fetches the service instance state' do + job.perform + + expect(client).to have_received(:fetch_service_instance_state) do |instance| + expect(instance.guid).to eq service_instance.guid + expect(instance.service_plan.guid).to eq service_instance.service_plan.guid + end + + db_service_instance = ManagedServiceInstance.first(guid: service_instance.guid) + expect(db_service_instance.state).to eq('creating') + end + + it 'should enqueue another fetch job' do + job.perform + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new).with(job, anything) + expect(enqueuer).to have_received(:enqueue) + end + end + + context 'when saving to the database fails' do + before do + allow(client).to receive(:fetch_service_instance_state) + allow(service_instance).to receive(:save) do |instance| + raise Sequel::ValidationFailed.new(instance) + end + end + + it 'should enqueue another fetch job' do + job.perform + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new).with(job, anything) + expect(enqueuer).to have_received(:enqueue) + end + end + + context 'when fetching the service instance from the broker fails' do + before do + allow(client).to receive(:fetch_service_instance_state) do |instance| + error = nil + raise HttpRequestError.new('something', '/some/path', :get, error) + end + end + + it 'should enqueue another fetch job' do + job.perform + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new).with(job, anything) + expect(enqueuer).to have_received(:enqueue) + end + end + end + + describe '#job_name_in_configuration' do + it 'returns the name of the job' do + expect(job.job_name_in_configuration).to eq(:service_instance_state_fetch) + end + end + + describe '#max_attempts' do + it 'returns 1' do + expect(job.max_attempts).to eq(1) + end + end + end + end +end diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 41dc6733451..56de382532e 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -16,6 +16,7 @@ module VCAP::Services::ServiceBrokers::V2 let(:http_client) { double('http_client') } let(:orphan_mitigator) { double('orphan_mitigator', cleanup_failed_provision: nil, cleanup_failed_bind: nil) } + let(:state_poller) { double('state_poller', poll_service_instance_state: nil) } before do allow(HttpClient).to receive(:new). @@ -25,6 +26,9 @@ module VCAP::Services::ServiceBrokers::V2 allow(VCAP::Services::ServiceBrokers::V2::OrphanMitigator).to receive(:new). and_return(orphan_mitigator) + allow(VCAP::Services::ServiceBrokers::V2::ServiceInstanceStatePoller).to receive(:new). + and_return(state_poller) + allow(http_client).to receive(:url).and_return(service_broker.broker_url) end @@ -98,7 +102,7 @@ module VCAP::Services::ServiceBrokers::V2 } end - let(:path) { "/v2/service_instances/#{instance.guid}" } + let(:path) { "/v2/service_instances/#{instance.guid}?accept_unavailable=true" } let(:response) { double('response') } let(:response_body) { response_data.to_json } let(:code) { '201' } @@ -117,7 +121,7 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) expect(http_client).to have_received(:put). - with("/v2/service_instances/#{instance.guid}", anything) + with("/v2/service_instances/#{instance.guid}?accept_unavailable=true", anything) end it 'makes a put request with correct message' do @@ -158,7 +162,27 @@ module VCAP::Services::ServiceBrokers::V2 expect(instance.credentials).to eq({}) end - context 'when the broker returns a state' do + context 'when the broker returns no state or the state is created, or available' do + let(:response_data) do + { + } + end + + it 'return immediately with the broker response' do + client = Client.new(client_attrs.merge(accept_unavailable: true)) + client.provision(instance) + + expect(instance.state).to eq('available') + expect(instance.state_description).to eq('') + end + + it 'does not enqueue a polling job' do + client.provision(instance) + expect(state_poller).to_not have_received(:poll_service_instance_state) + end + end + + context 'when the broker returns the state as creating' do let(:response_data) do { state: 'creating', @@ -173,6 +197,33 @@ module VCAP::Services::ServiceBrokers::V2 expect(instance.state).to eq('creating') expect(instance.state_description).to eq('10% done') end + + it 'enqueues a polling job' do + client.provision(instance) + expect(state_poller).to have_received(:poll_service_instance_state).with(client_attrs, instance) + end + end + + context 'when the broker returns the state as failed' do + let(:response_data) do + { + state: 'failed', + state_description: '100% failed' + } + end + + it 'return immediately with the broker response' do + client = Client.new(client_attrs.merge(accept_unavailable: true)) + client.provision(instance) + + expect(instance.state).to eq('failed') + expect(instance.state_description).to eq('100% failed') + end + + it 'does not enqueue a polling job' do + client.provision(instance) + expect(state_poller).to_not have_received(:poll_service_instance_state) + end end context 'when provision fails' do @@ -235,6 +286,56 @@ module VCAP::Services::ServiceBrokers::V2 end end + describe '#fetch_service_instance_state' do + let(:plan) { VCAP::CloudController::ServicePlan.make } + let(:space) { VCAP::CloudController::Space.make } + let(:instance) do + VCAP::CloudController::ManagedServiceInstance.new( + service_plan: plan, + space: space + ) + end + + let(:response_data) do + { + 'dashboard_url' => 'bar', + 'state' => 'created', + 'state_description' => '100% created' + } + end + + let(:path) { "/v2/service_instances/#{instance.guid}" } + let(:response) { double('response') } + let(:response_body) { response_data.to_json } + let(:code) { '200' } + let(:message) { 'OK' } + + before do + allow(http_client).to receive(:get).and_return(response) + + allow(response).to receive(:body).and_return(response_body) + allow(response).to receive(:code).and_return(code) + allow(response).to receive(:message).and_return(message) + end + + it 'makes a put request with correct path' do + client.fetch_service_instance_state(instance) + + expect(http_client).to have_received(:get). + with("/v2/service_instances/#{instance.guid}") + end + + it 'returns the instance given with new state values' do + returned_instance = client.fetch_service_instance_state(instance) + + expect(returned_instance).to be(instance) + + expect(returned_instance.dashboard_url).to eq('bar') + expect(returned_instance.state).to eq('created') + expect(returned_instance.state_description).to eq('100% created') + end + end + describe '#update_service_plan' do let(:old_plan) { VCAP::CloudController::ServicePlan.make } let(:new_plan) { VCAP::CloudController::ServicePlan.make } diff --git a/spec/unit/lib/services/service_brokers/v2/service_instance_state_poller_spec.rb b/spec/unit/lib/services/service_brokers/v2/service_instance_state_poller_spec.rb new file mode 100644 index 00000000000..0e27f5705b0 --- /dev/null +++ b/spec/unit/lib/services/service_brokers/v2/service_instance_state_poller_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +module VCAP::Services + module ServiceBrokers::V2 + describe ServiceInstanceStatePoller do + let(:client_attrs) { { uri: 'broker.com' } } + + let(:plan) { VCAP::CloudController::ServicePlan.make } + let(:space) { VCAP::CloudController::Space.make } + let(:service_instance) { VCAP::CloudController::ManagedServiceInstance.new(service_plan: plan, space: space) } + + describe 'poll_service_instance_state' do + it 'enqueues a ServiceInstanceStateFetch job' do + mock_enqueuer = double(:enqueuer, enqueue: nil) + allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) + + ServiceInstanceStatePoller.new.poll_service_instance_state(client_attrs, service_instance) + + expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| + expect(opts[:queue]).to eq 'cc-generic' + expect(opts[:run_at]).to be_within(0.01).of(Delayed::Job.db_time_now) + + expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceStateFetch + expect(job.name).to eq 'service-instance-state-fetch' + expect(job.client_attrs).to eq client_attrs + expect(job.service_instance_guid).to eq service_instance.guid + expect(job.service_plan_guid).to eq service_instance.service_plan.guid + end + + expect(mock_enqueuer).to have_received(:enqueue) + end + end + end + end +end From 367d350d0652addb27a5485b1b4e30585452e940 Mon Sep 17 00:00:00 2001 From: Jeff Hui Date: Fri, 16 Jan 2015 11:34:45 -0800 Subject: [PATCH 65/76] Fixed RuboCop offense. --- app/jobs/services/service_instance_state_fetch.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/services/service_instance_state_fetch.rb b/app/jobs/services/service_instance_state_fetch.rb index 848bf43da14..753cbcfd2dc 100644 --- a/app/jobs/services/service_instance_state_fetch.rb +++ b/app/jobs/services/service_instance_state_fetch.rb @@ -13,7 +13,7 @@ def perform if service_instance.state != 'available' retry_job end - rescue HttpRequestError, Sequel::Error => e + rescue HttpRequestError, Sequel::Error retry_job end From 9df987a58c630643d1fc2e79784170eafb3d30c1 Mon Sep 17 00:00:00 2001 From: Jeff Hui Date: Fri, 16 Jan 2015 11:30:59 -0800 Subject: [PATCH 66/76] Renamed accept_unavailable to accept_incomplete. [#86259450] --- lib/services/service_brokers/v2/client.rb | 2 +- .../lib/services/service_brokers/v2/client_spec.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/services/service_brokers/v2/client.rb b/lib/services/service_brokers/v2/client.rb index f0820866d99..45677349e25 100644 --- a/lib/services/service_brokers/v2/client.rb +++ b/lib/services/service_brokers/v2/client.rb @@ -19,7 +19,7 @@ def catalog # The broker is expected to guarantee uniqueness of instance_id. # raises ServiceBrokerConflict if the id is already in use def provision(instance) - path = "/v2/service_instances/#{instance.guid}?accept_unavailable=true" + path = "/v2/service_instances/#{instance.guid}?accepts_incomplete=true" response = @http_client.put(path, { service_id: instance.service.broker_provided_id, diff --git a/spec/unit/lib/services/service_brokers/v2/client_spec.rb b/spec/unit/lib/services/service_brokers/v2/client_spec.rb index 56de382532e..6a40bc05410 100644 --- a/spec/unit/lib/services/service_brokers/v2/client_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/client_spec.rb @@ -102,7 +102,7 @@ module VCAP::Services::ServiceBrokers::V2 } end - let(:path) { "/v2/service_instances/#{instance.guid}?accept_unavailable=true" } + let(:path) { "/v2/service_instances/#{instance.guid}?accepts_incomplete=true" } let(:response) { double('response') } let(:response_body) { response_data.to_json } let(:code) { '201' } @@ -121,7 +121,7 @@ module VCAP::Services::ServiceBrokers::V2 client.provision(instance) expect(http_client).to have_received(:put). - with("/v2/service_instances/#{instance.guid}?accept_unavailable=true", anything) + with("/v2/service_instances/#{instance.guid}?accepts_incomplete=true", anything) end it 'makes a put request with correct message' do @@ -169,7 +169,7 @@ module VCAP::Services::ServiceBrokers::V2 end it 'return immediately with the broker response' do - client = Client.new(client_attrs.merge(accept_unavailable: true)) + client = Client.new(client_attrs.merge(accepts_incomplete: true)) client.provision(instance) expect(instance.state).to eq('available') @@ -191,7 +191,7 @@ module VCAP::Services::ServiceBrokers::V2 end it 'return immediately with the broker response' do - client = Client.new(client_attrs.merge(accept_unavailable: true)) + client = Client.new(client_attrs.merge(accepts_incomplete: true)) client.provision(instance) expect(instance.state).to eq('creating') @@ -213,7 +213,7 @@ module VCAP::Services::ServiceBrokers::V2 end it 'return immediately with the broker response' do - client = Client.new(client_attrs.merge(accept_unavailable: true)) + client = Client.new(client_attrs.merge(accepts_incomplete: true)) client.provision(instance) expect(instance.state).to eq('failed') From fee82f3b2ca068de3aa8e5ff4c3de4aa790afa9e Mon Sep 17 00:00:00 2001 From: Jeff Hui Date: Fri, 16 Jan 2015 12:31:09 -0800 Subject: [PATCH 67/76] Set poll interval for service instance fetch job to be 1 minute. [#84254704] --- app/jobs/services/service_instance_state_fetch.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/services/service_instance_state_fetch.rb b/app/jobs/services/service_instance_state_fetch.rb index 753cbcfd2dc..3520c64505a 100644 --- a/app/jobs/services/service_instance_state_fetch.rb +++ b/app/jobs/services/service_instance_state_fetch.rb @@ -28,7 +28,8 @@ def max_attempts private def retry_job - opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now } + poll_interval = 1.minute + opts = { queue: 'cc-generic', run_at: Delayed::Job.db_time_now + poll_interval } VCAP::CloudController::Jobs::Enqueuer.new(self, opts).enqueue end end From 8f0cabeece5ac386d66932630054d59267dc48e8 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 16 Jan 2015 12:01:40 -0800 Subject: [PATCH 68/76] Use initializers instead of depending on Struct.new for Job classes Because we are no longer using Struct.new, we lost some behavior that we previously got for free. Most importantly, Struct.new initializers allow all arguments to be optional. We chose to make only a few of these arguments optional in our change; only initializers that are ever actually invoked with fewer arguments retain the originial method signature. Another interesting change is that Struct redefines equality: two structs are equal if all of their fields are equal. We've removed any attempt to depend on that functionality. --- app/jobs/audit_event_job.rb | 13 ++++++++++++- app/jobs/exception_catching_job.rb | 8 +++++++- app/jobs/request_job.rb | 9 ++++++++- app/jobs/runtime/app_bits_packer.rb | 10 +++++++++- app/jobs/runtime/app_events_cleanup.rb | 8 +++++++- app/jobs/runtime/app_usage_events_cleanup.rb | 8 +++++++- app/jobs/runtime/blobstore_delete.rb | 10 +++++++++- app/jobs/runtime/buildpack_installer.rb | 10 +++++++++- app/jobs/runtime/droplet_deletion.rb | 9 ++++++++- app/jobs/runtime/events_cleanup.rb | 8 +++++++- app/jobs/runtime/failed_jobs_cleanup.rb | 8 +++++++- app/jobs/runtime/model_deletion.rb | 9 ++++++++- app/jobs/runtime/pending_packages_cleanup.rb | 8 +++++++- app/jobs/services/service_instance_deprovision.rb | 11 ++++++++++- app/jobs/services/service_instance_state_fetch.rb | 11 ++++++++++- app/jobs/services/service_instance_unbind.rb | 12 +++++++++++- app/jobs/timeout_job.rb | 8 +++++++- spec/unit/jobs/runtime/blobstore_delete_spec.rb | 2 +- spec/unit/jobs/runtime/buildpack_installer_spec.rb | 10 ++++++---- spec/unit/jobs/runtime/legacy_jobs_spec.rb | 4 ++++ spec/unit/jobs/runtime/model_deletion_spec.rb | 6 +++--- spec/unit/models/runtime/droplet_spec.rb | 9 +++++---- 22 files changed, 162 insertions(+), 29 deletions(-) diff --git a/app/jobs/audit_event_job.rb b/app/jobs/audit_event_job.rb index 77facfa8d01..7c6988015fa 100644 --- a/app/jobs/audit_event_job.rb +++ b/app/jobs/audit_event_job.rb @@ -1,6 +1,17 @@ module VCAP::CloudController module Jobs - class AuditEventJob < Struct.new(:job, :event_repository, :event_creation_method, :event_type, :model, :params) + class AuditEventJob + attr_accessor :job, :event_repository, :event_creation_method, :event_type, :model, :params + + def initialize(job, event_repository, event_creation_method, event_type, model, params={}) + @job = job + @event_repository = event_repository + @event_creation_method = event_creation_method + @event_type = event_type + @model = model + @params = params + end + def perform job.perform event_repository.send(event_creation_method, event_type, model, params) diff --git a/app/jobs/exception_catching_job.rb b/app/jobs/exception_catching_job.rb index 8a4c7f908ed..08f282bdfe1 100644 --- a/app/jobs/exception_catching_job.rb +++ b/app/jobs/exception_catching_job.rb @@ -2,7 +2,13 @@ module VCAP::CloudController module Jobs - class ExceptionCatchingJob < Struct.new(:handler) + class ExceptionCatchingJob + attr_accessor :handler + + def initialize(handler) + @handler = handler + end + def perform handler.perform end diff --git a/app/jobs/request_job.rb b/app/jobs/request_job.rb index fc864c44b42..fd9c59b4521 100644 --- a/app/jobs/request_job.rb +++ b/app/jobs/request_job.rb @@ -1,6 +1,13 @@ module VCAP::CloudController module Jobs - class RequestJob < Struct.new(:job, :request_id) + class RequestJob + attr_accessor :job, :request_id + + def initialize(job, request_id) + @job = job + @request_id = request_id + end + def perform current_request_id = ::VCAP::Request.current_id begin diff --git a/app/jobs/runtime/app_bits_packer.rb b/app/jobs/runtime/app_bits_packer.rb index 448a81fd4d9..8502257ba5b 100644 --- a/app/jobs/runtime/app_bits_packer.rb +++ b/app/jobs/runtime/app_bits_packer.rb @@ -4,7 +4,15 @@ module VCAP::CloudController module Jobs module Runtime - class AppBitsPacker < Struct.new(:app_guid, :uploaded_compressed_path, :fingerprints) + class AppBitsPacker + attr_accessor :app_guid, :uploaded_compressed_path, :fingerprints + + def initialize(app_guid, uploaded_compressed_path, fingerprints) + @app_guid = app_guid + @uploaded_compressed_path = uploaded_compressed_path + @fingerprints = fingerprints + end + def perform logger = Steno.logger('cc.background') logger.info("Packing the app bits for app '#{app_guid}'") diff --git a/app/jobs/runtime/app_events_cleanup.rb b/app/jobs/runtime/app_events_cleanup.rb index 211f7606a42..a013abc1a06 100644 --- a/app/jobs/runtime/app_events_cleanup.rb +++ b/app/jobs/runtime/app_events_cleanup.rb @@ -1,7 +1,13 @@ module VCAP::CloudController module Jobs module Runtime - class AppEventsCleanup < Struct.new(:cutoff_age_in_days) + class AppEventsCleanup + attr_accessor :cutoff_age_in_days + + def initialize(cutoff_age_in_days) + @cutoff_age_in_days = cutoff_age_in_days + end + def perform old_app_events = AppEvent.where("created_at < CURRENT_TIMESTAMP - INTERVAL '?' DAY", cutoff_age_in_days.to_i) logger = Steno.logger('cc.background') diff --git a/app/jobs/runtime/app_usage_events_cleanup.rb b/app/jobs/runtime/app_usage_events_cleanup.rb index 6ba15a9ab11..729c0935751 100644 --- a/app/jobs/runtime/app_usage_events_cleanup.rb +++ b/app/jobs/runtime/app_usage_events_cleanup.rb @@ -3,7 +3,13 @@ module VCAP::CloudController module Jobs module Runtime - class AppUsageEventsCleanup < Struct.new(:cutoff_age_in_days) + class AppUsageEventsCleanup + attr_accessor :cutoff_age_in_days + + def initialize(cutoff_age_in_days) + @cutoff_age_in_days = cutoff_age_in_days + end + def perform logger = Steno.logger('cc.background') logger.info('Cleaning up old AppUsageEvent rows') diff --git a/app/jobs/runtime/blobstore_delete.rb b/app/jobs/runtime/blobstore_delete.rb index 0a205acc4b5..be9e3e821de 100644 --- a/app/jobs/runtime/blobstore_delete.rb +++ b/app/jobs/runtime/blobstore_delete.rb @@ -1,7 +1,15 @@ module VCAP::CloudController module Jobs module Runtime - class BlobstoreDelete < Struct.new(:key, :blobstore_name, :attributes) + class BlobstoreDelete + attr_accessor :key, :blobstore_name, :attributes + + def initialize(key, blobstore_name, attributes=nil) + @key = key + @blobstore_name = blobstore_name + @attributes = attributes + end + def perform logger = Steno.logger('cc.background') logger.info("Attempting delete of '#{key}' from blobstore '#{blobstore_name}'") diff --git a/app/jobs/runtime/buildpack_installer.rb b/app/jobs/runtime/buildpack_installer.rb index ea275a53e9e..95b7606fa90 100644 --- a/app/jobs/runtime/buildpack_installer.rb +++ b/app/jobs/runtime/buildpack_installer.rb @@ -1,7 +1,15 @@ module VCAP::CloudController module Jobs module Runtime - class BuildpackInstaller < Struct.new(:name, :file, :opts, :config) + class BuildpackInstaller + attr_accessor :name, :file, :opts + + def initialize(name, file, opts) + @name = name + @file = file + @opts = opts + end + def perform logger = Steno.logger('cc.background') logger.info "Installing buildpack #{name}" diff --git a/app/jobs/runtime/droplet_deletion.rb b/app/jobs/runtime/droplet_deletion.rb index c1d7d6a39ad..6a8cde6f013 100644 --- a/app/jobs/runtime/droplet_deletion.rb +++ b/app/jobs/runtime/droplet_deletion.rb @@ -1,7 +1,14 @@ module VCAP::CloudController module Jobs module Runtime - class DropletDeletion < Struct.new(:new_droplet_key, :old_droplet_key) + class DropletDeletion + attr_accessor :new_droplet_key, :old_droplet_key + + def initialize(new_droplet_key, old_droplet_key) + @new_droplet_key = new_droplet_key + @old_droplet_key = old_droplet_key + end + def perform logger = Steno.logger('cc.background') logger.info("Deleting droplet '#{new_droplet_key}' (and '#{old_droplet_key}') from droplet blobstore") diff --git a/app/jobs/runtime/events_cleanup.rb b/app/jobs/runtime/events_cleanup.rb index d825636ce44..b1eac37299e 100644 --- a/app/jobs/runtime/events_cleanup.rb +++ b/app/jobs/runtime/events_cleanup.rb @@ -1,7 +1,13 @@ module VCAP::CloudController module Jobs module Runtime - class EventsCleanup < Struct.new(:cutoff_age_in_days) + class EventsCleanup + attr_accessor :cutoff_age_in_days + + def initialize(cutoff_age_in_days) + @cutoff_age_in_days = cutoff_age_in_days + end + def perform old_events = Event.where("created_at < CURRENT_TIMESTAMP - INTERVAL '?' DAY", cutoff_age_in_days.to_i) logger = Steno.logger('cc.background') diff --git a/app/jobs/runtime/failed_jobs_cleanup.rb b/app/jobs/runtime/failed_jobs_cleanup.rb index fd41b968eaa..6feddb28a9c 100644 --- a/app/jobs/runtime/failed_jobs_cleanup.rb +++ b/app/jobs/runtime/failed_jobs_cleanup.rb @@ -1,7 +1,13 @@ module VCAP::CloudController module Jobs module Runtime - class FailedJobsCleanup < Struct.new(:cutoff_age_in_days) + class FailedJobsCleanup + attr_accessor :cutoff_age_in_days + + def initialize(cutoff_age_in_days) + @cutoff_age_in_days = cutoff_age_in_days + end + def perform old_delayed_jobs = Delayed::Job. where('failed_at is not null'). diff --git a/app/jobs/runtime/model_deletion.rb b/app/jobs/runtime/model_deletion.rb index 396a8e67f52..cb5a5f3f2aa 100644 --- a/app/jobs/runtime/model_deletion.rb +++ b/app/jobs/runtime/model_deletion.rb @@ -1,7 +1,14 @@ module VCAP::CloudController module Jobs module Runtime - class ModelDeletion < Struct.new(:model_class, :guid) + class ModelDeletion + attr_accessor :model_class, :guid + + def initialize(model_class, guid) + @model_class = model_class + @guid = guid + end + def perform logger = Steno.logger('cc.background') logger.info("Deleting model class '#{model_class}' with guid '#{guid}'") diff --git a/app/jobs/runtime/pending_packages_cleanup.rb b/app/jobs/runtime/pending_packages_cleanup.rb index af985198494..9cb8d040e65 100644 --- a/app/jobs/runtime/pending_packages_cleanup.rb +++ b/app/jobs/runtime/pending_packages_cleanup.rb @@ -1,7 +1,13 @@ module VCAP::CloudController module Jobs module Runtime - class PendingPackagesCleanup < Struct.new(:expiration_in_seconds) + class PendingPackagesCleanup + attr_accessor :expiration_in_seconds + + def initialize(expiration_in_seconds) + @expiration_in_seconds = expiration_in_seconds + end + def perform App.where("package_pending_since < ? - INTERVAL '?' SECOND", Sequel::CURRENT_TIMESTAMP, expiration_in_seconds.to_i).update( package_state: 'FAILED', diff --git a/app/jobs/services/service_instance_deprovision.rb b/app/jobs/services/service_instance_deprovision.rb index 4586d99ed57..26425246853 100644 --- a/app/jobs/services/service_instance_deprovision.rb +++ b/app/jobs/services/service_instance_deprovision.rb @@ -1,7 +1,16 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceDeprovision < Struct.new(:name, :client_attrs, :service_instance_guid, :service_plan_guid) + class ServiceInstanceDeprovision + attr_accessor :name, :client_attrs, :service_instance_guid, :service_plan_guid + + def initialize(name, client_attrs, service_instance_guid, service_plan_guid) + @name = name + @client_attrs = client_attrs + @service_instance_guid = service_instance_guid + @service_plan_guid = service_plan_guid + end + def perform logger = Steno.logger('cc-background') logger.info('There was an error during service instance provisioning. Attempting to delete potentially orphaned instance.') diff --git a/app/jobs/services/service_instance_state_fetch.rb b/app/jobs/services/service_instance_state_fetch.rb index 3520c64505a..6faced35cd0 100644 --- a/app/jobs/services/service_instance_state_fetch.rb +++ b/app/jobs/services/service_instance_state_fetch.rb @@ -1,7 +1,16 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceStateFetch < Struct.new(:name, :client_attrs, :service_instance_guid, :service_plan_guid) + class ServiceInstanceStateFetch + attr_accessor :name, :client_attrs, :service_instance_guid, :service_plan_guid + + def initialize(name, client_attrs, service_instance_guid, service_plan_guid) + @name = name + @client_attrs = client_attrs + @service_instance_guid = service_instance_guid + @service_plan_guid = service_plan_guid + end + def perform client = VCAP::Services::ServiceBrokers::V2::Client.new(client_attrs) service_plan = ServicePlan.first(guid: service_plan_guid) diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index fa19fecd806..43c02dabea2 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -1,7 +1,17 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceUnbind < Struct.new(:name, :client_attrs, :binding_guid, :service_instance_guid, :app_guid) + class ServiceInstanceUnbind + attr_accessor :name, :client_attrs, :binding_guid, :service_instance_guid, :app_guid + + def initialize(name, client_attrs, binding_guid, service_instance_guid, app_guid) + @name = name + @client_attrs = client_attrs + @binding_guid = binding_guid + @service_instance_guid = service_instance_guid + @app_guid = app_guid + end + def perform logger = Steno.logger('cc-background') logger.info('There was an error during service binding creation. Attempting to delete potentially orphaned binding.') diff --git a/app/jobs/timeout_job.rb b/app/jobs/timeout_job.rb index 3c77a5d3782..bcd66c59fba 100644 --- a/app/jobs/timeout_job.rb +++ b/app/jobs/timeout_job.rb @@ -1,6 +1,12 @@ module VCAP::CloudController module Jobs - class TimeoutJob < Struct.new(:job) + class TimeoutJob + attr_accessor :job + + def initialize(job) + @job = job + end + def perform Timeout.timeout max_run_time(job.job_name_in_configuration) do job.perform diff --git a/spec/unit/jobs/runtime/blobstore_delete_spec.rb b/spec/unit/jobs/runtime/blobstore_delete_spec.rb index ebfbba61c6c..a19333ccf60 100644 --- a/spec/unit/jobs/runtime/blobstore_delete_spec.rb +++ b/spec/unit/jobs/runtime/blobstore_delete_spec.rb @@ -4,7 +4,7 @@ module VCAP::CloudController module Jobs::Runtime describe BlobstoreDelete do let(:key) { 'key' } - let(:job) do + subject(:job) do BlobstoreDelete.new(key, :droplet_blobstore) end diff --git a/spec/unit/jobs/runtime/buildpack_installer_spec.rb b/spec/unit/jobs/runtime/buildpack_installer_spec.rb index c79ba2fb099..a9f3b9325c1 100644 --- a/spec/unit/jobs/runtime/buildpack_installer_spec.rb +++ b/spec/unit/jobs/runtime/buildpack_installer_spec.rb @@ -10,9 +10,11 @@ module Jobs::Runtime let(:options) { { enabled: true, locked: false, position: 1 } } - let(:job) { BuildpackInstaller.new(buildpack_name, zipfile, options, TestConfig.config) } + let(:job) { BuildpackInstaller.new(buildpack_name, zipfile, options) } - it { is_expected.to be_a_valid_job } + it 'is a valid job' do + expect(job).to be_a_valid_job + end describe '#perform' do context 'when the buildpack is enabled and unlocked' do @@ -34,7 +36,7 @@ module Jobs::Runtime it 'updates an existing buildpack' do buildpack1 = Buildpack.make(name: buildpack_name, key: 'new_key') - update_job = BuildpackInstaller.new(buildpack_name, zipfile2, { enabled: false }, TestConfig.config) + update_job = BuildpackInstaller.new(buildpack_name, zipfile2, { enabled: false }) update_job.perform buildpack2 = Buildpack.find(name: buildpack_name) @@ -48,7 +50,7 @@ module Jobs::Runtime context 'when the buildpack is locked' do it 'fails to update a locked buildpack' do buildpack = Buildpack.make(name: buildpack_name, locked: true) - update_job = BuildpackInstaller.new(buildpack_name, zipfile2, { enabled: false, locked: false }, TestConfig.config) + update_job = BuildpackInstaller.new(buildpack_name, zipfile2, { enabled: false, locked: false }) update_job.perform buildpack2 = Buildpack.find(name: buildpack_name) diff --git a/spec/unit/jobs/runtime/legacy_jobs_spec.rb b/spec/unit/jobs/runtime/legacy_jobs_spec.rb index 97ca2860eb4..687e2daa249 100644 --- a/spec/unit/jobs/runtime/legacy_jobs_spec.rb +++ b/spec/unit/jobs/runtime/legacy_jobs_spec.rb @@ -2,10 +2,12 @@ describe 'Legacy Jobs' do describe ::AppBitsPackerJob do + subject { ::AppBitsPackerJob.new('app-guid', 'path/to/compressed/file', 'the-fingerprint') } it { is_expected.to be_a(VCAP::CloudController::Jobs::Runtime::AppBitsPacker) } end describe ::BlobstoreDelete do + subject { ::BlobstoreDelete.new('key', 'blobstore-name') } it { is_expected.to be_a(VCAP::CloudController::Jobs::Runtime::BlobstoreDelete) } end @@ -15,6 +17,7 @@ end describe ::DropletDeletionJob do + subject { ::DropletDeletionJob.new('new-key', 'old-key') } it { is_expected.to be_a(VCAP::CloudController::Jobs::Runtime::DropletDeletion) } end @@ -24,6 +27,7 @@ end describe ::ModelDeletionJob do + subject { ::ModelDeletionJob.new(VCAP::CloudController::Space, 'space-guid') } it { is_expected.to be_a(VCAP::CloudController::Jobs::Runtime::ModelDeletion) } end end diff --git a/spec/unit/jobs/runtime/model_deletion_spec.rb b/spec/unit/jobs/runtime/model_deletion_spec.rb index d1fe7bb5977..1a70a53b9cf 100644 --- a/spec/unit/jobs/runtime/model_deletion_spec.rb +++ b/spec/unit/jobs/runtime/model_deletion_spec.rb @@ -6,14 +6,14 @@ module VCAP::CloudController module Jobs::Runtime describe ModelDeletion do + let(:space) { Space.make } + subject(:job) { ModelDeletion.new(Space, space.guid) } + it { is_expected.to be_a_valid_job } describe '#perform' do - let(:space) { Space.make } let!(:app) { App.make(space: space) } - subject(:job) { ModelDeletion.new(Space, space.guid) } - context 'deleting a space' do it 'can delete the space' do expect { job.perform }.to change { Space.count }.by(-1) diff --git a/spec/unit/models/runtime/droplet_spec.rb b/spec/unit/models/runtime/droplet_spec.rb index 673529432b7..c17df9fa68e 100644 --- a/spec/unit/models/runtime/droplet_spec.rb +++ b/spec/unit/models/runtime/droplet_spec.rb @@ -60,10 +60,11 @@ module VCAP::CloudController app = AppFactory.make droplet = app.current_droplet enqueuer = double('Enqueuer', enqueue: true) - expect(Jobs::Enqueuer).to receive(:new).with( - Jobs::Runtime::DropletDeletion.new(droplet.new_blobstore_key, droplet.old_blobstore_key), - queue: 'cc-generic' - ).and_return(enqueuer) + expect(Jobs::Enqueuer).to receive(:new) do |job, opts| + expect(job.new_droplet_key).to eq droplet.new_blobstore_key + expect(job.old_droplet_key).to eq droplet.old_blobstore_key + expect(opts[:queue]).to eq 'cc-generic' + end.and_return(enqueuer) droplet.destroy end end From 341cea9298a8029fc522d786919934f8b062c2e2 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 16 Jan 2015 14:29:16 -0800 Subject: [PATCH 69/76] Make all jobs inherit from CCJob --- app/jobs/audit_event_job.rb | 2 +- app/jobs/cc_job.rb | 9 +++++++++ app/jobs/exception_catching_job.rb | 2 +- app/jobs/request_job.rb | 2 +- app/jobs/runtime/app_bits_copier.rb | 2 +- app/jobs/runtime/app_bits_packer.rb | 2 +- app/jobs/runtime/app_events_cleanup.rb | 2 +- app/jobs/runtime/app_usage_events_cleanup.rb | 2 +- app/jobs/runtime/blobstore_delete.rb | 2 +- app/jobs/runtime/blobstore_upload.rb | 2 +- app/jobs/runtime/buildpack_installer.rb | 2 +- app/jobs/runtime/droplet_deletion.rb | 2 +- app/jobs/runtime/droplet_upload.rb | 2 +- app/jobs/runtime/events_cleanup.rb | 2 +- app/jobs/runtime/failed_jobs_cleanup.rb | 2 +- app/jobs/runtime/model_deletion.rb | 2 +- app/jobs/runtime/pending_packages_cleanup.rb | 2 +- .../services/service_instance_deprovision.rb | 2 +- .../services/service_instance_state_fetch.rb | 2 +- app/jobs/services/service_instance_unbind.rb | 2 +- app/jobs/timeout_job.rb | 2 +- lib/cloud_controller/jobs.rb | 1 + spec/unit/jobs/cc_job_spec.rb | 16 ++++++++++++++++ 23 files changed, 46 insertions(+), 20 deletions(-) create mode 100644 app/jobs/cc_job.rb create mode 100644 spec/unit/jobs/cc_job_spec.rb diff --git a/app/jobs/audit_event_job.rb b/app/jobs/audit_event_job.rb index 7c6988015fa..7c106871695 100644 --- a/app/jobs/audit_event_job.rb +++ b/app/jobs/audit_event_job.rb @@ -1,6 +1,6 @@ module VCAP::CloudController module Jobs - class AuditEventJob + class AuditEventJob < VCAP::CloudController::Jobs::CCJob attr_accessor :job, :event_repository, :event_creation_method, :event_type, :model, :params def initialize(job, event_repository, event_creation_method, event_type, model, params={}) diff --git a/app/jobs/cc_job.rb b/app/jobs/cc_job.rb new file mode 100644 index 00000000000..cade751539f --- /dev/null +++ b/app/jobs/cc_job.rb @@ -0,0 +1,9 @@ +module VCAP::CloudController + module Jobs + class CCJob + def reschedule_at(time, attempts) + time + (attempts**4) + 5 + end + end + end +end diff --git a/app/jobs/exception_catching_job.rb b/app/jobs/exception_catching_job.rb index 08f282bdfe1..9d275ee2512 100644 --- a/app/jobs/exception_catching_job.rb +++ b/app/jobs/exception_catching_job.rb @@ -2,7 +2,7 @@ module VCAP::CloudController module Jobs - class ExceptionCatchingJob + class ExceptionCatchingJob < VCAP::CloudController::Jobs::CCJob attr_accessor :handler def initialize(handler) diff --git a/app/jobs/request_job.rb b/app/jobs/request_job.rb index fd9c59b4521..690cf1b625c 100644 --- a/app/jobs/request_job.rb +++ b/app/jobs/request_job.rb @@ -1,6 +1,6 @@ module VCAP::CloudController module Jobs - class RequestJob + class RequestJob < VCAP::CloudController::Jobs::CCJob attr_accessor :job, :request_id def initialize(job, request_id) diff --git a/app/jobs/runtime/app_bits_copier.rb b/app/jobs/runtime/app_bits_copier.rb index aa119c9f96f..2c4628bc516 100644 --- a/app/jobs/runtime/app_bits_copier.rb +++ b/app/jobs/runtime/app_bits_copier.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class AppBitsCopier + class AppBitsCopier < VCAP::CloudController::Jobs::CCJob def initialize(src_app, dest_app, app_event_repo, user, email) @user = user @email = email diff --git a/app/jobs/runtime/app_bits_packer.rb b/app/jobs/runtime/app_bits_packer.rb index 8502257ba5b..d884b0a9b4c 100644 --- a/app/jobs/runtime/app_bits_packer.rb +++ b/app/jobs/runtime/app_bits_packer.rb @@ -4,7 +4,7 @@ module VCAP::CloudController module Jobs module Runtime - class AppBitsPacker + class AppBitsPacker < VCAP::CloudController::Jobs::CCJob attr_accessor :app_guid, :uploaded_compressed_path, :fingerprints def initialize(app_guid, uploaded_compressed_path, fingerprints) diff --git a/app/jobs/runtime/app_events_cleanup.rb b/app/jobs/runtime/app_events_cleanup.rb index a013abc1a06..1dfbaa24cbd 100644 --- a/app/jobs/runtime/app_events_cleanup.rb +++ b/app/jobs/runtime/app_events_cleanup.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class AppEventsCleanup + class AppEventsCleanup < VCAP::CloudController::Jobs::CCJob attr_accessor :cutoff_age_in_days def initialize(cutoff_age_in_days) diff --git a/app/jobs/runtime/app_usage_events_cleanup.rb b/app/jobs/runtime/app_usage_events_cleanup.rb index 729c0935751..138d5fa9e2e 100644 --- a/app/jobs/runtime/app_usage_events_cleanup.rb +++ b/app/jobs/runtime/app_usage_events_cleanup.rb @@ -3,7 +3,7 @@ module VCAP::CloudController module Jobs module Runtime - class AppUsageEventsCleanup + class AppUsageEventsCleanup < VCAP::CloudController::Jobs::CCJob attr_accessor :cutoff_age_in_days def initialize(cutoff_age_in_days) diff --git a/app/jobs/runtime/blobstore_delete.rb b/app/jobs/runtime/blobstore_delete.rb index be9e3e821de..0a0ad60125f 100644 --- a/app/jobs/runtime/blobstore_delete.rb +++ b/app/jobs/runtime/blobstore_delete.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class BlobstoreDelete + class BlobstoreDelete < VCAP::CloudController::Jobs::CCJob attr_accessor :key, :blobstore_name, :attributes def initialize(key, blobstore_name, attributes=nil) diff --git a/app/jobs/runtime/blobstore_upload.rb b/app/jobs/runtime/blobstore_upload.rb index 6b5c59aa43e..d2c1327495e 100644 --- a/app/jobs/runtime/blobstore_upload.rb +++ b/app/jobs/runtime/blobstore_upload.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class BlobstoreUpload + class BlobstoreUpload < VCAP::CloudController::Jobs::CCJob attr_reader :local_path, :blobstore_key, :blobstore_name attr_reader :max_attempts diff --git a/app/jobs/runtime/buildpack_installer.rb b/app/jobs/runtime/buildpack_installer.rb index 95b7606fa90..cfac8f1c698 100644 --- a/app/jobs/runtime/buildpack_installer.rb +++ b/app/jobs/runtime/buildpack_installer.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class BuildpackInstaller + class BuildpackInstaller < VCAP::CloudController::Jobs::CCJob attr_accessor :name, :file, :opts def initialize(name, file, opts) diff --git a/app/jobs/runtime/droplet_deletion.rb b/app/jobs/runtime/droplet_deletion.rb index 6a8cde6f013..1586584ea47 100644 --- a/app/jobs/runtime/droplet_deletion.rb +++ b/app/jobs/runtime/droplet_deletion.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class DropletDeletion + class DropletDeletion < VCAP::CloudController::Jobs::CCJob attr_accessor :new_droplet_key, :old_droplet_key def initialize(new_droplet_key, old_droplet_key) diff --git a/app/jobs/runtime/droplet_upload.rb b/app/jobs/runtime/droplet_upload.rb index a2b0779d04a..83a4d4e3bde 100644 --- a/app/jobs/runtime/droplet_upload.rb +++ b/app/jobs/runtime/droplet_upload.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class DropletUpload + class DropletUpload < VCAP::CloudController::Jobs::CCJob attr_reader :local_path, :app_id attr_reader :max_attempts diff --git a/app/jobs/runtime/events_cleanup.rb b/app/jobs/runtime/events_cleanup.rb index b1eac37299e..efe86ca4cef 100644 --- a/app/jobs/runtime/events_cleanup.rb +++ b/app/jobs/runtime/events_cleanup.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class EventsCleanup + class EventsCleanup < VCAP::CloudController::Jobs::CCJob attr_accessor :cutoff_age_in_days def initialize(cutoff_age_in_days) diff --git a/app/jobs/runtime/failed_jobs_cleanup.rb b/app/jobs/runtime/failed_jobs_cleanup.rb index 6feddb28a9c..a85677b6e86 100644 --- a/app/jobs/runtime/failed_jobs_cleanup.rb +++ b/app/jobs/runtime/failed_jobs_cleanup.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class FailedJobsCleanup + class FailedJobsCleanup < VCAP::CloudController::Jobs::CCJob attr_accessor :cutoff_age_in_days def initialize(cutoff_age_in_days) diff --git a/app/jobs/runtime/model_deletion.rb b/app/jobs/runtime/model_deletion.rb index cb5a5f3f2aa..fadfae73fbb 100644 --- a/app/jobs/runtime/model_deletion.rb +++ b/app/jobs/runtime/model_deletion.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class ModelDeletion + class ModelDeletion < VCAP::CloudController::Jobs::CCJob attr_accessor :model_class, :guid def initialize(model_class, guid) diff --git a/app/jobs/runtime/pending_packages_cleanup.rb b/app/jobs/runtime/pending_packages_cleanup.rb index 9cb8d040e65..310e3e98f1f 100644 --- a/app/jobs/runtime/pending_packages_cleanup.rb +++ b/app/jobs/runtime/pending_packages_cleanup.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Runtime - class PendingPackagesCleanup + class PendingPackagesCleanup < VCAP::CloudController::Jobs::CCJob attr_accessor :expiration_in_seconds def initialize(expiration_in_seconds) diff --git a/app/jobs/services/service_instance_deprovision.rb b/app/jobs/services/service_instance_deprovision.rb index 26425246853..be090b138ba 100644 --- a/app/jobs/services/service_instance_deprovision.rb +++ b/app/jobs/services/service_instance_deprovision.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceDeprovision + class ServiceInstanceDeprovision < VCAP::CloudController::Jobs::CCJob attr_accessor :name, :client_attrs, :service_instance_guid, :service_plan_guid def initialize(name, client_attrs, service_instance_guid, service_plan_guid) diff --git a/app/jobs/services/service_instance_state_fetch.rb b/app/jobs/services/service_instance_state_fetch.rb index 6faced35cd0..8d8ecc96222 100644 --- a/app/jobs/services/service_instance_state_fetch.rb +++ b/app/jobs/services/service_instance_state_fetch.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceStateFetch + class ServiceInstanceStateFetch < VCAP::CloudController::Jobs::CCJob attr_accessor :name, :client_attrs, :service_instance_guid, :service_plan_guid def initialize(name, client_attrs, service_instance_guid, service_plan_guid) diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index 43c02dabea2..d1adc66a025 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Jobs module Services - class ServiceInstanceUnbind + class ServiceInstanceUnbind < VCAP::CloudController::Jobs::CCJob attr_accessor :name, :client_attrs, :binding_guid, :service_instance_guid, :app_guid def initialize(name, client_attrs, binding_guid, service_instance_guid, app_guid) diff --git a/app/jobs/timeout_job.rb b/app/jobs/timeout_job.rb index bcd66c59fba..3ced82761c3 100644 --- a/app/jobs/timeout_job.rb +++ b/app/jobs/timeout_job.rb @@ -1,6 +1,6 @@ module VCAP::CloudController module Jobs - class TimeoutJob + class TimeoutJob < VCAP::CloudController::Jobs::CCJob attr_accessor :job def initialize(job) diff --git a/lib/cloud_controller/jobs.rb b/lib/cloud_controller/jobs.rb index 1074a5c42c8..3b5c0d93aa8 100644 --- a/lib/cloud_controller/jobs.rb +++ b/lib/cloud_controller/jobs.rb @@ -1,3 +1,4 @@ +require 'jobs/cc_job' require 'jobs/runtime/app_bits_packer' require 'jobs/runtime/app_bits_copier' require 'jobs/runtime/app_events_cleanup' diff --git a/spec/unit/jobs/cc_job_spec.rb b/spec/unit/jobs/cc_job_spec.rb new file mode 100644 index 00000000000..14852245989 --- /dev/null +++ b/spec/unit/jobs/cc_job_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +module VCAP::CloudController + module Jobs + describe CCJob do + describe '#reschedule_at' do + it 'uses the default from Delayed::Job' do + time = Time.now + attempts = 5 + job = CCJob.new + expect(job.reschedule_at(time, attempts)).to eq(time + (attempts**4) + 5) + end + end + end + end +end From 1672eedaaaacb15bfa1c264c2215e7dfc1608e97 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 16 Jan 2015 15:15:30 -0800 Subject: [PATCH 70/76] Wrapper jobs delegate reschedule_at to inner jobs [finishes #83710826] [finishes #83710822] --- app/jobs/audit_event_job.rb | 4 +++ app/jobs/exception_catching_job.rb | 4 +++ app/jobs/request_job.rb | 4 +++ app/jobs/timeout_job.rb | 4 +++ spec/api/api_version_spec.rb | 2 +- spec/api/documentation/job_api_spec.rb | 4 +++ spec/unit/jobs/audit_event_job_spec.rb | 26 ++++++++++++++ spec/unit/jobs/exception_catching_job_spec.rb | 10 +++++- spec/unit/jobs/request_job_spec.rb | 10 +++++- spec/unit/jobs/timeout_job_spec.rb | 14 ++++++++ .../v2/orphan_mitigator_spec.rb | 34 ++++++++++++++++--- 11 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 spec/unit/jobs/audit_event_job_spec.rb diff --git a/app/jobs/audit_event_job.rb b/app/jobs/audit_event_job.rb index 7c106871695..0a8308b09b6 100644 --- a/app/jobs/audit_event_job.rb +++ b/app/jobs/audit_event_job.rb @@ -24,6 +24,10 @@ def job_name_in_configuration def max_attempts 1 end + + def reschedule_at(time, attempts) + job.reschedule_at(time, attempts) + end end end end diff --git a/app/jobs/exception_catching_job.rb b/app/jobs/exception_catching_job.rb index 9d275ee2512..155844855bf 100644 --- a/app/jobs/exception_catching_job.rb +++ b/app/jobs/exception_catching_job.rb @@ -23,6 +23,10 @@ def max_attempts handler.max_attempts end + def reschedule_at(time, attempts) + handler.reschedule_at(time, attempts) + end + private def save_error(error_presenter, job) diff --git a/app/jobs/request_job.rb b/app/jobs/request_job.rb index 690cf1b625c..eae5b054026 100644 --- a/app/jobs/request_job.rb +++ b/app/jobs/request_job.rb @@ -21,6 +21,10 @@ def perform def max_attempts job.max_attempts end + + def reschedule_at(time, attempts) + job.reschedule_at(time, attempts) + end end end end diff --git a/app/jobs/timeout_job.rb b/app/jobs/timeout_job.rb index 3ced82761c3..3f0233c598b 100644 --- a/app/jobs/timeout_job.rb +++ b/app/jobs/timeout_job.rb @@ -24,6 +24,10 @@ def max_run_time(job_name_in_configuration) job_config = jobs_config[job_name_in_configuration] || jobs_config[:global] job_config[:timeout_in_seconds] end + + def reschedule_at(time, attempts) + job.reschedule_at(time, attempts) + end end end end diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index c6ff4aa6adc..af21f948633 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = 'cfaef9aa6c8588cfd6a23c3d89c20ce2f9857211' + API_FOLDER_CHECKSUM = '069366f2ce7ebc74edc8e561317dcd3e4ce92594' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/job_api_spec.rb b/spec/api/documentation/job_api_spec.rb index 3ef4ffdb842..ef849c811c4 100644 --- a/spec/api/documentation/job_api_spec.rb +++ b/spec/api/documentation/job_api_spec.rb @@ -19,6 +19,10 @@ def job_name_in_configuration def max_attempts 2 end + + def reschedule_at(time, attempts) + Time.now + 5 + end end before { Delayed::Job.delete_all } diff --git a/spec/unit/jobs/audit_event_job_spec.rb b/spec/unit/jobs/audit_event_job_spec.rb new file mode 100644 index 00000000000..bbe35adfb0d --- /dev/null +++ b/spec/unit/jobs/audit_event_job_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +module VCAP::CloudController + module Jobs + describe AuditEventJob do + let(:event_repository) { double(:event_repository) } + let(:event_creation_method) { :record_service_creation_event } + let(:event_type) { 'audit.service.create' } + let(:model) { Service.make } + let(:params) { {} } + + subject(:audit_event_job) do + AuditEventJob.new(job, event_repository, event_creation_method, event_type, model, params) + end + let(:job) { double(:job, perform: 'fake-perform', max_attempts: 1, reschedule_at: Time.now) } + + describe '#reschedule_at' do + it 'delegates to the handler' do + time = Time.now + attempts = 5 + expect(audit_event_job.reschedule_at(time, attempts)).to eq(job.reschedule_at(time, attempts)) + end + end + end + end +end diff --git a/spec/unit/jobs/exception_catching_job_spec.rb b/spec/unit/jobs/exception_catching_job_spec.rb index 81f07c43440..67cf47c402f 100644 --- a/spec/unit/jobs/exception_catching_job_spec.rb +++ b/spec/unit/jobs/exception_catching_job_spec.rb @@ -7,7 +7,7 @@ module Jobs ExceptionCatchingJob.new(handler) end - let(:handler) { double('Handler', perform: 'fake-perform', max_attempts: 1) } + let(:handler) { double('Handler', perform: 'fake-perform', max_attempts: 1, reschedule_at: Time.now) } context '#perform' do it 'delegates to the handler' do @@ -64,6 +64,14 @@ module Jobs exception_catching_job.error(job, 'exception') end end + + describe '#reschedule_at' do + it 'delegates to the handler' do + time = Time.now + attempts = 5 + expect(exception_catching_job.reschedule_at(time, attempts)).to eq(handler.reschedule_at(time, attempts)) + end + end end end end diff --git a/spec/unit/jobs/request_job_spec.rb b/spec/unit/jobs/request_job_spec.rb index 716136c2200..e70b6a091d0 100644 --- a/spec/unit/jobs/request_job_spec.rb +++ b/spec/unit/jobs/request_job_spec.rb @@ -2,7 +2,7 @@ module VCAP::CloudController::Jobs describe RequestJob do - let(:wrapped_job) { double('InnerJob', max_attempts: 2) } + let(:wrapped_job) { double('InnerJob', max_attempts: 2, reschedule_at: Time.now) } let(:request_id) { 'abc123' } subject(:request_job) { RequestJob.new(wrapped_job, request_id) } @@ -52,5 +52,13 @@ module VCAP::CloudController::Jobs expect(subject.max_attempts).to eq(2) end end + + describe '#reschedule_at' do + it 'delegates to the inner job' do + time = Time.now + attempts = 5 + expect(request_job.reschedule_at(time, attempts)).to eq(wrapped_job.reschedule_at(time, attempts)) + end + end end end diff --git a/spec/unit/jobs/timeout_job_spec.rb b/spec/unit/jobs/timeout_job_spec.rb index 53d8235d717..62adbb7b766 100644 --- a/spec/unit/jobs/timeout_job_spec.rb +++ b/spec/unit/jobs/timeout_job_spec.rb @@ -67,5 +67,19 @@ module VCAP::CloudController::Jobs end end end + + describe '#reschedule_at' do + before do + allow(job).to receive(:reschedule_at) do |time, attempts| + time + attempts + end + end + + it 'defers to the inner job' do + time = Time.now + attempts = 5 + expect(timeout_job.reschedule_at(time, attempts)).to eq(job.reschedule_at(time, attempts)) + end + end end end diff --git a/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb index 4d307bab7ff..e3413fbcc3e 100644 --- a/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb +++ b/spec/unit/lib/services/service_brokers/v2/orphan_mitigator_spec.rb @@ -8,7 +8,7 @@ module ServiceBrokers::V2 let(:plan) { VCAP::CloudController::ServicePlan.make } let(:space) { VCAP::CloudController::Space.make } let(:service_instance) { VCAP::CloudController::ManagedServiceInstance.new(service_plan: plan, space: space) } - let(:binding) do + let(:service_binding) do VCAP::CloudController::ServiceBinding.make( binding_options: { 'this' => 'that' } ) @@ -35,6 +35,18 @@ module ServiceBrokers::V2 expect(mock_enqueuer).to have_received(:enqueue) end + + specify 'the enqueued job has a reschedule_at define such that exponential backoff occurs' do + now = Time.now + + OrphanMitigator.new.cleanup_failed_provision(client_attrs, service_instance) + job = Delayed::Job.first.payload_object + expect(job).to respond_to :reschedule_at + + 10.times do |attempt| + expect(job.reschedule_at(now, attempt)).to be_within(0.01).of(now + (2**attempt).minutes) + end + end end describe 'cleanup_failed_bind' do @@ -42,7 +54,7 @@ module ServiceBrokers::V2 mock_enqueuer = double(:enqueuer, enqueue: nil) allow(VCAP::CloudController::Jobs::Enqueuer).to receive(:new).and_return(mock_enqueuer) - OrphanMitigator.new.cleanup_failed_bind(client_attrs, binding) + OrphanMitigator.new.cleanup_failed_bind(client_attrs, service_binding) expect(VCAP::CloudController::Jobs::Enqueuer).to have_received(:new) do |job, opts| expect(opts[:queue]).to eq 'cc-generic' @@ -51,13 +63,25 @@ module ServiceBrokers::V2 expect(job).to be_a VCAP::CloudController::Jobs::Services::ServiceInstanceUnbind expect(job.name).to eq 'service-instance-unbind' expect(job.client_attrs).to eq client_attrs - expect(job.binding_guid).to eq binding.guid - expect(job.service_instance_guid).to eq binding.service_instance.guid - expect(job.app_guid).to eq binding.app.guid + expect(job.binding_guid).to eq service_binding.guid + expect(job.service_instance_guid).to eq service_binding.service_instance.guid + expect(job.app_guid).to eq service_binding.app.guid end expect(mock_enqueuer).to have_received(:enqueue) end + + specify 'the enqueued job has a reschedule_at define such that exponential backoff occurs' do + now = Time.now + + OrphanMitigator.new.cleanup_failed_bind(client_attrs, service_binding) + job = Delayed::Job.first.payload_object + expect(job).to respond_to :reschedule_at + + 10.times do |attempt| + expect(job.reschedule_at(now, attempt)).to be_within(0.01).of(now + (2**attempt).minutes) + end + end end end end From c6c85147979feac513b604626750492c021943a5 Mon Sep 17 00:00:00 2001 From: David Sabeti and Whitney Schaefer Date: Fri, 16 Jan 2015 15:29:33 -0800 Subject: [PATCH 71/76] Rename binding => service_binding --- app/jobs/services/service_instance_unbind.rb | 5 ++--- spec/unit/jobs/services/service_instance_unbind_spec.rb | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/app/jobs/services/service_instance_unbind.rb b/app/jobs/services/service_instance_unbind.rb index d1adc66a025..a1acb3ee5c5 100644 --- a/app/jobs/services/service_instance_unbind.rb +++ b/app/jobs/services/service_instance_unbind.rb @@ -20,9 +20,8 @@ def perform app = VCAP::CloudController::App.first(guid: app_guid) service_instance = VCAP::CloudController::ServiceInstance.first(guid: service_instance_guid) - binding = VCAP::CloudController::ServiceBinding.new(guid: binding_guid, app: app, service_instance: service_instance) - - client.unbind(binding) + service_binding = VCAP::CloudController::ServiceBinding.new(guid: binding_guid, app: app, service_instance: service_instance) + client.unbind(service_binding) end def job_name_in_configuration diff --git a/spec/unit/jobs/services/service_instance_unbind_spec.rb b/spec/unit/jobs/services/service_instance_unbind_spec.rb index 2d0c00cd05f..534de26860c 100644 --- a/spec/unit/jobs/services/service_instance_unbind_spec.rb +++ b/spec/unit/jobs/services/service_instance_unbind_spec.rb @@ -8,9 +8,9 @@ module Jobs::Services let(:app_guid) { 'fake-app-guid' } let(:binding_guid) { 'fake-binding-guid' } - let(:binding) { instance_double('VCAP::CloudController::ServiceBinding') } + let(:service_binding) { instance_double('VCAP::CloudController::ServiceBinding') } before do - allow(VCAP::CloudController::ServiceBinding).to receive(:new).and_return(binding) + allow(VCAP::CloudController::ServiceBinding).to receive(:new).and_return(service_binding) end let(:name) { 'fake-name' } @@ -18,14 +18,14 @@ module Jobs::Services describe '#perform' do before do - allow(client).to receive(:unbind).with(binding) + allow(client).to receive(:unbind).with(service_binding) allow(VCAP::Services::ServiceBrokers::V2::Client).to receive(:new).and_return(client) end it 'unbinds the binding' do job.perform - expect(client).to have_received(:unbind).with(binding) + expect(client).to have_received(:unbind).with(service_binding) end end From 290f08e34acd25d0f0ef2bece3e64e648c6fa796 Mon Sep 17 00:00:00 2001 From: JT Archie and Luan Santos Date: Fri, 16 Jan 2015 15:55:06 -0800 Subject: [PATCH 72/76] Fix paginagion url for nested processes under apps [#79989902] --- app/controllers/v3/apps_controller.rb | 2 +- app/controllers/v3/processes_controller.rb | 2 +- app/presenters/v3/process_presenter.rb | 4 ++-- spec/api/api_version_spec.rb | 2 +- spec/api/documentation/v3/apps_api_spec.rb | 4 ++-- .../controllers/v3/processes_controller_spec.rb | 2 +- spec/unit/presenters/v3/process_presenter_spec.rb | 13 +++++++++---- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/app/controllers/v3/apps_controller.rb b/app/controllers/v3/apps_controller.rb index 96d908cd695..264f001fe13 100644 --- a/app/controllers/v3/apps_controller.rb +++ b/app/controllers/v3/apps_controller.rb @@ -85,7 +85,7 @@ def list_processes(guid) pagination_options = PaginationOptions.from_params(params) paginated_result = @process_handler.list(pagination_options, @access_context, app_guid: app.guid) - [HTTP::OK, @process_presenter.present_json_list(paginated_result)] + [HTTP::OK, @process_presenter.present_json_list(paginated_result, "/v3/apps/#{guid}/processes")] end put '/v3/apps/:guid/processes', :add_process diff --git a/app/controllers/v3/processes_controller.rb b/app/controllers/v3/processes_controller.rb index fc59d93c070..3b9579ba9c9 100644 --- a/app/controllers/v3/processes_controller.rb +++ b/app/controllers/v3/processes_controller.rb @@ -18,7 +18,7 @@ def list pagination_options = PaginationOptions.from_params(params) paginated_result = @processes_handler.list(pagination_options, @access_context) - [HTTP::OK, @process_presenter.present_json_list(paginated_result)] + [HTTP::OK, @process_presenter.present_json_list(paginated_result, '/v3/processes')] end get '/v3/processes/:guid', :show diff --git a/app/presenters/v3/process_presenter.rb b/app/presenters/v3/process_presenter.rb index 29f74be6739..5a4e96c8f69 100644 --- a/app/presenters/v3/process_presenter.rb +++ b/app/presenters/v3/process_presenter.rb @@ -10,12 +10,12 @@ def present_json(process) MultiJson.dump(process_hash(process), pretty: true) end - def present_json_list(paginated_result) + def present_json_list(paginated_result, base_url) processes = paginated_result.records process_hashes = processes.collect { |app| process_hash(app) } paginated_response = { - pagination: @pagination_presenter.present_pagination_hash(paginated_result, '/v3/processes'), + pagination: @pagination_presenter.present_pagination_hash(paginated_result, base_url), resources: process_hashes } diff --git a/spec/api/api_version_spec.rb b/spec/api/api_version_spec.rb index af21f948633..f7cd638742a 100644 --- a/spec/api/api_version_spec.rb +++ b/spec/api/api_version_spec.rb @@ -2,7 +2,7 @@ require 'digest/sha1' describe 'Stable API warning system', api_version_check: true do - API_FOLDER_CHECKSUM = '069366f2ce7ebc74edc8e561317dcd3e4ce92594' + API_FOLDER_CHECKSUM = 'f0c5604325a95cd780d3eab15b415a1651ba494f' it 'double-checks the version' do expect(VCAP::CloudController::Constants::API_VERSION).to eq('2.21.0') diff --git a/spec/api/documentation/v3/apps_api_spec.rb b/spec/api/documentation/v3/apps_api_spec.rb index bf0af2bc740..d96c9f89f6b 100644 --- a/spec/api/documentation/v3/apps_api_spec.rb +++ b/spec/api/documentation/v3/apps_api_spec.rb @@ -253,8 +253,8 @@ def do_request_with_error_handling expected_response = { 'pagination' => { 'total_results' => 1, - 'first' => { 'href' => '/v3/processes?page=1&per_page=50' }, - 'last' => { 'href' => '/v3/processes?page=1&per_page=50' }, + 'first' => { 'href' => "/v3/apps/#{guid}/processes?page=1&per_page=50" }, + 'last' => { 'href' => "/v3/apps/#{guid}/processes?page=1&per_page=50" }, 'next' => nil, 'previous' => nil, }, diff --git a/spec/unit/controllers/v3/processes_controller_spec.rb b/spec/unit/controllers/v3/processes_controller_spec.rb index a7a1d074d57..7330b59a17f 100644 --- a/spec/unit/controllers/v3/processes_controller_spec.rb +++ b/spec/unit/controllers/v3/processes_controller_spec.rb @@ -48,7 +48,7 @@ module VCAP::CloudController response_code, response_body = process_controller.list expect(processes_handler).to have_received(:list) - expect(process_presenter).to have_received(:present_json_list).with(list_response) + expect(process_presenter).to have_received(:present_json_list).with(list_response, '/v3/processes') expect(response_code).to eq(200) expect(response_body).to eq(expected_response) end diff --git a/spec/unit/presenters/v3/process_presenter_spec.rb b/spec/unit/presenters/v3/process_presenter_spec.rb index b83740c2794..a1fc294dfa5 100644 --- a/spec/unit/presenters/v3/process_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_presenter_spec.rb @@ -16,7 +16,7 @@ module VCAP::CloudController end describe '#present_json_list' do - let(:pagination_presenter) { double(:pagination_presenter, present_pagination_hash: 'pagination_stuff') } + let(:pagination_presenter) { double(:pagination_presenter) } let(:process1) { AppFactory.make } let(:process2) { AppFactory.make } let(:processes) { [process1, process2] } @@ -25,9 +25,14 @@ module VCAP::CloudController let(:per_page) { 1 } let(:total_results) { 2 } let(:paginated_result) { PaginatedResult.new(processes, total_results, PaginationOptions.new(page, per_page)) } + before do + allow(pagination_presenter).to receive(:present_pagination_hash) do |_, url| + "pagination-#{url}" + end + end it 'presents the processes as a json array under resources' do - json_result = presenter.present_json_list(paginated_result) + json_result = presenter.present_json_list(paginated_result, 'potato') result = MultiJson.load(json_result) guids = result['resources'].collect { |app_json| app_json['guid'] } @@ -35,10 +40,10 @@ module VCAP::CloudController end it 'includes pagination section' do - json_result = presenter.present_json_list(paginated_result) + json_result = presenter.present_json_list(paginated_result, 'bazooka') result = MultiJson.load(json_result) - expect(result['pagination']).to eq('pagination_stuff') + expect(result['pagination']).to eq('pagination-bazooka') end end end From e4beab6ec5769b2b1f002449b3bd5e6781fe05b4 Mon Sep 17 00:00:00 2001 From: Joseph Palermo and Sai To Yeung Date: Mon, 19 Jan 2015 10:37:08 -0800 Subject: [PATCH 73/76] Raise proper api error when user tries to upload package bits twice [#86380870] --- app/controllers/v3/packages_controller.rb | 6 ++++++ .../controllers/v3/packages_controller_spec.rb | 14 ++++++++++++++ vendor/errors | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/controllers/v3/packages_controller.rb b/app/controllers/v3/packages_controller.rb index e89cd73a70c..e95068c7943 100644 --- a/app/controllers/v3/packages_controller.rb +++ b/app/controllers/v3/packages_controller.rb @@ -48,6 +48,8 @@ def upload(package_guid) package_not_found! rescue PackagesHandler::Unauthorized unauthorized! + rescue PackagesHandler::BitsAlreadyUploaded + bits_already_uploaded! end get '/v3/packages/:guid', :show @@ -84,6 +86,10 @@ def unauthorized! raise VCAP::Errors::ApiError.new_from_details('NotAuthorized') end + def bits_already_uploaded! + raise VCAP::Errors::ApiError.new_from_details('PackageBitsAlreadyUploaded') + end + def unprocessable!(message) raise VCAP::Errors::ApiError.new_from_details('UnprocessableEntity', message) end diff --git a/spec/unit/controllers/v3/packages_controller_spec.rb b/spec/unit/controllers/v3/packages_controller_spec.rb index 6458ede58be..f4935efd354 100644 --- a/spec/unit/controllers/v3/packages_controller_spec.rb +++ b/spec/unit/controllers/v3/packages_controller_spec.rb @@ -208,6 +208,20 @@ module VCAP::CloudController end end end + + context 'when the bits have already been uploaded' do + before do + allow(packages_handler).to receive(:upload).and_raise(PackagesHandler::BitsAlreadyUploaded) + end + it 'returns a 400 PackageBitsAlreadyUploaded error' do + expect { + packages_controller.upload(package.guid) + }.to raise_error do |error| + expect(error.name).to eq 'PackageBitsAlreadyUploaded' + expect(error.response_code).to eq 400 + end + end + end end describe '#show' do diff --git a/vendor/errors b/vendor/errors index 9a645f7f513..b4aa6f3996e 160000 --- a/vendor/errors +++ b/vendor/errors @@ -1 +1 @@ -Subproject commit 9a645f7f513754c0fe2be5d89830df007b52cdf8 +Subproject commit b4aa6f3996e3d681ce3010c699511e5d73877ed0 From 62d32d01c0a73e07197bd0431345056422b98049 Mon Sep 17 00:00:00 2001 From: Luan Santos and Zach Robinson Date: Mon, 19 Jan 2015 16:41:16 -0800 Subject: [PATCH 74/76] Revert time consistency commits. Full revert list is: da665b7 - Decrease sensitivity on time based test (5 days ago) 0cdb27e - Fix incompatibility with MySQL 5.5 when setting timezone (6 days ago) 72a6713 - Fix rubocop offense (6 days ago) de41ab8 - Fix flaky time dependent test (6 days ago) eda570d - Fix time consistency, use DB time where possible (7 days ago) [#82077856] Conflicts: spec/unit/controllers/internal/bulk_apps_controller_spec.rb --- .../runtime/billing_events_controller.rb | 2 +- app/jobs/runtime/pending_packages_cleanup.rb | 3 +- app/models/runtime/app.rb | 2 +- app/models/runtime/app_start_event.rb | 2 +- app/models/runtime/app_stop_event.rb | 2 +- .../runtime/organization_start_event.rb | 2 +- app/models/services/service_create_event.rb | 2 +- app/models/services/service_delete_event.rb | 2 +- app/presenters/api/job_presenter.rb | 4 +-- .../runtime/app_event_repository.rb | 2 +- .../runtime/space_event_repository.rb | 6 ++-- app/repositories/services/event_repository.rb | 2 +- .../20130219194917_create_stacks_table.rb | 2 +- lib/cloud_controller.rb | 2 -- lib/cloud_controller/blobstore/blob.rb | 2 +- lib/cloud_controller/blobstore/client.rb | 4 +-- lib/cloud_controller/clock.rb | 4 +-- lib/cloud_controller/db.rb | 3 -- lib/cloud_controller/dea/client.rb | 4 +-- lib/cloud_controller/dea/pool.rb | 6 ++-- lib/cloud_controller/dea/stager_pool.rb | 4 +-- lib/cloud_controller/diagnostics.rb | 6 ++-- .../diego/instances_reporter.rb | 2 +- .../diego/service_registry.rb | 4 +-- lib/cloud_controller/resource_pool.rb | 4 +-- lib/vcap/rest_api/query.rb | 2 +- lib/vcap/uaa_token_decoder.rb | 4 +-- spec/support/fakes/blueprints.rb | 6 ++-- spec/support/integration/http.rb | 2 +- .../controllers/base/model_controller_spec.rb | 2 +- .../app_bits_upload_controller_spec.rb | 4 +-- .../app_usage_events_controller_spec.rb | 2 +- .../runtime/billing_events_controller_spec.rb | 2 +- .../runtime/events_controller_spec.rb | 6 ++-- .../services/snapshots_controller_spec.rb | 4 +-- .../jobs/runtime/blobstore_upload_spec.rb | 2 +- spec/unit/jobs/runtime/droplet_upload_spec.rb | 2 +- .../jobs/runtime/failed_jobs_cleanup_spec.rb | 6 ++-- .../runtime/pending_packages_cleanup_spec.rb | 8 ++--- .../cloud_controller/backends/runners_spec.rb | 4 +-- .../cloud_controller/blobstore/blob_spec.rb | 2 +- .../lib/cloud_controller/dea/client_spec.rb | 8 ++--- .../nats_messages/dea_advertisment_spec.rb | 8 ++--- .../nats_messages/stager_advertisment_spec.rb | 8 ++--- .../cloud_controller/dea/respondent_spec.rb | 31 ++++++++++--------- .../lib/cloud_controller/diagnostics_spec.rb | 6 ++-- .../diego/service_registry_spec.rb | 2 +- ...multi_response_message_bus_request_spec.rb | 4 +-- spec/unit/lib/vcap/rest_api/query_spec.rb | 4 +-- spec/unit/lib/vcap/uaa_token_decoder_spec.rb | 20 ++++++------ spec/unit/models/runtime/app_spec.rb | 4 +-- .../unit/models/runtime/billing_event_spec.rb | 8 ++--- spec/unit/models/runtime/event_spec.rb | 2 +- .../unit/presenters/api/job_presenter_spec.rb | 6 ++-- .../api/staging_job_presenter_spec.rb | 2 +- .../app_usage_event_repository_spec.rb | 2 +- 56 files changed, 122 insertions(+), 129 deletions(-) diff --git a/app/controllers/runtime/billing_events_controller.rb b/app/controllers/runtime/billing_events_controller.rb index 91b306d91e7..f03efcb18b5 100644 --- a/app/controllers/runtime/billing_events_controller.rb +++ b/app/controllers/runtime/billing_events_controller.rb @@ -38,7 +38,7 @@ def end_time def parse_date_param(param) str = @params[param] - Time.parse(str).utc if str + Time.parse(str).localtime if str rescue raise Errors::ApiError.new_from_details('BillingEventQueryInvalid') end diff --git a/app/jobs/runtime/pending_packages_cleanup.rb b/app/jobs/runtime/pending_packages_cleanup.rb index 310e3e98f1f..fe475885005 100644 --- a/app/jobs/runtime/pending_packages_cleanup.rb +++ b/app/jobs/runtime/pending_packages_cleanup.rb @@ -9,7 +9,8 @@ def initialize(expiration_in_seconds) end def perform - App.where("package_pending_since < ? - INTERVAL '?' SECOND", Sequel::CURRENT_TIMESTAMP, expiration_in_seconds.to_i).update( + cutoff_time = Time.now - expiration_in_seconds + App.where('package_pending_since < ?', cutoff_time).update( package_state: 'FAILED', staging_failed_reason: 'StagingTimeExpired', package_pending_since: nil, diff --git a/app/models/runtime/app.rb b/app/models/runtime/app.rb index 8d9255b9fc0..9e9b91560a1 100644 --- a/app/models/runtime/app.rb +++ b/app/models/runtime/app.rb @@ -424,7 +424,7 @@ def mark_as_failed_to_stage(reason='StagingError') def mark_for_restaging self.package_state = 'PENDING' self.staging_failed_reason = nil - self.package_pending_since = Sequel::CURRENT_TIMESTAMP + self.package_pending_since = Time.now end def buildpack diff --git a/app/models/runtime/app_start_event.rb b/app/models/runtime/app_start_event.rb index f6ffe2cbc3e..b5e936c8a43 100644 --- a/app/models/runtime/app_start_event.rb +++ b/app/models/runtime/app_start_event.rb @@ -35,7 +35,7 @@ def event_type def self.create_from_app(app) return unless app.space.organization.billing_enabled? AppStartEvent.create( - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, organization_guid: app.space.organization_guid, organization_name: app.space.organization.name, space_guid: app.space.guid, diff --git a/app/models/runtime/app_stop_event.rb b/app/models/runtime/app_stop_event.rb index 6661428cb3e..bd62c9f01ae 100644 --- a/app/models/runtime/app_stop_event.rb +++ b/app/models/runtime/app_stop_event.rb @@ -41,7 +41,7 @@ def create_from_app(app) end AppStopEvent.create( - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, organization_guid: app.space.organization_guid, organization_name: app.space.organization.name, space_guid: app.space.guid, diff --git a/app/models/runtime/organization_start_event.rb b/app/models/runtime/organization_start_event.rb index c996d80b100..79d8092e3a9 100644 --- a/app/models/runtime/organization_start_event.rb +++ b/app/models/runtime/organization_start_event.rb @@ -16,7 +16,7 @@ def event_type def self.create_from_org(org) raise BillingNotEnabled unless org.billing_enabled? OrganizationStartEvent.create( - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, organization_guid: org.guid, organization_name: org.name, ) diff --git a/app/models/services/service_create_event.rb b/app/models/services/service_create_event.rb index 36a4c19bd96..ea2c6cccf26 100644 --- a/app/models/services/service_create_event.rb +++ b/app/models/services/service_create_event.rb @@ -41,7 +41,7 @@ def self.create_from_service_instance(instance) return unless org.billing_enabled? ServiceCreateEvent.create( - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, organization_guid: org.guid, organization_name: org.name, space_guid: space.guid, diff --git a/app/models/services/service_delete_event.rb b/app/models/services/service_delete_event.rb index 5bdaa34181c..47e7fb50101 100644 --- a/app/models/services/service_delete_event.rb +++ b/app/models/services/service_delete_event.rb @@ -31,7 +31,7 @@ def self.create_from_service_instance(instance) return unless org.billing_enabled? ServiceDeleteEvent.create( - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, organization_guid: org.guid, organization_name: org.name, space_guid: space.guid, diff --git a/app/presenters/api/job_presenter.rb b/app/presenters/api/job_presenter.rb index a6d84bfee02..557a3d03e19 100644 --- a/app/presenters/api/job_presenter.rb +++ b/app/presenters/api/job_presenter.rb @@ -93,11 +93,11 @@ def guid end def created_at - Time.at(0).utc + Time.at(0) end def run_at - Time.at(0).utc + Time.at(0) end def cf_api_error diff --git a/app/repositories/runtime/app_event_repository.rb b/app/repositories/runtime/app_event_repository.rb index 49f63ea6eb7..5cdedb354f0 100644 --- a/app/repositories/runtime/app_event_repository.rb +++ b/app/repositories/runtime/app_event_repository.rb @@ -73,7 +73,7 @@ def create_app_audit_event(type, app, space, actor, metadata) Event.create( space: space, type: type, - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, actee: app.guid, actee_type: 'app', actee_name: app.name, diff --git a/app/repositories/runtime/space_event_repository.rb b/app/repositories/runtime/space_event_repository.rb index 6cbc78928cd..17f1d5e4c22 100644 --- a/app/repositories/runtime/space_event_repository.rb +++ b/app/repositories/runtime/space_event_repository.rb @@ -12,7 +12,7 @@ def record_space_create(space, actor, actor_name, request_attrs) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, metadata: { request: request_attrs } @@ -29,7 +29,7 @@ def record_space_update(space, actor, actor_name, request_attrs) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, metadata: { request: request_attrs } @@ -45,7 +45,7 @@ def record_space_delete_request(space, actor, actor_name, recursive) actor: actor.guid, actor_type: 'user', actor_name: actor_name, - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, space_guid: space.guid, organization_guid: space.organization.guid, metadata: { diff --git a/app/repositories/services/event_repository.rb b/app/repositories/services/event_repository.rb index a9f0e18f9a2..a4d05034ad4 100644 --- a/app/repositories/services/event_repository.rb +++ b/app/repositories/services/event_repository.rb @@ -214,7 +214,7 @@ def user_actor def create_event(type, actor_data, actee_data, metadata, space_data=nil) base_data = { type: type, - timestamp: Sequel::CURRENT_TIMESTAMP, + timestamp: Time.now, metadata: metadata } diff --git a/db/migrations/20130219194917_create_stacks_table.rb b/db/migrations/20130219194917_create_stacks_table.rb index 5abc484329a..eee64fbc9d5 100644 --- a/db/migrations/20130219194917_create_stacks_table.rb +++ b/db/migrations/20130219194917_create_stacks_table.rb @@ -19,7 +19,7 @@ guid: SecureRandom.uuid, name: 'lucid64', description: 'Ubuntu 10.04 on x86-64', - created_at: Sequel::CURRENT_TIMESTAMP, + created_at: Time.now, ) alter_table :apps do diff --git a/lib/cloud_controller.rb b/lib/cloud_controller.rb index fa09b318460..8c69955412d 100644 --- a/lib/cloud_controller.rb +++ b/lib/cloud_controller.rb @@ -21,8 +21,6 @@ require 'active_support/core_ext/object/to_query' require 'active_support/json/encoding' -Sequel.default_timezone = :utc - module VCAP::CloudController; end require 'vcap/errors/invalid_relation' diff --git a/lib/cloud_controller/blobstore/blob.rb b/lib/cloud_controller/blobstore/blob.rb index c480aaeadcc..2482604c88c 100644 --- a/lib/cloud_controller/blobstore/blob.rb +++ b/lib/cloud_controller/blobstore/blob.rb @@ -36,7 +36,7 @@ def download_uri_for_file end if file.respond_to?(:url) - return file.url(Time.now.utc + 3600) + return file.url(Time.now + 3600) end file.public_url end diff --git a/lib/cloud_controller/blobstore/client.rb b/lib/cloud_controller/blobstore/client.rb index 366c4dfeee5..801b880f4e9 100644 --- a/lib/cloud_controller/blobstore/client.rb +++ b/lib/cloud_controller/blobstore/client.rb @@ -49,7 +49,7 @@ def cp_r_to_blobstore(source_dir) end def cp_to_blobstore(source_path, destination_key, retries=2) - start = Time.now.utc + start = Time.now logger.info('blobstore.cp-start', destination_key: destination_key, source_path: source_path, bucket: @directory_key) size = -1 log_entry = 'blobstore.cp-skip' @@ -78,7 +78,7 @@ def cp_to_blobstore(source_path, destination_key, retries=2) log_entry = 'blobstore.cp-finish' end - duration = Time.now.utc - start + duration = Time.now - start logger.info(log_entry, destination_key: destination_key, duration_seconds: duration, diff --git a/lib/cloud_controller/clock.rb b/lib/cloud_controller/clock.rb index 0bdeea24107..5d845ad3ad4 100644 --- a/lib/cloud_controller/clock.rb +++ b/lib/cloud_controller/clock.rb @@ -21,7 +21,7 @@ def start def schedule_cleanup(name, klass, at) Clockwork.every(1.day, "#{name}.cleanup.job", at: at) do |_| - @logger.info("Queueing #{klass} at #{Time.now.utc}") + @logger.info("Queueing #{klass} at #{Time.now}") cutoff_age_in_days = @config.fetch(name.to_sym).fetch(:cutoff_age_in_days) job = klass.new(cutoff_age_in_days) Jobs::Enqueuer.new(job, queue: 'cc-generic').enqueue @@ -32,7 +32,7 @@ def schedule_frequent_cleanup(name, klass) config = @config.fetch(name.to_sym) Clockwork.every(config.fetch(:frequency_in_seconds), "#{name}.cleanup.job") do |_| - @logger.info("Queueing #{klass} at #{Time.now.utc}") + @logger.info("Queueing #{klass} at #{Time.now}") expiration = config.fetch(:expiration_in_seconds) job = klass.new(expiration) Jobs::Enqueuer.new(job, queue: 'cc-generic').enqueue diff --git a/lib/cloud_controller/db.rb b/lib/cloud_controller/db.rb index 87c8a0c8182..fd4168c0651 100644 --- a/lib/cloud_controller/db.rb +++ b/lib/cloud_controller/db.rb @@ -33,9 +33,6 @@ def self.connect(opts, logger) if db.database_type == :mysql Sequel::MySQL.default_collate = 'utf8_bin' - db.run("SET time_zone = '+0:00'") - elsif db.database_type == :postgres - db.run("SET time zone 'UTC'") end db diff --git a/lib/cloud_controller/dea/client.rb b/lib/cloud_controller/dea/client.rb index 35d7028108d..1496d52e81f 100644 --- a/lib/cloud_controller/dea/client.rb +++ b/lib/cloud_controller/dea/client.rb @@ -90,7 +90,7 @@ def find_all_instances(app) unless all_instances[index] all_instances[index] = { state: 'DOWN', - since: Time.now.utc.to_i, + since: Time.now.to_i, } end end @@ -234,7 +234,7 @@ def find_stats(app) unless stats[index] stats[index] = { state: 'DOWN', - since: Time.now.utc.to_i, + since: Time.now.to_i, } end end diff --git a/lib/cloud_controller/dea/pool.rb b/lib/cloud_controller/dea/pool.rb index aafd1d91295..3c1d4ece438 100644 --- a/lib/cloud_controller/dea/pool.rb +++ b/lib/cloud_controller/dea/pool.rb @@ -21,7 +21,7 @@ def register_subscriptions end def process_advertise_message(message) - advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.utc.to_i + @advertise_timeout) + advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.to_i + @advertise_timeout) mutex.synchronize do remove_advertisement_for_id(advertisement.dea_id) @@ -30,7 +30,7 @@ def process_advertise_message(message) end def process_shutdown_message(message) - fake_advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.utc.to_i + @advertise_timeout) + fake_advertisement = NatsMessages::DeaAdvertisement.new(message, Time.now.to_i + @advertise_timeout) mutex.synchronize do remove_advertisement_for_id(fake_advertisement.dea_id) @@ -69,7 +69,7 @@ def reserve_app_memory(dea_id, app_memory) attr_reader :message_bus def prune_stale_deas - now = Time.now.utc.to_i + now = Time.now.to_i @dea_advertisements.delete_if { |ad| ad.expired?(now) } end diff --git a/lib/cloud_controller/dea/stager_pool.rb b/lib/cloud_controller/dea/stager_pool.rb index 9aa2b50efca..dd5eda4926c 100644 --- a/lib/cloud_controller/dea/stager_pool.rb +++ b/lib/cloud_controller/dea/stager_pool.rb @@ -14,7 +14,7 @@ def initialize(config, message_bus, blobstore_url_generator) end def process_advertise_message(msg) - advertisement = NatsMessages::StagerAdvertisement.new(msg, Time.now.utc.to_i + @advertise_timeout) + advertisement = NatsMessages::StagerAdvertisement.new(msg, Time.now.to_i + @advertise_timeout) publish_buildpacks unless stager_in_pool?(advertisement.stager_id) mutex.synchronize do @@ -68,7 +68,7 @@ def top_5_stagers_for(memory, disk, stack) end def prune_stale_advertisements - now = Time.now.utc.to_i + now = Time.now.to_i @stager_advertisements.delete_if { |ad| ad.expired?(now) } end diff --git a/lib/cloud_controller/diagnostics.rb b/lib/cloud_controller/diagnostics.rb index 835d8fcef00..ba96194d868 100644 --- a/lib/cloud_controller/diagnostics.rb +++ b/lib/cloud_controller/diagnostics.rb @@ -2,7 +2,7 @@ module VCAP::CloudController class Diagnostics def self.collect(output_directory) data = { - time: Time.now.utc, + time: Time.now, threads: thread_data, varz: varz_data } @@ -27,7 +27,7 @@ def self.request_complete def self.request_info(request) { - start_time: Time.now.utc.to_f, + start_time: Time.now.to_f, request_id: ::VCAP::Request.current_id, request_method: request.request_method, request_uri: request_uri(request) @@ -57,7 +57,7 @@ def self.varz_data end def self.output_file_name - Time.now.utc.strftime("diag-#{Process.pid}-%Y%m%d-%H:%M:%S.%L.json") + Time.now.strftime("diag-#{Process.pid}-%Y%m%d-%H:%M:%S.%L.json") end end end diff --git a/lib/cloud_controller/diego/instances_reporter.rb b/lib/cloud_controller/diego/instances_reporter.rb index f007b6dee2e..097235dcc74 100644 --- a/lib/cloud_controller/diego/instances_reporter.rb +++ b/lib/cloud_controller/diego/instances_reporter.rb @@ -106,7 +106,7 @@ def fill_unreported_instances_with_down_instances(reported_instances, app) unless reported_instances[i] reported_instances[i] = { state: 'DOWN', - since: Time.now.utc.to_i, + since: Time.now.to_i, } end end diff --git a/lib/cloud_controller/diego/service_registry.rb b/lib/cloud_controller/diego/service_registry.rb index 4ad8ed4fa1a..b295da922ba 100644 --- a/lib/cloud_controller/diego/service_registry.rb +++ b/lib/cloud_controller/diego/service_registry.rb @@ -25,12 +25,12 @@ def tps_addrs attr_reader :message_bus def set_tps_addr(guid, addr, ttl) - expires_at = Time.now.utc + ttl + expires_at = Time.now + ttl tps_services[guid] = { addr: addr, expires_at: expires_at } end def expire_tps_addrs - now = Time.now.utc + now = Time.now tps_services.select! { |_, val| val[:expires_at] > now } end diff --git a/lib/cloud_controller/resource_pool.rb b/lib/cloud_controller/resource_pool.rb index 59196652889..2787d434b10 100644 --- a/lib/cloud_controller/resource_pool.rb +++ b/lib/cloud_controller/resource_pool.rb @@ -125,7 +125,7 @@ def overwrite_destination_with!(descriptor, destination) logger.debug 'resource_pool.download.starting', destination: destination - start = Time.now.utc + start = Time.now if @cdn && @cdn[:uri] logger.debug 'resource_pool.download.using-cdn' @@ -146,7 +146,7 @@ def overwrite_destination_with!(descriptor, destination) end end - took = Time.now.utc - start + took = Time.now - start logger.debug 'resource_pool.download.complete', took: took, destination: destination end diff --git a/lib/vcap/rest_api/query.rb b/lib/vcap/rest_api/query.rb index 72650f51591..64e808e9817 100644 --- a/lib/vcap/rest_api/query.rb +++ b/lib/vcap/rest_api/query.rb @@ -160,7 +160,7 @@ def clean_up_boolean(_, q_val) end def clean_up_datetime(q_val) - q_val.empty? ? nil : Time.parse(q_val).utc + q_val.empty? ? nil : Time.parse(q_val).localtime end def clean_up_integer(q_val) diff --git a/lib/vcap/uaa_token_decoder.rb b/lib/vcap/uaa_token_decoder.rb index 66c20c6f319..2e58de27267 100644 --- a/lib/vcap/uaa_token_decoder.rb +++ b/lib/vcap/uaa_token_decoder.rb @@ -59,9 +59,9 @@ def decode_token_with_asymmetric_key(auth_token) def decode_token_with_key(auth_token, options) options = { audience_ids: config[:resource_id] }.merge(options) - token = CF::UAA::TokenCoder.new(options).decode_at_reference_time(auth_token, Time.now.utc.to_i - @grace_period_in_seconds) + token = CF::UAA::TokenCoder.new(options).decode_at_reference_time(auth_token, Time.now.to_i - @grace_period_in_seconds) expiration_time = token['exp'] || token[:exp] - if expiration_time && expiration_time < Time.now.utc.to_i + if expiration_time && expiration_time < Time.now.to_i @logger.warn("token currently expired but accepted within grace period of #{@grace_period_in_seconds} seconds") end token diff --git a/spec/support/fakes/blueprints.rb b/spec/support/fakes/blueprints.rb index 9eeb5ca2126..a7db1d0c16c 100644 --- a/spec/support/fakes/blueprints.rb +++ b/spec/support/fakes/blueprints.rb @@ -201,13 +201,13 @@ module VCAP::CloudController end BillingEvent.blueprint do - timestamp { Time.now.utc } + timestamp { Time.now } organization_guid { Sham.guid } organization_name { Sham.name } end Event.blueprint do - timestamp { Time.now.utc } + timestamp { Time.now } type { Sham.name } actor { Sham.guid } actor_type { Sham.name } @@ -249,7 +249,7 @@ module VCAP::CloudController instance_index { Sham.instance_index } exit_status { Random.rand(256) } exit_description { Sham.description } - timestamp { Time.now.utc } + timestamp { Time.now } end ServiceCreateEvent.blueprint do diff --git a/spec/support/integration/http.rb b/spec/support/integration/http.rb index f9e71c61b89..0c4fd901984 100644 --- a/spec/support/integration/http.rb +++ b/spec/support/integration/http.rb @@ -5,7 +5,7 @@ module IntegrationHttp def admin_token token = { 'aud' => 'cloud_controller', - 'exp' => Time.now.utc.to_i + 10_000, + 'exp' => Time.now.to_i + 10_000, 'client_id' => Sham.guid, 'scope' => ['cloud_controller.admin'], } diff --git a/spec/unit/controllers/base/model_controller_spec.rb b/spec/unit/controllers/base/model_controller_spec.rb index 30eb9d18257..50e782aca2d 100644 --- a/spec/unit/controllers/base/model_controller_spec.rb +++ b/spec/unit/controllers/base/model_controller_spec.rb @@ -361,7 +361,7 @@ def run_delayed_job end describe '#enumerate' do - let(:timestamp) { Time.now.utc.change(usec: 0) } + let(:timestamp) { Time.now.change(usec: 0) } let!(:model1) { TestModel.make(created_at: timestamp) } let!(:model2) { TestModel.make(created_at: timestamp + 1.second) } let!(:model3) { TestModel.make(created_at: timestamp + 2.seconds) } diff --git a/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb b/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb index ae5a2247288..2e92e6922e3 100644 --- a/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb +++ b/spec/unit/controllers/runtime/app_bits_upload_controller_spec.rb @@ -172,7 +172,7 @@ def make_request it 'succeeds' do headers = headers_for(user) - Timecop.travel(Time.now.utc + 1.week + 100.seconds) do + Timecop.travel(Time.now + 1.week + 100.seconds) do put "/v2/apps/#{app_obj.guid}/bits", req_body, headers end expect(last_response.status).to eq(201) @@ -183,7 +183,7 @@ def make_request it 'fails to authorize the upload' do headers = headers_for(user) - Timecop.travel(Time.now.utc + 1.week + 10000.seconds) do + Timecop.travel(Time.now + 1.week + 10000.seconds) do put "/v2/apps/#{app_obj.guid}/bits", req_body, headers end expect(last_response.status).to eq(401) diff --git a/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb b/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb index e214bb95d63..0e1c1d4f05f 100644 --- a/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb +++ b/spec/unit/controllers/runtime/app_usage_events_controller_spec.rb @@ -84,7 +84,7 @@ module VCAP::CloudController expect(last_response).to be_successful expect(AppUsageEvent.count).to eq(1) expect(AppUsageEvent.last).to match_app(app) - expect(AppUsageEvent.last.created_at).to be_within(5.seconds).of(Time.now.utc) + expect(AppUsageEvent.last.created_at).to be_within(5.seconds).of(Time.now) end it 'returns 403 as a non-admin' do diff --git a/spec/unit/controllers/runtime/billing_events_controller_spec.rb b/spec/unit/controllers/runtime/billing_events_controller_spec.rb index 90dd6554076..5cca08ab364 100644 --- a/spec/unit/controllers/runtime/billing_events_controller_spec.rb +++ b/spec/unit/controllers/runtime/billing_events_controller_spec.rb @@ -17,7 +17,7 @@ module VCAP::CloudController BillingEvent.plugin :scissors BillingEvent.delete - timestamp = Time.new(2012, 01, 01, 00, 00, 01).utc + timestamp = Time.new(2012, 01, 01, 00, 00, 01) @start_time = timestamp @org_event = OrganizationStartEvent.make( diff --git a/spec/unit/controllers/runtime/events_controller_spec.rb b/spec/unit/controllers/runtime/events_controller_spec.rb index 08929ac017d..e233eeddced 100644 --- a/spec/unit/controllers/runtime/events_controller_spec.rb +++ b/spec/unit/controllers/runtime/events_controller_spec.rb @@ -33,9 +33,9 @@ module VCAP::CloudController describe 'default order' do it 'sorts by timestamp' do type = SecureRandom.uuid - Event.make(timestamp: Time.new(1990, 1, 1).utc, type: type, actor: 'earlier') - Event.make(timestamp: Time.new(2000, 1, 1).utc, type: type, actor: 'later') - Event.make(timestamp: Time.new(1995, 1, 1).utc, type: type, actor: 'middle') + Event.make(timestamp: Time.new(1990, 1, 1), type: type, actor: 'earlier') + Event.make(timestamp: Time.new(2000, 1, 1), type: type, actor: 'later') + Event.make(timestamp: Time.new(1995, 1, 1), type: type, actor: 'middle') get '/v2/events', {}, admin_headers parsed_body = MultiJson.load(last_response.body) diff --git a/spec/unit/controllers/services/snapshots_controller_spec.rb b/spec/unit/controllers/services/snapshots_controller_spec.rb index b79e8714df6..6bf07cafd7b 100644 --- a/spec/unit/controllers/services/snapshots_controller_spec.rb +++ b/spec/unit/controllers/services/snapshots_controller_spec.rb @@ -29,7 +29,7 @@ module VCAP::CloudController describe 'POST', '/v2/snapshots' do let(:new_name) { 'new name' } - let(:snapshot_created_at) { Time.now.utc.to_s } + let(:snapshot_created_at) { Time.now.to_s } let(:new_snapshot) { VCAP::Services::Api::SnapshotV2.new(snapshot_id: '1', name: 'foo', state: 'empty', size: 0, created_time: snapshot_created_at) } let(:payload) { MultiJson.dump( @@ -157,7 +157,7 @@ module VCAP::CloudController end it 'returns an list of snapshots' do - created_time = Time.now.utc.to_s + created_time = Time.now.to_s expect(service_instance).to receive(:enum_snapshots) do [VCAP::Services::Api::SnapshotV2.new( 'snapshot_id' => '1234', diff --git a/spec/unit/jobs/runtime/blobstore_upload_spec.rb b/spec/unit/jobs/runtime/blobstore_upload_spec.rb index 77bb022a9b1..833d82d7708 100644 --- a/spec/unit/jobs/runtime/blobstore_upload_spec.rb +++ b/spec/unit/jobs/runtime/blobstore_upload_spec.rb @@ -40,7 +40,7 @@ module Jobs::Runtime BlobstoreUpload.class_eval do def reschedule_at(_, _=nil) # induce the jobs to reschedule almost immediately instead of waiting around for the backoff algorithm - Time.now.utc + Time.now end end BlobstoreUpload.new(local_file.path, blobstore_key, blobstore_name) diff --git a/spec/unit/jobs/runtime/droplet_upload_spec.rb b/spec/unit/jobs/runtime/droplet_upload_spec.rb index 259ab00887b..55aef470331 100644 --- a/spec/unit/jobs/runtime/droplet_upload_spec.rb +++ b/spec/unit/jobs/runtime/droplet_upload_spec.rb @@ -74,7 +74,7 @@ module Jobs::Runtime DropletUpload.class_eval do def reschedule_at(_, _=nil) # induce the jobs to reschedule almost immediately instead of waiting around for the backoff algorithm - Time.now.utc + Time.now end end DropletUpload.new(local_file.path, app.id) diff --git a/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb b/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb index 449d244ba7f..fc3f1db34e0 100644 --- a/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/failed_jobs_cleanup_spec.rb @@ -42,7 +42,7 @@ def max_attempts end context 'non-failing jobs' do - let(:run_at) { Time.now.utc + 1.day } + let(:run_at) { Time.now + 1.day } let(:the_job) { SuccessJob.new } it 'the job is not removed' do @@ -53,7 +53,7 @@ def max_attempts end context 'failing jobs' do - let(:run_at) { Time.now.utc - 1.day } + let(:run_at) { Time.now - 1.day } let(:the_job) { FailingJob.new } context 'when younger than specified cut-off' do @@ -65,7 +65,7 @@ def max_attempts end context 'when older than specified cut-off' do - let(:run_at) { Time.now.utc - 3.days } + let(:run_at) { Time.now - 3.days } it 'removes the job' do expect { diff --git a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb index c0097e69820..bcb3c9cc813 100644 --- a/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb +++ b/spec/unit/jobs/runtime/pending_packages_cleanup_spec.rb @@ -15,8 +15,8 @@ module Jobs::Runtime describe '#perform' do context 'with packages which have been pending for too long' do - let!(:app1) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 1.minute) } - let!(:app2) { AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds - 2.minutes) } + let!(:app1) { AppFactory.make(package_pending_since: Time.now - expiration_in_seconds - 1.second) } + let!(:app2) { AppFactory.make(package_pending_since: Time.now - expiration_in_seconds - 2.second) } before do cleanup_job.perform @@ -41,8 +41,8 @@ module Jobs::Runtime end it "ignores apps that haven't been pending for too long" do - app1 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 1.minute) - app2 = AppFactory.make(package_pending_since: Time.now.utc - expiration_in_seconds + 2.minutes) + app1 = AppFactory.make(package_pending_since: Time.now - expiration_in_seconds + 1.second) + app2 = AppFactory.make(package_pending_since: Time.now - expiration_in_seconds + 2.second) cleanup_job.perform app1.reload diff --git a/spec/unit/lib/cloud_controller/backends/runners_spec.rb b/spec/unit/lib/cloud_controller/backends/runners_spec.rb index a34a211182f..619de880685 100644 --- a/spec/unit/lib/cloud_controller/backends/runners_spec.rb +++ b/spec/unit/lib/cloud_controller/backends/runners_spec.rb @@ -246,7 +246,7 @@ def make_dea_app(options={}) end it 'does not return deleted apps' do - deleted_app = make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) + deleted_app = make_diego_app(id: 6, state: 'STARTED', deleted_at: DateTime.current) batch = runners.diego_apps(100, 0) @@ -561,7 +561,7 @@ def make_dea_app(options={}) end it 'does not return deleted apps' do - deleted_app = make_dea_app(id: 6, state: 'STARTED', deleted_at: DateTime.now.utc) + deleted_app = make_dea_app(id: 6, state: 'STARTED', deleted_at: DateTime.current) batch = runners.dea_apps(100, 0) diff --git a/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb b/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb index 2395e3d589d..d44849e4216 100644 --- a/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/blob_spec.rb @@ -36,7 +36,7 @@ module Blobstore it 'is valid for an hour' do Timecop.freeze do - now = Time.now.utc + now = Time.now expect(file).to receive(:url).with(now + 3600) blob.download_url end diff --git a/spec/unit/lib/cloud_controller/dea/client_spec.rb b/spec/unit/lib/cloud_controller/dea/client_spec.rb index 8b84f40dfed..34c49bacf47 100644 --- a/spec/unit/lib/cloud_controller/dea/client_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/client_spec.rb @@ -605,7 +605,7 @@ module VCAP::CloudController 'stats' => stats, } - allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } + allow(Time).to receive(:now) { 1 } message_bus.respond_to_synchronous_request('dea.find.droplet', [instance]) @@ -654,7 +654,7 @@ module VCAP::CloudController 'stats' => stats, } - allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } + allow(Time).to receive(:now).and_return(1) message_bus.respond_to_synchronous_request('dea.find.droplet', [instance_0, instance_1, instance_2]) @@ -781,7 +781,7 @@ module VCAP::CloudController with(app, search_options, { expected: 2 }). and_return([starting_instance, running_instance]) - allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } + allow(Time).to receive(:now) { 1 } app_instances = Dea::Client.find_all_instances(app) expect(app_instances).to eq({ @@ -811,7 +811,7 @@ module VCAP::CloudController with(app, search_options, { expected: 2 }). and_return([]) - allow(Time).to receive(:now) { double(:utc_time, to_f: 1.0, utc: 1) } + allow(Time).to receive(:now) { 1 } app_instances = Dea::Client.find_all_instances(app) expect(app_instances).to eq({ diff --git a/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb b/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb index 9a3e49ac3e7..f0c2b459051 100644 --- a/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/nats_messages/dea_advertisment_spec.rb @@ -16,7 +16,7 @@ module Dea::NatsMessages } } end - let(:expires) { Time.now.utc.to_i + 10 } + let(:expires) { Time.now.to_i + 10 } subject(:ad) { DeaAdvertisement.new(message, expires) } @@ -37,13 +37,13 @@ module Dea::NatsMessages end describe '#expired?' do - let(:now) { Time.now.utc } + let(:now) { Time.now } context 'when the time since the advertisment is greater than or equal 10 seconds' do it 'returns true' do Timecop.freeze now do ad Timecop.travel now + 11.seconds do - expect(ad).to be_expired(Time.now.utc) + expect(ad).to be_expired(Time.now) end end end @@ -54,7 +54,7 @@ module Dea::NatsMessages Timecop.freeze now do ad Timecop.travel now + 9.seconds do - expect(ad).to_not be_expired(Time.now.utc) + expect(ad).to_not be_expired(Time.now) end end end diff --git a/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb b/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb index 1563adae8ba..d5e079777dc 100644 --- a/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/nats_messages/stager_advertisment_spec.rb @@ -11,7 +11,7 @@ module Dea::NatsMessages 'available_memory' => 1024, } end - let(:expires) { Time.now.utc.to_i + 10 } + let(:expires) { Time.now.to_i + 10 } subject(:ad) { StagerAdvertisement.new(message, expires) } @@ -28,13 +28,13 @@ module Dea::NatsMessages end describe '#expired?' do - let(:now) { Time.now.utc } + let(:now) { Time.now } context 'when the time since the advertisment is greater than or equal to 10 seconds' do it 'returns true' do Timecop.freeze now do ad Timecop.freeze now + 10.seconds do - expect(ad).to be_expired(Time.now.utc) + expect(ad).to be_expired(Time.now) end end end @@ -45,7 +45,7 @@ module Dea::NatsMessages Timecop.freeze now do ad Timecop.freeze now + 9.seconds do - expect(ad).to_not be_expired(Time.now.utc) + expect(ad).to_not be_expired(Time.now) end end end diff --git a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb index 447486a9551..569c15be024 100644 --- a/spec/unit/lib/cloud_controller/dea/respondent_spec.rb +++ b/spec/unit/lib/cloud_controller/dea/respondent_spec.rb @@ -51,23 +51,24 @@ module VCAP::CloudController context 'when the app crashed' do context 'the app described in the event exists' do it 'adds a record in the Events table' do - time = Time.new(1990, 07, 06) - stub_const('Sequel::CURRENT_TIMESTAMP', time) - respondent.process_droplet_exited_message(payload) + time = Time.now + Timecop.freeze(time) do + respondent.process_droplet_exited_message(payload) - app_event = Event.find(actee: app.guid) + app_event = Event.find(actee: app.guid) - expect(app_event).to be - expect(app_event.space).to eq(app.space) - expect(app_event.type).to eq('app.crash') - expect(app_event.timestamp.to_i).to eq(time.to_i) - expect(app_event.actor_type).to eq('app') - expect(app_event.actor).to eq(app.guid) - expect(app_event.metadata['instance']).to eq(payload['instance']) - expect(app_event.metadata['index']).to eq(payload['index']) - expect(app_event.metadata['exit_status']).to eq(payload['exit_status']) - expect(app_event.metadata['exit_description']).to eq(payload['exit_description']) - expect(app_event.metadata['reason']).to eq(reason) + expect(app_event).to be + expect(app_event.space).to eq(app.space) + expect(app_event.type).to eq('app.crash') + expect(app_event.timestamp.to_i).to eq(time.to_i) + expect(app_event.actor_type).to eq('app') + expect(app_event.actor).to eq(app.guid) + expect(app_event.metadata['instance']).to eq(payload['instance']) + expect(app_event.metadata['index']).to eq(payload['index']) + expect(app_event.metadata['exit_status']).to eq(payload['exit_status']) + expect(app_event.metadata['exit_description']).to eq(payload['exit_description']) + expect(app_event.metadata['reason']).to eq(reason) + end end end end diff --git a/spec/unit/lib/cloud_controller/diagnostics_spec.rb b/spec/unit/lib/cloud_controller/diagnostics_spec.rb index 8640181f349..c07c4164e7e 100644 --- a/spec/unit/lib/cloud_controller/diagnostics_spec.rb +++ b/spec/unit/lib/cloud_controller/diagnostics_spec.rb @@ -34,7 +34,7 @@ def current_request end it 'populates the start time to now' do - now = Time.now.utc + now = Time.now Timecop.freeze now do expect(current_request[:start_time]).to be_within(0.01).of(now.to_f) end @@ -97,9 +97,9 @@ def current_request describe 'file name' do it 'uses a file name that includes a time stamp' do - Timecop.freeze Time.now.utc do + Timecop.freeze Time.now do filename = Diagnostics.collect(output_dir) - timestamp = Time.now.utc.strftime('%Y%m%d-%H:%M:%S.%L') + timestamp = Time.now.strftime('%Y%m%d-%H:%M:%S.%L') expect(filename).to match_regex(/#{timestamp}/) end end diff --git a/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb b/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb index d2c288e6765..94c67cfd201 100644 --- a/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/service_registry_spec.rb @@ -23,7 +23,7 @@ module VCAP::CloudController::Diego context 'when a broadcast message has expired' do before do - Timecop.travel(Time.now.utc + 30) + Timecop.travel(Time.now + 30) end it 'no longer contains the expired ip address' do diff --git a/spec/unit/lib/multi_response_message_bus_request_spec.rb b/spec/unit/lib/multi_response_message_bus_request_spec.rb index 149baa2a74b..2a521940f7f 100644 --- a/spec/unit/lib/multi_response_message_bus_request_spec.rb +++ b/spec/unit/lib/multi_response_message_bus_request_spec.rb @@ -172,7 +172,7 @@ end it 'cancels timeout' do - t = Time.now.utc + t = Time.now multi_response_message_bus_request.on_response(100) { |*args| raise 'Must never be called' } multi_response_message_bus_request.request({}) multi_response_message_bus_request.ignore_subsequent_responses @@ -180,7 +180,7 @@ # if timeout timer does not get cancelled # this test will take ~100s instead of less than 100s # (Use within 50s instead of 0.1s since system might be busy.) - expect(Time.now.utc).to be_within(50).of(t) + expect(Time.now).to be_within(50).of(t) end it 'raises error when request was not made' do diff --git a/spec/unit/lib/vcap/rest_api/query_spec.rb b/spec/unit/lib/vcap/rest_api/query_spec.rb index 8044206a806..adb9290cc6d 100644 --- a/spec/unit/lib/vcap/rest_api/query_spec.rb +++ b/spec/unit/lib/vcap/rest_api/query_spec.rb @@ -28,13 +28,13 @@ class Subscriber < Sequel::Model a = Author.create(num_val: i + 1, str_val: "str #{i}", published: (i == 0), - published_at: (i == 0) ? nil : Time.at(0).utc + i) + published_at: (i == 0) ? nil : Time.at(0) + i) 2.times do |j| a.add_book(Book.create(num_val: j + 1, str_val: "str #{i} #{j}")) end end - @owner_nil_num = Author.create(str_val: 'no num', published: false, published_at: Time.at(0).utc + @num_authors) + @owner_nil_num = Author.create(str_val: 'no num', published: false, published_at: Time.at(0) + @num_authors) @queryable_attributes = Set.new(%w(num_val str_val author_id book_id published published_at)) end diff --git a/spec/unit/lib/vcap/uaa_token_decoder_spec.rb b/spec/unit/lib/vcap/uaa_token_decoder_spec.rb index f1d61305e02..00e40b73caf 100644 --- a/spec/unit/lib/vcap/uaa_token_decoder_spec.rb +++ b/spec/unit/lib/vcap/uaa_token_decoder_spec.rb @@ -44,12 +44,12 @@ module VCAP end describe '#decode_token' do - before { Timecop.freeze(Time.now.utc) } + before { Timecop.freeze(Time.now) } after { Timecop.return } context 'when symmetric key is used' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } end before { config_hash[:symmetric_secret] = 'symmetric-key' } @@ -81,7 +81,7 @@ module VCAP context 'when token is valid' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } end it 'successfully decodes token and caches key' do @@ -118,7 +118,7 @@ module VCAP context 'when token has invalid audience' do let(:token_content) do - { 'aud' => 'invalid-audience', 'payload' => 123, 'exp' => Time.now.utc.to_i + 10_000 } + { 'aud' => 'invalid-audience', 'payload' => 123, 'exp' => Time.now.to_i + 10_000 } end it 'raises an BadToken error' do @@ -131,7 +131,7 @@ module VCAP context 'when token has expired' do let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i } end it 'raises a BadToken error' do @@ -155,14 +155,14 @@ module VCAP subject { described_class.new(config_hash, 100) } let(:token_content) do - { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.utc.to_i } + { 'aud' => 'resource-id', 'payload' => 123, 'exp' => Time.now.to_i } end let(:token) { generate_token(rsa_key, token_content) } context 'and the token is currently expired but had not expired within the grace period' do it 'decodes the token and logs a warning about expiration within the grace period' do - token_content['exp'] = Time.now.utc.to_i - 50 + token_content['exp'] = Time.now.to_i - 50 expect(logger).to receive(:warn).with(/token currently expired but accepted within grace period of 100 seconds/i) expect(subject.decode_token("bearer #{token}")).to eq token_content end @@ -170,7 +170,7 @@ module VCAP context 'and the token expired outside of the grace period' do it 'raises and logs a warning about the expired token' do - token_content['exp'] = Time.now.utc.to_i - 150 + token_content['exp'] = Time.now.to_i - 150 expect(logger).to receive(:warn).with(/token expired/i) expect { subject.decode_token("bearer #{token}") @@ -182,14 +182,14 @@ module VCAP subject { described_class.new(config_hash, -10) } it 'sets the grace period to be 0 instead' do - token_content['exp'] = Time.now.utc.to_i + token_content['exp'] = Time.now.to_i expired_token = generate_token(rsa_key, token_content) allow(logger).to receive(:warn) expect { subject.decode_token("bearer #{expired_token}") }.to raise_error(VCAP::UaaTokenDecoder::BadToken) - token_content['exp'] = Time.now.utc.to_i + 1 + token_content['exp'] = Time.now.to_i + 1 valid_token = generate_token(rsa_key, token_content) expect(subject.decode_token("bearer #{valid_token}")).to eq token_content end diff --git a/spec/unit/models/runtime/app_spec.rb b/spec/unit/models/runtime/app_spec.rb index ece11f528fd..245ed870f92 100644 --- a/spec/unit/models/runtime/app_spec.rb +++ b/spec/unit/models/runtime/app_spec.rb @@ -1528,11 +1528,9 @@ def self.it_does_not_mark_for_re_staging it 'updates the package_pending_since date to current' do app.package_pending_since = nil - app.save expect { app.mark_for_restaging - app.save - }.to change { app.reload.package_pending_since }.from(nil).to(kind_of(Time)) + }.to change { app.package_pending_since }.from(nil).to(kind_of(Time)) end end diff --git a/spec/unit/models/runtime/billing_event_spec.rb b/spec/unit/models/runtime/billing_event_spec.rb index 300d8461ad1..ab8ec151e2b 100644 --- a/spec/unit/models/runtime/billing_event_spec.rb +++ b/spec/unit/models/runtime/billing_event_spec.rb @@ -9,13 +9,11 @@ module VCAP::CloudController it { is_expected.to have_timestamp_columns } describe '.create' do - let(:values) { - { - timestamp: Time.now.utc, + let(:values) { { + timestamp: Time.now, organization_guid: 'abc', organization_name: 'def', - } - } + } } context 'when billing event writing is enabled' do before do diff --git a/spec/unit/models/runtime/event_spec.rb b/spec/unit/models/runtime/event_spec.rb index a82bcabe908..bee15a61541 100644 --- a/spec/unit/models/runtime/event_spec.rb +++ b/spec/unit/models/runtime/event_spec.rb @@ -13,7 +13,7 @@ module VCAP::CloudController actee: 'jtravolta', actee_type: 'Scientologist', actee_name: 'John Travolta', - timestamp: Time.new(1997, 6, 27).utc, + timestamp: Time.new(1997, 6, 27), metadata: { 'popcorn_price' => '$(arm + leg)' }, space: space ) diff --git a/spec/unit/presenters/api/job_presenter_spec.rb b/spec/unit/presenters/api/job_presenter_spec.rb index fb968fcd0d7..d26312c4a40 100644 --- a/spec/unit/presenters/api/job_presenter_spec.rb +++ b/spec/unit/presenters/api/job_presenter_spec.rb @@ -4,7 +4,7 @@ describe '#to_hash' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:run_at) { Time.now.utc.months_since(1) } + allow(job).to receive(:run_at) { Time.now.months_since(1) } job end @@ -30,7 +30,7 @@ context 'when the job has started' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:locked_at) { Time.now.utc } + allow(job).to receive(:locked_at) { Time.now } job end @@ -56,7 +56,7 @@ expect(JobPresenter.new(job).to_hash).to eq( metadata: { guid: '0', - created_at: Time.at(0).utc.iso8601, + created_at: Time.at(0).iso8601, url: '/v2/jobs/0' }, entity: { diff --git a/spec/unit/presenters/api/staging_job_presenter_spec.rb b/spec/unit/presenters/api/staging_job_presenter_spec.rb index 0968ff6f2d5..e344f9969cd 100644 --- a/spec/unit/presenters/api/staging_job_presenter_spec.rb +++ b/spec/unit/presenters/api/staging_job_presenter_spec.rb @@ -5,7 +5,7 @@ describe '#to_hash' do let(:job) do job = Delayed::Job.enqueue double(:obj, perform: nil) - allow(job).to receive(:run_at) { Time.now.utc.months_since(1) } + allow(job).to receive(:run_at) { Time.now.months_since(1) } job end diff --git a/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb b/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb index 47ac463ddc9..907c8e49209 100644 --- a/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb +++ b/spec/unit/repositories/runtime/app_usage_event_repository_spec.rb @@ -202,7 +202,7 @@ module Repositories::Runtime before do AppUsageEvent.dataset.delete - old = Time.now.utc - 999.days + old = Time.now - 999.days 3.times do event = repository.create_from_app(App.make) From b0e4288d4f368f9ae17378d801a482f6b829ca80 Mon Sep 17 00:00:00 2001 From: Michael Fraenkel Date: Fri, 16 Jan 2015 10:36:30 -0800 Subject: [PATCH 75/76] Add network egress rules for Diego - flow network egress rules for staging and desiring [#79330234] Signed-off-by: Hristo Iliev Signed-off-by: Atul Kshirsagar --- lib/cloud_controller/diego/common/protocol.rb | 33 ++++++++++ lib/cloud_controller/diego/docker/protocol.rb | 2 + .../diego/traditional/protocol.rb | 2 + .../diego/common/protocol_spec.rb | 40 ++++++++++++ .../diego/docker/protocol_spec.rb | 7 +++ .../cloud_controller/diego/messenger_spec.rb | 2 + .../diego/traditional/protocol_spec.rb | 61 ++++++++----------- 7 files changed, 113 insertions(+), 34 deletions(-) diff --git a/lib/cloud_controller/diego/common/protocol.rb b/lib/cloud_controller/diego/common/protocol.rb index 516bb8a1563..70b4ae4c58b 100644 --- a/lib/cloud_controller/diego/common/protocol.rb +++ b/lib/cloud_controller/diego/common/protocol.rb @@ -6,6 +6,15 @@ def stop_index_request(app, index) ['diego.stop.index', stop_index_message(app, index).to_json] end + def staging_egress_rules + staging_security_groups = SecurityGroup.where(staging_default: true).all + EgressNetworkRulesPresenter.new(staging_security_groups).to_array.collect { |sg| transform_rule(sg) }.flatten + end + + def running_egress_rules(app) + EgressNetworkRulesPresenter.new(app.space.security_groups).to_array.collect { |sg| transform_rule(sg) }.flatten + end + private def stop_index_message(app, index) @@ -14,6 +23,30 @@ def stop_index_message(app, index) 'index' => index, } end + + def transform_rule(rule) + protocol = rule['protocol'] + template = { + 'protocol' => protocol, + 'destination' => rule['destination'], + } + + case protocol + when 'icmp' + template['icmp_info'] = { 'type' => rule['type'], 'code' => rule['code'] } + when 'tcp', 'udp' + range = rule['ports'].split('-') + if range.size == 1 + template['ports'] = range[0].split(',').collect(&:to_i) + else + template['port_range'] = { 'start' => range[0].to_i, 'end' => range[1].to_i } + end + end + + template['log'] = rule['log'] if rule['log'] + + template + end end end end diff --git a/lib/cloud_controller/diego/docker/protocol.rb b/lib/cloud_controller/diego/docker/protocol.rb index 229cb7a4f01..3a975d89428 100644 --- a/lib/cloud_controller/diego/docker/protocol.rb +++ b/lib/cloud_controller/diego/docker/protocol.rb @@ -22,6 +22,7 @@ def stage_app_message(app, staging_timeout) 'file_descriptors' => app.file_descriptors, 'stack' => app.stack.name, 'docker_image' => app.docker_image, + 'egress_rules' => @common_protocol.staging_egress_rules, 'timeout' => staging_timeout, } end @@ -49,6 +50,7 @@ def desire_app_message(app) 'log_guid' => app.guid, 'docker_image' => app.docker_image, 'health_check_type' => app.health_check_type, + 'egress_rules' => @common_protocol.running_egress_rules(app), 'etag' => app.updated_at.to_f.to_s } diff --git a/lib/cloud_controller/diego/traditional/protocol.rb b/lib/cloud_controller/diego/traditional/protocol.rb index c8ccc26db19..d0511da0b66 100644 --- a/lib/cloud_controller/diego/traditional/protocol.rb +++ b/lib/cloud_controller/diego/traditional/protocol.rb @@ -38,6 +38,7 @@ def stage_app_message(app, staging_timeout) 'droplet_upload_uri' => @blobstore_url_generator.droplet_upload_url(app), 'build_artifacts_cache_download_uri' => @blobstore_url_generator.buildpack_cache_download_url(app), 'build_artifacts_cache_upload_uri' => @blobstore_url_generator.buildpack_cache_upload_url(app), + 'egress_rules' => @common_protocol.staging_egress_rules, 'timeout' => staging_timeout, } end @@ -57,6 +58,7 @@ def desire_app_message(app) 'routes' => app.uris, 'log_guid' => app.guid, 'health_check_type' => app.health_check_type, + 'egress_rules' => @common_protocol.running_egress_rules(app), 'etag' => app.updated_at.to_f.to_s } diff --git a/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb index 3ffd26e639a..e4d07830c1f 100644 --- a/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/common/protocol_spec.rb @@ -18,6 +18,46 @@ module Common expect(request.last).to match_json({ 'process_guid' => 'guid-versioned', 'index' => 33 }) end end + + describe 'staging_egress_rules' do + before do + SecurityGroup.make(rules: [{ 'protocol' => 'udp', 'ports' => '8080-9090', 'destination' => '198.41.191.47/1' }], staging_default: true) + SecurityGroup.make(rules: [{ 'protocol' => 'tcp', 'ports' => '8080,9090', 'destination' => '198.41.191.48/1', 'log' => true }], staging_default: true) + SecurityGroup.make(rules: [{ 'protocol' => 'tcp', 'ports' => '443', 'destination' => '198.41.191.49/1' }], staging_default: true) + SecurityGroup.make(rules: [{ 'protocol' => 'icmp', 'destination' => '0.0.0.0/0', 'type' => 0, 'code' => 1 }], staging_default: true) + SecurityGroup.make(rules: [{ 'protocol' => 'tcp', 'ports' => '80', 'destination' => '0.0.0.0/0' }], staging_default: false) + end + + it 'includes egress security group staging information by aggregating all sg with staging_default=true' do + expect(protocol.staging_egress_rules).to match_array([ + { 'protocol' => 'udp', 'port_range' => { 'start' => 8080, 'end' => 9090 }, 'destination' => '198.41.191.47/1' }, + { 'protocol' => 'tcp', 'ports' => [8080, 9090], 'destination' => '198.41.191.48/1', 'log' => true }, + { 'protocol' => 'tcp', 'ports' => [443], 'destination' => '198.41.191.49/1' }, + { 'protocol' => 'icmp', 'icmp_info' => { 'type' => 0, 'code' => 1 }, 'destination' => '0.0.0.0/0' }, + ]) + end + end + + describe 'running_egress_rules' do + let(:app) { AppFactory.make } + let(:sg_default_rules_1) { [{ 'protocol' => 'udp', 'ports' => '8080', 'destination' => '198.41.191.47/1' }] } + let(:sg_default_rules_2) { [{ 'protocol' => 'tcp', 'ports' => '9090-9095', 'destination' => '198.41.191.48/1', 'log' => true }] } + let(:sg_for_space_rules) { [{ 'protocol' => 'udp', 'ports' => '1010,2020', 'destination' => '198.41.191.49/1' }] } + + before do + SecurityGroup.make(rules: sg_default_rules_1, running_default: true) + SecurityGroup.make(rules: sg_default_rules_2, running_default: true) + app.space.add_security_group(SecurityGroup.make(rules: sg_for_space_rules)) + end + + it 'should provide the egress rules in the start message' do + expect(protocol.running_egress_rules(app)).to match_array([ + { 'protocol' => 'udp', 'ports' => [8080], 'destination' => '198.41.191.47/1' }, + { 'protocol' => 'tcp', 'port_range' => { 'start' => 9090, 'end' => 9095 }, 'destination' => '198.41.191.48/1', 'log' => true }, + { 'protocol' => 'udp', 'ports' => [1010, 2020], 'destination' => '198.41.191.49/1' }, + ]) + end + end end end end diff --git a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb index 30301eace31..c9478c701ce 100644 --- a/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/docker/protocol_spec.rb @@ -21,6 +21,11 @@ module Docker Protocol.new(common_protocol) end + before do + allow(common_protocol).to receive(:staging_egress_rules).and_return(['staging_egress_rule']) + allow(common_protocol).to receive(:running_egress_rules).with(app).and_return(['running_egress_rule']) + end + describe '#stage_app_request' do subject(:request) do protocol.stage_app_request(app, 900) @@ -47,6 +52,7 @@ module Docker 'file_descriptors' => app.file_descriptors, 'stack' => app.stack.name, 'docker_image' => app.docker_image, + 'egress_rules' => ['staging_egress_rule'], 'timeout' => 900, }) end @@ -84,6 +90,7 @@ module Docker 'log_guid' => app.guid, 'docker_image' => app.docker_image, 'health_check_type' => app.health_check_type, + 'egress_rules' => ['running_egress_rule'], 'etag' => app.updated_at.to_f.to_s, }) end diff --git a/spec/unit/lib/cloud_controller/diego/messenger_spec.rb b/spec/unit/lib/cloud_controller/diego/messenger_spec.rb index 6303d4a01cd..2b0dfbc8dfb 100644 --- a/spec/unit/lib/cloud_controller/diego/messenger_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/messenger_spec.rb @@ -50,6 +50,7 @@ module Diego 'app_bits_download_uri' => 'http://app-package.com', 'buildpacks' => Traditional::BuildpackEntryGenerator.new(blobstore_url_generator).buildpack_entries(app), 'droplet_upload_uri' => 'http://droplet-upload-uri', + 'egress_rules' => [], 'timeout' => 90, } @@ -77,6 +78,7 @@ module Diego 'health_check_type' => app.health_check_type, 'health_check_timeout_in_seconds' => 120, 'log_guid' => app.guid, + 'egress_rules' => [], 'etag' => app.updated_at.to_f.to_s, } end diff --git a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb index 4d3a1732105..a5b208d7970 100644 --- a/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/traditional/protocol_spec.rb @@ -16,18 +16,21 @@ module Traditional let(:common_protocol) { double(:common_protocol) } + let(:app) do + AppFactory.make + end + subject(:protocol) do Protocol.new(blobstore_url_generator, common_protocol) end - describe '#stage_app_request' do - let(:app) do - AppFactory.make - end + before do + allow(common_protocol).to receive(:staging_egress_rules).and_return(['staging_egress_rule']) + allow(common_protocol).to receive(:running_egress_rules).with(app).and_return(['running_egress_rule']) + end - subject(:request) do - protocol.stage_app_request(app, 900) - end + describe '#stage_app_request' do + let(:request) { protocol.stage_app_request(app, 900) } it 'returns arguments intended for CfMessageBus::MessageBus#publish' do expect(request.size).to eq(2) @@ -37,37 +40,36 @@ module Traditional end describe '#stage_app_message' do - let(:staging_app) { AppFactory.make } - subject(:message) { protocol.stage_app_message(staging_app, 900) } + let(:message) { protocol.stage_app_message(app, 900) } before do - staging_app.update(staging_task_id: 'fake-staging-task-id') # Mimic Diego::Messenger#send_stage_request + app.update(staging_task_id: 'fake-staging-task-id') # Mimic Diego::Messenger#send_stage_request end it 'is a nats message with the appropriate staging subject and payload' do buildpack_entry_generator = BuildpackEntryGenerator.new(blobstore_url_generator) expect(message).to eq( - 'app_id' => staging_app.guid, + 'app_id' => app.guid, 'task_id' => 'fake-staging-task-id', - 'memory_mb' => staging_app.memory, - 'disk_mb' => staging_app.disk_quota, - 'file_descriptors' => staging_app.file_descriptors, - 'environment' => Environment.new(staging_app).as_json, - 'stack' => staging_app.stack.name, + 'memory_mb' => app.memory, + 'disk_mb' => app.disk_quota, + 'file_descriptors' => app.file_descriptors, + 'environment' => Environment.new(app).as_json, + 'stack' => app.stack.name, 'build_artifacts_cache_download_uri' => 'http://buildpack-artifacts-cache.com', 'build_artifacts_cache_upload_uri' => 'http://buildpack-artifacts-cache.up.com', 'app_bits_download_uri' => 'http://app-package.com', - 'buildpacks' => buildpack_entry_generator.buildpack_entries(staging_app), + 'buildpacks' => buildpack_entry_generator.buildpack_entries(app), 'droplet_upload_uri' => 'http://droplet-upload-uri', + 'egress_rules' => ['staging_egress_rule'], 'timeout' => 900, ) end end describe '#desire_app_request' do - let(:app) { AppFactory.make } - subject(:request) { protocol.desire_app_request(app) } + let(:request) { protocol.desire_app_request(app) } it 'returns arguments intended for CfMessageBus::MessageBus#publish' do expect(request.size).to eq(2) @@ -95,15 +97,13 @@ module Traditional ) end + let(:message) { protocol.desire_app_message(app) } + before do environment = instance_double(Environment, as_json: [{ 'name' => 'fake', 'value' => 'environment' }]) allow(Environment).to receive(:new).with(app).and_return(environment) end - subject(:message) do - protocol.desire_app_message(app) - end - it 'is a messsage with the information nsync needs to desire the app' do expect(message).to eq( 'disk_mb' => 222, @@ -120,6 +120,7 @@ module Traditional 'start_command' => 'the-custom-command', 'execution_metadata' => 'staging-metadata', 'routes' => ['fake-uris'], + 'egress_rules' => ['running_egress_rule'], 'etag' => '12345.6789' ) end @@ -136,14 +137,8 @@ module Traditional end describe '#stop_staging_app_request' do - let(:app) do - AppFactory.make - end let(:task_id) { 'staging_task_id' } - - subject(:request) do - protocol.stop_staging_app_request(app, task_id) - end + let(:request) { protocol.stop_staging_app_request(app, task_id) } it 'returns an array of arguments including the subject and message' do expect(request.size).to eq(2) @@ -153,20 +148,18 @@ module Traditional end describe '#stop_staging_message' do - let(:staging_app) { AppFactory.make } let(:task_id) { 'staging_task_id' } - subject(:message) { protocol.stop_staging_message(staging_app, task_id) } + let(:message) { protocol.stop_staging_message(app, task_id) } it 'is a nats message with the appropriate staging subject and payload' do expect(message).to eq( - 'app_id' => staging_app.guid, + 'app_id' => app.guid, 'task_id' => task_id, ) end end describe '#stop_index_request' do - let(:app) { AppFactory.make } before { allow(common_protocol).to receive(:stop_index_request) } it 'delegates to the common protocol' do From 2f3953362d7b9374e1c66b99422e29ef0f57ab91 Mon Sep 17 00:00:00 2001 From: shashidharatd Date: Mon, 29 Dec 2014 18:54:12 +0530 Subject: [PATCH 76/76] Configurable file_descriptors limit for warden container file_descriptors for a warden container is made configurable through deployment manifest. Currently the default is 16384 which is read from the database table and is not configurable. To configure, cf operator needs to add the following config variable under cc cc: container_file_descriptor_limit: 4096 [#82011156] --- app/models/runtime/app.rb | 4 ++++ bosh-templates/cloud_controller_api.yml.erb | 2 ++ bosh-templates/cloud_controller_clock.yml.erb | 2 ++ bosh-templates/cloud_controller_worker.yml.erb | 2 ++ lib/cloud_controller/config.rb | 2 ++ spec/unit/models/runtime/app_spec.rb | 11 +++++++++++ 6 files changed, 23 insertions(+) diff --git a/app/models/runtime/app.rb b/app/models/runtime/app.rb index 9e9b91560a1..177defc442f 100644 --- a/app/models/runtime/app.rb +++ b/app/models/runtime/app.rb @@ -138,6 +138,10 @@ def before_save self.memory ||= Config.config[:default_app_memory] self.disk_quota ||= Config.config[:default_app_disk_in_mb] + if Config.config[:container_file_descriptor_limit] + self.file_descriptors ||= Config.config[:container_file_descriptor_limit] + end + set_new_version if version_needs_to_be_updated? AppStopEvent.create_from_app(self) if generate_stop_event? diff --git a/bosh-templates/cloud_controller_api.yml.erb b/bosh-templates/cloud_controller_api.yml.erb index 352511dff45..dc5322f7aab 100644 --- a/bosh-templates/cloud_controller_api.yml.erb +++ b/bosh-templates/cloud_controller_api.yml.erb @@ -74,6 +74,8 @@ default_app_memory: <%= p("cc.default_app_memory") %> default_app_disk_in_mb: <%= p("cc.default_app_disk_in_mb") %> maximum_app_disk_in_mb: <%= p("cc.maximum_app_disk_in_mb") %> +container_file_descriptor_limit: <%= p("cc.container_file_descriptor_limit") %> + request_timeout_in_seconds: <%= p("request_timeout_in_seconds") %> cc_partition: <%= p("cc.cc_partition") %> diff --git a/bosh-templates/cloud_controller_clock.yml.erb b/bosh-templates/cloud_controller_clock.yml.erb index dcf3105e372..bb1ca892399 100644 --- a/bosh-templates/cloud_controller_clock.yml.erb +++ b/bosh-templates/cloud_controller_clock.yml.erb @@ -78,6 +78,8 @@ default_app_memory: <%= p("cc.default_app_memory") %> default_app_disk_in_mb: <%= p("cc.default_app_disk_in_mb") %> maximum_app_disk_in_mb: <%= p("cc.maximum_app_disk_in_mb") %> +container_file_descriptor_limit: <%= p("cc.container_file_descriptor_limit") %> + request_timeout_in_seconds: <%= p("request_timeout_in_seconds") %> cc_partition: <%= p("cc.cc_partition") %> diff --git a/bosh-templates/cloud_controller_worker.yml.erb b/bosh-templates/cloud_controller_worker.yml.erb index 103bf882ae0..6561f94af46 100644 --- a/bosh-templates/cloud_controller_worker.yml.erb +++ b/bosh-templates/cloud_controller_worker.yml.erb @@ -74,6 +74,8 @@ default_app_memory: <%= p("cc.default_app_memory") %> default_app_disk_in_mb: <%= p("cc.default_app_disk_in_mb") %> maximum_app_disk_in_mb: <%= p("cc.maximum_app_disk_in_mb") %> +container_file_descriptor_limit: <%= p("cc.container_file_descriptor_limit") %> + request_timeout_in_seconds: <%= p("request_timeout_in_seconds") %> cc_partition: <%= p("cc.cc_partition") %> diff --git a/lib/cloud_controller/config.rb b/lib/cloud_controller/config.rb index f81b88b1553..d4742a3b1cb 100644 --- a/lib/cloud_controller/config.rb +++ b/lib/cloud_controller/config.rb @@ -44,6 +44,8 @@ class Config < VCAP::Config optional(:maximum_app_disk_in_mb) => Fixnum, :maximum_health_check_timeout => Fixnum, + optional(:container_file_descriptor_limit) => Fixnum, + optional(:allow_debug) => bool, optional(:login) => { diff --git a/spec/unit/models/runtime/app_spec.rb b/spec/unit/models/runtime/app_spec.rb index 245ed870f92..b531abec61a 100644 --- a/spec/unit/models/runtime/app_spec.rb +++ b/spec/unit/models/runtime/app_spec.rb @@ -1643,6 +1643,17 @@ def self.it_does_not_mark_for_re_staging expect(app.disk_quota).to eq(512) end end + + describe 'container_file_descriptor_limit' do + before do + TestConfig.override({ container_file_descriptor_limit: 200 }) + end + + it 'uses the container_file_descriptor_limit config variable' do + app = App.create_from_hash(name: 'awesome app', space_guid: space.guid) + expect(app.file_descriptors).to eq(200) + end + end end describe 'saving' do