Configurable file_descriptors limit for warden container

.travis.yml

@@ -19,13 +19,13 @@ before_script:
 script:
   - bundle exec rake $TASKS
 
+after_success:
+  - "[ $TRAVIS_BRANCH == 'master' -a $UPLOAD_DOCUMENTATION == 'true' ] && travis-artifacts upload --path doc/api --target-path $TRAVIS_BUILD_ID && echo https://s3-us-east-1.amazonaws.com/cc-api-docs/$TRAVIS_BUILD_ID/index.html"
+
 services:
   - mysql
   - postgresql
 
-after_script: # and this only on success, fail the whole build if it fails
-  - "[ $TRAVIS_TEST_RESULT == 0 -a $TRAVIS_BRANCH == 'master' ] && travis-artifacts upload --path doc/api --target-path $TRAVIS_BUILD_ID && echo https://s3-us-east-1.amazonaws.com/cc-api-docs/$TRAVIS_BUILD_ID/index.html"
-
 env:
   global:
     # API Docs
@@ -39,5 +39,5 @@ env:
     - COVERAGE=true DB=postgres TASKS=spec:all
     - DB=mysql TASKS=spec:all
 
-    - DB=postgres TASKS=spec:api
+    - DB=postgres TASKS=spec:api UPLOAD_DOCUMENTATION=true
     - TASKS=rubocop

app/access/v3/package_access.rb

@@ -0,0 +1,34 @@
+module VCAP::CloudController
+  class PackageModelAccess
+    include Allowy::AccessControl
+
+    def read?(package)
+      return true if context.roles.admin?
+
+      has_read_scope = SecurityContext.scopes.include?('cloud_controller.read')
+      user_visible = Space.user_visible(context.user).where(guid: package.space_guid).count > 0
+
+      has_read_scope && user_visible
+    end
+
+    def create?(_, space)
+      return true if context.roles.admin?
+
+      has_write_scope = SecurityContext.scopes.include?('cloud_controller.write')
+      is_space_developer = space && space.developers.include?(context.user)
+      org_active = space && space.organization.active?
+
+      has_write_scope && is_space_developer && org_active
+    end
+
+    def delete?(package, space)
+      create?(package, space)
+    end
+
+    def upload?(package, space)
+      return true if context.roles.admin?
+      return false unless FeatureFlag.enabled?('app_bits_upload')
+      create?(package, space)
+    end
+  end
+end

app/collection_transformers/username_populator.rb

@@ -0,0 +1,15 @@
+module VCAP::CloudController
+  class UsernamePopulator
+    attr_reader :uaa_client
+
+    def initialize(uaa_client)
+      @uaa_client = uaa_client
+    end
+
+    def transform(users)
+      user_ids = users.collect(&:guid)
+      username_mapping = uaa_client.usernames_for_ids(user_ids)
+      users.each { |user| user.username = username_mapping[user.guid] }
+    end
+  end
+end

app/controllers/base/model_controller.rb

@@ -5,6 +5,8 @@ module VCAP::CloudController::RestController
   class ModelController < BaseController
     include Routes
 
+    attr_reader :object_renderer, :collection_renderer
+
     def inject_dependencies(dependencies)
       super
       @object_renderer = dependencies.fetch(:object_renderer)
@@ -74,11 +76,15 @@ def update(guid)
     def do_delete(obj)
       raise_if_has_associations!(obj) if v2_api? && !recursive?
       model_deletion_job = Jobs::Runtime::ModelDeletion.new(obj.class, obj.guid)
+      enqueue_deletion_job(model_deletion_job)
+    end
+
+    def enqueue_deletion_job(deletion_job)
       if async?
-        job = Jobs::Enqueuer.new(model_deletion_job, queue: 'cc-generic').enqueue
+        job = Jobs::Enqueuer.new(deletion_job, queue: 'cc-generic').enqueue
         [HTTP::ACCEPTED, JobPresenter.new(job).to_json]
       else
-        model_deletion_job.perform
+        deletion_job.perform
         [HTTP::NO_CONTENT, nil]
       end
     end
@@ -130,7 +136,9 @@ def enumerate_related(guid, name)
         @opts
       )
 
-      collection_renderer.render_json(
+      associated_controller_instance = CloudController::ControllerFactory.new(@config, @logger, @env, @params, @body, @sinatra).create_controller(associated_controller)
+
+      associated_controller_instance.collection_renderer.render_json(
         associated_controller,
         filtered_dataset,
         associated_path,
@@ -228,10 +236,6 @@ def model
       self.class.model
     end
 
-    protected
-
-    attr_reader :object_renderer, :collection_renderer
-
     private
 
     def enumerate_dataset

app/controllers/internal/bulk_apps_controller.rb

@@ -2,6 +2,7 @@
 require 'controllers/base/base_controller'
 require 'cloud_controller/diego/client'
 require 'cloud_controller/internal_api'
+require 'cloud_controller/diego/process_guid'
 
 module VCAP::CloudController
   class BulkAppsController < RestController::BaseController
@@ -21,9 +22,34 @@ def bulk_apps
       bulk_token = MultiJson.load(params.fetch('token'))
       last_id = Integer(bulk_token['id'] || 0)
 
-      dependency_locator = ::CloudController::DependencyLocator.instance
-      runners = dependency_locator.runners
+      if params['format'] == 'fingerprint'
+        bulk_fingerprint_format(batch_size, last_id)
+      else
+        bulk_desire_app_format(batch_size, last_id)
+      end
+    rescue IndexError => e
+      raise ApiError.new_from_details('BadQueryParameter', e.message)
+    end
+
+    get '/internal/bulk/apps', :bulk_apps
+
+    def filtered_bulk_apps
+      raise ApiError.new_from_details('MessageParseError', 'Missing request body') if body.length == 0
+      payload = MultiJson.load(body)
+
+      apps = runners.diego_apps_from_process_guids(payload)
+      messages = apps.map { |app| runners.runner_for_app(app).desire_app_message }
+
+      MultiJson.dump(messages)
+    rescue MultiJson::ParseError => e
+      raise ApiError.new_from_details('MessageParseError', e.message)
+    end
+
+    post '/internal/bulk/apps', :filtered_bulk_apps
 
+    private
+
+    def bulk_desire_app_format(batch_size, last_id)
       apps = runners.diego_apps(batch_size, last_id)
       messages = apps.map { |app| runners.runner_for_app(app).desire_app_message }
       id_for_next_token = apps.empty? ? nil : apps.last.id
@@ -32,10 +58,24 @@ def bulk_apps
         apps: messages,
         token: { 'id' => id_for_next_token }
       )
-    rescue IndexError => e
-      raise ApiError.new_from_details('BadQueryParameter', e.message)
     end
 
-    get '/internal/bulk/apps', :bulk_apps
+    def bulk_fingerprint_format(batch_size, last_id)
+      id_for_next_token = nil
+      messages = runners.diego_apps_cache_data(batch_size, last_id).map do |id, guid, version, updated|
+        id_for_next_token = id
+        { 'process_guid' => Diego::ProcessGuid.from(guid, version), 'etag' => updated.to_f.to_s }
+      end
+
+      MultiJson.dump(
+        fingerprints: messages,
+        token: { 'id' => id_for_next_token }
+      )
+    end
+
+    def runners
+      dependency_locator = ::CloudController::DependencyLocator.instance
+      @runners ||= dependency_locator.runners
+    end
   end
 end

app/controllers/runtime/users_controller.rb

@@ -1,5 +1,9 @@
 module VCAP::CloudController
   class UsersController < RestController::ModelController
+    def self.dependencies
+      [:username_populating_collection_renderer]
+    end
+
     define_attributes do
       attribute :guid, String, exclude_in: :update
       attribute :admin, Message::Boolean, default: false
@@ -33,6 +37,11 @@ def delete(guid)
       do_delete(find_guid_and_validate_access(:delete, guid))
     end
 
+    def inject_dependencies(dependencies)
+      super
+      @collection_renderer = dependencies[:username_populating_collection_renderer]
+    end
+
     define_messages
     define_routes
   end

app/controllers/services/service_bindings_controller.rb

@@ -24,19 +24,14 @@ def inject_dependencies(dependencies)
 
     post path, :create
     def create
-      json_msg = self.class::CreateMessage.decode(body)
-
-      @request_attrs = json_msg.extract(stringify_keys: true)
+      @request_attrs = self.class::CreateMessage.decode(body).extract(stringify_keys: true)
 
       logger.debug 'cc.create', model: self.class.model_class_name, attributes: request_attrs
 
       raise InvalidRequest unless request_attrs
 
-      instance_guid = request_attrs['service_instance_guid']
-      app_guid      = request_attrs['app_guid']
-
-      validate_service_instance(instance_guid)
-      validate_app(app_guid)
+      validate_service_instance(request_attrs['service_instance_guid'])
+      validate_app(request_attrs['app_guid'])
 
       service_binding = ServiceBinding.new(@request_attrs)
       validate_access(:create, service_binding)
@@ -63,13 +58,7 @@ def delete(guid)
       deletion_job = Jobs::Runtime::ModelDeletion.new(ServiceBinding, guid)
       delete_and_audit_job = Jobs::AuditEventJob.new(deletion_job, @services_event_repository, :record_service_binding_event, :delete, service_binding)
 
-      if async?
-        job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue
-        [HTTP::ACCEPTED, JobPresenter.new(job).to_json]
-      else
-        delete_and_audit_job.perform
-        [HTTP::NO_CONTENT, nil]
-      end
+      enqueue_deletion_job(delete_and_audit_job)
     end
 
     private

app/controllers/services/service_instances_controller.rb

@@ -200,13 +200,7 @@ def delete(guid)
       event_method = service_instance.type == 'managed_service_instance' ?  :record_service_instance_event : :record_user_provided_service_instance_event
       delete_and_audit_job = Jobs::AuditEventJob.new(deletion_job, @services_event_repository, event_method, :delete, service_instance, {})
 
-      if async?
-        job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue
-        [HTTP::ACCEPTED, JobPresenter.new(job).to_json]
-      else
-        delete_and_audit_job.perform
-        [HTTP::NO_CONTENT, nil]
-      end
+      enqueue_deletion_job(delete_and_audit_job)
     end
 
     def get_filtered_dataset_for_enumeration(model, ds, qp, opts)

app/controllers/services/service_plan_visibilities_controller.rb

@@ -81,13 +81,7 @@ def delete(guid)
         {}
       )
 
-      if async?
-        job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue
-        [HTTP::ACCEPTED, JobPresenter.new(job).to_json]
-      else
-        delete_and_audit_job.perform
-        [HTTP::NO_CONTENT, nil]
-      end
+      enqueue_deletion_job(delete_and_audit_job)
     end
 
     define_messages

app/controllers/services/user_provided_service_instances_controller.rb

@@ -99,13 +99,7 @@ def delete(guid)
         {}
       )
 
-      if async?
-        job = Jobs::Enqueuer.new(delete_and_audit_job, queue: 'cc-generic').enqueue
-        [HTTP::ACCEPTED, JobPresenter.new(job).to_json]
-      else
-        delete_and_audit_job.perform
-        [HTTP::NO_CONTENT, nil]
-      end
+      enqueue_deletion_job(delete_and_audit_job)
     end
 
     define_messages

app/controllers/v3/apps_controller.rb

@@ -1,6 +1,7 @@
 require 'presenters/v3/app_presenter'
 require 'handlers/processes_handler'
 require 'handlers/apps_handler'
+require 'cloud_controller/paging/pagination_options'
 
 module VCAP::CloudController
   class AppsV3Controller < RestController::BaseController
@@ -15,6 +16,14 @@ def inject_dependencies(dependencies)
       @process_presenter = dependencies[:process_presenter]
     end
 
+    get '/v3/apps', :list
+    def list
+      pagination_options = PaginationOptions.from_params(params)
+      paginated_result   = @app_handler.list(pagination_options, @access_context)
+
+      [HTTP::OK, @app_presenter.present_json_list(paginated_result)]
+    end
+
     get '/v3/apps/:guid', :show
     def show(guid)
       app = @app_handler.show(guid, @access_context)
@@ -73,7 +82,10 @@ def list_processes(guid)
       app = @app_handler.show(guid, @access_context)
       app_not_found! if app.nil?
 
-      [HTTP::OK, @process_presenter.present_json_list(app.processes)]
+      pagination_options = PaginationOptions.from_params(params)
+      paginated_result   = @process_handler.list(pagination_options, @access_context, app_guid: app.guid)
+
+      [HTTP::OK, @process_presenter.present_json_list(paginated_result, "/v3/apps/#{guid}/processes")]
     end
 
     put '/v3/apps/:guid/processes', :add_process