Skip to content

Logstash 2.2.1 never stops using logstash-input-jmx-2.0.2 log contains: The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.#16

Open
caspermout wants to merge 2 commits into
logstash-plugins:mainfrom
caspermout:master

Conversation

@caspermout
Copy link
Copy Markdown

{:timestamp=>"2016-02-17T09:28:10.667000+0100", :message=>#<LogStash::PipelineReporter::Snapshot:0x86f90f @DaTa={:events_filtered=>410875, :events_consumed=>410995, :worker_count=>2, :inflight_count=>244, :worker_states=>[{:status=>"run", :alive=>true, :index=>0, :inflight_count=>120}, {:status=>"sleep", :alive=>true, :index=>1, :inflight_count=>124}], :output_info=>[{:type=>"elasticsearch", :config=>{"hosts"=>"localhost"}, :is_multi_worker=>true, :events_received=>410875, :workers=>[<LogStash::Outputs::ElasticSearch hosts=>["localhost"], codec=><LogStash::Codecs::Plain charset=>"UTF-8">, workers=>1, index=>"logstash-%{+YYYY.MM.dd}", manage_template=>true, template_name=>"logstash", template_overwrite=>false, flush_size=>500, idle_flush_time=>1, doc_as_upsert=>false, max_retries=>3, script_type=>"inline", script_var_name=>"event", scripted_upsert=>false, retry_max_interval=>2, retry_max_items=>500, action=>"index", path=>"/", ssl_certificate_verification=>true, sniffing=>false, sniffing_delay=>5>, <LogStash::Outputs::ElasticSearch hosts=>["localhost"], codec=><LogStash::Codecs::Plain charset=>"UTF-8">, workers=>1, index=>"logstash-%{+YYYY.MM.dd}", manage_template=>true, template_name=>"logstash", template_overwrite=>false, flush_size=>500, idle_flush_time=>1, doc_as_upsert=>false, max_retries=>3, script_type=>"inline", script_var_name=>"event", scripted_upsert=>false, retry_max_interval=>2, retry_max_items=>500, action=>"index", path=>"/", ssl_certificate_verification=>true, sniffing=>false, sniffing_delay=>5>], :busy_workers=>0}], :thread_info=>[{"thread_id"=>18, "name"=>"[base]<jmx", "plugin"=>nil, "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-input-jmx-2.0.2/lib/logstash/inputs/jmx.rb:351:in sleep'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-jmx-2.0.2/lib/logstash/inputs/jmx.rb:351:inrun'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:331:in inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:325:instart_input'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-input-jmx-2.0.2/lib/logstash/inputs/jmx.rb:351:in sleep'"}, {"thread_id"=>35, "name"=>"[base]>worker0", "plugin"=>["LogStash::Filters::Mutate", {"gsub"=>["short_message", "^(.{0,20}).*", "\\1"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:328:ingsub'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:311:in each'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:311:ingsub'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:221:in filter'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/filters/base.rb:151:inmulti_filter'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/filters/base.rb:148:in each'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/filters/base.rb:148:inmulti_filter'", "(eval):160:in filter_func'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:256:infilter_batch'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:254:in inject'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:254:infilter_batch'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:212:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:190:instart_workers'"], "blocked_on"=>nil, "status"=>"run", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:328:in gsub'"}, {"thread_id"=>36, "name"=>"[base]>worker1", "plugin"=>["LogStash::Filters::Mutate", {"gsub"=>["short_message", "^(.{0,20}).*", "\\1"]}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:298:insynchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:298:in inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:223:inworker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:190:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:298:insynchronize'"}], :stalling_threads_info=>[{"thread_id"=>18, "name"=>"[base]<jmx", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-input-jmx-2.0.2/lib/logstash/inputs/jmx.rb:351:in sleep'"}, {"thread_id"=>35, "name"=>"[base]>worker0", "plugin"=>["LogStash::Filters::Mutate", {"gsub"=>["short_message", "^(.{0,20}).*", "\\1"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-2.0.3/lib/logstash/filters/mutate.rb:311:ineach'"}, {"thread_id"=>36, "name"=>"[base]>worker1", "plugin"=>["LogStash::Filters::Mutate", {"gsub"=>["short_message", "^(.{0,20}).*", "\1"]}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.2.1-java/lib/logstash/pipeline.rb:298:in `synchronize'"}]}>, :level=>:warn}
{:timestamp=>"2016-02-17T09:28:10.674000+0100", :message=>"The shutdown process appears to be stalled due to busy or blocked plugins. Check the logs for more information.", :level=>:error}

@ghost
Copy link
Copy Markdown

ghost commented Feb 17, 2016

Jenkins standing by to test this. If you aren't a maintainer, you can ignore this comment. Someone with commit access, please review this and clear it for Jenkins to run; then say 'jenkins, test it'.

@DEvil0000
Copy link
Copy Markdown

not fully fixing the shutdown issue for me - still gets killed at least sometimes..
hangs in:
/vendor/bundle/jruby/1.9/gems/logstash-input-jmx-2.0.4/lib/logstash/inputs/jmx.rb:352:in `sleep'"

@boernd
Copy link
Copy Markdown

boernd commented Nov 9, 2016

Any chance we get this PR merged? Fixes the issue for me (LS 5.0.0).

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

4 participants