From 54a6832a1563da2e7202fd22e116838019b8a500 Mon Sep 17 00:00:00 2001 From: Ayyappa Vemulkar Date: Mon, 18 Jan 2016 16:37:01 -0800 Subject: [PATCH 1/4] limited info level messages to have max length of 2000 characters --- lib/contextual_logging/logstash_message_formatter.rb | 2 +- spec/contextual_logging_spec.rb | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/contextual_logging/logstash_message_formatter.rb b/lib/contextual_logging/logstash_message_formatter.rb index 1f9f97a..d7d4ff7 100644 --- a/lib/contextual_logging/logstash_message_formatter.rb +++ b/lib/contextual_logging/logstash_message_formatter.rb @@ -1,6 +1,7 @@ module ContextualLogging class LogstashMessageFormatter def format(severity, message, extra_context) + message = message[0..1999] if severity == 'INFO' msg_hash = HashWithIndifferentAccess.new('message' => message, 'log_level' => severity) logstash_data = extra_context.merge(msg_hash) logstash_event = LogStash::Event.new(logstash_data) @@ -8,4 +9,3 @@ def format(severity, message, extra_context) end end end - diff --git a/spec/contextual_logging_spec.rb b/spec/contextual_logging_spec.rb index 1963fdb..9a1020f 100644 --- a/spec/contextual_logging_spec.rb +++ b/spec/contextual_logging_spec.rb @@ -1,4 +1,10 @@ require 'spec_helper' -describe ContextualLogging do +describe ContextualLogging::LogstashMessageFormatter do + message_formatter = ContextualLogging::LogstashMessageFormatter.new + it 'should limit message size to 2000 characters when severity level is INFO' do + message = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join + log_message = message_formatter.format('INFO', message, {}) + expect(eval(log_message)[:message].length).to equal 2000 + end end From cc2cc42584d42c5c22548d5b6f2c250f712d8982 Mon Sep 17 00:00:00 2001 From: Ayyappa Vemulkar Date: Tue, 19 Jan 2016 14:15:09 -0800 Subject: [PATCH 2/4] changed to create a new hash when severity is INFO --- .../logstash_message_formatter.rb | 20 +++++++++++++++++-- spec/contextual_logging_spec.rb | 15 +++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/contextual_logging/logstash_message_formatter.rb b/lib/contextual_logging/logstash_message_formatter.rb index d7d4ff7..ddb2285 100644 --- a/lib/contextual_logging/logstash_message_formatter.rb +++ b/lib/contextual_logging/logstash_message_formatter.rb @@ -1,9 +1,25 @@ module ContextualLogging class LogstashMessageFormatter def format(severity, message, extra_context) - message = message[0..1999] if severity == 'INFO' + max_length = 2000 + + if severity == 'INFO' + message = message[0...max_length] + limited_extra_context = HashWithIndifferentAccess.new() + extra_context.each do |key,value| + if value.class == HashWithIndifferentAccess + limited_extra_context[key] = HashWithIndifferentAccess.new() + value.each do |k,v| + limited_extra_context[key][k] = v.inspect[0...max_length] + end + else + limited_extra_context[key] = value + end + end + end + msg_hash = HashWithIndifferentAccess.new('message' => message, 'log_level' => severity) - logstash_data = extra_context.merge(msg_hash) + logstash_data = (severity == 'INFO') ? limited_extra_context.merge(msg_hash) : extra_context.merge(msg_hash) logstash_event = LogStash::Event.new(logstash_data) logstash_event.to_json end diff --git a/spec/contextual_logging_spec.rb b/spec/contextual_logging_spec.rb index 9a1020f..993ed7f 100644 --- a/spec/contextual_logging_spec.rb +++ b/spec/contextual_logging_spec.rb @@ -5,6 +5,19 @@ it 'should limit message size to 2000 characters when severity level is INFO' do message = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join log_message = message_formatter.format('INFO', message, {}) - expect(eval(log_message)[:message].length).to equal 2000 + expect(JSON[log_message]['message'].length).to equal 2000 end + + it 'should limit extra context values that are two levels deep to 2000 characters when severity level is INFO' do + message = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join + extra_context = HashWithIndifferentAccess.new() + extra_context['params'] = HashWithIndifferentAccess.new(nodes: {}) + extra_context['text'] = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join + extra_context['params']['nodes']['a'] = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join + + log_message = message_formatter.format('INFO', message, extra_context) + expect(JSON[log_message]['params']['nodes'].length).to equal 2000 + expect(JSON[log_message]['text'].length).to equal 3000 + end + end From bfc5d01fa253ce9668bd4e57acbd5b08ef943a3f Mon Sep 17 00:00:00 2001 From: Ayyappa Vemulkar Date: Tue, 19 Jan 2016 15:18:06 -0800 Subject: [PATCH 3/4] added a constant, removed ternary, fixed test --- .../logstash_message_formatter.rb | 28 +++++++++++++------ spec/contextual_logging_spec.rb | 3 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/contextual_logging/logstash_message_formatter.rb b/lib/contextual_logging/logstash_message_formatter.rb index ddb2285..5924a1b 100644 --- a/lib/contextual_logging/logstash_message_formatter.rb +++ b/lib/contextual_logging/logstash_message_formatter.rb @@ -1,16 +1,20 @@ module ContextualLogging class LogstashMessageFormatter - def format(severity, message, extra_context) - max_length = 2000 + MAX_MESSAGE_LENGTH = 2000 + def format(severity, message, extra_context) if severity == 'INFO' - message = message[0...max_length] - limited_extra_context = HashWithIndifferentAccess.new() + truncated_message = message[0...MAX_MESSAGE_LENGTH] + limited_extra_context = HashWithIndifferentAccess.new extra_context.each do |key,value| - if value.class == HashWithIndifferentAccess - limited_extra_context[key] = HashWithIndifferentAccess.new() + if value.is_a?(Hash) + limited_extra_context[key] = HashWithIndifferentAccess.new value.each do |k,v| - limited_extra_context[key][k] = v.inspect[0...max_length] + if v.is_a?(String) + limited_extra_context[key][k] = v[0...MAX_MESSAGE_LENGTH] + else + limited_extra_context[key][k] = v.inspect[0...MAX_MESSAGE_LENGTH] + end end else limited_extra_context[key] = value @@ -18,8 +22,14 @@ def format(severity, message, extra_context) end end - msg_hash = HashWithIndifferentAccess.new('message' => message, 'log_level' => severity) - logstash_data = (severity == 'INFO') ? limited_extra_context.merge(msg_hash) : extra_context.merge(msg_hash) + if severity == 'INFO' + msg_hash = HashWithIndifferentAccess.new('message' => truncated_message, 'log_level' => severity) + logstash_data = limited_extra_context.merge(msg_hash) + else + msg_hash = HashWithIndifferentAccess.new('message' => message, 'log_level' => severity) + logstash_data = extra_context.merge(msg_hash) + end + logstash_event = LogStash::Event.new(logstash_data) logstash_event.to_json end diff --git a/spec/contextual_logging_spec.rb b/spec/contextual_logging_spec.rb index 993ed7f..e4db045 100644 --- a/spec/contextual_logging_spec.rb +++ b/spec/contextual_logging_spec.rb @@ -10,7 +10,7 @@ it 'should limit extra context values that are two levels deep to 2000 characters when severity level is INFO' do message = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join - extra_context = HashWithIndifferentAccess.new() + extra_context = HashWithIndifferentAccess.new extra_context['params'] = HashWithIndifferentAccess.new(nodes: {}) extra_context['text'] = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join extra_context['params']['nodes']['a'] = (0...3000).map { ('a'..'z').to_a[rand(26)] }.join @@ -19,5 +19,4 @@ expect(JSON[log_message]['params']['nodes'].length).to equal 2000 expect(JSON[log_message]['text'].length).to equal 3000 end - end From 313bcf74f698a00da721db86203c6b66930dcc7b Mon Sep 17 00:00:00 2001 From: Ayyappa Vemulkar Date: Fri, 22 Jan 2016 15:38:11 -0800 Subject: [PATCH 4/4] cleaned up stuff and broke things into smaller functions --- .../logstash_message_formatter.rb | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/contextual_logging/logstash_message_formatter.rb b/lib/contextual_logging/logstash_message_formatter.rb index 5924a1b..a7017e2 100644 --- a/lib/contextual_logging/logstash_message_formatter.rb +++ b/lib/contextual_logging/logstash_message_formatter.rb @@ -4,34 +4,43 @@ class LogstashMessageFormatter def format(severity, message, extra_context) if severity == 'INFO' - truncated_message = message[0...MAX_MESSAGE_LENGTH] - limited_extra_context = HashWithIndifferentAccess.new - extra_context.each do |key,value| - if value.is_a?(Hash) - limited_extra_context[key] = HashWithIndifferentAccess.new - value.each do |k,v| - if v.is_a?(String) - limited_extra_context[key][k] = v[0...MAX_MESSAGE_LENGTH] - else - limited_extra_context[key][k] = v.inspect[0...MAX_MESSAGE_LENGTH] - end - end - else - limited_extra_context[key] = value - end - end - end - - if severity == 'INFO' - msg_hash = HashWithIndifferentAccess.new('message' => truncated_message, 'log_level' => severity) - logstash_data = limited_extra_context.merge(msg_hash) + truncated_message = truncate(message) + limited_extra_context = truncate_log_extra_context(extra_context) else - msg_hash = HashWithIndifferentAccess.new('message' => message, 'log_level' => severity) - logstash_data = extra_context.merge(msg_hash) + truncated_message = message + limited_extra_context = extra_context end + logstash_data = HashWithIndifferentAccess.new( + 'message' => truncated_message, + 'log_level' => severity + ).merge(limited_extra_context) + logstash_event = LogStash::Event.new(logstash_data) logstash_event.to_json end + + private + + def truncate(message) + message[0...MAX_MESSAGE_LENGTH] + end + + def truncate_log_extra_context(extra_context) + extra_context.each_with_object(HashWithIndifferentAccess.new) do |(key, value), limited_extra_context| + if value.is_a?(Hash) + limited_extra_context[key] = HashWithIndifferentAccess.new + value.each do |k, v| + if v.is_a?(String) + limited_extra_context[key][k] = truncate(v) + else + limited_extra_context[key][k] = truncate(v.inspect) + end + end + else + limited_extra_context[key] = value + end + end + end end end