From 3d52e532a2637d648eacb5abc73b48a3e68c9449 Mon Sep 17 00:00:00 2001 From: Fred Vollmer Date: Tue, 8 Aug 2023 11:18:11 -0400 Subject: [PATCH 1/4] Optionally send Error-level logs to Cloud Error Reporting --- README.md | 26 +++++++++- .../StackdriverFormatterTests.cs | 49 +++++++++++++------ .../StackdriverJsonFormatter.cs | 12 ++++- 3 files changed, 69 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a1d21f7..bcfb410 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Be sure to add `.ReadFrom.Configuration(configuration)` to your Serilog setup fi ### Configuration Options -The class `StackdriverJsonFormatter` has two optional arguments: +The class `StackdriverJsonFormatter` has some optional arguments. #### checkForPayloadLimit @@ -61,6 +61,30 @@ Stackdriver will break the long line into multiple lines, which will break searc Default `true`. If the Serilog Message Template should be included in the logs, e.g. ` { ... "MessageTemplate" : "Hello from {name:l}" ... }` +#### markErrorsForErrorReporting +Default `false`. If the `@type` property of the logs should be set to `type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent` when the log level is Error or above. +This causes GCP to send these logs to Cloud Error Reporting. See [documentation](https://cloud.google.com/error-reporting/docs/grouping-errors). + #### valueFormatter Defaults to `new JsonValueFormatter(typeTagName: "$type")`. A valid Serilog JSON Formatter. + +Options can be passed in the `StackdriverJsonFormatter` constructor, or set in appsettings.json: +```json +"Serilog": { + "Using": [ + "Serilog.Sinks.Console" + ], + "WriteTo": [ + { + "Name": "Console", + "Args": { + "formatter": { + "type": "Redbox.Serilog.Stackdriver.StackdriverJsonFormatter, Redbox.Serilog.Stackdriver", + "markErrorsForErrorReporting": true, + "includeMessageTemplate": false + } + } + }] +} +``` diff --git a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs index 492809c..cdcf20e 100644 --- a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs +++ b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs @@ -12,21 +12,21 @@ namespace Redbox.Serilog.Stackdriver.Tests public class StackdriverFormatterTests { private static readonly DateTimeOffset DateTimeOffset = DateTimeOffset.Now; - + [Fact] public void Test_StackdriverFormatter_Format() { var propertyName = "greeting"; var propertyValue = "hello"; - var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Debug, new Exception(), - new MessageTemplate("{greeting}", - new MessageTemplateToken[] { new PropertyToken(propertyName, propertyValue, "l") }), + var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Debug, new Exception(), + new MessageTemplate("{greeting}", + new MessageTemplateToken[] { new PropertyToken(propertyName, propertyValue, "l") }), new LogEventProperty[0]); - + using var writer = new StringWriter(); new StackdriverJsonFormatter().Format(logEvent, writer); var logDict = GetLogLineAsDictionary(writer.ToString()); - + AssertValidLogLine(logDict); Assert.True(logDict["message"] == propertyValue); } @@ -36,14 +36,14 @@ public void Test_StackdrvierFormatter_FormatLong() { // Creates a large string > 200kb var token = new TextToken(new string('*', 51200)); - var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Debug, - new Exception(), new MessageTemplate("{0}", new MessageTemplateToken[] { token }), + var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Debug, + new Exception(), new MessageTemplate("{0}", new MessageTemplateToken[] { token }), new LogEventProperty[0]); - + using var writer = new StringWriter(); new StackdriverJsonFormatter().Format(logEvent, writer); var lines = SplitLogLogs(writer.ToString()); - + // The log created was longer than Stackdriver's soft limit of 256 bytes // This means the json will be spread out onto two lines, breaking search // In this scenario the library should add an additional log event informing @@ -56,6 +56,23 @@ public void Test_StackdrvierFormatter_FormatLong() AssertValidLogLine(errorLogLineDict, hasException: false); } + [Fact] + public void Test_StackdriverFormatter_MarkErrorsForErrorReporting() + { + // Creates a large string > 200kb + var token = new TextToken("test error"); + var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Error, + null, new MessageTemplate("{0}", new MessageTemplateToken[] { token }), + Array.Empty()); + + using var writer = new StringWriter(); + new StackdriverJsonFormatter(markErrorsForErrorReporting: true).Format(logEvent, writer); + var logDict = GetLogLineAsDictionary(writer.ToString()); + + AssertValidLogLine(logDict, false); + Assert.True(logDict["@type"] == "type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent"); + } + private string[] SplitLogLogs(string logLines) { return logLines.Split("\n").Where(l => !string.IsNullOrWhiteSpace(l)).ToArray(); @@ -76,22 +93,22 @@ private Dictionary GetLogLineAsDictionary(string log) /// /// /// - private void AssertValidLogLine(Dictionary logDict, + private void AssertValidLogLine(Dictionary logDict, bool hasException = true) { Assert.True(logDict.ContainsKey("message")); Assert.NotEmpty(logDict["message"]); - + Assert.True(logDict.ContainsKey("timestamp")); var timestamp = DateTimeOffset.UtcDateTime.ToString("O"); Assert.Equal(logDict["timestamp"], timestamp); - + Assert.True(logDict.ContainsKey("fingerprint")); Assert.NotEmpty(logDict["fingerprint"]); - + Assert.True(logDict.ContainsKey("severity")); Assert.NotEmpty(logDict["severity"]); - + Assert.True(logDict.ContainsKey(("MessageTemplate"))); Assert.NotEmpty(logDict["MessageTemplate"]); @@ -102,4 +119,4 @@ private void AssertValidLogLine(Dictionary logDict, } } } -} +} \ No newline at end of file diff --git a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs index 3dba015..7f76e21 100644 --- a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs +++ b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs @@ -40,14 +40,17 @@ public class StackdriverJsonFormatter : ITextFormatter private readonly bool _checkForPayloadLimit; private readonly bool _includeMessageTemplate; + private readonly bool _markErrorsForErrorReporting; private readonly JsonValueFormatter _valueFormatter; public StackdriverJsonFormatter(bool checkForPayloadLimit = true, bool includeMessageTemplate = true, - JsonValueFormatter valueFormatter = null) + JsonValueFormatter valueFormatter = null, + bool markErrorsForErrorReporting = false) { _checkForPayloadLimit = checkForPayloadLimit; _includeMessageTemplate = includeMessageTemplate; + _markErrorsForErrorReporting = markErrorsForErrorReporting; _valueFormatter = valueFormatter ?? new JsonValueFormatter(typeTagName: "$type"); } @@ -118,6 +121,13 @@ public void FormatEvent(LogEvent logEvent, TextWriter originalOutput, JsonValueF output.Write(",\"MessageTemplate\":"); JsonValueFormatter.WriteQuotedJsonString(logEvent.MessageTemplate.Text, output); } + + // Give logs with severity: ERROR a type of ReportedErrorEvent + if (_markErrorsForErrorReporting && logEvent.Level >= LogEventLevel.Error) + { + output.Write(",\"@type\":"); + output.Write("\"type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent\""); + } // Custom Properties passed in by code logging foreach (var property in logEvent.Properties) From 92ef42bbfec9bb9ddfaccbb99414c920703b535e Mon Sep 17 00:00:00 2001 From: Fred Vollmer Date: Tue, 8 Aug 2023 11:19:17 -0400 Subject: [PATCH 2/4] undo punctuation change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bcfb410..3ed1958 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Be sure to add `.ReadFrom.Configuration(configuration)` to your Serilog setup fi ### Configuration Options -The class `StackdriverJsonFormatter` has some optional arguments. +The class `StackdriverJsonFormatter` has some optional arguments: #### checkForPayloadLimit From c0f28d840e0ed1042f233cd8f035c87c1723fb65 Mon Sep 17 00:00:00 2001 From: Fred Vollmer Date: Tue, 8 Aug 2023 14:51:04 -0400 Subject: [PATCH 3/4] Include source context --- .../StackdriverFormatterTests.cs | 67 +++++++++---------- .../StackdriverJsonFormatter.cs | 8 +++ 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs index cdcf20e..01db2dc 100644 --- a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs +++ b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Serilog.Events; using Serilog.Parsing; using Xunit; @@ -25,10 +26,10 @@ public void Test_StackdriverFormatter_Format() using var writer = new StringWriter(); new StackdriverJsonFormatter().Format(logEvent, writer); - var logDict = GetLogLineAsDictionary(writer.ToString()); + var log = JObject.Parse(writer.ToString()); - AssertValidLogLine(logDict); - Assert.True(logDict["message"] == propertyValue); + AssertValidLogLine(log); + Assert.True(log.Value("message") == propertyValue); } [Fact] @@ -50,10 +51,10 @@ public void Test_StackdrvierFormatter_FormatLong() // the user of this issue Assert.True(lines.Length == 2); // Validate each line is valid json - var ourLogLineDict = GetLogLineAsDictionary(lines[0]); - AssertValidLogLine(ourLogLineDict); - var errorLogLineDict = GetLogLineAsDictionary(lines[1]); - AssertValidLogLine(errorLogLineDict, hasException: false); + var ourLogLine = JObject.Parse(lines[0]); + AssertValidLogLine(ourLogLine); + var errorLogLine = JObject.Parse(lines[1]); + AssertValidLogLine(errorLogLine, hasException: false); } [Fact] @@ -63,14 +64,18 @@ public void Test_StackdriverFormatter_MarkErrorsForErrorReporting() var token = new TextToken("test error"); var logEvent = new LogEvent(DateTimeOffset, LogEventLevel.Error, null, new MessageTemplate("{0}", new MessageTemplateToken[] { token }), - Array.Empty()); + new[] { new LogEventProperty("SourceContext", new ScalarValue("the source context")) }); using var writer = new StringWriter(); new StackdriverJsonFormatter(markErrorsForErrorReporting: true).Format(logEvent, writer); - var logDict = GetLogLineAsDictionary(writer.ToString()); - - AssertValidLogLine(logDict, false); - Assert.True(logDict["@type"] == "type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent"); + var log = JObject.Parse(writer.ToString()); + + AssertValidLogLine(log, false); + Assert.Equal( + "type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent", + log.Value("@type") + ); + Assert.Equal("the source context", log.SelectToken("context.reportLocation.filePath")?.Value()); } private string[] SplitLogLogs(string logLines) @@ -78,44 +83,34 @@ private string[] SplitLogLogs(string logLines) return logLines.Split("\n").Where(l => !string.IsNullOrWhiteSpace(l)).ToArray(); } - /// - /// Gets a log line in json format as a dictionary of string pairs - /// - /// - /// - private Dictionary GetLogLineAsDictionary(string log) - { - return JsonConvert.DeserializeObject>(log); - } - /// /// Asserts required fields in log output are set and have valid values /// - /// + /// /// - private void AssertValidLogLine(Dictionary logDict, + private void AssertValidLogLine(JObject log, bool hasException = true) { - Assert.True(logDict.ContainsKey("message")); - Assert.NotEmpty(logDict["message"]); + Assert.True(log.ContainsKey("message")); + Assert.NotEmpty(log.Value("message")); - Assert.True(logDict.ContainsKey("timestamp")); + Assert.True(log.ContainsKey("timestamp")); var timestamp = DateTimeOffset.UtcDateTime.ToString("O"); - Assert.Equal(logDict["timestamp"], timestamp); + Assert.Equal(log.Value("timestamp").ToString("O"), timestamp); - Assert.True(logDict.ContainsKey("fingerprint")); - Assert.NotEmpty(logDict["fingerprint"]); + Assert.True(log.ContainsKey("fingerprint")); + Assert.NotEmpty(log.Value("fingerprint")); - Assert.True(logDict.ContainsKey("severity")); - Assert.NotEmpty(logDict["severity"]); + Assert.True(log.ContainsKey("severity")); + Assert.NotEmpty(log.Value("severity")); - Assert.True(logDict.ContainsKey(("MessageTemplate"))); - Assert.NotEmpty(logDict["MessageTemplate"]); + Assert.True(log.ContainsKey(("MessageTemplate"))); + Assert.NotEmpty(log.Value("MessageTemplate")); if (hasException) { - Assert.True(logDict.ContainsKey("exception")); - Assert.NotEmpty(logDict["exception"]); + Assert.True(log.ContainsKey("exception")); + Assert.NotEmpty(log.Value("exception")); } } } diff --git a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs index 7f76e21..aedd0d0 100644 --- a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs +++ b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs @@ -127,6 +127,14 @@ public void FormatEvent(LogEvent logEvent, TextWriter originalOutput, JsonValueF { output.Write(",\"@type\":"); output.Write("\"type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent\""); + + logEvent.Properties.TryGetValue("SourceContext", out var sourceContext); + if (sourceContext != null) + { + output.Write(",\"context\":{\"reportLocation\":{"); + WriteKeyValue(output, valueFormatter, "filePath", sourceContext, false); + output.Write("}}"); + } } // Custom Properties passed in by code logging From 1492bad06538c4fa55442f9b228cc6ec753a9bb0 Mon Sep 17 00:00:00 2001 From: Fred Vollmer Date: Tue, 8 Aug 2023 16:35:45 -0400 Subject: [PATCH 4/4] Add serviceContext --- README.md | 10 ++++++++++ .../StackdriverFormatterTests.cs | 10 ++++++++++ .../StackdriverJsonFormatter.cs | 20 ++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ed1958..fcb26ec 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,16 @@ Default `true`. If the Serilog Message Template should be included in the logs, Default `false`. If the `@type` property of the logs should be set to `type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent` when the log level is Error or above. This causes GCP to send these logs to Cloud Error Reporting. See [documentation](https://cloud.google.com/error-reporting/docs/grouping-errors). +#### serviceName +Defaults to the executing assembly name. Sets the service name used by GCP error reporting. + +#### serviceVersion +Defaults to the executing assembly version. Sets the service version used by GCP error reporting. + +#### markErrorsForErrorReporting +Default `false`. If the `@type` property of the logs should be set to `type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent` when the log level is Error or above. +This causes GCP to send these logs to Cloud Error Reporting. See [documentation](https://cloud.google.com/error-reporting/docs/grouping-errors). + #### valueFormatter Defaults to `new JsonValueFormatter(typeTagName: "$type")`. A valid Serilog JSON Formatter. diff --git a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs index 01db2dc..c965f2b 100644 --- a/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs +++ b/src/Redbox.Serilog.Stackdriver.Tests/StackdriverFormatterTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Serilog.Events; @@ -71,11 +72,20 @@ public void Test_StackdriverFormatter_MarkErrorsForErrorReporting() var log = JObject.Parse(writer.ToString()); AssertValidLogLine(log, false); + + // @type Assert.Equal( "type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent", log.Value("@type") ); + + // Report location Assert.Equal("the source context", log.SelectToken("context.reportLocation.filePath")?.Value()); + + // Service context + var assemblyName = Assembly.GetEntryAssembly()?.GetName(); + Assert.Equal(assemblyName?.Name, log.SelectToken("serviceContext.service")?.Value()); + Assert.Equal(assemblyName?.Version?.ToString(), log.SelectToken("serviceContext.version")?.Value()); } private string[] SplitLogLogs(string logLines) diff --git a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs index aedd0d0..c68f8ee 100644 --- a/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs +++ b/src/Redbox.Serilog.Stackdriver/StackdriverJsonFormatter.cs @@ -26,6 +26,7 @@ using Serilog.Parsing; using System.Collections.Generic; using System.Linq; +using System.Reflection; namespace Redbox.Serilog.Stackdriver { @@ -42,16 +43,24 @@ public class StackdriverJsonFormatter : ITextFormatter private readonly bool _includeMessageTemplate; private readonly bool _markErrorsForErrorReporting; private readonly JsonValueFormatter _valueFormatter; + private readonly LogEventPropertyValue _assemblyVersion; + private readonly LogEventPropertyValue _assemblyName; public StackdriverJsonFormatter(bool checkForPayloadLimit = true, bool includeMessageTemplate = true, JsonValueFormatter valueFormatter = null, - bool markErrorsForErrorReporting = false) + bool markErrorsForErrorReporting = false, + string serviceName = null, + string serviceVersion = null) { _checkForPayloadLimit = checkForPayloadLimit; _includeMessageTemplate = includeMessageTemplate; _markErrorsForErrorReporting = markErrorsForErrorReporting; _valueFormatter = valueFormatter ?? new JsonValueFormatter(typeTagName: "$type"); + + var assemblyName = Assembly.GetEntryAssembly()?.GetName(); + _assemblyName = new ScalarValue(serviceName ?? assemblyName?.Name); + _assemblyVersion = new ScalarValue(serviceVersion ?? assemblyName?.Version.ToString()); } /// @@ -125,9 +134,12 @@ public void FormatEvent(LogEvent logEvent, TextWriter originalOutput, JsonValueF // Give logs with severity: ERROR a type of ReportedErrorEvent if (_markErrorsForErrorReporting && logEvent.Level >= LogEventLevel.Error) { + // Set @type so that Cloud Error Reporting will recognize this error output.Write(",\"@type\":"); output.Write("\"type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent\""); + // If SourceContext is defined, set context.reportLocation to this value. + // We use filePath, the highest-level parameter of reportLocation, as it does not have a className parameter. logEvent.Properties.TryGetValue("SourceContext", out var sourceContext); if (sourceContext != null) { @@ -135,6 +147,12 @@ public void FormatEvent(LogEvent logEvent, TextWriter originalOutput, JsonValueF WriteKeyValue(output, valueFormatter, "filePath", sourceContext, false); output.Write("}}"); } + + // Set the serviceContext + output.Write(",\"serviceContext\":{"); + WriteKeyValue(output, valueFormatter, "service", _assemblyName, false); + WriteKeyValue(output, valueFormatter, "version", _assemblyVersion); + output.Write("}"); } // Custom Properties passed in by code logging