diff --git a/SS14.Admin/AdminLogs/Export/ExportConfiguration.cs b/SS14.Admin/AdminLogs/Export/ExportConfiguration.cs new file mode 100644 index 0000000..c5748ab --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/ExportConfiguration.cs @@ -0,0 +1,16 @@ +namespace SS14.Admin.AdminLogs.Export; + +public class ExportConfiguration +{ + public const string Name = "Export"; + + /// + /// The maximum amount of export processes that can be queued up before new export requests will be rejected + /// + public int ProcessQueueMaxSize { get; set; } = 6; + + /// + /// This is the directory for containing generated exports + /// + public string ExportDirectory { get; set; } = "export"; +} diff --git a/SS14.Admin/AdminLogs/Export/ExportProcessItem.cs b/SS14.Admin/AdminLogs/Export/ExportProcessItem.cs new file mode 100644 index 0000000..dc51bf9 --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/ExportProcessItem.cs @@ -0,0 +1,9 @@ +namespace SS14.Admin.AdminLogs.Export; + +public sealed record ExportProcessItem( + string? Search = null, + DateTime? FromDate = null, + DateTime? ToDate = null, + int? RoundId = null, + bool UseCompression = false +); diff --git a/SS14.Admin/AdminLogs/Export/LogExportBackgroundService.cs b/SS14.Admin/AdminLogs/Export/LogExportBackgroundService.cs new file mode 100644 index 0000000..7795eb4 --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/LogExportBackgroundService.cs @@ -0,0 +1,42 @@ +using Serilog; +using ILogger = Serilog.ILogger; + +namespace SS14.Admin.AdminLogs.Export; + +public sealed class LogExportBackgroundService : BackgroundService +{ + private readonly IServiceProvider _serviceProvider; + private readonly LogExportQueue _queue; + private readonly ILogger _log; + + public LogExportBackgroundService(IServiceProvider provider, LogExportQueue queue, IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + _queue = queue; + + _log = Log.ForContext(); + } + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + { + _log.Information("{ServiceName} started", nameof(LogExportBackgroundService)); + return ProcessQueueAsync(stoppingToken); + } + + private async Task ProcessQueueAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + var item = await _queue.DequeueAsync(ct); + await ProcessTask(item, ct); + } + } + + private async Task ProcessTask(ExportProcessItem item, CancellationToken ct) + { + using var scope = _serviceProvider.CreateScope(); + var exporter = scope.ServiceProvider.GetRequiredService(); + var filename = await exporter.Export(item, ct); + await _queue.ReportFinishedExport(filename); + } +} diff --git a/SS14.Admin/AdminLogs/Export/LogExportExtension.cs b/SS14.Admin/AdminLogs/Export/LogExportExtension.cs new file mode 100644 index 0000000..e80f8d4 --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/LogExportExtension.cs @@ -0,0 +1,47 @@ +using System.Text.Json; +using Microsoft.AspNetCore.Routing.Constraints; +using Microsoft.Extensions.Options; + +namespace SS14.Admin.AdminLogs.Export; + +public static class LogExportExtension +{ + public static void MapLogExportEndpoints(this IEndpointRouteBuilder endpoints) + { + endpoints.MapGet("/logs/export/poll", async (CancellationToken ct, LogExportQueue queue) => + { + var timoutCt = CancellationTokenSource.CreateLinkedTokenSource(ct); + timoutCt.CancelAfter(TimeSpan.FromSeconds(120)); + using var channel = queue.CreateReportChannel(); + var filename = await channel.Listen(timoutCt.Token); + + return Results.Ok(filename); + }).RequireAuthorization(); + + endpoints.MapGet("/logs/export/list", async (IOptions config) => + { + var exportPath = Path.Combine(Directory.GetCurrentDirectory(), config.Value.ExportDirectory); + var files = Directory.EnumerateFiles(exportPath, "*.csv*"); + + return Results.Ok(files.Select(Path.GetFileName)); + + }).RequireAuthorization(); + + endpoints.MapGet("/logs/export/download/{filename}", async (string filename, IOptions config) => + { + var extension = Path.GetExtension(filename); + var mimetype = extension switch + { + ".gz" => "application/x-gzip", + ".csv" => "text/csv", + _ => null + }; + + if (mimetype == null) + return Results.NotFound(); + var basePath = Path.Combine(Directory.GetCurrentDirectory(), config.Value.ExportDirectory); + var path = Path.Combine(basePath, Path.GetFileName(filename)); + return !File.Exists(path) ? Results.NotFound() : Results.File(path, contentType: mimetype); + }).RequireAuthorization(); + } +} diff --git a/SS14.Admin/AdminLogs/Export/LogExportQueue.cs b/SS14.Admin/AdminLogs/Export/LogExportQueue.cs new file mode 100644 index 0000000..3af171a --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/LogExportQueue.cs @@ -0,0 +1,96 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Options; + +namespace SS14.Admin.AdminLogs.Export; + +public sealed class LogExportQueue +{ + private readonly IOptions _configuration; + private readonly Channel _queue; + + private readonly List> _reportChannels = []; + + public int MaxItemCount => _configuration.Value.ProcessQueueMaxSize; + public int Count => _queue.Reader.Count; + + public LogExportQueue(IOptions configuration) + { + _configuration = configuration; + + var channelConfiguration = new BoundedChannelOptions(MaxItemCount) + { + FullMode = BoundedChannelFullMode.DropWrite + }; + + _queue = Channel.CreateBounded(channelConfiguration); + } + + public async Task TryQueueProcessItem(ExportProcessItem item) + { + if (Count == MaxItemCount) + return false; + + await _queue.Writer.WriteAsync(item); + return true; + } + + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + return await _queue.Reader.ReadAsync(cancellationToken); + } + + public ReportChannel CreateReportChannel() + { + var channelOptions = new BoundedChannelOptions(1) + { + SingleReader = true, + SingleWriter = true, + AllowSynchronousContinuations = false, + FullMode = BoundedChannelFullMode.DropOldest, + }; + + var channel = Channel.CreateBounded(channelOptions); + _reportChannels.Add(channel); + + return new ReportChannel( + channel, + new WeakReference>>(_reportChannels) + ); + } + + public async Task ReportFinishedExport(string filename) + { + foreach (var channel in _reportChannels) + { + await channel.Writer.WriteAsync(filename); + } + } + + public sealed record ReportChannel : IDisposable + { + private readonly Channel _channel; + private readonly WeakReference>> _channels; + + public ReportChannel(Channel channel, WeakReference>> channels) + { + _channel = channel; + _channels = channels; + } + + + public async ValueTask Listen(CancellationToken ct) + { + return await _channel.Reader.ReadAsync(ct); + } + + public void Dispose() + { + _channel.Writer.TryComplete(); + + if (!_channels.TryGetTarget(out var channels)) + return; + + channels.Remove(_channel); + } + } +} diff --git a/SS14.Admin/AdminLogs/Export/LogExporter.cs b/SS14.Admin/AdminLogs/Export/LogExporter.cs new file mode 100644 index 0000000..ac7525d --- /dev/null +++ b/SS14.Admin/AdminLogs/Export/LogExporter.cs @@ -0,0 +1,104 @@ +using System.IO.Compression; +using Content.Server.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; + +namespace SS14.Admin.AdminLogs.Export; + +public sealed class LogExporter +{ + private const char ColumnSeparator = ','; + private const char Quote = '"'; + private const string EscapedQuote = "\"\""; + + private readonly PostgresServerDbContext _context; + private readonly IOptions _configuration; + + public LogExporter(PostgresServerDbContext context, IOptions configuration) + { + _context = context; + _configuration = configuration; + } + + public async Task Export(ExportProcessItem item, CancellationToken ct) + { + // Prevent accidentally exporting all logs + if ((!item.FromDate.HasValue || !item.ToDate.HasValue) && !item.RoundId.HasValue) + throw new Exception("Neither date or round id filter is set correctly for log export."); + + var filename = $"{DateTime.UtcNow.ToShortDateString()}-{Guid.NewGuid()}_log_export.csv{(item.UseCompression ? ".gz" : "")}"; + var path = Path.Combine(_configuration.Value.ExportDirectory, filename); + await using var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write); + + if (item.UseCompression) + { + await using var compressionStream = new GZipStream(fileStream, CompressionMode.Compress, leaveOpen: true); + await using var writer = new StreamWriter(compressionStream); + await WriteCsv(writer, item, ct); + } + else + { + await using var writer = new StreamWriter(fileStream); + await WriteCsv(writer, item, ct); + } + + return filename; + } + + private async Task WriteCsv(StreamWriter writer, ExportProcessItem item, CancellationToken ct) + { + var query = _context.AdminLog + .AsNoTracking() + .AsQueryable(); + + if (item.RoundId.HasValue) + query = query.Where(e => e.RoundId == item.RoundId); + + if (item is { FromDate: not null, ToDate: not null }) + query = query.Where(e => item.FromDate.Value.Date.ToUniversalTime() <= e.Date + && e.Date <= item.ToDate.Value.Date.ToUniversalTime()); + + if (item.Search != null) + query = query.Where(e => EF.Functions.ToTsVector("english", e.Message).Matches(item.Search)); + + await WriteCsvHeader(writer); + + await foreach (var log in query.AsAsyncEnumerable().WithCancellation(ct)) + { + await writer.WriteAsync(log.Date.ToString("O")); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(log.Id.ToString()); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(log.RoundId.ToString()); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(log.Impact.ToString()); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(log.Type.ToString()); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(Quote); + await writer.WriteAsync(log.Message.Replace(Quote.ToString(), EscapedQuote)); + await writer.WriteAsync(Quote); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync(Quote); + await writer.WriteAsync(log.Json.RootElement.GetRawText().Replace(Quote.ToString(), EscapedQuote)); + await writer.WriteAsync(Quote); + } + } + + private async Task WriteCsvHeader(StreamWriter writer) + { + await writer.WriteAsync("timestamp"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("round_id"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("id"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("impact"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("type"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("message"); + await writer.WriteAsync(ColumnSeparator); + await writer.WriteAsync("json"); + } +} diff --git a/SS14.Admin/Pages/Logs/Export.cshtml b/SS14.Admin/Pages/Logs/Export.cshtml new file mode 100644 index 0000000..cd7cfb4 --- /dev/null +++ b/SS14.Admin/Pages/Logs/Export.cshtml @@ -0,0 +1,115 @@ +@page +@model SS14.Admin.Pages.Logs.Export + +@{ + ViewData["Title"] = "Admin Logs"; +} + +
+
+

Export admin logs as CSV. A maximum of @Export.MaxDays days can be exported at once.

+

Check use round id to use the round id instead of the date

+
+ + + + + @**@ +
+
+ + +
+
+ + +
+
+ @Model.ErrorMessage + + + + + +
+
+
+ +
+
    +@foreach (var filename in Model.ListExportedFiles()) +{ +
  • @filename
    • +} +@if (Model.Processing) +{ +
  • +
    + +
    + +
    • +} +
+
+ +@section scripts { + + @if (Model.Processing) + { + + } + +} + diff --git a/SS14.Admin/Pages/Logs/Export.cshtml.cs b/SS14.Admin/Pages/Logs/Export.cshtml.cs new file mode 100644 index 0000000..f09c0a8 --- /dev/null +++ b/SS14.Admin/Pages/Logs/Export.cshtml.cs @@ -0,0 +1,83 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using Microsoft.Extensions.Options; +using SS14.Admin.AdminLogs.Export; + +namespace SS14.Admin.Pages.Logs; + +public class Export : PageModel +{ + public const int MaxDays = 4; + public const string FromAfterToError = "From date needs to be before to date"; + public const string DateRangeToLargeError = "Selected date range larger than 7 days"; + public const string RoundIdMissingError = "Round id is required"; + + private readonly LogExportQueue _queue; + private readonly IOptions _config; + + [BindProperty, DisplayFormat(DataFormatString = "{0:yyyy-MM-ddTHH:mm}", ApplyFormatInEditMode = true)] + public DateTime FromDate { get; set; } = DateTime.Now.AddDays(-1); + + [BindProperty, DisplayFormat(DataFormatString = "{0:yyyy-MM-ddTHH:mm}", ApplyFormatInEditMode = true)] + public DateTime ToDate { get; set; } = DateTime.Now; + + [BindProperty] + public bool UseRoundId { get; set; } + + [BindProperty] + public int? RoundId { get; set; } + + [BindProperty] + public string? SearchText { get; set; } + + [BindProperty] + public bool UseCompression { get; set; } = true; + + public string? ErrorMessage { get; set; } + + public bool Processing { get; set; } + + public Export(LogExportQueue queue, IOptions config) + { + _queue = queue; + _config = config; + } + + public void OnGet() + { + + } + + public async Task OnPost(CancellationToken ct) + { + switch (UseRoundId) + { + case false when FromDate > ToDate: + ErrorMessage = FromAfterToError; + return; + case false when (ToDate - FromDate).TotalDays > MaxDays: + ErrorMessage = DateRangeToLargeError; + return; + case true when !RoundId.HasValue: + ErrorMessage = RoundIdMissingError; + break; + } + + DateTime? from = !UseRoundId ? FromDate : null; + DateTime? to = !UseRoundId ? ToDate : null; + var roundId = UseRoundId ? RoundId : null; + + var item = new ExportProcessItem(SearchText, from, to, roundId, UseCompression); + await _queue.TryQueueProcessItem(item); + Processing = true; + } + + public List ListExportedFiles() + { + var exportPath = Path.Combine(Directory.GetCurrentDirectory(), _config.Value.ExportDirectory); + var files = Directory.EnumerateFiles(exportPath, "*.csv*"); + + return files.Select(Path.GetFileName).ToList(); + } +} diff --git a/SS14.Admin/Pages/Shared/_Layout.cshtml b/SS14.Admin/Pages/Shared/_Layout.cshtml index 23bee72..6c54dfa 100644 --- a/SS14.Admin/Pages/Shared/_Layout.cshtml +++ b/SS14.Admin/Pages/Shared/_Layout.cshtml @@ -67,6 +67,9 @@
  • Logs
    • +
  • + Export Logs +
  • Characters
    • diff --git a/SS14.Admin/Startup.cs b/SS14.Admin/Startup.cs index 1b36179..2e00589 100644 --- a/SS14.Admin/Startup.cs +++ b/SS14.Admin/Startup.cs @@ -5,6 +5,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Protocols.OpenIdConnect; using Serilog; +using SS14.Admin.AdminLogs.Export; using SS14.Admin.Helpers; using SS14.Admin.SignIn; @@ -26,8 +27,12 @@ public void ConfigureServices(IServiceCollection services) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddSingleton(); services.AddHttpContextAccessor(); + services.AddHostedService(); + var connStr = Configuration.GetConnectionString("DefaultConnection"); if (connStr == null) throw new InvalidOperationException("Need to specify DefaultConnection connection string"); @@ -124,6 +129,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { endpoints.MapRazorPages(); endpoints.MapControllers(); + endpoints.MapLogExportEndpoints(); }); } }