From 3c4df59a543e8603dd86ff55dbd18af2f9a918df Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Tue, 10 Mar 2026 10:31:10 +0100 Subject: [PATCH 1/8] Added audit log retention period configuration --- provider-integration/im2/pkg/config/00_module.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/provider-integration/im2/pkg/config/00_module.go b/provider-integration/im2/pkg/config/00_module.go index 329e9cf1fc..e5837b813b 100644 --- a/provider-integration/im2/pkg/config/00_module.go +++ b/provider-integration/im2/pkg/config/00_module.go @@ -291,6 +291,10 @@ type ProviderConfiguration struct { ProviderBranding ProviderBranding `yaml:"providerBranding"` ProviderBrandingImageAbsolutePath map[string]string + AuditLog struct { + retentionPeriodInDays int `yaml:"retentionPeriodInDays"` + } + Hosts struct { UCloud HostInfo Self HostInfo @@ -395,6 +399,13 @@ func parseProvider(filePath string, provider *yaml.Node) (bool, ProviderConfigur if providerBranding != nil { populateProviderBranding(&cfg, filePath, providerBranding) } + + // Audit log section + auditLog, _ := cfgutil.GetChildOrNil(filePath, provider, "auditLog") + if auditLog != nil { + cfgutil.Decode(filePath, auditLog, &cfg.AuditLog, &success) + } + // Hosts section hosts := cfgutil.RequireChild(filePath, provider, "hosts", &success) ucloudHost := cfgutil.RequireChild(filePath, hosts, "ucloud", &success) From b9bac7204124ee81fbf97d895565414b58e21e89 Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Tue, 10 Mar 2026 12:48:05 +0100 Subject: [PATCH 2/8] Added retention cleanup logic for audit logs --- .../im2/pkg/config/00_module.go | 8 ++- .../im2/pkg/controller/00_module.go | 1 + .../im2/pkg/controller/audit_log.go | 67 +++++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 provider-integration/im2/pkg/controller/audit_log.go diff --git a/provider-integration/im2/pkg/config/00_module.go b/provider-integration/im2/pkg/config/00_module.go index e5837b813b..a68d273e2d 100644 --- a/provider-integration/im2/pkg/config/00_module.go +++ b/provider-integration/im2/pkg/config/00_module.go @@ -285,15 +285,17 @@ type ProviderBranding struct { Sections []ProviderBrandingSection `yaml:"sections"` ProductDescription []ProviderBrandingProductDescription `yaml:"productDescription"` } +type AuditLog struct { + RetentionPeriodInDays int `yaml:"retentionPeriodInDays"` +} + type ProviderConfiguration struct { Id string ProviderBranding ProviderBranding `yaml:"providerBranding"` ProviderBrandingImageAbsolutePath map[string]string - AuditLog struct { - retentionPeriodInDays int `yaml:"retentionPeriodInDays"` - } + AuditLog AuditLog `yaml:"auditLog"` Hosts struct { UCloud HostInfo diff --git a/provider-integration/im2/pkg/controller/00_module.go b/provider-integration/im2/pkg/controller/00_module.go index a458d7da63..89d2f5c9f2 100644 --- a/provider-integration/im2/pkg/controller/00_module.go +++ b/provider-integration/im2/pkg/controller/00_module.go @@ -36,6 +36,7 @@ func Init(mux *http.ServeMux) { initTasks() initSshKeys() initProviderBranding() + initAuditLog() initLiveness() if RunsServerCode() { diff --git a/provider-integration/im2/pkg/controller/audit_log.go b/provider-integration/im2/pkg/controller/audit_log.go new file mode 100644 index 0000000000..f55071c766 --- /dev/null +++ b/provider-integration/im2/pkg/controller/audit_log.go @@ -0,0 +1,67 @@ +package controller + +import ( + "os" + "path/filepath" + "strings" + "time" + + "ucloud.dk/pkg/config" + "ucloud.dk/shared/pkg/log" +) + +var auditLogFolder = "/mnt/storage/audit" + +func initAuditLog() { + + retentionDays := config.Provider.AuditLog.RetentionPeriodInDays + go func() { + cleanupLogs(retentionDays) // run once at startup + + ticker := time.NewTicker(24 * time.Hour) + defer ticker.Stop() + + for range ticker.C { + cleanupLogs(retentionDays) + } + }() +} + +func cleanupLogs(retentionDays int) { + files, err := os.ReadDir(auditLogFolder) + if err != nil { + log.Error("Could not read audit log folder: %s", err) + return + } + + cutoff := time.Now().AddDate(0, 0, -retentionDays) + today := time.Now().Format("2006-01-02") + + for _, file := range files { + name := file.Name() + + if !strings.HasPrefix(name, "audit-") || !strings.HasSuffix(name, ".log") { + continue + } + + dateStr := strings.TrimSuffix(strings.TrimPrefix(name, "audit-"), ".log") + + if dateStr == today { + continue + } + + fileDate, err := time.Parse("2006-01-02", dateStr) + if err != nil { + continue + } + if fileDate.Before(cutoff) { + fullPath := filepath.Join(auditLogFolder, name) + err := os.Remove(fullPath) + if err != nil { + log.Error("Could not remove file: %s", err) + } else { + log.Info("Removed audit log file: %s, since it's older than %d days", fullPath, retentionDays) + } + } + } +} From 0e40509ec53a26aed0803ef93643055b14201ab6 Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Wed, 11 Mar 2026 08:38:56 +0100 Subject: [PATCH 3/8] renamed to job audit log for differentiation --- provider-integration/im2/pkg/config/00_module.go | 12 ++++++------ provider-integration/im2/pkg/controller/00_module.go | 2 +- .../controller/{audit_log.go => job_audit_log.go} | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) rename provider-integration/im2/pkg/controller/{audit_log.go => job_audit_log.go} (82%) diff --git a/provider-integration/im2/pkg/config/00_module.go b/provider-integration/im2/pkg/config/00_module.go index a68d273e2d..e67981d0eb 100644 --- a/provider-integration/im2/pkg/config/00_module.go +++ b/provider-integration/im2/pkg/config/00_module.go @@ -285,7 +285,7 @@ type ProviderBranding struct { Sections []ProviderBrandingSection `yaml:"sections"` ProductDescription []ProviderBrandingProductDescription `yaml:"productDescription"` } -type AuditLog struct { +type JobAuditLog struct { RetentionPeriodInDays int `yaml:"retentionPeriodInDays"` } @@ -295,7 +295,7 @@ type ProviderConfiguration struct { ProviderBranding ProviderBranding `yaml:"providerBranding"` ProviderBrandingImageAbsolutePath map[string]string - AuditLog AuditLog `yaml:"auditLog"` + JobAuditLog JobAuditLog `yaml:"jobAuditLog"` Hosts struct { UCloud HostInfo @@ -402,10 +402,10 @@ func parseProvider(filePath string, provider *yaml.Node) (bool, ProviderConfigur populateProviderBranding(&cfg, filePath, providerBranding) } - // Audit log section - auditLog, _ := cfgutil.GetChildOrNil(filePath, provider, "auditLog") - if auditLog != nil { - cfgutil.Decode(filePath, auditLog, &cfg.AuditLog, &success) + // Job audit log section + jobAuditLog, _ := cfgutil.GetChildOrNil(filePath, provider, "jobAuditLog") + if jobAuditLog != nil { + cfgutil.Decode(filePath, jobAuditLog, &cfg.JobAuditLog, &success) } // Hosts section diff --git a/provider-integration/im2/pkg/controller/00_module.go b/provider-integration/im2/pkg/controller/00_module.go index 89d2f5c9f2..5f2a22c050 100644 --- a/provider-integration/im2/pkg/controller/00_module.go +++ b/provider-integration/im2/pkg/controller/00_module.go @@ -36,7 +36,7 @@ func Init(mux *http.ServeMux) { initTasks() initSshKeys() initProviderBranding() - initAuditLog() + initJobAuditLog() initLiveness() if RunsServerCode() { diff --git a/provider-integration/im2/pkg/controller/audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go similarity index 82% rename from provider-integration/im2/pkg/controller/audit_log.go rename to provider-integration/im2/pkg/controller/job_audit_log.go index f55071c766..e8da2bfaff 100644 --- a/provider-integration/im2/pkg/controller/audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -10,11 +10,11 @@ import ( "ucloud.dk/shared/pkg/log" ) -var auditLogFolder = "/mnt/storage/audit" +var jobAuditLogFolder = "/mnt/storage/audit" -func initAuditLog() { +func initJobAuditLog() { - retentionDays := config.Provider.AuditLog.RetentionPeriodInDays + retentionDays := config.Provider.JobAuditLog.RetentionPeriodInDays go func() { cleanupLogs(retentionDays) // run once at startup @@ -28,7 +28,7 @@ func initAuditLog() { } func cleanupLogs(retentionDays int) { - files, err := os.ReadDir(auditLogFolder) + files, err := os.ReadDir(jobAuditLogFolder) if err != nil { log.Error("Could not read audit log folder: %s", err) return @@ -55,7 +55,7 @@ func cleanupLogs(retentionDays int) { continue } if fileDate.Before(cutoff) { - fullPath := filepath.Join(auditLogFolder, name) + fullPath := filepath.Join(jobAuditLogFolder, name) err := os.Remove(fullPath) if err != nil { log.Error("Could not remove file: %s", err) From 7acc0b4a39baedf5a36f3c07acb8be111df3d664 Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Wed, 11 Mar 2026 09:51:50 +0100 Subject: [PATCH 4/8] Changed to use regex for job audit --- .../im2/pkg/controller/job_audit_log.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/provider-integration/im2/pkg/controller/job_audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go index e8da2bfaff..3887d30bba 100644 --- a/provider-integration/im2/pkg/controller/job_audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -3,7 +3,7 @@ package controller import ( "os" "path/filepath" - "strings" + "regexp" "time" "ucloud.dk/pkg/config" @@ -11,6 +11,7 @@ import ( ) var jobAuditLogFolder = "/mnt/storage/audit" +var jobAuditFileRegex = regexp.MustCompile(`^audit-(\d+)-(\d{4}-\d{2}-\d{2})\.jsonl$`) func initJobAuditLog() { @@ -40,11 +41,14 @@ func cleanupLogs(retentionDays int) { for _, file := range files { name := file.Name() - if !strings.HasPrefix(name, "audit-") || !strings.HasSuffix(name, ".log") { + matches := jobAuditFileRegex.FindStringSubmatch(name) + if matches == nil { continue } - dateStr := strings.TrimSuffix(strings.TrimPrefix(name, "audit-"), ".log") + // matches[1] = rank (string) + // matches[2] = date + dateStr := matches[2] if dateStr == today { continue @@ -54,6 +58,7 @@ func cleanupLogs(retentionDays int) { if err != nil { continue } + if fileDate.Before(cutoff) { fullPath := filepath.Join(jobAuditLogFolder, name) err := os.Remove(fullPath) From 982eb50380ce239863335e09d74625f0b254c463 Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Thu, 12 Mar 2026 13:36:09 +0100 Subject: [PATCH 5/8] Updated cleanupLogs function to also consider child directories. --- .../im2/pkg/controller/job_audit_log.go | 72 ++++++++++++------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/provider-integration/im2/pkg/controller/job_audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go index 3887d30bba..ed27ab3cfb 100644 --- a/provider-integration/im2/pkg/controller/job_audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -29,44 +29,64 @@ func initJobAuditLog() { } func cleanupLogs(retentionDays int) { - files, err := os.ReadDir(jobAuditLogFolder) - if err != nil { - log.Error("Could not read audit log folder: %s", err) - return - } - cutoff := time.Now().AddDate(0, 0, -retentionDays) today := time.Now().Format("2006-01-02") - for _, file := range files { - name := file.Name() - - matches := jobAuditFileRegex.FindStringSubmatch(name) - if matches == nil { - continue + walkErr := filepath.WalkDir(jobAuditLogFolder, func(path string, d os.DirEntry, err error) error { + if err != nil { + log.Error("Walk error: %s", err) + return nil } + // Only process files + if !d.IsDir() { + name := d.Name() + matches := jobAuditFileRegex.FindStringSubmatch(name) + if matches == nil { + return nil + } + + dateStr := matches[2] + + if dateStr == today { + return nil + } - // matches[1] = rank (string) - // matches[2] = date - dateStr := matches[2] + fileDate, err := time.Parse("2006-01-02", dateStr) + if err != nil { + return nil + } - if dateStr == today { - continue + if fileDate.Before(cutoff) { + err := os.Remove(path) + if err != nil { + log.Error("Could not remove file: %s", err) + } else { + log.Info("Removed audit log file: %s, since it is older than %d days", path, retentionDays) + } + } + + return nil + } + // After walking a directory, check if it's empty (skip root) + if path == jobAuditLogFolder { + return nil } - fileDate, err := time.Parse("2006-01-02", dateStr) + entries, err := os.ReadDir(path) if err != nil { - continue + return nil } - if fileDate.Before(cutoff) { - fullPath := filepath.Join(jobAuditLogFolder, name) - err := os.Remove(fullPath) - if err != nil { - log.Error("Could not remove file: %s", err) - } else { - log.Info("Removed audit log file: %s, since it's older than %d days", fullPath, retentionDays) + // Delete the child folder if it is empty + if len(entries) == 0 { + err := os.Remove(path) + if err == nil { + log.Info("Removed empty audit log folder: %s", path) } } + return nil + }) + if walkErr != nil { + log.Error("Error cleaning up job audit logs: %s", walkErr) } } From 8cb585ba85ee0e0d238f6bb15ebcbd06cff20fbc Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Fri, 13 Mar 2026 13:53:58 +0100 Subject: [PATCH 6/8] Shorten the frequency of log to 4 hours instead --- provider-integration/im2/pkg/controller/job_audit_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider-integration/im2/pkg/controller/job_audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go index ed27ab3cfb..5c50f5a74f 100644 --- a/provider-integration/im2/pkg/controller/job_audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -19,7 +19,7 @@ func initJobAuditLog() { go func() { cleanupLogs(retentionDays) // run once at startup - ticker := time.NewTicker(24 * time.Hour) + ticker := time.NewTicker(4 * time.Hour) defer ticker.Stop() for range ticker.C { From e415f9f6f6b89d67cef186a28aaf49bcc074162d Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Mon, 16 Mar 2026 10:34:54 +0100 Subject: [PATCH 7/8] Getting mount point from Kubernetes config --- .../im2/pkg/controller/job_audit_log.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/provider-integration/im2/pkg/controller/job_audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go index 5c50f5a74f..b1fcb811f9 100644 --- a/provider-integration/im2/pkg/controller/job_audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -10,25 +10,29 @@ import ( "ucloud.dk/shared/pkg/log" ) -var jobAuditLogFolder = "/mnt/storage/audit" var jobAuditFileRegex = regexp.MustCompile(`^audit-(\d+)-(\d{4}-\d{2}-\d{2})\.jsonl$`) func initJobAuditLog() { - + cfgK8s := config.Services.Kubernetes() + if cfgK8s == nil { + log.Error("Job audit log server failed to initialize Kubernetes config") + return + } + jobAuditLogFolder := filepath.Join(cfgK8s.FileSystem.MountPoint, "audit") retentionDays := config.Provider.JobAuditLog.RetentionPeriodInDays go func() { - cleanupLogs(retentionDays) // run once at startup + cleanupLogs(jobAuditLogFolder, retentionDays) // run once at startup ticker := time.NewTicker(4 * time.Hour) defer ticker.Stop() for range ticker.C { - cleanupLogs(retentionDays) + cleanupLogs(jobAuditLogFolder, retentionDays) } }() } -func cleanupLogs(retentionDays int) { +func cleanupLogs(jobAuditLogFolder string, retentionDays int) { cutoff := time.Now().AddDate(0, 0, -retentionDays) today := time.Now().Format("2006-01-02") From af615a2140185c0bb6117b44c2d9b6c5a9e0b293 Mon Sep 17 00:00:00 2001 From: Dan Vu Date: Mon, 16 Mar 2026 10:36:44 +0100 Subject: [PATCH 8/8] Renamed variable --- provider-integration/im2/pkg/controller/job_audit_log.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/provider-integration/im2/pkg/controller/job_audit_log.go b/provider-integration/im2/pkg/controller/job_audit_log.go index b1fcb811f9..1041469998 100644 --- a/provider-integration/im2/pkg/controller/job_audit_log.go +++ b/provider-integration/im2/pkg/controller/job_audit_log.go @@ -13,12 +13,12 @@ import ( var jobAuditFileRegex = regexp.MustCompile(`^audit-(\d+)-(\d{4}-\d{2}-\d{2})\.jsonl$`) func initJobAuditLog() { - cfgK8s := config.Services.Kubernetes() - if cfgK8s == nil { - log.Error("Job audit log server failed to initialize Kubernetes config") + k8sCfg := config.Services.Kubernetes() + if k8sCfg == nil { + log.Error("Job audit log server failed to get Kubernetes config") return } - jobAuditLogFolder := filepath.Join(cfgK8s.FileSystem.MountPoint, "audit") + jobAuditLogFolder := filepath.Join(k8sCfg.FileSystem.MountPoint, "audit") retentionDays := config.Provider.JobAuditLog.RetentionPeriodInDays go func() { cleanupLogs(jobAuditLogFolder, retentionDays) // run once at startup