NGINX Configs to Reference Remote External Files

internal/config/config.go

@@ -50,6 +50,10 @@ const (
 	regexLabelPattern = "^[a-zA-Z0-9]([a-zA-Z0-9-_]{0,254}[a-zA-Z0-9])?$"
 )
 
+var domainRegex = regexp.MustCompile(
+	`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`,
+)
+
 var viperInstance = viper.NewWithOptions(viper.KeyDelimiter(KeyDelimiter))
 
 func RegisterRunner(r func(cmd *cobra.Command, args []string)) {
@@ -157,6 +161,7 @@ func ResolveConfig() (*Config, error) {
 		Features:           viperInstance.GetStringSlice(FeaturesKey),
 		Labels:             resolveLabels(),
 		LibDir:             viperInstance.GetString(LibDirPathKey),
+		ExternalDataSource: resolveExternalDataSource(),
 		SyslogServer:       resolveSyslogServer(),
 	}
 
@@ -475,6 +480,7 @@ func registerFlags() {
 	registerCollectorFlags(fs)
 	registerClientFlags(fs)
 	registerDataPlaneFlags(fs)
+	registerExternalDataSourceFlags(fs)
 
 	fs.SetNormalizeFunc(normalizeFunc)
 
@@ -489,6 +495,24 @@ func registerFlags() {
 	})
 }
 
+func registerExternalDataSourceFlags(fs *flag.FlagSet) {
+	fs.String(
+		ExternalDataSourceProxyUrlKey,
+		DefExternalDataSourceProxyUrl,
+		"Url to the proxy service for fetching external files.",
+	)
+	fs.StringSlice(
+		ExternalDataSourceAllowDomainsKey,
+		[]string{},
+		"List of allowed domains for external data sources.",
+	)
+	fs.Int64(
+		ExternalDataSourceMaxBytesKey,
+		DefExternalDataSourceMaxBytes,
+		"Maximum size in bytes for external data sources.",
+	)
+}
+
 func registerDataPlaneFlags(fs *flag.FlagSet) {
 	fs.Duration(
 		NginxReloadMonitoringPeriodKey,
@@ -628,6 +652,11 @@ func registerClientFlags(fs *flag.FlagSet) {
 		DefMaxFileSize,
 		"Max file size in bytes.",
 	)
+	fs.Duration(
+		ClientFileDownloadTimeoutKey,
+		DefClientFileDownloadTimeout,
+		"Timeout value in seconds, for downloading a file during a config apply.",
+	)
 
 	fs.Int(
 		ClientGRPCMaxParallelFileOperationsKey,
@@ -1120,6 +1149,7 @@ func resolveClient() *Client {
 			RandomizationFactor: viperInstance.GetFloat64(ClientBackoffRandomizationFactorKey),
 			Multiplier:          viperInstance.GetFloat64(ClientBackoffMultiplierKey),
 		},
+		FileDownloadTimeout: viperInstance.GetDuration(ClientFileDownloadTimeoutKey),
 	}
 }
 
@@ -1560,3 +1590,36 @@ func areCommandServerProxyTLSSettingsSet() bool {
 		viperInstance.IsSet(CommandServerProxyTLSSkipVerifyKey) ||
 		viperInstance.IsSet(CommandServerProxyTLSServerNameKey)
 }
+
+func resolveExternalDataSource() *ExternalDataSource {
+	proxyURLStruct := ProxyURL{
+		URL: viperInstance.GetString(ExternalDataSourceProxyUrlKey),
+	}
+	externalDataSource := &ExternalDataSource{
+		ProxyURL:       proxyURLStruct,
+		AllowedDomains: viperInstance.GetStringSlice(ExternalDataSourceAllowDomainsKey),
+		MaxBytes:       viperInstance.GetInt64(ExternalDataSourceMaxBytesKey),
+	}
+
+	if err := validateAllowedDomains(externalDataSource.AllowedDomains); err != nil {
+		return nil
+	}
+
+	return externalDataSource
+}
+
+func validateAllowedDomains(domains []string) error {
+	if len(domains) == 0 {
+		return nil
+	}
+
+	for _, domain := range domains {
+		// Validating syntax using the RFC-compliant regex
+		if !domainRegex.MatchString(domain) || domain == "" {
+			slog.Error("domain specified in allowed_domains is invalid", "domain", domain)
+			return errors.New("invalid domain found in allowed_domains")
+		}
+	}
+
+	return nil
+}

internal/config/config_test.go

@@ -1425,6 +1425,13 @@ func createConfig() *Config {
 			config.FeatureCertificates, config.FeatureFileWatcher, config.FeatureMetrics,
 			config.FeatureAPIAction, config.FeatureLogsNap,
 		},
+		ExternalDataSource: &ExternalDataSource{
+			ProxyURL: ProxyURL{
+				URL: "http://proxy.example.com",
+			},
+			AllowedDomains: []string{"example.com", "api.example.com"},
+			MaxBytes:       1048576,
+		},
 	}
 }
 
@@ -1569,3 +1576,73 @@ func TestValidateLabel(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateAllowedDomains(t *testing.T) {
+	tests := []struct {
+		name    string
+		domains []string
+		wantErr bool
+	}{
+		{
+			name:    "Test 1: Success: Empty slice",
+			domains: []string{},
+			wantErr: false,
+		},
+		{
+			name:    "Test 2: Success: Nil slice",
+			domains: nil,
+			wantErr: false,
+		},
+		{
+			name:    "Test 3: Success: Valid domains",
+			domains: []string{"example.com", "api.nginx.com", "sub.domain.io"},
+			wantErr: false,
+		},
+		{
+			name:    "Test 4: Failure: Domain contains space",
+			domains: []string{"valid.com", "bad domain.com"},
+			wantErr: true,
+		},
+		{
+			name:    "Test 5: Failure: Empty string domain",
+			domains: []string{"valid.com", ""},
+			wantErr: true,
+		},
+		{
+			name:    "Test 6: Failure: Domain contains forward slash /",
+			domains: []string{"domain.com/path"},
+			wantErr: true,
+		},
+		{
+			name:    "Test 7: Failure: Domain contains backward slash \\",
+			domains: []string{"domain.com\\path"},
+			wantErr: true,
+		},
+		{
+			name:    "Test 8: Failure: Mixed valid and invalid (first is invalid)",
+			domains: []string{" only.com", "good.com"},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var logBuffer bytes.Buffer
+			logHandler := slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelError})
+
+			originalLogger := slog.Default()
+			slog.SetDefault(slog.New(logHandler))
+			defer slog.SetDefault(originalLogger)
+
+			actualErr := validateAllowedDomains(tt.domains)
+
+			if tt.wantErr {
+				require.Error(t, actualErr, "Expected an error but got nil.")
+				assert.Contains(t, logBuffer.String(), "domain specified in allowed_domains is invalid",
+					"Expected the error log message to be present in the output.")
+			} else {
+				assert.NoError(t, actualErr, "Did not expect an error but got one: %v", actualErr)
+			}
+		})
+	}
+}

internal/config/defaults.go

@@ -82,6 +82,8 @@ const (
 	DefBackoffMaxInterval         = 20 * time.Second
 	DefBackoffMaxElapsedTime      = 1 * time.Minute
 
+	DefClientFileDownloadTimeout = 60 * time.Second
+
 	// Watcher defaults
 	DefInstanceWatcherMonitoringFrequency       = 5 * time.Second
 	DefInstanceHealthWatcherMonitoringFrequency = 5 * time.Second
@@ -114,6 +116,9 @@ const (
 
 	// File defaults
 	DefLibDir = "/var/lib/nginx-agent"
+
+	DefExternalDataSourceProxyUrl = ""
+	DefExternalDataSourceMaxBytes = 100 * 1024 * 1024 // default 100MB
 )
 
 func DefaultFeatures() []string {

internal/config/flags.go

@@ -25,6 +25,7 @@ const (
 	InstanceHealthWatcherMonitoringFrequencyKey = "watchers_instance_health_watcher_monitoring_frequency"
 	FileWatcherKey                              = "watchers_file_watcher"
 	LibDirPathKey                               = "lib_dir"
+	ExternalDataSourceRootKey                   = "external_data_source"
 )
 
 var (
@@ -47,6 +48,7 @@ var (
 	ClientBackoffMaxElapsedTimeKey      = pre(ClientRootKey) + "backoff_max_elapsed_time"
 	ClientBackoffRandomizationFactorKey = pre(ClientRootKey) + "backoff_randomization_factor"
 	ClientBackoffMultiplierKey          = pre(ClientRootKey) + "backoff_multiplier"
+	ClientFileDownloadTimeoutKey        = pre(ClientRootKey) + "file_download_timeout"
 
 	CollectorConfigPathKey                      = pre(CollectorRootKey) + "config_path"
 	CollectorAdditionalConfigPathsKey           = pre(CollectorRootKey) + "additional_config_paths"
@@ -141,6 +143,11 @@ var (
 
 	FileWatcherMonitoringFrequencyKey = pre(FileWatcherKey) + "monitoring_frequency"
 	NginxExcludeFilesKey              = pre(FileWatcherKey) + "exclude_files"
+
+	ExternalDataSourceProxyKey        = pre(ExternalDataSourceRootKey) + "proxy"
+	ExternalDataSourceProxyUrlKey     = pre(ExternalDataSourceProxyKey) + "url"
+	ExternalDataSourceMaxBytesKey     = pre(ExternalDataSourceRootKey) + "max_bytes"
+	ExternalDataSourceAllowDomainsKey = pre(ExternalDataSourceRootKey) + "allowed_domains"
 )
 
 func pre(prefixes ...string) string {

internal/config/testdata/nginx-agent.conf

@@ -183,3 +183,11 @@ collector:
   log:      
     level: "INFO"
     path: "/var/log/nginx-agent/opentelemetry-collector-agent.log"
+
+external_data_source:
+  proxy:
+    url: "http://proxy.example.com"
+  allowed_domains:
+    - example.com
+    - api.example.com
+  max_bytes: 1048576