feat: baseline test

Author: samrose

nix/packages/cis-audit/scanner/internal/config/defaults.go

@@ -31,6 +31,29 @@ var DefaultExclusions = Config{
 		"/boot/config-*",
 		"/boot/initrd.img-*",
 		"/boot/vmlinuz-*",
+
+		// Development headers (not security-relevant)
+		"/usr/include/*",
+
+		// Python cache files (regenerated, not security-relevant)
+		"*/__pycache__/*",
+		"*.pyc",
+
+		// User cache directories (dynamic, user-specific)
+		"*/.cache/*",
+
+		// Nix build logs and var (deployment artifacts)
+		"/nix/var/*",
+
+		// Dynamic linker cache (regenerated)
+		"/etc/ld.so.cache",
+
+		// Shell history (dynamic, user-specific)
+		"*/.bash_history",
+		"*/.zsh_history",
+
+		// Ansible cache (deployment artifacts)
+		"*/.ansible/*",
 	},
 
 	ShallowDirs: []string{
@@ -39,6 +62,13 @@ var DefaultExclusions = Config{
 
 		// PostgreSQL data directory - contents are dynamic database state
 		"/data/pgdata",
+
+		// Deployment/provisioning tools - internal implementation details
+		"/opt/saltstack",
+
+		// Locally installed software - deep internals not security-relevant
+		"/usr/local/share",
+		"/usr/local/lib",
 	},
 
 	KernelParams: []string{

nix/packages/cis-audit/scanner/internal/config/loader.go

@@ -181,9 +181,45 @@ func removeItems(slice []string, itemsToRemove []string) []string {
 }
 
 // IsPathExcluded checks if a given path matches any exclusion pattern.
-// Supports glob patterns (*, ?, []).
+// Supports glob patterns (*, ?, []) and special patterns:
+// - /dir/* matches anything under /dir/
+// - */__pycache__/* matches __pycache__ directories anywhere
+// - *.pyc matches any .pyc file
+// - */.bash_history matches .bash_history file anywhere
 func (c *Config) IsPathExcluded(path string) bool {
 	for _, pattern := range c.Paths {
+		// Handle patterns that match anywhere in the path (starting with *)
+		if strings.HasPrefix(pattern, "*") {
+			// Pattern like *.pyc - match file extension
+			if strings.HasPrefix(pattern, "*.") {
+				suffix := strings.TrimPrefix(pattern, "*")
+				if strings.HasSuffix(path, suffix) {
+					return true
+				}
+				continue
+			}
+			// Pattern like */__pycache__/* or */.cache/* - match directory component anywhere
+			// Pattern like */.bash_history - match file anywhere
+			if strings.HasPrefix(pattern, "*/") {
+				// Get the part after */
+				rest := strings.TrimPrefix(pattern, "*/")
+
+				if strings.HasSuffix(pattern, "/*") {
+					// Directory pattern: */.cache/* should match /.cache/ anywhere
+					dirName := strings.TrimSuffix(rest, "/*")
+					if strings.Contains(path, "/"+dirName+"/") {
+						return true
+					}
+				} else {
+					// File pattern: */.bash_history should match /.bash_history at end
+					if strings.HasSuffix(path, "/"+rest) {
+						return true
+					}
+				}
+				continue
+			}
+		}
+
 		matched, err := filepath.Match(pattern, path)
 		if err != nil {
 			// Invalid pattern, skip

nix/packages/cis-audit/scanner/internal/config/loader_test.go

@@ -217,6 +217,84 @@ func TestLoad_ShallowDirs(t *testing.T) {
 	}
 }
 
+func TestIsPathExcluded_PycacheAndPyc(t *testing.T) {
+	cfg := &Config{
+		Paths: []string{
+			"*/__pycache__/*",
+			"*.pyc",
+		},
+	}
+
+	// Should be excluded
+	excluded := []string{
+		"/opt/saltstack/salt/lib/python3.10/__pycache__/locks.cpython-310.pyc",
+		"/usr/lib/python3/__pycache__/abc.cpython-310.pyc",
+		"/home/user/project/__pycache__/module.pyc",
+		"/some/path/file.pyc",
+		"/another/deeply/nested/file.pyc",
+	}
+
+	for _, path := range excluded {
+		if !cfg.IsPathExcluded(path) {
+			t.Errorf("Expected %s to be excluded", path)
+		}
+	}
+
+	// Should NOT be excluded
+	notExcluded := []string{
+		"/opt/saltstack/salt/lib/python3.10/locks.py",
+		"/usr/lib/python3/abc.py",
+		"/etc/passwd",
+		"/home/user/project/module.py",
+	}
+
+	for _, path := range notExcluded {
+		if cfg.IsPathExcluded(path) {
+			t.Errorf("Expected %s to NOT be excluded", path)
+		}
+	}
+}
+
+func TestIsPathExcluded_CacheAndHistory(t *testing.T) {
+	cfg := &Config{
+		Paths: []string{
+			"*/.cache/*",
+			"*/.bash_history",
+			"*/.ansible/*",
+		},
+	}
+
+	// Should be excluded
+	excluded := []string{
+		"/home/ubuntu/.cache/nix/eval-cache-v6/something.sqlite",
+		"/home/wal-g/.cache/nix/fetcher-cache-v4.sqlite",
+		"/root/.cache/pip/something",
+		"/home/ubuntu/.bash_history",
+		"/root/.bash_history",
+		"/home/ubuntu/.ansible/galaxy_cache/api.json",
+	}
+
+	for _, path := range excluded {
+		if !cfg.IsPathExcluded(path) {
+			t.Errorf("Expected %s to be excluded", path)
+		}
+	}
+
+	// Should NOT be excluded
+	notExcluded := []string{
+		"/home/ubuntu/.bashrc",
+		"/home/ubuntu/.profile",
+		"/etc/passwd",
+		"/home/ubuntu/cache/not-dotcache",
+	}
+
+	for _, path := range notExcluded {
+		if cfg.IsPathExcluded(path) {
+			t.Errorf("Expected %s to NOT be excluded", path)
+		}
+	}
+}
+
 // Helper function to check if a slice contains a string
 func contains(slice []string, item string) bool {
 	for _, s := range slice {

testinfra/test_ami_nix.py

@@ -10,6 +10,7 @@
 from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
 from time import sleep
 import paramiko
+from pathlib import Path
 
 # if EXECUTION_ID is not set, use a default value that includes the user and hostname
 RUN_ID = os.environ.get(
@@ -219,6 +220,16 @@ def run_ssh_command(ssh, command, timeout=None):
     }
 
 
+def upload_file_via_sftp(ssh, local_path, remote_path):
+    """Upload a file to the remote host via SFTP."""
+    sftp = ssh.open_sftp()
+    try:
+        sftp.put(local_path, remote_path)
+        logger.info(f"Uploaded {local_path} to {remote_path}")
+    finally:
+        sftp.close()
+
+
 # scope='session' uses the same container for all the tests;
 # scope='function' uses a new container per test function.
 @pytest.fixture(scope="session")
@@ -813,3 +824,79 @@ def test_postgrest_read_only_session_attrs(host):
                 print("Warning: Failed to restart PostgreSQL after restoring config")
         else:
             print("Warning: Failed to restore PostgreSQL configuration")
+
+
+def test_cis_baseline_audit(host):
+    """Run CIS baseline audit against the machine and report results.
+
+    This test uploads the current baseline.yml from the repo and uses
+    cis-audit to validate the machine against it. The test reports findings
+    but does not fail the build - it's for visibility into configuration drift.
+    """
+    git_sha = os.environ.get("GITHUB_SHA", "HEAD")
+
+    # Find the baseline file relative to the test file location
+    test_dir = Path(__file__).parent.parent
+    baseline_path = test_dir / "audit-specs" / "baselines" / "baseline.yml"
+
+    if not baseline_path.exists():
+        print(f"\n⚠️  Baseline file not found at {baseline_path}")
+        print("Skipping CIS baseline audit - no baseline file available")
+        pytest.skip("Baseline file not found")
+        return
+
+    print(f"\n{'='*60}")
+    print("CIS BASELINE AUDIT")
+    print(f"{'='*60}")
+    print(f"Baseline file: {baseline_path}")
+
+    # Upload baseline file to the instance
+    remote_baseline_path = "/tmp/baseline.yml"
+    try:
+        upload_file_via_sftp(host["ssh"], str(baseline_path), remote_baseline_path)
+        print(f"✓ Uploaded baseline to {remote_baseline_path}")
+    except Exception as e:
+        print(f"✗ Failed to upload baseline file: {e}")
+        pytest.skip(f"Failed to upload baseline: {e}")
+        return
+
+    # Install cis-audit via nix
+    print("\nInstalling cis-audit tool...")
+    install_cmd = f"nix profile install github:supabase/postgres/{git_sha}#cis-audit --refresh 2>&1"
+    result = run_ssh_command(host["ssh"], install_cmd, timeout=300)
+    if not result["succeeded"]:
+        print(f"Warning: {result['stderr'][:500]}")
+
+    # Run cis-audit with documentation format for readable output
+    print("\nRunning CIS baseline validation...")
+    print(f"{'-'*60}")
+
+    # Use the uploaded baseline file (local path, not bundled)
+    validate_cmd = f"~/.nix-profile/bin/cis-audit --spec {remote_baseline_path} --format documentation 2>&1"
+    result = run_ssh_command(host["ssh"], validate_cmd, timeout=600)
+
+    # Print full output for visibility in GitHub Actions logs
+    print(result["stdout"])
+    if result["stderr"]:
+        print(f"\nStderr:\n{result['stderr']}")
+
+    print(f"{'-'*60}")
+
+    # Also run with tap format to get summary counts
+    validate_tap_cmd = f"~/.nix-profile/bin/cis-audit --spec {remote_baseline_path} --format tap 2>&1 | tail -10"
+    result_tap = run_ssh_command(host["ssh"], validate_tap_cmd, timeout=600)
+
+    print(f"\nSummary:")
+    print(result_tap["stdout"])
+
+    # Clean up
+    run_ssh_command(host["ssh"], f"rm -f {remote_baseline_path}")
+
+    print(f"{'='*60}")
+    print("CIS BASELINE AUDIT COMPLETE")
+    print(f"{'='*60}\n")
+
+    # Note: This test intentionally does not assert/fail on validation results
+    # It's meant to provide visibility into configuration state
+    # To make this test fail on drift, uncomment the following:
+    # assert result["succeeded"], "CIS baseline validation found differences"