diff --git a/age/keysource.go b/age/keysource.go index 4eb9a200ed..711814d563 100644 --- a/age/keysource.go +++ b/age/keysource.go @@ -17,6 +17,7 @@ import ( "filippo.io/age/armor" "filippo.io/age/plugin" "github.com/sirupsen/logrus" + "github.com/getsops/sops/v3/fsio" "golang.org/x/crypto/ssh" "github.com/getsops/sops/v3/logging" @@ -423,7 +424,7 @@ func (key *MasterKey) loadIdentities() (ParsedIdentities, []string, errSet) { } if ageKeyFile, ok := os.LookupEnv(SopsAgeKeyFileEnv); ok { - f, err := os.Open(ageKeyFile) + f, err := fsio.Open(ageKeyFile) if err != nil { errs = append(errs, fmt.Errorf("failed to open %s file: %w", SopsAgeKeyFileEnv, err)) } else { @@ -456,7 +457,7 @@ func (key *MasterKey) loadIdentities() (ParsedIdentities, []string, errSet) { errs = append(errs, fmt.Errorf("user config directory could not be determined: %w", err)) } else if userConfigDir != "" { ageKeyFilePath := filepath.Join(userConfigDir, filepath.FromSlash(SopsAgeKeyUserConfigPath)) - f, err := os.Open(ageKeyFilePath) + f, err := fsio.Open(ageKeyFilePath) if err != nil && !errors.Is(err, os.ErrNotExist) { errs = append(errs, fmt.Errorf("failed to open file: %w", err)) } else if errors.Is(err, os.ErrNotExist) && len(readers) == 0 && len(identities) == 0 { diff --git a/age/ssh_parse.go b/age/ssh_parse.go index 404b88263f..06cd3473b0 100644 --- a/age/ssh_parse.go +++ b/age/ssh_parse.go @@ -12,11 +12,10 @@ package age import ( "fmt" - "io" - "os" "filippo.io/age" "filippo.io/age/agessh" + "github.com/getsops/sops/v3/fsio" "golang.org/x/crypto/ssh" ) @@ -26,15 +25,10 @@ import ( // error is returned. func readPublicKeyFile(privateKeyPath string) (ssh.PublicKey, error) { publicKeyPath := privateKeyPath + ".pub" - f, err := os.Open(publicKeyPath) + contents, err := fsio.Read(publicKeyPath) if err != nil { return nil, fmt.Errorf("failed to obtain public %q key for %q SSH key: %w", publicKeyPath, privateKeyPath, err) } - defer f.Close() - contents, err := io.ReadAll(f) - if err != nil { - return nil, fmt.Errorf("failed to read %q: %w", publicKeyPath, err) - } pubKey, _, _, _, err := ssh.ParseAuthorizedKey(contents) if err != nil { return nil, fmt.Errorf("failed to parse %q: %w", publicKeyPath, err) @@ -46,12 +40,7 @@ func readPublicKeyFile(privateKeyPath string) (ssh.PublicKey, error) { // private key file. If the private key file is encrypted, it will configure // the identity to prompt for a passphrase. func parseSSHIdentityFromPrivateKeyFile(keyPath string) (age.Identity, error) { - keyFile, err := os.Open(keyPath) - if err != nil { - return nil, fmt.Errorf("failed to open file: %w", err) - } - defer keyFile.Close() - contents, err := io.ReadAll(keyFile) + contents, err := fsio.Read(keyPath) if err != nil { return nil, fmt.Errorf("failed to read file: %w", err) } diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 330c3bc8eb..cb87388e7c 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -33,6 +33,7 @@ import ( publishcmd "github.com/getsops/sops/v3/cmd/sops/subcommand/publish" "github.com/getsops/sops/v3/cmd/sops/subcommand/updatekeys" "github.com/getsops/sops/v3/config" + "github.com/getsops/sops/v3/fsio" "github.com/getsops/sops/v3/gcpkms" "github.com/getsops/sops/v3/hckms" "github.com/getsops/sops/v3/hcvault" @@ -75,6 +76,8 @@ func warnMoreThanOnePositionalArgument(c *cli.Context) { } func main() { + defer fsio.ClearCache() + cli.VersionPrinter = version.PrintVersion app := cli.NewApp() diff --git a/fsio/fsio.go b/fsio/fsio.go new file mode 100644 index 0000000000..4766b26bb5 --- /dev/null +++ b/fsio/fsio.go @@ -0,0 +1,88 @@ +package fsio + +import ( + "bytes" + "io" + "os" + "path/filepath" + "sync" +) + +type cacheEntry struct { + mu sync.RWMutex + data []byte +} + +var fileStreamCache sync.Map + +// ClearCache wipes the cached stream secrets from memory by overwriting +// the byte slices with zeros before deleting them from the map. +// +// If you are using SOPS as a library, you should call ClearCache after +// completing decryption/encryption operations to ensure no sensitive key +// data remains in memory. +func ClearCache() { + fileStreamCache.Range(func(key, value any) bool { + if entry, ok := value.(*cacheEntry); ok { + entry.mu.Lock() + for i := range entry.data { + entry.data[i] = 0 + } + entry.mu.Unlock() + } + fileStreamCache.Delete(key) + return true + }) +} + +// Read reads a file from the given path. If it is a stream (e.g., /dev/fd/* or /proc/*) +// it caches the content in memory to avoid issues with multiple reads from the same stream. +func Read(path string) ([]byte, error) { + if absPath, err := filepath.Abs(path); err == nil { + path = absPath + } + fileInfo, err := os.Stat(path) + isStream := err == nil && + (fileInfo.Mode()&os.ModeNamedPipe != 0 || fileInfo.Mode()&os.ModeCharDevice != 0 || fileInfo.Mode()&os.ModeSocket != 0) + + if isStream { + if value, ok := fileStreamCache.Load(path); ok { + if entry, ok := value.(*cacheEntry); ok { + entry.mu.RLock() + defer entry.mu.RUnlock() + b := make([]byte, len(entry.data)) + copy(b, entry.data) + return b, nil + } + } + } + + b, err := os.ReadFile(path) + if err == nil && isStream { + cachedBytes := make([]byte, len(b)) + copy(cachedBytes, b) + fileStreamCache.Store(path, &cacheEntry{data: cachedBytes}) + } + return b, err +} + +// Open opens a file from the given path. If it is a stream, it loads the content +// into the cache and returns a reader over the cached bytes. +func Open(path string) (io.ReadCloser, error) { + if absPath, err := filepath.Abs(path); err == nil { + path = absPath + } + fileInfo, err := os.Stat(path) + isStream := err == nil && + (fileInfo.Mode()&os.ModeNamedPipe != 0 || fileInfo.Mode()&os.ModeCharDevice != 0 || fileInfo.Mode()&os.ModeSocket != 0) + + if isStream { + b, err := Read(path) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(b)), nil + } + + return os.Open(path) +} diff --git a/fsio/fsio_test.go b/fsio/fsio_test.go new file mode 100644 index 0000000000..907a1862ae --- /dev/null +++ b/fsio/fsio_test.go @@ -0,0 +1,31 @@ +package fsio + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestReadRegularFile(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "test.txt") + content := []byte("regular file content") + + err := os.WriteFile(filePath, content, 0o600) + if err != nil { + t.Fatalf("failed to write temp file: %v", err) + } + + b1, err := Read(filePath) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if !bytes.Equal(b1, content) { + t.Errorf("expected %q, got %q", content, b1) + } + + if _, ok := fileStreamCache.Load(filePath); ok { + t.Error("expected regular file NOT to be cached, but it was") + } +} diff --git a/fsio/fsio_unix_test.go b/fsio/fsio_unix_test.go new file mode 100644 index 0000000000..e6600769c1 --- /dev/null +++ b/fsio/fsio_unix_test.go @@ -0,0 +1,291 @@ +//go:build !windows + +package fsio + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "syscall" + "testing" +) + +func TestReadStreamFile(t *testing.T) { + tempDir := t.TempDir() + fifoPath := filepath.Join(tempDir, "test_fifo") + + err := syscall.Mkfifo(fifoPath, 0o600) + if err != nil { + t.Fatalf("failed to create named pipe: %v", err) + } + + content := []byte("super secret key stream") + + go func() { + f, err := os.OpenFile(fifoPath, os.O_WRONLY, 0) + if err != nil { + return + } + defer f.Close() + _, _ = f.Write(content) + }() + + b1, err := Read(fifoPath) + if err != nil { + t.Fatalf("first Read failed: %v", err) + } + if !bytes.Equal(b1, content) { + t.Errorf("expected %q, got %q", content, b1) + } + + if _, ok := fileStreamCache.Load(fifoPath); !ok { + t.Error("expected stream file to be cached, but it was not") + } + + b2, err := Read(fifoPath) + if err != nil { + t.Fatalf("second Read failed: %v", err) + } + if !bytes.Equal(b2, content) { + t.Errorf("expected cached %q, got %q", content, b2) + } + + r, err := Open(fifoPath) + if err != nil { + t.Fatalf("Open failed: %v", err) + } + defer r.Close() + b3, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + if !bytes.Equal(b3, content) { + t.Errorf("expected read cached %q, got %q", content, b3) + } + + // Get the cache entry before clearing to verify it gets zeroed + entryVal, ok := fileStreamCache.Load(fifoPath) + if !ok { + t.Fatal("expected entry to be in cache") + } + entry := entryVal.(*cacheEntry) + + // Clear cache and check that bytes are zeroed + ClearCache() + + // The cached slice should be zeroed + entry.mu.RLock() + for i, b := range entry.data { + if b != 0 { + t.Errorf("expected cached byte at index %d to be zeroed, got %d", i, b) + } + } + entry.mu.RUnlock() + + if _, ok := fileStreamCache.Load(fifoPath); ok { + t.Error("expected stream file cache to be cleared, but it was still present") + } +} + +func TestConcurrentReadAndClear(t *testing.T) { + tempDir := t.TempDir() + fifoPath := filepath.Join(tempDir, "concurrent_fifo") + + err := syscall.Mkfifo(fifoPath, 0o600) + if err != nil { + t.Fatalf("failed to create named pipe: %v", err) + } + + stop := make(chan struct{}) + go func() { + for { + select { + case <-stop: + return + default: + } + f, err := os.OpenFile(fifoPath, os.O_WRONLY, 0) + if err != nil { + return + } + _, _ = f.Write([]byte("secret_value")) + f.Close() + } + }() + + _, err = Read(fifoPath) + if err != nil { + t.Fatalf("initial read failed: %v", err) + } + + done := make(chan bool) + go func() { + for range 100 { + ClearCache() + _, _ = Read(fifoPath) + } + done <- true + }() + + go func() { + for range 100 { + b, err := Read(fifoPath) + if err == nil { + _ = len(b) + if len(b) > 0 { + _ = b[0] + } + } + } + done <- true + }() + + <-done + <-done + + close(stop) + dummy, err := os.OpenFile(fifoPath, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err == nil { + dummy.Close() + } +} + +func TestAnonymousPipe(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + defer r.Close() + + path := filepath.Join("/proc/self/fd", fmt.Sprintf("%d", r.Fd())) + + content := []byte("anonymous pipe secret key") + + go func() { + defer w.Close() + _, _ = w.Write(content) + }() + + rCloser1, err := Open(path) + if err != nil { + t.Fatalf("first Open failed: %v", err) + } + b1, err := io.ReadAll(rCloser1) + rCloser1.Close() + if err != nil { + t.Fatalf("first ReadAll failed: %v", err) + } + if !bytes.Equal(b1, content) { + t.Errorf("expected %q, got %q", content, b1) + } + + if _, ok := fileStreamCache.Load(path); !ok { + t.Error("expected anonymous pipe to be cached, but it was not") + } + + rCloser2, err := Open(path) + if err != nil { + t.Fatalf("second Open failed: %v", err) + } + b2, err := io.ReadAll(rCloser2) + rCloser2.Close() + if err != nil { + t.Fatalf("second ReadAll failed: %v", err) + } + if !bytes.Equal(b2, content) { + t.Errorf("expected cached %q, got %q", content, b2) + } + + ClearCache() +} + +func TestConcurrentReads(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + defer r.Close() + + path := filepath.Join("/proc/self/fd", fmt.Sprintf("%d", r.Fd())) + content := []byte("concurrent shared secret key") + + go func() { + defer w.Close() + _, _ = w.Write(content) + }() + + _, err = Read(path) + if err != nil { + t.Fatalf("initial read failed: %v", err) + } + + const numGoroutines = 50 + var wg sync.WaitGroup + wg.Add(numGoroutines) + + for range numGoroutines { + go func() { + defer wg.Done() + b, err := Read(path) + if err != nil { + t.Errorf("Read failed: %v", err) + return + } + if !bytes.Equal(b, content) { + t.Errorf("expected cached %q, got %q", content, b) + } + }() + } + + wg.Wait() + ClearCache() +} + +func TestReadCanonicalization(t *testing.T) { + tempDir := t.TempDir() + fifoPath := filepath.Join(tempDir, "canonical_fifo") + + err := syscall.Mkfifo(fifoPath, 0o600) + if err != nil { + t.Fatalf("failed to create named pipe: %v", err) + } + + content := []byte("canonicalized stream content") + + go func() { + f, err := os.OpenFile(fifoPath, os.O_WRONLY, 0) + if err != nil { + return + } + defer f.Close() + _, _ = f.Write(content) + }() + + absPath := filepath.Join(tempDir, "canonical_fifo") + altPath := filepath.Join(tempDir, "..", filepath.Base(tempDir), "canonical_fifo") + + b1, err := Read(altPath) + if err != nil { + t.Fatalf("first Read with altPath failed: %v", err) + } + if !bytes.Equal(b1, content) { + t.Errorf("expected %q, got %q", content, b1) + } + + b2, err := Read(absPath) + if err != nil { + t.Fatalf("second Read with absPath failed: %v", err) + } + if !bytes.Equal(b2, content) { + t.Errorf("expected cached %q, got %q", content, b2) + } + + if _, ok := fileStreamCache.Load(absPath); !ok { + t.Errorf("expected stream to be cached under absolute path %q, but it was not", absPath) + } + + ClearCache() +} diff --git a/gcpkms/keysource.go b/gcpkms/keysource.go index 2acbdd9c0c..090f53de91 100644 --- a/gcpkms/keysource.go +++ b/gcpkms/keysource.go @@ -12,6 +12,7 @@ import ( kms "cloud.google.com/go/kms/apiv1" "cloud.google.com/go/kms/apiv1/kmspb" "github.com/sirupsen/logrus" + "github.com/getsops/sops/v3/fsio" "golang.org/x/oauth2" "google.golang.org/api/option" "google.golang.org/grpc" @@ -358,7 +359,7 @@ func (key *MasterKey) newKMSClient(ctx context.Context) (*kms.KeyManagementClien func getGoogleCredentials() ([]byte, error) { if defaultCredentials, ok := os.LookupEnv(SopsGoogleCredentialsEnv); ok && len(defaultCredentials) > 0 { if _, err := os.Stat(defaultCredentials); err == nil { - return os.ReadFile(defaultCredentials) + return fsio.Read(defaultCredentials) } return []byte(defaultCredentials), nil } diff --git a/hcvault/keysource.go b/hcvault/keysource.go index c60e11cfcb..092a46a790 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -1,12 +1,10 @@ package hcvault import ( - "bytes" "context" "encoding/base64" "errors" "fmt" - "io" "net/http" "net/url" "os" @@ -17,6 +15,7 @@ import ( "time" "github.com/hashicorp/vault/api" + "github.com/getsops/sops/v3/fsio" "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" @@ -441,20 +440,14 @@ func userVaultToken() (string, error) { } tokenPath := filepath.Join(homePath, defaultTokenFile) - f, err := os.Open(tokenPath) + b, err := fsio.Read(tokenPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return "", nil } return "", err } - defer f.Close() - - buf := bytes.NewBuffer(nil) - if _, err := io.Copy(buf, f); err != nil { - return "", err - } - return strings.TrimSpace(buf.String()), nil + return strings.TrimSpace(string(b)), nil } // engineAndKeyFromPath returns the engine path and key name from the full diff --git a/pgp/keysource.go b/pgp/keysource.go index 63e31027ce..7ce0b341bf 100644 --- a/pgp/keysource.go +++ b/pgp/keysource.go @@ -22,6 +22,7 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/getsops/sops/v3/fsio" gpgagent "github.com/getsops/gopgagent" "github.com/sirupsen/logrus" "golang.org/x/term" @@ -169,7 +170,7 @@ func (d GnuPGHome) ImportContext(ctx context.Context, armoredKey []byte) error { // It returns an error if the GnuPGHome does not pass Validate, or if the // import failed. func (d GnuPGHome) ImportFile(path string) error { - b, err := os.ReadFile(path) + b, err := fsio.Read(path) if err != nil { return fmt.Errorf("cannot read armored key data from file: %w", err) } @@ -594,7 +595,7 @@ func (key *MasterKey) passphrasePrompt() func(keys []openpgp.Key, symmetric bool // Unsupported keys are ignored as long as at least a single valid key is // found. func loadRing(path string) (openpgp.EntityList, error) { - f, err := os.Open(path) + f, err := fsio.Open(path) if err != nil { return nil, err }