diff --git a/internal/etw/processors/chain_windows.go b/internal/etw/processors/chain_windows.go index d3651b0e7..77f38b293 100644 --- a/internal/etw/processors/chain_windows.go +++ b/internal/etw/processors/chain_windows.go @@ -57,7 +57,7 @@ func NewChain( chain.addProcessor(newRegistryProcessor(hsnap)) } if config.EventSource.EnableImageEvents { - chain.addProcessor(newImageProcessor(psnap)) + chain.addProcessor(newModuleProcessor(psnap)) } if config.EventSource.EnableNetEvents { chain.addProcessor(newNetProcessor()) diff --git a/internal/etw/processors/fs_windows.go b/internal/etw/processors/fs_windows.go index 4159afc49..304d22af1 100644 --- a/internal/etw/processors/fs_windows.go +++ b/internal/etw/processors/fs_windows.go @@ -44,8 +44,6 @@ var ( fileObjectMisses = expvar.NewInt("fs.file.objects.misses") fileObjectHandleHits = expvar.NewInt("fs.file.object.handle.hits") fileReleaseCount = expvar.NewInt("fs.file.releases") - - fsFileCharacteristicsRateLimits = expvar.NewInt("fs.file.characteristics.rate.limits") ) type fsProcessor struct { @@ -243,23 +241,6 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) { } } - // parse PE data for created files and append parameters - if ev.IsCreateDisposition() && ev.IsSuccess() { - if !f.lim.Allow() { - fsFileCharacteristicsRateLimits.Add(1) - return ev, nil - } - path := ev.GetParamAsString(params.FilePath) - c, err := parseImageFileCharacteristics(path) - if err != nil { - return ev, nil - } - ev.AppendParam(params.FileIsDLL, params.Bool, c.isDLL) - ev.AppendParam(params.FileIsDriver, params.Bool, c.isDriver) - ev.AppendParam(params.FileIsExecutable, params.Bool, c.isExe) - ev.AppendParam(params.FileIsDotnet, params.Bool, c.isDotnet) - } - return ev, nil case event.ReleaseFile: fileReleaseCount.Add(1) diff --git a/internal/etw/processors/fs_windows_test.go b/internal/etw/processors/fs_windows_test.go index 6b754c7b4..d8de0d9bb 100644 --- a/internal/etw/processors/fs_windows_test.go +++ b/internal/etw/processors/fs_windows_test.go @@ -19,6 +19,10 @@ package processors import ( + "os" + "reflect" + "testing" + "github.com/rabbitstack/fibratus/pkg/config" "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" @@ -31,9 +35,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "os" - "reflect" - "testing" ) func TestFsProcessor(t *testing.T) { @@ -166,10 +167,6 @@ func TestFsProcessor(t *testing.T) { assert.Equal(t, "Success", e.GetParamAsString(params.NTStatus)) assert.Equal(t, "File", e.GetParamAsString(params.FileType)) assert.Equal(t, "CREATE", e.GetParamAsString(params.FileOperation)) - assert.True(t, e.Params.MustGetBool(params.FileIsExecutable)) - assert.False(t, e.Params.MustGetBool(params.FileIsDotnet)) - assert.False(t, e.Params.MustGetBool(params.FileIsDLL)) - assert.False(t, e.Params.MustGetBool(params.FileIsDriver)) }, }, { @@ -195,33 +192,6 @@ func TestFsProcessor(t *testing.T) { assert.Empty(t, fsProcessor.files) }, }, - { - "parse created file characteristics", - &event.Event{ - Type: event.CreateFile, - Category: event.File, - Params: event.Params{ - params.FileObject: {Name: params.FileObject, Type: params.Uint64, Value: uint64(18446738026482168384)}, - params.ThreadID: {Name: params.ThreadID, Type: params.Uint32, Value: uint32(1484)}, - params.FileCreateOptions: {Name: params.FileCreateOptions, Type: params.Uint32, Value: uint32(1223456)}, - params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: exe}, - params.FileShareMask: {Name: params.FileShareMask, Type: params.Uint32, Value: uint32(5)}, - params.FileIrpPtr: {Name: params.FileIrpPtr, Type: params.Uint64, Value: uint64(1234543123112321)}, - params.FileOperation: {Name: params.FileOperation, Type: params.Uint64, Value: uint64(2)}, - }, - }, - nil, - func() *handle.SnapshotterMock { - hsnap := new(handle.SnapshotterMock) - return hsnap - }, - func(e *event.Event, t *testing.T, hsnap *handle.SnapshotterMock, p Processor) { - fsProcessor := p.(*fsProcessor) - assert.True(t, e.WaitEnqueue) - assert.Contains(t, fsProcessor.irps, uint64(1234543123112321)) - assert.True(t, reflect.DeepEqual(e, fsProcessor.irps[1234543123112321])) - }, - }, { "unmap view file", &event.Event{ diff --git a/internal/etw/processors/image_windows.go b/internal/etw/processors/image_windows.go deleted file mode 100644 index 9ab7d2b2f..000000000 --- a/internal/etw/processors/image_windows.go +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2019-2020 by Nedim Sabic Sabic - * https://www.fibratus.io - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package processors - -import ( - "expvar" - "github.com/rabbitstack/fibratus/pkg/event" - "github.com/rabbitstack/fibratus/pkg/event/params" - "github.com/rabbitstack/fibratus/pkg/ps" - "sync" - "time" -) - -var imageFileCharacteristicsCacheHits = expvar.NewInt("image.file.characteristics.cache.hits") - -var modTTL = time.Minute * 10 - -type imageProcessor struct { - psnap ps.Snapshotter - mods map[string]*imageFileCharacteristics - mu sync.Mutex - purger *time.Ticker - quit chan struct{} -} - -func newImageProcessor(psnap ps.Snapshotter) Processor { - m := &imageProcessor{ - psnap: psnap, - mods: make(map[string]*imageFileCharacteristics), - purger: time.NewTicker(time.Minute), - quit: make(chan struct{}, 1), - } - - go m.purge() - - return m -} - -func (*imageProcessor) Name() ProcessorType { return Image } - -func (m *imageProcessor) ProcessEvent(e *event.Event) (*event.Event, bool, error) { - if e.IsLoadImageInternal() { - // state management - return e, false, m.psnap.AddModule(e) - } - - if e.IsLoadImage() { - // is image characteristics data cached? - path := e.GetParamAsString(params.ImagePath) - key := path + e.GetParamAsString(params.ImageCheckSum) - - m.mu.Lock() - defer m.mu.Unlock() - c, ok := m.mods[key] - if !ok { - // parse PE image data - var err error - c, err = parseImageFileCharacteristics(path) - if err != nil { - return e, false, m.psnap.AddModule(e) - } - c.keepalive() - m.mods[key] = c - } else { - imageFileCharacteristicsCacheHits.Add(1) - c.keepalive() - } - - // append event parameters - e.AppendParam(params.FileIsDLL, params.Bool, c.isDLL) - e.AppendParam(params.FileIsDriver, params.Bool, c.isDriver) - e.AppendParam(params.FileIsExecutable, params.Bool, c.isExe) - e.AppendParam(params.FileIsDotnet, params.Bool, c.isDotnet) - } - - if e.IsUnloadImage() { - pid := e.Params.MustGetPid() - addr := e.Params.TryGetAddress(params.ImageBase) - if pid == 0 { - pid = e.PID - } - return e, false, m.psnap.RemoveModule(pid, addr) - } - - if e.IsLoadImage() || e.IsImageRundown() { - return e, false, m.psnap.AddModule(e) - } - return e, true, nil -} - -func (m *imageProcessor) Close() { - m.quit <- struct{}{} -} - -func (m *imageProcessor) purge() { - for { - select { - case <-m.purger.C: - m.mu.Lock() - for key, mod := range m.mods { - if time.Since(mod.accessed) > modTTL { - delete(m.mods, key) - } - } - m.mu.Unlock() - case <-m.quit: - return - } - } -} diff --git a/internal/etw/processors/module_windows.go b/internal/etw/processors/module_windows.go new file mode 100644 index 000000000..a3ce31c02 --- /dev/null +++ b/internal/etw/processors/module_windows.go @@ -0,0 +1,61 @@ +/* + * Copyright 2019-2020 by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processors + +import ( + "github.com/rabbitstack/fibratus/pkg/event" + "github.com/rabbitstack/fibratus/pkg/event/params" + "github.com/rabbitstack/fibratus/pkg/ps" +) + +type moduleProcessor struct { + psnap ps.Snapshotter +} + +func newModuleProcessor(psnap ps.Snapshotter) Processor { + m := &moduleProcessor{psnap: psnap} + + return m +} + +func (*moduleProcessor) Name() ProcessorType { return Image } + +func (m *moduleProcessor) ProcessEvent(e *event.Event) (*event.Event, bool, error) { + if e.IsLoadImageInternal() { + // state management + return e, false, m.psnap.AddModule(e) + } + + if e.IsUnloadImage() { + pid := e.Params.MustGetPid() + addr := e.Params.TryGetAddress(params.ImageBase) + if pid == 0 { + pid = e.PID + } + return e, false, m.psnap.RemoveModule(pid, addr) + } + + if e.IsLoadImage() || e.IsImageRundown() { + return e, false, m.psnap.AddModule(e) + } + + return e, true, nil +} + +func (m *moduleProcessor) Close() {} diff --git a/internal/etw/processors/image_windows_test.go b/internal/etw/processors/module_windows_test.go similarity index 92% rename from internal/etw/processors/image_windows_test.go rename to internal/etw/processors/module_windows_test.go index 0e9cb8ba4..3379c89e3 100644 --- a/internal/etw/processors/image_windows_test.go +++ b/internal/etw/processors/module_windows_test.go @@ -19,6 +19,10 @@ package processors import ( + "os" + "path/filepath" + "testing" + "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" "github.com/rabbitstack/fibratus/pkg/ps" @@ -27,12 +31,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "os" - "path/filepath" - "testing" ) -func TestImageProcessor(t *testing.T) { +func TestModuleProcessor(t *testing.T) { var tests = []struct { name string e *event.Event @@ -84,11 +85,6 @@ func TestImageProcessor(t *testing.T) { }, func(e *event.Event, t *testing.T, psnap *ps.SnapshotterMock) { psnap.AssertNumberOfCalls(t, "AddModule", 1) - // should be enriched with image characteristics params - assert.True(t, e.Params.MustGetBool(params.FileIsDLL)) - assert.True(t, e.Params.MustGetBool(params.FileIsDotnet)) - assert.False(t, e.Params.MustGetBool(params.FileIsExecutable)) - assert.False(t, e.Params.MustGetBool(params.FileIsDriver)) }, }, { @@ -119,7 +115,7 @@ func TestImageProcessor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { psnap := tt.psnap() - p := newImageProcessor(psnap) + p := newModuleProcessor(psnap) var err error tt.e, _, err = p.ProcessEvent(tt.e) require.NoError(t, err) diff --git a/internal/etw/processors/processor.go b/internal/etw/processors/processor.go index dcb929223..ea3209987 100644 --- a/internal/etw/processors/processor.go +++ b/internal/etw/processors/processor.go @@ -20,10 +20,6 @@ package processors import ( "github.com/rabbitstack/fibratus/pkg/event" - libntfs "github.com/rabbitstack/fibratus/pkg/fs/ntfs" - "github.com/rabbitstack/fibratus/pkg/pe" - "os" - "time" ) // ProcessorType is an alias for the event processor type @@ -82,63 +78,3 @@ func (typ ProcessorType) String() string { return "unknown" } } - -type imageFileCharacteristics struct { - isExe bool - isDLL bool - isDriver bool - isDotnet bool - accessed time.Time -} - -func (c *imageFileCharacteristics) keepalive() { - c.accessed = time.Now() -} - -// parseImageFileCharacteristics parses the PE structure for the file path -// residing in the given event parameters. The preferred method for reading -// the PE metadata is by directly accessing the file. -// If this operation fails, the file data is read form the raw device and -// the blob is passed to the PE parser. -// The given event is decorated with various parameters extracted from PE -// data. Most notably, parameters that indicate whether the file is a DLL, -// executable image, or a Windows driver. -func parseImageFileCharacteristics(path string) (*imageFileCharacteristics, error) { - var pefile *pe.PE - - f, err := os.Open(path) - if err != nil { - // read file data blob from raw device - // if the regular file access fails - ntfs := libntfs.NewFS() - data, n, err := ntfs.Read(path, 0, int64(os.Getpagesize())) - defer ntfs.Close() - if err != nil { - return nil, err - } - if n > 0 { - data = data[:n] - } - // parse PE file from byte slice - pefile, err = pe.ParseBytes(data, pe.WithSections(), pe.WithSymbols(), pe.WithCLR()) - if err != nil { - return nil, err - } - } else { - defer f.Close() - // parse PE file from on-disk file - pefile, err = pe.ParseFile(path, pe.WithSections(), pe.WithSymbols(), pe.WithCLR()) - if err != nil { - return nil, err - } - } - - c := &imageFileCharacteristics{ - isExe: pefile.IsExecutable, - isDLL: pefile.IsDLL, - isDriver: pefile.IsDriver, - isDotnet: pefile.IsDotnet, - } - - return c, nil -} diff --git a/pkg/filter/accessor_windows.go b/pkg/filter/accessor_windows.go index a8da074ed..8eee0e4e7 100644 --- a/pkg/filter/accessor_windows.go +++ b/pkg/filter/accessor_windows.go @@ -716,12 +716,11 @@ func (l *fileAccessor) Get(f Field, e *event.Event) (params.Value, error) { return isLOLDriver(f.Name, e) } return false, nil - case fields.FileIsDLL: - return e.Params.GetBool(params.FileIsDLL) - case fields.FileIsDriver: - return e.Params.GetBool(params.FileIsDriver) - case fields.FileIsExecutable: - return e.Params.GetBool(params.FileIsExecutable) + case fields.FileIsDLL, fields.FileIsDriver, fields.FileIsExecutable: + if e.IsCreateDisposition() && e.IsSuccess() { + return getFileInfo(f.Name, e) + } + return false, nil case fields.FilePID: return e.Params.GetPid() case fields.FileKey: @@ -873,14 +872,13 @@ func (*moduleAccessor) Get(f Field, e *event.Event) (params.Value, error) { return isLOLDriver(f.Name, e) } return false, nil - case fields.ImageIsDLL, fields.ModuleIsDLL: - return e.Params.GetBool(params.FileIsDLL) - case fields.ImageIsDriver, fields.ModuleIsDriver: - return e.Params.GetBool(params.FileIsDriver) - case fields.ImageIsExecutable, fields.ModuleIsExecutable: - return e.Params.GetBool(params.FileIsExecutable) - case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: - return e.Params.GetBool(params.FileIsDotnet) + case fields.ImageIsDLL, fields.ModuleIsDLL, fields.ImageIsDriver, + fields.ModuleIsDriver, fields.ImageIsExecutable, fields.ModuleIsExecutable, + fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: + if e.IsLoadImage() { + return getFileInfo(f.Name, e) + } + return false, nil } return nil, nil diff --git a/pkg/filter/filter_test.go b/pkg/filter/filter_test.go index 7e7111092..9acfaf9ae 100644 --- a/pkg/filter/filter_test.go +++ b/pkg/filter/filter_test.go @@ -1016,6 +1016,7 @@ func TestModuleFilter(t *testing.T) { {`module.signature.subject icontains 'Microsoft Corporation'`, true}, {`module.pe.is_dotnet`, false}, {`module.path.stem endswith 'System32\\kernel32'`, true}, + {`module.is_dll`, true}, {`dll.path.stem endswith 'System32\\kernel32'`, true}, {`dll.signature.type = 'EMBEDDED'`, true}, {`dll.signature.level = 'AUTHENTICODE'`, true}, diff --git a/pkg/filter/ql/functions/minidump.go b/pkg/filter/ql/functions/minidump.go index 9ef1b1720..1c9ae4fba 100644 --- a/pkg/filter/ql/functions/minidump.go +++ b/pkg/filter/ql/functions/minidump.go @@ -20,8 +20,9 @@ package functions import ( "encoding/binary" - "io" - "os" + "time" + + "github.com/rabbitstack/fibratus/pkg/sys" ) // The 4-byte magic number at the start of a minidump file @@ -36,18 +37,11 @@ func (f IsMinidump) Call(args []interface{}) (interface{}, bool) { } path := parseString(0, args) - file, err := os.Open(path) - if err != nil { - return false, true - } - defer file.Close() - - var header [4]byte - _, err = io.ReadFull(file, header[:]) - if err != nil { - return false, true + b, err := sys.ReadFile(path, 4, time.Second) + if err != nil || len(b) < 4 { + return nil, false } - isMinidumpSignature := binary.LittleEndian.Uint32(header[:]) == minidumpSignature + isMinidumpSignature := binary.LittleEndian.Uint32(b[:]) == minidumpSignature return isMinidumpSignature, true } diff --git a/pkg/filter/util.go b/pkg/filter/util.go index ccf4a05e7..18c479332 100644 --- a/pkg/filter/util.go +++ b/pkg/filter/util.go @@ -20,6 +20,7 @@ package filter import ( "encoding/hex" + "fmt" "net" "path/filepath" "strings" @@ -27,6 +28,7 @@ import ( "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" "github.com/rabbitstack/fibratus/pkg/filter/fields" + "github.com/rabbitstack/fibratus/pkg/fs" "github.com/rabbitstack/fibratus/pkg/util/bytes" "github.com/rabbitstack/fibratus/pkg/util/loldrivers" "github.com/rabbitstack/fibratus/pkg/util/signature" @@ -72,6 +74,53 @@ func initLOLDriversClient(flds []Field) { } } +// getFileInfo obtains the file information for created files and loaded modules. +// Appends the file data to the event parameters, so subsequent field extractions +// will already have the needed info. +func getFileInfo(f fields.Field, e *event.Event) (params.Value, error) { + switch f { + case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL: + if e.Params.Contains(params.FileIsDLL) { + return e.Params.GetBool(params.FileIsDLL) + } + case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver: + if e.Params.Contains(params.FileIsDriver) { + return e.Params.GetBool(params.FileIsDriver) + } + case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable: + if e.Params.Contains(params.FileIsExecutable) { + return e.Params.GetBool(params.FileIsExecutable) + } + case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: + if e.Params.Contains(params.FileIsDotnet) { + return e.Params.GetBool(params.FileIsDotnet) + } + } + + fileinfo, err := fs.GetFileInfo(e.GetParamAsString(params.FilePath)) + if err != nil { + return nil, err + } + + e.AppendParam(params.FileIsDLL, params.Bool, fileinfo.IsDLL) + e.AppendParam(params.FileIsDriver, params.Bool, fileinfo.IsDriver) + e.AppendParam(params.FileIsExecutable, params.Bool, fileinfo.IsExecutable) + e.AppendParam(params.FileIsDotnet, params.Bool, fileinfo.IsDotnet) + + switch f { + case fields.FileIsDLL, fields.ImageIsDLL, fields.ModuleIsDLL: + return fileinfo.IsDLL, nil + case fields.FileIsDriver, fields.ModuleIsDriver, fields.ImageIsDriver: + return fileinfo.IsDriver, nil + case fields.FileIsExecutable, fields.ImageIsExecutable, fields.ModuleIsExecutable: + return fileinfo.IsExecutable, nil + case fields.ImageIsDotnet, fields.ModuleIsDotnet, fields.DllIsDotnet: + return fileinfo.IsDotnet, nil + } + + return nil, fmt.Errorf("unexpected field: %s", f) +} + // getSignature tries to find the module signature mapped to the given address. // If the signature is not found in the cache, then a fresh signature instance // is created and verified. diff --git a/pkg/filter/util_test.go b/pkg/filter/util_test.go new file mode 100644 index 000000000..eb4e61c7d --- /dev/null +++ b/pkg/filter/util_test.go @@ -0,0 +1,77 @@ +/* + * Copyright 2021-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter + +import ( + "os" + "path/filepath" + "testing" + + "github.com/rabbitstack/fibratus/pkg/event" + "github.com/rabbitstack/fibratus/pkg/event/params" + "github.com/rabbitstack/fibratus/pkg/filter/fields" + "github.com/stretchr/testify/require" +) + +func TestGetFileInfo(t *testing.T) { + path, err := os.Executable() + require.NoError(t, err) + + var tests = []struct { + e *event.Event + f func(*testing.T, *event.Event) + fld fields.Field + }{ + { + e: &event.Event{ + Name: "CreateFile", + Params: map[string]*event.Param{ + params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: path}, + }, + }, + f: func(t *testing.T, e *event.Event) { + require.True(t, e.Params.MustGetBool(params.FileIsExecutable)) + }, + fld: fields.FileIsExecutable, + }, + { + e: &event.Event{ + Name: "CreateFile", + Params: map[string]*event.Param{ + params.FilePath: {Name: params.FilePath, Type: params.UnicodeString, Value: filepath.Join(os.Getenv("SystemRoot"), "System32", "kernel32.dll")}, + }, + }, + f: func(t *testing.T, e *event.Event) { + require.True(t, e.Params.MustGetBool(params.FileIsDLL)) + }, + fld: fields.ModuleIsDLL, + }, + } + + for _, tt := range tests { + t.Run(tt.e.GetParamAsString(params.FilePath), func(t *testing.T) { + v, err := getFileInfo(tt.fld, tt.e) + require.NotNil(t, v) + require.NoError(t, err) + if tt.f != nil { + tt.f(t, tt.e) + } + }) + } +} diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 1c0acc405..003e6ad24 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -23,12 +23,16 @@ package fs import ( "expvar" - "github.com/rabbitstack/fibratus/pkg/sys" - "golang.org/x/sys/windows" + "fmt" "os" "path/filepath" "strings" "unsafe" + + "github.com/rabbitstack/fibratus/pkg/pe" + "github.com/rabbitstack/fibratus/pkg/sys" + "github.com/rabbitstack/fibratus/pkg/util/wildcard" + "golang.org/x/sys/windows" ) const ( @@ -48,6 +52,70 @@ const ( devConsole = 0x00000050 ) +// FileInfo represents file metadata. +type FileInfo struct { + IsExecutable bool + IsDLL bool + IsDriver bool + IsDotnet bool +} + +var skippedPatterns = []string{ + `?:\$WinREAgent\Scratch\*`, + `?:\WINDOWS\WinSxS\*`, + `?:\Windows\WinSxS\*`, + `?:\WINDOWS\CbsTemp\*`, + `?:\Windows\CbsTemp\*`, + `?:\WINDOWS\SoftwareDistribution\*`, + `?:\Windows\SoftwareDistribution\*`, +} + +// ErrSkippedFile signals the file processing is skipped. +var ErrSkippedFile = func(path string) error { return fmt.Errorf("skipped file: %s", path) } + +var parserOpts = []pe.Option{ + pe.WithSections(), + pe.WithSymbols(), + pe.WithCLR(), +} + +// GetFileInfo returns file metadata for the given path. +// The file metadata consists of information extracted +// from the Portable Executable headers. +func GetFileInfo(path string) (*FileInfo, error) { + for _, pat := range skippedPatterns { + if wildcard.Match(pat, path) { + return nil, ErrSkippedFile(path) + } + } + + ext := filepath.Ext(path) + switch strings.ToLower(ext) { + case ".exe": + return &FileInfo{IsExecutable: true}, nil + case ".sys": + return &FileInfo{IsDriver: true}, nil + case ".dll": + pefile, err := pe.ParseFile(path, parserOpts...) + if err != nil { + return &FileInfo{IsDLL: true}, nil + } + return &FileInfo{IsDLL: true, IsDotnet: pefile.IsDotnet}, nil + } + + pefile, err := pe.ParseFile(path, parserOpts...) + if err != nil { + return nil, err + } + + return &FileInfo{ + IsExecutable: pefile.IsExecutable, + IsDLL: pefile.IsDLL, + IsDriver: pefile.IsDriver, + IsDotnet: pefile.IsDotnet, + }, nil +} + // queryVolumeCalls represents the number of times the query volume function was called var queryVolumeCalls = expvar.NewInt("file.query.volume.info.calls") diff --git a/pkg/fs/file_test.go b/pkg/fs/file_test.go index 3e416e4e6..a3aa94ce5 100644 --- a/pkg/fs/file_test.go +++ b/pkg/fs/file_test.go @@ -22,8 +22,10 @@ package fs import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetFileType(t *testing.T) { @@ -55,3 +57,50 @@ func TestGetFileType(t *testing.T) { }) } } + +func TestGetFileInfo(t *testing.T) { + var tests = []struct { + path string + fileinfo *FileInfo + err error + }{ + { + `C:\System32\cmd.exe`, + &FileInfo{IsExecutable: true}, + nil, + }, + { + `C:\System32\kernel32.dll`, + &FileInfo{IsDLL: true}, + nil, + }, + { + `C:\Temp\afs.sys`, + &FileInfo{IsDriver: true}, + nil, + }, + { + `../pe/_fixtures/054299e09cea38df2b84e6b29348b418.bin`, + &FileInfo{IsDriver: true}, + nil, + }, + { + `C:\WINDOWS\SoftwareDistribution\Temp\combase.dll`, + nil, + ErrSkippedFile(`C:\WINDOWS\SoftwareDistribution\Temp\combase.dll`), + }, + { + `../pe/_fixtures/mscorlib.dll`, + &FileInfo{IsDLL: true, IsDotnet: true}, + nil, + }, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + fileinfo, err := GetFileInfo(tt.path) + require.Equal(t, tt.err, err) + assert.Equal(t, tt.fileinfo, fileinfo) + }) + } +} diff --git a/pkg/pe/parser.go b/pkg/pe/parser.go index 32b381bb8..b7a872986 100644 --- a/pkg/pe/parser.go +++ b/pkg/pe/parser.go @@ -24,6 +24,11 @@ import ( "errors" "expvar" "fmt" + "os" + "path/filepath" + "strings" + "time" + "github.com/rabbitstack/fibratus/pkg/sys" "github.com/rabbitstack/fibratus/pkg/util/format" "github.com/rabbitstack/fibratus/pkg/util/va" @@ -31,10 +36,6 @@ import ( peparserlog "github.com/saferwall/pe/log" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" - "os" - "path/filepath" - "strings" - "time" ) var ( @@ -49,6 +50,7 @@ var ( directoryParseErrors = expvar.NewInt("pe.directory.parse.errors") versionResourcesParseErrors = expvar.NewInt("pe.version.resources.parse.errors") imphashErrors = expvar.NewInt("pe.imphash.errors") + parserWarnings = expvar.NewMap("pe.parser.warnings") ) type opts struct { @@ -237,7 +239,7 @@ func (l Logger) Log(level peparserlog.Level, keyvals ...interface{}) error { case peparserlog.LevelInfo: log.Info(keyvals[1:]...) case peparserlog.LevelWarn: - log.Warn(keyvals[1:]...) + parserWarnings.Add(fmt.Sprintf("%s", keyvals[1:]), 1) case peparserlog.LevelError, peparserlog.LevelFatal: log.Error(keyvals[1:]...) default: diff --git a/pkg/sys/fs.go b/pkg/sys/fs.go index 5e1b8efe8..62bcc074e 100644 --- a/pkg/sys/fs.go +++ b/pkg/sys/fs.go @@ -19,6 +19,10 @@ package sys import ( + "fmt" + "syscall" + "time" + "golang.org/x/sys/windows" ) @@ -102,3 +106,81 @@ func GetMappedFile(process windows.Handle, addr uintptr) string { } return "" } + +// WaitTimeout indicates the time-out interval +// elapsed, and the object's state is nonsignaled. +const WaitTimeout = 0x00000102 + +// ReadFile reads the file asynchronously with the specified number +// of bytes to read and the timeout after which the I/O operation +// is cancelled. +func ReadFile(path string, size int, timeout time.Duration) ([]byte, error) { + pathUTF16, err := syscall.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + + // open file asynchronously + handle, err := windows.CreateFile( + pathUTF16, + windows.GENERIC_READ, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + nil, + windows.OPEN_EXISTING, + windows.FILE_FLAG_OVERLAPPED, + 0, + ) + if err != nil { + return nil, err + } + //nolint:errcheck + defer windows.CloseHandle(handle) + + event, err := windows.CreateEvent(nil, 1, 0, nil) + if err != nil { + return nil, err + } + //nolint:errcheck + defer windows.CloseHandle(event) + + var overlapped windows.Overlapped + overlapped.HEvent = event + + buf := make([]byte, size) // chunk size + var n uint32 + + err = windows.ReadFile(handle, buf, &n, &overlapped) + if err != nil && err != windows.ERROR_IO_PENDING { + return nil, err + } + // synchronous completion + if err == nil && int(n) <= size { + return buf[:n], nil + } + + // wait for I/O operation to complete + wait, err := windows.WaitForSingleObject(event, uint32(timeout.Milliseconds())) + switch wait { + case windows.WAIT_FAILED: + return nil, err + case WaitTimeout: + // cancel the I/O + _ = windows.CancelIoEx(handle, &overlapped) + + // must wait until cancellation completes + _, _ = windows.WaitForSingleObject(event, 1000) + err = windows.GetOverlappedResult(handle, &overlapped, &n, false) + if err == windows.ERROR_OPERATION_ABORTED { + return nil, fmt.Errorf("timeout reading file %s", path) + } + return nil, err + } + + // get the result + err = windows.GetOverlappedResult(handle, &overlapped, &n, false) + if err != nil { + return nil, err + } + + return buf[:n], nil +} diff --git a/pkg/sys/fs_test.go b/pkg/sys/fs_test.go new file mode 100644 index 000000000..c6375c6de --- /dev/null +++ b/pkg/sys/fs_test.go @@ -0,0 +1,61 @@ +//go:build windows + +/* + * Copyright 2021-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sys + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestReadFile(t *testing.T) { + var tests = []struct { + name string + f func() (string, error) + err error + b []byte + }{ + { + "read file", + func() (string, error) { + path := filepath.Join(t.TempDir(), "test.txt") + + return path, os.WriteFile(path, []byte("hello world"), 0644) + }, + nil, + []byte{104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := tt.f() + require.NoError(t, err) + defer os.Remove(path) + b, err := ReadFile(path, 4096, time.Second*1) + require.Equal(t, tt.err, err) + require.Equal(t, tt.b, b) + }) + } +} diff --git a/pkg/yara/scanner.go b/pkg/yara/scanner.go index 513ea64ce..746a27c3a 100644 --- a/pkg/yara/scanner.go +++ b/pkg/yara/scanner.go @@ -25,18 +25,19 @@ import ( "encoding/json" "expvar" "fmt" + "os" + "path/filepath" + "strings" + "time" + "github.com/google/uuid" "github.com/rabbitstack/fibratus/pkg/alertsender" "github.com/rabbitstack/fibratus/pkg/event/params" - libntfs "github.com/rabbitstack/fibratus/pkg/fs/ntfs" "github.com/rabbitstack/fibratus/pkg/sys" "github.com/rabbitstack/fibratus/pkg/util/signature" "github.com/rabbitstack/fibratus/pkg/util/va" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" - "os" - "path/filepath" - "strings" "github.com/hillu/go-yara/v4" "github.com/rabbitstack/fibratus/pkg/event" @@ -257,9 +258,7 @@ func (s scanner) Scan(e *event.Event) (bool, error) { return false, nil } // read ADS data - ntfs := libntfs.NewFS() - data, n, err := ntfs.Read(filename, 0, 1024*1024) - defer ntfs.Close() + data, err := sys.ReadFile(filename, 1024*1024, time.Second) if err != nil { return false, nil } diff --git a/rules/credential_access_lsass_memory_dumping.yml b/rules/credential_access_lsass_memory_dumping.yml index 61310df03..f3f0153f4 100644 --- a/rules/credential_access_lsass_memory_dumping.yml +++ b/rules/credential_access_lsass_memory_dumping.yml @@ -1,6 +1,6 @@ name: LSASS memory dumping via legitimate or offensive tools id: 335795af-246b-483e-8657-09a30c102e63 -version: 1.2.0 +version: 1.2.1 description: | Detects an attempt to dump the LSAAS memory to the disk by employing legitimate tools such as procdump, Task Manager, Process Explorer or built-in Windows tools @@ -32,7 +32,16 @@ condition: > '?:\\ProgramData\\Microsoft\\Windows Defender\\*\\MsMpEng.exe' ) | - |create_file and (file.extension iin ('.dmp', '.mdmp', '.dump') or is_minidump(file.path))| + |create_new_file and + file.path not imatches + ( + '?:\\$WinREAgent\\Scratch\\*', + '?:\\Windows\\WinSxS\\*', + '?:\\Windows\\CbsTemp\\*', + '?:\\Windows\\SoftwareDistribution\\*' + ) and + (file.extension iin ('.dmp', '.mdmp', '.dump') or is_minidump(file.path)) + | output: > Detected an attempt by `%1.ps.name` process to access and read diff --git a/rules/credential_access_lsass_process_clone_creation_via_reflection.yml b/rules/credential_access_lsass_process_clone_creation_via_reflection.yml index 26ebc8b57..2d913047c 100644 --- a/rules/credential_access_lsass_process_clone_creation_via_reflection.yml +++ b/rules/credential_access_lsass_process_clone_creation_via_reflection.yml @@ -1,6 +1,6 @@ name: LSASS process clone creation via reflection id: cdf3810a-4832-446a-ac9d-d108cf2e313c -version: 1.0.3 +version: 1.0.4 description: | Identifies the creation of an LSASS clone process via RtlCreateProcessReflection API function. Adversaries can use this technique to dump credentials material from the LSASS fork and evade @@ -20,7 +20,7 @@ references: - https://s3cur3th1ssh1t.github.io/Reflective-Dump-Tools/ condition: > - spawn_process and + spawn_process and ps.name ~= 'lsass.exe' and ps.parent.name ~= 'lsass.exe' and thread.callstack.symbols imatches ('ntdll.dll!RtlCloneUserProcess', 'ntdll.dll!RtlCreateProcessReflection') action: diff --git a/rules/defense_evasion_dll_sideloading_via_copied_binary.yml b/rules/defense_evasion_dll_sideloading_via_copied_binary.yml index b732eff31..babe5a957 100644 --- a/rules/defense_evasion_dll_sideloading_via_copied_binary.yml +++ b/rules/defense_evasion_dll_sideloading_via_copied_binary.yml @@ -1,6 +1,6 @@ name: DLL Side-Loading via a copied binary id: 80798e2c-6c37-472b-936c-1d2d6b95ff3c -version: 1.0.6 +version: 1.0.7 description: | Identifies when a binary is copied to a directory and shortly followed by the loading of an unsigned DLL from the same directory. Adversaries may @@ -21,8 +21,9 @@ condition: > sequence maxspan 8m |create_file and - file.is_exec and ps.sid not in ('S-1-5-18', 'S-1-5-19', 'S-1-5-20') and - thread.callstack.symbols imatches ('*CopyFile*', '*MoveFile*') + ps.sid not in ('S-1-5-18', 'S-1-5-19', 'S-1-5-20') and + thread.callstack.symbols imatches ('*CopyFile*', '*MoveFile*') and + (file.extension ~= '.exe' or file.is_exec) | by file.path |(load_dll) and dir(image.path) ~= dir(ps.exe) and diff --git a/rules/defense_evasion_dll_sideloading_via_microsoft_office_dropped_file.yml b/rules/defense_evasion_dll_sideloading_via_microsoft_office_dropped_file.yml index 381672054..3f4fc8743 100644 --- a/rules/defense_evasion_dll_sideloading_via_microsoft_office_dropped_file.yml +++ b/rules/defense_evasion_dll_sideloading_via_microsoft_office_dropped_file.yml @@ -1,6 +1,6 @@ name: DLL Side-Loading via Microsoft Office dropped file id: d808175d-c4f8-459d-b17f-ca9a88890c04 -version: 1.0.4 +version: 1.0.5 description: | Identifies Microsoft Office process creating a DLL or other variant of an executable object which is later loaded by a trusted binary. Adversaries may exploit this behavior by delivering malicious @@ -20,8 +20,8 @@ condition: > sequence maxspan 6m |create_file and - (file.extension iin ('.dll', '.cpl', '.ocx') or file.is_dll) and - ps.name iin msoffice_binaries + ps.name iin msoffice_binaries and + (file.extension iin ('.dll', '.cpl', '.ocx') or file.is_dll) | by file.path |(load_unsigned_or_untrusted_dll) and ps.name not iin msoffice_binaries and ps.signature.trusted = true and diff --git a/rules/defense_evasion_dotnet_assembly_loaded_by_unmanaged_process.yml b/rules/defense_evasion_dotnet_assembly_loaded_by_unmanaged_process.yml index 3ccb1c28f..c8b18a2cd 100644 --- a/rules/defense_evasion_dotnet_assembly_loaded_by_unmanaged_process.yml +++ b/rules/defense_evasion_dotnet_assembly_loaded_by_unmanaged_process.yml @@ -1,6 +1,6 @@ name: .NET assembly loaded by unmanaged process id: 34be8bd1-1143-4fa8-bed4-ae2566b1394a -version: 1.0.10 +version: 1.0.11 description: | Identifies the loading of the .NET assembly by an unmanaged process. Adversaries can load the CLR runtime inside unmanaged process and execute the assembly via the ICLRRuntimeHost::ExecuteInDefaultAppDomain method. @@ -17,14 +17,18 @@ references: condition: > (load_unsigned_or_untrusted_module) and - ps.exe != '' and ps.pe.is_dotnet = false and - (dll.pe.is_dotnet or thread.callstack.modules imatches ('*clr.dll')) and dll.path not imatches ( '?:\\Windows\\assembly\\*\\*.ni.dll', '?:\\Program Files\\WindowsPowerShell\\Modules\\*\\*.dll', - '?:\\Windows\\Microsoft.NET\\assembly\\*\\*.dll' + '?:\\Windows\\Microsoft.NET\\assembly\\*\\*.dll', + '?:\\$WinREAgent\\Scratch\\*', + '?:\\Windows\\WinSxS\\*', + '?:\\Windows\\CbsTemp\\*', + '?:\\Windows\\SoftwareDistribution\\*' ) and + ps.exe != '' and ps.pe.is_dotnet = false and + (dll.pe.is_dotnet or thread.callstack.modules imatches ('*clr.dll')) and ps.exe not imatches ( '?:\\Program Files\\WindowsApps\\*\\CrossDeviceService.exe', diff --git a/rules/initial_access_execution_via_microsoft_office_process.yml b/rules/initial_access_execution_via_microsoft_office_process.yml index 2a7ece313..50f6ea3d4 100644 --- a/rules/initial_access_execution_via_microsoft_office_process.yml +++ b/rules/initial_access_execution_via_microsoft_office_process.yml @@ -1,6 +1,6 @@ name: Execution via Microsoft Office process id: a10ebe66-1b55-4005-a374-840f1e2933a3 -version: 1.0.3 +version: 1.0.4 description: Identifies the execution of the file dropped by Microsoft Office process. labels: @@ -17,7 +17,7 @@ labels: condition: > sequence maxspan 1h - |create_file and (file.extension iin executable_extensions or file.is_exec) and ps.name iin msoffice_binaries| by file.path + |create_file and ps.name iin msoffice_binaries and (file.extension iin executable_extensions or file.is_exec)| by file.path |spawn_process and ps.parent.name iin msoffice_binaries| by ps.exe min-engine-version: 3.0.0 diff --git a/rules/initial_access_suspicious_dll_loaded_by_microsoft_office_process.yml b/rules/initial_access_suspicious_dll_loaded_by_microsoft_office_process.yml index d1251a7ab..e0310b6cd 100644 --- a/rules/initial_access_suspicious_dll_loaded_by_microsoft_office_process.yml +++ b/rules/initial_access_suspicious_dll_loaded_by_microsoft_office_process.yml @@ -1,6 +1,6 @@ name: Suspicious DLL loaded by Microsoft Office process id: 5868518c-2a83-4b26-ad4b-f14f0b85e744 -version: 1.0.4 +version: 1.0.5 description: Identifies loading of recently dropped DLL by Microsoft Office process. labels: @@ -18,7 +18,8 @@ condition: > sequence maxspan 1h |create_file and - (file.extension iin module_extensions or file.is_dll) and ps.name iin msoffice_binaries and + ps.name iin msoffice_binaries and + (file.extension iin module_extensions or file.is_dll) and file.path not imatches '?:\\Program Files\\Microsoft Office\\Root\\Office*\\*.dll' | by file.name |load_module and ps.name iin msoffice_binaries| by module.name diff --git a/rules/macros/macros.yml b/rules/macros/macros.yml index af806ae84..9dd56adc8 100644 --- a/rules/macros/macros.yml +++ b/rules/macros/macros.yml @@ -25,6 +25,9 @@ - macro: create_file expr: evt.name = 'CreateFile' and file.operation != 'OPEN' and file.status = 'Success' +- macro: create_new_file + expr: evt.name = 'CreateFile' and file.operation = 'CREATE' and file.status = 'Success' + - macro: rename_file expr: evt.name = 'RenameFile' diff --git a/rules/persistence_suspicious_microsoft_office_template.yml b/rules/persistence_suspicious_microsoft_office_template.yml index 6ea6e9394..b00256ed3 100644 --- a/rules/persistence_suspicious_microsoft_office_template.yml +++ b/rules/persistence_suspicious_microsoft_office_template.yml @@ -1,6 +1,6 @@ name: Suspicious Microsoft Office template id: c4be3b30-9d23-4a33-b974-fb12e17487a2 -version: 1.0.4 +version: 1.0.5 description: | Detects when attackers drop macro-enabled files in specific folders to trigger their execution every time the victim user @@ -20,6 +20,7 @@ references: condition: > create_file and + ps.name not iin msoffice_binaries and file.path imatches ( '?:\\Users\\*\\AppData\\Roaming\\Microsoft\\Word\\Startup\\*', @@ -28,7 +29,6 @@ condition: > '?:\\Users\\*\\AppData\\Roaming\\Microsoft\\AddIns\\*', '?:\\Users\\*\\AppData\\Roaming\\Microsoft\\Outlook\\*.otm' ) and - ps.name not iin msoffice_binaries and ps.exe not imatches ( '?:\\Program Files\\*.exe', diff --git a/rules/persistence_unusual_file_written_in_startup_folder.yml b/rules/persistence_unusual_file_written_in_startup_folder.yml index 7d6934f9f..f6c1dd897 100644 --- a/rules/persistence_unusual_file_written_in_startup_folder.yml +++ b/rules/persistence_unusual_file_written_in_startup_folder.yml @@ -1,6 +1,6 @@ name: Unusual file written in Startup folder id: c5ffe15c-d94f-416b-bec7-c47f89843267 -version: 1.0.4 +version: 1.0.5 description: | Identifies suspicious files written to the startup folder that would allow adversaries to maintain persistence on the endpoint. @@ -17,8 +17,8 @@ labels: condition: > create_file and - (file.extension in ('.vbs', '.js', '.jar', '.exe', '.dll', '.com', '.ps1', '.hta', '.cmd', '.vbe') or (file.is_exec or file.is_dll)) and file.path imatches startup_locations and + (file.extension in ('.vbs', '.js', '.jar', '.exe', '.dll', '.com', '.ps1', '.hta', '.cmd', '.vbe') or (file.is_exec or file.is_dll)) and ps.exe not imatches ( '?:\\Windows\\System32\\wuauclt.exe', diff --git a/rules/privilege_escalation_vulnerable_or_malicious_driver_dropped.yml b/rules/privilege_escalation_vulnerable_or_malicious_driver_dropped.yml index 27dbac6e0..e9821bc17 100644 --- a/rules/privilege_escalation_vulnerable_or_malicious_driver_dropped.yml +++ b/rules/privilege_escalation_vulnerable_or_malicious_driver_dropped.yml @@ -1,6 +1,6 @@ name: Vulnerable or malicious driver dropped id: d4742163-cf68-4ebd-b9a2-3ad17bbf63d5 -version: 1.0.3 +version: 1.0.4 description: | Detects when adversaries drop a vulnerable/malicious driver onto a compromised system as a preparation for vulnerability @@ -16,7 +16,15 @@ references: - https://www.loldrivers.io/ condition: > - create_file and file.is_driver and (file.is_driver_vulnerable or file.is_driver_malicious) + create_file and + file.path not imatches + ( + '?:\\$WinREAgent\\Scratch\\*', + '?:\\Windows\\WinSxS\\*', + '?:\\Windows\\CbsTemp\\*', + '?:\\Windows\\SoftwareDistribution\\*' + ) and + (file.extension ~= '.sys' or file.is_driver) and (file.is_driver_vulnerable or file.is_driver_malicious) output: > Vulnerable or malicious %file.path driver dropped