fix(callstack): Treat surrogate process callstack

Process creation can happen through brokered
processes (Secondary Logon, App Info, etc). In
this case, the callstack is not reported by the
parent process. So, we obtain the callstack from
the surrogate process when the initial thread
is created inside the remote process.

Author: rabbitstack

pkg/event/event_windows.go

@@ -21,6 +21,11 @@ package event
 import (
 	"encoding/binary"
 	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"unsafe"
+
 	"github.com/rabbitstack/fibratus/pkg/event/params"
 	"github.com/rabbitstack/fibratus/pkg/sys"
 	"github.com/rabbitstack/fibratus/pkg/sys/etw"
@@ -29,10 +34,6 @@ import (
 	"github.com/rabbitstack/fibratus/pkg/util/hostname"
 	"github.com/rabbitstack/fibratus/pkg/util/ntstatus"
 	"golang.org/x/sys/windows"
-	"os"
-	"strings"
-	"sync"
-	"unsafe"
 )
 
 var (
@@ -266,6 +267,17 @@ func (e *Event) IsOpenDisposition() bool {
 	return e.IsCreateFile() && e.Params.MustGetUint32(params.FileOperation) == windows.FILE_OPEN
 }
 
+// IsCreateRemoteThread indicates if the remote thread creation occurred.
+func (e *Event) IsCreateRemoteThread() bool {
+	return e.Type == CreateThread && e.PID != e.Params.MustGetPid()
+}
+
+// IsSurrogateProcess indicates if the process creation event parent id
+// differs from the real process parent identifier.
+func (e *Event) IsSurrogateProcess() bool {
+	return e.IsCreateProcess() && e.Params.MustGetUint32(params.ProcessParentID) != e.Params.MustGetUint32(params.ProcessRealParentID)
+}
+
 // StackID returns the integer that is used to identify the callstack present in the StackWalk event.
 func (e *Event) StackID() uint64 { return uint64(e.PID + e.Tid) }
 

pkg/event/stackwalk.go

@@ -20,11 +20,12 @@ package event
 
 import (
 	"expvar"
+	"sync"
+	"time"
+
 	"github.com/rabbitstack/fibratus/pkg/event/params"
 	"github.com/rabbitstack/fibratus/pkg/util/multierror"
 	log "github.com/sirupsen/logrus"
-	"sync"
-	"time"
 )
 
 // maxQueueTTLPeriod specifies the maximum period
@@ -61,6 +62,8 @@ type StackwalkDecorator struct {
 
 	flusher *time.Ticker
 	quit    chan struct{}
+
+	procs map[uint32]*Event // stores CreateProcess events with surrogate parent
 }
 
 // NewStackwalkDecorator creates a new callstack return
@@ -70,6 +73,7 @@ func NewStackwalkDecorator(q *Queue) *StackwalkDecorator {
 	s := &StackwalkDecorator{
 		q:       q,
 		buckets: make(map[uint64][]*Event),
+		procs:   make(map[uint32]*Event),
 		flusher: time.NewTicker(flusherInterval),
 		quit:    make(chan struct{}, 1),
 	}
@@ -84,6 +88,13 @@ func (s *StackwalkDecorator) Push(e *Event) {
 	s.mux.Lock()
 	defer s.mux.Unlock()
 
+	// the process is created on behalf of brokered
+	// process and the callstack return addresses
+	// need to be obtained from the surrogate process
+	if e.IsSurrogateProcess() {
+		s.procs[e.Params.MustGetPid()] = e
+	}
+
 	// append the event to the bucket indexed by stack id
 	id := e.StackID()
 	q, ok := s.buckets[id]
@@ -121,9 +132,39 @@ func (s *StackwalkDecorator) Pop(e *Event) *Event {
 		return e
 	}
 
+	if evt.IsSurrogateProcess() && s.procs[evt.Params.MustGetPid()] != nil {
+		delete(s.procs, evt.Params.MustGetPid())
+	}
+
 	callstack := e.Params.MustGetSlice(params.Callstack)
 	evt.AppendParam(params.Callstack, params.Slice, callstack)
 
+	// obtain the callstack from the CreateThread event
+	// generated by the surrogate process, such as Seclogon.
+	// If the remote process id is present in the procs map
+	// the stack is attached to the cached event and then
+	// pushed to the queue immediately
+	if (evt.IsCreateRemoteThread() && evt.PS != nil) &&
+		(evt.PS.IsSeclogonSvc() || evt.PS.IsAppinfoSvc()) {
+		pid := evt.Params.MustGetPid()
+		ev, ok := s.procs[pid]
+		if ok {
+			ev.AppendParam(params.Callstack, params.Slice, callstack)
+			_ = s.q.push(ev)
+			delete(s.procs, pid)
+			// find the most recent CreateProcess event and
+			// remove it from buckets as we have the callstack
+			qu := s.buckets[ev.StackID()]
+			for i := len(qu) - 1; i >= 0; i-- {
+				proc := qu[i]
+				if !proc.IsCreateProcess() && proc.Params.MustGetPid() != pid {
+					continue
+				}
+				s.buckets[ev.StackID()] = append(qu[:i], qu[i+1:]...)
+			}
+		}
+	}
+
 	return evt
 }
 

pkg/event/stackwalk_test.go

@@ -19,12 +19,14 @@
 package event
 
 import (
+	"testing"
+	"time"
+
 	"github.com/rabbitstack/fibratus/pkg/event/params"
 	"github.com/rabbitstack/fibratus/pkg/fs"
+	pstypes "github.com/rabbitstack/fibratus/pkg/ps/types"
 	"github.com/rabbitstack/fibratus/pkg/util/va"
 	"github.com/stretchr/testify/assert"
-	"testing"
-	"time"
 )
 
 func TestStackwalkDecorator(t *testing.T) {
@@ -90,6 +92,77 @@ func TestStackwalkDecorator(t *testing.T) {
 	assert.Equal(t, "C:\\Windows\\system32\\user32.dll", evt.GetParamAsString(params.FilePath))
 }
 
+func TestStackwalkDecoratorSurrogateProcess(t *testing.T) {
+	q := NewQueue(50, false, true)
+	cd := NewStackwalkDecorator(q)
+
+	e := &Event{
+		Type:      CreateProcess,
+		Tid:       2484,
+		PID:       859,
+		CPU:       1,
+		Seq:       2,
+		Name:      "CreateProcess",
+		Timestamp: time.Now(),
+		Category:  Process,
+		Params: Params{
+			params.ProcessID:           {Name: params.ProcessID, Type: params.PID, Value: uint32(859)},
+			params.ProcessParentID:     {Name: params.ProcessParentID, Type: params.PID, Value: uint32(4523)},
+			params.ProcessRealParentID: {Name: params.ProcessRealParentID, Type: params.PID, Value: uint32(8846)},
+		},
+	}
+
+	e1 := &Event{
+		Type:      CreateThread,
+		Tid:       2484,
+		PID:       1411,
+		CPU:       1,
+		Seq:       3,
+		Name:      "CreateThread",
+		Timestamp: time.Now(),
+		Category:  Thread,
+		PS: &pstypes.PS{
+			Name:    "svchost.exe",
+			Exe:     `C:\WINDOWS\system32\svchost.exe`,
+			Cmdline: `C:\WINDOWS\system32\svchost.exe -k netsvcs -p -s seclogon`,
+		},
+		Params: Params{
+			params.ProcessID: {Name: params.ProcessID, Type: params.PID, Value: uint32(859)},
+		},
+	}
+
+	cd.Push(e)
+
+	assert.Len(t, cd.buckets[e.StackID()], 1)
+	assert.Len(t, cd.buckets[e1.StackID()], 0)
+	assert.Len(t, cd.procs, 1)
+
+	cd.Push(e1)
+	assert.Len(t, cd.buckets[e1.StackID()], 1)
+
+	sw := &Event{
+		Type:      StackWalk,
+		Tid:       2484,
+		PID:       1411,
+		CPU:       1,
+		Seq:       4,
+		Name:      "StackWalk",
+		Timestamp: time.Now(),
+		Params: Params{
+			params.Callstack: {Name: params.Callstack, Type: params.Slice, Value: []va.Address{0x7ffb5eb70dc4, 0x7ffb5c191deb, 0x7ffb3138592e}},
+		},
+	}
+
+	thread := cd.Pop(sw)
+	proc := <-q.Events()
+	assert.Equal(t, CreateProcess, proc.Type)
+	assert.Equal(t, CreateThread, thread.Type)
+	assert.Len(t, cd.buckets[e.StackID()], 0)
+	assert.Len(t, cd.buckets[e1.StackID()], 0)
+	assert.True(t, proc.Params.Contains(params.Callstack))
+	assert.True(t, thread.Params.Contains(params.Callstack))
+}
+
 func init() {
 	maxQueueTTLPeriod = time.Second * 2
 	flusherInterval = time.Second

pkg/ps/types/types_windows.go

@@ -21,20 +21,23 @@ package types
 import (
 	"encoding/binary"
 	"fmt"
+	"path/filepath"
+	"strings"
+	"sync"
+
 	"github.com/rabbitstack/fibratus/pkg/sys"
 	"github.com/rabbitstack/fibratus/pkg/util/cmdline"
 	"github.com/rabbitstack/fibratus/pkg/util/va"
+	"github.com/rabbitstack/fibratus/pkg/util/wildcard"
 	"golang.org/x/sys/windows"
-	"path/filepath"
-	"strings"
-	"sync"
 
 	"github.com/rabbitstack/fibratus/pkg/cap/section"
 	htypes "github.com/rabbitstack/fibratus/pkg/handle/types"
 	"github.com/rabbitstack/fibratus/pkg/pe"
 
-	"github.com/rabbitstack/fibratus/pkg/util/bootid"
 	"time"
+
+	"github.com/rabbitstack/fibratus/pkg/util/bootid"
 )
 
 // PS encapsulates process' state such as allocated resources and other metadata.
@@ -291,6 +294,21 @@ func (ps *PS) Ancestors() []string {
 	return ancestors
 }
 
+// IsSeclogonSvc returns true if this is the Secondary Logon Service process.
+func (ps *PS) IsSeclogonSvc() bool {
+	return ps.IsSvchost() && strings.HasSuffix(ps.Cmdline, "-s seclogon")
+}
+
+// IsAppinfoSvc returns true if this is the AppInfo Service process.
+func (ps *PS) IsAppinfoSvc() bool {
+	return ps.IsSvchost() && strings.HasSuffix(ps.Cmdline, "-s Appinfo")
+}
+
+// IsSvchost returns true if this is the Service Host process.
+func (ps *PS) IsSvchost() bool {
+	return wildcard.Match(`?:\windows\system32\svchost.exe`, strings.ToLower(ps.Exe))
+}
+
 // Thread stores metadata about a thread that's executing in process's address space.
 type Thread struct {
 	// Tid is the unique identifier of thread inside the process.

pkg/ps/types/types_windows_test.go

@@ -19,13 +19,14 @@
 package types
 
 import (
+	"os"
+	"testing"
+	"time"
+
 	"github.com/rabbitstack/fibratus/pkg/util/bootid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/windows"
-	"os"
-	"testing"
-	"time"
 )
 
 func TestVisit(t *testing.T) {
@@ -97,3 +98,35 @@ func TestUUID(t *testing.T) {
 	tsUUID := (bootid.Read() << 30) + uint64(os.Getpid()) | uint64(now.UnixNano())
 	assert.True(t, ps2.UUID() > 0 && ps2.UUID() != tsUUID)
 }
+
+func TestIsSeclogonSvc(t *testing.T) {
+	var tests = []struct {
+		ps *PS
+		ok bool
+	}{
+		{&PS{Name: "svchost.exe", Exe: `C:\WINDOWS\system32\svchost.exe`, Cmdline: `C:\WINDOWS\system32\svchost.exe -k netsvcs -p -s Appinfo`}, false},
+		{&PS{Name: "svchost.exe", Exe: `C:\WINDOWS\system32\svchost.exe`, Cmdline: `C:\WINDOWS\system32\svchost.exe -k netsvcs -p -s seclogon`}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.ps.Cmdline, func(t *testing.T) {
+			assert.Equal(t, tt.ok, tt.ps.IsSeclogonSvc())
+		})
+	}
+}
+
+func TestIsAppinfoSvc(t *testing.T) {
+	var tests = []struct {
+		ps *PS
+		ok bool
+	}{
+		{&PS{Name: "svchost.exe", Exe: `C:\WINDOWS\system32\svchost.exe`, Cmdline: `C:\WINDOWS\system32\svchost.exe -k netsvcs -p -s Appinfo`}, true},
+		{&PS{Name: "svchost.exe", Exe: `C:\WINDOWS\system32\svchost.exe`, Cmdline: `C:\WINDOWS\System32\svchost.exe -k LocalServiceNoNetwork -p -s DPS`}, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.ps.Cmdline, func(t *testing.T) {
+			assert.Equal(t, tt.ok, tt.ps.IsAppinfoSvc())
+		})
+	}
+}