diff --git a/runtime/internal/lib/runtime/pprof_linkname_llgo.go b/runtime/internal/lib/runtime/pprof_linkname_llgo.go index e7222ac61e..4488dbd97f 100644 --- a/runtime/internal/lib/runtime/pprof_linkname_llgo.go +++ b/runtime/internal/lib/runtime/pprof_linkname_llgo.go @@ -73,11 +73,6 @@ func pprof_goroutineLeakProfileWithLabels(p []StackRecord, labels []unsafe.Point return 0, true } -//go:linkname pprof_memProfileInternal runtime.pprof_memProfileInternal -func pprof_memProfileInternal(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { - return 0, true -} - //go:linkname pprof_blockProfileInternal runtime.pprof_blockProfileInternal func pprof_blockProfileInternal(p []BlockProfileRecord) (n int, ok bool) { return 0, true diff --git a/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go new file mode 100644 index 0000000000..172930c92b --- /dev/null +++ b/runtime/internal/lib/runtime/pprof_memprofile_go123_llgo.go @@ -0,0 +1,50 @@ +//go:build (darwin || linux) && go1.23 + +package runtime + +import _ "unsafe" + +type pprofMemProfileRecord struct { + AllocBytes, FreeBytes int64 + AllocObjects, FreeObjects int64 + Stack []uintptr +} + +//go:linkname pprof_memProfileInternal runtime.pprof_memProfileInternal +func pprof_memProfileInternal(p []pprofMemProfileRecord, inuseZero bool) (n int, ok bool) { + n, _ = MemProfile(nil, inuseZero) + if len(p) < n { + return n, false + } + if n == 0 { + return 0, true + } + var records [64]MemProfileRecord + if n > len(records) { + return n, false + } + n, ok = MemProfile(records[:n], inuseZero) + if !ok { + return n, false + } + for i := 0; i < n; i++ { + p[i] = pprofMemProfileRecord{ + AllocBytes: records[i].AllocBytes, + FreeBytes: records[i].FreeBytes, + AllocObjects: records[i].AllocObjects, + FreeObjects: records[i].FreeObjects, + Stack: pprofMemProfileStack(&records[i]), + } + } + return n, true +} + +func pprofMemProfileStack(r *MemProfileRecord) []uintptr { + stack := r.Stack() + if len(stack) == 0 { + return nil + } + out := make([]uintptr, len(stack)) + copy(out, stack) + return out +} diff --git a/runtime/internal/lib/runtime/pprof_memprofile_pre_go123_llgo.go b/runtime/internal/lib/runtime/pprof_memprofile_pre_go123_llgo.go new file mode 100644 index 0000000000..d7c148b64b --- /dev/null +++ b/runtime/internal/lib/runtime/pprof_memprofile_pre_go123_llgo.go @@ -0,0 +1,10 @@ +//go:build (darwin || linux) && !go1.23 + +package runtime + +import _ "unsafe" + +//go:linkname pprof_memProfileInternal runtime.pprof_memProfileInternal +func pprof_memProfileInternal(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + return MemProfile(p, inuseZero) +} diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 8f7045430b..5b8155d0e8 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -2,18 +2,16 @@ package runtime -// StackRecord is a minimal placeholder for runtime/pprof. +import llrt "github.com/goplus/llgo/runtime/internal/runtime" + type StackRecord struct { Stack []uintptr } -// MemProfileRecord is a minimal placeholder for runtime/pprof. type MemProfileRecord struct { - AllocBytes int64 - FreeBytes int64 - AllocObjects int64 - FreeObjects int64 - Stack []uintptr + AllocBytes, FreeBytes int64 + AllocObjects, FreeObjects int64 + Stack0 [32]uintptr } func (r *MemProfileRecord) InUseBytes() int64 { @@ -24,6 +22,15 @@ func (r *MemProfileRecord) InUseObjects() int64 { return r.AllocObjects - r.FreeObjects } +func (r *MemProfileRecord) Stack() []uintptr { + for i, pc := range r.Stack0 { + if pc == 0 { + return r.Stack0[:i] + } + } + return r.Stack0[:] +} + // BlockProfileRecord is a minimal placeholder for runtime/pprof. type BlockProfileRecord struct { Count int64 @@ -32,7 +39,31 @@ type BlockProfileRecord struct { } func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { - return 0, false + n, _ = llrt.MemProfile(nil, inuseZero) + if len(p) < n { + return n, false + } + if n == 0 { + return 0, true + } + var records [64]llrt.MemProfileRecord + if n > len(records) { + return n, false + } + n, ok = llrt.MemProfile(records[:n], inuseZero) + if !ok { + return n, false + } + for i := 0; i < n; i++ { + p[i] = MemProfileRecord{ + AllocBytes: records[i].AllocBytes, + FreeBytes: records[i].FreeBytes, + AllocObjects: records[i].AllocObjects, + FreeObjects: records[i].FreeObjects, + Stack0: records[i].Stack0, + } + } + return n, true } func BlockProfile(p []BlockProfileRecord) (n int, ok bool) { diff --git a/runtime/internal/runtime/memprofile.go b/runtime/internal/runtime/memprofile.go new file mode 100644 index 0000000000..c6221ca1dd --- /dev/null +++ b/runtime/internal/runtime/memprofile.go @@ -0,0 +1,112 @@ +package runtime + +// MemProfileRecord describes allocations aggregated by size class. +type MemProfileRecord struct { + AllocBytes, FreeBytes int64 + AllocObjects, FreeObjects int64 + Stack0 [32]uintptr +} + +func (r *MemProfileRecord) InUseBytes() int64 { + return r.AllocBytes - r.FreeBytes +} + +func (r *MemProfileRecord) InUseObjects() int64 { + return r.AllocObjects - r.FreeObjects +} + +func (r *MemProfileRecord) Stack() []uintptr { + for i, pc := range r.Stack0 { + if pc == 0 { + return r.Stack0[:i] + } + } + return r.Stack0[:] +} + +type memProfileBucket struct { + size uintptr + + objects memProfileCounter +} + +var memProfileBuckets = [...]memProfileBucket{ + {size: 16}, + {size: 32}, + {size: 64}, + {size: 128}, + {size: 256}, + {size: 512}, + {size: 1024}, + {size: 2048}, + {size: 4096}, + {size: 8192}, + {size: 16384}, + {size: 32768}, + {size: 65536}, + {size: 131072}, + {size: 262144}, + {size: 524288}, + {size: 1048576}, + {size: 2097152}, + {size: 4194304}, + {size: 8388608}, + {size: 16777216}, + {size: 33554432}, + {size: 67108864}, + {size: 134217728}, + {size: 268435456}, + {size: 536870912}, + {size: 1073741824}, +} + +func recordMemProfileAlloc(size uintptr) { + if size == 0 { + return + } + size = memProfileSizeClass(size) + for i := range memProfileBuckets { + b := &memProfileBuckets[i] + if b.size == size { + memProfileAddObject(&b.objects) + return + } + } +} + +func memProfileSizeClass(size uintptr) uintptr { + if size <= 16 { + return 16 + } + for _, b := range memProfileBuckets { + if size <= b.size { + return b.size + } + } + return memProfileBuckets[len(memProfileBuckets)-1].size +} + +func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) { + for i := range memProfileBuckets { + if memProfileLoadObjects(&memProfileBuckets[i].objects) != 0 { + n++ + } + } + if len(p) < n { + return n, false + } + j := 0 + for i := range memProfileBuckets { + b := &memProfileBuckets[i] + objects := memProfileLoadObjects(&b.objects) + if objects == 0 { + continue + } + p[j] = MemProfileRecord{ + AllocBytes: int64(uint64(b.size) * uint64(objects)), + AllocObjects: int64(objects), + } + j++ + } + return n, true +} diff --git a/runtime/internal/runtime/memprofile_atomic.go b/runtime/internal/runtime/memprofile_atomic.go new file mode 100644 index 0000000000..d31ae8e9f0 --- /dev/null +++ b/runtime/internal/runtime/memprofile_atomic.go @@ -0,0 +1,15 @@ +//go:build !baremetal + +package runtime + +import "github.com/goplus/llgo/runtime/internal/clite/sync/atomic" + +type memProfileCounter = uint64 + +func memProfileAddObject(p *memProfileCounter) { + atomic.Add(p, memProfileCounter(1)) +} + +func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { + return atomic.Load(p) +} diff --git a/runtime/internal/runtime/memprofile_baremetal.go b/runtime/internal/runtime/memprofile_baremetal.go new file mode 100644 index 0000000000..29ba9ab6d9 --- /dev/null +++ b/runtime/internal/runtime/memprofile_baremetal.go @@ -0,0 +1,13 @@ +//go:build baremetal + +package runtime + +type memProfileCounter = uintptr + +func memProfileAddObject(p *memProfileCounter) { + *p = *p + 1 +} + +func memProfileLoadObjects(p *memProfileCounter) memProfileCounter { + return *p +} diff --git a/runtime/internal/runtime/z_gc.go b/runtime/internal/runtime/z_gc.go index 04764a106a..94c741afd3 100644 --- a/runtime/internal/runtime/z_gc.go +++ b/runtime/internal/runtime/z_gc.go @@ -28,12 +28,15 @@ import ( // AllocU allocates uninitialized memory. func AllocU(size uintptr) unsafe.Pointer { - return bdwgc.Malloc(size) + ret := bdwgc.Malloc(size) + recordMemProfileAlloc(size) + return ret } // AllocZ allocates zero-initialized memory. func AllocZ(size uintptr) unsafe.Pointer { ret := bdwgc.Malloc(size) + recordMemProfileAlloc(size) return c.Memset(ret, 0, size) } diff --git a/runtime/internal/runtime/z_gc_baremetal.go b/runtime/internal/runtime/z_gc_baremetal.go index afccff9705..cda752a9cf 100644 --- a/runtime/internal/runtime/z_gc_baremetal.go +++ b/runtime/internal/runtime/z_gc_baremetal.go @@ -26,12 +26,16 @@ import ( // AllocU allocates uninitialized memory. func AllocU(size uintptr) unsafe.Pointer { - return tinygogc.Alloc(size) + ret := tinygogc.Alloc(size) + recordMemProfileAlloc(size) + return ret } // AllocZ allocates zero-initialized memory. func AllocZ(size uintptr) unsafe.Pointer { - return tinygogc.Alloc(size) + ret := tinygogc.Alloc(size) + recordMemProfileAlloc(size) + return ret } // AddCleanupPtr is not implemented in baremetal builds because tinygogc diff --git a/runtime/internal/runtime/z_nogc.go b/runtime/internal/runtime/z_nogc.go index 7cbc30c2f0..2f33774682 100644 --- a/runtime/internal/runtime/z_nogc.go +++ b/runtime/internal/runtime/z_nogc.go @@ -27,12 +27,15 @@ import ( // AllocU allocates uninitialized memory. func AllocU(size uintptr) unsafe.Pointer { - return c.Malloc(size) + ret := c.Malloc(size) + recordMemProfileAlloc(size) + return ret } // AllocZ allocates zero-initialized memory. func AllocZ(size uintptr) unsafe.Pointer { ret := c.Malloc(size) + recordMemProfileAlloc(size) return c.Memset(ret, 0, size) } diff --git a/test/go/memprofile/memprofile_test.go b/test/go/memprofile/memprofile_test.go new file mode 100644 index 0000000000..2e4a9d4644 --- /dev/null +++ b/test/go/memprofile/memprofile_test.go @@ -0,0 +1,96 @@ +package memprofile + +import ( + "bytes" + "fmt" + "runtime" + "runtime/pprof" + "testing" +) + +var tinySink []*int32 + +func TestRuntimeMemProfileReportsTinyAllocations(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + const n = 4096 + tinySink = make([]*int32, 0, n) + for i := 0; i < n; i++ { + p := new(int32) + *p = int32(i) + tinySink = append(tinySink, p) + } + runtime.GC() + runtime.GC() + + records := readMemProfile(t) + wantBytes := int64(n * 4) + for _, r := range records { + inUseObjects := r.InUseObjects() + inUseBytes := r.InUseBytes() + if inUseObjects <= 0 || inUseBytes <= 0 { + continue + } + if got := len(r.Stack()); got > len(r.Stack0) { + t.Fatalf("MemProfileRecord.Stack length = %d, want <= %d", got, len(r.Stack0)) + } + if inUseBytes/inUseObjects == 16 && inUseBytes >= wantBytes { + return + } + } + t.Fatalf("MemProfile did not report tiny allocations totaling at least %d bytes: %#v", wantBytes, records) +} + +func TestRuntimePprofHeapProfileReportsTinyAllocations(t *testing.T) { + oldRate := runtime.MemProfileRate + runtime.MemProfileRate = 1 + defer func() { + runtime.MemProfileRate = oldRate + }() + + const n = 4096 + allocateTinyObjects(n) + runtime.GC() + runtime.GC() + + var buf bytes.Buffer + if err := pprof.Lookup("heap").WriteTo(&buf, 1); err != nil { + t.Fatalf("heap profile WriteTo failed: %v", err) + } + + var inUseObjects, inUseBytes, allocObjects, allocBytes, rate int64 + if _, err := fmt.Fscanf(bytes.NewReader(buf.Bytes()), "heap profile: %d: %d [%d: %d] @ heap/%d", + &inUseObjects, &inUseBytes, &allocObjects, &allocBytes, &rate); err != nil { + t.Fatalf("failed to parse heap profile header: %v\n%s", err, buf.String()) + } + wantBytes := int64(n * 4) + if inUseObjects <= 0 || allocObjects <= 0 || inUseBytes < wantBytes || allocBytes < wantBytes { + t.Fatalf("heap profile totals = %d: %d [%d: %d], want live allocation bytes >= %d\n%s", + inUseObjects, inUseBytes, allocObjects, allocBytes, wantBytes, buf.String()) + } +} + +func readMemProfile(t *testing.T) []runtime.MemProfileRecord { + t.Helper() + var records []runtime.MemProfileRecord + for { + n, ok := runtime.MemProfile(records, false) + if ok { + return records[:n] + } + records = make([]runtime.MemProfileRecord, n+10) + } +} + +func allocateTinyObjects(n int) { + tinySink = make([]*int32, 0, n) + for i := 0; i < n; i++ { + p := new(int32) + *p = int32(i) + tinySink = append(tinySink, p) + } +} diff --git a/test/goroot/xfail.yaml b/test/goroot/xfail.yaml index a5a66d10b4..b8c7226fa8 100644 --- a/test/goroot/xfail.yaml +++ b/test/goroot/xfail.yaml @@ -2331,10 +2331,6 @@ xfails: directive: run case: deferfin.go reason: latest main goroot run failure on darwin/arm64 - - platform: darwin/arm64 - directive: run - case: finprofiled.go - reason: latest main goroot run failure on darwin/arm64 - platform: darwin/arm64 directive: run case: heapsampling.go @@ -2556,11 +2552,6 @@ xfails: directive: run case: deferfin.go reason: go1.25 goroot run failure on linux/amd64 - - version: go1.25 - platform: linux/amd64 - directive: run - case: finprofiled.go - reason: go1.25 goroot run failure on linux/amd64 - version: go1.25 platform: linux/amd64 directive: run @@ -2872,11 +2863,6 @@ xfails: directive: run case: deferfin.go reason: go1.24 goroot run failure on linux/amd64 - - version: go1.24 - platform: linux/amd64 - directive: run - case: finprofiled.go - reason: go1.24 goroot run failure on linux/amd64 - version: go1.24 platform: linux/amd64 directive: run @@ -3432,11 +3418,6 @@ xfails: directive: run case: devirtualization_nil_panics.go reason: go1.26 goroot run failure on linux/amd64 - - version: go1.26 - platform: linux/amd64 - directive: run - case: finprofiled.go - reason: go1.26 goroot run failure on linux/amd64 - version: go1.26 platform: linux/amd64 directive: run