fix(init): derive folded init-error reporting from invoke outcome and rapid's init duration

Reporting fixes for the on-demand folded-into-invoke (suppressed init) path:

- The invoke handler no longer unconditionally forces error status from the
  recorded init failure: the REPORT Status/Error Type lines and the /error
  routing now apply only when the invocation actually failed, and the
  recorded failure is cleared once an invocation succeeds. Previously one
  init failure permanently tainted a recovered environment - every later
  (successful) invocation was posted to /error with an error REPORT line.
  Persistent failures keep re-emitting the full failure envelope per invoke.

- RecordInitError captures the failure type detected by rapidcore for
  runtimes that crashed WITHOUT calling /init/error (sys.exit, segfault,
  invalid entrypoint), so those failures render the same
  INIT_REPORT(phase=invoke) and REPORT Status/Error Type lines as the
  /init/error-reported flavor instead of a healthy-looking envelope.

- REPORT's Init Duration now uses rapid's authoritative Init-phase
  measurement captured from the INIT_REPORT(phase=init) lifecycle event,
  instead of wall-clock time.Since(initStart) at invoke arrival, which
  wrongly included the idle gap between init completion and the first
  invoke (minutes for provisioned concurrency). It is also omitted for
  non-on-demand invokes, matching AWS, which reports provisioned
  concurrency init in separate provisioning-time log streams.

- Document that SnapStart environments intentionally use the on-demand
  error model in LocalStack (they initialize lazily at the first invoke,
  not at version publish).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: joe4dev

cmd/localstack/custom_interop.go

@@ -34,6 +34,9 @@ type CustomInteropServer struct {
 	// logCollector accumulates the runtime's stdout/stderr plus the synthetic START/REPORT/
 	// INIT_REPORT lines that are flushed to LocalStack with each invocation's logs.
 	logCollector *LogCollector
+	// eventsAPI provides rapid's authoritative Init-phase duration (see events.go), used for
+	// the REPORT/INIT_REPORT log lines instead of wall-clock measurements at invoke arrival.
+	eventsAPI *LocalStackEventsAPI
 	// initStart is set once in Init() and warmStart is flipped on the first invoke.
 	// Both are accessed only from the single sequential init -> invoke flow (the RIE
 	// processes one invocation at a time), so they need no additional synchronization.
@@ -51,12 +54,20 @@ type CustomInteropServer struct {
 	// initErrorType holds rapidcore's scrubbed fatal error type (e.g. Runtime.Unknown) when init
 	// failed, used to render the INIT_REPORT(phase=invoke) and REPORT Status/Error Type lines for
 	// the on-demand folded-into-invoke path. Stores a string; empty/unset means init did not fail.
+	// It persists while invocations keep failing (each one re-runs the init as a suppressed init
+	// and AWS re-emits the failure envelope), and is cleared by the invoke handler once an
+	// invocation succeeds so a recovered environment is not tainted by the original failure.
 	initErrorType atomic.Value
 	// onDemand is true for on-demand functions, where AWS folds a failed cold-start init into
 	// the first invocation (suppressed init). For these we do NOT report init failures via
 	// /status/error; instead we signal ready and let the first invoke surface the error with
-	// the full INIT_REPORT/START/END/REPORT envelope. Provisioned concurrency, SnapStart, and
-	// Managed Instances keep the provisioning-time /status/error model.
+	// the full INIT_REPORT/START/END/REPORT envelope. Provisioned concurrency and Managed
+	// Instances keep the provisioning-time /status/error model. SnapStart environments are
+	// also classified on-demand here (LocalStack sets AWS_LAMBDA_INITIALIZATION_TYPE=on-demand
+	// for them and initializes them lazily at the first invoke, not at version publish), so the
+	// fold-into-invoke model applies to them too.
+	// TODO: set AWS_LAMBDA_INITIALIZATION_TYPE=snap-start on the LocalStack side for env-var
+	// parity with AWS once SnapStart environments get their own initialization type.
 	onDemand bool
 }
 
@@ -121,13 +132,14 @@ func (l *LocalStackAdapter) SendResult(invokeId string, body []byte, isError boo
 	return nil
 }
 
-func NewCustomInteropServer(lsOpts *LsOpts, adapter *LocalStackAdapter, delegate interop.Server, logCollector *LogCollector) (server *CustomInteropServer) {
+func NewCustomInteropServer(lsOpts *LsOpts, adapter *LocalStackAdapter, delegate interop.Server, logCollector *LogCollector, eventsAPI *LocalStackEventsAPI) (server *CustomInteropServer) {
 	server = &CustomInteropServer{
 		delegate:          delegate.(*rapidcore.Server),
 		port:              lsOpts.InteropPort,
 		upstreamEndpoint:  lsOpts.RuntimeEndpoint,
 		localStackAdapter: adapter,
 		logCollector:      logCollector,
+		eventsAPI:         eventsAPI,
 		onDemand:          GetenvWithDefault("AWS_LAMBDA_INITIALIZATION_TYPE", "on-demand") == "on-demand",
 	}
 
@@ -153,17 +165,27 @@ func NewCustomInteropServer(lsOpts *LsOpts, adapter *LocalStackAdapter, delegate
 
 				initErrType, _ := server.initErrorType.Load().(string)
 
+				// First invocation into a successfully initialized on-demand environment: REPORT
+				// carries the Init phase duration as measured by rapid (init start -> init end).
+				// Provisioned concurrency / Managed Instances initialize at provisioning time and
+				// AWS omits Init Duration from their invokes' REPORT lines.
 				initDuration := ""
-				if !server.warmStart && !server.initTimedOut.Load() && initErrType == "" {
-					initTimeMS := float64(time.Since(server.initStart).Nanoseconds()) / float64(time.Millisecond)
-					initDuration = fmt.Sprintf("Init Duration: %.2f ms\t", initTimeMS)
+				if server.onDemand && !server.warmStart && !server.initTimedOut.Load() && initErrType == "" {
+					if initTimeMS, ok := server.eventsAPI.InitDurationMS(); ok {
+						initDuration = fmt.Sprintf("Init Duration: %.2f ms\t", initTimeMS)
+					}
 				}
 				server.warmStart = true
 
 				// On-demand init failure folded into this invocation (AWS suppressed init): emit
-				// the INIT_REPORT(phase=invoke) line before START (emitted during Invoke below).
+				// the INIT_REPORT(phase=invoke) line before START (emitted during Invoke below),
+				// reporting the failed init's duration (rapid's measurement when available; the
+				// wall-clock fallback covers inits that died before emitting INIT_REPORT).
 				if initErrType != "" {
-					initTimeMS := float64(time.Since(server.initStart).Nanoseconds()) / float64(time.Millisecond)
+					initTimeMS, ok := server.eventsAPI.InitDurationMS()
+					if !ok {
+						initTimeMS = float64(time.Since(server.initStart).Nanoseconds()) / float64(time.Millisecond)
+					}
 					_, _ = fmt.Fprintf(logCollector,
 						"INIT_REPORT Init Duration: %.2f ms\tPhase: invoke\tStatus: error\tError Type: %s\n",
 						initTimeMS, initErrType)
@@ -219,11 +241,19 @@ func NewCustomInteropServer(lsOpts *LsOpts, adapter *LocalStackAdapter, delegate
 						log.Fatalln(err)
 					}
 				}
-				// On-demand init failure folded into this invocation: the REPORT carries the
-				// failure status and rapidcore's scrubbed fatal error type (e.g. Runtime.Unknown).
+				// On-demand init failure folded into this invocation: when the suppressed init
+				// re-run (and thus the invoke) failed again, the REPORT carries the failure status
+				// and rapidcore's scrubbed fatal error type (e.g. Runtime.Unknown). When the
+				// invocation succeeded (the suppressed re-init recovered from a transient init
+				// failure), the result stands on its own — AWS reports it as successful — and the
+				// cached init failure is cleared so later invocations are not tainted by it.
 				if initErrType != "" {
-					isErr = true
-					status = "Status: error\tError Type: " + initErrType
+					if err != nil {
+						isErr = true
+						status = "Status: error\tError Type: " + initErrType
+					} else {
+						server.initErrorType.Store("")
+					}
 				}
 				// optional sleep. can be used for debugging purposes
 				if lsOpts.PostInvokeWaitMS != "" {
@@ -365,6 +395,21 @@ func (c *CustomInteropServer) SendInitError(errType fatalerror.ErrorType, errMsg
 	}
 }
 
+// RecordInitError records the structured init failure detected by rapidcore for runtimes that
+// failed WITHOUT calling /init/error (crash, sys.exit, invalid entrypoint), so the on-demand
+// folded-into-invoke path renders the same INIT_REPORT(phase=invoke) and REPORT Status/Error
+// Type lines as the /init/error-reported flavor. It must not overwrite a type already recorded
+// by SendInitErrorResponse: the runtime-reported error is the authoritative one.
+func (c *CustomInteropServer) RecordInitError(errType fatalerror.ErrorType) {
+	if recorded, _ := c.initErrorType.Load().(string); recorded != "" {
+		return
+	}
+	if errType == "" {
+		errType = fatalerror.RuntimeExit
+	}
+	c.initErrorType.Store(string(errType))
+}
+
 func (c *CustomInteropServer) GetCurrentInvokeID() string {
 	log.Traceln("GetCurrentInvokeID called")
 	return c.delegate.GetCurrentInvokeID()

cmd/localstack/events.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"fmt"
+	"sync/atomic"
 
 	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/interop"
 	"github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/rapidcore/standalone/telemetry"
+	lambdatelemetry "github.com/aws/aws-lambda-runtime-interface-emulator/internal/lambda/telemetry"
 )
 
 // LocalStackEventsAPI rides rapidcore's invoke lifecycle events to emit the synthetic START
@@ -15,6 +17,12 @@ import (
 type LocalStackEventsAPI struct {
 	*telemetry.StandaloneEventsAPI
 	logCollector *LogCollector
+	// initDurationMS holds rapid's authoritative measurement of the Init phase (init start ->
+	// init end, monotonic), captured from the INIT_REPORT(phase=init) lifecycle event emitted
+	// by doRuntimeDomainInit. The /invoke handler renders it as the first invocation's REPORT
+	// "Init Duration" instead of re-measuring at invoke arrival, which would wrongly include
+	// the idle gap between init completion and the first invoke dispatch.
+	initDurationMS atomic.Value // float64
 }
 
 func NewLocalStackEventsAPI(logCollector *LogCollector) *LocalStackEventsAPI {
@@ -28,3 +36,17 @@ func (e *LocalStackEventsAPI) SendInvokeStart(data interop.InvokeStartData) erro
 	_, _ = fmt.Fprintf(e.logCollector, "START RequestId: %s Version: %s\n", data.RequestID, data.Version)
 	return e.StandaloneEventsAPI.SendInvokeStart(data)
 }
+
+func (e *LocalStackEventsAPI) SendInitReport(data interop.InitReportData) error {
+	if data.Phase == lambdatelemetry.InitInsideInitPhase {
+		e.initDurationMS.Store(data.Metrics.DurationMs)
+	}
+	return e.StandaloneEventsAPI.SendInitReport(data)
+}
+
+// InitDurationMS returns rapid's measured duration of the startup Init phase and whether one
+// was recorded (false e.g. when the init phase timed out and never completed).
+func (e *LocalStackEventsAPI) InitDurationMS() (float64, bool) {
+	durationMS, ok := e.initDurationMS.Load().(float64)
+	return durationMS, ok
+}

cmd/localstack/main.go

@@ -238,7 +238,7 @@ func main() {
 	runDaemon(d) // async
 
 	defaultInterop := sandbox.DefaultInteropServer()
-	interopServer := NewCustomInteropServer(lsOpts, lsAdapter, defaultInterop, logCollector)
+	interopServer := NewCustomInteropServer(lsOpts, lsAdapter, defaultInterop, logCollector, lsEventsAPI)
 	sandbox.SetInteropServer(interopServer)
 	if len(handler) > 0 {
 		sandbox.SetHandler(handler)
@@ -323,7 +323,11 @@ func main() {
 		// invoke, which surfaces the cached init error (or a runtime-exit error) together with
 		// the full INIT_REPORT/START/END/REPORT log envelope. SendInitErrorResponse has already
 		// cached the structured error (without reporting via /status/error for on-demand).
+		// Record the failure type detected by rapidcore so runtimes that crashed WITHOUT
+		// calling /init/error still get the error envelope (no-op when /init/error already
+		// recorded the runtime-reported type).
 		log.Debugln("Init failed; deferring to first invocation (on-demand suppressed init).")
+		interopServer.RecordInitError(initResp.InitErrorType)
 	case err != nil:
 		// PC/SnapStart/MI, or an init-phase reset: report the failure now and exit. When the
 		// runtime reported its own error via /init/error, SendInitErrorResponse already