usertrap: disable syscall patching when ptraced

Workaround the issue in #12266.

FUTURE_COPYBARA_INTEGRATE_REVIEW=#12325 from trail-of-forks:ptrace-issue 3c52fed
PiperOrigin-RevId: 840404296

Author: ret2libc

pkg/sentry/platform/systrap/usertrap/usertrap_amd64.go

@@ -193,6 +193,22 @@ func (s *State) PatchSyscall(ctx context.Context, ac *arch.Context64, mm memoryM
 		return fmt.Errorf("no task found")
 	}
 
+	// Skip syscall patching when the task is being ptraced, because
+	// single-stepping and other debugger features are incompatible with
+	// the "syshandler" routine used to handle patched syscalls (see
+	// syshandler_amd64.S). This incompatibility can result in inconsistent
+	// process states and failures (e.g. SIGSEGV).
+	// TODO(gvisor.dev/issue/11649): for a full fix we'd need to roll back
+	//     existing patched syscalls, in case the traced program was patched
+	//     before being traced (e.g. PTRACE_ATTACH on an already running
+	//     process).
+	if task.Tracer() != nil {
+		if s.nextTrap > 0 {
+			ctx.Warningf("LIKELY ERROR: Attached tracer to process with patched syscalls (traps %d)! Systrap is not fully compatible with ptrace/debuggers, program may die unexpectedly soon!", s.nextTrap)
+		}
+		return nil
+	}
+
 	s.mu.Lock()
 	defer s.mu.Unlock()