From 3e7729fcca34d81161d1b0b4036d81b9f09b6531 Mon Sep 17 00:00:00 2001 From: mograph Date: Fri, 8 May 2026 15:01:57 -0400 Subject: [PATCH 1/4] feat: background color compositing for transparent windows + improved AI zoom suggestions for mobile - Add canvas compositing pipeline in useScreenRecorder to fill transparent window areas with a user-chosen color instead of encoding as black (H.264/VP8 have no alpha channel) - Add recording background color picker to HUD bar: preset swatches, transparent option, and custom color wheel via @uiw/react-color-colorful - Fix smoothedAutoFocusRef not writing back clamped position, causing auto-zoom trail to start outside the phone content area - Add detectClickCandidates() in zoomSuggestionUtils to generate zoom regions from click timestamps (picks up every mobile tap via uiohook-napi) - Lower MIN_DWELL_DURATION_MS from 450ms to 120ms so brief cursor pauses at click points are detected even without Accessibility permission - Use shorter fixed duration (1000ms) and pre-tap anchor (-100ms) for click-based zoom candidates so rapid taps don't produce overlapping regions - Remove inter-suggestion blocking in handleSuggestZooms so every tap gets a zoom region on the timeline regardless of adjacency - Pass cursorClickTimestamps from VideoEditor to TimelineEditor Co-Authored-By: Claude Sonnet 4.6 --- PR_DESCRIPTION.md | 70 ++++++++++ src/components/launch/LaunchWindow.tsx | 125 ++++++++++++++++++ src/components/video-editor/VideoEditor.tsx | 65 +++++++-- src/components/video-editor/VideoPlayback.tsx | 95 +++++++++++-- .../video-editor/timeline/TimelineEditor.tsx | 31 ++--- .../timeline/zoomSuggestionUtils.ts | 41 +++++- src/hooks/useScreenRecorder.ts | 49 ++++++- 7 files changed, 431 insertions(+), 45 deletions(-) create mode 100644 PR_DESCRIPTION.md diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 000000000..ab4029fb7 --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,70 @@ +# Pull Request Template + +## Description + +This PR adds two major features: + +1. **Recording background color compositing** — when recording a transparent window (e.g. iPhone Mirroring, iOS Simulator), the transparent areas are now filled with a user-chosen solid color via an off-screen canvas pipeline, instead of encoding as black. + +2. **Improved AI zoom suggestions for mobile recordings** — the "Suggest Zooms" wand now generates a zoom region for every tap/click during recording, not just long cursor dwells. Click timestamps and short cursor pauses (120ms+) are both used as candidates, and suggestions no longer block each other so all interactions appear on the timeline. + +## Motivation + +**Background compositing**: When recording a transparent Electron window (e.g. a mirrored iPhone in a frameless window), H.264/VP8 video codecs have no alpha channel — transparent areas encode as solid black. Users had no way to control or replace that background color. + +**AI zoom suggestions**: The previous dwell-detection algorithm required 450ms+ of cursor stillness, missing rapid navigation taps on mobile recordings. Suggestions were also silently dropped when they overlapped each other, causing most mobile interactions to be skipped entirely. + +## Type of Change +- [x] New Feature +- [x] Bug Fix +- [ ] Refactor / Code Cleanup +- [ ] Documentation Update +- [ ] Other (please specify) + +## Related Issue(s) + + +## Screenshots / Video + + + +## Testing + +**Background color picker:** +1. Launch a transparent window recording (iPhone Mirroring or iOS Simulator) +2. Before starting recording, click the colored circle in the HUD bar +3. Select a color from the presets or the custom color wheel +4. Record and export — the background should be the chosen color instead of black +5. Select the transparent (checkerboard) option — **known bug**: transparent areas still export as black because video codecs do not support alpha. This option is preserved in the UI for future codec support but does not currently work. + +**AI zoom suggestions (mobile):** +1. Record an iPhone Mirroring session with several quick tap navigations (~1s apart) +2. Open the recording in the editor +3. Click the magic wand "Suggest Zooms" button on the zoom timeline row +4. A zoom region should appear for each tap, even if they are adjacent or overlapping +5. Pre-existing manually-created zoom regions are respected and will not be overwritten + +## Known Bug + +**Transparent background does not work** — selecting the checkerboard/transparent swatch sets `captureBackgroundColor` to `null`, which bypasses canvas compositing and passes the raw video track to `MediaRecorder`. Because H.264 (and VP8/VP9) have no alpha channel, any transparent pixels in the source window are encoded as black. The transparent option is present in the UI but produces the same black result as having no compositing. A fix would require a codec that supports alpha (e.g. HEVC with alpha, or ProRes 4444), which is not currently supported by `MediaRecorder` in Chromium/Electron. + +## Checklist +- [x] I have performed a self-review of my code. +- [ ] I have added any necessary screenshots or videos. +- [ ] I have linked related issue(s) and updated the changelog if applicable. + +--- + +## Files Changed + +| File | Change | +|------|--------| +| `src/hooks/useScreenRecorder.ts` | Canvas compositing pipeline; `captureBackgroundColor` state | +| `src/components/launch/LaunchWindow.tsx` | HUD color picker (presets, transparent swatch, custom wheel) | +| `src/components/video-editor/VideoPlayback.tsx` | Write clamped auto-focus position back to `smoothedAutoFocusRef` | +| `src/components/video-editor/timeline/zoomSuggestionUtils.ts` | `detectClickCandidates()`; lowered `MIN_DWELL_DURATION_MS` 450→120ms; per-candidate `durationMs`/`startOffsetMs` | +| `src/components/video-editor/timeline/TimelineEditor.tsx` | Merge click+dwell candidates; removed inter-suggestion blocking; pass `cursorClickTimestamps` prop | +| `src/components/video-editor/VideoEditor.tsx` | Pass `cursorClickTimestamps` to `TimelineEditor` | + +--- +*Thank you for contributing!* diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index bffbd9c9a..2dede0410 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -28,6 +28,8 @@ import { requestCameraAccess } from "../../lib/requestCameraAccess"; import { formatTimePadded } from "../../utils/timeUtils"; import { AudioLevelMeter } from "../ui/audio-level-meter"; import { Button } from "../ui/button"; +import Colorful from "@uiw/react-color-colorful"; +import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover"; import { Tooltip } from "../ui/tooltip"; import styles from "./LaunchWindow.module.css"; @@ -106,8 +108,14 @@ export function LaunchWindow() { setWebcamEnabled, webcamDeviceId, setWebcamDeviceId, + captureBackgroundColor, + setCaptureBackgroundColor, } = useScreenRecorder(); + const [isBgPickerOpen, setIsBgPickerOpen] = useState(false); + const [showCustomWheel, setShowCustomWheel] = useState(false); + const BG_PRESETS = ["#000000", "#ffffff", "#1e90ff", "#00b140"] as const; + const showMicControls = microphoneEnabled && !recording; const showWebcamControls = webcamEnabled && !recording; @@ -541,6 +549,123 @@ export function LaunchWindow() { + {/* Background color for compositing */} + {!recording && ( + + +
+ +
+ + +
Recording Background
+
+ {/* Transparent swatch */} + + ); + })()} +
+ + {/* Inline color wheel */} + {showCustomWheel && ( +
+ setCaptureBackgroundColor(c.hex)} + disableAlpha={true} + style={{ borderRadius: "8px", width: "220px" }} + /> +
+ )} + + + )} + {/* Record/Stop group */} - ); - })} +
+
+ Zoom size + + {selectedZoomScale != null ? `${Math.round(selectedZoomScale * 100)}%` : "—"} + +
+ onZoomScaleChange?.(v)} + onValueCommit={() => onZoomScaleCommit?.()} + disabled={!zoomEnabled} + className="w-full [&_[role=slider]]:bg-[#34B27B] [&_[role=slider]]:border-[#34B27B] [&_[role=slider]]:h-3 [&_[role=slider]]:w-3" + /> +
+ Less + More +
+ {zoomEnabled && ( +
+
Zoom shape
+ +
+ )} {!zoomEnabled && (

{t("zoom.selectRegion")}

)} @@ -689,7 +707,6 @@ export function SettingsPanel({ )} - {zoomEnabled && ( + + + {t("links.reportBug")} - {onSaveDiagnostic && ( - - )}