From 0b622244e05beccb15bf54dd9c478d885ccb9d89 Mon Sep 17 00:00:00 2001 From: Charles Anim Date: Sun, 1 Nov 2026 20:18:37 -0500 Subject: [PATCH] feat: add native pointer interaction and drawing captures Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 177a4e89-321a-4dda-aa20-96bb2d75b334 --- .github/workflows/build.yml | 2 + Package.swift | 3 +- README.md | 52 +++- Sources/TinyHand/AppDelegate.swift | 36 +++ Sources/TinyHand/Capture.swift | 129 ++++++++++ Sources/TinyHand/E2ETest.swift | 170 +++++++++++++- Sources/TinyHand/Ink.swift | 18 ++ Sources/TinyHand/InputTap.swift | 133 +++++++---- Sources/TinyHand/OverlayController.swift | 287 ++++++++++++++++++++--- Sources/TinyHand/SettingsView.swift | 8 +- Sources/TinyHand/Toolbar.swift | 52 ++++ Sources/TinyHand/WelcomeWindow.swift | 2 +- Sources/TinyHand/main.swift | 7 +- Tests/TinyHandTests/CaptureTests.swift | 76 ++++++ Tests/TinyHandTests/InputTapTests.swift | 85 +++++++ scripts/capture-evidence.py | 62 +++++ scripts/capture-fixture.swift | 37 +++ 17 files changed, 1060 insertions(+), 99 deletions(-) create mode 100644 Sources/TinyHand/Capture.swift create mode 100644 Tests/TinyHandTests/CaptureTests.swift create mode 100644 Tests/TinyHandTests/InputTapTests.swift create mode 100644 scripts/capture-evidence.py create mode 100644 scripts/capture-fixture.swift diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1cd008..8665428 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,5 +11,7 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + - name: Test input routing and image export + run: swift test - name: Build (universal) run: swift build -c release --arch arm64 --arch x86_64 diff --git a/Package.swift b/Package.swift index c536a5e..eaf0acd 100644 --- a/Package.swift +++ b/Package.swift @@ -13,6 +13,7 @@ let package = Package( dependencies: [.product(name: "Sparkle", package: "Sparkle")], path: "Sources/TinyHand", linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@executable_path/../Frameworks"])] - ) + ), + .testTarget(name: "TinyHandTests", dependencies: ["TinyHand"]) ] ) diff --git a/README.md b/README.md index e1f8647..acadb59 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,10 @@ macOS 13 or later, Apple Silicon and Intel. ## Install -Open `TinyHand.dmg`, drag TinyHand to Applications and allow **Accessibility** when asked. It needs that to turn your clicks into taps. TinyHand updates itself. +Open `TinyHand.dmg`, drag TinyHand to Applications and allow **Accessibility** when asked. It needs that to animate clicks and draw. Screenshot mode also asks for **Screen Recording** permission when you first save a capture. + +This fork adds drawing export, native pointer interaction, and freeform screenshots. +The download links above still refer to the upstream release, not these fork features. ## Use @@ -25,14 +28,57 @@ Open `TinyHand.dmg`, drag TinyHand to Applications and allow **Accessibility** w |---|---| | Click the menu bar icon, or **⌃⌥H** | Turn the pointer on or off | | Right-click the menu bar icon | Menu and Settings | -| Click | Tap (nothing underneath gets clicked) | -| Hold and drag | Draw on the screen | +| **Pointer** mode (default) | Click, double-click, right-click, and drag in macOS normally | +| **Draw** mode | Hold and drag to draw without clicking anything underneath | +| **Screenshot** mode | Drag a freeform shape, release, then click **Save** | +| Save button / **Save Drawing as PNG…** in the menu | Export your strokes to a transparent PNG | +| Hold **⌥ Option** in Draw or Screenshot mode | Temporarily interact with the app underneath (configurable in Settings) | | **⌃⌥X** | Clear the drawing | | **Esc** | Clear everything and turn the pointer off | While the pointer is on, a small toolbar at the bottom of the screen lets you pick one of 36 hands or a laser dot, mute the tap sound, or exit. +### Save drawings and screenshots + +In **Draw** mode, the save button exports all strokes since the last clear, including +lines that have faded. The PNG is cropped to the artwork with a transparent background; +the desktop and pointer are not included. Strokes on multiple displays retain their +desktop-relative positions. Clear, Esc, and turning the pointer off discard the drawing. + +In **Screenshot** mode, draw around the content you want. The outline closes +automatically; click the save button or **Save Selected Area as PNG…** in the menu. +TinyHand captures the current screen contents inside the shape, cropped to its bounds, +with transparency outside it. The pointer, selection outline, toolbar, and drawing +overlays are excluded. A selection stays on the display where it started; draw again to +replace it, or clear it to select on another display. + +Both actions open a standard save dialog so you choose the filename and destination. +Cancelling leaves your drawing or selection available to save again. If macOS denies +screen access, allow TinyHand in **System Settings → Privacy & Security → Screen +Recording**, then quit and reopen it if prompted. + +### Development + +```sh +swift build +swift test +swift run TinyHand --e2e-test +``` + +The E2E check needs Accessibility permission and a logged-in macOS desktop. +To also verify real PNG saves and record the flow over a privacy-safe native fixture, +grant Screen Recording permission and run `python3 scripts/capture-evidence.py dist/evidence`. +The output directory must not already exist. The fixture covers the primary display +during the recording and is closed when the check finishes. + +If SwiftPM reports `safe.bareRepository is 'explicit'`, scope the workaround to the +build command rather than changing your global Git settings: + +```sh +GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.bareRepository GIT_CONFIG_VALUE_0=all swift test +``` + ## License MIT. See [LICENSE](LICENSE). diff --git a/Sources/TinyHand/AppDelegate.swift b/Sources/TinyHand/AppDelegate.swift index aa6dd5f..1a70c3e 100644 --- a/Sources/TinyHand/AppDelegate.swift +++ b/Sources/TinyHand/AppDelegate.swift @@ -13,6 +13,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { private var toggleItem: NSMenuItem? private var clearItem: NSMenuItem? private var soundItem: NSMenuItem? + private var saveDrawingItem: NSMenuItem? + private var saveScreenshotItem: NSMenuItem? + private var modeItems: [NSMenuItem] = [] private var toggleHotKey: UInt32? private var clearHotKey: UInt32? @@ -27,6 +30,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { overlay.onSoundChanged = { [weak self] in self?.refreshMenu() } + overlay.onCaptureChanged = { [weak self] in self?.refreshMenu() } overlay.extraPassThrough = { [weak self] in guard let frame = self?.statusItem.button?.window?.frame else { return [] @@ -71,6 +75,20 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { clearItem = clear soundItem = sound menu.addItem(toggle) + menu.autoenablesItems = false + for (index, mode) in PointerMode.allCases.enumerated() { + let item = NSMenuItem(title: "\(mode.title) Mode", action: #selector(selectMode(_:)), keyEquivalent: "") + item.tag = index + item.image = NSImage(systemSymbolName: mode.symbol, accessibilityDescription: mode.title) + modeItems.append(item) + menu.addItem(item) + } + let saveDrawing = NSMenuItem(title: "Save Drawing as PNG…", action: #selector(saveDrawing), keyEquivalent: "") + let saveScreenshot = NSMenuItem(title: "Save Selected Area as PNG…", action: #selector(saveScreenshot), keyEquivalent: "") + saveDrawingItem = saveDrawing + saveScreenshotItem = saveScreenshot + menu.addItem(saveDrawing) + menu.addItem(saveScreenshot) menu.addItem(clear) menu.addItem(sound) menu.addItem(.separator()) @@ -127,6 +145,11 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { toggleItem?.title = "\(on ? "Turn Pointer Off" : "Turn Pointer On") \(Preferences.toggleShortcut.display)" clearItem?.title = "Clear Drawing \(Preferences.clearShortcut.display)" soundItem?.state = overlay.soundOn ? .on : .off + saveDrawingItem?.isEnabled = overlay.canSaveDrawing + saveScreenshotItem?.isEnabled = overlay.canSaveScreenshot + for item in modeItems { + item.state = PointerMode.allCases[item.tag] == overlay.mode ? .on : .off + } permissionItem?.isHidden = AXIsProcessTrusted() } @@ -187,6 +210,19 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { overlay.clearInk() } + @objc private func selectMode(_ sender: NSMenuItem) { + overlay.setMode(PointerMode.allCases[sender.tag]) + if !overlay.isActive { overlay.activate() } + } + + @objc private func saveDrawing() { + overlay.saveDrawing() + } + + @objc private func saveScreenshot() { + overlay.saveScreenshot() + } + @objc private func toggleSound() { overlay.soundOn.toggle() } diff --git a/Sources/TinyHand/Capture.swift b/Sources/TinyHand/Capture.swift new file mode 100644 index 0000000..91f2ffc --- /dev/null +++ b/Sources/TinyHand/Capture.swift @@ -0,0 +1,129 @@ +import AppKit +import ScreenCaptureKit + +enum CaptureError: LocalizedError { + case emptyDrawing, emptySelection, imageCreation, displayUnavailable, permission, captureFailed(String) + + var errorDescription: String? { + switch self { + case .emptyDrawing: return "Draw something first, then choose Save Drawing." + case .emptySelection: return "Draw a shape with an area at least 4 points wide and tall." + case .imageCreation: return "The PNG could not be created. Try a smaller drawing or selection." + case .displayUnavailable: return "The selected display is no longer available." + case .permission: return "Allow TinyHand in System Settings > Privacy & Security > Screen Recording, then try again. macOS may require you to quit and reopen TinyHand." + case .captureFailed(let detail): return "Screen capture failed: \(detail)" + } + } +} + +enum Capture { + static func drawing(_ strokes: [Ink.Stroke], scale: CGFloat) throws -> CGImage { + guard !strokes.isEmpty else { throw CaptureError.emptyDrawing } + let bounds = strokes.reduce(CGRect.null) { + $0.union($1.path.boundingBoxOfPath.insetBy(dx: -$1.width / 2 - 2, dy: -$1.width / 2 - 2)) + }.integral + let context = try bitmap(bounds: bounds, scale: scale) + context.setLineCap(.round) + context.setLineJoin(.round) + for stroke in strokes { + context.setStrokeColor(stroke.color) + context.setLineWidth(stroke.width) + context.addPath(stroke.path) + context.strokePath() + } + guard let image = context.makeImage() else { throw CaptureError.imageCreation } + return image + } + + static func selection(_ points: [CGPoint]) throws -> CGPath { + guard points.count >= 3, let first = points.first else { throw CaptureError.emptySelection } + let path = CGMutablePath() + path.addLines(between: points) + path.closeSubpath() + let bounds = path.boundingBoxOfPath + let hasArea = zip(points.dropFirst(), points.dropFirst(2)).contains { pair in + let a = pair.0, b = pair.1 + let left: CGFloat = (a.x - first.x) * (b.y - first.y) + let right: CGFloat = (a.y - first.y) * (b.x - first.x) + return abs(left - right) >= 4 + } + guard bounds.width >= 4, bounds.height >= 4, hasArea else { throw CaptureError.emptySelection } + return path + } + + static func screenshot(_ image: CGImage, path: CGPath, screenSize: CGSize) throws -> CGImage { + let screen = CGRect(origin: .zero, size: screenSize) + let bounds = path.boundingBoxOfPath.integral.intersection(screen) + guard !bounds.isNull, bounds.width >= 4, bounds.height >= 4 else { throw CaptureError.emptySelection } + let context = try bitmap(bounds: bounds, scale: CGFloat(image.width) / screenSize.width) + context.addPath(path) + context.clip(using: .evenOdd) + context.draw(image, in: screen) + guard let result = context.makeImage() else { throw CaptureError.imageCreation } + return result + } + + private static func bitmap(bounds: CGRect, scale: CGFloat) throws -> CGContext { + let width = ceil(bounds.width * scale), height = ceil(bounds.height * scale) + guard scale.isFinite, scale > 0, width.isFinite, height.isFinite, + width > 0, height > 0, width * height <= 100_000_000, + let context = CGContext(data: nil, width: Int(width), height: Int(height), + bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { + throw CaptureError.imageCreation + } + context.scaleBy(x: scale, y: scale) + context.translateBy(x: -bounds.minX, y: -bounds.minY) + return context + } + + static func png(_ image: CGImage) throws -> Data { + guard let data = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) else { + throw CaptureError.imageCreation + } + return data + } + + static func screen(id: CGDirectDisplayID, frame: CGRect) async throws -> CGImage { + if #available(macOS 14.0, *) { + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + guard let display = content.displays.first(where: { $0.displayID == id }) else { + throw CaptureError.displayUnavailable + } + let apps = content.applications.filter { $0.processID == getpid() } + let filter = SCContentFilter(display: display, excludingApplications: apps, exceptingWindows: []) + let config = SCStreamConfiguration() + config.width = Int(CGFloat(display.width) * CGFloat(filter.pointPixelScale)) + config.height = Int(CGFloat(display.height) * CGFloat(filter.pointPixelScale)) + config.showsCursor = false + return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + } + // macOS 13 predates SCScreenshotManager. The system tool also omits the cursor. + return try await withCheckedThrowingContinuation { continuation in + DispatchQueue.global(qos: .userInitiated).async { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("tinyhand-\(UUID().uuidString).png") + defer { + if FileManager.default.fileExists(atPath: url.path) { + do { try FileManager.default.removeItem(at: url) } + catch { Log.write("temporary screenshot cleanup failed: \(error.localizedDescription)") } + } + } + do { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture") + process.arguments = ["-x", "-R\(Int(frame.minX)),\(Int(frame.minY)),\(Int(frame.width)),\(Int(frame.height))", url.path] + try process.run() + process.waitUntilExit() + guard process.terminationStatus == 0, + let image = NSBitmapImageRep(data: try Data(contentsOf: url))?.cgImage else { + throw CaptureError.captureFailed("screencapture exited with status \(process.terminationStatus)") + } + continuation.resume(returning: image) + } catch { + continuation.resume(throwing: error) + } + } + } + } +} diff --git a/Sources/TinyHand/E2ETest.swift b/Sources/TinyHand/E2ETest.swift index b40e29f..82bec55 100644 --- a/Sources/TinyHand/E2ETest.swift +++ b/Sources/TinyHand/E2ETest.swift @@ -3,10 +3,15 @@ import AppKit enum E2ETest { private final class CatcherView: NSView { var clicks = 0 + var drags = 0 + var rightClicks = 0 var scrolls = 0 override func mouseDown(with event: NSEvent) { clicks += 1 } + override func mouseDragged(with event: NSEvent) { drags += 1 } + override func rightMouseDown(with event: NSEvent) { rightClicks += 1 } + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } override func scrollWheel(with event: NSEvent) { scrolls += 1 } @@ -16,7 +21,7 @@ enum E2ETest { } } - static func run() { + static func run() -> Int32 { setlinebuf(stdout) let app = NSApplication.shared app.setActivationPolicy(.accessory) @@ -24,8 +29,18 @@ enum E2ETest { guard let screen = NSScreen.screens.first else { print("no screen") - return + return 1 } + guard AXIsProcessTrusted() else { + print("FAIL: Accessibility permission is required for the live E2E test") + return 1 + } + var failures = 0 + func check(_ value: Bool, _ message: String) { + print("\(value ? "PASS" : "FAIL"): \(message)") + if !value { failures += 1 } + } + let output = ProcessInfo.processInfo.environment["TH_CAPTURE_OUTPUT"].map { URL(fileURLWithPath: $0, isDirectory: true) } let size = CGSize(width: 520, height: 360) let frame = CGRect(x: screen.frame.midX - size.width / 2, y: screen.frame.midY - size.height / 2, width: size.width, height: size.height) @@ -48,7 +63,8 @@ enum E2ETest { } } func post(_ type: CGEventType, _ p: CGPoint) { - CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p, mouseButton: .left)? + let button: CGMouseButton = type == .rightMouseDown || type == .rightMouseUp ? .right : .left + CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: p, mouseButton: button)? .post(tap: .cghidEventTap) } func state(_ label: String, _ overlay: OverlayController) { @@ -57,16 +73,83 @@ enum E2ETest { print(String(format: "%-14@ mouse=(%.0f,%.0f) hidDown=%@ %@", label as NSString, m.x, m.y, down ? "yes" : "no", overlay.debugState)) } + func savePanel(to directory: URL) { + guard let panel = app.windows.compactMap({ $0 as? NSSavePanel }).first(where: { $0.isVisible }) else { + check(false, "capture opens a save dialog") + return + } + pump(0.5) + let destination = directory.appendingPathComponent(panel.nameFieldStringValue) + guard panel.directoryURL?.standardizedFileURL == directory.standardizedFileURL, + destination.deletingLastPathComponent().standardizedFileURL == directory.standardizedFileURL, + !FileManager.default.fileExists(atPath: destination.path) else { + check(false, "save destination is the empty test output directory") + panel.cancel(nil) + return + } + guard panel.isKeyWindow else { + check(false, "the save dialog owns keyboard focus") + panel.cancel(nil) + return + } + // Modern macOS hosts the save UI out of process; its public ok: is unavailable. + for down in [true, false] { + let key = CGEvent(keyboardEventSource: source, virtualKey: 0x24, keyDown: down) + key?.flags = [] + key?.post(tap: .cghidEventTap) + } + pump(1) + check(FileManager.default.fileExists(atPath: destination.path), "native save dialog writes \(destination.lastPathComponent)") + if panel.isVisible { panel.cancel(nil) } + } let savedStyle = PointerStyle.saved PointerStyle.saved = .lavender - let overlay = OverlayController() + let overlay = OverlayController(makeSavePanel: { + let panel = NSSavePanel() + if let output = output { panel.directoryURL = output } + return panel + }) + func selectMode(_ mode: PointerMode) { + guard let center = overlay.toolbarButtonCenter(mode) else { + check(false, "\(mode.title) control is available") + return + } + let p = CGPoint(x: center.x, y: screen.frame.height - center.y) + post(.mouseMoved, p) + pump(0.3) + post(.leftMouseDown, p) + post(.leftMouseUp, p) + pump(0.2) + check(overlay.mode == mode, "clicking the toolbar selects \(mode.title) mode") + post(.mouseMoved, start) + pump(0.1) + } let installed = overlay.activate() print("tap installed:", installed) + guard installed else { + overlay.deactivate() + PointerStyle.saved = savedStyle + catcher.orderOut(nil) + return 1 + } pump(0.3) post(.mouseMoved, start) pump(0.2) + post(.leftMouseDown, start) + pump(0.05) + post(.leftMouseDragged, CGPoint(x: start.x + 10, y: start.y)) + pump(0.05) + post(.leftMouseUp, start) + post(.rightMouseDown, start) + post(.rightMouseUp, start) + pump(0.2) + check(catcherView.clicks == 1 && catcherView.drags > 0 && catcherView.rightClicks == 1, + "Pointer delivers native left clicks, drags, and right clicks") + check(!overlay.canSaveDrawing, "Pointer gestures do not draw") + catcherView.clicks = 0 + selectMode(.draw) state("before press", overlay) post(.leftMouseDown, start) @@ -91,14 +174,30 @@ enum E2ETest { } pump(0.1) print("scroll ticks right after drawing that reached the app (of 5):", catcherView.scrolls) + check(catcherView.scrolls == 5, "scrolling still reaches the app") + check(overlay.canSaveDrawing, "completed drawing is exportable") - let before = overlay.inkStrokes + overlay.saveDrawing() + pump(0.5) + let drawingPanel = app.windows.compactMap { $0 as? NSSavePanel }.first { $0.isVisible } + check(drawingPanel != nil, "Save Drawing opens a native save dialog") + drawingPanel?.cancel(nil) + pump(0.2) + check(overlay.canSaveDrawing && overlay.isActive, "cancelling Save preserves the drawing and pointer") + if let output = output { + overlay.saveDrawing() + pump(0.5) + savePanel(to: output) + } + + let before = overlay.savedInkStrokes post(.leftMouseDown, start) pump(0.05) post(.leftMouseUp, start) pump(0.2) state("plain click", overlay) - print("plain click added strokes:", overlay.inkStrokes - before) + print("plain click added strokes:", overlay.savedInkStrokes - before) + check(overlay.savedInkStrokes == before, "a drawing-mode tap does not create a stroke") let primaryH = NSScreen.screens.first?.frame.height ?? 0 if let c = overlay.toolbarButtonCenter(.laser) { @@ -111,6 +210,8 @@ enum E2ETest { } print("style after clicking the toolbar's laser button:", overlay.style.rawValue) print("clicks that leaked to the catcher window:", catcherView.clicks) + check(overlay.style == .laser, "toolbar remains clickable in Draw mode") + check(catcherView.clicks == 0, "Draw mode does not click the underlying app") let clickPoint = start overlay.resetTapWait() @@ -134,6 +235,61 @@ enum E2ETest { } print(String(format: "longest a click waited to reach the tap during the stall: %.1f ms", overlay.tapMaxWaitMs)) print("clicks that leaked during the stall:", catcherView.clicks) + check(catcherView.clicks == 0, "Draw mode consumes clicks while the main thread stalls") + + selectMode(.screenshot) + post(.leftMouseDown, start) + pump(0.05) + for p in [CGPoint(x: start.x + 160, y: start.y), + CGPoint(x: start.x + 80, y: start.y + 100), start] { + post(.leftMouseDragged, p) + pump(0.05) + } + post(.leftMouseUp, start) + pump(0.2) + check(overlay.canSaveScreenshot, "a freeform screen selection is ready to capture") + check(catcherView.clicks == 0, "Screenshot mode does not click the underlying app") + if let output = output { + if CGPreflightScreenCaptureAccess() { + overlay.saveScreenshot() + let deadline = Date().addingTimeInterval(10) + while Date() < deadline && !app.windows.contains(where: { $0 is NSSavePanel && $0.isVisible }) { + pump(0.1) + } + if let panel = app.windows.compactMap({ $0 as? NSSavePanel }).first(where: { $0.isVisible }) { + panel.cancel(nil) + pump(0.2) + check(overlay.canSaveScreenshot && overlay.isActive, "cancelling screenshot Save preserves the selection") + overlay.saveScreenshot() + let deadline = Date().addingTimeInterval(10) + while Date() < deadline && !app.windows.contains(where: { $0 is NSSavePanel && $0.isVisible }) { + pump(0.1) + } + savePanel(to: output) + let url = output.appendingPathComponent("TinyHand Screenshot.png") + do { + let data = try Data(contentsOf: url) + if let rep = NSBitmapImageRep(data: data), + let inside = rep.colorAt(x: rep.pixelsWide / 2, y: rep.pixelsHigh / 4)?.usingColorSpace(.deviceRGB), + let outside = rep.colorAt(x: 0, y: rep.pixelsHigh - 2) { + check(outside.alphaComponent == 0, "the saved freeform screenshot is transparent outside the shape") + check(abs(inside.redComponent - 0.13) < 0.1 && abs(inside.blueComponent - 0.31) < 0.1, + "screen capture contains the separate fixture, not TinyHand overlays") + } else { + check(false, "saved screenshot can be decoded") + } + } catch { + check(false, "saved screenshot can be read: \(error.localizedDescription)") + } + } else { + check(false, "ScreenCaptureKit completes and opens Save") + } + } else { + check(false, "Screen Recording permission is required for capture evidence") + } + } + overlay.clearInk() + check(!overlay.canSaveScreenshot && !overlay.canSaveDrawing, "Clear discards both drawing and selection") for down in [true, false] { CGEvent(keyboardEventSource: source, virtualKey: 0x35, keyDown: down)?.post(tap: .cghidEventTap) @@ -141,11 +297,13 @@ enum E2ETest { } pump(0.2) print("active after Esc:", overlay.isActive) + check(!overlay.isActive, "Escape turns off the pointer") overlay.deactivate() PointerStyle.saved = savedStyle catcher.orderOut(nil) CGWarpMouseCursorPosition(saved) pump(0.1) + return failures == 0 ? 0 : 1 } } diff --git a/Sources/TinyHand/Ink.swift b/Sources/TinyHand/Ink.swift index edfcb28..6c2ffa7 100644 --- a/Sources/TinyHand/Ink.swift +++ b/Sources/TinyHand/Ink.swift @@ -1,6 +1,14 @@ import AppKit final class Ink { + struct Stroke { + let path: CGPath + let color: CGColor + let width: CGFloat + } + + private(set) var savedStrokes: [Stroke] = [] + var origin = CGPoint.zero let layer = CALayer() var keep: Bool { Preferences.fadeSeconds <= 0 } @@ -57,6 +65,11 @@ final class Ink { guard let s = stroke else { return } + if let path = s.path, let color = s.strokeColor { + let globalPath = CGMutablePath() + globalPath.addPath(path, transform: CGAffineTransform(translationX: origin.x, y: origin.y)) + savedStrokes.append(Stroke(path: globalPath, color: color, width: s.lineWidth)) + } stroke = nil points = [] freeze(s) @@ -67,6 +80,11 @@ final class Ink { } func clear() { + savedStrokes.removeAll() + clearVisible() + } + + func clearVisible() { stroke = nil points = [] layer.sublayers?.forEach { $0.removeFromSuperlayer() } diff --git a/Sources/TinyHand/InputTap.swift b/Sources/TinyHand/InputTap.swift index 8114e4b..4d05558 100644 --- a/Sources/TinyHand/InputTap.swift +++ b/Sources/TinyHand/InputTap.swift @@ -1,18 +1,47 @@ import AppKit +enum PointerMode: String, CaseIterable { + case pointer, draw, screenshot + + var title: String { + switch self { + case .pointer: return "Pointer" + case .draw: return "Draw" + case .screenshot: return "Screenshot" + } + } + + var symbol: String { + switch self { + case .pointer: return "cursorarrow" + case .draw: return "pencil.tip" + case .screenshot: return "viewfinder" + } + } +} + final class InputTap { - var onMouseDown: (() -> Void)? + var onMouseDown: ((CGPoint) -> Void)? var onInteractDown: (() -> Void)? var onInteractUp: (() -> Void)? - var onMouseUp: (() -> Void)? - var onLeftDragged: (() -> Void)? + var onMouseUp: ((CGPoint) -> Void)? + var onLeftDragged: ((CGPoint) -> Void)? var onPassThroughPress: ((CGPoint) -> Void)? private let lock = NSLock() private var _active = false private var _passThroughRects: [CGRect] = [] private var _interactKey: CGEventFlags = [] - private var _interactPress = false + private var _mode = PointerMode.pointer + private enum Route { + case native(CGEventFlags), controls, drawing + } + private var presses: [Int64: Route] = [:] + + var mode: PointerMode { + get { lock.withLock { _mode } } + set { lock.withLock { _mode = newValue } } + } var passThroughRects: [CGRect] { get { lock.withLock { _passThroughRects } } @@ -29,10 +58,14 @@ final class InputTap { } var interactPressing: Bool { - lock.withLock { _interactPress } + lock.withLock { + presses.values.contains { + if case .native(let flags) = $0 { return !flags.isEmpty } + return false + } + } } - private var passingThrough = false private(set) var maxWaitMs: Double = 0 private var tap: CFMachPort? private var thread: Thread? @@ -45,16 +78,12 @@ final class InputTap { return true } - let mask: CGEventMask = - (1 << CGEventType.leftMouseDown.rawValue) | - (1 << CGEventType.leftMouseUp.rawValue) | - (1 << CGEventType.rightMouseDown.rawValue) | - (1 << CGEventType.rightMouseUp.rawValue) | - (1 << CGEventType.otherMouseDown.rawValue) | - (1 << CGEventType.otherMouseUp.rawValue) | - (1 << CGEventType.leftMouseDragged.rawValue) | - (1 << CGEventType.rightMouseDragged.rawValue) | - (1 << CGEventType.otherMouseDragged.rawValue) + let types: [CGEventType] = [ + .leftMouseDown, .leftMouseUp, .rightMouseDown, .rightMouseUp, + .otherMouseDown, .otherMouseUp, .leftMouseDragged, + .rightMouseDragged, .otherMouseDragged, + ] + let mask = types.reduce(CGEventMask(0)) { $0 | (CGEventMask(1) << $1.rawValue) } let refcon = Unmanaged.passUnretained(self).toOpaque() guard let tap = CGEvent.tapCreate( @@ -96,7 +125,7 @@ final class InputTap { func setActive(_ value: Bool) { lock.withLock { _active = value - _interactPress = false + presses.removeAll() } guard let tap = tap else { return @@ -118,7 +147,7 @@ final class InputTap { maxWaitMs = 0 } - private func handle(type: CGEventType, event: CGEvent) -> Unmanaged? { + func handle(type: CGEventType, event: CGEvent) -> Unmanaged? { if event.timestamp > 0 { let wait = Double(DispatchTime.now().uptimeNanoseconds &- event.timestamp) / 1_000_000 if wait < 600_000 { @@ -139,55 +168,61 @@ final class InputTap { return Unmanaged.passUnretained(event) } - let key = interactKey - switch type { - case .leftMouseDown, .rightMouseDown, .otherMouseDown: - if !key.isEmpty && (event.flags.contains(key) || CGEventSource.flagsState(.hidSystemState).contains(key)) { - lock.withLock { _interactPress = true } - DispatchQueue.main.async { [weak self] in self?.onInteractDown?() } + let isDown = [.leftMouseDown, .rightMouseDown, .otherMouseDown].contains(type) + let isUp = [.leftMouseUp, .rightMouseUp, .otherMouseUp].contains(type) + let button = event.getIntegerValueField(.mouseEventButtonNumber) + let location = event.location + // Latch the route and modifier on mouse-down, independently for each button. + let route: Route = lock.withLock { + if isDown { + if _passThroughRects.contains(where: { $0.contains(location) }) { + presses[button] = .controls + } else if _mode == .pointer { + presses[button] = .native([]) + } else if !_interactKey.isEmpty && event.flags.contains(_interactKey) { + presses[button] = .native(_interactKey) + } else { + presses[button] = .drawing + } } - default: - break + let route = presses[button] ?? .native([]) + if isUp { presses[button] = nil } + return route } - if lock.withLock({ _interactPress }) { - event.flags.remove(key) - if type == .leftMouseUp || type == .rightMouseUp || type == .otherMouseUp { - lock.withLock { _interactPress = false } + + switch route { + case .native(let modifier): + event.flags.remove(modifier) + if isDown { + DispatchQueue.main.async { [weak self] in self?.onInteractDown?() } + } else if isUp { DispatchQueue.main.async { [weak self] in self?.onInteractUp?() } } return Unmanaged.passUnretained(event) - } - - switch type { - case .leftMouseDown, .rightMouseDown, .otherMouseDown: - passingThrough = passThroughRects.contains { $0.contains(event.location) } - if passingThrough { - let location = event.location + case .controls: + if isDown { DispatchQueue.main.async { [weak self] in self?.onPassThroughPress?(location) } } - default: - break - } - if passingThrough { - if type == .leftMouseUp || type == .rightMouseUp || type == .otherMouseUp { - passingThrough = false - } return Unmanaged.passUnretained(event) + case .drawing: + break } switch type { - case .leftMouseDown, .rightMouseDown, .otherMouseDown: - DispatchQueue.main.async { [weak self] in self?.onMouseDown?() } + case .leftMouseDown: + DispatchQueue.main.async { [weak self] in self?.onMouseDown?(location) } return nil - case .leftMouseUp, .rightMouseUp, .otherMouseUp: - DispatchQueue.main.async { [weak self] in self?.onMouseUp?() } + case .leftMouseUp: + DispatchQueue.main.async { [weak self] in self?.onMouseUp?(location) } return nil case .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: if type == .leftMouseDragged { - DispatchQueue.main.async { [weak self] in self?.onLeftDragged?() } + DispatchQueue.main.async { [weak self] in self?.onLeftDragged?(location) } } event.type = .mouseMoved return Unmanaged.passUnretained(event) + case .rightMouseDown, .rightMouseUp, .otherMouseDown, .otherMouseUp: + return nil default: return Unmanaged.passUnretained(event) } diff --git a/Sources/TinyHand/OverlayController.swift b/Sources/TinyHand/OverlayController.swift index 4a60e95..8416b8c 100644 --- a/Sources/TinyHand/OverlayController.swift +++ b/Sources/TinyHand/OverlayController.swift @@ -1,5 +1,6 @@ import AppKit import Carbon.HIToolbox +import UniformTypeIdentifiers final class OverlayController { private final class OverlayWindow: NSWindow { @@ -19,6 +20,14 @@ final class OverlayController { private var downPoint: CGPoint? private var buttonSeenDown = false private var dragEvents = 0 + private(set) var mode = PointerMode.pointer + private let selectionLayer = CAShapeLayer() + private var selectionPoints: [CGPoint] = [] + private var selectionPath: CGPath? + private var exporting = false + private var savePanel: NSSavePanel? + private let makeSavePanel: () -> NSSavePanel + private var captureTask: Task? private let toolbar = Toolbar() private var revealUntil = Date.distantPast @@ -46,7 +55,8 @@ final class OverlayController { var onActiveChanged: (() -> Void)? var onNeedsPermission: (() -> Void)? - init() { + init(makeSavePanel: @escaping () -> NSSavePanel = { NSSavePanel() }) { + self.makeSavePanel = makeSavePanel let frame = (NSScreen.main ?? NSScreen.screens.first).map { $0.frame } ?? NSRect(x: 0, y: 0, width: 1440, height: 900) @@ -61,39 +71,55 @@ final class OverlayController { armView = ArmView(frame: NSRect(origin: .zero, size: frame.size)) window.contentView = armView - - inputTap.onMouseDown = { [weak self] in - guard let self = self else { + selectionLayer.strokeColor = NSColor.systemBlue.cgColor + selectionLayer.fillColor = NSColor.systemBlue.withAlphaComponent(0.12).cgColor + selectionLayer.lineWidth = 2 + selectionLayer.lineDashPattern = [6, 4] + selectionLayer.fillRule = .evenOdd + armView.layer?.addSublayer(selectionLayer) + + inputTap.onMouseDown = { [weak self] location in + guard let self = self, self.isActive, !self.exporting else { return } self.armView.press() self.armView.held = true self.tapSound.play() self.toolbar.hideHint() - self.downPoint = self.localMouse() + self.downPoint = self.localPoint(location) + if self.mode == .screenshot { + self.clearSelection() + self.selectionPoints = self.downPoint.map { [$0] } ?? [] + } self.dragEvents = 0 Log.write("press at \(self.downPoint.map { "\(Int($0.x)),\(Int($0.y))" } ?? "?")") self.noteButtonEvent() } - inputTap.onLeftDragged = { [weak self] in - self?.dragEvents += 1 - self?.drag() - self?.wake() + inputTap.onLeftDragged = { [weak self] location in + guard let self = self, self.isActive, !self.exporting else { return } + self.dragEvents += 1 + self.drag(to: self.localPoint(location)) + self.wake() } - inputTap.onMouseUp = { [weak self] in - self?.release() - self?.noteButtonEvent() + inputTap.onMouseUp = { [weak self] location in + guard let self = self, self.isActive, !self.exporting else { return } + self.drag(to: self.localPoint(location)) + self.release() + self.noteButtonEvent() } inputTap.onInteractDown = { [weak self] in - self?.armView.press() - self?.armView.held = true - self?.noteButtonEvent() + guard let self = self, self.isActive, !self.exporting else { return } + self.armView.press() + self.armView.held = true + self.tapSound.play() + self.noteButtonEvent() } inputTap.onInteractUp = { [weak self] in - self?.armView.held = false - self?.guardCursorUntil = Date().addingTimeInterval(2) + guard let self = self, self.isActive, !self.exporting else { return } + self.armView.held = false + self.guardCursorUntil = Date().addingTimeInterval(2) CursorHider.reassert(urgent: true) - self?.noteButtonEvent() + self.noteButtonEvent() } inputTap.onPassThroughPress = { [weak self] location in guard let screen = self?.currentScreen() else { @@ -119,6 +145,13 @@ final class OverlayController { self?.exit() self?.onNeedsPermission?() } + toolbar.onModeChanged = { [weak self] in self?.setMode($0) } + toolbar.onSave = { [weak self] in + guard let self = self else { return } + if self.mode == .screenshot { self.saveScreenshot() } + else { self.saveDrawing() } + } + toolbar.onClear = { [weak self] in self?.clearInk() } toolbar.select(armView.style) tapSound.enabled = soundOn toolbar.setSoundOn(soundOn) @@ -165,7 +198,8 @@ final class OverlayController { return } ownMenuOpen = suspended - inputTap.setActive(!suspended) + guard !exporting else { return } + inputTap.setActive(!suspended && !exporting) if let screen = currentScreen() { updateZones(mouse: NSEvent.mouseLocation, screen: screen) } @@ -185,6 +219,17 @@ final class OverlayController { armView.held = false downPoint = nil armView.ink.end() + if mode == .screenshot, !selectionPoints.isEmpty { + do { + selectionPath = try Capture.selection(selectionPoints) + revealUntil = Date().addingTimeInterval(10) + } catch { + clearSelection() + Log.write("selection rejected: \(error.localizedDescription)") + toolbar.showHint([("Draw a shape", "at least 4 points wide and tall")]) + } + } + refreshSave() } var style: PointerStyle { armView.style } @@ -199,20 +244,75 @@ final class OverlayController { toolbar.center(of: style) } + func toolbarButtonCenter(_ mode: PointerMode) -> CGPoint? { + toolbar.center(of: mode) + } + var inkStrokes: Int { armView.ink.layer.sublayers?.count ?? 0 } + var savedInkStrokes: Int { armView.ink.savedStrokes.count } var debugState: String { "held=\(armView.held) downPoint=\(downPoint != nil) drawing=\(armView.ink.isDrawing) strokes=\(inkStrokes)" } func clearInk() { + downPoint = nil + armView.held = false armView.ink.clear() + clearSelection() + refreshSave() } - private func drag() { - guard let start = downPoint, let p = localMouse() else { + var canSaveDrawing: Bool { !armView.ink.savedStrokes.isEmpty } + var canSaveScreenshot: Bool { selectionPath != nil } + var onCaptureChanged: (() -> Void)? + + func setMode(_ mode: PointerMode) { + guard !exporting else { return } + release() + clearSelection() + self.mode = mode + inputTap.mode = mode + // The overlay must never win hit-testing over the Dock in Pointer mode. + window.ignoresMouseEvents = true + refreshSave() + revealUntil = Date().addingTimeInterval(5) + if isActive { + let text = mode == .pointer ? "click and drag normally" + : mode == .draw ? "drag to draw, then save a PNG" + : "draw a shape, then click Save" + toolbar.showHint([(mode.title, text)]) + } + } + + private func refreshSave() { + toolbar.setMode(mode, canSave: mode == .screenshot ? canSaveScreenshot : canSaveDrawing) + onCaptureChanged?() + } + + private func clearSelection() { + selectionPoints = [] + selectionPath = nil + selectionLayer.path = nil + } + + private func drag(to point: CGPoint? = nil) { + guard !exporting, let start = downPoint, let p = point ?? localMouse() else { + return + } + if mode == .screenshot { + if let last = selectionPoints.last, hypot(p.x - last.x, p.y - last.y) < 1.5 { return } + selectionPoints.append(p) + let path = CGMutablePath() + path.addLines(between: selectionPoints) + path.closeSubpath() + CATransaction.begin() + CATransaction.setDisableActions(true) + selectionLayer.path = path + CATransaction.commit() return } + guard mode == .draw else { return } let ink = armView.ink if !ink.isDrawing { if hypot(p.x - start.x, p.y - start.y) < 4 { @@ -223,6 +323,12 @@ final class OverlayController { ink.extend(to: p) } + private func localPoint(_ cgPoint: CGPoint) -> CGPoint? { + guard let frame = currentFrame else { return nil } + let point = OverlayController.cgPoint(cgPoint) + return CGPoint(x: point.x - frame.minX, y: point.y - frame.minY) + } + private func localMouse() -> CGPoint? { guard let frame = currentFrame else { return nil @@ -277,10 +383,7 @@ final class OverlayController { revealUntil = Date().addingTimeInterval(3) if installed && !UserDefaults.standard.bool(forKey: OverlayController.hintShownKey) { UserDefaults.standard.set(true, forKey: OverlayController.hintShownKey) - var parts: [(key: String, text: String)] = [("Click", "to tap"), ("Drag", "to draw")] - if Preferences.interactKey != .off { - parts.append(("Hold \(Preferences.interactKey.symbol)", "to click for real")) - } + var parts: [(key: String, text: String)] = [("Pointer", "to click"), ("Draw", "to sketch"), ("Screenshot", "to select")] parts.append(("Esc", "to exit")) toolbar.showHint(parts) revealUntil = Date().addingTimeInterval(10) @@ -298,6 +401,9 @@ final class OverlayController { return } isActive = false + captureTask?.cancel() + captureTask = nil + savePanel?.cancel(nil) timer?.invalidate() timer = nil @@ -311,6 +417,8 @@ final class OverlayController { armView.held = false downPoint = nil armView.ink.clear() + clearSelection() + refreshSave() window.ignoresMouseEvents = true window.orderOut(nil) toolbar.hide() @@ -327,7 +435,7 @@ final class OverlayController { let f = toolbar.frame let near = f.insetBy(dx: -80, dy: 0).union(f.offsetBy(dx: 0, dy: 90)).insetBy(dx: 0, dy: -10).contains(mouse) let needsPermission = !inputTap.isInstalled - let drawing = armView.ink.isDrawing + let drawing = armView.ink.isDrawing || downPoint != nil toolbar.setRevealed(!drawing && (near || Date() < revealUntil || needsPermission)) let menuLikely = !menuRects.isEmpty || ownMenuOpen || Date() < menuOpeningUntil @@ -343,7 +451,7 @@ final class OverlayController { let overMenu = menuBar.contains(mouse) || menuRects.contains { $0.contains(cgMouse) } || Date() < menuOpeningUntil let key = Preferences.interactKey.cgFlags - let interacting = (!key.isEmpty && CGEventSource.flagsState(.hidSystemState).contains(key)) || inputTap.interactPressing + let interacting = mode != .pointer && ((!key.isEmpty && CGEventSource.flagsState(.hidSystemState).contains(key)) || inputTap.interactPressing) interactHeld = interacting menuHover = overMenu let normal = !drawing && !Preferences.interactKeepsHand && (interacting || overMenu || ownMenuOpen) @@ -372,7 +480,7 @@ final class OverlayController { } private func setEscEnabled(_ enabled: Bool) { - if enabled && isActive && escHotKey == nil { + if enabled && isActive && !exporting && escHotKey == nil { escHotKey = HotKeyCenter.shared.register(keyCode: UInt32(kVK_Escape), modifiers: 0) { [weak self] in self?.exit() } @@ -460,6 +568,10 @@ final class OverlayController { } private func currentScreen() -> NSScreen? { + if mode == .screenshot, !selectionPoints.isEmpty, + let screen = NSScreen.screens.first(where: { $0.frame == currentFrame }) { + return screen + } let mouse = NSEvent.mouseLocation return NSScreen.screens.first { NSMouseInRect(mouse, $0.frame, false) } ?? NSScreen.main @@ -536,6 +648,7 @@ final class OverlayController { } private func tick() { + guard !exporting else { return } let mouse = NSEvent.mouseLocation guard let screen = currentScreen() else { return @@ -543,12 +656,15 @@ final class OverlayController { if currentFrame != screen.frame { let moving = currentFrame != nil - currentFrame = screen.frame if moving { release() + clearSelection() revealUntil = Date().addingTimeInterval(1.5) } - armView.ink.clear() + currentFrame = screen.frame + armView.ink.clearVisible() + armView.ink.origin = screen.frame.origin + refreshSave() window.setFrame(screen.frame, display: false) armView.frame = NSRect(origin: .zero, size: screen.frame.size) toolbar.place(on: screen) @@ -561,7 +677,7 @@ final class OverlayController { updateZones(mouse: mouse, screen: screen) let overDock = DockArea.contains(OverlayController.cgPoint(mouse)) - let catchMouse = overDock && inputTap.isInstalled && !normalCursor && !interactHeld + let catchMouse = mode != .pointer && overDock && inputTap.isInstalled && !normalCursor && !interactHeld if window.ignoresMouseEvents == catchMouse { window.ignoresMouseEvents = !catchMouse } @@ -588,4 +704,113 @@ final class OverlayController { } drag() } + + func saveDrawing() { + guard !exporting else { return } + armView.ink.end() + do { + let image = try Capture.drawing(armView.ink.savedStrokes, scale: armView.ink.scale) + beginExport() + presentSave(image, name: "TinyHand Drawing") + } catch { + beginExport() + showCaptureError(error) + finishExport() + } + } + + func saveScreenshot() { + guard !exporting else { return } + guard let path = selectionPath, let frame = currentFrame, + let screen = NSScreen.screens.first(where: { $0.frame == frame }), + let id = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID else { + beginExport() + showCaptureError(selectionPath == nil ? CaptureError.emptySelection : CaptureError.displayUnavailable) + finishExport() + return + } + beginExport() + guard CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess() else { + showCaptureError(CaptureError.permission) + finishExport() + return + } + captureTask = Task { @MainActor [weak self] in + guard let self = self else { return } + do { + // Allow our windows to leave the compositor before the macOS 13 fallback. + try await Task.sleep(nanoseconds: 150_000_000) + let desktop = try await Capture.screen(id: id, frame: OverlayController.cgRect(frame)) + try Task.checkCancellation() + let image = try Capture.screenshot(desktop, path: path, screenSize: frame.size) + self.presentSave(image, name: "TinyHand Screenshot") + } catch is CancellationError { + self.finishExport() + } catch { + if !Task.isCancelled && self.isActive { self.showCaptureError(error) } + self.finishExport() + } + self.captureTask = nil + } + } + + private func beginExport() { + exporting = true + inputTap.setActive(false) + setEscEnabled(false) + window.orderOut(nil) + toolbar.hide() + CursorHider.show() + } + + private func finishExport() { + exporting = false + guard isActive else { return } + inputTap.setActive(!ownMenuOpen) + normalCursor = false + armView.pointerHidden = false + CursorHider.hide() + window.orderFrontRegardless() + toolbar.show() + revealUntil = Date().addingTimeInterval(5) + tick() + refreshSave() + } + + private func presentSave(_ image: CGImage, name: String) { + let panel = makeSavePanel() + panel.title = name + panel.nameFieldStringValue = "\(name).png" + panel.allowedContentTypes = [.png] + panel.canCreateDirectories = true + savePanel = panel + NSApp.activate(ignoringOtherApps: true) + panel.begin { [weak self] response in + guard let self = self else { return } + defer { + self.savePanel = nil + self.finishExport() + } + guard response == .OK, let url = panel.url else { return } + do { try Capture.png(image).write(to: url, options: .atomic) } + catch { self.showCaptureError(error) } + } + } + + private func showCaptureError(_ error: Error) { + Log.write("capture failed: \(error.localizedDescription)") + let alert = NSAlert() + alert.messageText = "Couldn't save capture" + alert.informativeText = error.localizedDescription + alert.alertStyle = .warning + alert.addButton(withTitle: "OK") + if case CaptureError.permission = error { + alert.addButton(withTitle: "Open Settings") + } + NSApp.activate(ignoringOtherApps: true) + if alert.runModal() == .alertSecondButtonReturn, + let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") { + NSWorkspace.shared.open(url) + } + } } diff --git a/Sources/TinyHand/SettingsView.swift b/Sources/TinyHand/SettingsView.swift index f2b0ddf..b3597cf 100644 --- a/Sources/TinyHand/SettingsView.swift +++ b/Sources/TinyHand/SettingsView.swift @@ -403,7 +403,7 @@ private struct GeneralPage: View { VStack(alignment: .leading, spacing: 22) { if !AXIsProcessTrusted() { Card(title: "Permission") { - Row("Accessibility", detail: "Needed so clicks become taps and you can draw.") { + Row("Accessibility", detail: "Needed to animate clicks and draw on your screen.") { PillButton(title: "Open Settings") { if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") { NSWorkspace.shared.open(url) @@ -423,8 +423,8 @@ private struct GeneralPage: View { if let error = model.shortcutError { Text(error).font(.system(size: 12)).foregroundColor(.red) } - Card(title: "Clicking for real") { - Row("Hold to use the app", detail: "Clicks, drags and scrolls reach the app.") { + Card(title: "While drawing or selecting") { + Row("Hold to use the app", detail: "Pointer mode always lets clicks and drags through.") { Picker("", selection: model.binding({ Preferences.interactKey }, { Preferences.interactKey = $0 })) { ForEach(InteractKey.allCases, id: \.self) { key in Text(key.title).tag(key) @@ -972,7 +972,7 @@ private struct DrawingPage: View { } } } - Row("Lines fade", detail: "Clear them any time with \(Preferences.clearShortcut.display).") { + Row("Lines fade", detail: "Faded lines can still be saved. \(Preferences.clearShortcut.display) discards them.") { Picker("", selection: model.binding({ Preferences.fadeSeconds }, { Preferences.fadeSeconds = $0 })) { ForEach(DrawingPage.choices, id: \.1) { choice in Text(choice.0).tag(choice.1) diff --git a/Sources/TinyHand/Toolbar.swift b/Sources/TinyHand/Toolbar.swift index 95086a0..8c53692 100644 --- a/Sources/TinyHand/Toolbar.swift +++ b/Sources/TinyHand/Toolbar.swift @@ -5,6 +5,9 @@ final class Toolbar { var onToggleSound: (() -> Void)? var onExit: (() -> Void)? var onFixPermission: (() -> Void)? + var onModeChanged: ((PointerMode) -> Void)? + var onSave: (() -> Void)? + var onClear: (() -> Void)? private final class Panel: NSPanel { override var canBecomeKey: Bool { false } @@ -25,6 +28,9 @@ final class Toolbar { private var inkOpen = false private let permissionButton = Button() private var soundButton: Button? + private var saveButton: Button? + private let modes = NSSegmentedControl(labels: PointerMode.allCases.map(\.title), + trackingMode: .selectOne, target: nil, action: nil) private(set) var revealed = false private let tray: Panel @@ -86,12 +92,35 @@ final class Toolbar { trayStack.bottomAnchor.constraint(equalTo: trayBackground.bottomAnchor), ]) + stack.addArrangedSubview(Toolbar.divider()) + modes.selectedSegment = 0 + for (index, width) in [CGFloat(72), 58, 104].enumerated() { + modes.setWidth(width, forSegment: index) + } + modes.segmentStyle = .rounded + modes.appearance = NSAppearance(named: .darkAqua) + modes.target = self + modes.action = #selector(modeChanged) + modes.setAccessibilityLabel("Pointer mode") + modes.setToolTip("Click and drag in macOS normally", forSegment: 0) + modes.setToolTip("Draw without clicking the app underneath", forSegment: 1) + modes.setToolTip("Draw a shape, then save the area inside it", forSegment: 2) + stack.addArrangedSubview(modes) stack.addArrangedSubview(Toolbar.divider()) let ink = makeButton(image: nil, tip: "Ink colour") { [weak self] in self?.toggleInk() } inkButton = ink stack.addArrangedSubview(ink) + let save = makeButton(symbol: "square.and.arrow.down", tip: "Save Drawing as PNG") { [weak self] in + self?.onSave?() + } + save.isEnabled = false + saveButton = save + stack.addArrangedSubview(save) + stack.addArrangedSubview(makeButton(symbol: "trash", tip: "Clear Drawing and Selection") { [weak self] in + self?.onClear?() + }) let sound = makeButton(symbol: "speaker.wave.2.fill", tip: "Tap sound") { [weak self] in self?.onToggleSound?() } @@ -130,6 +159,19 @@ final class Toolbar { } } + @objc private func modeChanged() { + guard PointerMode.allCases.indices.contains(modes.selectedSegment) else { return } + onModeChanged?(PointerMode.allCases[modes.selectedSegment]) + } + + func setMode(_ mode: PointerMode, canSave: Bool) { + modes.selectedSegment = PointerMode.allCases.firstIndex(of: mode) ?? 0 + let tip = mode == .screenshot ? "Save Selected Area as PNG" : "Save Drawing as PNG" + saveButton?.toolTip = tip + saveButton?.setAccessibilityLabel(tip) + saveButton?.isEnabled = canSave + } + private func toggleInk() { setInkOpen(!inkOpen) } @@ -249,6 +291,14 @@ final class Toolbar { return CGPoint(x: r.midX, y: r.midY) } + func center(of mode: PointerMode) -> CGPoint? { + guard let index = PointerMode.allCases.firstIndex(of: mode) else { return nil } + let r = panel.convertToScreen(modes.convert(modes.bounds, to: nil)) + let widths = PointerMode.allCases.indices.map { modes.width(forSegment: $0) } + let padding = (r.width - widths.reduce(0, +)) / 2 + return CGPoint(x: r.minX + padding + widths.prefix(index).reduce(0, +) + widths[index] / 2, y: r.midY) + } + func snapshot() -> NSBitmapImageRep? { guard let view = panel.contentView else { return nil @@ -386,6 +436,8 @@ final class Toolbar { func hide() { setInkOpen(false) + tray.orderOut(nil) + hint?.orderOut(nil) hideHint() revealed = false panel.alphaValue = 0 diff --git a/Sources/TinyHand/WelcomeWindow.swift b/Sources/TinyHand/WelcomeWindow.swift index ba5c74f..619d502 100644 --- a/Sources/TinyHand/WelcomeWindow.swift +++ b/Sources/TinyHand/WelcomeWindow.swift @@ -47,7 +47,7 @@ final class WelcomeWindow: NSObject { private func refresh() { title.stringValue = granted ? "You're all set" : "Allow Accessibility" - subtitle.stringValue = granted ? "Press \(Preferences.toggleShortcut.display) any time to start." : "So your clicks become taps, not real clicks." + subtitle.stringValue = granted ? "Press \(Preferences.toggleShortcut.display) any time to start." : "To animate clicks and draw on your screen." button.title = granted ? "Start" : "Open Settings" tip.isHidden = granted || askedAt.map { Date().timeIntervalSince($0) < 10 } ?? true } diff --git a/Sources/TinyHand/main.swift b/Sources/TinyHand/main.swift index 92f758e..a86de1c 100644 --- a/Sources/TinyHand/main.swift +++ b/Sources/TinyHand/main.swift @@ -12,7 +12,7 @@ if let idx = CommandLine.arguments.firstIndex(of: "--ax-status") { let mask: CGEventMask = 1 << CGEventType.leftMouseDown.rawValue let tap = CGEvent.tapCreate(tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap, eventsOfInterest: mask, callback: { _, _, e, _ in Unmanaged.passUnretained(e) }, userInfo: nil) - let text = "trusted=\(AXIsProcessTrusted()) tap=\(tap != nil) bundle=\(Bundle.main.bundlePath)\n" + let text = "trusted=\(AXIsProcessTrusted()) tap=\(tap != nil) screen=\(CGPreflightScreenCaptureAccess()) bundle=\(Bundle.main.bundlePath)\n" try? text.write(toFile: out, atomically: true, encoding: .utf8) exit(0) } @@ -28,7 +28,7 @@ if let idx = CommandLine.arguments.firstIndex(of: "--toolbar-test") { print("rendered \(out)") } if CommandLine.arguments.contains("--hint") { - bar.showHint([("Click", "to tap"), ("Drag", "to draw"), ("Hold ⌥", "to click for real"), ("Esc", "to exit")]) + bar.showHint([("Pointer", "to click"), ("Draw", "to sketch"), ("Screenshot", "to select"), ("Esc", "to exit")]) if let data = bar.snapshotHint()?.representation(using: .png, properties: [:]) { let hintOut = out.replacingOccurrences(of: ".png", with: "-hint.png") try? data.write(to: URL(fileURLWithPath: hintOut)) @@ -86,8 +86,7 @@ if let idx = CommandLine.arguments.firstIndex(of: "--make-hand"), CommandLine.ar } if CommandLine.arguments.contains("--e2e-test") { - E2ETest.run() - exit(0) + exit(E2ETest.run()) } if CommandLine.arguments.contains("--click-test") { diff --git a/Tests/TinyHandTests/CaptureTests.swift b/Tests/TinyHandTests/CaptureTests.swift new file mode 100644 index 0000000..8c2769b --- /dev/null +++ b/Tests/TinyHandTests/CaptureTests.swift @@ -0,0 +1,76 @@ +import AppKit +import XCTest +@testable import TinyHand + +final class CaptureTests: XCTestCase { + func testDrawingSurvivesFadingAndDisplayChangesUntilCleared() throws { + let ink = Ink() + ink.color = Ink.red + ink.origin = CGPoint(x: -100, y: 50) + ink.begin(at: CGPoint(x: 10, y: 10), width: 4) + ink.extend(to: CGPoint(x: 40, y: 30)) + ink.end() + ink.clearVisible() + XCTAssertEqual(ink.savedStrokes.count, 1) + XCTAssertEqual(ink.savedStrokes[0].path.boundingBoxOfPath.minX, -90) + let image = try Capture.drawing(ink.savedStrokes, scale: 2) + XCTAssertEqual(image.width, 76) + XCTAssertEqual(image.height, 56) + let png = try Capture.png(image) + XCTAssertEqual(Array(png.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10]) + let rep = try XCTUnwrap(NSBitmapImageRep(data: png)) + XCTAssertTrue(rep.hasAlpha) + XCTAssertEqual(try XCTUnwrap(rep.colorAt(x: 0, y: 0)).alphaComponent, 0) + ink.clear() + XCTAssertThrowsError(try Capture.drawing(ink.savedStrokes, scale: 2)) + } + + func testFreeformCropHasTransparentOutsideAndCorrectOrientationAtRetinaScale() throws { + let context = try XCTUnwrap(CGContext(data: nil, width: 200, height: 160, + bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)) + context.setFillColor(NSColor.red.cgColor) + context.fill(CGRect(x: 0, y: 0, width: 200, height: 80)) + context.setFillColor(NSColor.blue.cgColor) + context.fill(CGRect(x: 0, y: 80, width: 200, height: 80)) + let path = try Capture.selection([ + CGPoint(x: 10, y: 10), CGPoint(x: 90, y: 10), CGPoint(x: 10, y: 70), + ]) + let image = try Capture.screenshot(XCTUnwrap(context.makeImage()), path: path, + screenSize: CGSize(width: 100, height: 80)) + XCTAssertEqual(image.width, 160) + XCTAssertEqual(image.height, 120) + let rep = NSBitmapImageRep(cgImage: image) + XCTAssertEqual(try XCTUnwrap(rep.colorAt(x: 150, y: 5)).alphaComponent, 0) + XCTAssertGreaterThan(try XCTUnwrap(rep.colorAt(x: 5, y: 15)).blueComponent, 0.9) + XCTAssertGreaterThan(try XCTUnwrap(rep.colorAt(x: 5, y: 105)).redComponent, 0.9) + } + + func testSelectionRejectsClicksAndLinesButAllowsSelfIntersectingShapes() throws { + XCTAssertThrowsError(try Capture.selection([])) + XCTAssertThrowsError(try Capture.selection([.zero, CGPoint(x: 10, y: 10), CGPoint(x: 30, y: 30)])) + XCTAssertThrowsError(try Capture.selection([.zero, CGPoint(x: 2, y: 0), CGPoint(x: 0, y: 2)])) + XCTAssertNoThrow(try Capture.selection([ + .zero, CGPoint(x: 50, y: 50), CGPoint(x: 0, y: 50), CGPoint(x: 50, y: 0), + ])) + } + + func testSelectionIsClippedToDisplayEdges() throws { + let context = try XCTUnwrap(CGContext(data: nil, width: 100, height: 80, + bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)) + let path = try Capture.selection([ + CGPoint(x: -20, y: -10), CGPoint(x: 40, y: -10), + CGPoint(x: 40, y: 40), CGPoint(x: -20, y: 40), + ]) + let image = try Capture.screenshot(XCTUnwrap(context.makeImage()), path: path, + screenSize: CGSize(width: 100, height: 80)) + XCTAssertEqual(image.width, 40) + XCTAssertEqual(image.height, 40) + XCTAssertThrowsError(try Capture.drawing([ + Ink.Stroke(path: path, color: Ink.red, width: 4), + ], scale: .infinity)) + } +} diff --git a/Tests/TinyHandTests/InputTapTests.swift b/Tests/TinyHandTests/InputTapTests.swift new file mode 100644 index 0000000..86bd4b4 --- /dev/null +++ b/Tests/TinyHandTests/InputTapTests.swift @@ -0,0 +1,85 @@ +import AppKit +import XCTest +@testable import TinyHand + +final class InputTapTests: XCTestCase { + private var events: [CGEvent] = [] + + private func event(_ type: CGEventType, button: CGMouseButton = .left, + flags: CGEventFlags = [], point: CGPoint = CGPoint(x: 500, y: 500)) throws -> CGEvent { + let event = try XCTUnwrap(CGEvent(mouseEventSource: nil, mouseType: type, + mouseCursorPosition: point, mouseButton: button)) + event.flags = flags + // The event-tap callback returns unretained events, owned by its caller. + events.append(event) + return event + } + + func testPointerPassesNativeClicksDragsAndModifiersWithoutReposting() throws { + let tap = InputTap() + tap.setActive(true) + tap.interactKey = .maskAlternate + for (button, types) in [ + (CGMouseButton.left, [CGEventType.leftMouseDown, .leftMouseDragged, .leftMouseUp]), + (.right, [.rightMouseDown, .rightMouseDragged, .rightMouseUp]), + (.center, [.otherMouseDown, .otherMouseDragged, .otherMouseUp]), + ] { + for type in types { + let e = try event(type, button: button, flags: [.maskAlternate, .maskShift]) + e.setIntegerValueField(.mouseEventClickState, value: 2) + XCTAssertTrue(tap.handle(type: type, event: e)?.takeUnretainedValue() === e) + XCTAssertEqual(e.type, type) + XCTAssertEqual(e.flags, [.maskAlternate, .maskShift]) + XCTAssertEqual(e.getIntegerValueField(.mouseEventClickState), 2) + } + } + } + + func testDrawingAndScreenshotConsumeClicksAndDoNotLeakDrags() throws { + let tap = InputTap() + tap.setActive(true) + for mode in [PointerMode.draw, .screenshot] { + tap.mode = mode + XCTAssertNil(tap.handle(type: .leftMouseDown, event: try event(.leftMouseDown))) + let drag = try event(.leftMouseDragged) + XCTAssertNotNil(tap.handle(type: .leftMouseDragged, event: drag)) + XCTAssertEqual(drag.type, .mouseMoved) + XCTAssertNil(tap.handle(type: .leftMouseUp, event: try event(.leftMouseUp))) + } + } + + func testModifierAndRoutesStayLatchedForEachButton() throws { + let tap = InputTap() + tap.setActive(true) + tap.mode = .draw + tap.interactKey = .maskAlternate + let down = try event(.leftMouseDown, flags: [.maskAlternate, .maskShift]) + XCTAssertNotNil(tap.handle(type: .leftMouseDown, event: down)) + XCTAssertEqual(down.flags, .maskShift) + XCTAssertTrue(tap.interactPressing) + XCTAssertNil(tap.handle(type: .rightMouseDown, event: try event(.rightMouseDown, button: .right))) + tap.interactKey = .maskCommand + let drag = try event(.leftMouseDragged, flags: [.maskAlternate, .maskShift]) + XCTAssertNotNil(tap.handle(type: .leftMouseDragged, event: drag)) + XCTAssertEqual(drag.flags, .maskShift) + XCTAssertEqual(drag.type, .leftMouseDragged) + XCTAssertNil(tap.handle(type: .rightMouseUp, event: try event(.rightMouseUp, button: .right))) + XCTAssertTrue(tap.interactPressing) + XCTAssertNotNil(tap.handle(type: .leftMouseUp, event: try event(.leftMouseUp))) + XCTAssertFalse(tap.interactPressing) + } + + func testToolbarGestureAndModeChangeKeepOriginalRoute() throws { + let tap = InputTap() + tap.setActive(true) + tap.mode = .draw + tap.passThroughRects = [CGRect(x: 0, y: 0, width: 100, height: 100)] + XCTAssertNotNil(tap.handle(type: .leftMouseDown, event: try event(.leftMouseDown, point: CGPoint(x: 50, y: 50)))) + XCTAssertNotNil(tap.handle(type: .leftMouseUp, event: try event(.leftMouseUp))) + XCTAssertNil(tap.handle(type: .leftMouseDown, event: try event(.leftMouseDown))) + tap.mode = .pointer + XCTAssertNil(tap.handle(type: .leftMouseUp, event: try event(.leftMouseUp, point: CGPoint(x: 50, y: 50)))) + tap.setActive(false) + XCTAssertNotNil(tap.handle(type: .leftMouseDown, event: try event(.leftMouseDown))) + } +} diff --git a/scripts/capture-evidence.py b/scripts/capture-evidence.py new file mode 100644 index 0000000..7259c64 --- /dev/null +++ b/scripts/capture-evidence.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Record live capture tests over a separate, privacy-safe native window.""" + +import argparse +import os +from pathlib import Path +import select +import subprocess +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path, help="A new output directory") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + subprocess.run(["swift", "build"], cwd=root, check=True) + subprocess.run( + ["swiftc", str(root / "scripts/capture-fixture.swift"), "-o", str(output / "fixture")], + check=True, + ) + fixture = subprocess.Popen([str(output / "fixture")], stdout=subprocess.PIPE, text=True) + recorder = None + try: + if ( + not select.select([fixture.stdout], [], [], 10)[0] + or fixture.stdout.readline().strip() != "fixture ready" + ): + raise RuntimeError("The privacy-safe fixture did not become ready") + time.sleep(0.5) + recorder = subprocess.Popen( + ["/usr/sbin/screencapture", "-x", "-v", "-V", "20", "-m", str(output / "demo.mov")] + ) + result = subprocess.run( + [str(root / ".build/debug/TinyHand"), "--e2e-test"], + cwd=root, + env=dict(os.environ, TH_CAPTURE_OUTPUT=str(output)), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=60, + ) + (output / "e2e.log").write_text(result.stdout) + print(result.stdout, end="") + # Keep the fixture on screen until recording ends, even if a test failed. + recorder.wait(timeout=30) + result.check_returncode() + if recorder.returncode: + raise RuntimeError(f"Recording failed with status {recorder.returncode}") + finally: + if recorder is not None and recorder.poll() is None: + recorder.terminate() + recorder.wait(timeout=10) + if fixture.poll() is None: + fixture.terminate() + fixture.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/scripts/capture-fixture.swift b/scripts/capture-fixture.swift new file mode 100644 index 0000000..b558758 --- /dev/null +++ b/scripts/capture-fixture.swift @@ -0,0 +1,37 @@ +import AppKit + +final class FixtureView: NSView { + override func draw(_ rect: NSRect) { + NSColor(srgbRed: 0.13, green: 0.22, blue: 0.31, alpha: 1).setFill() + bounds.fill() + ("TinyHand" as NSString).draw(at: CGPoint(x: bounds.midX - 165, y: bounds.height * 0.74), + withAttributes: [.font: NSFont.systemFont(ofSize: 64, weight: .semibold), + .foregroundColor: NSColor.white]) + ("Pointer / Draw / Screenshot" as NSString).draw( + at: CGPoint(x: bounds.midX - 192, y: bounds.height * 0.68), + withAttributes: [.font: NSFont.systemFont(ofSize: 25, weight: .medium), + .foregroundColor: NSColor(srgbRed: 0.76, green: 0.85, blue: 0.91, alpha: 1)]) + ("Captured" as NSString).draw( + at: CGPoint(x: bounds.midX - 105, y: bounds.midY - 20), + withAttributes: [.font: NSFont.systemFont(ofSize: 16, weight: .medium), + .foregroundColor: NSColor.white]) + ("Native macOS capture fixture - no personal desktop content" as NSString).draw( + at: CGPoint(x: bounds.midX - 232, y: bounds.height * 0.20), + withAttributes: [.font: NSFont.systemFont(ofSize: 16), + .foregroundColor: NSColor(srgbRed: 0.76, green: 0.85, blue: 0.91, alpha: 1)]) + } +} + +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +guard let screen = NSScreen.screens.first else { exit(1) } +let window = NSWindow(contentRect: screen.frame, styleMask: .borderless, backing: .buffered, defer: false) +window.contentView = FixtureView(frame: CGRect(origin: .zero, size: screen.frame.size)) +window.level = .screenSaver +window.ignoresMouseEvents = true +window.isOpaque = true +window.orderFrontRegardless() +Timer.scheduledTimer(withTimeInterval: 90, repeats: false) { _ in app.terminate(nil) } +print("fixture ready") +fflush(stdout) +app.run()