Skip to content

Add screenshot capture settings for token and payload optimization #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antiartificial wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add screenshot capture settings for token and payload optimization #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 44 additions & 3 deletions leanring-buddy/CompanionManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,39 @@ final class CompanionManager: ObservableObject {
}
}

// MARK: - Screenshot Settings

/// When enabled, only the screen containing the cursor is captured and
/// sent to Claude. Reduces token usage on multi-monitor setups.
/// Off by default so Claude has full multi-monitor context.
@Published var captureOnlyPrimaryScreen: Bool = UserDefaults.standard.bool(forKey: "captureOnlyPrimaryScreen")

func setCaptureOnlyPrimaryScreen(_ enabled: Bool) {
captureOnlyPrimaryScreen = enabled
UserDefaults.standard.set(enabled, forKey: "captureOnlyPrimaryScreen")
}

/// JPEG compression quality for screenshots sent to Claude.
/// Lower values reduce payload size (faster uploads) but not token count.
/// Range: 0.0 (most compression) to 1.0 (least compression). Default: 0.8.
@Published var screenshotJPEGQuality: Double = UserDefaults.standard.object(forKey: "screenshotJPEGQuality") == nil
? 0.8
: UserDefaults.standard.double(forKey: "screenshotJPEGQuality")

func setScreenshotJPEGQuality(_ quality: Double) {
screenshotJPEGQuality = quality
UserDefaults.standard.set(quality, forKey: "screenshotJPEGQuality")
}

/// When enabled, captures only the frontmost application window instead
/// of the entire screen. Useful for reducing noise in screenshots.
@Published var captureActiveWindowOnly: Bool = UserDefaults.standard.bool(forKey: "captureActiveWindowOnly")

func setCaptureActiveWindowOnly(_ enabled: Bool) {
captureActiveWindowOnly = enabled
UserDefaults.standard.set(enabled, forKey: "captureActiveWindowOnly")
}

/// Whether the user has completed onboarding at least once. Persisted
/// to UserDefaults so the Start button only appears on first launch.
var hasCompletedOnboarding: Bool {
Expand Down Expand Up @@ -592,8 +625,12 @@ final class CompanionManager: ObservableObject {
voiceState = .processing

do {
// Capture all connected screens so the AI has full context
let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG()
// Capture screens using the user's screenshot settings
let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG(
captureOnlyPrimaryScreen: captureOnlyPrimaryScreen,
captureActiveWindowOnly: captureActiveWindowOnly,
jpegCompressionQuality: CGFloat(screenshotJPEGQuality)
)

guard !Task.isCancelled else { return }

Expand Down Expand Up @@ -970,7 +1007,11 @@ final class CompanionManager: ObservableObject {

Task {
do {
let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG()
let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG(
captureOnlyPrimaryScreen: true,
captureActiveWindowOnly: captureActiveWindowOnly,
jpegCompressionQuality: CGFloat(screenshotJPEGQuality)
)

// Only send the cursor screen so Claude can't pick something
// on a different monitor that we can't point at.
Expand Down
114 changes: 114 additions & 0 deletions leanring-buddy/CompanionPanelView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ struct CompanionPanelView: View {
// .padding(.horizontal, 16)
// }

if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted {
Spacer()
.frame(height: 12)

screenshotSettingsSection
.padding(.horizontal, 16)
}

if companionManager.hasCompletedOnboarding && companionManager.allPermissionsGranted {
Spacer()
.frame(height: 16)
Expand Down Expand Up @@ -641,6 +649,112 @@ struct CompanionPanelView: View {
.pointerCursor()
}

// MARK: - Screenshot Settings

private var screenshotSettingsSection: some View {
VStack(spacing: 2) {
Text("SCREENSHOT")
.font(.system(size: 10, weight: .semibold, design: .rounded))
.foregroundColor(DS.Colors.textTertiary)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.bottom, 6)

primaryScreenOnlyToggleRow

activeWindowOnlyToggleRow

jpegQualitySliderRow
}
}

private var primaryScreenOnlyToggleRow: some View {
HStack {
HStack(spacing: 8) {
Image(systemName: "display")
.font(.system(size: 12, weight: .medium))
.foregroundColor(DS.Colors.textTertiary)
.frame(width: 16)

Text("Primary screen only")
.font(.system(size: 13, weight: .medium))
.foregroundColor(DS.Colors.textSecondary)
}

Spacer()

Toggle("", isOn: Binding(
get: { companionManager.captureOnlyPrimaryScreen },
set: { companionManager.setCaptureOnlyPrimaryScreen($0) }
))
.toggleStyle(.switch)
.labelsHidden()
.tint(DS.Colors.accent)
.scaleEffect(0.8)
}
.padding(.vertical, 4)
}

private var activeWindowOnlyToggleRow: some View {
HStack {
HStack(spacing: 8) {
Image(systemName: "macwindow")
.font(.system(size: 12, weight: .medium))
.foregroundColor(DS.Colors.textTertiary)
.frame(width: 16)

Text("Active window only")
.font(.system(size: 13, weight: .medium))
.foregroundColor(DS.Colors.textSecondary)
}

Spacer()

Toggle("", isOn: Binding(
get: { companionManager.captureActiveWindowOnly },
set: { companionManager.setCaptureActiveWindowOnly($0) }
))
.toggleStyle(.switch)
.labelsHidden()
.tint(DS.Colors.accent)
.scaleEffect(0.8)
}
.padding(.vertical, 4)
}

private var jpegQualitySliderRow: some View {
VStack(spacing: 6) {
HStack {
HStack(spacing: 8) {
Image(systemName: "photo")
.font(.system(size: 12, weight: .medium))
.foregroundColor(DS.Colors.textTertiary)
.frame(width: 16)

Text("Image quality")
.font(.system(size: 13, weight: .medium))
.foregroundColor(DS.Colors.textSecondary)
}

Spacer()

Text("\(Int(companionManager.screenshotJPEGQuality * 100))%")
.font(.system(size: 11, weight: .medium, design: .monospaced))
.foregroundColor(DS.Colors.textTertiary)
}

Slider(
value: Binding(
get: { companionManager.screenshotJPEGQuality },
set: { companionManager.setScreenshotJPEGQuality($0) }
),
in: 0.3...1.0,
step: 0.1
)
.tint(DS.Colors.accent)
}
.padding(.vertical, 4)
}

// MARK: - DM Farza Button

private var dmFarzaButton: some View {
Expand Down
78 changes: 65 additions & 13 deletions leanring-buddy/CompanionScreenCaptureUtility.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,18 @@ struct CompanionScreenCapture {
@MainActor
enum CompanionScreenCaptureUtility {

/// Captures all connected displays as JPEG data, labeling each with
/// whether the user's cursor is on that screen. This gives the AI
/// full context across multiple monitors.
static func captureAllScreensAsJPEG() async throws -> [CompanionScreenCapture] {
/// Captures displays as JPEG data based on the provided settings.
///
/// - Parameters:
/// - captureOnlyPrimaryScreen: When true, only the screen containing the cursor is captured.
/// - captureActiveWindowOnly: When true, captures only the frontmost application window
/// instead of the full screen.
/// - jpegCompressionQuality: JPEG compression factor from 0.0 (max compression) to 1.0 (min compression).
static func captureAllScreensAsJPEG(
captureOnlyPrimaryScreen: Bool = false,
captureActiveWindowOnly: Bool = false,
jpegCompressionQuality: CGFloat = 0.8
) async throws -> [CompanionScreenCapture] {
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)

guard !content.displays.isEmpty else {
Expand Down Expand Up @@ -67,6 +75,16 @@ enum CompanionScreenCaptureUtility {
return false
}

// If capturing only the active window, find the frontmost non-own window
let activeWindow: SCWindow? = captureActiveWindowOnly
? content.windows.first(where: { window in
window.owningApplication?.bundleIdentifier != ownBundleIdentifier
&& window.isOnScreen
&& window.frame.width > 100
&& window.frame.height > 100
})
: nil

var capturedScreens: [CompanionScreenCapture] = []

for (displayIndex, display) in sortedDisplays.enumerated() {
Expand All @@ -78,17 +96,43 @@ enum CompanionScreenCaptureUtility {
width: CGFloat(display.width), height: CGFloat(display.height))
let isCursorScreen = displayFrame.contains(mouseLocation)

let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows)
// Skip non-cursor screens when primary-only mode is enabled
if captureOnlyPrimaryScreen && !isCursorScreen {
continue
}

let filter: SCContentFilter
if let activeWindow {
// Capture just the active window — no desktop or other windows
filter = SCContentFilter(desktopIndependentWindow: activeWindow)
} else {
filter = SCContentFilter(display: display, excludingWindows: ownAppWindows)
}

let configuration = SCStreamConfiguration()
let maxDimension = 1280
let aspectRatio = CGFloat(display.width) / CGFloat(display.height)
if display.width >= display.height {
configuration.width = maxDimension
configuration.height = Int(CGFloat(maxDimension) / aspectRatio)

if let activeWindow {
// Size the capture to the window's actual dimensions, capped at maxDimension
let windowWidth = Int(activeWindow.frame.width)
let windowHeight = Int(activeWindow.frame.height)
let windowAspectRatio = CGFloat(windowWidth) / CGFloat(windowHeight)
if windowWidth >= windowHeight {
configuration.width = min(windowWidth, maxDimension)
configuration.height = Int(CGFloat(configuration.width) / windowAspectRatio)
} else {
configuration.height = min(windowHeight, maxDimension)
configuration.width = Int(CGFloat(configuration.height) * windowAspectRatio)
}
} else {
configuration.height = maxDimension
configuration.width = Int(CGFloat(maxDimension) * aspectRatio)
let aspectRatio = CGFloat(display.width) / CGFloat(display.height)
if display.width >= display.height {
configuration.width = maxDimension
configuration.height = Int(CGFloat(maxDimension) / aspectRatio)
} else {
configuration.height = maxDimension
configuration.width = Int(CGFloat(maxDimension) * aspectRatio)
}
}

let cgImage = try await SCScreenshotManager.captureImage(
Expand All @@ -97,12 +141,15 @@ enum CompanionScreenCaptureUtility {
)

guard let jpegData = NSBitmapImageRep(cgImage: cgImage)
.representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else {
.representation(using: .jpeg, properties: [.compressionFactor: jpegCompressionQuality]) else {
continue
}

let screenLabel: String
if sortedDisplays.count == 1 {
if captureActiveWindowOnly, let activeWindow {
let appName = activeWindow.owningApplication?.applicationName ?? "unknown app"
screenLabel = "active window (\(appName)) — cursor screen"
} else if sortedDisplays.count == 1 || captureOnlyPrimaryScreen {
screenLabel = "user's screen (cursor is here)"
} else if isCursorScreen {
screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)"
Expand All @@ -120,6 +167,11 @@ enum CompanionScreenCaptureUtility {
screenshotWidthInPixels: configuration.width,
screenshotHeightInPixels: configuration.height
))

// Only need one screen when capturing the active window
if captureActiveWindowOnly {
break
}
}

guard !capturedScreens.isEmpty else {
Expand Down
Loading