From 8560de672a2cd362d92252ddba3a0704e0bb0ed4 Mon Sep 17 00:00:00 2001 From: marton Date: Wed, 13 May 2026 17:47:26 -0400 Subject: [PATCH 1/5] feat(macos): add ScreenCaptureKit display capture --- cmake/compile_definitions/macos.cmake | 3 + cmake/dependencies/macos.cmake | 1 + src/platform/macos/display.mm | 220 +++++++++++++++++- src/platform/macos/nv12_zero_device.cpp | 4 + src/platform/macos/sc_capture.h | 45 ++++ src/platform/macos/sc_capture.m | 289 ++++++++++++++++++++++++ 6 files changed, 554 insertions(+), 8 deletions(-) create mode 100644 src/platform/macos/sc_capture.h create mode 100644 src/platform/macos/sc_capture.m diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index 42a5dbe10f1..7c63468f3f9 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -35,6 +35,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES ${CORE_MEDIA_LIBRARY} ${CORE_VIDEO_LIBRARY} ${FOUNDATION_LIBRARY} + ${SCREEN_CAPTURE_KIT_LIBRARY} ${VIDEO_TOOLBOX_LIBRARY}) set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in") @@ -55,6 +56,8 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.h" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_capture.m" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h" ${APPLE_PLIST_FILE}) diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 5e225fdac21..e202cfb277b 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -9,6 +9,7 @@ FIND_LIBRARY(CORE_AUDIO_LIBRARY CoreAudio) FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia) FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) if(SUNSHINE_ENABLE_TRAY) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index be124b2d331..3b8b6d4b19c 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -2,6 +2,11 @@ * @file src/platform/macos/display.mm * @brief Definitions for display capture on macOS. */ +// standard includes +#include +#include +#include + // local includes #include "src/config.h" #include "src/logging.h" @@ -10,6 +15,7 @@ #include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" +#import "src/platform/macos/sc_capture.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg @@ -21,6 +27,104 @@ namespace platf { using namespace std::literals; + static bool process_frame(CMSampleBufferRef sampleBuffer, img_t *img) { + auto pixel_buffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixel_buffer) { + return false; + } + + auto new_sample_buffer = std::make_shared(sampleBuffer); + auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + + auto av_img = (av_img_t *) img; + + auto old_data_retainer = std::make_shared( + av_img->sample_buffer, + av_img->pixel_buffer, + img->data + ); + + av_img->sample_buffer = new_sample_buffer; + av_img->pixel_buffer = new_pixel_buffer; + img->data = new_pixel_buffer->data(); + + img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); + img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); + img->row_pitch = CVPixelBufferIsPlanar(new_pixel_buffer->buf) ? + (int) CVPixelBufferGetBytesPerRowOfPlane(new_pixel_buffer->buf, 0) : + (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); + img->pixel_pitch = img->row_pitch / img->width; + + old_data_retainer = nullptr; + return true; + } + + static void clear_pixel_buffer(CVPixelBufferRef pixel_buffer) { + CVPixelBufferLockBaseAddress(pixel_buffer, 0); + + if (CVPixelBufferIsPlanar(pixel_buffer)) { + for (size_t plane = 0; plane < CVPixelBufferGetPlaneCount(pixel_buffer); ++plane) { + auto *base = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane)); + auto bytes_per_row = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); + auto height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + std::memset(base, 0, bytes_per_row * height); + } + } else { + auto *base = static_cast(CVPixelBufferGetBaseAddress(pixel_buffer)); + std::memset(base, 0, CVPixelBufferGetBytesPerRow(pixel_buffer) * CVPixelBufferGetHeight(pixel_buffer)); + } + + CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + } + + static int make_dummy_img(img_t *img, int width, int height, OSType pixel_format, std::string_view backend_name) { + CVPixelBufferRef pixel_buffer = nullptr; + NSDictionary *attrs = @{ + (NSString *) kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + + auto status = CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + pixel_format, + (__bridge CFDictionaryRef) attrs, + &pixel_buffer + ); + + if (status != kCVReturnSuccess || !pixel_buffer) { + BOOST_LOG(error) << backend_name << " dummy_img: failed to create pixel buffer"sv; + return 1; + } + + clear_pixel_buffer(pixel_buffer); + + CMVideoFormatDescriptionRef format_desc = nullptr; + status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, &format_desc); + if (status != noErr || !format_desc) { + CVPixelBufferRelease(pixel_buffer); + BOOST_LOG(error) << backend_name << " dummy_img: failed to create format description"sv; + return 1; + } + + CMSampleTimingInfo timing = {kCMTimeInvalid, kCMTimeInvalid, kCMTimeInvalid}; + CMSampleBufferRef sample_buffer = nullptr; + status = CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixel_buffer, YES, nullptr, nullptr, format_desc, &timing, &sample_buffer); + CFRelease(format_desc); + + if (status != noErr || !sample_buffer) { + CVPixelBufferRelease(pixel_buffer); + BOOST_LOG(error) << backend_name << " dummy_img: failed to create sample buffer"sv; + return 1; + } + + auto ret = process_frame(sample_buffer, img) ? 0 : 1; + CFRelease(sample_buffer); + CVPixelBufferRelease(pixel_buffer); + + return ret; + } + struct av_display_t: public display_t { AVVideo *av_capture {}; CGDirectDisplayID display_id {}; @@ -151,32 +255,132 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } }; + struct sc_display_t: public display_t { + SCCapture *sc_capture {}; + CGDirectDisplayID display_id {}; + + ~sc_display_t() override { + [sc_capture stopCapture]; + [sc_capture release]; + } + + capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { + auto signal = [sc_capture captureVideo:^(CMSampleBufferRef sampleBuffer) { + std::shared_ptr img_out; + if (!pull_free_image_cb(img_out)) { + return false; + } + + if (!process_frame(sampleBuffer, img_out.get())) { + return true; + } + + if (!push_captured_image_cb(std::move(img_out), true)) { + return false; + } + + return true; + }]; + + if (!signal) { + BOOST_LOG(error) << "SCCapture failed to start video capture"sv; + return capture_e::error; + } + + while (dispatch_semaphore_wait(signal, dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC)) != 0) { + std::shared_ptr probe_img; + if (!pull_free_image_cb(probe_img)) { + [sc_capture stopCapture]; + break; + } + } + + return capture_e::ok; + } + + std::shared_ptr alloc_img() override { + return std::make_shared(); + } + + std::unique_ptr make_avcodec_encode_device(pix_fmt_e pix_fmt) override { + if (pix_fmt == pix_fmt_e::yuv420p) { + sc_capture.pixelFormat = kCVPixelFormatType_32BGRA; + + return std::make_unique(); + } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { + auto device = std::make_unique(); + + device->init(static_cast(sc_capture), pix_fmt, setResolution, setPixelFormat); + + return device; + } else { + BOOST_LOG(error) << "Unsupported Pixel Format."sv; + return nullptr; + } + } + + int dummy_img(img_t *img) override { + if (!platf::is_screen_capture_allowed()) { + return 1; + } + + return make_dummy_img(img, sc_capture.frameWidth, sc_capture.frameHeight, sc_capture.pixelFormat, "SCCapture"sv); + } + + static void setResolution(void *display, int width, int height) { + [static_cast(display) setFrameWidth:width frameHeight:height]; + } + + static void setPixelFormat(void *display, OSType pixelFormat) { + static_cast(display).pixelFormat = pixelFormat; + } + }; + std::shared_ptr display(platf::mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) { if (hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::videotoolbox) { BOOST_LOG(error) << "Could not initialize display with the given hw device type."sv; return nullptr; } - auto display = std::make_shared(); - // Default to main display - display->display_id = CGMainDisplayID(); + auto display_id = CGMainDisplayID(); // Print all displays available with it's name and id auto display_array = [AVVideo displayNames]; BOOST_LOG(info) << "Detecting displays"sv; for (NSDictionary *item in display_array) { - NSNumber *display_id = item[@"id"]; + NSNumber *item_display_id = item[@"id"]; // We need show display's product name and corresponding display number given by user NSString *name = item[@"displayName"]; // We are using CGGetActiveDisplayList that only returns active displays so hardcoded connected value in log to true - BOOST_LOG(info) << "Detected display: "sv << name.UTF8String << " (id: "sv << [NSString stringWithFormat:@"%@", display_id].UTF8String << ") connected: true"sv; - if (!display_name.empty() && std::atoi(display_name.c_str()) == [display_id unsignedIntValue]) { - display->display_id = [display_id unsignedIntValue]; + BOOST_LOG(info) << "Detected display: "sv << name.UTF8String << " (id: "sv << [NSString stringWithFormat:@"%@", item_display_id].UTF8String << ") connected: true"sv; + if (!display_name.empty() && std::atoi(display_name.c_str()) == [item_display_id unsignedIntValue]) { + display_id = [item_display_id unsignedIntValue]; + } + } + BOOST_LOG(info) << "Configuring selected display ("sv << display_id << ") to stream"sv; + + if (@available(macOS 12.3, *)) { + if ([SCCapture isAvailable]) { + auto display = std::make_shared(); + display->display_id = display_id; + display->sc_capture = [[SCCapture alloc] initWithDisplay:display_id frameRate:config.framerate]; + + if (display->sc_capture) { + display->width = display->sc_capture.frameWidth; + display->height = display->sc_capture.frameHeight; + display->env_width = display->width; + display->env_height = display->height; + + return display; + } + + BOOST_LOG(error) << "SCCapture setup failed, trying AVFoundation..."sv; } } - BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; + auto display = std::make_shared(); + display->display_id = display_id; display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; if (!display->av_capture) { diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index b4fb28cb736..b4d6aa33f47 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -29,6 +29,10 @@ namespace platf { int nv12_zero_device::convert(platf::img_t &img) { auto *av_img = (av_img_t *) &img; + if (!av_img->pixel_buffer || !av_img->pixel_buffer->buf) { + return -1; + } + // Release any existing CVPixelBuffer previously retained for encoding av_buffer_unref(&av_frame->buf[0]); diff --git a/src/platform/macos/sc_capture.h b/src/platform/macos/sc_capture.h new file mode 100644 index 00000000000..c449a5ed192 --- /dev/null +++ b/src/platform/macos/sc_capture.h @@ -0,0 +1,45 @@ +/** + * @file src/platform/macos/sc_capture.h + * @brief Declarations for ScreenCaptureKit-based display capture on macOS. + */ +#pragma once + +#import +#import +#import +#import + +API_AVAILABLE(macos(12.3)) +@interface SCCapture : NSObject + +#define kMaxDisplays 32 + +typedef bool (^VideoFrameCallbackBlock)(CMSampleBufferRef); + +@property(nonatomic, assign) CGDirectDisplayID displayID; +@property(nonatomic, assign) int frameRate; +@property(nonatomic, assign) OSType pixelFormat; +@property(nonatomic, assign) int frameWidth; +@property(nonatomic, assign) int frameHeight; + +@property(nonatomic, strong) SCStream *stream; +@property(nonatomic, strong) SCShareableContent *shareableContent; +@property(nonatomic, strong) dispatch_queue_t videoQueue; + +@property(nonatomic, copy) VideoFrameCallbackBlock videoCallback; +@property(nonatomic, strong) dispatch_semaphore_t captureSignal; +@property(nonatomic, assign) BOOL stopping; +@property(nonatomic, assign) CMSampleBufferRef lastValidSampleBuffer; + ++ (BOOL)isAvailable; ++ (NSArray *)displayNames; ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID; + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID + frameRate:(int)frameRate; + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback; +- (void)stopCapture; + +@end diff --git a/src/platform/macos/sc_capture.m b/src/platform/macos/sc_capture.m new file mode 100644 index 00000000000..23f94a4249a --- /dev/null +++ b/src/platform/macos/sc_capture.m @@ -0,0 +1,289 @@ +/** + * @file src/platform/macos/sc_capture.m + * @brief ScreenCaptureKit-based display capture implementation. + */ +#import "sc_capture.h" + +API_AVAILABLE(macos(12.3)) +@implementation SCCapture + ++ (BOOL)isAvailable { + if (@available(macOS 12.3, *)) { + return YES; + } + return NO; +} + ++ (NSArray *)displayNames { + CGDirectDisplayID displays[kMaxDisplays]; + uint32_t count; + if (CGGetActiveDisplayList(kMaxDisplays, displays, &count) != kCGErrorSuccess) { + return [NSArray array]; + } + + NSMutableArray *result = [NSMutableArray array]; + + for (uint32_t i = 0; i < count; i++) { + [result addObject:@{ + @"id": [NSNumber numberWithUnsignedInt:displays[i]], + @"name": [NSString stringWithFormat:@"%d", displays[i]], + @"displayName": [self getDisplayName:displays[i]] ?: @"Unknown Display", + }]; + } + + return [NSArray arrayWithArray:result]; +} + ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID { + for (NSScreen *screen in [NSScreen screens]) { + if ([screen.deviceDescription[@"NSScreenNumber"] isEqualToNumber:[NSNumber numberWithUnsignedInt:displayID]]) { + return screen.localizedName; + } + } + return nil; +} + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID + frameRate:(int)frameRate { + self = [super init]; + if (self) { + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); + + self.displayID = displayID; + self.frameRate = frameRate; + self.pixelFormat = kCVPixelFormatType_32BGRA; + + if (mode) { + self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); + self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); + CFRelease(mode); + } else { + self.frameWidth = (int) CGDisplayPixelsWide(displayID); + self.frameHeight = (int) CGDisplayPixelsHigh(displayID); + } + + dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, DISPATCH_QUEUE_PRIORITY_HIGH); + self.videoQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckVideoQueue", qos); + + dispatch_semaphore_t initSemaphore = dispatch_semaphore_create(0); + __block BOOL initSuccess = NO; + + [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) { + if (error) { + NSLog(@"[SCCapture] Failed to get shareable content: %@", error.localizedDescription); + } else { + self.shareableContent = content; + initSuccess = YES; + } + dispatch_semaphore_signal(initSemaphore); + }]; + + dispatch_semaphore_wait(initSemaphore, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + + if (!initSuccess) { + return nil; + } + } + return self; +} + +- (void)dealloc { + [self stopCapture]; + [super dealloc]; +} + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { + self.frameWidth = frameWidth; + self.frameHeight = frameHeight; +} + +- (SCDisplay *)findDisplayWithID:(CGDirectDisplayID)displayID { + for (SCDisplay *display in self.shareableContent.displays) { + if (display.displayID == displayID) { + return display; + } + } + return nil; +} + +- (SCDisplay *)findDisplayWithIDRetrying:(CGDirectDisplayID)displayID { + SCDisplay *display = [self findDisplayWithID:displayID]; + if (display) { + return display; + } + + for (int attempt = 1; attempt <= 3; attempt++) { + NSLog(@"[SCCapture] Display %u not found in SCShareableContent, refreshing (attempt %d/3)", displayID, attempt); + [NSThread sleepForTimeInterval:1.0]; + + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + __block BOOL success = NO; + + [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) { + if (!error && content) { + self.shareableContent = content; + success = YES; + } + dispatch_semaphore_signal(sem); + }]; + + dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + + if (success) { + display = [self findDisplayWithID:displayID]; + if (display) { + NSLog(@"[SCCapture] Found display %u after refresh", displayID); + return display; + } + } + } + + return nil; +} + +- (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { + @synchronized(self) { + if (self.stream) { + dispatch_semaphore_t stopSem = dispatch_semaphore_create(0); + [self.stream stopCaptureWithCompletionHandler:^(NSError *error) { + dispatch_semaphore_signal(stopSem); + }]; + dispatch_semaphore_wait(stopSem, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + self.stream = nil; + } + + self.stopping = NO; + self.videoCallback = videoCallback; + self.captureSignal = dispatch_semaphore_create(0); + + SCDisplay *display = [self findDisplayWithIDRetrying:self.displayID]; + if (!display) { + NSLog(@"[SCCapture] Display not found after retries: %u", self.displayID); + return nil; + } + + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:display excludingWindows:@[]]; + + SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init]; + config.width = self.frameWidth; + config.height = self.frameHeight; + config.minimumFrameInterval = CMTimeMake(1, self.frameRate); + config.pixelFormat = self.pixelFormat; + config.queueDepth = 5; + config.showsCursor = YES; + + NSError *error = nil; + self.stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:self]; + + if (!self.stream) { + NSLog(@"[SCCapture] Failed to create SCStream"); + return nil; + } + + if (![self.stream addStreamOutput:self type:SCStreamOutputTypeScreen sampleHandlerQueue:self.videoQueue error:&error]) { + NSLog(@"[SCCapture] Failed to add video output: %@", error.localizedDescription); + return nil; + } + + dispatch_semaphore_t startSemaphore = dispatch_semaphore_create(0); + __block BOOL startSuccess = NO; + + [self.stream startCaptureWithCompletionHandler:^(NSError *error) { + if (error) { + NSLog(@"[SCCapture] Failed to start capture: %@", error.localizedDescription); + } else { + NSLog(@"[SCCapture] Capture started successfully"); + startSuccess = YES; + } + dispatch_semaphore_signal(startSemaphore); + }]; + + dispatch_semaphore_wait(startSemaphore, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + + if (!startSuccess) { + return nil; + } + + return self.captureSignal; + } +} + +- (void)stopCapture { + @synchronized(self) { + if (self.stream) { + dispatch_semaphore_t stopSemaphore = dispatch_semaphore_create(0); + + [self.stream stopCaptureWithCompletionHandler:^(NSError *error) { + if (error) { + NSLog(@"[SCCapture] Error stopping capture: %@", error.localizedDescription); + } + dispatch_semaphore_signal(stopSemaphore); + }]; + + dispatch_semaphore_wait(stopSemaphore, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + self.stream = nil; + } + + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); + self.captureSignal = nil; + } + + self.videoCallback = nil; + + if (self.lastValidSampleBuffer) { + CFRelease(self.lastValidSampleBuffer); + self.lastValidSampleBuffer = NULL; + } + } +} + +#pragma mark - SCStreamDelegate + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + NSLog(@"[SCCapture] Stream stopped with error: %@", error.localizedDescription); + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); + } +} + +#pragma mark - SCStreamOutput + +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeScreen) { + return; + } + + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixelBuffer) { + @synchronized(self) { + if (self.lastValidSampleBuffer && !self.stopping && self.videoCallback) { + self.videoCallback(self.lastValidSampleBuffer); + } + } + return; + } + + @synchronized(self) { + if (self.lastValidSampleBuffer) { + CFRelease(self.lastValidSampleBuffer); + } + self.lastValidSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); + } + + if (self.stopping) { + return; + } + + if (self.videoCallback && !self.videoCallback(sampleBuffer)) { + self.stopping = YES; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + [self stopCapture]; + }); + } +} + +@end From 65f798600218fc007dbefa05aa92f7b6a60cd27d Mon Sep 17 00:00:00 2001 From: marton Date: Wed, 13 May 2026 19:57:25 -0400 Subject: [PATCH 2/5] fix formatting --- src/platform/macos/display.mm | 2 +- src/platform/macos/sc_capture.h | 26 +++++++++++++------------- src/platform/macos/sc_capture.m | 9 ++++++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 3b8b6d4b19c..be2052e6058 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -80,7 +80,7 @@ static void clear_pixel_buffer(CVPixelBufferRef pixel_buffer) { static int make_dummy_img(img_t *img, int width, int height, OSType pixel_format, std::string_view backend_name) { CVPixelBufferRef pixel_buffer = nullptr; NSDictionary *attrs = @{ - (NSString *) kCVPixelBufferIOSurfacePropertiesKey: @{}, + (NSString *) kCVPixelBufferIOSurfacePropertiesKey: @ {}, }; auto status = CVPixelBufferCreate( diff --git a/src/platform/macos/sc_capture.h b/src/platform/macos/sc_capture.h index c449a5ed192..412bec42bb8 100644 --- a/src/platform/macos/sc_capture.h +++ b/src/platform/macos/sc_capture.h @@ -10,26 +10,26 @@ #import API_AVAILABLE(macos(12.3)) -@interface SCCapture : NSObject +@interface SCCapture: NSObject #define kMaxDisplays 32 typedef bool (^VideoFrameCallbackBlock)(CMSampleBufferRef); -@property(nonatomic, assign) CGDirectDisplayID displayID; -@property(nonatomic, assign) int frameRate; -@property(nonatomic, assign) OSType pixelFormat; -@property(nonatomic, assign) int frameWidth; -@property(nonatomic, assign) int frameHeight; +@property (nonatomic, assign) CGDirectDisplayID displayID; +@property (nonatomic, assign) int frameRate; +@property (nonatomic, assign) OSType pixelFormat; +@property (nonatomic, assign) int frameWidth; +@property (nonatomic, assign) int frameHeight; -@property(nonatomic, strong) SCStream *stream; -@property(nonatomic, strong) SCShareableContent *shareableContent; -@property(nonatomic, strong) dispatch_queue_t videoQueue; +@property (nonatomic, strong) SCStream *stream; +@property (nonatomic, strong) SCShareableContent *shareableContent; +@property (nonatomic, strong) dispatch_queue_t videoQueue; -@property(nonatomic, copy) VideoFrameCallbackBlock videoCallback; -@property(nonatomic, strong) dispatch_semaphore_t captureSignal; -@property(nonatomic, assign) BOOL stopping; -@property(nonatomic, assign) CMSampleBufferRef lastValidSampleBuffer; +@property (nonatomic, copy) VideoFrameCallbackBlock videoCallback; +@property (nonatomic, strong) dispatch_semaphore_t captureSignal; +@property (nonatomic, assign) BOOL stopping; +@property (nonatomic, assign) CMSampleBufferRef lastValidSampleBuffer; + (BOOL)isAvailable; + (NSArray *)displayNames; diff --git a/src/platform/macos/sc_capture.m b/src/platform/macos/sc_capture.m index 23f94a4249a..767267dbc86 100644 --- a/src/platform/macos/sc_capture.m +++ b/src/platform/macos/sc_capture.m @@ -63,7 +63,10 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID } dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class( - DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, DISPATCH_QUEUE_PRIORITY_HIGH); + DISPATCH_QUEUE_SERIAL, + QOS_CLASS_USER_INITIATED, + DISPATCH_QUEUE_PRIORITY_HIGH + ); self.videoQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckVideoQueue", qos); dispatch_semaphore_t initSemaphore = dispatch_semaphore_create(0); @@ -251,8 +254,8 @@ - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { #pragma mark - SCStreamOutput - (void)stream:(SCStream *)stream - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - ofType:(SCStreamOutputType)type { + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { if (type != SCStreamOutputTypeScreen) { return; } From ebee5c227f86c427e6a0ef2dac6a4fa2a4e5c08f Mon Sep 17 00:00:00 2001 From: marton Date: Thu, 14 May 2026 07:41:27 -0400 Subject: [PATCH 3/5] fix(macos): poll ScreenCaptureKit for sparse updates --- src/platform/macos/display.mm | 56 ++++++++---- src/platform/macos/sc_capture.h | 12 +-- src/platform/macos/sc_capture.m | 154 ++++++++++++++++++++++++++------ 3 files changed, 171 insertions(+), 51 deletions(-) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index be2052e6058..3d5e1e406bd 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -16,6 +16,7 @@ #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" #import "src/platform/macos/sc_capture.h" +#include "src/utility.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg @@ -26,6 +27,7 @@ namespace platf { using namespace std::literals; + static constexpr auto SCKIT_SCREENSHOT_POLL_INTERVAL_NS = NSEC_PER_SEC / 60; static bool process_frame(CMSampleBufferRef sampleBuffer, img_t *img) { auto pixel_buffer = CMSampleBufferGetImageBuffer(sampleBuffer); @@ -265,31 +267,51 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { - auto signal = [sc_capture captureVideo:^(CMSampleBufferRef sampleBuffer) { - std::shared_ptr img_out; - if (!pull_free_image_cb(img_out)) { - return false; + auto signal = [sc_capture captureVideo]; + if (!signal) { + BOOST_LOG(error) << "SCCapture failed to start video capture"sv; + return capture_e::error; + } + + auto frame_signal = sc_capture.frameSignal; + + while (true) { + auto frame_status = dispatch_semaphore_wait(frame_signal, dispatch_time(DISPATCH_TIME_NOW, SCKIT_SCREENSHOT_POLL_INTERVAL_NS)); + if (dispatch_semaphore_wait(signal, DISPATCH_TIME_NOW) == 0) { + break; } - if (!process_frame(sampleBuffer, img_out.get())) { - return true; + CMSampleBufferRef sampleBuffer = nullptr; + if (frame_status == 0) { + sampleBuffer = [sc_capture copyLatestSampleBuffer]; + } else { + sampleBuffer = [sc_capture copyScreenshotSampleBuffer]; } - if (!push_captured_image_cb(std::move(img_out), true)) { - return false; + if (!sampleBuffer) { + std::shared_ptr probe_img; + if (!pull_free_image_cb(probe_img)) { + [sc_capture stopCapture]; + break; + } + continue; } - return true; - }]; + auto release_sample_buffer = util::fail_guard([sampleBuffer]() { + CFRelease(sampleBuffer); + }); - if (!signal) { - BOOST_LOG(error) << "SCCapture failed to start video capture"sv; - return capture_e::error; - } + std::shared_ptr img_out; + if (!pull_free_image_cb(img_out)) { + [sc_capture stopCapture]; + break; + } - while (dispatch_semaphore_wait(signal, dispatch_time(DISPATCH_TIME_NOW, 1 * NSEC_PER_SEC)) != 0) { - std::shared_ptr probe_img; - if (!pull_free_image_cb(probe_img)) { + if (!process_frame(sampleBuffer, img_out.get())) { + continue; + } + + if (!push_captured_image_cb(std::move(img_out), true)) { [sc_capture stopCapture]; break; } diff --git a/src/platform/macos/sc_capture.h b/src/platform/macos/sc_capture.h index 412bec42bb8..1c7f29860f7 100644 --- a/src/platform/macos/sc_capture.h +++ b/src/platform/macos/sc_capture.h @@ -14,8 +14,6 @@ API_AVAILABLE(macos(12.3)) #define kMaxDisplays 32 -typedef bool (^VideoFrameCallbackBlock)(CMSampleBufferRef); - @property (nonatomic, assign) CGDirectDisplayID displayID; @property (nonatomic, assign) int frameRate; @property (nonatomic, assign) OSType pixelFormat; @@ -23,13 +21,15 @@ typedef bool (^VideoFrameCallbackBlock)(CMSampleBufferRef); @property (nonatomic, assign) int frameHeight; @property (nonatomic, strong) SCStream *stream; +@property (nonatomic, strong) SCContentFilter *contentFilter; +@property (nonatomic, strong) SCStreamConfiguration *streamConfiguration; @property (nonatomic, strong) SCShareableContent *shareableContent; @property (nonatomic, strong) dispatch_queue_t videoQueue; -@property (nonatomic, copy) VideoFrameCallbackBlock videoCallback; @property (nonatomic, strong) dispatch_semaphore_t captureSignal; +@property (nonatomic, strong) dispatch_semaphore_t frameSignal; @property (nonatomic, assign) BOOL stopping; -@property (nonatomic, assign) CMSampleBufferRef lastValidSampleBuffer; +@property (nonatomic, assign) CMSampleBufferRef latestSampleBuffer; + (BOOL)isAvailable; + (NSArray *)displayNames; @@ -39,7 +39,9 @@ typedef bool (^VideoFrameCallbackBlock)(CMSampleBufferRef); frameRate:(int)frameRate; - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; -- (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback; +- (dispatch_semaphore_t)captureVideo; +- (CMSampleBufferRef)copyLatestSampleBuffer; +- (CMSampleBufferRef)copyScreenshotSampleBuffer; - (void)stopCapture; @end diff --git a/src/platform/macos/sc_capture.m b/src/platform/macos/sc_capture.m index 767267dbc86..9a4522efd90 100644 --- a/src/platform/macos/sc_capture.m +++ b/src/platform/macos/sc_capture.m @@ -7,6 +7,33 @@ API_AVAILABLE(macos(12.3)) @implementation SCCapture +static BOOL isCompleteScreenFrame(CMSampleBufferRef sampleBuffer) { + if (!CMSampleBufferIsValid(sampleBuffer)) { + return NO; + } + + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, NO); + if (!attachmentsArray || CFArrayGetCount(attachmentsArray) == 0) { + return NO; + } + + CFDictionaryRef attachments = (CFDictionaryRef) CFArrayGetValueAtIndex(attachmentsArray, 0); + if (!attachments) { + return NO; + } + + NSNumber *status = (NSNumber *) CFDictionaryGetValue(attachments, SCStreamFrameInfoStatus); + if (!status) { + return NO; + } + + return status.integerValue == SCFrameStatusComplete; +} + +static BOOL isUsableImageSampleBuffer(CMSampleBufferRef sampleBuffer) { + return sampleBuffer && CMSampleBufferIsValid(sampleBuffer) && CMSampleBufferGetImageBuffer(sampleBuffer); +} + + (BOOL)isAvailable { if (@available(macOS 12.3, *)) { return YES; @@ -145,7 +172,7 @@ - (SCDisplay *)findDisplayWithIDRetrying:(CGDirectDisplayID)displayID { return nil; } -- (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { +- (dispatch_semaphore_t)captureVideo { @synchronized(self) { if (self.stream) { dispatch_semaphore_t stopSem = dispatch_semaphore_create(0); @@ -156,9 +183,14 @@ - (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { self.stream = nil; } + if (self.latestSampleBuffer) { + CFRelease(self.latestSampleBuffer); + self.latestSampleBuffer = NULL; + } + self.stopping = NO; - self.videoCallback = videoCallback; self.captureSignal = dispatch_semaphore_create(0); + self.frameSignal = dispatch_semaphore_create(0); SCDisplay *display = [self findDisplayWithIDRetrying:self.displayID]; if (!display) { @@ -167,6 +199,8 @@ - (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { } SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:display excludingWindows:@[]]; + self.contentFilter = filter; + [filter release]; SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init]; config.width = self.frameWidth; @@ -175,9 +209,13 @@ - (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { config.pixelFormat = self.pixelFormat; config.queueDepth = 5; config.showsCursor = YES; + self.streamConfiguration = config; + [config release]; NSError *error = nil; - self.stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:self]; + SCStream *stream = [[SCStream alloc] initWithFilter:self.contentFilter configuration:self.streamConfiguration delegate:self]; + self.stream = stream; + [stream release]; if (!self.stream) { NSLog(@"[SCCapture] Failed to create SCStream"); @@ -205,6 +243,10 @@ - (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { dispatch_semaphore_wait(startSemaphore, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); if (!startSuccess) { + self.captureSignal = nil; + self.frameSignal = nil; + self.contentFilter = nil; + self.streamConfiguration = nil; return nil; } @@ -212,8 +254,59 @@ - (dispatch_semaphore_t)captureVideo:(VideoFrameCallbackBlock)videoCallback { } } +- (CMSampleBufferRef)copyLatestSampleBuffer { + @synchronized(self) { + CMSampleBufferRef sampleBuffer = self.latestSampleBuffer; + self.latestSampleBuffer = NULL; + return sampleBuffer; + } +} + +- (CMSampleBufferRef)copyScreenshotSampleBuffer { + if (@available(macOS 14.0, *)) { + SCContentFilter *filter = nil; + SCStreamConfiguration *config = nil; + + @synchronized(self) { + if (self.stopping || !self.contentFilter || !self.streamConfiguration) { + return NULL; + } + + filter = [self.contentFilter retain]; + config = [self.streamConfiguration retain]; + } + + dispatch_semaphore_t screenshotSemaphore = dispatch_semaphore_create(0); + __block BOOL timedOut = NO; + __block CMSampleBufferRef screenshotSampleBuffer = NULL; + + [SCScreenshotManager captureSampleBufferWithFilter:filter + configuration:config + completionHandler:^(CMSampleBufferRef sampleBuffer, NSError *error) { + if (!timedOut && !error && isUsableImageSampleBuffer(sampleBuffer)) { + screenshotSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); + } + + dispatch_semaphore_signal(screenshotSemaphore); + }]; + + if (dispatch_semaphore_wait(screenshotSemaphore, dispatch_time(DISPATCH_TIME_NOW, 500 * NSEC_PER_MSEC)) != 0) { + timedOut = YES; + } + + [filter release]; + [config release]; + + return screenshotSampleBuffer; + } + + return NULL; +} + - (void)stopCapture { @synchronized(self) { + self.stopping = YES; + if (self.stream) { dispatch_semaphore_t stopSemaphore = dispatch_semaphore_create(0); @@ -228,16 +321,20 @@ - (void)stopCapture { self.stream = nil; } - if (self.captureSignal) { - dispatch_semaphore_signal(self.captureSignal); - self.captureSignal = nil; + self.contentFilter = nil; + self.streamConfiguration = nil; + + if (self.latestSampleBuffer) { + CFRelease(self.latestSampleBuffer); + self.latestSampleBuffer = NULL; } - self.videoCallback = nil; + if (self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + } - if (self.lastValidSampleBuffer) { - CFRelease(self.lastValidSampleBuffer); - self.lastValidSampleBuffer = NULL; + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); } } } @@ -246,6 +343,9 @@ - (void)stopCapture { - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { NSLog(@"[SCCapture] Stream stopped with error: %@", error.localizedDescription); + if (self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + } if (self.captureSignal) { dispatch_semaphore_signal(self.captureSignal); } @@ -260,32 +360,28 @@ - (void)stream:(SCStream *)stream return; } - CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - if (!pixelBuffer) { - @synchronized(self) { - if (self.lastValidSampleBuffer && !self.stopping && self.videoCallback) { - self.videoCallback(self.lastValidSampleBuffer); - } - } + if (!isCompleteScreenFrame(sampleBuffer)) { + return; + } + + if (!isUsableImageSampleBuffer(sampleBuffer)) { return; } @synchronized(self) { - if (self.lastValidSampleBuffer) { - CFRelease(self.lastValidSampleBuffer); + if (self.stopping) { + return; } - self.lastValidSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); - } - if (self.stopping) { - return; - } + BOOL shouldSignal = self.latestSampleBuffer == NULL; + if (self.latestSampleBuffer) { + CFRelease(self.latestSampleBuffer); + } + self.latestSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); - if (self.videoCallback && !self.videoCallback(sampleBuffer)) { - self.stopping = YES; - dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ - [self stopCapture]; - }); + if (shouldSignal && self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + } } } From 58901ed0f21e81a3cbdaa100644309358351a540 Mon Sep 17 00:00:00 2001 From: marton Date: Fri, 15 May 2026 14:43:22 -0400 Subject: [PATCH 4/5] refactor, fix blurriness - SCK scaler is garbage --- src/platform/macos/display.mm | 28 ++- src/platform/macos/nv12_zero_device.cpp | 164 +++++++++++-- src/platform/macos/nv12_zero_device.h | 18 +- src/platform/macos/sc_capture.h | 8 +- src/platform/macos/sc_capture.m | 303 ++++++++++++++---------- 5 files changed, 364 insertions(+), 157 deletions(-) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 3d5e1e406bd..cb614a93e22 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -272,8 +272,21 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const BOOST_LOG(error) << "SCCapture failed to start video capture"sv; return capture_e::error; } + dispatch_retain(signal); auto frame_signal = sc_capture.frameSignal; + if (!frame_signal) { + BOOST_LOG(error) << "SCCapture failed to create frame signal"sv; + dispatch_release(signal); + [sc_capture stopCapture]; + return capture_e::error; + } + dispatch_retain(frame_signal); + + auto release_signals = util::fail_guard([signal, frame_signal]() { + dispatch_release(frame_signal); + dispatch_release(signal); + }); while (true) { auto frame_status = dispatch_semaphore_wait(frame_signal, dispatch_time(DISPATCH_TIME_NOW, SCKIT_SCREENSHOT_POLL_INTERVAL_NS)); @@ -281,14 +294,13 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const break; } - CMSampleBufferRef sampleBuffer = nullptr; - if (frame_status == 0) { - sampleBuffer = [sc_capture copyLatestSampleBuffer]; - } else { - sampleBuffer = [sc_capture copyScreenshotSampleBuffer]; - } + CMSampleBufferRef sampleBuffer = [sc_capture copyLatestSampleBuffer]; if (!sampleBuffer) { + if (frame_status != 0) { + [sc_capture requestScreenshotSampleBuffer]; + } + std::shared_ptr probe_img; if (!pull_free_image_cb(probe_img)) { [sc_capture stopCapture]; @@ -332,7 +344,9 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init(static_cast(sc_capture), pix_fmt, setResolution, setPixelFormat); + // SCK's scaler visibly softens even small upscales. Capture at native resolution + // and let VideoToolbox resize into the encoder-sized CVPixelBuffer instead. + device->init(static_cast(sc_capture), pix_fmt, setResolution, setPixelFormat, false); return device; } else { diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index b4d6aa33f47..0ba3199f27a 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -3,9 +3,12 @@ * @brief Definitions for NV12 zero copy device on macOS. */ // standard includes +#include +#include #include // local includes +#include "src/logging.h" #include "src/platform/macos/av_img_t.h" #include "src/platform/macos/nv12_zero_device.h" #include "src/video.h" @@ -26,44 +29,175 @@ namespace platf { util::safe_ptr av_frame; - int nv12_zero_device::convert(platf::img_t &img) { - auto *av_img = (av_img_t *) &img; + nv12_zero_device::~nv12_zero_device() { + if (transfer_session) { + VTPixelTransferSessionInvalidate(transfer_session); + CFRelease(transfer_session); + } - if (!av_img->pixel_buffer || !av_img->pixel_buffer->buf) { - return -1; + if (pixel_buffer_pool) { + CFRelease(pixel_buffer_pool); } + } - // Release any existing CVPixelBuffer previously retained for encoding + int nv12_zero_device::attach_pixel_buffer(CVPixelBufferRef pixel_buffer) { av_buffer_unref(&av_frame->buf[0]); - - // Attach an AVBufferRef to this frame which will retain ownership of the CVPixelBuffer - // until av_buffer_unref() is called (above) or the frame is freed with av_frame_free(). - // - // The presence of the AVBufferRef allows FFmpeg to simply add a reference to the buffer - // rather than having to perform a deep copy of the data buffers in avcodec_send_frame(). - av_frame->buf[0] = av_buffer_create((uint8_t *) CFRetain(av_img->pixel_buffer->buf), 0, free_buffer, nullptr, 0); + av_frame->buf[0] = av_buffer_create((uint8_t *) pixel_buffer, 0, free_buffer, nullptr, 0); + if (!av_frame->buf[0]) { + CVPixelBufferRelease(pixel_buffer); + return -1; + } // Place a CVPixelBufferRef at data[3] as required by AV_PIX_FMT_VIDEOTOOLBOX - av_frame->data[3] = (uint8_t *) av_img->pixel_buffer->buf; + av_frame->data[3] = (uint8_t *) pixel_buffer; + + return 0; + } + + int nv12_zero_device::ensure_pixel_buffer_pool(OSType pixel_format) { + if (pixel_buffer_pool && pool_pixel_format == pixel_format && pool_width == av_frame->width && pool_height == av_frame->height) { + return 0; + } + + if (pixel_buffer_pool) { + CFRelease(pixel_buffer_pool); + pixel_buffer_pool = nullptr; + } + + auto width = av_frame->width; + auto height = av_frame->height; + auto pixel_format_value = static_cast(pixel_format); + CFNumberRef width_number = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &width); + CFNumberRef height_number = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &height); + CFNumberRef pixel_format_number = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixel_format_value); + CFDictionaryRef io_surface_properties = CFDictionaryCreate( + kCFAllocatorDefault, + nullptr, + nullptr, + 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks + ); + + const void *keys[] = { + kCVPixelBufferPixelFormatTypeKey, + kCVPixelBufferWidthKey, + kCVPixelBufferHeightKey, + kCVPixelBufferIOSurfacePropertiesKey, + }; + const void *values[] = { + pixel_format_number, + width_number, + height_number, + io_surface_properties, + }; + CFDictionaryRef attrs = CFDictionaryCreate( + kCFAllocatorDefault, + keys, + values, + 4, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks + ); + + auto status = CVPixelBufferPoolCreate(kCFAllocatorDefault, nullptr, attrs, &pixel_buffer_pool); + + CFRelease(attrs); + CFRelease(io_surface_properties); + CFRelease(pixel_format_number); + CFRelease(height_number); + CFRelease(width_number); + + if (status != kCVReturnSuccess || !pixel_buffer_pool) { + BOOST_LOG(error) << "Failed to create VideoToolbox pixel transfer pool: " << status; + return -1; + } + + pool_pixel_format = pixel_format; + pool_width = width; + pool_height = height; + + return 0; + } + + int nv12_zero_device::ensure_transfer_session() { + if (transfer_session) { + return 0; + } + + auto status = VTPixelTransferSessionCreate(kCFAllocatorDefault, &transfer_session); + if (status != noErr || !transfer_session) { + BOOST_LOG(error) << "Failed to create VideoToolbox pixel transfer session: " << status; + return -1; + } + + VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Normal); + VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_RealTime, kCFBooleanTrue); + VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_DownsamplingMode, kVTDownsamplingMode_Average); return 0; } + CVPixelBufferRef nv12_zero_device::copy_scaled_pixel_buffer(CVPixelBufferRef pixel_buffer) { + if (CVPixelBufferGetWidth(pixel_buffer) == av_frame->width && CVPixelBufferGetHeight(pixel_buffer) == av_frame->height) { + return (CVPixelBufferRef) CFRetain(pixel_buffer); + } + + auto pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); + if (ensure_pixel_buffer_pool(pixel_format) || ensure_transfer_session()) { + return nullptr; + } + + CVPixelBufferRef scaled_pixel_buffer = nullptr; + auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixel_buffer_pool, &scaled_pixel_buffer); + if (status != kCVReturnSuccess || !scaled_pixel_buffer) { + BOOST_LOG(error) << "Failed to create VideoToolbox pixel transfer buffer: " << status; + return nullptr; + } + + status = VTPixelTransferSessionTransferImage(transfer_session, pixel_buffer, scaled_pixel_buffer); + if (status != noErr) { + BOOST_LOG(error) << "VideoToolbox pixel transfer failed: " << status; + CVPixelBufferRelease(scaled_pixel_buffer); + return nullptr; + } + + return scaled_pixel_buffer; + } + + int nv12_zero_device::convert(platf::img_t &img) { + auto *av_img = (av_img_t *) &img; + + if (!av_img->pixel_buffer || !av_img->pixel_buffer->buf) { + return -1; + } + + CVPixelBufferRef pixel_buffer = copy_scaled_pixel_buffer(av_img->pixel_buffer->buf); + if (!pixel_buffer) { + return -1; + } + + return attach_pixel_buffer(pixel_buffer); + } + int nv12_zero_device::set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) { this->frame = frame; av_frame.reset(frame); - resolution_fn(this->display, frame->width, frame->height); + if (resize_capture) { + resolution_fn(this->display, frame->width, frame->height); + } return 0; } - int nv12_zero_device::init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn) { + int nv12_zero_device::init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn, bool resize_capture) { pixel_format_fn(display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); this->display = display; this->resolution_fn = std::move(resolution_fn); + this->resize_capture = resize_capture; // we never use this pointer, but its existence is checked/used // by the platform independent code diff --git a/src/platform/macos/nv12_zero_device.h b/src/platform/macos/nv12_zero_device.h index 3aa7540c3f3..3173c5117ca 100644 --- a/src/platform/macos/nv12_zero_device.h +++ b/src/platform/macos/nv12_zero_device.h @@ -7,6 +7,9 @@ // local includes #include "src/platform/common.h" +// platform includes +#include + struct AVFrame; namespace platf { @@ -24,13 +27,26 @@ namespace platf { resolution_fn_t resolution_fn; using pixel_format_fn_t = std::function; - int init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn); + ~nv12_zero_device() override; + + int init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn, bool resize_capture = true); int convert(img_t &img) override; int set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override; private: util::safe_ptr av_frame; + VTPixelTransferSessionRef transfer_session {}; + CVPixelBufferPoolRef pixel_buffer_pool {}; + OSType pool_pixel_format {}; + int pool_width {}; + int pool_height {}; + bool resize_capture {}; + + int attach_pixel_buffer(CVPixelBufferRef pixel_buffer); + CVPixelBufferRef copy_scaled_pixel_buffer(CVPixelBufferRef pixel_buffer); + int ensure_pixel_buffer_pool(OSType pixel_format); + int ensure_transfer_session(); }; } // namespace platf diff --git a/src/platform/macos/sc_capture.h b/src/platform/macos/sc_capture.h index 1c7f29860f7..22ca9907d78 100644 --- a/src/platform/macos/sc_capture.h +++ b/src/platform/macos/sc_capture.h @@ -24,10 +24,10 @@ API_AVAILABLE(macos(12.3)) @property (nonatomic, strong) SCContentFilter *contentFilter; @property (nonatomic, strong) SCStreamConfiguration *streamConfiguration; @property (nonatomic, strong) SCShareableContent *shareableContent; -@property (nonatomic, strong) dispatch_queue_t videoQueue; +@property (nonatomic, assign) dispatch_queue_t videoQueue; -@property (nonatomic, strong) dispatch_semaphore_t captureSignal; -@property (nonatomic, strong) dispatch_semaphore_t frameSignal; +@property (nonatomic, assign) dispatch_semaphore_t captureSignal; +@property (nonatomic, assign) dispatch_semaphore_t frameSignal; @property (nonatomic, assign) BOOL stopping; @property (nonatomic, assign) CMSampleBufferRef latestSampleBuffer; @@ -41,7 +41,7 @@ API_AVAILABLE(macos(12.3)) - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; - (dispatch_semaphore_t)captureVideo; - (CMSampleBufferRef)copyLatestSampleBuffer; -- (CMSampleBufferRef)copyScreenshotSampleBuffer; +- (void)requestScreenshotSampleBuffer; - (void)stopCapture; @end diff --git a/src/platform/macos/sc_capture.m b/src/platform/macos/sc_capture.m index 9a4522efd90..a3a45a1ef61 100644 --- a/src/platform/macos/sc_capture.m +++ b/src/platform/macos/sc_capture.m @@ -4,36 +4,58 @@ */ #import "sc_capture.h" -API_AVAILABLE(macos(12.3)) -@implementation SCCapture +static SCShareableContent *copyShareableContent(void) { + __block SCShareableContent *shareableContent = nil; + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); -static BOOL isCompleteScreenFrame(CMSampleBufferRef sampleBuffer) { - if (!CMSampleBufferIsValid(sampleBuffer)) { - return NO; - } + [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) { + if (error) { + NSLog(@"[SCCapture] Failed to get shareable content: %@", error.localizedDescription); + } else { + shareableContent = [content retain]; + } - CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, NO); - if (!attachmentsArray || CFArrayGetCount(attachmentsArray) == 0) { - return NO; - } + dispatch_semaphore_signal(semaphore); + }]; - CFDictionaryRef attachments = (CFDictionaryRef) CFArrayGetValueAtIndex(attachmentsArray, 0); - if (!attachments) { - return NO; - } + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + dispatch_release(semaphore); - NSNumber *status = (NSNumber *) CFDictionaryGetValue(attachments, SCStreamFrameInfoStatus); - if (!status) { - return NO; + return shareableContent; +} + +static BOOL isCompleteScreenFrame(CMSampleBufferRef sampleBuffer) { + if (sampleBuffer && CMSampleBufferIsValid(sampleBuffer)) { + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, NO); + if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) { + CFDictionaryRef attachments = (CFDictionaryRef) CFArrayGetValueAtIndex(attachmentsArray, 0); + NSNumber *status = (NSNumber *) CFDictionaryGetValue(attachments, SCStreamFrameInfoStatus); + return status && status.integerValue == SCFrameStatusComplete; + } } - return status.integerValue == SCFrameStatusComplete; + return NO; } static BOOL isUsableImageSampleBuffer(CMSampleBufferRef sampleBuffer) { return sampleBuffer && CMSampleBufferIsValid(sampleBuffer) && CMSampleBufferGetImageBuffer(sampleBuffer); } +API_AVAILABLE(macos(12.3)) +@interface SCCapture () + +@property (nonatomic, assign) BOOL screenshotInFlight; + +- (void)finishScreenshotSampleBuffer:(CMSampleBufferRef)sampleBuffer + error:(NSError *)error + filter:(SCContentFilter *)filter + configuration:(SCStreamConfiguration *)config; + +@end + +API_AVAILABLE(macos(12.3)) +@implementation SCCapture + + (BOOL)isAvailable { if (@available(macOS 12.3, *)) { return YES; @@ -67,6 +89,7 @@ + (NSString *)getDisplayName:(CGDirectDisplayID)displayID { return screen.localizedName; } } + return nil; } @@ -96,30 +119,28 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID ); self.videoQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckVideoQueue", qos); - dispatch_semaphore_t initSemaphore = dispatch_semaphore_create(0); - __block BOOL initSuccess = NO; - - [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) { - if (error) { - NSLog(@"[SCCapture] Failed to get shareable content: %@", error.localizedDescription); - } else { - self.shareableContent = content; - initSuccess = YES; - } - dispatch_semaphore_signal(initSemaphore); - }]; - - dispatch_semaphore_wait(initSemaphore, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); - - if (!initSuccess) { + SCShareableContent *content = copyShareableContent(); + if (!content) { + [self release]; return nil; } + + self.shareableContent = content; + [content release]; } + return self; } - (void)dealloc { [self stopCapture]; + self.shareableContent = nil; + + if (self.videoQueue) { + dispatch_release(self.videoQueue); + self.videoQueue = NULL; + } + [super dealloc]; } @@ -134,6 +155,7 @@ - (SCDisplay *)findDisplayWithID:(CGDirectDisplayID)displayID { return display; } } + return nil; } @@ -147,54 +169,97 @@ - (SCDisplay *)findDisplayWithIDRetrying:(CGDirectDisplayID)displayID { NSLog(@"[SCCapture] Display %u not found in SCShareableContent, refreshing (attempt %d/3)", displayID, attempt); [NSThread sleepForTimeInterval:1.0]; - dispatch_semaphore_t sem = dispatch_semaphore_create(0); - __block BOOL success = NO; - - [SCShareableContent getShareableContentWithCompletionHandler:^(SCShareableContent *content, NSError *error) { - if (!error && content) { - self.shareableContent = content; - success = YES; - } - dispatch_semaphore_signal(sem); - }]; + SCShareableContent *content = copyShareableContent(); + if (!content) { + continue; + } - dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + self.shareableContent = content; + [content release]; - if (success) { - display = [self findDisplayWithID:displayID]; - if (display) { - NSLog(@"[SCCapture] Found display %u after refresh", displayID); - return display; - } + display = [self findDisplayWithID:displayID]; + if (display) { + NSLog(@"[SCCapture] Found display %u after refresh", displayID); + return display; } } return nil; } -- (dispatch_semaphore_t)captureVideo { +- (void)releaseCaptureSignals { + if (self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); + dispatch_release(self.frameSignal); + self.frameSignal = NULL; + } + + if (self.captureSignal) { + dispatch_semaphore_signal(self.captureSignal); + dispatch_release(self.captureSignal); + self.captureSignal = NULL; + } +} + +- (void)stopCurrentStream { + if (!self.stream) { + return; + } + + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); + + [self.stream stopCaptureWithCompletionHandler:^(NSError *error) { + if (error) { + NSLog(@"[SCCapture] Error stopping capture: %@", error.localizedDescription); + } + + dispatch_semaphore_signal(semaphore); + }]; + + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + dispatch_release(semaphore); + self.stream = nil; +} + +- (void)clearLatestSampleBuffer { + if (self.latestSampleBuffer) { + CFRelease(self.latestSampleBuffer); + self.latestSampleBuffer = NULL; + } +} + +- (void)storeSampleBuffer:(CMSampleBufferRef)sampleBuffer { @synchronized(self) { - if (self.stream) { - dispatch_semaphore_t stopSem = dispatch_semaphore_create(0); - [self.stream stopCaptureWithCompletionHandler:^(NSError *error) { - dispatch_semaphore_signal(stopSem); - }]; - dispatch_semaphore_wait(stopSem, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); - self.stream = nil; + if (self.stopping) { + return; } - if (self.latestSampleBuffer) { - CFRelease(self.latestSampleBuffer); - self.latestSampleBuffer = NULL; + BOOL shouldSignal = self.latestSampleBuffer == NULL; + + [self clearLatestSampleBuffer]; + self.latestSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); + + if (shouldSignal && self.frameSignal) { + dispatch_semaphore_signal(self.frameSignal); } + } +} + +- (dispatch_semaphore_t)captureVideo { + @synchronized(self) { + [self stopCurrentStream]; + [self clearLatestSampleBuffer]; + [self releaseCaptureSignals]; self.stopping = NO; + self.screenshotInFlight = NO; self.captureSignal = dispatch_semaphore_create(0); self.frameSignal = dispatch_semaphore_create(0); SCDisplay *display = [self findDisplayWithIDRetrying:self.displayID]; if (!display) { NSLog(@"[SCCapture] Display not found after retries: %u", self.displayID); + [self releaseCaptureSignals]; return nil; } @@ -209,6 +274,10 @@ - (dispatch_semaphore_t)captureVideo { config.pixelFormat = self.pixelFormat; config.queueDepth = 5; config.showsCursor = YES; + if (@available(macOS 14.0, *)) { + config.captureResolution = SCCaptureResolutionBest; + config.preservesAspectRatio = YES; + } self.streamConfiguration = config; [config release]; @@ -219,15 +288,22 @@ - (dispatch_semaphore_t)captureVideo { if (!self.stream) { NSLog(@"[SCCapture] Failed to create SCStream"); + self.contentFilter = nil; + self.streamConfiguration = nil; + [self releaseCaptureSignals]; return nil; } if (![self.stream addStreamOutput:self type:SCStreamOutputTypeScreen sampleHandlerQueue:self.videoQueue error:&error]) { NSLog(@"[SCCapture] Failed to add video output: %@", error.localizedDescription); + self.stream = nil; + self.contentFilter = nil; + self.streamConfiguration = nil; + [self releaseCaptureSignals]; return nil; } - dispatch_semaphore_t startSemaphore = dispatch_semaphore_create(0); + dispatch_semaphore_t semaphore = dispatch_semaphore_create(0); __block BOOL startSuccess = NO; [self.stream startCaptureWithCompletionHandler:^(NSError *error) { @@ -237,16 +313,18 @@ - (dispatch_semaphore_t)captureVideo { NSLog(@"[SCCapture] Capture started successfully"); startSuccess = YES; } - dispatch_semaphore_signal(startSemaphore); + + dispatch_semaphore_signal(semaphore); }]; - dispatch_semaphore_wait(startSemaphore, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER); + dispatch_release(semaphore); if (!startSuccess) { - self.captureSignal = nil; - self.frameSignal = nil; + self.stream = nil; self.contentFilter = nil; self.streamConfiguration = nil; + [self releaseCaptureSignals]; return nil; } @@ -262,80 +340,57 @@ - (CMSampleBufferRef)copyLatestSampleBuffer { } } -- (CMSampleBufferRef)copyScreenshotSampleBuffer { +- (void)finishScreenshotSampleBuffer:(CMSampleBufferRef)sampleBuffer + error:(NSError *)error + filter:(SCContentFilter *)filter + configuration:(SCStreamConfiguration *)config { + @synchronized(self) { + self.screenshotInFlight = NO; + } + + if (!error && isUsableImageSampleBuffer(sampleBuffer)) { + [self storeSampleBuffer:sampleBuffer]; + } + + [filter release]; + [config release]; +} + +- (void)requestScreenshotSampleBuffer { if (@available(macOS 14.0, *)) { SCContentFilter *filter = nil; SCStreamConfiguration *config = nil; @synchronized(self) { - if (self.stopping || !self.contentFilter || !self.streamConfiguration) { - return NULL; + if (self.stopping || self.screenshotInFlight || !self.contentFilter || !self.streamConfiguration) { + return; } + self.screenshotInFlight = YES; filter = [self.contentFilter retain]; config = [self.streamConfiguration retain]; } - dispatch_semaphore_t screenshotSemaphore = dispatch_semaphore_create(0); - __block BOOL timedOut = NO; - __block CMSampleBufferRef screenshotSampleBuffer = NULL; - [SCScreenshotManager captureSampleBufferWithFilter:filter configuration:config completionHandler:^(CMSampleBufferRef sampleBuffer, NSError *error) { - if (!timedOut && !error && isUsableImageSampleBuffer(sampleBuffer)) { - screenshotSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); - } - - dispatch_semaphore_signal(screenshotSemaphore); + [self finishScreenshotSampleBuffer:sampleBuffer error:error filter:filter configuration:config]; }]; - - if (dispatch_semaphore_wait(screenshotSemaphore, dispatch_time(DISPATCH_TIME_NOW, 500 * NSEC_PER_MSEC)) != 0) { - timedOut = YES; - } - - [filter release]; - [config release]; - - return screenshotSampleBuffer; } - - return NULL; } - (void)stopCapture { @synchronized(self) { self.stopping = YES; + self.screenshotInFlight = NO; - if (self.stream) { - dispatch_semaphore_t stopSemaphore = dispatch_semaphore_create(0); - - [self.stream stopCaptureWithCompletionHandler:^(NSError *error) { - if (error) { - NSLog(@"[SCCapture] Error stopping capture: %@", error.localizedDescription); - } - dispatch_semaphore_signal(stopSemaphore); - }]; - - dispatch_semaphore_wait(stopSemaphore, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); - self.stream = nil; - } + [self stopCurrentStream]; self.contentFilter = nil; self.streamConfiguration = nil; - if (self.latestSampleBuffer) { - CFRelease(self.latestSampleBuffer); - self.latestSampleBuffer = NULL; - } - - if (self.frameSignal) { - dispatch_semaphore_signal(self.frameSignal); - } - - if (self.captureSignal) { - dispatch_semaphore_signal(self.captureSignal); - } + [self clearLatestSampleBuffer]; + [self releaseCaptureSignals]; } } @@ -343,9 +398,11 @@ - (void)stopCapture { - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { NSLog(@"[SCCapture] Stream stopped with error: %@", error.localizedDescription); + if (self.frameSignal) { dispatch_semaphore_signal(self.frameSignal); } + if (self.captureSignal) { dispatch_semaphore_signal(self.captureSignal); } @@ -368,21 +425,7 @@ - (void)stream:(SCStream *)stream return; } - @synchronized(self) { - if (self.stopping) { - return; - } - - BOOL shouldSignal = self.latestSampleBuffer == NULL; - if (self.latestSampleBuffer) { - CFRelease(self.latestSampleBuffer); - } - self.latestSampleBuffer = (CMSampleBufferRef) CFRetain(sampleBuffer); - - if (shouldSignal && self.frameSignal) { - dispatch_semaphore_signal(self.frameSignal); - } - } + [self storeSampleBuffer:sampleBuffer]; } @end From 895a19cca7dee61dbd3f78d8c09da54dc3145267 Mon Sep 17 00:00:00 2001 From: marton Date: Fri, 15 May 2026 15:54:06 -0400 Subject: [PATCH 5/5] fix aspect ratio, because VideoToolbox letterbox does not do what it says --- src/platform/macos/nv12_zero_device.cpp | 125 +++++++++++++++++++++++- src/platform/macos/nv12_zero_device.h | 6 ++ 2 files changed, 126 insertions(+), 5 deletions(-) diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index 0ba3199f27a..bc5276cd2a3 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -4,9 +4,13 @@ */ // standard includes #include +#include #include #include +// platform includes +#include + // local includes #include "src/logging.h" #include "src/platform/macos/av_img_t.h" @@ -63,6 +67,7 @@ namespace platf { CFRelease(pixel_buffer_pool); pixel_buffer_pool = nullptr; } + black_surfaces.clear(); auto width = av_frame->width; auto height = av_frame->height; @@ -131,20 +136,87 @@ namespace platf { return -1; } - VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_Normal); + VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_CropSourceToCleanAperture); VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_RealTime, kCFBooleanTrue); VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_DownsamplingMode, kVTDownsamplingMode_Average); return 0; } + int nv12_zero_device::prefill_black(CVPixelBufferRef pixel_buffer) { + auto surface = CVPixelBufferGetIOSurface(pixel_buffer); + if (surface && black_surfaces.find(surface) != black_surfaces.end()) { + return 0; + } + + auto pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); + AVPixelFormat av_pixel_format; + switch (pixel_format) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + av_pixel_format = AV_PIX_FMT_NV12; + break; + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + av_pixel_format = AV_PIX_FMT_P010; + break; + default: + av_pixel_format = AV_PIX_FMT_NONE; + break; + } + + if (av_pixel_format == AV_PIX_FMT_NONE) { + BOOST_LOG(error) << "Unsupported pixel format for VideoToolbox black fill: " << pixel_format; + return -1; + } + + auto status = CVPixelBufferLockBaseAddress(pixel_buffer, 0); + if (status != kCVReturnSuccess) { + BOOST_LOG(error) << "Failed to lock VideoToolbox pixel transfer buffer: " << status; + return -1; + } + + uint8_t *data[4] {}; + ptrdiff_t linesize[4] {}; + auto plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + for (size_t plane = 0; plane < plane_count && plane < 4; ++plane) { + data[plane] = static_cast(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane)); + linesize[plane] = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); + } + + if (plane_count < 2 || !data[0] || !data[1]) { + CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + BOOST_LOG(error) << "VideoToolbox pixel transfer buffer has no writable YUV planes"; + return -1; + } + + auto result = av_image_fill_black( + data, + linesize, + av_pixel_format, + AVCOL_RANGE_MPEG, + static_cast(CVPixelBufferGetWidth(pixel_buffer)), + static_cast(CVPixelBufferGetHeight(pixel_buffer)) + ); + CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + + if (result < 0) { + BOOST_LOG(error) << "Failed to black-fill VideoToolbox pixel transfer buffer: " << result; + return -1; + } + + if (surface) { + black_surfaces.insert(surface); + } + + return 0; + } + CVPixelBufferRef nv12_zero_device::copy_scaled_pixel_buffer(CVPixelBufferRef pixel_buffer) { - if (CVPixelBufferGetWidth(pixel_buffer) == av_frame->width && CVPixelBufferGetHeight(pixel_buffer) == av_frame->height) { + auto source_pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); + if (source_pixel_format == output_pixel_format && CVPixelBufferGetWidth(pixel_buffer) == av_frame->width && CVPixelBufferGetHeight(pixel_buffer) == av_frame->height) { return (CVPixelBufferRef) CFRetain(pixel_buffer); } - auto pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); - if (ensure_pixel_buffer_pool(pixel_format) || ensure_transfer_session()) { + if (ensure_pixel_buffer_pool(output_pixel_format) || ensure_transfer_session()) { return nullptr; } @@ -155,6 +227,48 @@ namespace platf { return nullptr; } + if (prefill_black(scaled_pixel_buffer)) { + CVPixelBufferRelease(scaled_pixel_buffer); + return nullptr; + } + + auto width_scale = (double) av_frame->width / (double) CVPixelBufferGetWidth(pixel_buffer); + auto height_scale = (double) av_frame->height / (double) CVPixelBufferGetHeight(pixel_buffer); + auto scale = width_scale < height_scale ? width_scale : height_scale; + auto clean_width = (double) CVPixelBufferGetWidth(pixel_buffer) * scale; + auto clean_height = (double) CVPixelBufferGetHeight(pixel_buffer) * scale; + double zero = 0.0; + CFNumberRef clean_width_number = CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &clean_width); + CFNumberRef clean_height_number = CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &clean_height); + CFNumberRef clean_horizontal_offset = CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &zero); + CFNumberRef clean_vertical_offset = CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &zero); + const void *clean_keys[] = { + kCVImageBufferCleanApertureWidthKey, + kCVImageBufferCleanApertureHeightKey, + kCVImageBufferCleanApertureHorizontalOffsetKey, + kCVImageBufferCleanApertureVerticalOffsetKey, + }; + const void *clean_values[] = { + clean_width_number, + clean_height_number, + clean_horizontal_offset, + clean_vertical_offset, + }; + CFDictionaryRef clean_aperture = CFDictionaryCreate( + kCFAllocatorDefault, + clean_keys, + clean_values, + 4, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks + ); + VTSessionSetProperty(transfer_session, kVTPixelTransferPropertyKey_DestinationCleanAperture, clean_aperture); + CFRelease(clean_aperture); + CFRelease(clean_vertical_offset); + CFRelease(clean_horizontal_offset); + CFRelease(clean_height_number); + CFRelease(clean_width_number); + status = VTPixelTransferSessionTransferImage(transfer_session, pixel_buffer, scaled_pixel_buffer); if (status != noErr) { BOOST_LOG(error) << "VideoToolbox pixel transfer failed: " << status; @@ -193,7 +307,8 @@ namespace platf { } int nv12_zero_device::init(void *display, pix_fmt_e pix_fmt, resolution_fn_t resolution_fn, const pixel_format_fn_t &pixel_format_fn, bool resize_capture) { - pixel_format_fn(display, pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange); + output_pixel_format = pix_fmt == pix_fmt_e::nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange; + pixel_format_fn(display, output_pixel_format); this->display = display; this->resolution_fn = std::move(resolution_fn); diff --git a/src/platform/macos/nv12_zero_device.h b/src/platform/macos/nv12_zero_device.h index 3173c5117ca..ffb9d0f52f6 100644 --- a/src/platform/macos/nv12_zero_device.h +++ b/src/platform/macos/nv12_zero_device.h @@ -7,6 +7,9 @@ // local includes #include "src/platform/common.h" +// standard includes +#include + // platform includes #include @@ -38,15 +41,18 @@ namespace platf { util::safe_ptr av_frame; VTPixelTransferSessionRef transfer_session {}; CVPixelBufferPoolRef pixel_buffer_pool {}; + OSType output_pixel_format {}; OSType pool_pixel_format {}; int pool_width {}; int pool_height {}; bool resize_capture {}; + std::unordered_set black_surfaces; int attach_pixel_buffer(CVPixelBufferRef pixel_buffer); CVPixelBufferRef copy_scaled_pixel_buffer(CVPixelBufferRef pixel_buffer); int ensure_pixel_buffer_pool(OSType pixel_format); int ensure_transfer_session(); + int prefill_black(CVPixelBufferRef pixel_buffer); }; } // namespace platf