camera: Report rotation needed to account for device orientation.

Fixes #11476.

Author: icculus

include/SDL3/SDL_surface.h

@@ -241,6 +241,12 @@ extern SDL_DECLSPEC void SDLCALL SDL_DestroySurface(SDL_Surface *surface);
  *   left edge of the image, if this surface is being used as a cursor.
  * - `SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER`: the hotspot pixel offset from the
  *   top edge of the image, if this surface is being used as a cursor.
+ * - `SDL_PROP_SURFACE_ROTATION_NUMBER`: the number of degrees a surface's
+ *   data is meant to be rotated clockwise to make the image
+ *   right-side up. Default 0. This is used by the camera API, if a mobile
+ *   device is oriented differently than what its camera provides (i.e. -
+ *   the camera always provides portrait images but the phone is being held
+ *   in landscape orientation). Since SDL 3.4.0.
  *
  * \param surface the SDL_Surface structure to query.
  * \returns a valid property ID on success or 0 on failure; call
@@ -257,6 +263,7 @@ extern SDL_DECLSPEC SDL_PropertiesID SDLCALL SDL_GetSurfaceProperties(SDL_Surfac
 #define SDL_PROP_SURFACE_TONEMAP_OPERATOR_STRING            "SDL.surface.tonemap"
 #define SDL_PROP_SURFACE_HOTSPOT_X_NUMBER                   "SDL.surface.hotspot.x"
 #define SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER                   "SDL.surface.hotspot.y"
+#define SDL_PROP_SURFACE_ROTATION_NUMBER                    "SDL.surface.rotation"
 
 /**
  * Set the colorspace used by a surface.

src/camera/SDL_camera.c

@@ -150,7 +150,7 @@ static size_t GetFrameBufLen(const SDL_CameraSpec *spec)
     return wxh * SDL_BYTESPERPIXEL(fmt);
 }
 
-static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     const SDL_CameraSpec *spec = &device->actual_spec;
 
@@ -832,9 +832,10 @@ bool SDL_CameraThreadIterate(SDL_Camera *device)
     SDL_Surface *output_surface = NULL;
     SurfaceList *slist = NULL;
     Uint64 timestampNS = 0;
+    int rotation = 0;
 
     // AcquireFrame SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitDevice instead!
-    const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, &timestampNS);
+    const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, &timestampNS, &rotation);
 
     if (rc == SDL_CAMERA_FRAME_READY) {  // new frame acquired!
         #if DEBUG_CAMERA
@@ -928,6 +929,8 @@ bool SDL_CameraThreadIterate(SDL_Camera *device)
         acquired->pixels = NULL;
         acquired->pitch = 0;
 
+        SDL_SetNumberProperty(SDL_GetSurfaceProperties(output_surface), SDL_PROP_SURFACE_ROTATION_NUMBER, rotation);
+
         // make the filled output surface available to the app.
         SDL_LockMutex(device->lock);
         slist->next = device->filled_output_surfaces.next;

src/camera/SDL_syscamera.h

@@ -99,7 +99,7 @@ struct SDL_Camera
 
     // These are, initially, set from camera_driver, but we might swap them out with Zombie versions on disconnect/failure.
     bool (*WaitDevice)(SDL_Camera *device);
-    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS);
+    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation);
     void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame);
 
     // All supported formats/dimensions for this device.
@@ -167,13 +167,18 @@ struct SDL_Camera
     struct SDL_PrivateCameraData *hidden;
 };
 
+
+// Note that for AcquireFrame, `rotation` is degrees, with positive values rotating clockwise. This is the amount to rotate an image so it would be right-side up.
+// Rotations should be in 90 degree increments at this time (landscape to portrait, or upside down to right side up, etc).
+// Most platforms won't care about this, but mobile devices might need to deal with the device itself being physically rotated, causing the fixed-orientation camera to be presenting sideways images.
+
 typedef struct SDL_CameraDriverImpl
 {
     void (*DetectDevices)(void);
     bool (*OpenDevice)(SDL_Camera *device, const SDL_CameraSpec *spec);
     void (*CloseDevice)(SDL_Camera *device);
     bool (*WaitDevice)(SDL_Camera *device);
-    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS); // set frame->pixels, frame->pitch, and *timestampNS!
+    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation); // set frame->pixels, frame->pitch, *timestampNS, and *rotation!
     void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame); // Reclaim frame->pixels and frame->pitch!
     void (*FreeDeviceHandle)(SDL_Camera *device); // SDL is done with this device; free the handle from SDL_AddCamera()
     void (*Deinitialize)(void);

src/camera/android/SDL_camera_android.c

@@ -149,6 +149,7 @@ struct SDL_PrivateCameraData
     ACaptureRequest *request;
     ACameraCaptureSession *session;
     SDL_CameraSpec requested_spec;
+    int rotation;  // degrees to rotate clockwise to get from camera's static orientation to device's native orientation. Apply this plus current phone rotation to get upright image!
 };
 
 static bool SetErrorStr(const char *what, const char *errstr, const int rc)
@@ -295,7 +296,7 @@ static bool ANDROIDCAMERA_WaitDevice(SDL_Camera *device)
     return true;  // this isn't used atm, since we run our own thread via onImageAvailable callbacks.
 }
 
-static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY;
     media_status_t res;
@@ -366,6 +367,21 @@ static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_
 
     pAImage_delete(image);
 
+    int dev_rotation = 0;
+    switch (Android_JNI_GetDisplayCurrentOrientation()) {
+        case SDL_ORIENTATION_PORTRAIT: dev_rotation = 0; break;
+        case SDL_ORIENTATION_LANDSCAPE: dev_rotation = 90; break;
+        case SDL_ORIENTATION_PORTRAIT_FLIPPED: dev_rotation = 180; break;
+        case SDL_ORIENTATION_LANDSCAPE_FLIPPED: dev_rotation = 270; break;
+        default: SDL_assert(!"Unexpected device rotation!"); dev_rotation = 0; break;
+    }
+
+    if (device->position == SDL_CAMERA_POSITION_BACK_FACING) {
+        dev_rotation = -dev_rotation;  // we want to subtract this value, instead of add, if back-facing.
+    }
+
+    *rotation = dev_rotation + device->hidden->rotation;   // current phone orientation, static camera orientation in relation to phone.
+
     return result;
 }
 
@@ -494,10 +510,23 @@ static bool PrepareCamera(SDL_Camera *device)
     imglistener.context = device;
     imglistener.onImageAvailable = onImageAvailable;
 
+
+    const char *devid = (const char *) device->handle;
+
+    device->hidden->rotation = 0;
+    ACameraMetadata *metadata = NULL;
+    ACameraMetadata_const_entry orientationentry;
+    if (pACameraManager_getCameraCharacteristics(cameraMgr, devid, &metadata) == ACAMERA_OK) {
+        if (pACameraMetadata_getConstEntry(metadata, ACAMERA_SENSOR_ORIENTATION, &orientationentry) == ACAMERA_OK) {
+            device->hidden->rotation = (int) (*orientationentry.data.i32 % 360);
+        }
+        pACameraMetadata_free(metadata);
+    }
+
     // just in case SDL_OpenCamera is overwriting device->spec as CameraPermissionCallback runs, we work from a different copy.
     const SDL_CameraSpec *spec = &device->hidden->requested_spec;
 
-    if ((res = pACameraManager_openCamera(cameraMgr, (const char *) device->handle, &dev_callbacks, &device->hidden->device)) != ACAMERA_OK) {
+    if ((res = pACameraManager_openCamera(cameraMgr, devid, &dev_callbacks, &device->hidden->device)) != ACAMERA_OK) {
         return SetCameraError("Failed to open camera", res);
     } else if ((res2 = pAImageReader_new(spec->width, spec->height, format_sdl_to_android(spec->format), 10 /* nb buffers */, &device->hidden->reader)) != AMEDIA_OK) {
         return SetMediaError("Error AImageReader_new", res2);

src/camera/coremedia/SDL_camera_coremedia.m

@@ -29,6 +29,14 @@
 #import <AVFoundation/AVFoundation.h>
 #import <CoreMedia/CoreMedia.h>
 
+#if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+#define USE_UIKIT_DEVICE_ROTATION
+#endif
+
+#ifdef USE_UIKIT_DEVICE_ROTATION
+#import <UIKit/UIKit.h>
+#endif
+
 /*
  * Need to link with:: CoreMedia CoreVideo
  *
@@ -77,6 +85,9 @@ @interface SDLPrivateCameraData : NSObject
 @property(nonatomic, retain) AVCaptureSession *session;
 @property(nonatomic, retain) SDLCaptureVideoDataOutputSampleBufferDelegate *delegate;
 @property(nonatomic, assign) CMSampleBufferRef current_sample;
+#ifdef USE_UIKIT_DEVICE_ROTATION
+@property(nonatomic, assign) UIDeviceOrientation last_device_orientation;
+#endif
 @end
 
 @implementation SDLPrivateCameraData
@@ -146,7 +157,7 @@ static bool COREMEDIA_WaitDevice(SDL_Camera *device)
     return true;  // this isn't used atm, since we run our own thread out of Grand Central Dispatch.
 }
 
-static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY;
     SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden;
@@ -219,6 +230,37 @@ static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surf
 
     CVPixelBufferUnlockBaseAddress(image, 0);
 
+    #ifdef USE_UIKIT_DEVICE_ROTATION
+    UIDeviceOrientation device_orientation = [[UIDevice currentDevice] orientation];
+    if (!UIDeviceOrientationIsValidInterfaceOrientation(device_orientation)) {
+        device_orientation = hidden.last_device_orientation;  // possible the phone is laying flat or something went wrong, just stay with the last known-good orientation.
+    } else {
+        hidden.last_device_orientation = device_orientation;  // update the last known-good orientation for later.
+    }
+
+    const UIInterfaceOrientation ui_orientation = [UIApplication sharedApplication].statusBarOrientation;
+
+    // there is probably math for this, but this is easy to slap into a table.
+    // rotation = rotations[uiorientation-1][devorientation-1];
+    if (device->position == SDL_CAMERA_POSITION_BACK_FACING) {
+        static const int back_rotations[4][4] = {
+            {   90,  90,  90,  90 },  // ui portrait
+            {  270, 270, 270, 270 },  // ui portait upside down
+            {    0,   0,   0,   0 },  // ui landscape left
+            {  180, 180, 180, 180 }   // ui landscape right
+        };
+        *rotation = back_rotations[ui_orientation - 1][device_orientation - 1];
+    } else {
+        static const int front_rotations[4][4] = {
+            {   90,  90, 270, 270 },  // ui portrait
+            {  270, 270,  90,  90 },  // ui portait upside down
+            {    0,   0, 180, 180 },  // ui landscape left
+            {  180, 180,   0,   0 }   // ui landscape right
+        };
+        *rotation = front_rotations[ui_orientation - 1][device_orientation - 1];
+    }
+    #endif
+
     return result;
 }
 
@@ -231,6 +273,10 @@ static void COREMEDIA_ReleaseFrame(SDL_Camera *device, SDL_Surface *frame)
 static void COREMEDIA_CloseDevice(SDL_Camera *device)
 {
     if (device && device->hidden) {
+        #ifdef USE_UIKIT_DEVICE_ROTATION
+        [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
+        #endif
+
         SDLPrivateCameraData *hidden = (SDLPrivateCameraData *) CFBridgingRelease(device->hidden);
         device->hidden = NULL;
 
@@ -358,6 +404,28 @@ static bool COREMEDIA_OpenDevice(SDL_Camera *device, const SDL_CameraSpec *spec)
     hidden.session = session;
     hidden.delegate = delegate;
     hidden.current_sample = NULL;
+
+    #ifdef USE_UIKIT_DEVICE_ROTATION
+    // When using a camera, we turn on device orientation tracking. The docs note that this turns on
+    // the device's accelerometer, so I assume this burns power, so we don't leave this running all
+    // the time. These calls nest, so we just need to call the matching `end` message when we close.
+    // You _can_ get an actual events through this mechanism, but we just want to be able to call
+    // -[UIDevice orientation], which will update with real info while notificatons are enabled.
+    UIDevice *uidevice = [UIDevice currentDevice];
+    [uidevice beginGeneratingDeviceOrientationNotifications];
+    hidden.last_device_orientation = uidevice.orientation;
+    if (!UIDeviceOrientationIsValidInterfaceOrientation(hidden.last_device_orientation)) {
+        // accelerometer isn't ready yet or the phone is laying flat or something. Just try to guess from how the UI is oriented at the moment.
+        switch ([UIApplication sharedApplication].statusBarOrientation) {
+            case UIInterfaceOrientationPortrait: hidden.last_device_orientation = UIDeviceOrientationPortrait; break;
+            case UIInterfaceOrientationPortraitUpsideDown: hidden.last_device_orientation = UIDeviceOrientationPortraitUpsideDown; break;
+            case UIInterfaceOrientationLandscapeLeft: hidden.last_device_orientation = UIDeviceOrientationLandscapeRight; break;  // Apple docs say UI and device orientations are reversed in landscape.
+            case UIInterfaceOrientationLandscapeRight: hidden.last_device_orientation = UIDeviceOrientationLandscapeLeft; break;
+            default: hidden.last_device_orientation = UIDeviceOrientationPortrait; break;  // oh well.
+        }
+    }
+    #endif
+
     device->hidden = (struct SDL_PrivateCameraData *)CFBridgingRetain(hidden);
 
     [session startRunning];  // !!! FIXME: docs say this can block while camera warms up and shouldn't be done on main thread. Maybe push through `queue`?

src/camera/dummy/SDL_camera_dummy.c

@@ -38,7 +38,7 @@ static bool DUMMYCAMERA_WaitDevice(SDL_Camera *device)
     return SDL_Unsupported();
 }
 
-static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_Unsupported();
     return SDL_CAMERA_FRAME_ERROR;

src/camera/emscripten/SDL_camera_emscripten.c

@@ -39,7 +39,7 @@ static bool EMSCRIPTENCAMERA_WaitDevice(SDL_Camera *device)
     return false;
 }
 
-static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     void *rgba = SDL_malloc(device->actual_spec.width * device->actual_spec.height * 4);
     if (!rgba) {

src/camera/mediafoundation/SDL_camera_mediafoundation.c

@@ -430,7 +430,7 @@ static void SDLCALL CleanupIMFMediaBuffer(void *userdata, void *value)
     SDL_free(objs);
 }
 
-static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_assert(device->hidden->current_sample != NULL);
 
@@ -562,7 +562,7 @@ static SDL_CameraFrameResult MEDIAFOUNDATION_CopyFrame(SDL_Surface *frame, const
     return SDL_CAMERA_FRAME_READY;
 }
 
-static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_assert(device->hidden->current_sample != NULL);
 

src/camera/pipewire/SDL_camera_pipewire.c

@@ -577,7 +577,7 @@ static bool PIPEWIRECAMERA_WaitDevice(SDL_Camera *device)
     return true;
 }
 
-static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     struct pw_buffer *b;
 

src/camera/v4l2/SDL_camera_v4l2.c

@@ -125,7 +125,7 @@ static bool V4L2_WaitDevice(SDL_Camera *device)
     return false;
 }
 
-static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     const int fd = device->hidden->fd;
     const io_method io = device->hidden->io;