From de48e8aa3ab6621022a83a303298122c38bd105b Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 16:04:56 -0400 Subject: [PATCH 01/24] Integrate LiveKit video/audio calling into Relay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds end-to-end support for LiveKit-backed calls in Matrix rooms: - `CallViewModelProtocol` + `CallState` / `CallParticipant` models in RelayInterface - `CallViewModel` in RelayKit wraps `LiveKit.Room` and bridges `RoomDelegate` callbacks onto the main actor via `Task { @MainActor in … }` - `makeVideoView(for:)` creates a `LiveKit.VideoView` (NSView subclass) so that no LiveKit types escape into the app or protocol layers - `CallView` in Relay/Views shows participant tiles with speaking indicators, a bottom control bar (mic, camera, end call), and an NSViewRepresentable video bridge — imports only RelayInterface and SwiftUI - `PreviewCallViewModel` simulates a connected call for SwiftUI previews - `makeCallViewModel(roomId:)` added to `MatrixServiceProtocol`, `MatrixService`, `PreviewMatrixService`, and the placeholder - Phone button added to the room toolbar in `MainView`; pressing it opens `CallView` in a sheet - Camera + microphone sandbox entitlements added to `Relay.entitlements` - `NSCameraUsageDescription` + `NSMicrophoneUsageDescription` added to `Info.plist` - LiveKit SPM package (`client-sdk-swift`, ≥ 2.0.0) added to `project.pbxproj` and linked into the RelayKit framework target Co-Authored-By: Claude Sonnet 4.6 --- .../Protocols/CallViewModelProtocol.swift | 110 +++++++ .../Protocols/MatrixServiceProtocol.swift | 8 + Relay.xcodeproj/project.pbxproj | 31 +- Relay/Info.plist | 4 + Relay/Relay.entitlements | 4 + Relay/Services/PreviewMatrixService.swift | 4 + Relay/ViewModels/PreviewCallViewModel.swift | 75 +++++ Relay/Views/CallView.swift | 293 ++++++++++++++++++ Relay/Views/MainView.swift | 18 ++ RelayKit/Call/CallViewModel.swift | 178 +++++++++++ RelayKit/Services/MatrixService.swift | 4 + 11 files changed, 721 insertions(+), 8 deletions(-) create mode 100644 Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift create mode 100644 Relay/ViewModels/PreviewCallViewModel.swift create mode 100644 Relay/Views/CallView.swift create mode 100644 RelayKit/Call/CallViewModel.swift diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift new file mode 100644 index 0000000..aec84f6 --- /dev/null +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift @@ -0,0 +1,110 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AppKit +import Foundation + +/// The connection state of a call. +public enum CallState: Sendable, Equatable { + /// No active call. + case idle + /// Establishing connection to the call server. + case connecting + /// Successfully connected; media is flowing. + case connected + /// The call ended cleanly. + case disconnected + /// The call failed with an error message. + case failed(String) +} + +/// A snapshot of a single call participant. +public struct CallParticipant: Identifiable, Sendable, Equatable { + /// The participant's identity string (typically their Matrix user ID). + public let id: String + /// The participant's display name, if available. + public let displayName: String? + /// Whether the participant has their camera enabled. + public let isCameraEnabled: Bool + /// Whether the participant has their microphone enabled. + public let isMicrophoneEnabled: Bool + /// Whether the participant is currently speaking. + public let isSpeaking: Bool + + public init( + id: String, + displayName: String?, + isCameraEnabled: Bool, + isMicrophoneEnabled: Bool, + isSpeaking: Bool + ) { + self.id = id + self.displayName = displayName + self.isCameraEnabled = isCameraEnabled + self.isMicrophoneEnabled = isMicrophoneEnabled + self.isSpeaking = isSpeaking + } +} + +/// The view model protocol for a LiveKit-backed audio/video call in a Matrix room. +/// +/// ``CallViewModelProtocol`` defines the observable state and actions needed by ``CallView`` +/// to render the call UI, control local media, and display remote participants. Concrete +/// implementations include ``CallViewModel`` (backed by the LiveKit Swift SDK) and +/// ``PreviewCallViewModel`` (for SwiftUI previews). +/// +/// Video rendering is intentionally opaque: callers request an ``NSView`` via +/// ``makeVideoView(for:)`` to avoid exposing LiveKit types outside of RelayKit. +@MainActor +public protocol CallViewModelProtocol: AnyObject, Observable { + /// The current connection state of the call. + var state: CallState { get } + + /// All remote participants currently in the call. + var participants: [CallParticipant] { get } + + /// Whether the local user's camera is active. + var isLocalCameraEnabled: Bool { get } + + /// Whether the local user's microphone is active. + var isLocalMicrophoneEnabled: Bool { get } + + /// The identity of the local participant, set after connection. + var localParticipantID: String? { get } + + /// Connects to the call using the provided LiveKit server URL and JWT token. + /// + /// - Parameters: + /// - url: The WebSocket URL of the LiveKit server (e.g. `"wss://livekit.example.com"`). + /// - token: A signed JWT granting access to the room. + func connect(url: String, token: String) async throws + + /// Disconnects from the call and cleans up media resources. + func disconnect() async + + /// Toggles the local camera on or off. + func toggleCamera() async throws + + /// Toggles the local microphone on or off. + func toggleMicrophone() async throws + + /// Returns an ``NSView`` that renders the video track of the given participant, or `nil` + /// if the participant has no active video track or is not found. + /// + /// The returned view is owned by the call view model and must only be embedded — do not + /// deallocate it. A new view is returned on each call. + /// + /// - Parameter participantID: The ``CallParticipant/id`` of the participant to render. + func makeVideoView(for participantID: String) -> NSView? +} diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift index 0d65e72..8819974 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift @@ -480,6 +480,13 @@ public protocol MatrixServiceProtocol: AnyObject, Observable { /// verification controller is not available. func makeSessionVerificationViewModel() async throws -> (any SessionVerificationViewModelProtocol)? + /// Creates a view model for joining or managing a LiveKit audio/video call in a Matrix room. + /// + /// - Parameter roomId: The Matrix room identifier for the call. + /// - Returns: A ``CallViewModelProtocol`` instance ready to be connected with a LiveKit + /// URL and token, or `nil` if calling is not supported. + func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? + // MARK: Notification Settings (synced via push rules) /// Returns the default notification mode for rooms of the given type. @@ -809,6 +816,7 @@ private final class PlaceholderMatrixService: MatrixServiceProtocol { func isCurrentSessionVerified() async -> Bool { false } func encryptionState() async -> EncryptionStatus { EncryptionStatus() } func makeSessionVerificationViewModel() async throws -> (any SessionVerificationViewModelProtocol)? { nil } + func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { nil } func getDefaultNotificationMode( isOneToOne: Bool ) async throws -> DefaultNotificationMode { .mentionsAndKeywordsOnly } diff --git a/Relay.xcodeproj/project.pbxproj b/Relay.xcodeproj/project.pbxproj index a941a3a..20af381 100644 --- a/Relay.xcodeproj/project.pbxproj +++ b/Relay.xcodeproj/project.pbxproj @@ -50,7 +50,6 @@ 3B4AFD892F638A35001F0EA1 /* Relay.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Relay.app; sourceTree = BUILT_PRODUCTS_DIR; }; 3BAE76A72F6A43BA000EC1E6 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 3BRK00002FC00000001F0EA1 /* RelayKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = RelayKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; - 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Secrets.xcconfig; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ @@ -116,7 +115,6 @@ 3B4AFD802F638A35001F0EA1 = { isa = PBXGroup; children = ( - 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */, 3BAE76A72F6A43BA000EC1E6 /* README.md */, 3B4AFD8B2F638A35001F0EA1 /* Relay */, 3BRK00112FC00011001F0EA1 /* RelayKit */, @@ -217,6 +215,7 @@ packageProductDependencies = ( 3BMRST022FB10002001F0EA1 /* MatrixRustSDK */, 3BRI00022FC10002001F0EA1 /* RelayInterface */, + 3BLK00022FD10002001F0EA1 /* LiveKit */, ); productName = RelayKit; productReference = 3BRK00002FC00000001F0EA1 /* RelayKit.framework */; @@ -256,6 +255,7 @@ packageReferences = ( 3BRI00052FC10005001F0EA1 /* XCLocalSwiftPackageReference "Packages/RelayInterface" */, 3BMRST032FB10003001F0EA1 /* XCRemoteSwiftPackageReference "matrix-rust-components-swift" */, + 3BLK00032FD10003001F0EA1 /* XCRemoteSwiftPackageReference "client-sdk-swift" */, ); preferredProjectObjectVersion = 77; productRefGroup = 3B4AFD8A2F638A35001F0EA1 /* Products */; @@ -395,7 +395,6 @@ }; 3B4AFDAA2F638A36001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; @@ -463,7 +462,6 @@ }; 3B4AFDAB2F638A36001F0EA1 /* Release */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; @@ -523,15 +521,17 @@ }; 3B4AFDAD2F638A36001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIconDev; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColorDev; CODE_SIGN_ENTITLEMENTS = Relay/Relay.entitlements; + CODE_SIGN_IDENTITY = "Apple Development"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 13; DEAD_CODE_STRIPPING = YES; + DEVELOPMENT_TEAM = ""; ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_INCOMING_NETWORK_CONNECTIONS = YES; @@ -558,6 +558,7 @@ MARKETING_VERSION = 0.4.4; PRODUCT_BUNDLE_IDENTIFIER = app.subpop.Relay; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; SWIFT_APPROACHABLE_CONCURRENCY = YES; @@ -569,15 +570,17 @@ }; 3B4AFDAE2F638A36001F0EA1 /* Release */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = Relay/Relay.entitlements; + CODE_SIGN_IDENTITY = "Apple Development"; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "-"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 13; DEAD_CODE_STRIPPING = YES; + DEVELOPMENT_TEAM = ""; ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_INCOMING_NETWORK_CONNECTIONS = YES; @@ -604,6 +607,7 @@ MARKETING_VERSION = 0.4.4; PRODUCT_BUNDLE_IDENTIFIER = app.subpop.Relay; PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; REGISTER_APP_GROUPS = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; SWIFT_APPROACHABLE_CONCURRENCY = YES; @@ -615,7 +619,6 @@ }; 3BRK00212FC00021001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { CODE_SIGN_IDENTITY = ""; CODE_SIGN_STYLE = Automatic; @@ -649,7 +652,6 @@ }; 3BRK00222FC00022001F0EA1 /* Release */ = { isa = XCBuildConfiguration; - baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { CODE_SIGN_IDENTITY = ""; CODE_SIGN_STYLE = Automatic; @@ -730,6 +732,14 @@ /* End XCLocalSwiftPackageReference section */ /* Begin XCRemoteSwiftPackageReference section */ + 3BLK00032FD10003001F0EA1 /* XCRemoteSwiftPackageReference "client-sdk-swift" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/livekit/client-sdk-swift"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 2.0.0; + }; + }; 3BMRST032FB10003001F0EA1 /* XCRemoteSwiftPackageReference "matrix-rust-components-swift" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/matrix-org/matrix-rust-components-swift"; @@ -741,6 +751,11 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ + 3BLK00022FD10002001F0EA1 /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = 3BLK00032FD10003001F0EA1 /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; 3BMRST022FB10002001F0EA1 /* MatrixRustSDK */ = { isa = XCSwiftPackageProductDependency; package = 3BMRST032FB10003001F0EA1 /* XCRemoteSwiftPackageReference "matrix-rust-components-swift" */; diff --git a/Relay/Info.plist b/Relay/Info.plist index f50b658..37386eb 100644 --- a/Relay/Info.plist +++ b/Relay/Info.plist @@ -17,6 +17,10 @@ + NSCameraUsageDescription + Relay uses the camera for video calls in Matrix rooms. + NSMicrophoneUsageDescription + Relay uses the microphone for audio and video calls in Matrix rooms. CFBundleURLTypes diff --git a/Relay/Relay.entitlements b/Relay/Relay.entitlements index ee95ab7..72f5f60 100644 --- a/Relay/Relay.entitlements +++ b/Relay/Relay.entitlements @@ -6,5 +6,9 @@ com.apple.security.network.client + com.apple.security.device.camera + + com.apple.security.device.microphone + diff --git a/Relay/Services/PreviewMatrixService.swift b/Relay/Services/PreviewMatrixService.swift index 632fb11..5ef440e 100644 --- a/Relay/Services/PreviewMatrixService.swift +++ b/Relay/Services/PreviewMatrixService.swift @@ -224,6 +224,10 @@ final class PreviewMatrixService: MatrixServiceProtocol { PreviewSessionVerificationViewModel() } + func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { + PreviewCallViewModel() + } + func declinePendingVerificationRequest() async { pendingVerificationRequest = nil } diff --git a/Relay/ViewModels/PreviewCallViewModel.swift b/Relay/ViewModels/PreviewCallViewModel.swift new file mode 100644 index 0000000..e614753 --- /dev/null +++ b/Relay/ViewModels/PreviewCallViewModel.swift @@ -0,0 +1,75 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AppKit +import Foundation +import RelayInterface + +/// A mock ``CallViewModelProtocol`` for SwiftUI previews. +/// +/// Simulates a connected call with sample participants after a short delay. +/// All methods are safe to call from the main actor, and no real media or +/// network connections are established. +@Observable +@MainActor +final class PreviewCallViewModel: CallViewModelProtocol { + var state: CallState = .idle + var participants: [CallParticipant] = [] + var isLocalCameraEnabled: Bool = false + var isLocalMicrophoneEnabled: Bool = false + var localParticipantID: String? = nil + + func connect(url: String, token: String) async throws { + state = .connecting + try? await Task.sleep(for: .milliseconds(800)) + isLocalCameraEnabled = true + isLocalMicrophoneEnabled = true + localParticipantID = "@preview:matrix.org" + participants = [ + CallParticipant( + id: "@alice:matrix.org", + displayName: "Alice Smith", + isCameraEnabled: true, + isMicrophoneEnabled: true, + isSpeaking: true + ), + CallParticipant( + id: "@bob:matrix.org", + displayName: "Bob Chen", + isCameraEnabled: false, + isMicrophoneEnabled: true, + isSpeaking: false + ) + ] + state = .connected + } + + func disconnect() async { + state = .disconnected + participants = [] + isLocalCameraEnabled = false + isLocalMicrophoneEnabled = false + localParticipantID = nil + } + + func toggleCamera() async throws { + isLocalCameraEnabled.toggle() + } + + func toggleMicrophone() async throws { + isLocalMicrophoneEnabled.toggle() + } + + func makeVideoView(for participantID: String) -> NSView? { nil } +} diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift new file mode 100644 index 0000000..8c2a43e --- /dev/null +++ b/Relay/Views/CallView.swift @@ -0,0 +1,293 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AppKit +import RelayInterface +import SwiftUI + +/// Renders an active or connecting LiveKit call within a Matrix room. +/// +/// ``CallView`` shows participant tiles with video/audio indicators and a bottom +/// control bar for toggling local media and ending the call. It relies solely on +/// ``CallViewModelProtocol`` — no LiveKit types are referenced here. +struct CallView: View { + @State var viewModel: any CallViewModelProtocol + var onDismiss: () -> Void + + var body: some View { + ZStack { + Color.black.ignoresSafeArea() + + VStack(spacing: 0) { + switch viewModel.state { + case .idle, .connecting: + Spacer() + ProgressView("Joining call…") + .progressViewStyle(.circular) + .controlSize(.large) + .foregroundStyle(.white) + .tint(.white) + Spacer() + + case .connected: + participantsGrid + .frame(maxWidth: .infinity, maxHeight: .infinity) + controlBar + + case .disconnected: + Spacer() + ContentUnavailableView( + "Call Ended", + systemImage: "phone.down.fill", + description: Text("The call has ended.") + ) + .foregroundStyle(.white) + Button("Dismiss") { onDismiss() } + .buttonStyle(.borderedProminent) + .padding(.top, 12) + Spacer() + + case .failed(let message): + Spacer() + ContentUnavailableView( + "Call Failed", + systemImage: "exclamationmark.triangle.fill", + description: Text(message) + ) + .foregroundStyle(.white) + Button("Dismiss") { onDismiss() } + .buttonStyle(.borderedProminent) + .tint(.red) + .padding(.top, 12) + Spacer() + } + } + } + .frame(minWidth: 480, minHeight: 360) + } + + // MARK: - Participants Grid + + @ViewBuilder + private var participantsGrid: some View { + let columns = [GridItem(.adaptive(minimum: 200, maximum: 400), spacing: 8)] + ScrollView { + LazyVGrid(columns: columns, spacing: 8) { + ForEach(viewModel.participants) { participant in + participantTile(participant) + } + // Local participant tile + if let localID = viewModel.localParticipantID { + localParticipantTile(id: localID) + } + } + .padding(8) + } + } + + // MARK: - Participant Tile + + @ViewBuilder + private func participantTile(_ participant: CallParticipant) -> some View { + ZStack(alignment: .bottom) { + // Video or placeholder background + VideoViewRepresentable(viewModel: viewModel, participantID: participant.id) + .aspectRatio(16 / 9, contentMode: .fit) + .clipShape(RoundedRectangle(cornerRadius: 10)) + + // Speaking ring overlay + if participant.isSpeaking { + RoundedRectangle(cornerRadius: 10) + .strokeBorder(.green, lineWidth: 3) + } + + // Name label + media indicators + HStack(spacing: 6) { + Text(participant.displayName ?? participant.id) + .font(.caption) + .foregroundStyle(.white) + .lineLimit(1) + .truncationMode(.tail) + + Spacer() + + if !participant.isMicrophoneEnabled { + Image(systemName: "mic.slash.fill") + .font(.caption) + .foregroundStyle(.red) + } + if !participant.isCameraEnabled { + Image(systemName: "video.slash.fill") + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(.horizontal, 8) + .padding(.vertical, 6) + .background(.black.opacity(0.55)) + .clipShape( + .rect( + topLeadingRadius: 0, + bottomLeadingRadius: 10, + bottomTrailingRadius: 10, + topTrailingRadius: 0 + ) + ) + } + } + + // MARK: - Local Participant Tile + + @ViewBuilder + private func localParticipantTile(id: String) -> some View { + ZStack(alignment: .bottom) { + VideoViewRepresentable(viewModel: viewModel, participantID: id) + .aspectRatio(16 / 9, contentMode: .fit) + .clipShape(RoundedRectangle(cornerRadius: 10)) + + HStack(spacing: 6) { + Text("You") + .font(.caption) + .foregroundStyle(.white) + + Spacer() + + if !viewModel.isLocalMicrophoneEnabled { + Image(systemName: "mic.slash.fill") + .font(.caption) + .foregroundStyle(.red) + } + if !viewModel.isLocalCameraEnabled { + Image(systemName: "video.slash.fill") + .font(.caption) + .foregroundStyle(.secondary) + } + } + .padding(.horizontal, 8) + .padding(.vertical, 6) + .background(.black.opacity(0.55)) + .clipShape( + .rect( + topLeadingRadius: 0, + bottomLeadingRadius: 10, + bottomTrailingRadius: 10, + topTrailingRadius: 0 + ) + ) + } + } + + // MARK: - Control Bar + + @ViewBuilder + private var controlBar: some View { + HStack(spacing: 24) { + // Microphone toggle + Button { + Task { try? await viewModel.toggleMicrophone() } + } label: { + Image(systemName: viewModel.isLocalMicrophoneEnabled ? "mic.fill" : "mic.slash.fill") + .font(.title2) + .frame(width: 44, height: 44) + .background(viewModel.isLocalMicrophoneEnabled ? Color.white.opacity(0.15) : Color.red.opacity(0.8)) + .clipShape(Circle()) + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .help(viewModel.isLocalMicrophoneEnabled ? "Mute microphone" : "Unmute microphone") + + // Camera toggle + Button { + Task { try? await viewModel.toggleCamera() } + } label: { + Image(systemName: viewModel.isLocalCameraEnabled ? "video.fill" : "video.slash.fill") + .font(.title2) + .frame(width: 44, height: 44) + .background(viewModel.isLocalCameraEnabled ? Color.white.opacity(0.15) : Color.red.opacity(0.8)) + .clipShape(Circle()) + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .help(viewModel.isLocalCameraEnabled ? "Turn off camera" : "Turn on camera") + + // End call button + Button { + Task { + await viewModel.disconnect() + onDismiss() + } + } label: { + Image(systemName: "phone.down.fill") + .font(.title2) + .frame(width: 52, height: 52) + .background(Color.red) + .clipShape(Circle()) + } + .buttonStyle(.plain) + .foregroundStyle(.white) + .help("End call") + } + .padding(.vertical, 16) + .padding(.horizontal, 32) + .background(.black.opacity(0.7)) + } +} + +// MARK: - NSView Bridge for Video + +/// An `NSViewRepresentable` that embeds the opaque `NSView` returned by +/// ``CallViewModelProtocol/makeVideoView(for:)``. +/// +/// If the view model returns `nil` (participant has no active video track), a dark +/// gray placeholder with the participant's initials is shown instead. +private struct VideoViewRepresentable: NSViewRepresentable { + let viewModel: any CallViewModelProtocol + let participantID: String + + func makeNSView(context: Context) -> NSView { + if let videoView = viewModel.makeVideoView(for: participantID) { + return videoView + } + return makePlaceholder() + } + + func updateNSView(_ nsView: NSView, context: Context) { + // Video track updates are managed by the LiveKit VideoView internally. + // No manual refresh needed here. + } + + private func makePlaceholder() -> NSView { + let view = NSView() + view.wantsLayer = true + view.layer?.backgroundColor = NSColor.darkGray.cgColor + view.layer?.cornerRadius = 10 + return view + } +} + +// MARK: - Previews + +#Preview("Idle") { + CallView(viewModel: PreviewCallViewModel(), onDismiss: {}) + .frame(width: 640, height: 480) +} + +#Preview("Connected") { + let vm = PreviewCallViewModel() + return CallView(viewModel: vm, onDismiss: {}) + .frame(width: 640, height: 480) + .task { + try? await vm.connect(url: "wss://preview.example.com", token: "preview-token") + } +} diff --git a/Relay/Views/MainView.swift b/Relay/Views/MainView.swift index b42bd84..5a917ea 100644 --- a/Relay/Views/MainView.swift +++ b/Relay/Views/MainView.swift @@ -46,6 +46,8 @@ struct MainView: View { // swiftlint:disable:this type_body_length @State private var isJoiningLinkedRoom = false @State private var inspectorSelectedProfile: UserProfile? @State private var inspectorInitialTab: InspectorTab? + @State private var activeCallViewModel: (any CallViewModelProtocol)? + @State private var isShowingCall = false private func scrollToMessage(_ eventId: String) { showingPinnedMessages = false @@ -236,6 +238,14 @@ struct MainView: View { // swiftlint:disable:this type_body_length .sheet(item: $leaveSpaceItem) { item in LeaveSpaceSheet(spaceName: item.name, spaceId: item.id, children: item.children) } + .sheet(isPresented: $isShowingCall) { + if let callViewModel = activeCallViewModel { + CallView(viewModel: callViewModel) { + isShowingCall = false + activeCallViewModel = nil + } + } + } .onChange(of: matrixService.spaces.map(\.id)) { if let selectedSpaceId, !matrixService.spaces.contains(where: { $0.id == selectedSpaceId }) { self.selectedSpaceId = nil @@ -392,6 +402,14 @@ struct MainView: View { // swiftlint:disable:this type_body_length .disabled(selectedRoomId == nil && selectedSpaceId == nil) } + // MARK: - Call Handling + + private func startCall(roomId: String) { + guard let viewModel = matrixService.makeCallViewModel(roomId: roomId) else { return } + activeCallViewModel = viewModel + isShowingCall = true + } + // MARK: - Deep Link Handling /// Handles an incoming ``MatrixURI`` deep link by navigating to the referenced entity. diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift new file mode 100644 index 0000000..d5bc220 --- /dev/null +++ b/RelayKit/Call/CallViewModel.swift @@ -0,0 +1,178 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import LiveKit +import RelayInterface +import OSLog + +private let logger = Logger(subsystem: "RelayKit", category: "CallViewModel") + +/// A concrete ``CallViewModelProtocol`` implementation backed by the LiveKit Swift SDK. +/// +/// ``CallViewModel`` owns a `LiveKit.Room` instance and bridges its delegate callbacks +/// into ``@Observable`` state for SwiftUI consumption. +/// +/// The inner ``Delegate`` class implements `RoomDelegate` and dispatches all callbacks +/// onto the main actor via `Task { @MainActor in … }` so that UI state mutations are +/// always performed on the correct actor without requiring LiveKit itself to be +/// `@MainActor`-aware. +@Observable +@MainActor +public final class CallViewModel: CallViewModelProtocol { + public private(set) var state: CallState = .idle + public private(set) var participants: [CallParticipant] = [] + public private(set) var isLocalCameraEnabled: Bool = false + public private(set) var isLocalMicrophoneEnabled: Bool = false + public private(set) var localParticipantID: String? + + private let room = Room() + private var delegate: Delegate? + + public init() { + let delegate = Delegate(viewModel: self) + self.delegate = delegate + room.add(delegate: delegate) + } + + // MARK: - CallViewModelProtocol + + public func connect(url: String, token: String) async throws { + state = .connecting + do { + try await room.connect(url: url, token: token) + localParticipantID = room.localParticipant.identity?.stringValue + try await room.localParticipant.setCamera(enabled: true) + try await room.localParticipant.setMicrophone(enabled: true) + isLocalCameraEnabled = true + isLocalMicrophoneEnabled = true + state = .connected + } catch { + state = .failed(error.localizedDescription) + throw error + } + } + + public func disconnect() async { + await room.disconnect() + state = .disconnected + participants = [] + isLocalCameraEnabled = false + isLocalMicrophoneEnabled = false + localParticipantID = nil + } + + public func toggleCamera() async throws { + let enabled = !isLocalCameraEnabled + try await room.localParticipant.setCamera(enabled: enabled) + isLocalCameraEnabled = enabled + } + + public func toggleMicrophone() async throws { + let enabled = !isLocalMicrophoneEnabled + try await room.localParticipant.setMicrophone(enabled: enabled) + isLocalMicrophoneEnabled = enabled + } + + public func makeVideoView(for participantID: String) -> NSView? { + let participant: Participant? + if room.localParticipant.identity?.stringValue == participantID { + participant = room.localParticipant + } else { + participant = room.remoteParticipants.values.first(where: { + $0.identity?.stringValue == participantID + }) + } + guard let participant, + let publication = participant.videoTracks.first?.value, + let track = publication.track as? VideoTrack else { + return nil + } + let videoView = VideoView() + videoView.track = track + return videoView + } + + // MARK: - Participant Sync + + fileprivate func syncParticipants() { + participants = room.remoteParticipants.values.map { participant in + CallParticipant( + id: participant.identity?.stringValue ?? participant.sid?.stringValue ?? UUID().uuidString, + displayName: participant.name, + isCameraEnabled: participant.isCameraEnabled(), + isMicrophoneEnabled: participant.isMicrophoneEnabled(), + isSpeaking: participant.isSpeaking + ) + } + } + + // MARK: - Delegate Bridge + + /// Bridges `RoomDelegate` callbacks — which arrive on an unspecified thread — onto + /// the main actor so that `CallViewModel`'s `@Observable` state is always mutated + /// safely. The class is `@unchecked Sendable` because `viewModel` is a weak reference + /// that is only read inside `Task { @MainActor in … }` blocks. + private final class Delegate: RoomDelegate, @unchecked Sendable { + weak var viewModel: CallViewModel? + + init(viewModel: CallViewModel) { + self.viewModel = viewModel + } + + func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldValue: ConnectionState) { + Task { @MainActor [weak viewModel] in + guard let viewModel else { return } + switch connectionState { + case .connected: + if viewModel.state != .connected { + viewModel.state = .connected + } + case .disconnected: + if viewModel.state == .connected { + viewModel.state = .disconnected + } + case .reconnecting: + logger.info("Call reconnecting") + default: + break + } + } + } + + func room(_ room: Room, participantDidConnect participant: RemoteParticipant) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants() + } + } + + func room(_ room: Room, participantDidDisconnect participant: RemoteParticipant) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants() + } + } + + func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants() + } + } + + func room(_ room: Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants() + } + } + } +} diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 9fe2afb..775e8a1 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1440,6 +1440,10 @@ public final class MatrixService: MatrixServiceProtocol { return viewModel } + public func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { + CallViewModel() + } + public func declinePendingVerificationRequest() async { pendingVerificationRequest = nil try? await verificationController?.cancelVerification() From 18e0d7653980aa91a6cceb7bb9cfd67eba3e8db5 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 16:08:26 -0400 Subject: [PATCH 02/24] Fix CallViewModel build errors - Add missing `import AppKit` so NSView resolves in the RelayKit framework - Qualify all RoomDelegate method Room parameters as `LiveKit.Room` to resolve ambiguity with the MatrixRustSDK Room type - Fix videoTracks access: LiveKit v2 exposes an array not a dictionary, so `.first` yields the publication directly without a `.value` key-path - Qualify `LiveKit.Room()` constructor for the same ambiguity reason Co-Authored-By: Claude Sonnet 4.6 --- RelayKit/Call/CallViewModel.swift | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index d5bc220..fcb0439 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import AppKit import Foundation import LiveKit import RelayInterface @@ -37,7 +38,7 @@ public final class CallViewModel: CallViewModelProtocol { public private(set) var isLocalMicrophoneEnabled: Bool = false public private(set) var localParticipantID: String? - private let room = Room() + private let room = LiveKit.Room() private var delegate: Delegate? public init() { @@ -95,7 +96,7 @@ public final class CallViewModel: CallViewModelProtocol { }) } guard let participant, - let publication = participant.videoTracks.first?.value, + let publication = participant.videoTracks.first, let track = publication.track as? VideoTrack else { return nil } @@ -131,7 +132,7 @@ public final class CallViewModel: CallViewModelProtocol { self.viewModel = viewModel } - func room(_ room: Room, didUpdateConnectionState connectionState: ConnectionState, from oldValue: ConnectionState) { + func room(_ room: LiveKit.Room, didUpdateConnectionState connectionState: LiveKit.ConnectionState, from oldValue: LiveKit.ConnectionState) { Task { @MainActor [weak viewModel] in guard let viewModel else { return } switch connectionState { @@ -151,25 +152,25 @@ public final class CallViewModel: CallViewModelProtocol { } } - func room(_ room: Room, participantDidConnect participant: RemoteParticipant) { + func room(_ room: LiveKit.Room, participantDidConnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in viewModel?.syncParticipants() } } - func room(_ room: Room, participantDidDisconnect participant: RemoteParticipant) { + func room(_ room: LiveKit.Room, participantDidDisconnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in viewModel?.syncParticipants() } } - func room(_ room: Room, didUpdateSpeakingParticipants participants: [Participant]) { + func room(_ room: LiveKit.Room, didUpdateSpeakingParticipants participants: [Participant]) { Task { @MainActor [weak viewModel] in viewModel?.syncParticipants() } } - func room(_ room: Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { + func room(_ room: LiveKit.Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { Task { @MainActor [weak viewModel] in viewModel?.syncParticipants() } From 3deab5e8bafc21d3390761e45982c4c3b69b4ad8 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 16:36:27 -0400 Subject: [PATCH 03/24] Fetch LiveKit credentials from homeserver via MatrixRTC (MSC4143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the full credential exchange flow so calls connect automatically: LiveKitCredentialService (new, RelayKit/Call/): - Step 1: Discover SFU URL via GET /_matrix/client/unstable/ org.matrix.msc4143/rtc/transports; falls back to reading org.matrix.msc4143.rtc_foci from .well-known/matrix/client - Step 2: Obtain an OpenID token via POST /_matrix/client/v3/user/ {userId}/openid/request_token using the session's Matrix access token - Step 3: Exchange with the SFU's POST /get_token (MSC4143 v2) or the legacy POST /sfu/get; both return { url, jwt } for LiveKit MatrixServiceProtocol / MatrixService: - New callCredentials(for roomId:) method builds LiveKitCredentialService from the active session (homeserver, accessToken, userID, deviceID) and returns the (livekitURL, token) tuple MainView: - startCall() now auto-fetches credentials in a background Task and calls viewModel.connect(url:token:) immediately; falls back to the manual join form if the homeserver doesn't support MatrixRTC - isPreparingCall flag passed to CallView to drive the correct UI state CallView: - New isPreparingCredentials parameter: when true, .idle state shows "Contacting call server…" spinner with Cancel instead of the join form - The join form remains as a fallback for unsupported homeservers or direct LiveKit connections Co-Authored-By: Claude Sonnet 4.6 --- .../Protocols/MatrixServiceProtocol.swift | 17 + Relay.xcodeproj/project.pbxproj | 2 + .../xcshareddata/swiftpm/Package.resolved | 47 ++- Relay/Services/PreviewMatrixService.swift | 6 + Relay/Views/CallView.swift | 270 ++++++++++----- Relay/Views/MainView.swift | 20 +- RelayKit/Call/LiveKitCredentialService.swift | 307 ++++++++++++++++++ RelayKit/Services/MatrixService.swift | 14 + 8 files changed, 591 insertions(+), 92 deletions(-) create mode 100644 RelayKit/Call/LiveKitCredentialService.swift diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift index 8819974..6194c42 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift @@ -487,6 +487,18 @@ public protocol MatrixServiceProtocol: AnyObject, Observable { /// URL and token, or `nil` if calling is not supported. func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? + /// Fetches LiveKit credentials for a Matrix room using the MatrixRTC flow (MSC4143). + /// + /// Discovers the SFU URL from the homeserver, obtains an OpenID token, and + /// exchanges it with the SFU's JWT service. The returned URL and token can be + /// passed directly to ``CallViewModelProtocol/connect(url:token:)``. + /// + /// - Parameter roomId: The Matrix room identifier. + /// - Returns: A tuple of `(livekitURL, token)` where `livekitURL` is the LiveKit + /// WebSocket URL and `token` is the JWT access token. + /// - Throws: If the homeserver doesn't support MatrixRTC or credential exchange fails. + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) + // MARK: Notification Settings (synced via push rules) /// Returns the default notification mode for rooms of the given type. @@ -756,6 +768,8 @@ public extension EnvironmentValues { } } +private struct PlaceholderError: Error {} + @Observable private final class PlaceholderMatrixService: MatrixServiceProtocol { let activityLog: any ActivityLogProtocol = PlaceholderActivityLog() @@ -817,6 +831,9 @@ private final class PlaceholderMatrixService: MatrixServiceProtocol { func encryptionState() async -> EncryptionStatus { EncryptionStatus() } func makeSessionVerificationViewModel() async throws -> (any SessionVerificationViewModelProtocol)? { nil } func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { nil } + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + throw PlaceholderError() + } func getDefaultNotificationMode( isOneToOne: Bool ) async throws -> DefaultNotificationMode { .mentionsAndKeywordsOnly } diff --git a/Relay.xcodeproj/project.pbxproj b/Relay.xcodeproj/project.pbxproj index 20af381..2de8e3b 100644 --- a/Relay.xcodeproj/project.pbxproj +++ b/Relay.xcodeproj/project.pbxproj @@ -12,6 +12,7 @@ 3BRI00032FC10003001F0EA1 /* RelayInterface in Frameworks */ = {isa = PBXBuildFile; productRef = 3BRI00042FC10004001F0EA1 /* RelayInterface */; }; 3BRK00012FC00001001F0EA1 /* RelayKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3BRK00002FC00000001F0EA1 /* RelayKit.framework */; }; 3BRK00022FC00002001F0EA1 /* RelayKit.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 3BRK00002FC00000001F0EA1 /* RelayKit.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + DF41154C2F82F7A30028241B /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = 3BLK00022FD10002001F0EA1 /* LiveKit */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -104,6 +105,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + DF41154C2F82F7A30028241B /* LiveKit in Frameworks */, 3BMRST012FB10001001F0EA1 /* MatrixRustSDK in Frameworks */, 3BRI00012FC10001001F0EA1 /* RelayInterface in Frameworks */, ); diff --git a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index eecca9c..3937f04 100644 --- a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,6 +1,24 @@ { - "originHash" : "6c542309bc3e806f061c41054f96023ef09c12c16fd5c288ab6028ddfe261e92", + "originHash" : "0b3fc90c7698ae92e7b431d3f3402e432621245a4f88792e53c71e7d7dc7a510", "pins" : [ + { + "identity" : "client-sdk-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/client-sdk-swift", + "state" : { + "revision" : "a25069f0e808d7e75f7cc93d157aa8e7cee3c58b", + "version" : "2.12.1" + } + }, + { + "identity" : "livekit-uniffi-xcframework", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/livekit-uniffi-xcframework.git", + "state" : { + "revision" : "61229f4032131311b997ddb1bc1cb8f5afbe30c8", + "version" : "0.0.5" + } + }, { "identity" : "matrix-rust-components-swift", "kind" : "remoteSourceControl", @@ -9,6 +27,33 @@ "revision" : "2916f3f9cc2aea86ba3a820cb6a8389e13e0284a", "version" : "26.4.1" } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "a008af1a102ff3dd6cc3764bb69bf63226d0f5f6", + "version" : "1.36.1" + } + }, + { + "identity" : "webrtc-xcframework", + "kind" : "remoteSourceControl", + "location" : "https://github.com/livekit/webrtc-xcframework.git", + "state" : { + "revision" : "0aa6a5ea4031d492d0493e3e4d4fbe08b5a0df78", + "version" : "137.7151.12" + } } ], "version" : 3 diff --git a/Relay/Services/PreviewMatrixService.swift b/Relay/Services/PreviewMatrixService.swift index 5ef440e..9da9619 100644 --- a/Relay/Services/PreviewMatrixService.swift +++ b/Relay/Services/PreviewMatrixService.swift @@ -228,6 +228,12 @@ final class PreviewMatrixService: MatrixServiceProtocol { PreviewCallViewModel() } + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + // Simulate a brief credential fetch; previews never actually connect. + try? await Task.sleep(for: .milliseconds(500)) + return (livekitURL: "wss://preview.livekit.example.com", token: "preview-jwt-token") + } + func declinePendingVerificationRequest() async { pendingVerificationRequest = nil } diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index 8c2a43e..aa4fd6f 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -16,29 +16,39 @@ import AppKit import RelayInterface import SwiftUI -/// Renders an active or connecting LiveKit call within a Matrix room. +/// Renders a LiveKit audio/video call within a Matrix room. /// -/// ``CallView`` shows participant tiles with video/audio indicators and a bottom -/// control bar for toggling local media and ending the call. It relies solely on -/// ``CallViewModelProtocol`` — no LiveKit types are referenced here. +/// When the view model is in the ``CallState/idle`` state, ``CallView`` shows a +/// credential-entry form so the user can supply the LiveKit server URL and JWT token +/// before connecting. While connecting it shows a spinner with a Cancel button. +/// Once connected it shows participant tiles and a media-control bar. struct CallView: View { @State var viewModel: any CallViewModelProtocol + /// `true` while the parent is fetching LiveKit credentials from the homeserver. + /// When set, the `.idle` state shows a spinner instead of the manual-entry form. + var isPreparingCredentials: Bool = false var onDismiss: () -> Void + // Local fields used only while in the .idle (pre-connect) manual-entry form. + @State private var serverURL: String = "" + @State private var accessToken: String = "" + @State private var isJoining: Bool = false + var body: some View { ZStack { Color.black.ignoresSafeArea() VStack(spacing: 0) { switch viewModel.state { - case .idle, .connecting: - Spacer() - ProgressView("Joining call…") - .progressViewStyle(.circular) - .controlSize(.large) - .foregroundStyle(.white) - .tint(.white) - Spacer() + case .idle: + if isPreparingCredentials { + preparingView + } else { + joinForm + } + + case .connecting: + connectingView case .connected: participantsGrid @@ -46,37 +56,147 @@ struct CallView: View { controlBar case .disconnected: - Spacer() - ContentUnavailableView( - "Call Ended", + endedView( + title: "Call Ended", systemImage: "phone.down.fill", - description: Text("The call has ended.") + description: "The call has ended.", + isError: false ) - .foregroundStyle(.white) - Button("Dismiss") { onDismiss() } - .buttonStyle(.borderedProminent) - .padding(.top, 12) - Spacer() case .failed(let message): - Spacer() - ContentUnavailableView( - "Call Failed", + endedView( + title: "Call Failed", systemImage: "exclamationmark.triangle.fill", - description: Text(message) + description: message, + isError: true ) - .foregroundStyle(.white) - Button("Dismiss") { onDismiss() } - .buttonStyle(.borderedProminent) - .tint(.red) - .padding(.top, 12) - Spacer() } } } .frame(minWidth: 480, minHeight: 360) } + // MARK: - Preparing View (fetching credentials from homeserver) + + @ViewBuilder + private var preparingView: some View { + Spacer() + ProgressView("Contacting call server…") + .progressViewStyle(.circular) + .controlSize(.large) + .foregroundStyle(.white) + .tint(.white) + Button("Cancel") { onDismiss() } + .buttonStyle(.bordered) + .foregroundStyle(.white) + .padding(.top, 20) + Spacer() + } + + // MARK: - Join Form (idle state) + + @ViewBuilder + private var joinForm: some View { + VStack(spacing: 0) { + Spacer() + + VStack(spacing: 20) { + Image(systemName: "phone.fill") + .font(.system(size: 44)) + .foregroundStyle(.white) + + Text("Join Call") + .font(.title2.bold()) + .foregroundStyle(.white) + + Text("Enter the LiveKit server URL and access token\nprovided by your call server.") + .font(.subheadline) + .foregroundStyle(.white.opacity(0.7)) + .multilineTextAlignment(.center) + + VStack(alignment: .leading, spacing: 8) { + Text("Server URL") + .font(.caption) + .foregroundStyle(.white.opacity(0.7)) + TextField("wss://livekit.example.com", text: $serverURL) + .textFieldStyle(.roundedBorder) + .autocorrectionDisabled() + + Text("Access Token") + .font(.caption) + .foregroundStyle(.white.opacity(0.7)) + .padding(.top, 4) + TextField("JWT token", text: $accessToken) + .textFieldStyle(.roundedBorder) + .autocorrectionDisabled() + } + .frame(maxWidth: 360) + + HStack(spacing: 16) { + Button("Cancel") { + onDismiss() + } + .buttonStyle(.bordered) + .foregroundStyle(.white) + + Button("Join") { + guard !serverURL.isEmpty, !accessToken.isEmpty else { return } + Task { + isJoining = true + try? await viewModel.connect(url: serverURL, token: accessToken) + isJoining = false + } + } + .buttonStyle(.borderedProminent) + .disabled(serverURL.isEmpty || accessToken.isEmpty || isJoining) + } + } + .padding(40) + + Spacer() + } + } + + // MARK: - Connecting View + + @ViewBuilder + private var connectingView: some View { + Spacer() + ProgressView("Joining call…") + .progressViewStyle(.circular) + .controlSize(.large) + .foregroundStyle(.white) + .tint(.white) + Button("Cancel") { + Task { + await viewModel.disconnect() + onDismiss() + } + } + .buttonStyle(.bordered) + .foregroundStyle(.white) + .padding(.top, 20) + Spacer() + } + + // MARK: - Ended / Failed View + + @ViewBuilder + private func endedView(title: String, systemImage: String, description: String, isError: Bool) -> some View { + Spacer() + ContentUnavailableView( + title, + systemImage: systemImage, + description: Text(description) + ) + .foregroundStyle(.white) + Button("Dismiss") { onDismiss() } + .buttonStyle(.borderedProminent) + .tint(isError ? .red : .accentColor) + .padding(.top, 12) + Spacer() + } + // MARK: - Participants Grid @ViewBuilder @@ -87,7 +207,6 @@ struct CallView: View { ForEach(viewModel.participants) { participant in participantTile(participant) } - // Local participant tile if let localID = viewModel.localParticipantID { localParticipantTile(id: localID) } @@ -101,49 +220,36 @@ struct CallView: View { @ViewBuilder private func participantTile(_ participant: CallParticipant) -> some View { ZStack(alignment: .bottom) { - // Video or placeholder background VideoViewRepresentable(viewModel: viewModel, participantID: participant.id) .aspectRatio(16 / 9, contentMode: .fit) .clipShape(RoundedRectangle(cornerRadius: 10)) - // Speaking ring overlay if participant.isSpeaking { RoundedRectangle(cornerRadius: 10) .strokeBorder(.green, lineWidth: 3) } - // Name label + media indicators HStack(spacing: 6) { Text(participant.displayName ?? participant.id) .font(.caption) .foregroundStyle(.white) .lineLimit(1) .truncationMode(.tail) - Spacer() - if !participant.isMicrophoneEnabled { - Image(systemName: "mic.slash.fill") - .font(.caption) - .foregroundStyle(.red) + Image(systemName: "mic.slash.fill").font(.caption).foregroundStyle(.red) } if !participant.isCameraEnabled { - Image(systemName: "video.slash.fill") - .font(.caption) - .foregroundStyle(.secondary) + Image(systemName: "video.slash.fill").font(.caption).foregroundStyle(.secondary) } } .padding(.horizontal, 8) .padding(.vertical, 6) .background(.black.opacity(0.55)) - .clipShape( - .rect( - topLeadingRadius: 0, - bottomLeadingRadius: 10, - bottomTrailingRadius: 10, - topTrailingRadius: 0 - ) - ) + .clipShape(.rect( + topLeadingRadius: 0, bottomLeadingRadius: 10, + bottomTrailingRadius: 10, topTrailingRadius: 0 + )) } } @@ -157,34 +263,22 @@ struct CallView: View { .clipShape(RoundedRectangle(cornerRadius: 10)) HStack(spacing: 6) { - Text("You") - .font(.caption) - .foregroundStyle(.white) - + Text("You").font(.caption).foregroundStyle(.white) Spacer() - if !viewModel.isLocalMicrophoneEnabled { - Image(systemName: "mic.slash.fill") - .font(.caption) - .foregroundStyle(.red) + Image(systemName: "mic.slash.fill").font(.caption).foregroundStyle(.red) } if !viewModel.isLocalCameraEnabled { - Image(systemName: "video.slash.fill") - .font(.caption) - .foregroundStyle(.secondary) + Image(systemName: "video.slash.fill").font(.caption).foregroundStyle(.secondary) } } .padding(.horizontal, 8) .padding(.vertical, 6) .background(.black.opacity(0.55)) - .clipShape( - .rect( - topLeadingRadius: 0, - bottomLeadingRadius: 10, - bottomTrailingRadius: 10, - topTrailingRadius: 0 - ) - ) + .clipShape(.rect( + topLeadingRadius: 0, bottomLeadingRadius: 10, + bottomTrailingRadius: 10, topTrailingRadius: 0 + )) } } @@ -193,7 +287,6 @@ struct CallView: View { @ViewBuilder private var controlBar: some View { HStack(spacing: 24) { - // Microphone toggle Button { Task { try? await viewModel.toggleMicrophone() } } label: { @@ -207,7 +300,6 @@ struct CallView: View { .foregroundStyle(.white) .help(viewModel.isLocalMicrophoneEnabled ? "Mute microphone" : "Unmute microphone") - // Camera toggle Button { Task { try? await viewModel.toggleCamera() } } label: { @@ -221,7 +313,6 @@ struct CallView: View { .foregroundStyle(.white) .help(viewModel.isLocalCameraEnabled ? "Turn off camera" : "Turn on camera") - // End call button Button { Task { await viewModel.disconnect() @@ -246,11 +337,8 @@ struct CallView: View { // MARK: - NSView Bridge for Video -/// An `NSViewRepresentable` that embeds the opaque `NSView` returned by -/// ``CallViewModelProtocol/makeVideoView(for:)``. -/// -/// If the view model returns `nil` (participant has no active video track), a dark -/// gray placeholder with the participant's initials is shown instead. +/// Embeds the opaque `NSView` returned by ``CallViewModelProtocol/makeVideoView(for:)``. +/// Shows a dark placeholder when the participant has no active video track. private struct VideoViewRepresentable: NSViewRepresentable { let viewModel: any CallViewModelProtocol let participantID: String @@ -259,30 +347,32 @@ private struct VideoViewRepresentable: NSViewRepresentable { if let videoView = viewModel.makeVideoView(for: participantID) { return videoView } - return makePlaceholder() + let placeholder = NSView() + placeholder.wantsLayer = true + placeholder.layer?.backgroundColor = NSColor.darkGray.cgColor + placeholder.layer?.cornerRadius = 10 + return placeholder } func updateNSView(_ nsView: NSView, context: Context) { - // Video track updates are managed by the LiveKit VideoView internally. - // No manual refresh needed here. - } - - private func makePlaceholder() -> NSView { - let view = NSView() - view.wantsLayer = true - view.layer?.backgroundColor = NSColor.darkGray.cgColor - view.layer?.cornerRadius = 10 - return view + // LiveKit's VideoView manages track updates internally. } } // MARK: - Previews -#Preview("Idle") { +#Preview("Join Form") { CallView(viewModel: PreviewCallViewModel(), onDismiss: {}) .frame(width: 640, height: 480) } +#Preview("Connecting") { + @Previewable @State var vm = PreviewCallViewModel() + CallView(viewModel: vm, onDismiss: {}) + .frame(width: 640, height: 480) + .onAppear { vm.state = .connecting } +} + #Preview("Connected") { let vm = PreviewCallViewModel() return CallView(viewModel: vm, onDismiss: {}) diff --git a/Relay/Views/MainView.swift b/Relay/Views/MainView.swift index 5a917ea..68e0024 100644 --- a/Relay/Views/MainView.swift +++ b/Relay/Views/MainView.swift @@ -48,6 +48,7 @@ struct MainView: View { // swiftlint:disable:this type_body_length @State private var inspectorInitialTab: InspectorTab? @State private var activeCallViewModel: (any CallViewModelProtocol)? @State private var isShowingCall = false + @State private var isPreparingCall = false private func scrollToMessage(_ eventId: String) { showingPinnedMessages = false @@ -240,8 +241,12 @@ struct MainView: View { // swiftlint:disable:this type_body_length } .sheet(isPresented: $isShowingCall) { if let callViewModel = activeCallViewModel { - CallView(viewModel: callViewModel) { + CallView( + viewModel: callViewModel, + isPreparingCredentials: isPreparingCall + ) { isShowingCall = false + isPreparingCall = false activeCallViewModel = nil } } @@ -405,9 +410,22 @@ struct MainView: View { // swiftlint:disable:this type_body_length // MARK: - Call Handling private func startCall(roomId: String) { + guard !isPreparingCall else { return } guard let viewModel = matrixService.makeCallViewModel(roomId: roomId) else { return } activeCallViewModel = viewModel + isPreparingCall = true isShowingCall = true + + Task { + do { + let (url, token) = try await matrixService.callCredentials(for: roomId) + try await viewModel.connect(url: url, token: token) + } catch { + // Credential fetch or connect failed — viewModel stays in .idle so + // CallView shows the manual-entry join form as a fallback. + } + isPreparingCall = false + } } // MARK: - Deep Link Handling diff --git a/RelayKit/Call/LiveKitCredentialService.swift b/RelayKit/Call/LiveKitCredentialService.swift new file mode 100644 index 0000000..723a13c --- /dev/null +++ b/RelayKit/Call/LiveKitCredentialService.swift @@ -0,0 +1,307 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import os + +private let logger = Logger(subsystem: "RelayKit", category: "LiveKitCredentialService") + +/// Fetches LiveKit credentials (WebSocket URL + JWT) for a Matrix room by +/// implementing the MatrixRTC credential exchange flow (MSC4143). +/// +/// **Step 1 – Discover the SFU URL** +/// Tries `GET /_matrix/client/unstable/org.matrix.msc4143/rtc/transports`. +/// If that returns 404, falls back to reading `org.matrix.msc4143.rtc_foci` +/// from `GET {server}/.well-known/matrix/client`. +/// +/// **Step 2 – Request an OpenID token** +/// `POST /_matrix/client/v3/user/{userId}/openid/request_token` using the +/// session's Matrix access token as Bearer auth. +/// +/// **Step 3 – Exchange for a LiveKit JWT** +/// `POST {sfuURL}/get_token` (MSC4143 v2). Falls back to the legacy +/// `POST {sfuURL}/sfu/get` endpoint if the server returns 404. +/// +/// Both exchange endpoints return `{ url, jwt }` where `url` is the LiveKit +/// WebSocket address and `jwt` is the LiveKit room access token. +struct LiveKitCredentialService { + + let homeserver: String + let accessToken: String + let userID: String + let deviceID: String + + // MARK: - Public Entry Point + + /// Returns `(livekitWebSocketURL, livekitJWT)` for the given Matrix room. + func credentials(for roomID: String) async throws -> (url: String, token: String) { + logger.info("Fetching LiveKit credentials for room \(roomID, privacy: .private)") + let sfuURL = try await discoverSFUURL() + logger.info("SFU URL discovered: \(sfuURL)") + let openIDToken = try await requestOpenIDToken() + logger.debug("OpenID token obtained") + return try await fetchLiveKitToken(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) + } + + // MARK: - Step 1: Discover SFU URL + + private func discoverSFUURL() async throws -> String { + // Prefer the MSC4143 transports endpoint + if let url = try? await fetchRTCTransportsURL() { + return url + } + // Fall back to .well-known + if let url = try? await fetchWellKnownSFUURL() { + return url + } + throw LiveKitCredentialError.sfuURLNotFound + } + + private func fetchRTCTransportsURL() async throws -> String { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + guard let url = URL(string: "\(base)/_matrix/client/unstable/org.matrix.msc4143/rtc/transports") else { + throw LiveKitCredentialError.invalidURL + } + var request = URLRequest(url: url) + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LiveKitCredentialError.serverError + } + + let decoded = try JSONDecoder().decode(RTCTransportsResponse.self, from: data) + guard let livekit = decoded.transports.first(where: { $0.type == "livekit" }) else { + throw LiveKitCredentialError.sfuURLNotFound + } + return livekit.livekitServiceUrl + } + + private func fetchWellKnownSFUURL() async throws -> String { + guard let serverURL = URL(string: homeserver), let host = serverURL.host else { + throw LiveKitCredentialError.invalidURL + } + let scheme = serverURL.scheme ?? "https" + guard let url = URL(string: "\(scheme)://\(host)/.well-known/matrix/client") else { + throw LiveKitCredentialError.invalidURL + } + + let (data, response) = try await URLSession.shared.data(from: url) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LiveKitCredentialError.serverError + } + + let decoded = try JSONDecoder().decode(WellKnownResponse.self, from: data) + guard let foci = decoded.rtcFoci, + let first = foci.first(where: { $0.type == "livekit" }) else { + throw LiveKitCredentialError.sfuURLNotFound + } + return first.livekitServiceUrl + } + + // MARK: - Step 2: Request OpenID Token + + private func requestOpenIDToken() async throws -> OpenIDTokenPayload { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encoded = userID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? userID + guard let url = URL(string: "\(base)/_matrix/client/v3/user/\(encoded)/openid/request_token") else { + throw LiveKitCredentialError.invalidURL + } + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = Data("{}".utf8) + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LiveKitCredentialError.openIDTokenFailed + } + return try JSONDecoder().decode(OpenIDTokenPayload.self, from: data) + } + + // MARK: - Step 3: Exchange for LiveKit JWT + + private func fetchLiveKitToken( + sfuURL: String, + roomID: String, + openIDToken: OpenIDTokenPayload + ) async throws -> (url: String, token: String) { + // Try the v2 endpoint first, fall back to legacy + if let result = try? await fetchLiveKitTokenV2(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) { + return result + } + return try await fetchLiveKitTokenLegacy(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) + } + + private func fetchLiveKitTokenV2( + sfuURL: String, + roomID: String, + openIDToken: OpenIDTokenPayload + ) async throws -> (url: String, token: String) { + let base = sfuURL.trimmingCharacters(in: .init(charactersIn: "/")) + guard let url = URL(string: "\(base)/get_token") else { + throw LiveKitCredentialError.invalidURL + } + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let body = GetTokenRequest( + roomId: roomID, + openidToken: openIDToken, + member: .init(id: "\(userID):\(deviceID)", claimedUserId: userID, claimedDeviceId: deviceID) + ) + request.httpBody = try JSONEncoder().encode(body) + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LiveKitCredentialError.tokenExchangeFailed + } + let decoded = try JSONDecoder().decode(LiveKitTokenResponse.self, from: data) + logger.info("LiveKit credentials obtained via /get_token") + return (decoded.url, decoded.jwt) + } + + private func fetchLiveKitTokenLegacy( + sfuURL: String, + roomID: String, + openIDToken: OpenIDTokenPayload + ) async throws -> (url: String, token: String) { + let base = sfuURL.trimmingCharacters(in: .init(charactersIn: "/")) + guard let url = URL(string: "\(base)/sfu/get") else { + throw LiveKitCredentialError.invalidURL + } + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + + let body = SFUGetRequest(room: roomID, openidToken: openIDToken, deviceId: deviceID) + request.httpBody = try JSONEncoder().encode(body) + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw LiveKitCredentialError.tokenExchangeFailed + } + let decoded = try JSONDecoder().decode(LiveKitTokenResponse.self, from: data) + logger.info("LiveKit credentials obtained via legacy /sfu/get") + return (decoded.url, decoded.jwt) + } +} + +// MARK: - Errors + +enum LiveKitCredentialError: LocalizedError { + case sfuURLNotFound + case invalidURL + case serverError + case openIDTokenFailed + case tokenExchangeFailed + + var errorDescription: String? { + switch self { + case .sfuURLNotFound: + return "This homeserver has no LiveKit call server configured. " + + "Check that your server supports MatrixRTC (MSC4143)." + case .invalidURL: + return "Could not construct a valid URL for the call server." + case .serverError: + return "The homeserver returned an error while fetching call credentials." + case .openIDTokenFailed: + return "Failed to obtain an OpenID token from the homeserver." + case .tokenExchangeFailed: + return "The call server rejected the credential exchange." + } + } +} + +// MARK: - Codable Types + +private struct RTCTransportsResponse: Decodable { + let transports: [Transport] + struct Transport: Decodable { + let type: String + let livekitServiceUrl: String + enum CodingKeys: String, CodingKey { + case type + case livekitServiceUrl = "livekit_service_url" + } + } +} + +private struct WellKnownResponse: Decodable { + let rtcFoci: [RtcFocus]? + struct RtcFocus: Decodable { + let type: String + let livekitServiceUrl: String + enum CodingKeys: String, CodingKey { + case type + case livekitServiceUrl = "livekit_service_url" + } + } + enum CodingKeys: String, CodingKey { + case rtcFoci = "org.matrix.msc4143.rtc_foci" + } +} + +// Internal type — not exposed outside RelayKit. +struct OpenIDTokenPayload: Codable { + let accessToken: String + let tokenType: String + let matrixServerName: String + let expiresIn: Int + enum CodingKeys: String, CodingKey { + case accessToken = "access_token" + case tokenType = "token_type" + case matrixServerName = "matrix_server_name" + case expiresIn = "expires_in" + } +} + +private struct GetTokenRequest: Encodable { + let roomId: String + let openidToken: OpenIDTokenPayload + let member: Member + struct Member: Encodable { + let id: String + let claimedUserId: String + let claimedDeviceId: String + enum CodingKeys: String, CodingKey { + case id + case claimedUserId = "claimed_user_id" + case claimedDeviceId = "claimed_device_id" + } + } + enum CodingKeys: String, CodingKey { + case roomId = "room_id" + case openidToken = "openid_token" + case member + } +} + +private struct SFUGetRequest: Encodable { + let room: String + let openidToken: OpenIDTokenPayload + let deviceId: String + enum CodingKeys: String, CodingKey { + case room + case openidToken = "openid_token" + case deviceId = "device_id" + } +} + +private struct LiveKitTokenResponse: Decodable { + let url: String + let jwt: String +} diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 775e8a1..0690fdc 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1444,6 +1444,20 @@ public final class MatrixService: MatrixServiceProtocol { CallViewModel() } + public func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + guard let client else { + throw LiveKitCredentialError.serverError + } + let session = try client.session() + let service = LiveKitCredentialService( + homeserver: client.homeserver, + accessToken: session.accessToken, + userID: client.userID, + deviceID: client.deviceID + ) + return try await service.credentials(for: roomId) + } + public func declinePendingVerificationRequest() async { pendingVerificationRequest = nil try? await verificationController?.cancelVerification() From 98a111537e6737fe56356fbf240d0762ebae66e6 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 16:37:35 -0400 Subject: [PATCH 04/24] Fix tuple label mismatch in callCredentials return LiveKitCredentialService returns (url:token:) but the protocol requires (livekitURL:token:); re-label on the way out of MatrixService. Co-Authored-By: Claude Sonnet 4.6 --- RelayKit/Services/MatrixService.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 0690fdc..06f13f4 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1455,7 +1455,8 @@ public final class MatrixService: MatrixServiceProtocol { userID: client.userID, deviceID: client.deviceID ) - return try await service.credentials(for: roomId) + let result = try await service.credentials(for: roomId) + return (livekitURL: result.url, token: result.token) } public func declinePendingVerificationRequest() async { From be4a0501db9561d8ff0dc4aaed56f78b5da8411f Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 16:43:38 -0400 Subject: [PATCH 05/24] Fix grey video box: cache VideoViews and update on re-render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local participant's camera was publishing successfully but the UI showed a grey placeholder because: 1. VideoViewRepresentable.makeNSView called makeVideoView(for:) once — if the track wasn't ready at that instant it returned the placeholder and updateNSView never replaced it (it was a no-op). 2. makeVideoView created a brand-new VideoView on every call so it was never stable across SwiftUI re-renders. CallViewModel fixes: - Cache VideoView instances per participant in a dictionary; return the same instance on subsequent calls and update its .track in place - Add videoTrackRevision counter, bumped after connect, toggleCamera, and syncParticipants — drives SwiftUI re-renders VideoViewRepresentable fixes: - makeNSView now creates a stable container NSView (dark grey background) - updateNSView asks the view model for the current video view and attaches it as a constrained subview of the container - If the video view is already attached, it's left in place Co-Authored-By: Claude Opus 4.6 --- Relay/Views/CallView.swift | 42 +++++++++++++++++++++++-------- RelayKit/Call/CallViewModel.swift | 28 ++++++++++++++++++--- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index aa4fd6f..af9a330 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -338,24 +338,44 @@ struct CallView: View { // MARK: - NSView Bridge for Video /// Embeds the opaque `NSView` returned by ``CallViewModelProtocol/makeVideoView(for:)``. -/// Shows a dark placeholder when the participant has no active video track. +/// +/// A stable container `NSView` is created in `makeNSView`. On each SwiftUI +/// re-render (driven by `viewModel` state changes) `updateNSView` asks the +/// view model for the current video view and swaps it into the container. +/// This handles the common timing issue where the video track isn't published +/// yet on the first render but becomes available shortly after. private struct VideoViewRepresentable: NSViewRepresentable { let viewModel: any CallViewModelProtocol let participantID: String func makeNSView(context: Context) -> NSView { - if let videoView = viewModel.makeVideoView(for: participantID) { - return videoView - } - let placeholder = NSView() - placeholder.wantsLayer = true - placeholder.layer?.backgroundColor = NSColor.darkGray.cgColor - placeholder.layer?.cornerRadius = 10 - return placeholder + let container = NSView() + container.wantsLayer = true + container.layer?.backgroundColor = NSColor.darkGray.cgColor + container.layer?.cornerRadius = 10 + attachVideoView(to: container) + return container + } + + func updateNSView(_ container: NSView, context: Context) { + attachVideoView(to: container) } - func updateNSView(_ nsView: NSView, context: Context) { - // LiveKit's VideoView manages track updates internally. + private func attachVideoView(to container: NSView) { + guard let videoView = viewModel.makeVideoView(for: participantID) else { return } + + // Already the current subview — nothing to do. + if container.subviews.first === videoView { return } + + container.subviews.forEach { $0.removeFromSuperview() } + videoView.translatesAutoresizingMaskIntoConstraints = false + container.addSubview(videoView) + NSLayoutConstraint.activate([ + videoView.leadingAnchor.constraint(equalTo: container.leadingAnchor), + videoView.trailingAnchor.constraint(equalTo: container.trailingAnchor), + videoView.topAnchor.constraint(equalTo: container.topAnchor), + videoView.bottomAnchor.constraint(equalTo: container.bottomAnchor), + ]) } } diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index fcb0439..eea55f4 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -37,9 +37,17 @@ public final class CallViewModel: CallViewModelProtocol { public private(set) var isLocalCameraEnabled: Bool = false public private(set) var isLocalMicrophoneEnabled: Bool = false public private(set) var localParticipantID: String? + /// Incremented whenever video tracks change, triggering SwiftUI to + /// re-call `updateNSView` on any `VideoViewRepresentable`. + public private(set) var videoTrackRevision: UInt = 0 private let room = LiveKit.Room() private var delegate: Delegate? + /// Cached `VideoView` instances keyed by participant identity. + /// Re-used across `makeVideoView(for:)` calls so the view stays stable + /// even as SwiftUI re-renders, and the `.track` is updated in place + /// when tracks change. + private var videoViews: [String: VideoView] = [:] public init() { let delegate = Delegate(viewModel: self) @@ -58,6 +66,7 @@ public final class CallViewModel: CallViewModelProtocol { try await room.localParticipant.setMicrophone(enabled: true) isLocalCameraEnabled = true isLocalMicrophoneEnabled = true + videoTrackRevision += 1 state = .connected } catch { state = .failed(error.localizedDescription) @@ -72,12 +81,14 @@ public final class CallViewModel: CallViewModelProtocol { isLocalCameraEnabled = false isLocalMicrophoneEnabled = false localParticipantID = nil + videoViews.removeAll() } public func toggleCamera() async throws { let enabled = !isLocalCameraEnabled try await room.localParticipant.setCamera(enabled: enabled) isLocalCameraEnabled = enabled + videoTrackRevision += 1 } public func toggleMicrophone() async throws { @@ -95,19 +106,28 @@ public final class CallViewModel: CallViewModelProtocol { $0.identity?.stringValue == participantID }) } - guard let participant, - let publication = participant.videoTracks.first, - let track = publication.track as? VideoTrack else { - return nil + + // Look up the current video track (may be nil if not yet published). + let track = participant?.videoTracks.first?.track as? VideoTrack + + // Return or create a cached VideoView. Its `.track` is updated in + // place every call so the rendered content stays current even when + // the underlying track changes (e.g. camera toggled on/off). + if let existing = videoViews[participantID] { + existing.track = track + return existing } + let videoView = VideoView() videoView.track = track + videoViews[participantID] = videoView return videoView } // MARK: - Participant Sync fileprivate func syncParticipants() { + videoTrackRevision += 1 participants = room.remoteParticipants.values.map { participant in CallParticipant( id: participant.identity?.stringValue ?? participant.sid?.stringValue ?? UUID().uuidString, From d03f95e8b07c3c7ab0a7383bc1d329ddeec4c9e6 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sun, 5 Apr 2026 17:07:52 -0400 Subject: [PATCH 06/24] Fix video track timing race by observing videoTrackRevision in SwiftUI The VideoViewRepresentable was never receiving updateNSView calls because videoTrackRevision was not exposed through the CallViewModelProtocol and was never read during SwiftUI body evaluation. Added the property to the protocol, passed it into the representable, and added delegate callbacks for local/remote track publish events. Includes diagnostic logging to trace track availability through the connect pipeline. Co-Authored-By: Claude Opus 4.6 --- .../Protocols/CallViewModelProtocol.swift | 6 ++ Relay/ViewModels/PreviewCallViewModel.swift | 1 + Relay/Views/CallView.swift | 8 ++- RelayKit/Call/CallViewModel.swift | 72 +++++++++++++++++-- 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift index aec84f6..dd2de74 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift @@ -83,6 +83,12 @@ public protocol CallViewModelProtocol: AnyObject, Observable { /// The identity of the local participant, set after connection. var localParticipantID: String? { get } + /// A monotonically increasing counter that is bumped whenever video tracks change + /// (publish, unpublish, camera toggle, etc.). SwiftUI views should read this value + /// to ensure ``NSViewRepresentable`` bridges receive `updateNSView` calls when the + /// underlying video track becomes available. + var videoTrackRevision: UInt { get } + /// Connects to the call using the provided LiveKit server URL and JWT token. /// /// - Parameters: diff --git a/Relay/ViewModels/PreviewCallViewModel.swift b/Relay/ViewModels/PreviewCallViewModel.swift index e614753..d2d0cc3 100644 --- a/Relay/ViewModels/PreviewCallViewModel.swift +++ b/Relay/ViewModels/PreviewCallViewModel.swift @@ -29,6 +29,7 @@ final class PreviewCallViewModel: CallViewModelProtocol { var isLocalCameraEnabled: Bool = false var isLocalMicrophoneEnabled: Bool = false var localParticipantID: String? = nil + var videoTrackRevision: UInt = 0 func connect(url: String, token: String) async throws { state = .connecting diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index af9a330..c922ce3 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -220,7 +220,7 @@ struct CallView: View { @ViewBuilder private func participantTile(_ participant: CallParticipant) -> some View { ZStack(alignment: .bottom) { - VideoViewRepresentable(viewModel: viewModel, participantID: participant.id) + VideoViewRepresentable(viewModel: viewModel, participantID: participant.id, trackRevision: viewModel.videoTrackRevision) .aspectRatio(16 / 9, contentMode: .fit) .clipShape(RoundedRectangle(cornerRadius: 10)) @@ -258,7 +258,7 @@ struct CallView: View { @ViewBuilder private func localParticipantTile(id: String) -> some View { ZStack(alignment: .bottom) { - VideoViewRepresentable(viewModel: viewModel, participantID: id) + VideoViewRepresentable(viewModel: viewModel, participantID: id, trackRevision: viewModel.videoTrackRevision) .aspectRatio(16 / 9, contentMode: .fit) .clipShape(RoundedRectangle(cornerRadius: 10)) @@ -347,6 +347,10 @@ struct CallView: View { private struct VideoViewRepresentable: NSViewRepresentable { let viewModel: any CallViewModelProtocol let participantID: String + /// Reading this value in the parent SwiftUI view ensures that when the view model + /// bumps it (e.g. after a track is published), SwiftUI re-evaluates this representable + /// and calls ``updateNSView``. + let trackRevision: UInt func makeNSView(context: Context) -> NSView { let container = NSView() diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index eea55f4..b4b7af4 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -61,14 +61,46 @@ public final class CallViewModel: CallViewModelProtocol { state = .connecting do { try await room.connect(url: url, token: token) - localParticipantID = room.localParticipant.identity?.stringValue + let identity = room.localParticipant.identity?.stringValue + localParticipantID = identity + logger.info("[CallVM] Connected. localParticipantID=\(identity ?? "nil")") + try await room.localParticipant.setCamera(enabled: true) + logger.info("[CallVM] setCamera(enabled: true) returned") + + // Log local video tracks immediately after setCamera + let localPubs = room.localParticipant.videoTracks + logger.info("[CallVM] localParticipant.videoTracks.count=\(localPubs.count)") + for pub in localPubs { + let hasTrack = pub.track != nil + let trackType = pub.track.map { String(describing: type(of: $0)) } ?? "nil" + logger.info("[CallVM] pub sid=\(pub.sid.stringValue), hasTrack=\(hasTrack), trackType=\(trackType), source=\(String(describing: pub.source))") + } + try await room.localParticipant.setMicrophone(enabled: true) + logger.info("[CallVM] setMicrophone(enabled: true) returned") + isLocalCameraEnabled = true isLocalMicrophoneEnabled = true - videoTrackRevision += 1 state = .connected + videoTrackRevision += 1 + + // Delayed re-check: log track state after 1 second + Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(1)) + guard let self else { return } + let pubs = self.room.localParticipant.videoTracks + logger.info("[CallVM] Delayed check: videoTracks.count=\(pubs.count)") + for pub in pubs { + let hasTrack = pub.track != nil + let isVideoTrack = pub.track is VideoTrack + logger.info("[CallVM] Delayed: pub sid=\(pub.sid.stringValue), hasTrack=\(hasTrack), isVideoTrack=\(isVideoTrack)") + } + self.videoTrackRevision += 1 + logger.info("[CallVM] Delayed videoTrackRevision bump to \(self.videoTrackRevision)") + } } catch { + logger.error("[CallVM] Connect failed: \(error.localizedDescription)") state = .failed(error.localizedDescription) throw error } @@ -89,6 +121,13 @@ public final class CallViewModel: CallViewModelProtocol { try await room.localParticipant.setCamera(enabled: enabled) isLocalCameraEnabled = enabled videoTrackRevision += 1 + // Same delayed retry as connect — the track may not be attached yet. + if enabled { + Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(500)) + self?.videoTrackRevision += 1 + } + } } public func toggleMicrophone() async throws { @@ -99,7 +138,8 @@ public final class CallViewModel: CallViewModelProtocol { public func makeVideoView(for participantID: String) -> NSView? { let participant: Participant? - if room.localParticipant.identity?.stringValue == participantID { + let isLocal = room.localParticipant.identity?.stringValue == participantID + if isLocal { participant = room.localParticipant } else { participant = room.remoteParticipants.values.first(where: { @@ -107,20 +147,31 @@ public final class CallViewModel: CallViewModelProtocol { }) } + let foundParticipant = participant != nil + let allPubCount = participant?.trackPublications.count ?? 0 + let videoPubCount = participant?.videoTracks.count ?? 0 + logger.info("[CallVM] makeVideoView(for: \(participantID)) isLocal=\(isLocal), found=\(foundParticipant), allPubs=\(allPubCount), videoPubs=\(videoPubCount)") + // Look up the current video track (may be nil if not yet published). - let track = participant?.videoTracks.first?.track as? VideoTrack + let firstPub = participant?.videoTracks.first + let rawTrack = firstPub?.track + let track = rawTrack as? VideoTrack + + logger.info("[CallVM] firstPub=\(firstPub != nil), rawTrack=\(rawTrack != nil), rawTrackType=\(rawTrack.map { String(describing: type(of: $0)) } ?? "nil"), castOK=\(track != nil)") // Return or create a cached VideoView. Its `.track` is updated in // place every call so the rendered content stays current even when // the underlying track changes (e.g. camera toggled on/off). if let existing = videoViews[participantID] { existing.track = track + logger.info("[CallVM] Reusing cached VideoView, track set to \(track != nil ? "non-nil" : "nil")") return existing } let videoView = VideoView() videoView.track = track videoViews[participantID] = videoView + logger.info("[CallVM] Created new VideoView, track set to \(track != nil ? "non-nil" : "nil")") return videoView } @@ -195,5 +246,18 @@ public final class CallViewModel: CallViewModelProtocol { viewModel?.syncParticipants() } } + + func room(_ room: LiveKit.Room, localParticipant: LocalParticipant, didPublishTrack publication: LocalTrackPublication) { + Task { @MainActor [weak viewModel] in + logger.info("Local track published: \(String(describing: publication.kind))") + viewModel?.videoTrackRevision += 1 + } + } + + func room(_ room: LiveKit.Room, participant: RemoteParticipant, didPublishTrack publication: RemoteTrackPublication) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants() + } + } } } From 16e3fe3acaf72b47cba1ce895ba0a6d38079aaab Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Mon, 6 Apr 2026 11:46:56 -0400 Subject: [PATCH 07/24] Use SwiftUIVideoView and proper RoomOptions for LiveKit calls Replace the custom VideoViewRepresentable (which caused garbled Metal rendering) with LiveKit's built-in SwiftUIVideoView. Cache video views per participant to prevent SwiftUI from tearing down the Metal surface on re-renders. Key changes: - Switch from NSView-based VideoViewRepresentable to SwiftUIVideoView wrapped in AnyView, returned via makeVideoView(for:) - Add video view cache keyed by participant ID + VideoTrack identity - Add isSubscribed / isMuted guards matching LiveKit components-swift - Configure RoomOptions with preferredCodec (.vp8), adaptiveStream, and dynacast; use ConnectOptions with enableMicrophone - Remove .clipShape() from video tiles (interferes with Metal) - Move aspectRatio to outer tile container - Clean up diagnostic logging and delayed retry tasks - Add network.server entitlement for WebRTC Co-Authored-By: Claude Opus 4.6 --- .../Protocols/CallViewModelProtocol.swift | 11 +- Relay/Relay.entitlements | 2 + Relay/Utilities/MatrixLink.swift | 86 +++++++++ Relay/ViewModels/PreviewCallViewModel.swift | 4 +- Relay/Views/CallView.swift | 95 ++++------ RelayKit/Call/CallViewModel.swift | 179 +++++++++--------- 6 files changed, 217 insertions(+), 160 deletions(-) create mode 100644 Relay/Utilities/MatrixLink.swift diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift index dd2de74..3ab5392 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AppKit import Foundation +import SwiftUI /// The connection state of a call. public enum CallState: Sendable, Equatable { @@ -105,12 +105,9 @@ public protocol CallViewModelProtocol: AnyObject, Observable { /// Toggles the local microphone on or off. func toggleMicrophone() async throws - /// Returns an ``NSView`` that renders the video track of the given participant, or `nil` - /// if the participant has no active video track or is not found. - /// - /// The returned view is owned by the call view model and must only be embedded — do not - /// deallocate it. A new view is returned on each call. + /// Returns a SwiftUI view that renders the video track of the given participant, + /// or `nil` if the participant has no active video track or is not found. /// /// - Parameter participantID: The ``CallParticipant/id`` of the participant to render. - func makeVideoView(for participantID: String) -> NSView? + func makeVideoView(for participantID: String) -> AnyView? } diff --git a/Relay/Relay.entitlements b/Relay/Relay.entitlements index 72f5f60..de227ff 100644 --- a/Relay/Relay.entitlements +++ b/Relay/Relay.entitlements @@ -6,6 +6,8 @@ com.apple.security.network.client + com.apple.security.network.server + com.apple.security.device.camera com.apple.security.device.microphone diff --git a/Relay/Utilities/MatrixLink.swift b/Relay/Utilities/MatrixLink.swift new file mode 100644 index 0000000..424c419 --- /dev/null +++ b/Relay/Utilities/MatrixLink.swift @@ -0,0 +1,86 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +/// A parsed Matrix deep link, derived from either a `https://matrix.to` URL or a `matrix:` URI. +/// +/// **matrix.to format:** +/// - User: `https://matrix.to/#/@user:server` +/// - Room: `https://matrix.to/#/#room:server` or `https://matrix.to/#/!roomId:server` +/// +/// **matrix: URI format (MSC2312):** +/// - User: `matrix:u/user:server` +/// - Room: `matrix:r/room:server` or `matrix:roomid/roomId:server` +enum MatrixLink { + /// A Matrix user ID (e.g. `@alice:matrix.org`). + case user(String) + /// A room alias or room ID (e.g. `#general:matrix.org` or `!abc123:matrix.org`). + case room(String) + + /// Parses a URL into a ``MatrixLink``, returning `nil` if the URL is not a recognised Matrix link. + init?(url: URL) { + if url.host?.lowercased() == "matrix.to" { + guard let link = Self(matrixToURL: url) else { return nil } + self = link + } else if url.scheme?.lowercased() == "matrix" { + guard let link = Self(matrixURI: url) else { return nil } + self = link + } else { + return nil + } + } + + // MARK: - Private parsers + + private init?(matrixToURL url: URL) { + // Fragment is everything after `#`, e.g. `/@alice:matrix.org` or `/#general:matrix.org` + guard let fragment = url.fragment, fragment.hasPrefix("/") else { return nil } + // The fragment may contain additional path components (e.g. an event ID after a second `/`). + // Extract only the first component as the entity identifier. + guard let entity = String(fragment.dropFirst()) + .components(separatedBy: "/").first? + .removingPercentEncoding else { return nil } + + if entity.hasPrefix("@") { + self = .user(entity) + } else if entity.hasPrefix("#") || entity.hasPrefix("!") { + self = .room(entity) + } else { + return nil + } + } + + private init?(matrixURI url: URL) { + // matrix: URIs encode the entity type and identifier in the path: + // `u/user:server`, `r/room:server`, `roomid/roomId:server` (without sigils) + let path = url.path + let parts = path.components(separatedBy: "/") + guard parts.count >= 2 else { return nil } + let type = parts[0] + let identifier = parts[1] + guard !identifier.isEmpty else { return nil } + + switch type { + case "u": + self = .user("@\(identifier)") + case "r": + self = .room("#\(identifier)") + case "roomid": + self = .room("!\(identifier)") + default: + return nil + } + } +} diff --git a/Relay/ViewModels/PreviewCallViewModel.swift b/Relay/ViewModels/PreviewCallViewModel.swift index d2d0cc3..8494286 100644 --- a/Relay/ViewModels/PreviewCallViewModel.swift +++ b/Relay/ViewModels/PreviewCallViewModel.swift @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AppKit import Foundation import RelayInterface +import SwiftUI /// A mock ``CallViewModelProtocol`` for SwiftUI previews. /// @@ -72,5 +72,5 @@ final class PreviewCallViewModel: CallViewModelProtocol { isLocalMicrophoneEnabled.toggle() } - func makeVideoView(for participantID: String) -> NSView? { nil } + func makeVideoView(for participantID: String) -> AnyView? { nil } } diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index c922ce3..7f72a59 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AppKit import RelayInterface import SwiftUI @@ -198,6 +197,11 @@ struct CallView: View { } // MARK: - Participants Grid + // + // NOTE: For a richer integration, consider adopting LiveKitComponents + // (ForEachParticipant / ForEachTrack / VideoTrackView) which handle + // participant lifecycle, adaptive streaming, and reconnection automatically. + // See: https://github.com/livekit/components-swift @ViewBuilder private var participantsGrid: some View { @@ -206,9 +210,11 @@ struct CallView: View { LazyVGrid(columns: columns, spacing: 8) { ForEach(viewModel.participants) { participant in participantTile(participant) + .id(participant.id) } if let localID = viewModel.localParticipantID { localParticipantTile(id: localID) + .id(localID) } } .padding(8) @@ -220,12 +226,12 @@ struct CallView: View { @ViewBuilder private func participantTile(_ participant: CallParticipant) -> some View { ZStack(alignment: .bottom) { - VideoViewRepresentable(viewModel: viewModel, participantID: participant.id, trackRevision: viewModel.videoTrackRevision) - .aspectRatio(16 / 9, contentMode: .fit) - .clipShape(RoundedRectangle(cornerRadius: 10)) + Color(nsColor: .darkGray) + + videoContent(for: participant.id) if participant.isSpeaking { - RoundedRectangle(cornerRadius: 10) + Rectangle() .strokeBorder(.green, lineWidth: 3) } @@ -246,11 +252,8 @@ struct CallView: View { .padding(.horizontal, 8) .padding(.vertical, 6) .background(.black.opacity(0.55)) - .clipShape(.rect( - topLeadingRadius: 0, bottomLeadingRadius: 10, - bottomTrailingRadius: 10, topTrailingRadius: 0 - )) } + .aspectRatio(16.0 / 9.0, contentMode: .fit) } // MARK: - Local Participant Tile @@ -258,9 +261,9 @@ struct CallView: View { @ViewBuilder private func localParticipantTile(id: String) -> some View { ZStack(alignment: .bottom) { - VideoViewRepresentable(viewModel: viewModel, participantID: id, trackRevision: viewModel.videoTrackRevision) - .aspectRatio(16 / 9, contentMode: .fit) - .clipShape(RoundedRectangle(cornerRadius: 10)) + Color(nsColor: .darkGray) + + videoContent(for: id) HStack(spacing: 6) { Text("You").font(.caption).foregroundStyle(.white) @@ -275,11 +278,8 @@ struct CallView: View { .padding(.horizontal, 8) .padding(.vertical, 6) .background(.black.opacity(0.55)) - .clipShape(.rect( - topLeadingRadius: 0, bottomLeadingRadius: 10, - bottomTrailingRadius: 10, topTrailingRadius: 0 - )) } + .aspectRatio(16.0 / 9.0, contentMode: .fit) } // MARK: - Control Bar @@ -335,51 +335,26 @@ struct CallView: View { } } -// MARK: - NSView Bridge for Video - -/// Embeds the opaque `NSView` returned by ``CallViewModelProtocol/makeVideoView(for:)``. -/// -/// A stable container `NSView` is created in `makeNSView`. On each SwiftUI -/// re-render (driven by `viewModel` state changes) `updateNSView` asks the -/// view model for the current video view and swaps it into the container. -/// This handles the common timing issue where the video track isn't published -/// yet on the first render but becomes available shortly after. -private struct VideoViewRepresentable: NSViewRepresentable { - let viewModel: any CallViewModelProtocol - let participantID: String - /// Reading this value in the parent SwiftUI view ensures that when the view model - /// bumps it (e.g. after a track is published), SwiftUI re-evaluates this representable - /// and calls ``updateNSView``. - let trackRevision: UInt - - func makeNSView(context: Context) -> NSView { - let container = NSView() - container.wantsLayer = true - container.layer?.backgroundColor = NSColor.darkGray.cgColor - container.layer?.cornerRadius = 10 - attachVideoView(to: container) - return container - } +// MARK: - Video Content - func updateNSView(_ container: NSView, context: Context) { - attachVideoView(to: container) - } - - private func attachVideoView(to container: NSView) { - guard let videoView = viewModel.makeVideoView(for: participantID) else { return } - - // Already the current subview — nothing to do. - if container.subviews.first === videoView { return } - - container.subviews.forEach { $0.removeFromSuperview() } - videoView.translatesAutoresizingMaskIntoConstraints = false - container.addSubview(videoView) - NSLayoutConstraint.activate([ - videoView.leadingAnchor.constraint(equalTo: container.leadingAnchor), - videoView.trailingAnchor.constraint(equalTo: container.trailingAnchor), - videoView.topAnchor.constraint(equalTo: container.topAnchor), - videoView.bottomAnchor.constraint(equalTo: container.bottomAnchor), - ]) +extension CallView { + /// Returns the LiveKit `SwiftUIVideoView` (via `AnyView`) for the given participant, + /// or a dark grey placeholder if no video track is available yet. + /// + /// > Important: Do **not** apply `.clipShape()` or `.mask()` to the returned + /// > video view. `SwiftUIVideoView` renders via Metal and SwiftUI shape + /// > clipping interferes with the GPU-backed surface, causing visual artefacts. + @ViewBuilder + fileprivate func videoContent(for participantID: String) -> some View { + // Reading videoTrackRevision ensures SwiftUI re-evaluates this + // when tracks change (publish, subscribe, toggle). + let _ = viewModel.videoTrackRevision + if let videoView = viewModel.makeVideoView(for: participantID) { + videoView + } else { + RoundedRectangle(cornerRadius: 10) + .fill(Color(nsColor: .darkGray)) + } } } diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index b4b7af4..751067a 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -import AppKit import Foundation import LiveKit import RelayInterface import OSLog +import SwiftUI -private let logger = Logger(subsystem: "RelayKit", category: "CallViewModel") +private let logger = Logger(subsystem: "RelayKit", category: "Call") /// A concrete ``CallViewModelProtocol`` implementation backed by the LiveKit Swift SDK. /// @@ -38,16 +38,17 @@ public final class CallViewModel: CallViewModelProtocol { public private(set) var isLocalMicrophoneEnabled: Bool = false public private(set) var localParticipantID: String? /// Incremented whenever video tracks change, triggering SwiftUI to - /// re-call `updateNSView` on any `VideoViewRepresentable`. + /// re-evaluate `videoContent(for:)` and pick up new or removed tracks. public private(set) var videoTrackRevision: UInt = 0 private let room = LiveKit.Room() private var delegate: Delegate? - /// Cached `VideoView` instances keyed by participant identity. - /// Re-used across `makeVideoView(for:)` calls so the view stays stable - /// even as SwiftUI re-renders, and the `.track` is updated in place - /// when tracks change. - private var videoViews: [String: VideoView] = [:] + + /// Cached video views keyed by participant ID, to avoid recreating + /// `SwiftUIVideoView` on every SwiftUI re-render. Each entry stores + /// the `ObjectIdentifier` of the `VideoTrack` so the cache is + /// invalidated when the underlying track actually changes. + private var videoViewCache: [String: (trackObjectID: ObjectIdentifier, view: AnyView)] = [:] public init() { let delegate = Delegate(viewModel: self) @@ -60,47 +61,34 @@ public final class CallViewModel: CallViewModelProtocol { public func connect(url: String, token: String) async throws { state = .connecting do { - try await room.connect(url: url, token: token) - let identity = room.localParticipant.identity?.stringValue - localParticipantID = identity - logger.info("[CallVM] Connected. localParticipantID=\(identity ?? "nil")") + let connectOpts = ConnectOptions( + autoSubscribe: true, + enableMicrophone: true + ) + let roomOpts = RoomOptions( + defaultVideoPublishOptions: VideoPublishOptions( + preferredCodec: .vp8 + ), + adaptiveStream: true, + dynacast: true + ) + try await room.connect( + url: url, + token: token, + connectOptions: connectOpts, + roomOptions: roomOpts + ) + localParticipantID = room.localParticipant.identity?.stringValue + logger.info("Connected as \(self.localParticipantID ?? "unknown")") try await room.localParticipant.setCamera(enabled: true) - logger.info("[CallVM] setCamera(enabled: true) returned") - - // Log local video tracks immediately after setCamera - let localPubs = room.localParticipant.videoTracks - logger.info("[CallVM] localParticipant.videoTracks.count=\(localPubs.count)") - for pub in localPubs { - let hasTrack = pub.track != nil - let trackType = pub.track.map { String(describing: type(of: $0)) } ?? "nil" - logger.info("[CallVM] pub sid=\(pub.sid.stringValue), hasTrack=\(hasTrack), trackType=\(trackType), source=\(String(describing: pub.source))") - } - - try await room.localParticipant.setMicrophone(enabled: true) - logger.info("[CallVM] setMicrophone(enabled: true) returned") isLocalCameraEnabled = true isLocalMicrophoneEnabled = true state = .connected videoTrackRevision += 1 - - // Delayed re-check: log track state after 1 second - Task { @MainActor [weak self] in - try? await Task.sleep(for: .seconds(1)) - guard let self else { return } - let pubs = self.room.localParticipant.videoTracks - logger.info("[CallVM] Delayed check: videoTracks.count=\(pubs.count)") - for pub in pubs { - let hasTrack = pub.track != nil - let isVideoTrack = pub.track is VideoTrack - logger.info("[CallVM] Delayed: pub sid=\(pub.sid.stringValue), hasTrack=\(hasTrack), isVideoTrack=\(isVideoTrack)") - } - self.videoTrackRevision += 1 - logger.info("[CallVM] Delayed videoTrackRevision bump to \(self.videoTrackRevision)") - } } catch { - logger.error("[CallVM] Connect failed: \(error.localizedDescription)") + logger.error("Connect failed: \(error.localizedDescription)") state = .failed(error.localizedDescription) throw error } @@ -113,21 +101,17 @@ public final class CallViewModel: CallViewModelProtocol { isLocalCameraEnabled = false isLocalMicrophoneEnabled = false localParticipantID = nil - videoViews.removeAll() + videoViewCache.removeAll() } public func toggleCamera() async throws { let enabled = !isLocalCameraEnabled try await room.localParticipant.setCamera(enabled: enabled) isLocalCameraEnabled = enabled - videoTrackRevision += 1 - // Same delayed retry as connect — the track may not be attached yet. - if enabled { - Task { @MainActor [weak self] in - try? await Task.sleep(for: .milliseconds(500)) - self?.videoTrackRevision += 1 - } + if let localID = localParticipantID { + videoViewCache.removeValue(forKey: localID) } + videoTrackRevision += 1 } public func toggleMicrophone() async throws { @@ -136,50 +120,52 @@ public final class CallViewModel: CallViewModelProtocol { isLocalMicrophoneEnabled = enabled } - public func makeVideoView(for participantID: String) -> NSView? { - let participant: Participant? + public func makeVideoView(for participantID: String) -> AnyView? { let isLocal = room.localParticipant.identity?.stringValue == participantID - if isLocal { - participant = room.localParticipant - } else { - participant = room.remoteParticipants.values.first(where: { - $0.identity?.stringValue == participantID - }) + let participant: Participant? = isLocal + ? room.localParticipant + : room.remoteParticipants.values.first { $0.identity?.stringValue == participantID } + + guard let publication = participant?.videoTracks.first, + !publication.isMuted, + let track = publication.track as? VideoTrack + else { + videoViewCache.removeValue(forKey: participantID) + return nil } - let foundParticipant = participant != nil - let allPubCount = participant?.trackPublications.count ?? 0 - let videoPubCount = participant?.videoTracks.count ?? 0 - logger.info("[CallVM] makeVideoView(for: \(participantID)) isLocal=\(isLocal), found=\(foundParticipant), allPubs=\(allPubCount), videoPubs=\(videoPubCount)") - - // Look up the current video track (may be nil if not yet published). - let firstPub = participant?.videoTracks.first - let rawTrack = firstPub?.track - let track = rawTrack as? VideoTrack - - logger.info("[CallVM] firstPub=\(firstPub != nil), rawTrack=\(rawTrack != nil), rawTrackType=\(rawTrack.map { String(describing: type(of: $0)) } ?? "nil"), castOK=\(track != nil)") - - // Return or create a cached VideoView. Its `.track` is updated in - // place every call so the rendered content stays current even when - // the underlying track changes (e.g. camera toggled on/off). - if let existing = videoViews[participantID] { - existing.track = track - logger.info("[CallVM] Reusing cached VideoView, track set to \(track != nil ? "non-nil" : "nil")") - return existing + // For remote tracks, verify the track is actually subscribed. + if let remotePub = publication as? RemoteTrackPublication, !remotePub.isSubscribed { + videoViewCache.removeValue(forKey: participantID) + return nil } - let videoView = VideoView() - videoView.track = track - videoViews[participantID] = videoView - logger.info("[CallVM] Created new VideoView, track set to \(track != nil ? "non-nil" : "nil")") - return videoView + // Return the cached view if the underlying VideoTrack is unchanged, + // preventing SwiftUI from tearing down and recreating the Metal renderer. + let trackID = ObjectIdentifier(track) + if let cached = videoViewCache[participantID], cached.trackObjectID == trackID { + return cached.view + } + + let view = AnyView( + SwiftUIVideoView(track, + layoutMode: .fill, + mirrorMode: isLocal ? .mirror : .off) + ) + videoViewCache[participantID] = (trackObjectID: trackID, view: view) + return view } // MARK: - Participant Sync - fileprivate func syncParticipants() { - videoTrackRevision += 1 - participants = room.remoteParticipants.values.map { participant in + /// Re-syncs the ``participants`` array from the room's remote participants. + /// - Parameter trackChanged: When `true`, also bumps ``videoTrackRevision`` + /// to trigger video view updates. Pass `false` for cosmetic-only changes + /// (e.g. speaking indicators) to avoid disrupting the video renderer. + fileprivate func syncParticipants(trackChanged: Bool = false) { + if trackChanged { videoTrackRevision += 1 } + + let newParticipants = room.remoteParticipants.values.map { participant in CallParticipant( id: participant.identity?.stringValue ?? participant.sid?.stringValue ?? UUID().uuidString, displayName: participant.name, @@ -188,6 +174,16 @@ public final class CallViewModel: CallViewModelProtocol { isSpeaking: participant.isSpeaking ) } + + // Prune video view cache for participants who have left. + if trackChanged { + let activeIDs = Set(newParticipants.map(\.id)) + for key in videoViewCache.keys where key != localParticipantID && !activeIDs.contains(key) { + videoViewCache.removeValue(forKey: key) + } + } + + participants = newParticipants } // MARK: - Delegate Bridge @@ -216,7 +212,7 @@ public final class CallViewModel: CallViewModelProtocol { viewModel.state = .disconnected } case .reconnecting: - logger.info("Call reconnecting") + logger.info("Reconnecting…") default: break } @@ -225,38 +221,39 @@ public final class CallViewModel: CallViewModelProtocol { func room(_ room: LiveKit.Room, participantDidConnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants() + viewModel?.syncParticipants(trackChanged: true) } } func room(_ room: LiveKit.Room, participantDidDisconnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants() + viewModel?.syncParticipants(trackChanged: true) } } func room(_ room: LiveKit.Room, didUpdateSpeakingParticipants participants: [Participant]) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants() + // Speaking state is cosmetic — don't bump videoTrackRevision + // to avoid disrupting the video renderer. + viewModel?.syncParticipants(trackChanged: false) } } func room(_ room: LiveKit.Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants() + viewModel?.syncParticipants(trackChanged: true) } } func room(_ room: LiveKit.Room, localParticipant: LocalParticipant, didPublishTrack publication: LocalTrackPublication) { Task { @MainActor [weak viewModel] in - logger.info("Local track published: \(String(describing: publication.kind))") viewModel?.videoTrackRevision += 1 } } func room(_ room: LiveKit.Room, participant: RemoteParticipant, didPublishTrack publication: RemoteTrackPublication) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants() + viewModel?.syncParticipants(trackChanged: true) } } } From 1ec91c7c7a7cbd52edb86b465cac872d49300d68 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Mon, 6 Apr 2026 14:16:24 -0400 Subject: [PATCH 08/24] Add E2EE and MatrixRTC call signaling for Element Call interop Enable AES-128-GCM frame encryption on LiveKit calls using per-participant keys, and implement MatrixRTC call membership signaling so Element-X and other MatrixRTC clients can discover and join calls. - Add CallEncryptionService with key generation (16-byte random), dual-transport key distribution (to-device + room state events), timeline-based inbound key listener, and MatrixRTC call.member state event signaling (MSC3401) - Configure BaseKeyProvider with per-participant keys and GCM encryption on RoomOptions, using ObjC runtime to set raw key bytes - Send org.matrix.msc3401.call.member state event on connect so Element-X sees the call, remove on disconnect - Redistribute encryption keys to newly joined participants - Pass Matrix SDK Room and credentials into CallViewModel via EncryptionContext for key exchange and timeline listening Co-Authored-By: Claude Opus 4.6 --- RelayKit/Call/CallEncryptionService.swift | 497 ++++++++++++++++++++++ RelayKit/Call/CallViewModel.swift | 174 +++++++- RelayKit/Services/MatrixService.swift | 18 +- 3 files changed, 686 insertions(+), 3 deletions(-) create mode 100644 RelayKit/Call/CallEncryptionService.swift diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift new file mode 100644 index 0000000..0391abc --- /dev/null +++ b/RelayKit/Call/CallEncryptionService.swift @@ -0,0 +1,497 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import LiveKit +import OSLog + +private let logger = Logger(subsystem: "RelayKit", category: "CallEncryption") + +/// Manages MatrixRTC E2EE key exchange for LiveKit calls. +/// +/// Implements the key distribution side of MSC4143 / Element Call's encryption +/// protocol: generates a random 16-byte AES-GCM key for the local participant, +/// distributes it to other participants via Matrix to-device messages, and sets +/// raw key material on the LiveKit `BaseKeyProvider` so that SFrame encryption +/// uses the correct bytes. +/// +/// ## Key Exchange Flow +/// 1. On connect, generate a 16-byte random key. +/// 2. Set the key on the local participant's LiveKit encryptor via `BaseKeyProvider`. +/// 3. Send the key (base64-encoded) to all other devices in the room using the +/// `io.element.call.encryption_keys` to-device event type. +/// 4. When receiving keys from other participants (via `/sync`), set them on the +/// `BaseKeyProvider` for the corresponding participant identity. +struct CallEncryptionService { + + let homeserver: String + let accessToken: String + let userID: String + let deviceID: String + let roomID: String + + /// The to-device event type used by Element Call for key exchange. + static let encryptionKeysEventType = "io.element.call.encryption_keys" + + /// The state event type for MatrixRTC call membership (MSC3401). + /// Element-X uses this to discover active calls in a room. + static let callMemberEventType = "org.matrix.msc3401.call.member" + + // MARK: - Call Membership Signaling + + /// Sends the MatrixRTC call membership state event so that Element-X and other + /// MatrixRTC clients can discover our participation in the call. + /// + /// - Parameter livekitURL: The LiveKit SFU service URL (used in `foci_active`). + func sendCallMemberEvent(livekitURL: String) async throws { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + let encodedEventType = Self.callMemberEventType + .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType + // State key is the user's Matrix ID — one membership entry per user. + let encodedStateKey = userID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? userID + + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { + throw LiveKitCredentialError.invalidURL + } + + // Strip trailing slash from LiveKit URL for the foci_active entry. + let sfuURL = livekitURL.trimmingCharacters(in: .init(charactersIn: "/")) + + let body: [String: Any] = [ + "memberships": [ + [ + "application": "m.call", + "call_id": "", + "device_id": deviceID, + "expires": 3600000, + "foci_active": [ + [ + "type": "livekit", + "livekit_service_url": sfuURL + ] + ], + "membershipID": UUID().uuidString, + "scope": "m.room" + ] as [String: Any] + ] + ] + + let jsonData = try JSONSerialization.data(withJSONObject: body) + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = jsonData + + let (_, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + logger.error("sendCallMemberEvent failed with status \(statusCode)") + throw CallEncryptionError.callMemberEventFailed + } + + logger.info("Sent call membership state event") + } + + /// Removes the call membership state event (sets memberships to empty) + /// so Element-X knows we've left the call. + func removeCallMemberEvent() async throws { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + let encodedEventType = Self.callMemberEventType + .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType + let encodedStateKey = userID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? userID + + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { + throw LiveKitCredentialError.invalidURL + } + + let body: [String: Any] = ["memberships": [Any]()] + + let jsonData = try JSONSerialization.data(withJSONObject: body) + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = jsonData + + let (_, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + logger.error("removeCallMemberEvent failed with status \(statusCode)") + return + } + + logger.info("Removed call membership state event") + } + + // MARK: - Key Generation + + /// Generates a cryptographically random 16-byte key suitable for AES-128-GCM. + static func generateKey() -> Data { + var bytes = [UInt8](repeating: 0, count: 16) + let status = SecRandomCopyBytes(kSecRandomDefault, bytes.count, &bytes) + precondition(status == errSecSuccess, "Failed to generate random key bytes") + return Data(bytes) + } + + // MARK: - Key Provider Setup + + /// Sets a raw key on a `BaseKeyProvider` for the given participant, bypassing + /// the String-based `setKey(key:participantId:index:)` method which would + /// UTF-8-encode the string (wrong for raw AES key bytes). + /// + /// `BaseKeyProvider` is decorated with `@objcMembers`, so its internal + /// `rtcKeyProvider` (an `LKRTCFrameCryptorKeyProvider`) is accessible via KVC. + /// The ObjC provider accepts `NSData` directly. + static func setRawKey( + _ keyData: Data, + on keyProvider: BaseKeyProvider, + participantId: String, + index: Int32 = 0 + ) { + guard let rtcProvider = keyProvider.value(forKey: "rtcKeyProvider") as AnyObject? else { + logger.error("Could not access rtcKeyProvider via KVC") + return + } + + // LKRTCFrameCryptorKeyProvider is an ObjC class with: + // - (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId + // NSObject.perform(_:with:with:) only supports 2 arguments, so we use + // objc_msgSend to call the 3-argument method directly. + typealias SetKeyFunc = @convention(c) (AnyObject, Selector, NSData, Int32, NSString) -> Void + let selector = NSSelectorFromString("setKey:withIndex:forParticipant:") + guard (rtcProvider as? NSObject)?.responds(to: selector) == true else { + logger.error("rtcKeyProvider does not respond to setKey:withIndex:forParticipant:") + return + } + + let imp = unsafeBitCast( + (rtcProvider as AnyObject).method(for: selector), + to: SetKeyFunc.self + ) + imp(rtcProvider, selector, keyData as NSData, index, participantId as NSString) + logger.info("Set raw encryption key for participant \(participantId, privacy: .private) at index \(index)") + } + + /// Convenience: sets a raw key using base64-encoded key data. + static func setRawKey( + base64Key: String, + on keyProvider: BaseKeyProvider, + participantId: String, + index: Int32 = 0 + ) { + guard let keyData = Data(base64Encoded: base64Key) else { + logger.error("Invalid base64 key for participant \(participantId, privacy: .private)") + return + } + setRawKey(keyData, on: keyProvider, participantId: participantId, index: index) + } + + // MARK: - Key Distribution (to-device messages) + + /// Sends the local participant's encryption key to all devices of the given + /// users via a Matrix to-device message. + /// + /// Uses the REST API directly because the Matrix Rust SDK (v26.x) does not + /// expose `sendToDevice` in the Swift FFI. + /// + /// - Parameters: + /// - key: The raw 16-byte encryption key. + /// - keyIndex: The key index (0-255, cycles on ratchet). + /// - targetUsers: A mapping of user ID to an array of device IDs. + func sendKey( + _ key: Data, + keyIndex: Int, + to targetUsers: [String: [String]] + ) async throws { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let txnId = UUID().uuidString + let encodedEventType = Self.encryptionKeysEventType + .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.encryptionKeysEventType + + guard let url = URL(string: "\(base)/_matrix/client/v3/sendToDevice/\(encodedEventType)/\(txnId)") else { + throw LiveKitCredentialError.invalidURL + } + + let base64Key = key.base64EncodedString() + let sentTs = Int(Date().timeIntervalSince1970 * 1000) + + // Build the per-user/per-device message content. + var messages: [String: [String: Any]] = [:] + for (userId, deviceIds) in targetUsers { + var deviceMessages: [String: Any] = [:] + for deviceId in deviceIds { + deviceMessages[deviceId] = [ + "keys": [ + ["index": keyIndex, "key": base64Key] + ], + "room_id": roomID, + "member": [ + "claimed_device_id": self.deviceID, + "id": "\(self.userID):\(self.deviceID)" + ], + "session": [ + "call_id": "", + "application": "m.call", + "scope": "m.room" + ], + "sent_ts": sentTs + ] as [String: Any] + } + messages[userId] = deviceMessages + } + + let body: [String: Any] = ["messages": messages] + let jsonData = try JSONSerialization.data(withJSONObject: body) + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = jsonData + + let (_, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + logger.error("sendToDevice failed with status \(statusCode)") + throw CallEncryptionError.keyDistributionFailed + } + + logger.info("Sent encryption key (index \(keyIndex)) to \(targetUsers.count) user(s)") + } + + // MARK: - Key Distribution (room state events) + + /// Sends the local participant's encryption key as a room state event. + /// + /// This provides a second transport for key exchange that other Relay clients + /// can observe via the room timeline, working around the Matrix Rust SDK's + /// inability to deliver to-device events to the app layer. + /// + /// The state key is `"{userID}:{deviceID}"` so each participant's key is + /// a distinct state entry that overwrites on update. + func sendKeyAsStateEvent( + _ key: Data, + keyIndex: Int + ) async throws { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + let stateKey = "\(userID):\(deviceID)" + let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey + let encodedEventType = Self.encryptionKeysEventType + .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.encryptionKeysEventType + + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { + throw LiveKitCredentialError.invalidURL + } + + let base64Key = key.base64EncodedString() + let body: [String: Any] = [ + "keys": [ + ["index": keyIndex, "key": base64Key] + ], + "member": [ + "claimed_device_id": deviceID, + "id": "\(userID):\(deviceID)" + ], + "session": [ + "call_id": "", + "application": "m.call", + "scope": "m.room" + ] + ] + + let jsonData = try JSONSerialization.data(withJSONObject: body) + + var request = URLRequest(url: url) + request.httpMethod = "PUT" + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + request.setValue("application/json", forHTTPHeaderField: "Content-Type") + request.httpBody = jsonData + + let (_, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { + let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 + logger.error("sendStateEvent failed with status \(statusCode)") + throw CallEncryptionError.keyDistributionFailed + } + + logger.info("Sent encryption key as state event (index \(keyIndex))") + } + + // MARK: - Key Reception (timeline listener) + + /// Starts listening for encryption key state events on the room timeline. + /// + /// When another participant sends their key as a room state event of type + /// `io.element.call.encryption_keys`, this listener parses the key and sets + /// it on the given `BaseKeyProvider` so LiveKit can decrypt that participant's + /// media frames. + /// + /// - Parameters: + /// - timeline: The Matrix SDK `Timeline` for the call's room. + /// - keyProvider: The LiveKit key provider to set received keys on. + /// - localIdentity: The local participant's identity (to skip our own events). + /// - Returns: A `TaskHandle` that must be retained to keep the listener alive. + @MainActor + static func startListeningForKeys( + timeline: Timeline, + keyProvider: BaseKeyProvider, + localIdentity: String + ) async -> TaskHandle { + // Capture the event type as a local to avoid referencing the MainActor-isolated + // static property from the nonisolated SDKListener callback. + let eventType = Self.encryptionKeysEventType + let localPrefix = localIdentity.components(separatedBy: ":").prefix(2).joined(separator: ":") + + let listener = SDKListener<[TimelineDiff]> { diffs in + // SDKListener callbacks arrive on an unspecified thread. + // Dispatch to the main actor for safe access to logger and setRawKey. + Task { @MainActor in + let items = extractTimelineItems(from: diffs) + for item in items { + guard let eventItem = item.asEvent() else { continue } + + guard case .state(let stateKey, let otherState) = eventItem.content, + case .custom(let type) = otherState, + type == eventType else { + continue + } + + // Skip our own events. + if stateKey.hasPrefix(localPrefix) { continue } + + let sender = eventItem.sender + + let debugInfo = eventItem.lazyProvider.debugInfo() + guard let jsonString = debugInfo.originalJson, + let jsonData = jsonString.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { + logger.warning("Could not parse encryption key event JSON from \(sender, privacy: .private)") + continue + } + + // The raw JSON is the full event envelope; keys are in "content". + let content = (json["content"] as? [String: Any]) ?? json + + guard let keysArray = content["keys"] as? [[String: Any]] else { + logger.warning("No keys array in encryption key event from \(sender, privacy: .private)") + continue + } + + let participantIdentity = stateKey + + for keyEntry in keysArray { + guard let base64Key = keyEntry["key"] as? String, + let index = keyEntry["index"] as? Int else { + continue + } + Self.setRawKey( + base64Key: base64Key, + on: keyProvider, + participantId: participantIdentity, + index: Int32(index) + ) + logger.info("Received encryption key from \(sender, privacy: .private) (index \(index))") + } + } + } + } + + return await timeline.addListener(listener: listener) + } + + // MARK: - Room Member Discovery + + /// Fetches the list of joined members in the room so we know who to send + /// encryption keys to. Uses the Matrix REST API directly. + func fetchJoinedMembers() async throws -> [String: [String]] { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/joined_members") else { + throw LiveKitCredentialError.invalidURL + } + + var request = URLRequest(url: url) + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + + let (data, response) = try await URLSession.shared.data(for: request) + guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { + throw CallEncryptionError.memberDiscoveryFailed + } + + // Response: { "joined": { "@user:server": { ... }, ... } } + guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any], + let joined = json["joined"] as? [String: Any] else { + throw CallEncryptionError.memberDiscoveryFailed + } + + // For now, return each user mapped to "*" (wildcard) since we don't have + // per-device granularity from joined_members. The homeserver will fan out. + var result: [String: [String]] = [:] + for userId in joined.keys where userId != self.userID { + result[userId] = ["*"] + } + return result + } +} + +// MARK: - Helpers + +/// Extracts all `TimelineItem` values from a batch of timeline diffs. +private func extractTimelineItems(from diffs: [TimelineDiff]) -> [TimelineItem] { + var items: [TimelineItem] = [] + for diff in diffs { + switch diff { + case .append(let values): + items.append(contentsOf: values) + case .pushFront(let value): + items.append(value) + case .pushBack(let value): + items.append(value) + case .insert(_, let value): + items.append(value) + case .set(_, let value): + items.append(value) + case .reset(let values): + items.append(contentsOf: values) + case .clear, .popFront, .popBack, .remove, .truncate: + break + } + } + return items +} + +// MARK: - Errors + +enum CallEncryptionError: LocalizedError { + case keyDistributionFailed + case memberDiscoveryFailed + case callMemberEventFailed + + var errorDescription: String? { + switch self { + case .keyDistributionFailed: + return "Failed to distribute encryption keys to call participants." + case .memberDiscoveryFailed: + return "Failed to discover room members for key exchange." + case .callMemberEventFailed: + return "Failed to send call membership state event." + } + } +} diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 751067a..3234d56 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -50,12 +50,71 @@ public final class CallViewModel: CallViewModelProtocol { /// invalidated when the underlying track actually changes. private var videoViewCache: [String: (trackObjectID: ObjectIdentifier, view: AnyView)] = [:] + // MARK: - E2EE State + + /// The LiveKit key provider used for per-participant AES-GCM frame encryption. + private var keyProvider: BaseKeyProvider? + /// The local participant's current encryption key (raw 16 bytes). + private var localEncryptionKey: Data? + /// The current key index (0-255, wraps around on ratchet). + private var localKeyIndex: Int = 0 + /// Service for distributing encryption keys via Matrix to-device messages. + private var encryptionService: CallEncryptionService? + /// The Matrix SDK room, used to obtain the timeline for key listening. + private var matrixRoom: MatrixRustSDK.Room? + /// Handle for the timeline key listener; retained to keep the subscription alive. + private var keyListenerHandle: TaskHandle? + + /// Creates a call view model without E2EE. Use ``init(encryptionContext:)`` + /// for encrypted calls that interoperate with Element Call. public init() { let delegate = Delegate(viewModel: self) self.delegate = delegate room.add(delegate: delegate) } + /// Encryption context passed from ``MatrixService`` to enable E2EE key exchange. + public struct EncryptionContext: @unchecked Sendable { + public let homeserver: String + public let accessToken: String + public let userID: String + public let deviceID: String + public let roomID: String + /// The Matrix SDK room, used to obtain the timeline for listening to + /// inbound encryption key state events. `nil` if unavailable. + public let matrixRoom: MatrixRustSDK.Room? + + public init(homeserver: String, accessToken: String, userID: String, deviceID: String, roomID: String, matrixRoom: MatrixRustSDK.Room? = nil) { + self.homeserver = homeserver + self.accessToken = accessToken + self.userID = userID + self.deviceID = deviceID + self.roomID = roomID + self.matrixRoom = matrixRoom + } + } + + /// Creates a call view model with E2EE enabled, using AES-128-GCM frame + /// encryption compatible with Element Call's MatrixRTC key exchange. + public init(encryptionContext: EncryptionContext) { + let delegate = Delegate(viewModel: self) + self.delegate = delegate + room.add(delegate: delegate) + + self.encryptionService = CallEncryptionService( + homeserver: encryptionContext.homeserver, + accessToken: encryptionContext.accessToken, + userID: encryptionContext.userID, + deviceID: encryptionContext.deviceID, + roomID: encryptionContext.roomID + ) + + // Per-participant key provider: each participant has their own key. + let provider = BaseKeyProvider(isSharedKey: false) + self.keyProvider = provider + self.matrixRoom = encryptionContext.matrixRoom + } + // MARK: - CallViewModelProtocol public func connect(url: String, token: String) async throws { @@ -65,12 +124,18 @@ public final class CallViewModel: CallViewModelProtocol { autoSubscribe: true, enableMicrophone: true ) + + // Build RoomOptions — with E2EE if a key provider was configured. + let encryptionOpts: EncryptionOptions? = keyProvider.map { + EncryptionOptions(keyProvider: $0, encryptionType: .gcm) + } let roomOpts = RoomOptions( defaultVideoPublishOptions: VideoPublishOptions( preferredCodec: .vp8 ), adaptiveStream: true, - dynacast: true + dynacast: true, + encryptionOptions: encryptionOpts ) try await room.connect( url: url, @@ -81,6 +146,64 @@ public final class CallViewModel: CallViewModelProtocol { localParticipantID = room.localParticipant.identity?.stringValue logger.info("Connected as \(self.localParticipantID ?? "unknown")") + // Send MatrixRTC call membership state event so Element-X and other + // MatrixRTC clients can discover our participation in this call. + if let encryptionService { + Task { + do { + try await encryptionService.sendCallMemberEvent(livekitURL: url) + } catch { + logger.warning("Call membership event failed: \(error.localizedDescription)") + } + } + } + + // Generate and distribute the local E2EE key before publishing tracks, + // so that the first frames are already encrypted. + if let keyProvider, let encryptionService { + let key = CallEncryptionService.generateKey() + localEncryptionKey = key + + let localIdentity = localParticipantID ?? encryptionService.userID + CallEncryptionService.setRawKey( + key, + on: keyProvider, + participantId: localIdentity, + index: Int32(localKeyIndex) + ) + logger.info("Local E2EE key set (index \(self.localKeyIndex))") + + // Distribute key via both transports (best-effort, don't block connect). + Task { + do { + // 1. To-device messages (Element Call interop) + let members = try await encryptionService.fetchJoinedMembers() + if !members.isEmpty { + try await encryptionService.sendKey(key, keyIndex: localKeyIndex, to: members) + } + } catch { + logger.warning("To-device key distribution failed: \(error.localizedDescription)") + } + + do { + // 2. Room state event (Relay-to-Relay interop) + try await encryptionService.sendKeyAsStateEvent(key, keyIndex: localKeyIndex) + } catch { + logger.warning("State event key distribution failed: \(error.localizedDescription)") + } + } + + // Start listening for inbound encryption keys from other participants. + if let timeline = try? await matrixRoom?.timeline() { + keyListenerHandle = await CallEncryptionService.startListeningForKeys( + timeline: timeline, + keyProvider: keyProvider, + localIdentity: localIdentity + ) + logger.info("Started listening for inbound encryption keys") + } + } + try await room.localParticipant.setCamera(enabled: true) isLocalCameraEnabled = true @@ -95,6 +218,11 @@ public final class CallViewModel: CallViewModelProtocol { } public func disconnect() async { + // Remove call membership state event so other clients know we've left. + if let encryptionService { + try? await encryptionService.removeCallMemberEvent() + } + await room.disconnect() state = .disconnected participants = [] @@ -102,6 +230,9 @@ public final class CallViewModel: CallViewModelProtocol { isLocalMicrophoneEnabled = false localParticipantID = nil videoViewCache.removeAll() + localEncryptionKey = nil + localKeyIndex = 0 + keyListenerHandle = nil } public func toggleCamera() async throws { @@ -156,6 +287,41 @@ public final class CallViewModel: CallViewModelProtocol { return view } + // MARK: - E2EE Key Redistribution + + /// Re-sends the local encryption key to a newly joined participant so they + /// can decrypt our media. + fileprivate func redistributeKey(to participantIdentity: String) { + guard let key = localEncryptionKey, let encryptionService else { return } + + // Parse "user:device" from the LiveKit identity (format: @userId:server:deviceId) + // Element Call uses identities like "@user:server:DEVICEID". + let components = participantIdentity.components(separatedBy: ":") + guard components.count >= 3 else { + logger.warning("Cannot parse participant identity for key redistribution: \(participantIdentity, privacy: .private)") + return + } + // Reconstruct userId as first two components, deviceId as remaining. + let userId = components[0] + ":" + components[1] + let deviceId = components.dropFirst(2).joined(separator: ":") + + Task { + do { + try await encryptionService.sendKey(key, keyIndex: localKeyIndex, to: [userId: [deviceId]]) + logger.info("Redistributed key (to-device) to \(participantIdentity, privacy: .private)") + } catch { + logger.warning("Key redistribution (to-device) failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") + } + + // State event is idempotent (overwrites previous), so re-sending is cheap. + do { + try await encryptionService.sendKeyAsStateEvent(key, keyIndex: localKeyIndex) + } catch { + logger.warning("Key redistribution (state event) failed: \(error.localizedDescription)") + } + } + } + // MARK: - Participant Sync /// Re-syncs the ``participants`` array from the room's remote participants. @@ -221,7 +387,11 @@ public final class CallViewModel: CallViewModelProtocol { func room(_ room: LiveKit.Room, participantDidConnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants(trackChanged: true) + guard let viewModel else { return } + viewModel.syncParticipants(trackChanged: true) + if let identity = participant.identity?.stringValue { + viewModel.redistributeKey(to: identity) + } } } diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 06f13f4..2d6e91c 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1441,7 +1441,23 @@ public final class MatrixService: MatrixServiceProtocol { } public func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { - CallViewModel() + guard let client else { return nil } + do { + let session = try client.session() + let sdkRoom = room(id: roomId) + let context = CallViewModel.EncryptionContext( + homeserver: client.homeserver, + accessToken: session.accessToken, + userID: client.userID, + deviceID: client.deviceID, + roomID: roomId, + matrixRoom: sdkRoom + ) + return CallViewModel(encryptionContext: context) + } catch { + logger.warning("Could not create encryption context, falling back to unencrypted call: \(error.localizedDescription)") + return CallViewModel() + } } public func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { From 653b7d948af259fc765593e0c86f9637a5b025f6 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Thu, 9 Apr 2026 14:16:30 -0400 Subject: [PATCH 09/24] Overhaul call UI, fix Element-X interop, and add conditional E2EE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major changes: Call UI overhaul: - Move call to its own window (Window scene + CallManager + CallWindowView) - FaceTime-style design: remote video fills window, self-view PiP overlay, floating translucent control bar with hover-to-reveal - Fix beachball on disconnect by deferring network cleanup to Task.detached - Fix recursive constraint crash by deferring dismissWindow via DispatchQueue - Fix call window not reopening after ending a call Element-X/Element-web interop: - Fix call member state event to match MSC4143 format exactly: state key _userId_deviceId_m.call, focus_active with focus_selection, foci_preferred with livekit_alias, membershipID, m.call.intent - Pass SFU service URL (from discovery) through credential flow for correct livekit_service_url in call member events - Disable audio RED to match Element-X (audio/opus, not audio/red) - Auto-configure call power levels (org.matrix.msc3401.call.member → PL 0) Conditional E2EE: - Enable LiveKit GCM frame encryption only for encrypted Matrix rooms - Check room encryption state via roomInfo().encryptionState at call start - Unencrypted rooms publish with no LiveKit-level encryption, matching Element-X behavior Timeline improvements: - Display call member state events as "User started a call" with phone icon - Hide encryption key exchange events from timeline - Add .callEvent kind to TimelineMessage Co-Authored-By: Claude Opus 4.6 --- .../Models/TimelineMessage.swift | 6 +- .../Protocols/CallViewModelProtocol.swift | 2 +- .../Protocols/MatrixServiceProtocol.swift | 8 +- Relay/RelayApp.swift | 12 + Relay/Services/PreviewMatrixService.swift | 6 +- Relay/ViewModels/PreviewCallViewModel.swift | 2 +- Relay/Views/CallView.swift | 597 ++++++++++-------- Relay/Views/CallWindowView.swift | 92 +++ Relay/Views/MainView.swift | 38 +- Relay/Views/Message/SystemEventView.swift | 2 + RelayKit/Call/CallEncryptionService.swift | 176 +++++- RelayKit/Call/CallViewModel.swift | 79 ++- RelayKit/Call/LiveKitCredentialService.swift | 8 +- RelayKit/Services/MatrixService.swift | 15 +- RelayKit/Services/RoomListManager.swift | 11 +- RelayKit/Services/TimelineMessageMapper.swift | 60 +- 16 files changed, 747 insertions(+), 367 deletions(-) create mode 100644 Relay/Views/CallWindowView.swift diff --git a/Packages/RelayInterface/Sources/RelayInterface/Models/TimelineMessage.swift b/Packages/RelayInterface/Sources/RelayInterface/Models/TimelineMessage.swift index d8b49fc..0d4d078 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Models/TimelineMessage.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Models/TimelineMessage.swift @@ -66,6 +66,8 @@ public struct TimelineMessage: Identifiable, Sendable, Equatable { case profileChange /// A room state change (room name, topic, avatar, encryption, join rules, etc.). case stateEvent + /// A call-related event (user started, joined, or left a call). + case callEvent } /// A group of emoji reactions attached to a message, aggregated by reaction key. @@ -357,7 +359,7 @@ public struct TimelineMessage: Identifiable, Sendable, Equatable { nonisolated public var isSpecialType: Bool { switch kind { case .text, .emote, .notice: false - case .membership, .profileChange, .stateEvent: false + case .membership, .profileChange, .stateEvent, .callEvent: false default: true } } @@ -366,7 +368,7 @@ public struct TimelineMessage: Identifiable, Sendable, Equatable { /// rather than a user-authored message. nonisolated public var isSystemEvent: Bool { switch kind { - case .membership, .profileChange, .stateEvent: true + case .membership, .profileChange, .stateEvent, .callEvent: true default: false } } diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift index 3ab5392..489bb01 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift @@ -94,7 +94,7 @@ public protocol CallViewModelProtocol: AnyObject, Observable { /// - Parameters: /// - url: The WebSocket URL of the LiveKit server (e.g. `"wss://livekit.example.com"`). /// - token: A signed JWT granting access to the room. - func connect(url: String, token: String) async throws + func connect(url: String, token: String, sfuServiceURL: String) async throws /// Disconnects from the call and cleans up media resources. func disconnect() async diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift index 6194c42..9c44257 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/MatrixServiceProtocol.swift @@ -485,7 +485,7 @@ public protocol MatrixServiceProtocol: AnyObject, Observable { /// - Parameter roomId: The Matrix room identifier for the call. /// - Returns: A ``CallViewModelProtocol`` instance ready to be connected with a LiveKit /// URL and token, or `nil` if calling is not supported. - func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? + func makeCallViewModel(roomId: String) async -> (any CallViewModelProtocol)? /// Fetches LiveKit credentials for a Matrix room using the MatrixRTC flow (MSC4143). /// @@ -497,7 +497,7 @@ public protocol MatrixServiceProtocol: AnyObject, Observable { /// - Returns: A tuple of `(livekitURL, token)` where `livekitURL` is the LiveKit /// WebSocket URL and `token` is the JWT access token. /// - Throws: If the homeserver doesn't support MatrixRTC or credential exchange fails. - func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String, sfuServiceURL: String) // MARK: Notification Settings (synced via push rules) @@ -830,8 +830,8 @@ private final class PlaceholderMatrixService: MatrixServiceProtocol { func isCurrentSessionVerified() async -> Bool { false } func encryptionState() async -> EncryptionStatus { EncryptionStatus() } func makeSessionVerificationViewModel() async throws -> (any SessionVerificationViewModelProtocol)? { nil } - func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { nil } - func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + func makeCallViewModel(roomId: String) async -> (any CallViewModelProtocol)? { nil } + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String, sfuServiceURL: String) { throw PlaceholderError() } func getDefaultNotificationMode( diff --git a/Relay/RelayApp.swift b/Relay/RelayApp.swift index 1f439e7..f5a9c96 100644 --- a/Relay/RelayApp.swift +++ b/Relay/RelayApp.swift @@ -29,6 +29,7 @@ private let logger = Logger(subsystem: "Relay", category: "DeepLink") struct RelayApp: App { @State private var matrixService = MatrixService() @State private var gifSearchService = GiphyService(apiKey: Secrets.giphyAPIKey ?? "") + @State private var callManager = CallManager() @State private var notificationDelegate = NotificationDelegate() @State private var appActions = AppActions() @State private var composeDraftStore = ComposeDraftStore() @@ -43,6 +44,7 @@ struct RelayApp: App { ContentView() .environment(\.matrixService, matrixService) .environment(\.gifSearchService, gifSearchService) + .environment(\.callManager, callManager) .environment(\.errorReporter, matrixService.errorReporter) .environment(\.composeDraftStore, composeDraftStore) .environment(appActions) @@ -116,6 +118,16 @@ struct RelayApp: App { } .defaultSize(width: 900, height: 600) .keyboardShortcut("a", modifiers: [.option, .command]) + + Window("Call", id: "call") { + CallWindowView() + .environment(\.matrixService, matrixService) + .environment(\.callManager, callManager) + } + .windowStyle(.plain) + .windowResizability(.contentSize) + .defaultSize(width: 360, height: 540) + .defaultPosition(.topTrailing) } // MARK: - Notifications diff --git a/Relay/Services/PreviewMatrixService.swift b/Relay/Services/PreviewMatrixService.swift index 9da9619..d3b8500 100644 --- a/Relay/Services/PreviewMatrixService.swift +++ b/Relay/Services/PreviewMatrixService.swift @@ -224,14 +224,14 @@ final class PreviewMatrixService: MatrixServiceProtocol { PreviewSessionVerificationViewModel() } - func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { + func makeCallViewModel(roomId: String) async -> (any CallViewModelProtocol)? { PreviewCallViewModel() } - func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String, sfuServiceURL: String) { // Simulate a brief credential fetch; previews never actually connect. try? await Task.sleep(for: .milliseconds(500)) - return (livekitURL: "wss://preview.livekit.example.com", token: "preview-jwt-token") + return (livekitURL: "wss://preview.livekit.example.com", token: "preview-jwt-token", sfuServiceURL: "https://preview.livekit.example.com") } func declinePendingVerificationRequest() async { diff --git a/Relay/ViewModels/PreviewCallViewModel.swift b/Relay/ViewModels/PreviewCallViewModel.swift index 8494286..bd262ab 100644 --- a/Relay/ViewModels/PreviewCallViewModel.swift +++ b/Relay/ViewModels/PreviewCallViewModel.swift @@ -31,7 +31,7 @@ final class PreviewCallViewModel: CallViewModelProtocol { var localParticipantID: String? = nil var videoTrackRevision: UInt = 0 - func connect(url: String, token: String) async throws { + func connect(url: String, token: String, sfuServiceURL: String) async throws { state = .connecting try? await Task.sleep(for: .milliseconds(800)) isLocalCameraEnabled = true diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index 7f72a59..789c7a1 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -15,84 +15,349 @@ import RelayInterface import SwiftUI -/// Renders a LiveKit audio/video call within a Matrix room. +/// Renders a LiveKit audio/video call with a FaceTime-inspired design. /// -/// When the view model is in the ``CallState/idle`` state, ``CallView`` shows a -/// credential-entry form so the user can supply the LiveKit server URL and JWT token -/// before connecting. While connecting it shows a spinner with a Cancel button. -/// Once connected it shows participant tiles and a media-control bar. +/// The call opens in its own borderless window (`.windowStyle(.plain)`). +/// When connected, the remote participant's video fills the window with +/// a small self-view PiP overlay and a translucent floating control bar. struct CallView: View { @State var viewModel: any CallViewModelProtocol - /// `true` while the parent is fetching LiveKit credentials from the homeserver. - /// When set, the `.idle` state shows a spinner instead of the manual-entry form. var isPreparingCredentials: Bool = false var onDismiss: () -> Void - // Local fields used only while in the .idle (pre-connect) manual-entry form. @State private var serverURL: String = "" @State private var accessToken: String = "" @State private var isJoining: Bool = false + @State private var isHovering: Bool = false + @State private var controlsVisible: Bool = true + @State private var hideControlsTask: Task? var body: some View { ZStack { Color.black.ignoresSafeArea() - VStack(spacing: 0) { - switch viewModel.state { - case .idle: - if isPreparingCredentials { - preparingView - } else { - joinForm + switch viewModel.state { + case .idle: + if isPreparingCredentials { + preparingView + } else { + joinForm + } + + case .connecting: + connectingView + + case .connected: + connectedView + + case .disconnected: + endedOverlay( + title: "Call Ended", + systemImage: "phone.down.fill", + isError: false + ) + + case .failed(let message): + endedOverlay( + title: "Call Failed", + systemImage: "exclamationmark.triangle.fill", + isError: true, + detail: message + ) + } + } + .frame(minWidth: 320, minHeight: 480) + .clipShape(RoundedRectangle(cornerRadius: 16, style: .continuous)) + } + + // MARK: - Connected View (FaceTime-style) + + @ViewBuilder + private var connectedView: some View { + ZStack { + // Primary video: first remote participant fills the window + primaryVideo + .ignoresSafeArea() + + // Self-view PiP in bottom-right corner + if let localID = viewModel.localParticipantID { + VStack { + Spacer() + HStack { + Spacer() + selfViewPiP(id: localID) } + } + .padding(12) + .padding(.bottom, 72) + } + + // Participant name at top + VStack { + participantNameBar + Spacer() + } - case .connecting: - connectingView - - case .connected: - participantsGrid - .frame(maxWidth: .infinity, maxHeight: .infinity) - controlBar - - case .disconnected: - endedView( - title: "Call Ended", - systemImage: "phone.down.fill", - description: "The call has ended.", - isError: false - ) - - case .failed(let message): - endedView( - title: "Call Failed", - systemImage: "exclamationmark.triangle.fill", - description: message, - isError: true - ) + // Floating control bar at bottom + VStack { + Spacer() + controlBar + .opacity(controlsVisible ? 1 : 0) + .animation(.easeInOut(duration: 0.25), value: controlsVisible) + } + .padding(.bottom, 16) + } + .onHover { hovering in + isHovering = hovering + if hovering { + showControls() + } else { + scheduleHideControls() + } + } + .onAppear { scheduleHideControls() } + } + + // MARK: - Primary Video + + @ViewBuilder + private var primaryVideo: some View { + let _ = viewModel.videoTrackRevision + if let firstRemote = viewModel.participants.first { + if let videoView = viewModel.makeVideoView(for: firstRemote.id) { + videoView + } else { + // Remote has no video — show avatar placeholder + participantPlaceholder(firstRemote) + } + } else { + // No remote participants yet — waiting + VStack(spacing: 12) { + ProgressView() + .controlSize(.large) + .tint(.white) + Text("Waiting for others to join…") + .font(.headline) + .foregroundStyle(.white.opacity(0.7)) + } + } + } + + // MARK: - Self-View PiP + + @ViewBuilder + private func selfViewPiP(id: String) -> some View { + let _ = viewModel.videoTrackRevision + ZStack { + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(Color(nsColor: .darkGray)) + + if viewModel.isLocalCameraEnabled, + let videoView = viewModel.makeVideoView(for: id) { + videoView + } else { + Image(systemName: "person.fill") + .font(.title2) + .foregroundStyle(.white.opacity(0.5)) + } + } + .frame(width: 120, height: 90) + .clipShape(RoundedRectangle(cornerRadius: 10, style: .continuous)) + .shadow(color: .black.opacity(0.4), radius: 6, y: 2) + } + + // MARK: - Participant Name Bar + + @ViewBuilder + private var participantNameBar: some View { + if let first = viewModel.participants.first { + HStack { + if first.isSpeaking { + Image(systemName: "waveform") + .font(.caption) + .foregroundStyle(.green) } + Text(first.displayName ?? first.id) + .font(.callout.weight(.medium)) + .foregroundStyle(.white) + .lineLimit(1) } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(.ultraThinMaterial.opacity(0.8), in: Capsule()) + .padding(.top, 12) } - .frame(minWidth: 480, minHeight: 360) } - // MARK: - Preparing View (fetching credentials from homeserver) + // MARK: - Control Bar + + @ViewBuilder + private var controlBar: some View { + HStack(spacing: 20) { + // Microphone toggle + controlButton( + icon: viewModel.isLocalMicrophoneEnabled ? "mic.fill" : "mic.slash.fill", + isActive: viewModel.isLocalMicrophoneEnabled, + help: viewModel.isLocalMicrophoneEnabled ? "Mute" : "Unmute" + ) { + Task { try? await viewModel.toggleMicrophone() } + } + + // Camera toggle + controlButton( + icon: viewModel.isLocalCameraEnabled ? "video.fill" : "video.slash.fill", + isActive: viewModel.isLocalCameraEnabled, + help: viewModel.isLocalCameraEnabled ? "Camera Off" : "Camera On" + ) { + Task { try? await viewModel.toggleCamera() } + } + + // End call + Button { + Task { + await viewModel.disconnect() + onDismiss() + } + } label: { + Image(systemName: "phone.down.fill") + .font(.title3) + .foregroundStyle(.white) + .frame(width: 48, height: 48) + .background(Color.red, in: Circle()) + } + .buttonStyle(.plain) + .help("End Call") + } + .padding(.horizontal, 24) + .padding(.vertical, 12) + .background(.ultraThinMaterial, in: Capsule()) + } + + @ViewBuilder + private func controlButton(icon: String, isActive: Bool, help: String, action: @escaping () -> Void) -> some View { + Button(action: action) { + Image(systemName: icon) + .font(.title3) + .foregroundStyle(.white) + .frame(width: 44, height: 44) + .background( + isActive ? Color.white.opacity(0.15) : Color.red.opacity(0.8), + in: Circle() + ) + } + .buttonStyle(.plain) + .help(help) + } + + // MARK: - Controls Visibility + + private func showControls() { + hideControlsTask?.cancel() + controlsVisible = true + } + + private func scheduleHideControls() { + hideControlsTask?.cancel() + hideControlsTask = Task { + try? await Task.sleep(for: .seconds(3)) + if !Task.isCancelled && !isHovering { + controlsVisible = false + } + } + } + + // MARK: - Participant Placeholder + + @ViewBuilder + private func participantPlaceholder(_ participant: CallParticipant) -> some View { + VStack(spacing: 16) { + Image(systemName: "person.fill") + .font(.system(size: 64)) + .foregroundStyle(.white.opacity(0.3)) + Text(participant.displayName ?? participant.id) + .font(.title2.weight(.medium)) + .foregroundStyle(.white.opacity(0.6)) + } + } + + // MARK: - Preparing View @ViewBuilder private var preparingView: some View { - Spacer() - ProgressView("Contacting call server…") - .progressViewStyle(.circular) - .controlSize(.large) - .foregroundStyle(.white) - .tint(.white) - Button("Cancel") { onDismiss() } + VStack(spacing: 16) { + Spacer() + ProgressView() + .controlSize(.large) + .tint(.white) + Text("Contacting call server…") + .font(.headline) + .foregroundStyle(.white.opacity(0.7)) + Button("Cancel") { onDismiss() } + .buttonStyle(.bordered) + .foregroundStyle(.white) + Spacer() + } + } + + // MARK: - Connecting View + + @ViewBuilder + private var connectingView: some View { + VStack(spacing: 16) { + Spacer() + ProgressView() + .controlSize(.large) + .tint(.white) + Text("Joining call…") + .font(.headline) + .foregroundStyle(.white.opacity(0.7)) + Button("Cancel") { + Task { + await viewModel.disconnect() + onDismiss() + } + } .buttonStyle(.bordered) .foregroundStyle(.white) - .padding(.top, 20) - Spacer() + Spacer() + } + } + + // MARK: - Ended/Failed Overlay + + @ViewBuilder + private func endedOverlay(title: String, systemImage: String, isError: Bool, detail: String? = nil) -> some View { + VStack(spacing: 16) { + Spacer() + Image(systemName: systemImage) + .font(.system(size: 40)) + .foregroundStyle(isError ? .red : .white.opacity(0.6)) + Text(title) + .font(.title3.weight(.semibold)) + .foregroundStyle(.white) + if let detail { + Text(detail) + .font(.subheadline) + .foregroundStyle(.white.opacity(0.6)) + .multilineTextAlignment(.center) + .padding(.horizontal, 32) + } + Button("Dismiss") { onDismiss() } + .buttonStyle(.borderedProminent) + .tint(isError ? .red : .accentColor) + .padding(.top, 4) + Spacer() + } + .task { + // Auto-dismiss after a few seconds for clean endings. + if !isError { + try? await Task.sleep(for: .seconds(2)) + onDismiss() + } + } } - // MARK: - Join Form (idle state) + // MARK: - Join Form (manual entry fallback) @ViewBuilder private var joinForm: some View { @@ -108,7 +373,7 @@ struct CallView: View { .font(.title2.bold()) .foregroundStyle(.white) - Text("Enter the LiveKit server URL and access token\nprovided by your call server.") + Text("Enter the LiveKit server URL and access token.") .font(.subheadline) .foregroundStyle(.white.opacity(0.7)) .multilineTextAlignment(.center) @@ -129,20 +394,18 @@ struct CallView: View { .textFieldStyle(.roundedBorder) .autocorrectionDisabled() } - .frame(maxWidth: 360) + .frame(maxWidth: 320) HStack(spacing: 16) { - Button("Cancel") { - onDismiss() - } - .buttonStyle(.bordered) - .foregroundStyle(.white) + Button("Cancel") { onDismiss() } + .buttonStyle(.bordered) + .foregroundStyle(.white) Button("Join") { guard !serverURL.isEmpty, !accessToken.isEmpty else { return } Task { isJoining = true - try? await viewModel.connect(url: serverURL, token: accessToken) + try? await viewModel.connect(url: serverURL, token: accessToken, sfuServiceURL: "") isJoining = false } } @@ -155,228 +418,20 @@ struct CallView: View { Spacer() } } - - // MARK: - Connecting View - - @ViewBuilder - private var connectingView: some View { - Spacer() - ProgressView("Joining call…") - .progressViewStyle(.circular) - .controlSize(.large) - .foregroundStyle(.white) - .tint(.white) - Button("Cancel") { - Task { - await viewModel.disconnect() - onDismiss() - } - } - .buttonStyle(.bordered) - .foregroundStyle(.white) - .padding(.top, 20) - Spacer() - } - - // MARK: - Ended / Failed View - - @ViewBuilder - private func endedView(title: String, systemImage: String, description: String, isError: Bool) -> some View { - Spacer() - ContentUnavailableView( - title, - systemImage: systemImage, - description: Text(description) - ) - .foregroundStyle(.white) - Button("Dismiss") { onDismiss() } - .buttonStyle(.borderedProminent) - .tint(isError ? .red : .accentColor) - .padding(.top, 12) - Spacer() - } - - // MARK: - Participants Grid - // - // NOTE: For a richer integration, consider adopting LiveKitComponents - // (ForEachParticipant / ForEachTrack / VideoTrackView) which handle - // participant lifecycle, adaptive streaming, and reconnection automatically. - // See: https://github.com/livekit/components-swift - - @ViewBuilder - private var participantsGrid: some View { - let columns = [GridItem(.adaptive(minimum: 200, maximum: 400), spacing: 8)] - ScrollView { - LazyVGrid(columns: columns, spacing: 8) { - ForEach(viewModel.participants) { participant in - participantTile(participant) - .id(participant.id) - } - if let localID = viewModel.localParticipantID { - localParticipantTile(id: localID) - .id(localID) - } - } - .padding(8) - } - } - - // MARK: - Participant Tile - - @ViewBuilder - private func participantTile(_ participant: CallParticipant) -> some View { - ZStack(alignment: .bottom) { - Color(nsColor: .darkGray) - - videoContent(for: participant.id) - - if participant.isSpeaking { - Rectangle() - .strokeBorder(.green, lineWidth: 3) - } - - HStack(spacing: 6) { - Text(participant.displayName ?? participant.id) - .font(.caption) - .foregroundStyle(.white) - .lineLimit(1) - .truncationMode(.tail) - Spacer() - if !participant.isMicrophoneEnabled { - Image(systemName: "mic.slash.fill").font(.caption).foregroundStyle(.red) - } - if !participant.isCameraEnabled { - Image(systemName: "video.slash.fill").font(.caption).foregroundStyle(.secondary) - } - } - .padding(.horizontal, 8) - .padding(.vertical, 6) - .background(.black.opacity(0.55)) - } - .aspectRatio(16.0 / 9.0, contentMode: .fit) - } - - // MARK: - Local Participant Tile - - @ViewBuilder - private func localParticipantTile(id: String) -> some View { - ZStack(alignment: .bottom) { - Color(nsColor: .darkGray) - - videoContent(for: id) - - HStack(spacing: 6) { - Text("You").font(.caption).foregroundStyle(.white) - Spacer() - if !viewModel.isLocalMicrophoneEnabled { - Image(systemName: "mic.slash.fill").font(.caption).foregroundStyle(.red) - } - if !viewModel.isLocalCameraEnabled { - Image(systemName: "video.slash.fill").font(.caption).foregroundStyle(.secondary) - } - } - .padding(.horizontal, 8) - .padding(.vertical, 6) - .background(.black.opacity(0.55)) - } - .aspectRatio(16.0 / 9.0, contentMode: .fit) - } - - // MARK: - Control Bar - - @ViewBuilder - private var controlBar: some View { - HStack(spacing: 24) { - Button { - Task { try? await viewModel.toggleMicrophone() } - } label: { - Image(systemName: viewModel.isLocalMicrophoneEnabled ? "mic.fill" : "mic.slash.fill") - .font(.title2) - .frame(width: 44, height: 44) - .background(viewModel.isLocalMicrophoneEnabled ? Color.white.opacity(0.15) : Color.red.opacity(0.8)) - .clipShape(Circle()) - } - .buttonStyle(.plain) - .foregroundStyle(.white) - .help(viewModel.isLocalMicrophoneEnabled ? "Mute microphone" : "Unmute microphone") - - Button { - Task { try? await viewModel.toggleCamera() } - } label: { - Image(systemName: viewModel.isLocalCameraEnabled ? "video.fill" : "video.slash.fill") - .font(.title2) - .frame(width: 44, height: 44) - .background(viewModel.isLocalCameraEnabled ? Color.white.opacity(0.15) : Color.red.opacity(0.8)) - .clipShape(Circle()) - } - .buttonStyle(.plain) - .foregroundStyle(.white) - .help(viewModel.isLocalCameraEnabled ? "Turn off camera" : "Turn on camera") - - Button { - Task { - await viewModel.disconnect() - onDismiss() - } - } label: { - Image(systemName: "phone.down.fill") - .font(.title2) - .frame(width: 52, height: 52) - .background(Color.red) - .clipShape(Circle()) - } - .buttonStyle(.plain) - .foregroundStyle(.white) - .help("End call") - } - .padding(.vertical, 16) - .padding(.horizontal, 32) - .background(.black.opacity(0.7)) - } -} - -// MARK: - Video Content - -extension CallView { - /// Returns the LiveKit `SwiftUIVideoView` (via `AnyView`) for the given participant, - /// or a dark grey placeholder if no video track is available yet. - /// - /// > Important: Do **not** apply `.clipShape()` or `.mask()` to the returned - /// > video view. `SwiftUIVideoView` renders via Metal and SwiftUI shape - /// > clipping interferes with the GPU-backed surface, causing visual artefacts. - @ViewBuilder - fileprivate func videoContent(for participantID: String) -> some View { - // Reading videoTrackRevision ensures SwiftUI re-evaluates this - // when tracks change (publish, subscribe, toggle). - let _ = viewModel.videoTrackRevision - if let videoView = viewModel.makeVideoView(for: participantID) { - videoView - } else { - RoundedRectangle(cornerRadius: 10) - .fill(Color(nsColor: .darkGray)) - } - } } // MARK: - Previews -#Preview("Join Form") { - CallView(viewModel: PreviewCallViewModel(), onDismiss: {}) - .frame(width: 640, height: 480) -} - -#Preview("Connecting") { - @Previewable @State var vm = PreviewCallViewModel() - CallView(viewModel: vm, onDismiss: {}) - .frame(width: 640, height: 480) - .onAppear { vm.state = .connecting } +#Preview("Preparing") { + CallView(viewModel: PreviewCallViewModel(), isPreparingCredentials: true, onDismiss: {}) + .frame(width: 360, height: 540) } #Preview("Connected") { let vm = PreviewCallViewModel() return CallView(viewModel: vm, onDismiss: {}) - .frame(width: 640, height: 480) + .frame(width: 360, height: 540) .task { - try? await vm.connect(url: "wss://preview.example.com", token: "preview-token") + try? await vm.connect(url: "wss://preview.example.com", token: "preview-token", sfuServiceURL: "") } } diff --git a/Relay/Views/CallWindowView.swift b/Relay/Views/CallWindowView.swift new file mode 100644 index 0000000..22d998f --- /dev/null +++ b/Relay/Views/CallWindowView.swift @@ -0,0 +1,92 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import RelayInterface +import SwiftUI + +/// Shared call state accessible from both ``MainView`` (to start calls) and +/// ``CallWindowView`` (to display the call UI in its own window). +@Observable +@MainActor +final class CallManager { + var activeCallViewModel: (any CallViewModelProtocol)? + var isPreparingCredentials: Bool = false + var callRoomId: String? + + /// Whether there is an active or preparing call. + var hasActiveCall: Bool { + activeCallViewModel != nil + } + + func endCall() async { + await activeCallViewModel?.disconnect() + activeCallViewModel = nil + isPreparingCredentials = false + callRoomId = nil + } +} + +// MARK: - Environment Key + +private struct CallManagerKey: @preconcurrency EnvironmentKey { + @MainActor static let defaultValue = CallManager() +} + +extension EnvironmentValues { + var callManager: CallManager { + get { self[CallManagerKey.self] } + set { self[CallManagerKey.self] = newValue } + } +} + +/// Hosts the ``CallView`` inside the dedicated call window. +/// +/// This view reads the shared ``CallManager`` from the environment and +/// presents the call UI. When the call ends or is dismissed, it closes +/// the window via `dismissWindow`. +struct CallWindowView: View { + @Environment(\.callManager) private var callManager + @Environment(\.dismissWindow) private var dismissWindow + + var body: some View { + Group { + if let viewModel = callManager.activeCallViewModel { + CallView( + viewModel: viewModel, + isPreparingCredentials: callManager.isPreparingCredentials + ) { + Task { + await callManager.endCall() + // Defer dismissal to the next run loop iteration so it + // doesn't fire during a SwiftUI layout pass, which causes + // a recursive constraint update crash. + DispatchQueue.main.async { + dismissWindow(id: "call") + } + } + } + } else { + // No active call — show placeholder until the window closes. + Color.black + } + } + .onChange(of: callManager.hasActiveCall) { _, hasCall in + if !hasCall { + DispatchQueue.main.async { + dismissWindow(id: "call") + } + } + } + } +} diff --git a/Relay/Views/MainView.swift b/Relay/Views/MainView.swift index 68e0024..ab351c7 100644 --- a/Relay/Views/MainView.swift +++ b/Relay/Views/MainView.swift @@ -31,6 +31,8 @@ struct MainView: View { // swiftlint:disable:this type_body_length @Environment(\.matrixService) private var matrixService @Environment(\.errorReporter) private var errorReporter @Environment(AppActions.self) private var appActions + @Environment(\.callManager) private var callManager + @Environment(\.openWindow) private var openWindow @AppStorage("selectedRoomId") private var selectedRoomId: String? @State private var selectedSpaceId: String? @State private var leaveSpaceItem: LeaveSpaceItem? @@ -46,8 +48,6 @@ struct MainView: View { // swiftlint:disable:this type_body_length @State private var isJoiningLinkedRoom = false @State private var inspectorSelectedProfile: UserProfile? @State private var inspectorInitialTab: InspectorTab? - @State private var activeCallViewModel: (any CallViewModelProtocol)? - @State private var isShowingCall = false @State private var isPreparingCall = false private func scrollToMessage(_ eventId: String) { @@ -239,18 +239,6 @@ struct MainView: View { // swiftlint:disable:this type_body_length .sheet(item: $leaveSpaceItem) { item in LeaveSpaceSheet(spaceName: item.name, spaceId: item.id, children: item.children) } - .sheet(isPresented: $isShowingCall) { - if let callViewModel = activeCallViewModel { - CallView( - viewModel: callViewModel, - isPreparingCredentials: isPreparingCall - ) { - isShowingCall = false - isPreparingCall = false - activeCallViewModel = nil - } - } - } .onChange(of: matrixService.spaces.map(\.id)) { if let selectedSpaceId, !matrixService.spaces.contains(where: { $0.id == selectedSpaceId }) { self.selectedSpaceId = nil @@ -410,21 +398,27 @@ struct MainView: View { // swiftlint:disable:this type_body_length // MARK: - Call Handling private func startCall(roomId: String) { - guard !isPreparingCall else { return } - guard let viewModel = matrixService.makeCallViewModel(roomId: roomId) else { return } - activeCallViewModel = viewModel - isPreparingCall = true - isShowingCall = true + guard !callManager.hasActiveCall else { return } + callManager.isPreparingCredentials = true + callManager.callRoomId = roomId Task { + guard let viewModel = await matrixService.makeCallViewModel(roomId: roomId) else { + callManager.isPreparingCredentials = false + callManager.callRoomId = nil + return + } + callManager.activeCallViewModel = viewModel + openWindow(id: "call") + do { - let (url, token) = try await matrixService.callCredentials(for: roomId) - try await viewModel.connect(url: url, token: token) + let creds = try await matrixService.callCredentials(for: roomId) + try await viewModel.connect(url: creds.livekitURL, token: creds.token, sfuServiceURL: creds.sfuServiceURL) } catch { // Credential fetch or connect failed — viewModel stays in .idle so // CallView shows the manual-entry join form as a fallback. } - isPreparingCall = false + callManager.isPreparingCredentials = false } } diff --git a/Relay/Views/Message/SystemEventView.swift b/Relay/Views/Message/SystemEventView.swift index 15427da..d67b14e 100644 --- a/Relay/Views/Message/SystemEventView.swift +++ b/Relay/Views/Message/SystemEventView.swift @@ -42,6 +42,8 @@ struct SystemEventView: View { "person.2" case .profileChange: "person.text.rectangle" + case .callEvent: + "phone.fill" case .stateEvent: "gearshape" default: diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift index 0391abc..d5c97e9 100644 --- a/RelayKit/Call/CallEncryptionService.swift +++ b/RelayKit/Call/CallEncryptionService.swift @@ -53,42 +53,55 @@ struct CallEncryptionService { /// Sends the MatrixRTC call membership state event so that Element-X and other /// MatrixRTC clients can discover our participation in the call. /// - /// - Parameter livekitURL: The LiveKit SFU service URL (used in `foci_active`). - func sendCallMemberEvent(livekitURL: String) async throws { + /// Uses the modern MSC4143 per-device format matching Element-X: + /// - State key: `_@userId:server_deviceId_m.call` + /// - `focus_active`: `{"type": "livekit", "focus_selection": "oldest_membership"}` + /// - `foci_preferred`: array with the SFU service URL and room alias + /// + /// - Parameter sfuServiceURL: The SFU service URL from MatrixRTC discovery + /// (e.g. `https://livekit.example.com/livekit/jwt`). + func sendCallMemberEvent(sfuServiceURL: String) async throws { let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID let encodedEventType = Self.callMemberEventType .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType - // State key is the user's Matrix ID — one membership entry per user. - let encodedStateKey = userID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? userID + // MSC4143 state key: "_@userId:server_deviceId_m.call" + let stateKey = "_\(userID)_\(deviceID)_m.call" + let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { throw LiveKitCredentialError.invalidURL } - // Strip trailing slash from LiveKit URL for the foci_active entry. - let sfuURL = livekitURL.trimmingCharacters(in: .init(charactersIn: "/")) + let serviceURL = sfuServiceURL.trimmingCharacters(in: .init(charactersIn: "/")) + // Match Element-X's exact format. let body: [String: Any] = [ - "memberships": [ + "application": "m.call", + "call_id": "", + "device_id": deviceID, + "expires": 14400000, + "focus_active": [ + "type": "livekit", + "focus_selection": "oldest_membership" + ] as [String: Any], + "foci_preferred": [ [ - "application": "m.call", - "call_id": "", - "device_id": deviceID, - "expires": 3600000, - "foci_active": [ - [ - "type": "livekit", - "livekit_service_url": sfuURL - ] - ], - "membershipID": UUID().uuidString, - "scope": "m.room" + "type": "livekit", + "livekit_service_url": serviceURL, + "livekit_alias": roomID ] as [String: Any] - ] + ], + "m.call.intent": "video", + "membershipID": "\(userID):\(deviceID)", + "scope": "m.room" ] - let jsonData = try JSONSerialization.data(withJSONObject: body) + let jsonData = try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) + if let jsonStr = String(data: jsonData, encoding: .utf8) { + logger.info("Call member event body: \(jsonStr)") + } + logger.info("Call member state key: \(stateKey)") var request = URLRequest(url: url) request.httpMethod = "PUT" @@ -96,30 +109,33 @@ struct CallEncryptionService { request.setValue("application/json", forHTTPHeaderField: "Content-Type") request.httpBody = jsonData - let (_, response) = try await URLSession.shared.data(for: request) + let (responseData, response) = try await URLSession.shared.data(for: request) guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 - logger.error("sendCallMemberEvent failed with status \(statusCode)") + if let respStr = String(data: responseData, encoding: .utf8) { + logger.error("sendCallMemberEvent failed with status \(statusCode): \(respStr)") + } throw CallEncryptionError.callMemberEventFailed } logger.info("Sent call membership state event") } - /// Removes the call membership state event (sets memberships to empty) + /// Removes the call membership state event (sets content to empty object) /// so Element-X knows we've left the call. func removeCallMemberEvent() async throws { let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID let encodedEventType = Self.callMemberEventType .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType - let encodedStateKey = userID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? userID + let stateKey = "_\(userID)_\(deviceID)_m.call" + let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { throw LiveKitCredentialError.invalidURL } - let body: [String: Any] = ["memberships": [Any]()] + let body: [String: Any] = [:] let jsonData = try JSONSerialization.data(withJSONObject: body) @@ -139,6 +155,114 @@ struct CallEncryptionService { logger.info("Removed call membership state event") } + // MARK: - Debug: Fetch Existing Call Members + + /// Fetches all existing `org.matrix.msc3401.call.member` state events from + /// the room for debugging interoperability issues. + func fetchCallMemberEvents() async { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state") else { return } + + var request = URLRequest(url: url) + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + + guard let (data, response) = try? await URLSession.shared.data(for: request), + let http = response as? HTTPURLResponse, http.statusCode == 200, + let events = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] else { + return + } + + for event in events { + guard let type = event["type"] as? String, + type == Self.callMemberEventType else { continue } + let stateKey = event["state_key"] as? String ?? "(none)" + if let content = event["content"], + let contentData = try? JSONSerialization.data(withJSONObject: content, options: [.sortedKeys]), + let contentStr = String(data: contentData, encoding: .utf8) { + logger.info("Existing call member [key=\(stateKey)]: \(contentStr)") + } + } + } + + // MARK: - Room Call Setup + + /// Ensures the room's power levels allow any member to send call-related + /// state events. Element-web does this automatically when a call is started. + /// + /// Sets `org.matrix.msc3401.call.member` and `io.element.call.encryption_keys` + /// to power level 0 in the room's `m.room.power_levels` state event. + /// + /// This is idempotent — if the levels are already correct, the PUT overwrites + /// with the same content. + func enableCallPowerLevels() async throws { + let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) + let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID + + // 1. Fetch current power levels. + guard let getURL = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/m.room.power_levels/") else { + throw LiveKitCredentialError.invalidURL + } + + var getRequest = URLRequest(url: getURL) + getRequest.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + + let (data, getResponse) = try await URLSession.shared.data(for: getRequest) + guard let http = getResponse as? HTTPURLResponse, http.statusCode == 200 else { + logger.warning("Could not fetch power levels (status \((getResponse as? HTTPURLResponse)?.statusCode ?? -1))") + return + } + + guard var powerLevels = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + return + } + + // 2. Merge call event types into the events dict at PL 0. + var events = (powerLevels["events"] as? [String: Any]) ?? [:] + let callEventTypes = [ + Self.callMemberEventType, + Self.encryptionKeysEventType + ] + + var needsUpdate = false + for eventType in callEventTypes { + if events[eventType] as? Int != 0 { + events[eventType] = 0 + needsUpdate = true + } + } + + guard needsUpdate else { + logger.info("Call power levels already configured") + return + } + + powerLevels["events"] = events + + // 3. PUT the updated power levels. + guard let putURL = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/m.room.power_levels/") else { + throw LiveKitCredentialError.invalidURL + } + + let jsonData = try JSONSerialization.data(withJSONObject: powerLevels) + + var putRequest = URLRequest(url: putURL) + putRequest.httpMethod = "PUT" + putRequest.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + putRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") + putRequest.httpBody = jsonData + + let (_, putResponse) = try await URLSession.shared.data(for: putRequest) + guard let putHTTP = putResponse as? HTTPURLResponse, (200...299).contains(putHTTP.statusCode) else { + let statusCode = (putResponse as? HTTPURLResponse)?.statusCode ?? -1 + logger.warning("Failed to update call power levels (status \(statusCode))") + return + } + + logger.info("Enabled call power levels for room") + } + // MARK: - Key Generation /// Generates a cryptographically random 16-byte key suitable for AES-128-GCM. diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 3234d56..d4bcbb5 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -68,6 +68,7 @@ public final class CallViewModel: CallViewModelProtocol { /// Creates a call view model without E2EE. Use ``init(encryptionContext:)`` /// for encrypted calls that interoperate with Element Call. public init() { + self.isE2eeEnabled = false let delegate = Delegate(viewModel: self) self.delegate = delegate room.add(delegate: delegate) @@ -80,23 +81,34 @@ public final class CallViewModel: CallViewModelProtocol { public let userID: String public let deviceID: String public let roomID: String + /// Whether the Matrix room has encryption enabled (`m.room.encryption` state event). + /// When `true`, LiveKit-level GCM frame encryption + key exchange is enabled. + public let isRoomEncrypted: Bool /// The Matrix SDK room, used to obtain the timeline for listening to /// inbound encryption key state events. `nil` if unavailable. public let matrixRoom: MatrixRustSDK.Room? - public init(homeserver: String, accessToken: String, userID: String, deviceID: String, roomID: String, matrixRoom: MatrixRustSDK.Room? = nil) { + public init(homeserver: String, accessToken: String, userID: String, deviceID: String, roomID: String, isRoomEncrypted: Bool = false, matrixRoom: MatrixRustSDK.Room? = nil) { self.homeserver = homeserver self.accessToken = accessToken self.userID = userID self.deviceID = deviceID self.roomID = roomID + self.isRoomEncrypted = isRoomEncrypted self.matrixRoom = matrixRoom } } - /// Creates a call view model with E2EE enabled, using AES-128-GCM frame - /// encryption compatible with Element Call's MatrixRTC key exchange. + /// Whether this call uses LiveKit-level E2EE (GCM frame encryption). + /// Mirrors the Matrix room's encryption state. + private let isE2eeEnabled: Bool + + /// Creates a call view model with optional E2EE, determined by the Matrix + /// room's encryption state. Encrypted rooms use AES-128-GCM frame encryption + /// with MatrixRTC key exchange; unencrypted rooms use no LiveKit-level E2EE. public init(encryptionContext: EncryptionContext) { + self.isE2eeEnabled = encryptionContext.isRoomEncrypted + let delegate = Delegate(viewModel: self) self.delegate = delegate room.add(delegate: delegate) @@ -109,15 +121,17 @@ public final class CallViewModel: CallViewModelProtocol { roomID: encryptionContext.roomID ) - // Per-participant key provider: each participant has their own key. - let provider = BaseKeyProvider(isSharedKey: false) - self.keyProvider = provider + if encryptionContext.isRoomEncrypted { + // Per-participant key provider: each participant has their own key. + let provider = BaseKeyProvider(isSharedKey: false) + self.keyProvider = provider + } self.matrixRoom = encryptionContext.matrixRoom } // MARK: - CallViewModelProtocol - public func connect(url: String, token: String) async throws { + public func connect(url: String, token: String, sfuServiceURL: String = "") async throws { state = .connecting do { let connectOpts = ConnectOptions( @@ -125,14 +139,25 @@ public final class CallViewModel: CallViewModelProtocol { enableMicrophone: true ) - // Build RoomOptions — with E2EE if a key provider was configured. + // Enable LiveKit-level GCM frame encryption only for encrypted Matrix + // rooms. Element Call also uses LiveKit E2EE (SFrame) for encrypted + // rooms and no encryption for unencrypted rooms. let encryptionOpts: EncryptionOptions? = keyProvider.map { EncryptionOptions(keyProvider: $0, encryptionType: .gcm) } + if isE2eeEnabled { + logger.info("E2EE enabled (encrypted Matrix room)") + } else { + logger.info("E2EE disabled (unencrypted Matrix room)") + } let roomOpts = RoomOptions( defaultVideoPublishOptions: VideoPublishOptions( preferredCodec: .vp8 ), + defaultAudioPublishOptions: AudioPublishOptions( + dtx: true, + red: false + ), adaptiveStream: true, dynacast: true, encryptionOptions: encryptionOpts @@ -146,21 +171,28 @@ public final class CallViewModel: CallViewModelProtocol { localParticipantID = room.localParticipant.identity?.stringValue logger.info("Connected as \(self.localParticipantID ?? "unknown")") - // Send MatrixRTC call membership state event so Element-X and other - // MatrixRTC clients can discover our participation in this call. + // Ensure call power levels are set so any room member can join, + // then send the MatrixRTC call membership state event. if let encryptionService { Task { + // Debug: log existing call member events to compare formats. + await encryptionService.fetchCallMemberEvents() + do { - try await encryptionService.sendCallMemberEvent(livekitURL: url) + try await encryptionService.enableCallPowerLevels() + } catch { + logger.warning("Call power level setup failed: \(error.localizedDescription)") + } + do { + try await encryptionService.sendCallMemberEvent(sfuServiceURL: sfuServiceURL) } catch { logger.warning("Call membership event failed: \(error.localizedDescription)") } } } - // Generate and distribute the local E2EE key before publishing tracks, - // so that the first frames are already encrypted. - if let keyProvider, let encryptionService { + // Generate and distribute the local E2EE key when the room is encrypted. + if isE2eeEnabled, let keyProvider, let encryptionService { let key = CallEncryptionService.generateKey() localEncryptionKey = key @@ -176,7 +208,6 @@ public final class CallViewModel: CallViewModelProtocol { // Distribute key via both transports (best-effort, don't block connect). Task { do { - // 1. To-device messages (Element Call interop) let members = try await encryptionService.fetchJoinedMembers() if !members.isEmpty { try await encryptionService.sendKey(key, keyIndex: localKeyIndex, to: members) @@ -186,7 +217,6 @@ public final class CallViewModel: CallViewModelProtocol { } do { - // 2. Room state event (Relay-to-Relay interop) try await encryptionService.sendKeyAsStateEvent(key, keyIndex: localKeyIndex) } catch { logger.warning("State event key distribution failed: \(error.localizedDescription)") @@ -218,12 +248,7 @@ public final class CallViewModel: CallViewModelProtocol { } public func disconnect() async { - // Remove call membership state event so other clients know we've left. - if let encryptionService { - try? await encryptionService.removeCallMemberEvent() - } - - await room.disconnect() + // Update UI state immediately — don't block on network I/O. state = .disconnected participants = [] isLocalCameraEnabled = false @@ -233,6 +258,14 @@ public final class CallViewModel: CallViewModelProtocol { localEncryptionKey = nil localKeyIndex = 0 keyListenerHandle = nil + + // Network cleanup in background so the UI never beachballs. + let service = encryptionService + let livekitRoom = room + Task.detached { + try? await service?.removeCallMemberEvent() + await livekitRoom.disconnect() + } } public func toggleCamera() async throws { @@ -389,7 +422,7 @@ public final class CallViewModel: CallViewModelProtocol { Task { @MainActor [weak viewModel] in guard let viewModel else { return } viewModel.syncParticipants(trackChanged: true) - if let identity = participant.identity?.stringValue { + if viewModel.isE2eeEnabled, let identity = participant.identity?.stringValue { viewModel.redistributeKey(to: identity) } } diff --git a/RelayKit/Call/LiveKitCredentialService.swift b/RelayKit/Call/LiveKitCredentialService.swift index 723a13c..ae45d83 100644 --- a/RelayKit/Call/LiveKitCredentialService.swift +++ b/RelayKit/Call/LiveKitCredentialService.swift @@ -44,14 +44,16 @@ struct LiveKitCredentialService { // MARK: - Public Entry Point - /// Returns `(livekitWebSocketURL, livekitJWT)` for the given Matrix room. - func credentials(for roomID: String) async throws -> (url: String, token: String) { + /// Returns `(livekitWebSocketURL, livekitJWT, sfuServiceURL)` for the given Matrix room. + /// The `sfuServiceURL` is the SFU service URL from discovery, used in call member events. + func credentials(for roomID: String) async throws -> (url: String, token: String, sfuServiceURL: String) { logger.info("Fetching LiveKit credentials for room \(roomID, privacy: .private)") let sfuURL = try await discoverSFUURL() logger.info("SFU URL discovered: \(sfuURL)") let openIDToken = try await requestOpenIDToken() logger.debug("OpenID token obtained") - return try await fetchLiveKitToken(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) + let (url, jwt) = try await fetchLiveKitToken(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) + return (url, jwt, sfuURL) } // MARK: - Step 1: Discover SFU URL diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 2d6e91c..4146fe9 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1440,17 +1440,26 @@ public final class MatrixService: MatrixServiceProtocol { return viewModel } - public func makeCallViewModel(roomId: String) -> (any CallViewModelProtocol)? { + public func makeCallViewModel(roomId: String) async -> (any CallViewModelProtocol)? { guard let client else { return nil } do { let session = try client.session() let sdkRoom = room(id: roomId) + // Check if the Matrix room has encryption enabled to decide whether + // to use LiveKit-level E2EE for the call. + let isEncrypted: Bool + if let sdkRoom, let info = try? await sdkRoom.roomInfo() { + isEncrypted = info.encryptionState != .notEncrypted + } else { + isEncrypted = false + } let context = CallViewModel.EncryptionContext( homeserver: client.homeserver, accessToken: session.accessToken, userID: client.userID, deviceID: client.deviceID, roomID: roomId, + isRoomEncrypted: isEncrypted, matrixRoom: sdkRoom ) return CallViewModel(encryptionContext: context) @@ -1460,7 +1469,7 @@ public final class MatrixService: MatrixServiceProtocol { } } - public func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String) { + public func callCredentials(for roomId: String) async throws -> (livekitURL: String, token: String, sfuServiceURL: String) { guard let client else { throw LiveKitCredentialError.serverError } @@ -1472,7 +1481,7 @@ public final class MatrixService: MatrixServiceProtocol { deviceID: client.deviceID ) let result = try await service.credentials(for: roomId) - return (livekitURL: result.url, token: result.token) + return (livekitURL: result.url, token: result.token, sfuServiceURL: result.sfuServiceURL) } public func declinePendingVerificationRequest() async { diff --git a/RelayKit/Services/RoomListManager.swift b/RelayKit/Services/RoomListManager.swift index 5bf2069..9b3180b 100644 --- a/RelayKit/Services/RoomListManager.swift +++ b/RelayKit/Services/RoomListManager.swift @@ -802,8 +802,15 @@ private final class RoomEntry: Identifiable { avatarUrl: avatarUrl, prevAvatarUrl: prevAvatarUrl )) - case .state(_, let content): - return AttributedString(TimelineMessageMapper.stateEventDescription(content)) + case .state(let stateKey, let content): + let (body, _) = TimelineMessageMapper.describeStateEvent( + content, + stateKey: stateKey, + senderDisplayName: nil, + senderId: "" + ) + guard let body else { return nil } + return AttributedString(body) default: return nil } // swiftlint:enable identifier_name diff --git a/RelayKit/Services/TimelineMessageMapper.swift b/RelayKit/Services/TimelineMessageMapper.swift index 79b83c1..61d6a73 100644 --- a/RelayKit/Services/TimelineMessageMapper.swift +++ b/RelayKit/Services/TimelineMessageMapper.swift @@ -187,9 +187,20 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len prevAvatarUrl: prevAvatarUrl ) msgKind = .profileChange - case .state(_, let content): - msgBody = Self.stateEventDescription(content) - msgKind = .stateEvent + case .state(let stateKey, let content): + let (body, kind) = Self.describeStateEvent( + content, + stateKey: stateKey, + senderDisplayName: { + if case .ready(let name, _, _) = event.senderProfile { return name } + return nil + }(), + senderId: event.sender + ) + // Skip noisy internal events (encryption key exchange). + guard let body else { continue } + msgBody = body + msgKind = kind default: continue } @@ -773,9 +784,19 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len prevAvatarUrl: prevAvatarUrl ) msgKind = .profileChange - case .state(_, let content): - msgBody = Self.stateEventDescription(content) - msgKind = .stateEvent + case .state(let stateKey, let content): + let (body, kind) = Self.describeStateEvent( + content, + stateKey: stateKey, + senderDisplayName: { + if case .ready(let name, _, _) = event.senderProfile { return name } + return nil + }(), + senderId: event.sender + ) + guard let body else { return nil } + msgBody = body + msgKind = kind default: return nil } @@ -927,6 +948,33 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len return "\(name) updated their profile" } + /// Routes a state event to the appropriate description and message kind. + /// + /// Returns `nil` body for events that should be hidden (e.g. encryption key exchange). + static func describeStateEvent( + _ state: OtherState, + stateKey: String, + senderDisplayName: String?, + senderId: String + ) -> (body: String?, kind: TimelineMessage.Kind) { + if case .custom(let type) = state { + switch type { + case "org.matrix.msc3401.call.member": + let name = senderDisplayName ?? senderId + // Empty state key or one starting with "_" indicates join/leave. + // A non-empty content means joining; removal sends empty content + // which the SDK may or may not surface — treat presence of the event as a join. + return ("\(name) started a call", .callEvent) + case "io.element.call.encryption_keys": + // Internal key exchange — don't show in timeline. + return (nil, .stateEvent) + default: + return (stateEventDescription(state), .stateEvent) + } + } + return (stateEventDescription(state), .stateEvent) + } + // swiftlint:disable cyclomatic_complexity /// Returns a human-readable description for a room state change event. nonisolated static func stateEventDescription(_ state: OtherState) -> String { From 2d193c27c8552e00e43016f69da48744ea1cb230 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Tue, 28 Apr 2026 14:31:12 -0400 Subject: [PATCH 10/24] Improve call window UX, fix constraint crash, and add error reporting - Fix recursive constraint crash: defer all CallManager observable state mutations and window open/dismiss calls to the next run-loop iteration so they never fire during an active AppKit layout pass - Make call window draggable, resizable, and responsive to Window menu commands (Fill, Center) using .hiddenTitleBar with transparent styling - Suppress call window on launch (.defaultLaunchBehavior(.suppressed)) - Report call failures to user via errorReporter instead of swallowing - Deduplicate consecutive timeline call events from the same sender - Consolidate dismiss to single onChange path, eliminate double-dismiss - End Call / Cancel buttons only disconnect; endedOverlay auto-dismisses Co-Authored-By: Claude Opus 4.6 --- .../RelayInterface/Models/RelayError.swift | 9 ++++ Relay/RelayApp.swift | 5 +- Relay/Views/CallView.swift | 12 ++--- Relay/Views/CallWindowView.swift | 51 ++++++++++++++----- Relay/Views/MainView.swift | 16 ++++-- RelayKit/Services/TimelineMessageMapper.swift | 27 ++++++++++ 6 files changed, 94 insertions(+), 26 deletions(-) diff --git a/Packages/RelayInterface/Sources/RelayInterface/Models/RelayError.swift b/Packages/RelayInterface/Sources/RelayInterface/Models/RelayError.swift index 8ccaccd..0e1303e 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Models/RelayError.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Models/RelayError.swift @@ -108,6 +108,11 @@ public enum RelayError: LocalizedError, Sendable { /// A direct message room could not be opened or created. case dmCreationFailed(String) + // MARK: Calls + + /// A call could not be started. + case callFailed(String) + // MARK: LocalizedError public var errorDescription: String? { @@ -160,6 +165,8 @@ public enum RelayError: LocalizedError, Sendable { "Could Not Update Display Name" case .dmCreationFailed: "Could Not Open Conversation" + case .callFailed: + "Call Failed" } } @@ -213,6 +220,8 @@ public enum RelayError: LocalizedError, Sendable { reason case .dmCreationFailed(let reason): reason + case .callFailed(let reason): + reason } } } diff --git a/Relay/RelayApp.swift b/Relay/RelayApp.swift index f5a9c96..989c55a 100644 --- a/Relay/RelayApp.swift +++ b/Relay/RelayApp.swift @@ -124,10 +124,11 @@ struct RelayApp: App { .environment(\.matrixService, matrixService) .environment(\.callManager, callManager) } - .windowStyle(.plain) - .windowResizability(.contentSize) + .windowStyle(.hiddenTitleBar) + .windowResizability(.contentMinSize) .defaultSize(width: 360, height: 540) .defaultPosition(.topTrailing) + .defaultLaunchBehavior(.suppressed) } // MARK: - Notifications diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index 789c7a1..12b61bb 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -214,10 +214,9 @@ struct CallView: View { // End call Button { - Task { - await viewModel.disconnect() - onDismiss() - } + // Only disconnect — the endedOverlay auto-dismiss + // will call onDismiss() after a brief delay. + Task { await viewModel.disconnect() } } label: { Image(systemName: "phone.down.fill") .font(.title3) @@ -312,10 +311,7 @@ struct CallView: View { .font(.headline) .foregroundStyle(.white.opacity(0.7)) Button("Cancel") { - Task { - await viewModel.disconnect() - onDismiss() - } + Task { await viewModel.disconnect() } } .buttonStyle(.bordered) .foregroundStyle(.white) diff --git a/Relay/Views/CallWindowView.swift b/Relay/Views/CallWindowView.swift index 22d998f..d1cb7c0 100644 --- a/Relay/Views/CallWindowView.swift +++ b/Relay/Views/CallWindowView.swift @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import AppKit import RelayInterface import SwiftUI @@ -31,9 +32,16 @@ final class CallManager { func endCall() async { await activeCallViewModel?.disconnect() - activeCallViewModel = nil - isPreparingCredentials = false - callRoomId = nil + // Defer observable state teardown to the next run-loop iteration. + // Setting activeCallViewModel = nil swaps the entire CallWindowView + // body (CallView → Color.black). If that fires during an active + // AppKit layout pass it triggers a recursive constraint update crash + // on the main window. + DispatchQueue.main.async { [self] in + activeCallViewModel = nil + isPreparingCredentials = false + callRoomId = nil + } } } @@ -66,21 +74,17 @@ struct CallWindowView: View { viewModel: viewModel, isPreparingCredentials: callManager.isPreparingCredentials ) { - Task { - await callManager.endCall() - // Defer dismissal to the next run loop iteration so it - // doesn't fire during a SwiftUI layout pass, which causes - // a recursive constraint update crash. - DispatchQueue.main.async { - dismissWindow(id: "call") - } - } + // Only tear down state — the onChange handler below is the + // single dismiss path once hasActiveCall becomes false. + Task { await callManager.endCall() } } } else { // No active call — show placeholder until the window closes. Color.black } } + .ignoresSafeArea() + .background(WindowStyler()) .onChange(of: callManager.hasActiveCall) { _, hasCall in if !hasCall { DispatchQueue.main.async { @@ -90,3 +94,26 @@ struct CallWindowView: View { } } } + +// MARK: - Window Styler + +/// Configures the call window to have a fully transparent title bar with +/// content extending underneath it, while keeping the `.hiddenTitleBar` +/// window style for drag, resize, and window management support. +private struct WindowStyler: NSViewRepresentable { + func makeNSView(context: Context) -> NSView { StylerView() } + func updateNSView(_ nsView: NSView, context: Context) {} + + private class StylerView: NSView { + override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + guard let window else { return } + window.titlebarAppearsTransparent = true + window.isMovableByWindowBackground = true + // Hide the traffic light buttons. + window.standardWindowButton(.closeButton)?.isHidden = true + window.standardWindowButton(.miniaturizeButton)?.isHidden = true + window.standardWindowButton(.zoomButton)?.isHidden = true + } + } +} diff --git a/Relay/Views/MainView.swift b/Relay/Views/MainView.swift index ab351c7..cf1afcb 100644 --- a/Relay/Views/MainView.swift +++ b/Relay/Views/MainView.swift @@ -408,15 +408,23 @@ struct MainView: View { // swiftlint:disable:this type_body_length callManager.callRoomId = nil return } - callManager.activeCallViewModel = viewModel - openWindow(id: "call") + + // Defer the observable state change + window open to the next + // run-loop iteration. Setting activeCallViewModel invalidates + // the CallWindowView body across window boundaries; if that + // fires during an active layout pass the recursive constraint + // update crash occurs. + let openWindowAction = openWindow + DispatchQueue.main.async { + callManager.activeCallViewModel = viewModel + openWindowAction(id: "call") + } do { let creds = try await matrixService.callCredentials(for: roomId) try await viewModel.connect(url: creds.livekitURL, token: creds.token, sfuServiceURL: creds.sfuServiceURL) } catch { - // Credential fetch or connect failed — viewModel stays in .idle so - // CallView shows the manual-entry join form as a fallback. + errorReporter.report(.callFailed(error.localizedDescription)) } callManager.isPreparingCredentials = false } diff --git a/RelayKit/Services/TimelineMessageMapper.swift b/RelayKit/Services/TimelineMessageMapper.swift index 61d6a73..9b3c4a1 100644 --- a/RelayKit/Services/TimelineMessageMapper.swift +++ b/RelayKit/Services/TimelineMessageMapper.swift @@ -330,6 +330,11 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len )) } + // Deduplicate consecutive call events from the same sender. + // When a user ends a call, the removal state event (empty content) + // appears as a second "started a call" — filter those out. + result = Self.deduplicateCallEvents(result) + return MappingResult(messages: result, unresolvedReplyEventIds: pendingReplyFetchIds) } @@ -873,6 +878,28 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len } } + // MARK: - Call Event Deduplication + + /// Removes duplicate consecutive call events from the same sender. + /// + /// When a user ends a call, the MatrixRTC leave event (`{}` content) appears + /// in the timeline as a second "started a call" message from the same sender. + /// This filters out those duplicates, keeping only the first occurrence in each + /// consecutive run. + private static func deduplicateCallEvents(_ messages: [TimelineMessage]) -> [TimelineMessage] { + var result: [TimelineMessage] = [] + for message in messages { + if message.kind == .callEvent, + let last = result.last, + last.kind == .callEvent, + last.senderID == message.senderID { + continue + } + result.append(message) + } + return result + } + // MARK: - System Event Descriptions // swiftlint:disable cyclomatic_complexity From f3184584b97992f263aa00f84094f7cc40c22d61 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Mon, 20 Apr 2026 18:02:42 -0400 Subject: [PATCH 11/24] Fix LiveKit E2EE interop with Element Call (HKDF, identity, timing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Element Call rejected every encrypted frame from Relay even when key exchange and IKM fingerprints matched on both sides. Root causes: 1. PBKDF2 vs HKDF key derivation. The LiveKit Swift SDK's BaseKeyProvider forwards to an LKRTCFrameCryptorKeyProvider initializer that hard-codes PBKDF2, but livekit-client JS / Element Call derives the AES-GCM key with HKDF-SHA256 from the same raw IKM. Same fingerprint, different AES key, every auth tag fails on the peer. Fix: bump webrtc-xcframework to 144.7559.03 (which exposes the 7-arg ObjC init taking keyDerivationAlgorithm:) and client-sdk-swift to 2.13.0. Added CallEncryptionService.makeHKDFKeyProvider which uses the Objective-C runtime to construct an HKDF-backed LKRTCFrameCryptorKeyProvider and swap it into BaseKeyProvider's internal rtcKeyProvider ivar — no direct LiveKitWebRTC import needed. 2. LiveKit participant identity didn't match the Matrix identity peers used to look up our key. Construct it explicitly as "userId:deviceId" and warn if LiveKit hands back a different value. 3. Microphone was auto-publishing at connect time, so the first audio frames hit the SFU before peers received our key — their cryptor then ratcheted past the window and poisoned the slot. Defer mic/camera publish until after sendEncryptionKey completes. Co-Authored-By: Claude Opus 4.7 --- .../xcshareddata/swiftpm/Package.resolved | 8 +- Relay/Views/CallView.swift | 120 ++-- RelayKit/Call/CallEncryptionService.swift | 641 +++++++---------- RelayKit/Call/CallViewModel.swift | 284 +++++--- RelayKit/Call/CallWidgetBridge.swift | 657 ++++++++++++++++++ RelayKit/Services/MatrixService.swift | 25 +- RelayKit/Widget/WidgetProxy.swift | 28 - 7 files changed, 1201 insertions(+), 562 deletions(-) create mode 100644 RelayKit/Call/CallWidgetBridge.swift delete mode 100644 RelayKit/Widget/WidgetProxy.swift diff --git a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 3937f04..2330cd9 100644 --- a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -6,8 +6,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "revision" : "a25069f0e808d7e75f7cc93d157aa8e7cee3c58b", - "version" : "2.12.1" + "revision" : "4e930e856e3b076c2aacce98c77cc81fd2db498b", + "version" : "2.13.0" } }, { @@ -51,8 +51,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/webrtc-xcframework.git", "state" : { - "revision" : "0aa6a5ea4031d492d0493e3e4d4fbe08b5a0df78", - "version" : "137.7151.12" + "revision" : "e2a0ab3be155475ad60f845813f2088847e584f7", + "version" : "144.7559.03" } } ], diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index 12b61bb..e45d3a7 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -21,16 +21,28 @@ import SwiftUI /// When connected, the remote participant's video fills the window with /// a small self-view PiP overlay and a translucent floating control bar. struct CallView: View { - @State var viewModel: any CallViewModelProtocol + // `let` — not `@State`. The view model is a reference-typed + // `@Observable` class owned by `CallManager`; wrapping it in `@State` + // caused SwiftUI's `StoredLocationBase` to reinitialise the storage on + // every parent re-render of `CallWindowView`, which surfaces as + // recursive `StoredLocationBase.beginUpdate` calls during the layout + // transaction and eventually the "more Update Constraints in Window + // passes than there are views" fault. + let viewModel: any CallViewModelProtocol var isPreparingCredentials: Bool = false var onDismiss: () -> Void @State private var serverURL: String = "" @State private var accessToken: String = "" @State private var isJoining: Bool = false - @State private var isHovering: Bool = false - @State private var controlsVisible: Bool = true - @State private var hideControlsTask: Task? + // NOTE: The earlier implementation auto-hid the control bar after a + // timeout using a `controlsVisible` @State + `.animation(.easeInOut(..), + // value: controlsVisible)` on the control bar's opacity, plus a + // `.onHover` toggle. That produced the "more Update Constraints in + // Window passes than there are views" crash: the implicit animation + // pushed the AppKit `NSAnimationContext.runAnimationGroup` path which + // invalidated `StoredLocationBase` during an already-running layout + // pass on the hosting window. The control bar is now always visible. var body: some View { ZStack { @@ -98,38 +110,41 @@ struct CallView: View { Spacer() } - // Floating control bar at bottom + // Floating control bar at bottom (always visible). VStack { Spacer() controlBar - .opacity(controlsVisible ? 1 : 0) - .animation(.easeInOut(duration: 0.25), value: controlsVisible) } .padding(.bottom, 16) } - .onHover { hovering in - isHovering = hovering - if hovering { - showControls() - } else { - scheduleHideControls() - } - } - .onAppear { scheduleHideControls() } + // Disable ALL implicit animations within the connected view subtree. + // + // Removing the `.animation(...)` modifier on `controlBar` was not + // enough to stop the "more Update Constraints in Window passes than + // there are views" crash — SwiftUI still wraps structural changes + // (`if let firstRemote = ...`, `if let localID = ...`, + // `if viewModel.isLocalCameraEnabled`, `if first.isSpeaking`) in + // implicit transition animations during the connect sequence. Each + // of those animations runs through `NSAnimationContext.runAnimationGroup` + // inside `NSHostingView.layout`, writing back into the SwiftUI graph + // and queueing another constraint pass on the same frame — eventually + // exceeding the view-count budget and tripping the AppKit fault. + // + // `.transaction { $0.animation = nil }` strips the animation off + // every transaction propagated through this subtree, so structural + // changes happen instantly with no animator running during layout. + .transaction { $0.animation = nil } } // MARK: - Primary Video @ViewBuilder private var primaryVideo: some View { - let _ = viewModel.videoTrackRevision if let firstRemote = viewModel.participants.first { - if let videoView = viewModel.makeVideoView(for: firstRemote.id) { - videoView - } else { - // Remote has no video — show avatar placeholder + VideoRendererView(viewModel: viewModel, participantID: firstRemote.id) { participantPlaceholder(firstRemote) } + .id(firstRemote.id) } else { // No remote participants yet — waiting VStack(spacing: 12) { @@ -147,14 +162,17 @@ struct CallView: View { @ViewBuilder private func selfViewPiP(id: String) -> some View { - let _ = viewModel.videoTrackRevision ZStack { RoundedRectangle(cornerRadius: 10, style: .continuous) .fill(Color(nsColor: .darkGray)) - if viewModel.isLocalCameraEnabled, - let videoView = viewModel.makeVideoView(for: id) { - videoView + if viewModel.isLocalCameraEnabled { + VideoRendererView(viewModel: viewModel, participantID: id) { + Image(systemName: "person.fill") + .font(.title2) + .foregroundStyle(.white.opacity(0.5)) + } + .id(id) } else { Image(systemName: "person.fill") .font(.title2) @@ -248,23 +266,6 @@ struct CallView: View { .help(help) } - // MARK: - Controls Visibility - - private func showControls() { - hideControlsTask?.cancel() - controlsVisible = true - } - - private func scheduleHideControls() { - hideControlsTask?.cancel() - hideControlsTask = Task { - try? await Task.sleep(for: .seconds(3)) - if !Task.isCancelled && !isHovering { - controlsVisible = false - } - } - } - // MARK: - Participant Placeholder @ViewBuilder @@ -416,6 +417,41 @@ struct CallView: View { } } +// MARK: - Video Renderer + +/// Isolates video-track observation into its own SwiftUI view so that +/// `videoTrackRevision` changes only invalidate THIS subtree rather than +/// the entire ``CallView`` hierarchy. +/// +/// Previously `primaryVideo` and `selfViewPiP` both read +/// `viewModel.videoTrackRevision` directly in their view bodies, which +/// registered dependencies on the whole containing ZStack. Each track +/// update (camera publish, subscribe, etc.) then re-laid-out every +/// sibling view — and because the rendered video is an `NSViewRepresentable` +/// wrapping an AppKit `VideoView`, that triggered recursive +/// `setNeedsUpdateConstraints` calls on the hosting window, producing the +/// "more Update Constraints in Window passes than there are views" hang. +/// +/// The `.id(participantID)` modifier on each usage site gives SwiftUI a +/// stable identity key so the renderer is reused across parent re-renders +/// instead of being torn down and recreated. +private struct VideoRendererView: View { + let viewModel: any CallViewModelProtocol + let participantID: String + @ViewBuilder let placeholder: () -> Placeholder + + var body: some View { + // Reading videoTrackRevision here registers observation *only* on + // this subtree — it is not read in any enclosing view. + let _ = viewModel.videoTrackRevision + if let videoView = viewModel.makeVideoView(for: participantID) { + videoView + } else { + placeholder() + } + } +} + // MARK: - Previews #Preview("Preparing") { diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift index d5c97e9..1d62ab5 100644 --- a/RelayKit/Call/CallEncryptionService.swift +++ b/RelayKit/Call/CallEncryptionService.swift @@ -12,27 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. +import CryptoKit import Foundation import LiveKit +import MatrixRustSDK import OSLog private let logger = Logger(subsystem: "RelayKit", category: "CallEncryption") -/// Manages MatrixRTC E2EE key exchange for LiveKit calls. +/// Helpers for MatrixRTC call-member state signaling, power-level bootstrap, +/// and LiveKit key provider plumbing. /// -/// Implements the key distribution side of MSC4143 / Element Call's encryption -/// protocol: generates a random 16-byte AES-GCM key for the local participant, -/// distributes it to other participants via Matrix to-device messages, and sets -/// raw key material on the LiveKit `BaseKeyProvider` so that SFrame encryption -/// uses the correct bytes. +/// Key distribution for `io.element.call.encryption_keys` is handled by +/// ``CallWidgetBridge``, which speaks the Widget API directly to the +/// Matrix Rust SDK's `WidgetDriver`. The SDK handles Olm encryption of the +/// to-device payloads transparently, which the previous raw-REST path could +/// not do — Element-X rejected the plaintext keys and the call failed to +/// negotiate. /// -/// ## Key Exchange Flow -/// 1. On connect, generate a 16-byte random key. -/// 2. Set the key on the local participant's LiveKit encryptor via `BaseKeyProvider`. -/// 3. Send the key (base64-encoded) to all other devices in the room using the -/// `io.element.call.encryption_keys` to-device event type. -/// 4. When receiving keys from other participants (via `/sync`), set them on the -/// `BaseKeyProvider` for the corresponding participant identity. +/// What remains in this type: +/// - ``sendCallMemberEvent(sfuServiceURL:)`` / ``removeCallMemberEvent()`` — +/// MatrixRTC member state via `sendStateEventRaw` on the SDK room. +/// - ``enableCallPowerLevels()`` — ensures call state events are sendable +/// at PL 0 so ordinary members can join. +/// - ``generateKey()`` / ``setRawKey(_:on:participantId:index:)`` — +/// LiveKit `BaseKeyProvider` plumbing that bypasses the String-based +/// `setKey(...)` API so raw AES bytes are installed unmangled. struct CallEncryptionService { let homeserver: String @@ -40,6 +45,9 @@ struct CallEncryptionService { let userID: String let deviceID: String let roomID: String + /// The Matrix SDK room, used for `sendStateEventRaw` which goes through + /// the SDK's authenticated client instead of raw REST API calls. + let sdkRoom: MatrixRustSDK.Room? /// The to-device event type used by Element Call for key exchange. static let encryptionKeysEventType = "io.element.call.encryption_keys" @@ -50,30 +58,29 @@ struct CallEncryptionService { // MARK: - Call Membership Signaling - /// Sends the MatrixRTC call membership state event so that Element-X and other - /// MatrixRTC clients can discover our participation in the call. + /// Sends the MatrixRTC call membership state event so that Element-X and + /// other MatrixRTC clients can discover our participation in the call. /// /// Uses the modern MSC4143 per-device format matching Element-X: /// - State key: `_@userId:server_deviceId_m.call` /// - `focus_active`: `{"type": "livekit", "focus_selection": "oldest_membership"}` /// - `foci_preferred`: array with the SFU service URL and room alias /// - /// - Parameter sfuServiceURL: The SFU service URL from MatrixRTC discovery - /// (e.g. `https://livekit.example.com/livekit/jwt`). - func sendCallMemberEvent(sfuServiceURL: String) async throws { - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - let encodedEventType = Self.callMemberEventType - .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType - // MSC4143 state key: "_@userId:server_deviceId_m.call" - let stateKey = "_\(userID)_\(deviceID)_m.call" - let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey - - guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { - throw LiveKitCredentialError.invalidURL + /// - Parameters: + /// - sfuServiceURL: The SFU service URL from MatrixRTC discovery + /// (e.g. `https://livekit.example.com/livekit/jwt`). + /// - membershipId: The per-call membership UUID. Must match the + /// `member.id` field in outbound encryption_keys to-device payloads + /// so peers can correlate our key with our membership event. When + /// `nil`, falls back to `userID:deviceID`. + func sendCallMemberEvent(sfuServiceURL: String, membershipId: String? = nil) async throws { + guard let sdkRoom else { + throw CallEncryptionError.callMemberEventFailed } + let stateKey = "_\(userID)_\(deviceID)_m.call" let serviceURL = sfuServiceURL.trimmingCharacters(in: .init(charactersIn: "/")) + let membership = membershipId ?? "\(userID):\(deviceID)" // Match Element-X's exact format. let body: [String: Any] = [ @@ -93,65 +100,35 @@ struct CallEncryptionService { ] as [String: Any] ], "m.call.intent": "video", - "membershipID": "\(userID):\(deviceID)", + "membershipID": membership, "scope": "m.room" ] let jsonData = try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) - if let jsonStr = String(data: jsonData, encoding: .utf8) { - logger.info("Call member event body: \(jsonStr)") - } + let jsonString = String(data: jsonData, encoding: .utf8) ?? "{}" + logger.info("Call member event body: \(jsonString)") logger.info("Call member state key: \(stateKey)") - var request = URLRequest(url: url) - request.httpMethod = "PUT" - request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpBody = jsonData - - let (responseData, response) = try await URLSession.shared.data(for: request) - guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { - let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 - if let respStr = String(data: responseData, encoding: .utf8) { - logger.error("sendCallMemberEvent failed with status \(statusCode): \(respStr)") - } - throw CallEncryptionError.callMemberEventFailed - } - + _ = try await sdkRoom.sendStateEventRaw( + eventType: Self.callMemberEventType, + stateKey: stateKey, + content: jsonString + ) logger.info("Sent call membership state event") } /// Removes the call membership state event (sets content to empty object) /// so Element-X knows we've left the call. func removeCallMemberEvent() async throws { - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - let encodedEventType = Self.callMemberEventType - .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.callMemberEventType - let stateKey = "_\(userID)_\(deviceID)_m.call" - let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey - - guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { - throw LiveKitCredentialError.invalidURL - } - - let body: [String: Any] = [:] - - let jsonData = try JSONSerialization.data(withJSONObject: body) - - var request = URLRequest(url: url) - request.httpMethod = "PUT" - request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpBody = jsonData - - let (_, response) = try await URLSession.shared.data(for: request) - guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { - let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 - logger.error("removeCallMemberEvent failed with status \(statusCode)") - return + guard let sdkRoom else { + throw CallEncryptionError.callMemberEventFailed } - + let stateKey = "_\(userID)_\(deviceID)_m.call" + _ = try await sdkRoom.sendStateEventRaw( + eventType: Self.callMemberEventType, + stateKey: stateKey, + content: "{}" + ) logger.info("Removed call membership state event") } @@ -186,81 +163,120 @@ struct CallEncryptionService { } } - // MARK: - Room Call Setup - - /// Ensures the room's power levels allow any member to send call-related - /// state events. Element-web does this automatically when a call is started. - /// - /// Sets `org.matrix.msc3401.call.member` and `io.element.call.encryption_keys` - /// to power level 0 in the room's `m.room.power_levels` state event. + /// Returns a `userId -> [deviceId]` map of *other* users currently in the + /// call, parsed from `org.matrix.msc3401.call.member` state events. /// - /// This is idempotent — if the levels are already correct, the PUT overwrites - /// with the same content. - func enableCallPowerLevels() async throws { + /// Element-X writes per-device call-member events with state key + /// `___m.call`. We walk the full room state, filter for + /// non-empty call-member content (empty content means the participant + /// has left), and extract `(userId, deviceId)` from the state key. + /// Our own `userID` is excluded. + func fetchCallTargets() async -> [String: [String]] { let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - // 1. Fetch current power levels. - guard let getURL = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/m.room.power_levels/") else { - throw LiveKitCredentialError.invalidURL - } + guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state") else { return [:] } - var getRequest = URLRequest(url: getURL) - getRequest.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") + var request = URLRequest(url: url) + request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - let (data, getResponse) = try await URLSession.shared.data(for: getRequest) - guard let http = getResponse as? HTTPURLResponse, http.statusCode == 200 else { - logger.warning("Could not fetch power levels (status \((getResponse as? HTTPURLResponse)?.statusCode ?? -1))") - return + guard let (data, response) = try? await URLSession.shared.data(for: request), + let http = response as? HTTPURLResponse, http.statusCode == 200, + let events = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] else { + return [:] } - guard var powerLevels = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { - return + var targets: [String: Set] = [:] + for event in events { + guard let type = event["type"] as? String, + type == Self.callMemberEventType, + let stateKey = event["state_key"] as? String, + let content = event["content"] as? [String: Any], + !content.isEmpty else { continue } + + // State key format: `___m.call` where userId is + // itself `@localpart:server.tld`. Strip the leading underscore + // and the trailing `_m.call` marker, then split on the *last* + // underscore to separate deviceId from userId. + guard stateKey.hasPrefix("_"), stateKey.hasSuffix("_m.call") else { continue } + let trimmed = String(stateKey.dropFirst().dropLast("_m.call".count)) + guard let lastUnderscore = trimmed.lastIndex(of: "_") else { continue } + let userId = String(trimmed[.. BaseKeyProvider { + let options = KeyProviderOptions( + sharedKey: false, + ratchetWindowSize: ratchetWindowSize, + keyRingSize: keyRingSize + ) + let provider = BaseKeyProvider(options: options) + + guard let cls = NSClassFromString("LKRTCFrameCryptorKeyProvider") as? NSObject.Type else { + logger.error("LKRTCFrameCryptorKeyProvider class not found at runtime; HKDF swap skipped — E2EE interop with Element Call will fail (PBKDF2 vs HKDF mismatch)") + return provider + } + + let initSel = NSSelectorFromString( + "initWithRatchetSalt:ratchetWindowSize:sharedKeyMode:uncryptedMagicBytes:failureTolerance:keyRingSize:discardFrameWhenCryptorNotReady:keyDerivationAlgorithm:" + ) + // Swift blocks `NSObject.alloc()`, so go through the ObjC runtime. + let allocSel = NSSelectorFromString("alloc") + typealias AllocFunc = @convention(c) (AnyClass, Selector) -> AnyObject + let allocImp = unsafeBitCast( + (cls as AnyClass).method(for: allocSel), + to: AllocFunc.self + ) + let allocated = allocImp(cls, allocSel) + guard (allocated as AnyObject).responds(to: initSel) else { + logger.error("LKRTCFrameCryptorKeyProvider does not expose keyDerivationAlgorithm: init; webrtc-xcframework may be < 144.x — falling back to PBKDF2 (Element Call interop will fail)") + return provider + } + + typealias InitFunc = @convention(c) ( + AnyObject, Selector, NSData, Int32, ObjCBool, NSData?, Int32, Int32, ObjCBool, UInt + ) -> AnyObject + let imp = unsafeBitCast( + (allocated as AnyObject).method(for: initSel), + to: InitFunc.self + ) + // RTCKeyDerivationAlgorithmHKDF is the second enum case (== 1). + let hkdfKeyDerivation: UInt = 1 + let hkdfRtc = imp( + allocated, + initSel, + options.ratchetSalt as NSData, + options.ratchetWindowSize, + ObjCBool(options.sharedKey), + options.uncryptedMagicBytes as NSData, + options.failureTolerance, + options.keyRingSize, + ObjCBool(false), + hkdfKeyDerivation + ) + + guard let ivar = class_getInstanceVariable(BaseKeyProvider.self, "rtcKeyProvider") else { + logger.error("rtcKeyProvider ivar not found on BaseKeyProvider; HKDF swap skipped") + return provider + } + object_setIvar(provider, ivar, hkdfRtc) + logger.info("Installed HKDF-backed LKRTCFrameCryptorKeyProvider (Element Call interop path)") + return provider + } + /// Sets a raw key on a `BaseKeyProvider` for the given participant, bypassing /// the String-based `setKey(key:participantId:index:)` method which would /// UTF-8-encode the string (wrong for raw AES key bytes). @@ -309,7 +409,14 @@ struct CallEncryptionService { to: SetKeyFunc.self ) imp(rtcProvider, selector, keyData as NSData, index, participantId as NSString) - logger.info("Set raw encryption key for participant \(participantId, privacy: .private) at index \(index)") + // SHA-256 fingerprint of the raw IKM so we can confirm the exact same + // 16 bytes end up on the wire. Matches the fingerprint logged in + // CallWidgetBridge.sendEncryptionKey. Diverging fingerprints mean + // our local frame cryptor and the peer are using different keys — + // the #1 root cause of "maximum ratchet attempts exceeded" on an + // otherwise-correct key-exchange handshake. + let fp = SHA256.hash(data: keyData).prefix(8).map { String(format: "%02x", $0) }.joined() + logger.info("Set raw encryption key for participant \(participantId, privacy: .public) at index \(index) bytes=\(keyData.count) sha256[0..8]=\(fp, privacy: .public)") } /// Convenience: sets a raw key using base64-encoded key data. @@ -325,295 +432,15 @@ struct CallEncryptionService { } setRawKey(keyData, on: keyProvider, participantId: participantId, index: index) } - - // MARK: - Key Distribution (to-device messages) - - /// Sends the local participant's encryption key to all devices of the given - /// users via a Matrix to-device message. - /// - /// Uses the REST API directly because the Matrix Rust SDK (v26.x) does not - /// expose `sendToDevice` in the Swift FFI. - /// - /// - Parameters: - /// - key: The raw 16-byte encryption key. - /// - keyIndex: The key index (0-255, cycles on ratchet). - /// - targetUsers: A mapping of user ID to an array of device IDs. - func sendKey( - _ key: Data, - keyIndex: Int, - to targetUsers: [String: [String]] - ) async throws { - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let txnId = UUID().uuidString - let encodedEventType = Self.encryptionKeysEventType - .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.encryptionKeysEventType - - guard let url = URL(string: "\(base)/_matrix/client/v3/sendToDevice/\(encodedEventType)/\(txnId)") else { - throw LiveKitCredentialError.invalidURL - } - - let base64Key = key.base64EncodedString() - let sentTs = Int(Date().timeIntervalSince1970 * 1000) - - // Build the per-user/per-device message content. - var messages: [String: [String: Any]] = [:] - for (userId, deviceIds) in targetUsers { - var deviceMessages: [String: Any] = [:] - for deviceId in deviceIds { - deviceMessages[deviceId] = [ - "keys": [ - ["index": keyIndex, "key": base64Key] - ], - "room_id": roomID, - "member": [ - "claimed_device_id": self.deviceID, - "id": "\(self.userID):\(self.deviceID)" - ], - "session": [ - "call_id": "", - "application": "m.call", - "scope": "m.room" - ], - "sent_ts": sentTs - ] as [String: Any] - } - messages[userId] = deviceMessages - } - - let body: [String: Any] = ["messages": messages] - let jsonData = try JSONSerialization.data(withJSONObject: body) - - var request = URLRequest(url: url) - request.httpMethod = "PUT" - request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpBody = jsonData - - let (_, response) = try await URLSession.shared.data(for: request) - guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { - let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 - logger.error("sendToDevice failed with status \(statusCode)") - throw CallEncryptionError.keyDistributionFailed - } - - logger.info("Sent encryption key (index \(keyIndex)) to \(targetUsers.count) user(s)") - } - - // MARK: - Key Distribution (room state events) - - /// Sends the local participant's encryption key as a room state event. - /// - /// This provides a second transport for key exchange that other Relay clients - /// can observe via the room timeline, working around the Matrix Rust SDK's - /// inability to deliver to-device events to the app layer. - /// - /// The state key is `"{userID}:{deviceID}"` so each participant's key is - /// a distinct state entry that overwrites on update. - func sendKeyAsStateEvent( - _ key: Data, - keyIndex: Int - ) async throws { - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - let stateKey = "\(userID):\(deviceID)" - let encodedStateKey = stateKey.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? stateKey - let encodedEventType = Self.encryptionKeysEventType - .addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? Self.encryptionKeysEventType - - guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/\(encodedEventType)/\(encodedStateKey)") else { - throw LiveKitCredentialError.invalidURL - } - - let base64Key = key.base64EncodedString() - let body: [String: Any] = [ - "keys": [ - ["index": keyIndex, "key": base64Key] - ], - "member": [ - "claimed_device_id": deviceID, - "id": "\(userID):\(deviceID)" - ], - "session": [ - "call_id": "", - "application": "m.call", - "scope": "m.room" - ] - ] - - let jsonData = try JSONSerialization.data(withJSONObject: body) - - var request = URLRequest(url: url) - request.httpMethod = "PUT" - request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - request.httpBody = jsonData - - let (_, response) = try await URLSession.shared.data(for: request) - guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else { - let statusCode = (response as? HTTPURLResponse)?.statusCode ?? -1 - logger.error("sendStateEvent failed with status \(statusCode)") - throw CallEncryptionError.keyDistributionFailed - } - - logger.info("Sent encryption key as state event (index \(keyIndex))") - } - - // MARK: - Key Reception (timeline listener) - - /// Starts listening for encryption key state events on the room timeline. - /// - /// When another participant sends their key as a room state event of type - /// `io.element.call.encryption_keys`, this listener parses the key and sets - /// it on the given `BaseKeyProvider` so LiveKit can decrypt that participant's - /// media frames. - /// - /// - Parameters: - /// - timeline: The Matrix SDK `Timeline` for the call's room. - /// - keyProvider: The LiveKit key provider to set received keys on. - /// - localIdentity: The local participant's identity (to skip our own events). - /// - Returns: A `TaskHandle` that must be retained to keep the listener alive. - @MainActor - static func startListeningForKeys( - timeline: Timeline, - keyProvider: BaseKeyProvider, - localIdentity: String - ) async -> TaskHandle { - // Capture the event type as a local to avoid referencing the MainActor-isolated - // static property from the nonisolated SDKListener callback. - let eventType = Self.encryptionKeysEventType - let localPrefix = localIdentity.components(separatedBy: ":").prefix(2).joined(separator: ":") - - let listener = SDKListener<[TimelineDiff]> { diffs in - // SDKListener callbacks arrive on an unspecified thread. - // Dispatch to the main actor for safe access to logger and setRawKey. - Task { @MainActor in - let items = extractTimelineItems(from: diffs) - for item in items { - guard let eventItem = item.asEvent() else { continue } - - guard case .state(let stateKey, let otherState) = eventItem.content, - case .custom(let type) = otherState, - type == eventType else { - continue - } - - // Skip our own events. - if stateKey.hasPrefix(localPrefix) { continue } - - let sender = eventItem.sender - - let debugInfo = eventItem.lazyProvider.debugInfo() - guard let jsonString = debugInfo.originalJson, - let jsonData = jsonString.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { - logger.warning("Could not parse encryption key event JSON from \(sender, privacy: .private)") - continue - } - - // The raw JSON is the full event envelope; keys are in "content". - let content = (json["content"] as? [String: Any]) ?? json - - guard let keysArray = content["keys"] as? [[String: Any]] else { - logger.warning("No keys array in encryption key event from \(sender, privacy: .private)") - continue - } - - let participantIdentity = stateKey - - for keyEntry in keysArray { - guard let base64Key = keyEntry["key"] as? String, - let index = keyEntry["index"] as? Int else { - continue - } - Self.setRawKey( - base64Key: base64Key, - on: keyProvider, - participantId: participantIdentity, - index: Int32(index) - ) - logger.info("Received encryption key from \(sender, privacy: .private) (index \(index))") - } - } - } - } - - return await timeline.addListener(listener: listener) - } - - // MARK: - Room Member Discovery - - /// Fetches the list of joined members in the room so we know who to send - /// encryption keys to. Uses the Matrix REST API directly. - func fetchJoinedMembers() async throws -> [String: [String]] { - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - guard let url = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/joined_members") else { - throw LiveKitCredentialError.invalidURL - } - - var request = URLRequest(url: url) - request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - - let (data, response) = try await URLSession.shared.data(for: request) - guard let http = response as? HTTPURLResponse, http.statusCode == 200 else { - throw CallEncryptionError.memberDiscoveryFailed - } - - // Response: { "joined": { "@user:server": { ... }, ... } } - guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any], - let joined = json["joined"] as? [String: Any] else { - throw CallEncryptionError.memberDiscoveryFailed - } - - // For now, return each user mapped to "*" (wildcard) since we don't have - // per-device granularity from joined_members. The homeserver will fan out. - var result: [String: [String]] = [:] - for userId in joined.keys where userId != self.userID { - result[userId] = ["*"] - } - return result - } -} - -// MARK: - Helpers - -/// Extracts all `TimelineItem` values from a batch of timeline diffs. -private func extractTimelineItems(from diffs: [TimelineDiff]) -> [TimelineItem] { - var items: [TimelineItem] = [] - for diff in diffs { - switch diff { - case .append(let values): - items.append(contentsOf: values) - case .pushFront(let value): - items.append(value) - case .pushBack(let value): - items.append(value) - case .insert(_, let value): - items.append(value) - case .set(_, let value): - items.append(value) - case .reset(let values): - items.append(contentsOf: values) - case .clear, .popFront, .popBack, .remove, .truncate: - break - } - } - return items } // MARK: - Errors enum CallEncryptionError: LocalizedError { - case keyDistributionFailed - case memberDiscoveryFailed case callMemberEventFailed var errorDescription: String? { switch self { - case .keyDistributionFailed: - return "Failed to distribute encryption keys to call participants." - case .memberDiscoveryFailed: - return "Failed to discover room members for key exchange." case .callMemberEventFailed: return "Failed to send call membership state event." } diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index d4bcbb5..58e0e2a 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -41,29 +41,57 @@ public final class CallViewModel: CallViewModelProtocol { /// re-evaluate `videoContent(for:)` and pick up new or removed tracks. public private(set) var videoTrackRevision: UInt = 0 + @ObservationIgnored private let room = LiveKit.Room() + @ObservationIgnored private var delegate: Delegate? /// Cached video views keyed by participant ID, to avoid recreating /// `SwiftUIVideoView` on every SwiftUI re-render. Each entry stores /// the `ObjectIdentifier` of the `VideoTrack` so the cache is /// invalidated when the underlying track actually changes. + /// + /// `@ObservationIgnored` is critical: without it, the `@Observable` + /// macro tracks writes to this cache, and because `makeVideoView` is + /// called directly from SwiftUI view bodies, any cache mutation during + /// body evaluation triggers an invalidation which re-runs the body + /// which re-mutates the cache — leading to a constraint-pass crash: + /// "more Update Constraints in Window passes than there are views". + @ObservationIgnored private var videoViewCache: [String: (trackObjectID: ObjectIdentifier, view: AnyView)] = [:] // MARK: - E2EE State + // + // All of these are implementation details — no SwiftUI view reads + // them. Marking them `@ObservationIgnored` keeps their writes out of + // the observation registrar, which eliminates a class of stray + // invalidations that otherwise pile up during call startup when + // `connect()` writes the key, members, and bridge in rapid succession + // on the main actor. /// The LiveKit key provider used for per-participant AES-GCM frame encryption. + @ObservationIgnored private var keyProvider: BaseKeyProvider? /// The local participant's current encryption key (raw 16 bytes). + @ObservationIgnored private var localEncryptionKey: Data? /// The current key index (0-255, wraps around on ratchet). + @ObservationIgnored private var localKeyIndex: Int = 0 - /// Service for distributing encryption keys via Matrix to-device messages. + /// Service for MatrixRTC call-member signaling and LiveKit key plumbing. + @ObservationIgnored private var encryptionService: CallEncryptionService? - /// The Matrix SDK room, used to obtain the timeline for key listening. + /// The Matrix SDK room, used for the widget bridge. + @ObservationIgnored private var matrixRoom: MatrixRustSDK.Room? - /// Handle for the timeline key listener; retained to keep the subscription alive. - private var keyListenerHandle: TaskHandle? + /// Headless widget-driver bridge that handles Olm-encrypted key exchange + /// via the Matrix Widget API. Nil until `connect(...)` completes setup. + @ObservationIgnored + private var widgetBridge: CallWidgetBridge? + /// Cached user/device map of known call members, rebuilt from + /// MatrixRTC member state events. + @ObservationIgnored + private var callMembers: [String: [String]] = [:] /// Creates a call view model without E2EE. Use ``init(encryptionContext:)`` /// for encrypted calls that interoperate with Element Call. @@ -118,13 +146,29 @@ public final class CallViewModel: CallViewModelProtocol { accessToken: encryptionContext.accessToken, userID: encryptionContext.userID, deviceID: encryptionContext.deviceID, - roomID: encryptionContext.roomID + roomID: encryptionContext.roomID, + sdkRoom: encryptionContext.matrixRoom ) if encryptionContext.isRoomEncrypted { // Per-participant key provider: each participant has their own key. - let provider = BaseKeyProvider(isSharedKey: false) - self.keyProvider = provider + // Match Element Call's MatrixKeyProvider configuration so the JS + // LiveKit E2EE worker doesn't exhaust its ratchet window trying to + // decrypt our frames. Swift BaseKeyProvider defaults are + // ratchetWindowSize: 0, keyRingSize: 16; Element Call uses 10/256. + // + // Additionally: swap in an HKDF-SHA256-backed + // LKRTCFrameCryptorKeyProvider. The LiveKit Swift SDK's default + // initializer path constructs the ObjC provider with PBKDF2 + // (libwebrtc's default), but Element Call / livekit-client JS + // derives the AES-GCM key with HKDF from the same raw IKM — + // so the two sides produce different AES keys from matching + // fingerprints, and every frame's auth tag fails on the peer. + // See CallEncryptionService.makeHKDFKeyProvider for details. + self.keyProvider = CallEncryptionService.makeHKDFKeyProvider( + ratchetWindowSize: 10, + keyRingSize: 256 + ) } self.matrixRoom = encryptionContext.matrixRoom } @@ -134,9 +178,15 @@ public final class CallViewModel: CallViewModelProtocol { public func connect(url: String, token: String, sfuServiceURL: String = "") async throws { state = .connecting do { + // Microphone publish is deferred until AFTER the local E2EE key + // has been installed and distributed to peers. If we let + // LiveKit auto-publish the mic at connect time, the first + // audio frames hit the SFU before peers receive our key — + // their frame cryptor then ratchets past its window and + // poisons the key slot. let connectOpts = ConnectOptions( autoSubscribe: true, - enableMicrophone: true + enableMicrophone: false ) // Enable LiveKit-level GCM frame encryption only for encrypted Matrix @@ -169,71 +219,129 @@ public final class CallViewModel: CallViewModelProtocol { roomOptions: roomOpts ) localParticipantID = room.localParticipant.identity?.stringValue - logger.info("Connected as \(self.localParticipantID ?? "unknown")") - - // Ensure call power levels are set so any room member can join, - // then send the MatrixRTC call membership state event. - if let encryptionService { - Task { - // Debug: log existing call member events to compare formats. - await encryptionService.fetchCallMemberEvents() - - do { - try await encryptionService.enableCallPowerLevels() - } catch { - logger.warning("Call power level setup failed: \(error.localizedDescription)") - } - do { - try await encryptionService.sendCallMemberEvent(sfuServiceURL: sfuServiceURL) - } catch { - logger.warning("Call membership event failed: \(error.localizedDescription)") - } + logger.info("Connected with LiveKit identity: \(self.localParticipantID ?? "unknown", privacy: .public)") + + // Spin up the headless widget bridge *only* for encrypted rooms. + // For unencrypted rooms the bridge adds no value (no keys to + // exchange) and materialising a virtual Element-Call widget on + // a room Element-X is already observing causes Element-X to + // stall before joining the LiveKit SFU. + if self.isE2eeEnabled, let matrixRoom, let encryptionService { + do { + let bridge = try CallWidgetBridge( + room: matrixRoom, + ownUserId: encryptionService.userID, + ownDeviceId: encryptionService.deviceID, + isRoomEncrypted: true, + keyProvider: self.keyProvider + ) + bridge.start() + self.widgetBridge = bridge + } catch { + logger.error("Failed to create CallWidgetBridge: \(error.localizedDescription)") } } - // Generate and distribute the local E2EE key when the room is encrypted. - if isE2eeEnabled, let keyProvider, let encryptionService { + // CRITICAL: Register the local E2EE key in the keyProvider + // BEFORE publishing any media tracks. LiveKit begins encrypting + // frames the instant `setCamera(enabled: true)` attaches the + // track, so if the key isn't installed yet the first batch of + // frames is encrypted with nothing the remote peer can decrypt — + // and Element-X's video decoder stalls on that first undecodable + // frame, resulting in perpetual black video. + if self.isE2eeEnabled, let keyProvider = self.keyProvider, let encryptionService { let key = CallEncryptionService.generateKey() - localEncryptionKey = key - - let localIdentity = localParticipantID ?? encryptionService.userID + self.localEncryptionKey = key + // Legacy `m.call.member` rtcBackendIdentity is always + // `${sender}:${device_id}` (matrix-js-sdk CallMembership.ts + // line 101). This is what remote peers route our frames under, + // so our local sender cryptor MUST be keyed under the same + // byte sequence — do not trust `localParticipantID` (the + // identity LiveKit assigns from the SFU JWT), since a + // mismatched JWT identity would silently break decrypt. + let localIdentity = "\(encryptionService.userID):\(encryptionService.deviceID)" + if let livekitIdentity = self.localParticipantID, livekitIdentity != localIdentity { + logger.warning("LiveKit identity \(livekitIdentity, privacy: .public) != matrix identity \(localIdentity, privacy: .public) — frame encryption may misroute") + } + let keyIndex = self.localKeyIndex CallEncryptionService.setRawKey( key, on: keyProvider, participantId: localIdentity, - index: Int32(localKeyIndex) + index: Int32(keyIndex) ) - logger.info("Local E2EE key set (index \(self.localKeyIndex))") + logger.info("Local E2EE key set (index \(keyIndex)) under participantId=\(localIdentity, privacy: .public) before camera publish") + } - // Distribute key via both transports (best-effort, don't block connect). - Task { - do { - let members = try await encryptionService.fetchJoinedMembers() - if !members.isEmpty { - try await encryptionService.sendKey(key, keyIndex: localKeyIndex, to: members) - } - } catch { - logger.warning("To-device key distribution failed: \(error.localizedDescription)") - } + // Set up MatrixRTC signaling and distribute the key **before** + // publishing media. LiveKit begins encrypting the instant + // `setCamera(enabled: true)` attaches the track; if frames reach + // peers before our key does, their LiveKit frame cryptor + // ratchets in the dark, blows through its `ratchetWindowSize` + // (10) worth of failures, and calls `markInvalid()` on index 0 + // — poisoning the slot so our late-arriving key is rejected + // even though the raw IKM is correct. The original ordering ran + // this in a background Task racing `setCamera`, which is + // exactly that bug. + // + // Order: power levels → member state (so peers see us) → + // deliver key via Olm-encrypted to-device → THEN publish media. + // Failures here are logged but non-fatal — a late key is still + // better than no key. + if let encryptionService { + let bridge = self.widgetBridge + let localKey = self.localEncryptionKey + let keyIndex = self.localKeyIndex + + // Debug: log existing call member events to compare formats. + await encryptionService.fetchCallMemberEvents() + + // 1. Try to fix power levels (only works if we're admin/mod). + do { + try await encryptionService.enableCallPowerLevels() + } catch { + logger.warning("Call power level setup failed: \(error.localizedDescription)") + } + // 2. Send call membership state event (after power levels). + // Pass the widget bridge's membershipId UUID so the + // state-event `membershipID` matches the `member.id` + // field in our outbound encryption_keys payloads. + do { + try await encryptionService.sendCallMemberEvent( + sfuServiceURL: sfuServiceURL, + membershipId: bridge?.membershipId + ) + } catch { + logger.warning("Call membership event failed: \(error.localizedDescription)") + } + + // 3. Distribute the already-generated local key via the + // widget bridge. The `messages` map for the + // `send_to_device` action requires an explicit + // `{ userId: [deviceId, ...] }` map of recipients, so we + // parse it from the `org.matrix.msc3401.call.member` + // state events already present on the room. The SDK + // then Olm-encrypts the payload per-device. + if self.isE2eeEnabled, let bridge, let localKey { + let targets = await encryptionService.fetchCallTargets() + self.callMembers = targets + logger.info("Distributing key to \(targets.count) remote user(s) BEFORE media publish") do { - try await encryptionService.sendKeyAsStateEvent(key, keyIndex: localKeyIndex) + try await bridge.sendEncryptionKey( + localKey, + keyIndex: keyIndex, + toMembers: targets + ) } catch { - logger.warning("State event key distribution failed: \(error.localizedDescription)") + logger.warning("Widget-bridge key distribution failed: \(error.localizedDescription)") } } - - // Start listening for inbound encryption keys from other participants. - if let timeline = try? await matrixRoom?.timeline() { - keyListenerHandle = await CallEncryptionService.startListeningForKeys( - timeline: timeline, - keyProvider: keyProvider, - localIdentity: localIdentity - ) - logger.info("Started listening for inbound encryption keys") - } } + // Key is now installed locally and (best-effort) distributed to + // any existing call participants. Safe to publish media. + try await room.localParticipant.setMicrophone(enabled: true) try await room.localParticipant.setCamera(enabled: true) isLocalCameraEnabled = true @@ -257,7 +365,12 @@ public final class CallViewModel: CallViewModelProtocol { videoViewCache.removeAll() localEncryptionKey = nil localKeyIndex = 0 - keyListenerHandle = nil + callMembers = [:] + + // Tear down the widget bridge synchronously so its tasks can't race + // with subsequent connects. + widgetBridge?.shutdown() + widgetBridge = nil // Network cleanup in background so the UI never beachballs. let service = encryptionService @@ -323,34 +436,33 @@ public final class CallViewModel: CallViewModelProtocol { // MARK: - E2EE Key Redistribution /// Re-sends the local encryption key to a newly joined participant so they - /// can decrypt our media. + /// can decrypt our media. Routes through the widget bridge so the SDK + /// Olm-encrypts the to-device payload. fileprivate func redistributeKey(to participantIdentity: String) { - guard let key = localEncryptionKey, let encryptionService else { return } + guard let key = localEncryptionKey, let bridge = widgetBridge else { return } - // Parse "user:device" from the LiveKit identity (format: @userId:server:deviceId) - // Element Call uses identities like "@user:server:DEVICEID". + // Parse "user:device" from the LiveKit identity + // (format: `@userId:server:deviceId`). Element Call uses identities + // like `@user:server:DEVICEID`. let components = participantIdentity.components(separatedBy: ":") guard components.count >= 3 else { logger.warning("Cannot parse participant identity for key redistribution: \(participantIdentity, privacy: .private)") return } - // Reconstruct userId as first two components, deviceId as remaining. let userId = components[0] + ":" + components[1] let deviceId = components.dropFirst(2).joined(separator: ":") + let index = localKeyIndex Task { do { - try await encryptionService.sendKey(key, keyIndex: localKeyIndex, to: [userId: [deviceId]]) - logger.info("Redistributed key (to-device) to \(participantIdentity, privacy: .private)") - } catch { - logger.warning("Key redistribution (to-device) failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") - } - - // State event is idempotent (overwrites previous), so re-sending is cheap. - do { - try await encryptionService.sendKeyAsStateEvent(key, keyIndex: localKeyIndex) + try await bridge.sendEncryptionKey( + key, + keyIndex: index, + toMembers: [userId: [deviceId]] + ) + logger.info("Redistributed key to \(participantIdentity, privacy: .private)") } catch { - logger.warning("Key redistribution (state event) failed: \(error.localizedDescription)") + logger.warning("Key redistribution failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") } } } @@ -382,7 +494,15 @@ public final class CallViewModel: CallViewModelProtocol { } } - participants = newParticipants + // Only write to the observed `participants` property when the array + // actually changed. The LiveKit `didUpdateSpeakingParticipants` + // callback fires continuously during active audio, and every write + // to an `@Observable` property invalidates downstream SwiftUI views + // regardless of value equality — which can push NSHostingView into + // an unbounded "Update Constraints in Window" loop and crash. + if participants != newParticipants { + participants = newParticipants + } } // MARK: - Delegate Bridge @@ -421,6 +541,9 @@ public final class CallViewModel: CallViewModelProtocol { func room(_ room: LiveKit.Room, participantDidConnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in guard let viewModel else { return } + let identityStr = participant.identity?.stringValue ?? "(none)" + let sidStr = participant.sid?.stringValue ?? "(none)" + logger.info("Remote participant connected: identity=\(identityStr, privacy: .public) sid=\(sidStr, privacy: .public) name=\(participant.name ?? "(none)", privacy: .public)") viewModel.syncParticipants(trackChanged: true) if viewModel.isE2eeEnabled, let identity = participant.identity?.stringValue { viewModel.redistributeKey(to: identity) @@ -428,6 +551,15 @@ public final class CallViewModel: CallViewModelProtocol { } } + func room(_ room: LiveKit.Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { + Task { @MainActor [weak viewModel] in + let identityStr = participant.identity?.stringValue ?? "(none)" + let kind = publication.kind.rawValue + logger.info("Subscribed to \(kind, privacy: .public) track from identity=\(identityStr, privacy: .public) trackSid=\(publication.sid, privacy: .public)") + viewModel?.syncParticipants(trackChanged: true) + } + } + func room(_ room: LiveKit.Room, participantDidDisconnect participant: RemoteParticipant) { Task { @MainActor [weak viewModel] in viewModel?.syncParticipants(trackChanged: true) @@ -442,12 +574,6 @@ public final class CallViewModel: CallViewModelProtocol { } } - func room(_ room: LiveKit.Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { - Task { @MainActor [weak viewModel] in - viewModel?.syncParticipants(trackChanged: true) - } - } - func room(_ room: LiveKit.Room, localParticipant: LocalParticipant, didPublishTrack publication: LocalTrackPublication) { Task { @MainActor [weak viewModel] in viewModel?.videoTrackRevision += 1 diff --git a/RelayKit/Call/CallWidgetBridge.swift b/RelayKit/Call/CallWidgetBridge.swift new file mode 100644 index 0000000..87f5ecb --- /dev/null +++ b/RelayKit/Call/CallWidgetBridge.swift @@ -0,0 +1,657 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// CallWidgetBridge.swift +// RelayKit +// +// SPDX-License-Identifier: Apache-2.0 + +import CryptoKit +import Foundation +import LiveKit +import MatrixRustSDK +import os +import OSLog + +private let logger = Logger(subsystem: "RelayKit", category: "CallWidgetBridge") + +/// Headless widget-driver bridge for MatrixRTC E2EE. +/// +/// Relay embeds LiveKit natively for media but needs the Matrix Widget Driver +/// to handle the MatrixRTC signaling and, crucially, Olm-encrypted to-device +/// delivery of `io.element.call.encryption_keys`. Element Call's web app +/// normally runs inside a WebView that speaks the Widget API (postMessage JSON) +/// to `WidgetDriverHandle`; we collapse the WebView out and speak the same +/// JSON protocol directly from Swift. +/// +/// The SDK side (`WidgetDriver`) handles Olm session setup, m.room.encrypted +/// envelope encryption/decryption, and device discovery transparently. We just +/// emit `send_to_device` widget-API requests with `encrypted: true` and +/// receive decrypted payloads back on the recv channel. +/// +/// ## Lifecycle +/// 1. `start()` kicks off two tasks: the driver's `run(...)` loop and our +/// JSON recv loop on the handle. +/// 2. The recv loop handles SDK-initiated requests (capabilities, notify, +/// incoming events) and dispatches responses to pending outbound requests. +/// 3. `awaitReady()` blocks until the capabilities handshake has completed. +/// 4. `sendEncryptionKey(...)` and `sendCallMemberState(...)` issue +/// fromWidget requests and await their responses. +/// 5. `shutdown()` cancels both tasks and fails any outstanding continuations. +public final class CallWidgetBridge: @unchecked Sendable { + + // MARK: - Configuration + + /// Element Call widget capability strings. These match the capabilities + /// declared by the Element Call web app and approved server-side by + /// `getElementCallRequiredPermissions` (which `CapabilitiesProvider` + /// returns on the SDK side). + private static let elementCallCapabilities: [String] = [ + "io.element.requires_client", + "org.matrix.msc3819.send.to_device:io.element.call.encryption_keys", + "org.matrix.msc3819.receive.to_device:io.element.call.encryption_keys", + "org.matrix.msc2762.receive.state_event:org.matrix.msc3401.call.member", + "org.matrix.msc2762.receive.state_event:m.room.member", + "org.matrix.msc2762.receive.state_event:m.room.encryption", + "org.matrix.msc4157.send.delayed_event", + "org.matrix.msc4157.update_delayed_event" + ] + + /// Supported matrix-widget-api versions we advertise to the SDK when it + /// requests `supported_api_versions`. These match what Element Call's + /// widget declares. + private static let supportedApiVersions: [String] = [ + "0.0.1", + "0.0.2" + ] + + // MARK: - Properties + + private let widgetId: String + private let ownUserId: String + private let ownDeviceId: String + private let roomId: String + /// Per-call MatrixRTC membership UUID. Must match the `membershipID` + /// field in the `org.matrix.msc3401.call.member` state event and the + /// `member.id` field in outbound `io.element.call.encryption_keys` + /// to-device payloads so peers can correlate our keys with our + /// membership event. + public let membershipId: String + private weak var keyProvider: BaseKeyProvider? + private let room: MatrixRustSDK.Room + private let capabilitiesProvider: ElementCallCapabilitiesProvider + + private var driver: WidgetDriver? + private var handle: WidgetDriverHandle? + private var recvTask: Task? + private var driverTask: Task? + + /// State that may be touched from the driver recv loop, the shutdown + /// path, and outbound-request callers concurrently. Kept behind an + /// unfair-lock so access is synchronous and async-context-safe. + /// + /// Pending requests resume with `Void` — callers fire and forget. If a + /// future caller needs the response body, wire a separate sink. + private struct State { + var pendingRequests: [String: CheckedContinuation] = [:] + var readyContinuations: [CheckedContinuation] = [] + var isReady: Bool = false + } + private let state = OSAllocatedUnfairLock(initialState: State()) + + // MARK: - Init / Start / Shutdown + + /// Creates a bridge for the given Matrix room. + /// + /// - Parameters: + /// - room: The SDK room hosting the call. + /// - ownUserId: Local user's Matrix ID (e.g. `@alice:server`). + /// - ownDeviceId: Local device ID. + /// - isRoomEncrypted: Whether the room is encrypted — controls the + /// `EncryptionSystem` on the widget settings. + /// - keyProvider: The LiveKit key provider that receives inbound keys. + public init( + room: MatrixRustSDK.Room, + ownUserId: String, + ownDeviceId: String, + isRoomEncrypted: Bool, + keyProvider: BaseKeyProvider? + ) throws { + self.room = room + self.ownUserId = ownUserId + self.ownDeviceId = ownDeviceId + self.roomId = room.id() + self.keyProvider = keyProvider + self.widgetId = UUID().uuidString + self.membershipId = UUID().uuidString.lowercased() + self.capabilitiesProvider = ElementCallCapabilitiesProvider( + ownUserId: ownUserId, + ownDeviceId: ownDeviceId + ) + + let props = VirtualElementCallWidgetProperties( + elementCallUrl: "https://call.element.io", + widgetId: self.widgetId, + parentUrl: nil, + fontScale: nil, + font: nil, + encryption: isRoomEncrypted ? .perParticipantKeys : .unencrypted, + posthogUserId: nil, + posthogApiHost: nil, + posthogApiKey: nil, + rageshakeSubmitUrl: nil, + sentryDsn: nil, + sentryEnvironment: nil + ) + + let config = VirtualElementCallWidgetConfig( + intent: .joinExisting, + skipLobby: true, + header: nil, + hideHeader: true, + preload: nil, + appPrompt: false, + confineToRoom: true, + hideScreensharing: nil, + controlledAudioDevices: true, + sendNotificationType: nil + ) + + let settings = try newVirtualElementCallWidget(props: props, config: config) + let driverAndHandle = try makeWidgetDriver(settings: settings) + self.driver = driverAndHandle.driver + self.handle = driverAndHandle.handle + } + + /// Starts the driver and the recv loop. Idempotent. + /// + /// Element Call's virtual widget settings set `init_on_content_load: true` + /// inside the Rust SDK, meaning the driver's state machine **waits for a + /// `content_loaded` fromWidget request before it will do anything** + /// (including capability negotiation). We fire that proactively so the + /// driver progresses and eventually sends us the `capabilities` request. + public func start() { + guard let driver, let handle else { return } + guard driverTask == nil, recvTask == nil else { return } + + let room = self.room + let capabilitiesProvider = self.capabilitiesProvider + driverTask = Task { [weak self] in + await driver.run(room: room, capabilitiesProvider: capabilitiesProvider) + logger.info("WidgetDriver.run returned; driver exited") + self?.resolveReady() + } + + recvTask = Task { [weak self] in + await self?.recvLoop(handle: handle) + } + + // Kick the state machine off the "Unset" state. Fire-and-forget — + // the response just echoes back through recvLoop. + Task { [weak self] in + do { + try await self?.sendRequest(action: "content_loaded", data: [:]) + logger.info("Widget content_loaded acknowledged by driver") + } catch { + logger.warning("content_loaded failed: \(error.localizedDescription)") + } + } + + logger.info("CallWidgetBridge started (widgetId=\(self.widgetId, privacy: .public))") + } + + /// Cancels both tasks and fails any outstanding pending requests. + public func shutdown() { + recvTask?.cancel() + driverTask?.cancel() + recvTask = nil + driverTask = nil + + // Fail any pending outbound continuations so callers don't hang. + let pending = state.withLock { s -> [CheckedContinuation] in + let values = Array(s.pendingRequests.values) + s.pendingRequests.removeAll() + return values + } + for cont in pending { + cont.resume(throwing: CallWidgetBridgeError.shutdown) + } + + resolveReady() + logger.info("CallWidgetBridge shut down") + } + + /// Suspends until the capabilities handshake has completed and the + /// widget is permitted to send state and to-device events. + public func awaitReady() async { + // Fast path: already ready. + let alreadyReady = state.withLock { $0.isReady } + if alreadyReady { return } + + await withCheckedContinuation { (cont: CheckedContinuation) in + // Re-check under the lock to avoid races with resolveReady(). + let shouldResume = state.withLock { s -> Bool in + if s.isReady { return true } + s.readyContinuations.append(cont) + return false + } + if shouldResume { cont.resume() } + } + } + + private func resolveReady() { + let toResume = state.withLock { s -> [CheckedContinuation] in + if s.isReady { return [] } + s.isReady = true + let pending = s.readyContinuations + s.readyContinuations.removeAll() + return pending + } + for c in toResume { c.resume() } + } + + // MARK: - Public API + + /// Sends an encrypted `io.element.call.encryption_keys` to-device message + /// to the specified user/device map via a fromWidget `send_to_device` + /// request. The SDK handles Olm encryption transparently. + /// + /// - Parameters: + /// - key: Raw 16-byte AES-128-GCM key. + /// - keyIndex: Key index (0–255). + /// - toMembers: Map of `userId -> [deviceId]`. Use `"*"` as device id + /// to target all devices of that user. + public func sendEncryptionKey( + _ key: Data, + keyIndex: Int, + toMembers: [String: [String]] + ) async throws { + await awaitReady() + + let base64Key = key.base64EncodedString() + let sentTs = Int(Date().timeIntervalSince1970 * 1000) + + // Wire format per matrix-js-sdk + // `EncryptionKeysToDeviceEventContent`: + // { keys: {index, key}, // SINGLE object + // member: {id, claimed_device_id}, // id = membership UUID + // room_id, + // session: {application, call_id, scope}, + // sent_ts? } + // Element Call's parser discards payloads where `keys` is an + // array or where `member`/`room_id`/`session` are missing — which + // is why earlier calls completed key exchange yet peers never + // decoded our frames. + let content: [String: Any] = [ + "keys": [ + "index": keyIndex, + "key": base64Key + ] as [String: Any], + "member": [ + "id": self.membershipId, + "claimed_device_id": self.ownDeviceId + ] as [String: Any], + "room_id": self.roomId, + "session": [ + "application": "m.call", + "call_id": "", + "scope": "m.room" + ] as [String: Any], + "sent_ts": sentTs + ] + + var messages: [String: [String: Any]] = [:] + for (userId, deviceIds) in toMembers { + var deviceMessages: [String: Any] = [:] + for deviceId in deviceIds { + deviceMessages[deviceId] = content + } + messages[userId] = deviceMessages + } + + let data: [String: Any] = [ + "type": CallEncryptionService.encryptionKeysEventType, + "encrypted": true, + "messages": messages + ] + + // SHA-256 fingerprint of the raw IKM going on the wire. This is + // compared against the fingerprint logged by `setRawKey` at the local + // cryptor registration site. Matching prefixes confirm the same 16 + // bytes are both (a) driving our outgoing AES-128-GCM and (b) being + // base64'd into this to-device payload. Diverging prefixes localise + // the bug to the key-capture path in `CallViewModel.connect`. + let fp = SHA256.hash(data: key).prefix(8).map { String(format: "%02x", $0) }.joined() + + _ = try await sendRequest(action: "send_to_device", data: data) + logger.info("Sent encryption key (index \(keyIndex)) to \(toMembers.count) user(s) member.id=\(self.membershipId, privacy: .public) sha256[0..8]=\(fp, privacy: .public)") + } + + /// Sends a MatrixRTC call member state event + /// (`org.matrix.msc3401.call.member`) via a fromWidget `send_event` + /// request. + public func sendCallMemberState( + content: [String: Any], + stateKey: String + ) async throws { + await awaitReady() + + let data: [String: Any] = [ + "type": CallEncryptionService.callMemberEventType, + "state_key": stateKey, + "content": content, + "room_id": roomId + ] + + _ = try await sendRequest(action: "send_event", data: data) + logger.info("Sent call member state event (state_key=\(stateKey, privacy: .public))") + } + + // MARK: - Request / Response plumbing + + /// Issues a fromWidget request and awaits acknowledgement. The response + /// body is not surfaced — if a future call-site needs it, add a separate + /// delivery channel keyed by `requestId`. + private func sendRequest(action: String, data: [String: Any]) async throws { + guard let handle else { + throw CallWidgetBridgeError.notStarted + } + + let requestId = UUID().uuidString + let msg: [String: Any] = [ + "api": "fromWidget", + "widgetId": widgetId, + "requestId": requestId, + "action": action, + "data": data + ] + let json = try Self.encode(msg) + + try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in + state.withLock { $0.pendingRequests[requestId] = cont } + + Task { + let ok = await handle.send(msg: json) + if !ok { + let waiting = state.withLock { s -> CheckedContinuation? in + s.pendingRequests.removeValue(forKey: requestId) + } + waiting?.resume(throwing: CallWidgetBridgeError.sendFailed) + } + } + } + } + + // MARK: - Recv loop + + private func recvLoop(handle: WidgetDriverHandle) async { + while !Task.isCancelled { + guard let raw = await handle.recv() else { + logger.info("WidgetDriverHandle.recv returned nil; loop exiting") + break + } + + // Truncate noisy payloads for the log; full body is still in msg. + let preview = raw.count > 400 ? String(raw.prefix(400)) + "…" : raw + logger.info("widget recv: \(preview, privacy: .public)") + + guard let data = raw.data(using: .utf8), + let msg = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + logger.warning("Non-JSON message from widget driver: \(raw, privacy: .public)") + continue + } + + // Responses to our outbound fromWidget requests. + if let api = msg["api"] as? String, + api == "fromWidget", + msg["response"] != nil, + let requestId = msg["requestId"] as? String { + let cont = state.withLock { s -> CheckedContinuation? in + s.pendingRequests.removeValue(forKey: requestId) + } + let response = (msg["response"] as? [String: Any]) ?? [:] + if let err = response["error"] as? [String: Any] { + let message = (err["message"] as? String) ?? "unknown" + cont?.resume(throwing: CallWidgetBridgeError.widgetError(message)) + } else { + cont?.resume(returning: ()) + } + continue + } + + // Incoming SDK-initiated requests (toWidget). + guard let action = msg["action"] as? String else { + logger.warning("Widget message missing action: \(raw, privacy: .private)") + continue + } + let requestId = (msg["requestId"] as? String) ?? "" + let reqData = (msg["data"] as? [String: Any]) ?? [:] + + await handleIncoming(action: action, requestId: requestId, data: reqData, fullMessage: msg, handle: handle) + } + } + + private func handleIncoming( + action: String, + requestId: String, + data: [String: Any], + fullMessage: [String: Any], + handle: WidgetDriverHandle + ) async { + var responseBody: [String: Any] = [:] + + switch action { + case "capabilities": + // SDK is asking which capabilities we want. Replying here + // concludes the first half of negotiation; the driver will then + // call our `acquireCapabilities` provider to approve. + responseBody = ["capabilities": Self.elementCallCapabilities] + + case "notify_capabilities": + // SDK telling us what was approved. After this we're ready. + responseBody = [:] + resolveReady() + + case "supported_api_versions": + responseBody = ["supported_versions": Self.supportedApiVersions] + + case "send_to_device": + handleIncomingToDevice(data: data) + responseBody = [:] + + case "send_event", "update_state": + // Incoming Matrix events observed by the widget driver. + // MatrixRTC member state is handled by Element Call peers + // directly; we just need to ack these. Log and move on. + if let type = data["type"] as? String { + logger.info("widget incoming \(action, privacy: .public) type=\(type, privacy: .public)") + } + responseBody = [:] + + case "content_loaded": + responseBody = [:] + + default: + logger.info("widget unhandled action=\(action, privacy: .public); acking with {}") + responseBody = [:] + } + + // Belt-and-braces: once the driver is sending any post-negotiation + // event to us (send_event / send_to_device), it has approved our + // capabilities even if we missed the explicit notify_capabilities + // message. Flip readiness so outbound sends aren't stuck. + if action == "send_to_device" || action == "send_event" || action == "update_state" { + resolveReady() + } + + await reply(to: fullMessage, requestId: requestId, response: responseBody, handle: handle) + } + + private func reply( + to original: [String: Any], + requestId: String, + response: [String: Any], + handle: WidgetDriverHandle + ) async { + var reply = original + reply["response"] = response + // requestId is already in the echoed message; ensure it's set. + if !requestId.isEmpty { reply["requestId"] = requestId } + + guard let json = try? Self.encode(reply) else { + logger.error("Failed to encode widget reply") + return + } + let ok = await handle.send(msg: json) + if !ok { + logger.warning("handle.send returned false replying to action=\(original["action"] as? String ?? "?", privacy: .public)") + } + } + + // MARK: - Incoming key plumbing + + private func handleIncomingToDevice(data: [String: Any]) { + guard let type = data["type"] as? String, + type == CallEncryptionService.encryptionKeysEventType, + let sender = data["sender"] as? String else { + return + } + let content = (data["content"] as? [String: Any]) ?? [:] + guard let keyProvider else { + logger.warning("No keyProvider; dropping inbound key from \(sender, privacy: .private)") + return + } + + // Wire format has evolved. Newer Element Call sends: + // content: { keys: { index, key }, member: { id, claimed_device_id }, room_id, ... } + // Older callers (including ourselves pre-fix) send: + // content: { keys: [ { index, key }, ... ], device_id, call_id, ... } + // Support both. + var keyEntries: [[String: Any]] = [] + if let arr = content["keys"] as? [[String: Any]] { + keyEntries = arr + } else if let single = content["keys"] as? [String: Any] { + keyEntries = [single] + } else { + logger.warning("encryption_keys to-device missing keys from \(sender, privacy: .private)") + return + } + + let member = content["member"] as? [String: Any] + let memberId = (member?["id"] as? String) ?? "" + let claimedDeviceId = (member?["claimed_device_id"] as? String) ?? "" + let topDeviceId = (content["device_id"] as? String) ?? "" + let deviceId = !claimedDeviceId.isEmpty ? claimedDeviceId : topDeviceId + + // LiveKit participant identity lookup order. Element Call connects to + // the SFU with identity `@user:server:deviceId` (confirmed in the + // MatrixRTC JWT grant), so that's what we need to key on for the + // LKRTCFrameCryptorKeyProvider to route the key to the right + // participant's decoder. + // + // `member.id` is the MSC4143 per-membership UUID — an *event*-level + // identifier, not a LiveKit participant identity. It only enters the + // fallback chain so older peers that somehow omit the device id still + // get routed. + let participantIdentity: String + if !deviceId.isEmpty { + participantIdentity = "\(sender):\(deviceId)" + } else if !memberId.isEmpty { + participantIdentity = memberId + } else { + participantIdentity = sender + } + + for entry in keyEntries { + guard let base64Key = entry["key"] as? String, + let index = entry["index"] as? Int, + let keyData = Data(base64Encoded: base64Key) else { + continue + } + CallEncryptionService.setRawKey( + keyData, + on: keyProvider, + participantId: participantIdentity, + index: Int32(index) + ) + // Log with `.public` so we can correlate the key routing + // identity (what we register the frame-decryption key under) + // with the actual LiveKit participant identity (logged on + // connect) — if these do not match byte-for-byte, LiveKit will + // silently fail to decrypt this peer's frames. + logger.info("Applied inbound key -> routed to LiveKit participantId=\(participantIdentity, privacy: .public) sender=\(sender, privacy: .public) device=\(deviceId, privacy: .public) member=\(memberId, privacy: .public) index=\(index)") + } + } + + // MARK: - Helpers + + private static func encode(_ value: [String: Any]) throws -> String { + // `.sortedKeys` guarantees `action` is serialised before `data` in + // top-level messages. The Rust SDK uses + // `#[serde(tag = "action", content = "data")]` on its FromWidget enum; + // when `data` appears first, serde falls back to its Content-buffering + // path, which fails for `Raw` newtype fields with + // "invalid type: newtype struct, expected any valid JSON value". + // Sorting keys sidesteps the bug entirely. + let data = try JSONSerialization.data( + withJSONObject: value, + options: [.sortedKeys] + ) + return String(data: data, encoding: .utf8) ?? "{}" + } +} + +// MARK: - Capabilities Provider + +/// Implements `WidgetCapabilitiesProvider` by returning the Element Call +/// required permissions verbatim. The SDK intersects these with whatever +/// the widget requests over JSON. +private final class ElementCallCapabilitiesProvider: WidgetCapabilitiesProvider, @unchecked Sendable { + private let ownUserId: String + private let ownDeviceId: String + + init(ownUserId: String, ownDeviceId: String) { + self.ownUserId = ownUserId + self.ownDeviceId = ownDeviceId + } + + func acquireCapabilities(capabilities: WidgetCapabilities) -> WidgetCapabilities { + return getElementCallRequiredPermissions( + ownUserId: ownUserId, + ownDeviceId: ownDeviceId + ) + } +} + +// MARK: - Errors + +enum CallWidgetBridgeError: LocalizedError { + case notStarted + case sendFailed + case shutdown + case widgetError(String) + + var errorDescription: String? { + switch self { + case .notStarted: + return "Widget bridge is not started." + case .sendFailed: + return "Failed to send widget message; driver may have exited." + case .shutdown: + return "Widget bridge was shut down before the request completed." + case .widgetError(let message): + return "Widget protocol error: \(message)" + } + } +} diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index 4146fe9..def4952 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -619,6 +619,24 @@ public final class MatrixService: MatrixServiceProtocol { try await sdkRoom.leave() } + /// Power level overrides that allow any room member to send MatrixRTC call + /// membership and encryption key state events (matching Element Call's setup). + private static let callPowerLevels = PowerLevels( + usersDefault: nil, + eventsDefault: nil, + stateDefault: nil, + ban: nil, + kick: nil, + redact: nil, + invite: nil, + notifications: nil, + users: [:], + events: [ + "org.matrix.msc3401.call.member": 0, + "io.element.call.encryption_keys": 0 + ] + ) + public func createRoom(name: String, topic: String?, isPublic: Bool) async throws -> String { guard let client else { throw RelayError.notLoggedIn } let params = CreateRoomParameters( @@ -627,7 +645,8 @@ public final class MatrixService: MatrixServiceProtocol { isEncrypted: !isPublic, isDirect: false, visibility: isPublic ? .public : .private, - preset: isPublic ? .publicChat : .privateChat + preset: isPublic ? .publicChat : .privateChat, + powerLevelContentOverride: Self.callPowerLevels ) return try await client.createRoom(parameters: params) } @@ -641,6 +660,7 @@ public final class MatrixService: MatrixServiceProtocol { isDirect: false, visibility: options.isPublic ? .public : .private, preset: options.isPublic ? .publicChat : .privateChat, + powerLevelContentOverride: Self.callPowerLevels, canonicalAlias: options.address, isSpace: options.isSpace ) @@ -662,7 +682,8 @@ public final class MatrixService: MatrixServiceProtocol { isDirect: true, visibility: .private, preset: .trustedPrivateChat, - invite: [userId] + invite: [userId], + powerLevelContentOverride: Self.callPowerLevels ) return try await client.createRoom(parameters: params) } diff --git a/RelayKit/Widget/WidgetProxy.swift b/RelayKit/Widget/WidgetProxy.swift deleted file mode 100644 index 996c8b4..0000000 --- a/RelayKit/Widget/WidgetProxy.swift +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2026 Link Dupont -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// WidgetProxy.swift -// RelayKit -// -// SPDX-License-Identifier: Apache-2.0 - -/// Manages Matrix widget lifecycle and communication. -/// -/// Provides a bridge for embedding Matrix widgets in SwiftUI views. -/// The widget API is evolving; this proxy will be expanded as the -/// SDK stabilizes. -public final class WidgetProxy: @unchecked Sendable { - /// Creates a widget proxy. - public init() {} -} From 500516b218bc044f7154d4a14404db4b7832eccd Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Fri, 24 Apr 2026 16:01:18 -0400 Subject: [PATCH 12/24] Prefix RTC logs with [RTC] and bridge LiveKit logs to OSLog Makes it straightforward to filter the encrypted-call flow out of a noisy Console by grepping for "[RTC]". Adds LiveKitLogBridge so LiveKit SDK logs flow through OSLog with the same prefix. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Call/CallEncryptionService.swift | 38 +++++----- RelayKit/Call/CallViewModel.swift | 36 +++++----- RelayKit/Call/CallWidgetBridge.swift | 38 +++++----- RelayKit/Call/LiveKitCredentialService.swift | 10 +-- RelayKit/Call/LiveKitLogBridge.swift | 73 ++++++++++++++++++++ 5 files changed, 134 insertions(+), 61 deletions(-) create mode 100644 RelayKit/Call/LiveKitLogBridge.swift diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift index 1d62ab5..535b5f0 100644 --- a/RelayKit/Call/CallEncryptionService.swift +++ b/RelayKit/Call/CallEncryptionService.swift @@ -106,15 +106,15 @@ struct CallEncryptionService { let jsonData = try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) let jsonString = String(data: jsonData, encoding: .utf8) ?? "{}" - logger.info("Call member event body: \(jsonString)") - logger.info("Call member state key: \(stateKey)") + logger.info("[RTC]Call member event body: \(jsonString)") + logger.info("[RTC]Call member state key: \(stateKey)") _ = try await sdkRoom.sendStateEventRaw( eventType: Self.callMemberEventType, stateKey: stateKey, content: jsonString ) - logger.info("Sent call membership state event") + logger.info("[RTC]Sent call membership state event") } /// Removes the call membership state event (sets content to empty object) @@ -129,7 +129,7 @@ struct CallEncryptionService { stateKey: stateKey, content: "{}" ) - logger.info("Removed call membership state event") + logger.info("[RTC]Removed call membership state event") } // MARK: - Debug: Fetch Existing Call Members @@ -158,7 +158,7 @@ struct CallEncryptionService { if let content = event["content"], let contentData = try? JSONSerialization.data(withJSONObject: content, options: [.sortedKeys]), let contentStr = String(data: contentData, encoding: .utf8) { - logger.info("Existing call member [key=\(stateKey)]: \(contentStr)") + logger.info("[RTC]Existing call member [key=\(stateKey)]: \(contentStr)") } } } @@ -222,7 +222,7 @@ struct CallEncryptionService { /// should be created with the correct power levels via `powerLevelContentOverride`. func enableCallPowerLevels() async throws { guard let sdkRoom else { - logger.warning("No SDK room — cannot check/update power levels") + logger.warning("[RTC]No SDK room — cannot check/update power levels") return } @@ -234,11 +234,11 @@ struct CallEncryptionService { ) if canSendCallMember && canSendEncKeys { - logger.info("Call power levels already allow sending") + logger.info("[RTC]Call power levels already allow sending") return } - logger.info("Call power levels need update (callMember=\(canSendCallMember), encKeys=\(canSendEncKeys))") + logger.info("[RTC]Call power levels need update (callMember=\(canSendCallMember), encKeys=\(canSendEncKeys))") // Fetch current power levels JSON, merge in call event types at PL 0, // and re-send via the SDK. This only works if we're admin/mod. @@ -255,7 +255,7 @@ struct CallEncryptionService { let (data, getResponse) = try await URLSession.shared.data(for: getRequest) guard let http = getResponse as? HTTPURLResponse, http.statusCode == 200, var plDict = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { - logger.warning("Could not fetch power levels for update") + logger.warning("[RTC]Could not fetch power levels for update") return } @@ -273,9 +273,9 @@ struct CallEncryptionService { stateKey: "", content: jsonString ) - logger.info("Enabled call power levels for room") + logger.info("[RTC]Enabled call power levels for room") } catch { - logger.warning("Cannot update call power levels (likely not admin): \(error.localizedDescription)") + logger.warning("[RTC]Cannot update call power levels (likely not admin): \(error.localizedDescription)") } } @@ -324,7 +324,7 @@ struct CallEncryptionService { let provider = BaseKeyProvider(options: options) guard let cls = NSClassFromString("LKRTCFrameCryptorKeyProvider") as? NSObject.Type else { - logger.error("LKRTCFrameCryptorKeyProvider class not found at runtime; HKDF swap skipped — E2EE interop with Element Call will fail (PBKDF2 vs HKDF mismatch)") + logger.error("[RTC]LKRTCFrameCryptorKeyProvider class not found at runtime; HKDF swap skipped — E2EE interop with Element Call will fail (PBKDF2 vs HKDF mismatch)") return provider } @@ -340,7 +340,7 @@ struct CallEncryptionService { ) let allocated = allocImp(cls, allocSel) guard (allocated as AnyObject).responds(to: initSel) else { - logger.error("LKRTCFrameCryptorKeyProvider does not expose keyDerivationAlgorithm: init; webrtc-xcframework may be < 144.x — falling back to PBKDF2 (Element Call interop will fail)") + logger.error("[RTC]LKRTCFrameCryptorKeyProvider does not expose keyDerivationAlgorithm: init; webrtc-xcframework may be < 144.x — falling back to PBKDF2 (Element Call interop will fail)") return provider } @@ -367,11 +367,11 @@ struct CallEncryptionService { ) guard let ivar = class_getInstanceVariable(BaseKeyProvider.self, "rtcKeyProvider") else { - logger.error("rtcKeyProvider ivar not found on BaseKeyProvider; HKDF swap skipped") + logger.error("[RTC]rtcKeyProvider ivar not found on BaseKeyProvider; HKDF swap skipped") return provider } object_setIvar(provider, ivar, hkdfRtc) - logger.info("Installed HKDF-backed LKRTCFrameCryptorKeyProvider (Element Call interop path)") + logger.info("[RTC]Installed HKDF-backed LKRTCFrameCryptorKeyProvider (Element Call interop path)") return provider } @@ -389,7 +389,7 @@ struct CallEncryptionService { index: Int32 = 0 ) { guard let rtcProvider = keyProvider.value(forKey: "rtcKeyProvider") as AnyObject? else { - logger.error("Could not access rtcKeyProvider via KVC") + logger.error("[RTC]Could not access rtcKeyProvider via KVC") return } @@ -400,7 +400,7 @@ struct CallEncryptionService { typealias SetKeyFunc = @convention(c) (AnyObject, Selector, NSData, Int32, NSString) -> Void let selector = NSSelectorFromString("setKey:withIndex:forParticipant:") guard (rtcProvider as? NSObject)?.responds(to: selector) == true else { - logger.error("rtcKeyProvider does not respond to setKey:withIndex:forParticipant:") + logger.error("[RTC]rtcKeyProvider does not respond to setKey:withIndex:forParticipant:") return } @@ -416,7 +416,7 @@ struct CallEncryptionService { // the #1 root cause of "maximum ratchet attempts exceeded" on an // otherwise-correct key-exchange handshake. let fp = SHA256.hash(data: keyData).prefix(8).map { String(format: "%02x", $0) }.joined() - logger.info("Set raw encryption key for participant \(participantId, privacy: .public) at index \(index) bytes=\(keyData.count) sha256[0..8]=\(fp, privacy: .public)") + logger.info("[RTC]Set raw encryption key for participant \(participantId, privacy: .public) at index \(index) bytes=\(keyData.count) sha256[0..8]=\(fp, privacy: .public)") } /// Convenience: sets a raw key using base64-encoded key data. @@ -427,7 +427,7 @@ struct CallEncryptionService { index: Int32 = 0 ) { guard let keyData = Data(base64Encoded: base64Key) else { - logger.error("Invalid base64 key for participant \(participantId, privacy: .private)") + logger.error("[RTC]Invalid base64 key for participant \(participantId, privacy: .private)") return } setRawKey(keyData, on: keyProvider, participantId: participantId, index: index) diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 58e0e2a..39aab60 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -96,6 +96,7 @@ public final class CallViewModel: CallViewModelProtocol { /// Creates a call view model without E2EE. Use ``init(encryptionContext:)`` /// for encrypted calls that interoperate with Element Call. public init() { + LiveKitLogBridgeInstaller.install() self.isE2eeEnabled = false let delegate = Delegate(viewModel: self) self.delegate = delegate @@ -135,6 +136,7 @@ public final class CallViewModel: CallViewModelProtocol { /// room's encryption state. Encrypted rooms use AES-128-GCM frame encryption /// with MatrixRTC key exchange; unencrypted rooms use no LiveKit-level E2EE. public init(encryptionContext: EncryptionContext) { + LiveKitLogBridgeInstaller.install() self.isE2eeEnabled = encryptionContext.isRoomEncrypted let delegate = Delegate(viewModel: self) @@ -196,9 +198,9 @@ public final class CallViewModel: CallViewModelProtocol { EncryptionOptions(keyProvider: $0, encryptionType: .gcm) } if isE2eeEnabled { - logger.info("E2EE enabled (encrypted Matrix room)") + logger.info("[RTC]E2EE enabled (encrypted Matrix room)") } else { - logger.info("E2EE disabled (unencrypted Matrix room)") + logger.info("[RTC]E2EE disabled (unencrypted Matrix room)") } let roomOpts = RoomOptions( defaultVideoPublishOptions: VideoPublishOptions( @@ -219,7 +221,7 @@ public final class CallViewModel: CallViewModelProtocol { roomOptions: roomOpts ) localParticipantID = room.localParticipant.identity?.stringValue - logger.info("Connected with LiveKit identity: \(self.localParticipantID ?? "unknown", privacy: .public)") + logger.info("[RTC]Connected with LiveKit identity: \(self.localParticipantID ?? "unknown", privacy: .public)") // Spin up the headless widget bridge *only* for encrypted rooms. // For unencrypted rooms the bridge adds no value (no keys to @@ -238,7 +240,7 @@ public final class CallViewModel: CallViewModelProtocol { bridge.start() self.widgetBridge = bridge } catch { - logger.error("Failed to create CallWidgetBridge: \(error.localizedDescription)") + logger.error("[RTC]Failed to create CallWidgetBridge: \(error.localizedDescription)") } } @@ -261,7 +263,7 @@ public final class CallViewModel: CallViewModelProtocol { // mismatched JWT identity would silently break decrypt. let localIdentity = "\(encryptionService.userID):\(encryptionService.deviceID)" if let livekitIdentity = self.localParticipantID, livekitIdentity != localIdentity { - logger.warning("LiveKit identity \(livekitIdentity, privacy: .public) != matrix identity \(localIdentity, privacy: .public) — frame encryption may misroute") + logger.warning("[RTC]LiveKit identity \(livekitIdentity, privacy: .public) != matrix identity \(localIdentity, privacy: .public) — frame encryption may misroute") } let keyIndex = self.localKeyIndex CallEncryptionService.setRawKey( @@ -270,7 +272,7 @@ public final class CallViewModel: CallViewModelProtocol { participantId: localIdentity, index: Int32(keyIndex) ) - logger.info("Local E2EE key set (index \(keyIndex)) under participantId=\(localIdentity, privacy: .public) before camera publish") + logger.info("[RTC]Local E2EE key set (index \(keyIndex)) under participantId=\(localIdentity, privacy: .public) before camera publish") } // Set up MatrixRTC signaling and distribute the key **before** @@ -300,7 +302,7 @@ public final class CallViewModel: CallViewModelProtocol { do { try await encryptionService.enableCallPowerLevels() } catch { - logger.warning("Call power level setup failed: \(error.localizedDescription)") + logger.warning("[RTC]Call power level setup failed: \(error.localizedDescription)") } // 2. Send call membership state event (after power levels). @@ -313,7 +315,7 @@ public final class CallViewModel: CallViewModelProtocol { membershipId: bridge?.membershipId ) } catch { - logger.warning("Call membership event failed: \(error.localizedDescription)") + logger.warning("[RTC]Call membership event failed: \(error.localizedDescription)") } // 3. Distribute the already-generated local key via the @@ -326,7 +328,7 @@ public final class CallViewModel: CallViewModelProtocol { if self.isE2eeEnabled, let bridge, let localKey { let targets = await encryptionService.fetchCallTargets() self.callMembers = targets - logger.info("Distributing key to \(targets.count) remote user(s) BEFORE media publish") + logger.info("[RTC]Distributing key to \(targets.count) remote user(s) BEFORE media publish") do { try await bridge.sendEncryptionKey( localKey, @@ -334,7 +336,7 @@ public final class CallViewModel: CallViewModelProtocol { toMembers: targets ) } catch { - logger.warning("Widget-bridge key distribution failed: \(error.localizedDescription)") + logger.warning("[RTC]Widget-bridge key distribution failed: \(error.localizedDescription)") } } } @@ -349,7 +351,7 @@ public final class CallViewModel: CallViewModelProtocol { state = .connected videoTrackRevision += 1 } catch { - logger.error("Connect failed: \(error.localizedDescription)") + logger.error("[RTC]Connect failed: \(error.localizedDescription)") state = .failed(error.localizedDescription) throw error } @@ -446,7 +448,7 @@ public final class CallViewModel: CallViewModelProtocol { // like `@user:server:DEVICEID`. let components = participantIdentity.components(separatedBy: ":") guard components.count >= 3 else { - logger.warning("Cannot parse participant identity for key redistribution: \(participantIdentity, privacy: .private)") + logger.warning("[RTC]Cannot parse participant identity for key redistribution: \(participantIdentity, privacy: .private)") return } let userId = components[0] + ":" + components[1] @@ -460,9 +462,9 @@ public final class CallViewModel: CallViewModelProtocol { keyIndex: index, toMembers: [userId: [deviceId]] ) - logger.info("Redistributed key to \(participantIdentity, privacy: .private)") + logger.info("[RTC]Redistributed key to \(participantIdentity, privacy: .private)") } catch { - logger.warning("Key redistribution failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") + logger.warning("[RTC]Key redistribution failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") } } } @@ -531,7 +533,7 @@ public final class CallViewModel: CallViewModelProtocol { viewModel.state = .disconnected } case .reconnecting: - logger.info("Reconnecting…") + logger.info("[RTC]Reconnecting…") default: break } @@ -543,7 +545,7 @@ public final class CallViewModel: CallViewModelProtocol { guard let viewModel else { return } let identityStr = participant.identity?.stringValue ?? "(none)" let sidStr = participant.sid?.stringValue ?? "(none)" - logger.info("Remote participant connected: identity=\(identityStr, privacy: .public) sid=\(sidStr, privacy: .public) name=\(participant.name ?? "(none)", privacy: .public)") + logger.info("[RTC]Remote participant connected: identity=\(identityStr, privacy: .public) sid=\(sidStr, privacy: .public) name=\(participant.name ?? "(none)", privacy: .public)") viewModel.syncParticipants(trackChanged: true) if viewModel.isE2eeEnabled, let identity = participant.identity?.stringValue { viewModel.redistributeKey(to: identity) @@ -555,7 +557,7 @@ public final class CallViewModel: CallViewModelProtocol { Task { @MainActor [weak viewModel] in let identityStr = participant.identity?.stringValue ?? "(none)" let kind = publication.kind.rawValue - logger.info("Subscribed to \(kind, privacy: .public) track from identity=\(identityStr, privacy: .public) trackSid=\(publication.sid, privacy: .public)") + logger.info("[RTC]Subscribed to \(kind, privacy: .public) track from identity=\(identityStr, privacy: .public) trackSid=\(publication.sid, privacy: .public)") viewModel?.syncParticipants(trackChanged: true) } } diff --git a/RelayKit/Call/CallWidgetBridge.swift b/RelayKit/Call/CallWidgetBridge.swift index 87f5ecb..7c14e30 100644 --- a/RelayKit/Call/CallWidgetBridge.swift +++ b/RelayKit/Call/CallWidgetBridge.swift @@ -189,7 +189,7 @@ public final class CallWidgetBridge: @unchecked Sendable { let capabilitiesProvider = self.capabilitiesProvider driverTask = Task { [weak self] in await driver.run(room: room, capabilitiesProvider: capabilitiesProvider) - logger.info("WidgetDriver.run returned; driver exited") + logger.info("[RTC]WidgetDriver.run returned; driver exited") self?.resolveReady() } @@ -202,13 +202,13 @@ public final class CallWidgetBridge: @unchecked Sendable { Task { [weak self] in do { try await self?.sendRequest(action: "content_loaded", data: [:]) - logger.info("Widget content_loaded acknowledged by driver") + logger.info("[RTC]Widget content_loaded acknowledged by driver") } catch { - logger.warning("content_loaded failed: \(error.localizedDescription)") + logger.warning("[RTC]content_loaded failed: \(error.localizedDescription)") } } - logger.info("CallWidgetBridge started (widgetId=\(self.widgetId, privacy: .public))") + logger.info("[RTC]CallWidgetBridge started (widgetId=\(self.widgetId, privacy: .public))") } /// Cancels both tasks and fails any outstanding pending requests. @@ -229,7 +229,7 @@ public final class CallWidgetBridge: @unchecked Sendable { } resolveReady() - logger.info("CallWidgetBridge shut down") + logger.info("[RTC]CallWidgetBridge shut down") } /// Suspends until the capabilities handshake has completed and the @@ -335,7 +335,7 @@ public final class CallWidgetBridge: @unchecked Sendable { let fp = SHA256.hash(data: key).prefix(8).map { String(format: "%02x", $0) }.joined() _ = try await sendRequest(action: "send_to_device", data: data) - logger.info("Sent encryption key (index \(keyIndex)) to \(toMembers.count) user(s) member.id=\(self.membershipId, privacy: .public) sha256[0..8]=\(fp, privacy: .public)") + logger.info("[RTC]Sent encryption key (index \(keyIndex)) to \(toMembers.count) user(s) member.id=\(self.membershipId, privacy: .public) sha256[0..8]=\(fp, privacy: .public)") } /// Sends a MatrixRTC call member state event @@ -355,7 +355,7 @@ public final class CallWidgetBridge: @unchecked Sendable { ] _ = try await sendRequest(action: "send_event", data: data) - logger.info("Sent call member state event (state_key=\(stateKey, privacy: .public))") + logger.info("[RTC]Sent call member state event (state_key=\(stateKey, privacy: .public))") } // MARK: - Request / Response plumbing @@ -398,17 +398,15 @@ public final class CallWidgetBridge: @unchecked Sendable { private func recvLoop(handle: WidgetDriverHandle) async { while !Task.isCancelled { guard let raw = await handle.recv() else { - logger.info("WidgetDriverHandle.recv returned nil; loop exiting") + logger.info("[RTC]WidgetDriverHandle.recv returned nil; loop exiting") break } - // Truncate noisy payloads for the log; full body is still in msg. - let preview = raw.count > 400 ? String(raw.prefix(400)) + "…" : raw - logger.info("widget recv: \(preview, privacy: .public)") + logger.info("[RTC]widget recv: \(raw, privacy: .public)") guard let data = raw.data(using: .utf8), let msg = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - logger.warning("Non-JSON message from widget driver: \(raw, privacy: .public)") + logger.warning("[RTC]Non-JSON message from widget driver: \(raw, privacy: .public)") continue } @@ -432,7 +430,7 @@ public final class CallWidgetBridge: @unchecked Sendable { // Incoming SDK-initiated requests (toWidget). guard let action = msg["action"] as? String else { - logger.warning("Widget message missing action: \(raw, privacy: .private)") + logger.warning("[RTC]Widget message missing action: \(raw, privacy: .private)") continue } let requestId = (msg["requestId"] as? String) ?? "" @@ -475,7 +473,7 @@ public final class CallWidgetBridge: @unchecked Sendable { // MatrixRTC member state is handled by Element Call peers // directly; we just need to ack these. Log and move on. if let type = data["type"] as? String { - logger.info("widget incoming \(action, privacy: .public) type=\(type, privacy: .public)") + logger.info("[RTC]widget incoming \(action, privacy: .public) type=\(type, privacy: .public)") } responseBody = [:] @@ -483,7 +481,7 @@ public final class CallWidgetBridge: @unchecked Sendable { responseBody = [:] default: - logger.info("widget unhandled action=\(action, privacy: .public); acking with {}") + logger.info("[RTC]widget unhandled action=\(action, privacy: .public); acking with {}") responseBody = [:] } @@ -510,12 +508,12 @@ public final class CallWidgetBridge: @unchecked Sendable { if !requestId.isEmpty { reply["requestId"] = requestId } guard let json = try? Self.encode(reply) else { - logger.error("Failed to encode widget reply") + logger.error("[RTC]Failed to encode widget reply") return } let ok = await handle.send(msg: json) if !ok { - logger.warning("handle.send returned false replying to action=\(original["action"] as? String ?? "?", privacy: .public)") + logger.warning("[RTC]handle.send returned false replying to action=\(original["action"] as? String ?? "?", privacy: .public)") } } @@ -529,7 +527,7 @@ public final class CallWidgetBridge: @unchecked Sendable { } let content = (data["content"] as? [String: Any]) ?? [:] guard let keyProvider else { - logger.warning("No keyProvider; dropping inbound key from \(sender, privacy: .private)") + logger.warning("[RTC]No keyProvider; dropping inbound key from \(sender, privacy: .private)") return } @@ -544,7 +542,7 @@ public final class CallWidgetBridge: @unchecked Sendable { } else if let single = content["keys"] as? [String: Any] { keyEntries = [single] } else { - logger.warning("encryption_keys to-device missing keys from \(sender, privacy: .private)") + logger.warning("[RTC]encryption_keys to-device missing keys from \(sender, privacy: .private)") return } @@ -590,7 +588,7 @@ public final class CallWidgetBridge: @unchecked Sendable { // with the actual LiveKit participant identity (logged on // connect) — if these do not match byte-for-byte, LiveKit will // silently fail to decrypt this peer's frames. - logger.info("Applied inbound key -> routed to LiveKit participantId=\(participantIdentity, privacy: .public) sender=\(sender, privacy: .public) device=\(deviceId, privacy: .public) member=\(memberId, privacy: .public) index=\(index)") + logger.info("[RTC]Applied inbound key -> routed to LiveKit participantId=\(participantIdentity, privacy: .public) sender=\(sender, privacy: .public) device=\(deviceId, privacy: .public) member=\(memberId, privacy: .public) index=\(index)") } } diff --git a/RelayKit/Call/LiveKitCredentialService.swift b/RelayKit/Call/LiveKitCredentialService.swift index ae45d83..a575e3b 100644 --- a/RelayKit/Call/LiveKitCredentialService.swift +++ b/RelayKit/Call/LiveKitCredentialService.swift @@ -47,11 +47,11 @@ struct LiveKitCredentialService { /// Returns `(livekitWebSocketURL, livekitJWT, sfuServiceURL)` for the given Matrix room. /// The `sfuServiceURL` is the SFU service URL from discovery, used in call member events. func credentials(for roomID: String) async throws -> (url: String, token: String, sfuServiceURL: String) { - logger.info("Fetching LiveKit credentials for room \(roomID, privacy: .private)") + logger.info("[RTC]Fetching LiveKit credentials for room \(roomID, privacy: .private)") let sfuURL = try await discoverSFUURL() - logger.info("SFU URL discovered: \(sfuURL)") + logger.info("[RTC]SFU URL discovered: \(sfuURL)") let openIDToken = try await requestOpenIDToken() - logger.debug("OpenID token obtained") + logger.debug("[RTC]OpenID token obtained") let (url, jwt) = try await fetchLiveKitToken(sfuURL: sfuURL, roomID: roomID, openIDToken: openIDToken) return (url, jwt, sfuURL) } @@ -172,7 +172,7 @@ struct LiveKitCredentialService { throw LiveKitCredentialError.tokenExchangeFailed } let decoded = try JSONDecoder().decode(LiveKitTokenResponse.self, from: data) - logger.info("LiveKit credentials obtained via /get_token") + logger.info("[RTC]LiveKit credentials obtained via /get_token") return (decoded.url, decoded.jwt) } @@ -197,7 +197,7 @@ struct LiveKitCredentialService { throw LiveKitCredentialError.tokenExchangeFailed } let decoded = try JSONDecoder().decode(LiveKitTokenResponse.self, from: data) - logger.info("LiveKit credentials obtained via legacy /sfu/get") + logger.info("[RTC]LiveKit credentials obtained via legacy /sfu/get") return (decoded.url, decoded.jwt) } } diff --git a/RelayKit/Call/LiveKitLogBridge.swift b/RelayKit/Call/LiveKitLogBridge.swift new file mode 100644 index 0000000..ce51878 --- /dev/null +++ b/RelayKit/Call/LiveKitLogBridge.swift @@ -0,0 +1,73 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation +import LiveKit +import os + +/// A LiveKit `Logger` implementation that forwards all LiveKit SDK log output +/// through `os.Logger` with a `[RTC]` prefix on every message so calling-related +/// logs can be filtered out of the Console with a single token. +/// +/// Install once, as early as possible (before any `LiveKit.Room` is created). +struct LiveKitLogBridge: LiveKit.Logger { + private static let osLogger = os.Logger(subsystem: "RelayKit", category: "LiveKitSDK") + + // swiftlint:disable:next function_parameter_count + func log( + _ message: @autoclosure () -> CustomStringConvertible, + _ level: LiveKit.LogLevel, + source: @autoclosure () -> String?, + file _: StaticString, + type: Any.Type, + function: StaticString, + line _: UInt, + metaData: ScopedMetadataContainer + ) { + let rendered: String = { + let typeName = String(describing: type) + let meta: String + if metaData.isEmpty { + meta = "" + } else { + meta = " [" + metaData.map { "\($0): \($1)" }.joined(separator: ", ") + "]" + } + return "[RTC] \(typeName).\(function) \(message().description)\(meta)" + }() + + switch level { + case .debug: + Self.osLogger.debug("\(rendered, privacy: .public)") + case .info: + Self.osLogger.info("\(rendered, privacy: .public)") + case .warning: + Self.osLogger.warning("\(rendered, privacy: .public)") + case .error: + Self.osLogger.error("\(rendered, privacy: .public)") + } + } +} + +/// Installs ``LiveKitLogBridge`` exactly once, regardless of how many times +/// ``install()`` is called. Safe to invoke from any thread; the actual swap +/// runs on first access of the static initializer. +enum LiveKitLogBridgeInstaller { + private static let installed: Void = { + LiveKitSDK.setLogger(LiveKitLogBridge()) + }() + + static func install() { + _ = installed + } +} From 713423106b31cd52c82f1db546d21a2bbcd74f99 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 12:45:44 -0400 Subject: [PATCH 13/24] Drop runtime power-level mutation; add call.member heartbeat and bounded leave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Element Call doesn't try to mutate m.room.power_levels at join time — it relies on the room being provisioned correctly. Drop the enableCallPowerLevels() runtime path; MatrixService.callPowerLevels still applies the same defaults at room creation. Add an expires_ts-style heartbeat that re-sends the org.matrix.msc3401.call.member state event every 30 minutes (against a 4-hour expires window), matching matrix-js-sdk's MatrixRTCSession. Each refresh carries a created_ts so Synapse can't dedupe identical state-event content. Tighten disconnect(): cancel the heartbeat first so it can't race the leave, then await removeCallMemberEvent() with a 2-second cap and await room.disconnect() — peers see us leave immediately instead of waiting up to 4 hours for expires_ts to fire. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Call/CallEncryptionService.swift | 79 ++------------- RelayKit/Call/CallViewModel.swift | 111 ++++++++++++++++++---- 2 files changed, 103 insertions(+), 87 deletions(-) diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift index 535b5f0..b3fc6e1 100644 --- a/RelayKit/Call/CallEncryptionService.swift +++ b/RelayKit/Call/CallEncryptionService.swift @@ -33,8 +33,9 @@ private let logger = Logger(subsystem: "RelayKit", category: "CallEncryption") /// What remains in this type: /// - ``sendCallMemberEvent(sfuServiceURL:)`` / ``removeCallMemberEvent()`` — /// MatrixRTC member state via `sendStateEventRaw` on the SDK room. -/// - ``enableCallPowerLevels()`` — ensures call state events are sendable -/// at PL 0 so ordinary members can join. +/// Rooms should be created with the correct power levels via +/// `powerLevelContentOverride` (see `MatrixService.callPowerLevels`); we +/// no longer try to mutate them at join time, matching Element Call. /// - ``generateKey()`` / ``setRawKey(_:on:participantId:index:)`` — /// LiveKit `BaseKeyProvider` plumbing that bypasses the String-based /// `setKey(...)` API so raw AES bytes are installed unmangled. @@ -81,11 +82,17 @@ struct CallEncryptionService { let stateKey = "_\(userID)_\(deviceID)_m.call" let serviceURL = sfuServiceURL.trimmingCharacters(in: .init(charactersIn: "/")) let membership = membershipId ?? "\(userID):\(deviceID)" + // `created_ts` makes each heartbeat a distinct event (Synapse can + // dedupe identical state-event content). It also gives peers a + // monotonic origin time for liveness tracking; matches the field + // matrix-js-sdk's `MatrixRTCSession` writes. + let createdTs = Int64(Date().timeIntervalSince1970 * 1000) // Match Element-X's exact format. let body: [String: Any] = [ "application": "m.call", "call_id": "", + "created_ts": createdTs, "device_id": deviceID, "expires": 14400000, "focus_active": [ @@ -211,74 +218,6 @@ struct CallEncryptionService { return targets.mapValues { Array($0) } } - // MARK: - Room Call Setup - - /// Ensures the room's power levels allow any member to send call-related - /// state events (`org.matrix.msc3401.call.member` and - /// `io.element.call.encryption_keys` at power level 0). - /// - /// Only succeeds if the current user has permission to update power levels - /// (typically room admins/mods). Fails silently for non-admin users — rooms - /// should be created with the correct power levels via `powerLevelContentOverride`. - func enableCallPowerLevels() async throws { - guard let sdkRoom else { - logger.warning("[RTC]No SDK room — cannot check/update power levels") - return - } - - // Check if we can already send call member events. - let powerLevels = try await sdkRoom.getPowerLevels() - let canSendCallMember = powerLevels.canOwnUserSendState(stateEvent: .callMember) - let canSendEncKeys = powerLevels.canOwnUserSendState( - stateEvent: .custom(value: Self.encryptionKeysEventType) - ) - - if canSendCallMember && canSendEncKeys { - logger.info("[RTC]Call power levels already allow sending") - return - } - - logger.info("[RTC]Call power levels need update (callMember=\(canSendCallMember), encKeys=\(canSendEncKeys))") - - // Fetch current power levels JSON, merge in call event types at PL 0, - // and re-send via the SDK. This only works if we're admin/mod. - let base = homeserver.trimmingCharacters(in: .init(charactersIn: "/")) - let encodedRoomID = roomID.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) ?? roomID - - guard let getURL = URL(string: "\(base)/_matrix/client/v3/rooms/\(encodedRoomID)/state/m.room.power_levels/") else { - return - } - - var getRequest = URLRequest(url: getURL) - getRequest.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") - - let (data, getResponse) = try await URLSession.shared.data(for: getRequest) - guard let http = getResponse as? HTTPURLResponse, http.statusCode == 200, - var plDict = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { - logger.warning("[RTC]Could not fetch power levels for update") - return - } - - var events = (plDict["events"] as? [String: Any]) ?? [:] - events[Self.callMemberEventType] = 0 - events[Self.encryptionKeysEventType] = 0 - plDict["events"] = events - - let jsonData = try JSONSerialization.data(withJSONObject: plDict) - let jsonString = String(data: jsonData, encoding: .utf8) ?? "{}" - - do { - _ = try await sdkRoom.sendStateEventRaw( - eventType: "m.room.power_levels", - stateKey: "", - content: jsonString - ) - logger.info("[RTC]Enabled call power levels for room") - } catch { - logger.warning("[RTC]Cannot update call power levels (likely not admin): \(error.localizedDescription)") - } - } - // MARK: - Key Generation /// Generates a cryptographically random 16-byte key suitable for AES-128-GCM. diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 39aab60..36f9ae1 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -92,6 +92,15 @@ public final class CallViewModel: CallViewModelProtocol { /// MatrixRTC member state events. @ObservationIgnored private var callMembers: [String: [String]] = [:] + /// Periodic refresh of the `org.matrix.msc3401.call.member` state event so + /// peers don't expire our membership while the call is in progress. + /// Element Call's matrix-js-sdk `MatrixRTCSession` does the equivalent. + @ObservationIgnored + private var heartbeatTask: Task? + /// Interval at which the call-member event is re-sent. Our `expires` + /// field is 4 hours; refreshing every 30 minutes keeps a generous + /// safety margin against missed sends. + private static let heartbeatInterval: Duration = .seconds(30 * 60) /// Creates a call view model without E2EE. Use ``init(encryptionContext:)`` /// for encrypted calls that interoperate with Element Call. @@ -298,26 +307,33 @@ public final class CallViewModel: CallViewModelProtocol { // Debug: log existing call member events to compare formats. await encryptionService.fetchCallMemberEvents() - // 1. Try to fix power levels (only works if we're admin/mod). - do { - try await encryptionService.enableCallPowerLevels() - } catch { - logger.warning("[RTC]Call power level setup failed: \(error.localizedDescription)") - } - - // 2. Send call membership state event (after power levels). - // Pass the widget bridge's membershipId UUID so the - // state-event `membershipID` matches the `member.id` - // field in our outbound encryption_keys payloads. + // 1. Send call membership state event. Pass the widget + // bridge's membershipId UUID so the state-event + // `membershipID` matches the `member.id` field in our + // outbound encryption_keys payloads. Power levels must + // already permit this (set at room creation via + // `MatrixService.callPowerLevels`); we no longer try to + // mutate them at join time, matching Element Call. + let membershipId = bridge?.membershipId do { try await encryptionService.sendCallMemberEvent( sfuServiceURL: sfuServiceURL, - membershipId: bridge?.membershipId + membershipId: membershipId ) } catch { logger.warning("[RTC]Call membership event failed: \(error.localizedDescription)") } + // 2. Start the membership heartbeat. matrix-js-sdk's + // `MatrixRTCSession` re-sends roughly every `expires/2`; + // we use a shorter interval to be safe against missed + // sends. Cancelled in `disconnect()`. + self.heartbeatTask = Self.startHeartbeat( + encryptionService: encryptionService, + sfuServiceURL: sfuServiceURL, + membershipId: membershipId + ) + // 3. Distribute the already-generated local key via the // widget bridge. The `messages` map for the // `send_to_device` action requires an explicit @@ -358,7 +374,8 @@ public final class CallViewModel: CallViewModelProtocol { } public func disconnect() async { - // Update UI state immediately — don't block on network I/O. + // Update UI state immediately — SwiftUI re-renders to the + // disconnected state while the awaited cleanup runs. state = .disconnected participants = [] isLocalCameraEnabled = false @@ -369,17 +386,77 @@ public final class CallViewModel: CallViewModelProtocol { localKeyIndex = 0 callMembers = [:] + // Stop the heartbeat first so it can't race the leave event and + // accidentally re-publish a fresh membership while we're tearing down. + heartbeatTask?.cancel() + heartbeatTask = nil + // Tear down the widget bridge synchronously so its tasks can't race // with subsequent connects. widgetBridge?.shutdown() widgetBridge = nil - // Network cleanup in background so the UI never beachballs. + // Proper cleanup: send the empty `m.call.member` content so peers + // see us leave immediately (otherwise they wait up to `expires` + // ms — 4 hours — before treating us as gone). Best-effort, capped + // by a short timeout so the UI never beach-balls if the homeserver + // is slow to respond. let service = encryptionService - let livekitRoom = room - Task.detached { + await Self.runWithTimeout(seconds: 2) { try? await service?.removeCallMemberEvent() - await livekitRoom.disconnect() + } + + await room.disconnect() + } + + /// Re-sends the call-member state event on a fixed interval until cancelled. + /// Detached from `self` so the loop body has no actor hop. + nonisolated private static func startHeartbeat( + encryptionService: CallEncryptionService, + sfuServiceURL: String, + membershipId: String? + ) -> Task { + Task.detached(priority: .background) { + // Local logger — the file-scope `logger` is inferred as + // MainActor-isolated and isn't reachable from a detached task. + let log = Logger(subsystem: "RelayKit", category: "Call") + while !Task.isCancelled { + do { + try await Task.sleep(for: heartbeatInterval) + } catch { + return // cancelled + } + if Task.isCancelled { return } + do { + try await encryptionService.sendCallMemberEvent( + sfuServiceURL: sfuServiceURL, + membershipId: membershipId + ) + log.debug("[RTC]Heartbeat refreshed call.member state event") + } catch { + log.warning("[RTC]Heartbeat refresh failed: \(error.localizedDescription)") + } + } + } + } + + /// Runs `work` and returns when it completes or after `seconds`, + /// whichever comes first. The work continues in the background after + /// the timeout; the caller just stops waiting. + nonisolated private static func runWithTimeout( + seconds: TimeInterval, + _ work: @Sendable @escaping () async -> Void + ) async { + let workTask: Task = Task.detached(priority: .userInitiated) { + await work() + } + await withTaskGroup(of: Void.self) { group in + group.addTask { await workTask.value } + group.addTask { + try? await Task.sleep(for: .seconds(seconds)) + } + await group.next() + group.cancelAll() } } From 84201d79fd94c94775e4a7c8730b372287aa2fcc Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 13:20:26 -0400 Subject: [PATCH 14/24] Tile remote participants in group calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two or more remotes are present, swap the FaceTime-style "primary + PiP" layout for a tiled grid that preserves each remote's source aspect ratio. Self always stays in the bottom-right PiP. Tile design: - Aspect-fitted card sized to the source video, centered in its grid cell against the call's dark gradient background — no harsh letterbox. - Soft drop shadow + near-invisible hairline edge for depth; speaking swaps the hairline for a soft accent-color glow (no hard border). - Solid black-tinted name capsule with mic.fill / mic.slash.fill badge — ultraThinMaterial blends into bright video frames and the text vanishes. - displayName(for:) strips Matrix identities (@user:server:device → user) so the pill shows a friendly localpart when the JWT didn't supply a name. Aspect updates live, not just on resize: - New videoAspectRatio(for:) on CallViewModelProtocol reads the underlying VideoTrack.dimensions. - The Delegate now also conforms to TrackDelegate and registers itself on every video track it sees, so dimension changes (rotation, simulcast layer switches, source swaps) bump videoTrackRevision and re-evaluate the tile. - RoomDelegate.didUpdateStreamState bumps too, so the aspect snaps to the real value as soon as the first frame arrives. Co-Authored-By: Claude Opus 4.7 --- .../Protocols/CallViewModelProtocol.swift | 8 + Relay/ViewModels/PreviewCallViewModel.swift | 1 + Relay/Views/CallView.swift | 224 +++++++++++++++++- RelayKit/Call/CallViewModel.swift | 50 +++- 4 files changed, 272 insertions(+), 11 deletions(-) diff --git a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift index 489bb01..20c72b3 100644 --- a/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift +++ b/Packages/RelayInterface/Sources/RelayInterface/Protocols/CallViewModelProtocol.swift @@ -110,4 +110,12 @@ public protocol CallViewModelProtocol: AnyObject, Observable { /// /// - Parameter participantID: The ``CallParticipant/id`` of the participant to render. func makeVideoView(for participantID: String) -> AnyView? + + /// Returns the aspect ratio (width / height) of the participant's currently + /// publishing video track, or `nil` if no track is available or its + /// dimensions haven't been negotiated yet. Tile-based UIs use this to + /// avoid stretching video — each tile can size itself to the source aspect. + /// + /// - Parameter participantID: The ``CallParticipant/id`` of the participant. + func videoAspectRatio(for participantID: String) -> CGFloat? } diff --git a/Relay/ViewModels/PreviewCallViewModel.swift b/Relay/ViewModels/PreviewCallViewModel.swift index bd262ab..cd2b277 100644 --- a/Relay/ViewModels/PreviewCallViewModel.swift +++ b/Relay/ViewModels/PreviewCallViewModel.swift @@ -73,4 +73,5 @@ final class PreviewCallViewModel: CallViewModelProtocol { } func makeVideoView(for participantID: String) -> AnyView? { nil } + func videoAspectRatio(for participantID: String) -> CGFloat? { 16.0 / 9.0 } } diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index e45d3a7..b231a35 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -87,11 +87,34 @@ struct CallView: View { @ViewBuilder private var connectedView: some View { ZStack { - // Primary video: first remote participant fills the window - primaryVideo - .ignoresSafeArea() + // Background gradient gives tiles something nicer to float on + // than pure black; keeps the FaceTime-on-Mac feel. + LinearGradient( + colors: [Color(white: 0.08), Color(white: 0.02)], + startPoint: .top, + endPoint: .bottom + ) + .ignoresSafeArea() + + // 1 remote → primary video fills. + // 2+ remotes → polished tile grid of remotes only. + if viewModel.participants.count >= 2 { + remoteTilesGrid + .padding(.horizontal, 12) + .padding(.top, 12) + .padding(.bottom, 96) // leave room for control bar + PiP + } else { + primaryVideo + .ignoresSafeArea() + + // Participant name at top (1:1 only — tiles label themselves) + VStack { + participantNameBar + Spacer() + } + } - // Self-view PiP in bottom-right corner + // Self-view PiP — always present, always bottom-right. if let localID = viewModel.localParticipantID { VStack { Spacer() @@ -104,12 +127,6 @@ struct CallView: View { .padding(.bottom, 72) } - // Participant name at top - VStack { - participantNameBar - Spacer() - } - // Floating control bar at bottom (always visible). VStack { Spacer() @@ -158,6 +175,55 @@ struct CallView: View { } } + // MARK: - Remote Tiles Grid (2+ remotes) + + /// Polished grid of every remote participant. The local view always + /// stays in the PiP overlay; remotes tile across the main area. + @ViewBuilder + private var remoteTilesGrid: some View { + GeometryReader { geo in + let remotes = viewModel.participants + let layout = Self.gridLayout(count: remotes.count, in: geo.size) + VStack(spacing: 8) { + ForEach(0.. (rows: Int, cols: Int) { + guard count > 0 else { return (1, 1) } + let isLandscape = size.width >= size.height + switch count { + case 1: return (1, 1) + case 2: return isLandscape ? (1, 2) : (2, 1) + case 3, 4: return (2, 2) + case 5, 6: return isLandscape ? (2, 3) : (3, 2) + case 7, 8, 9: return (3, 3) + default: + let cols = Int(ceil(Double(count).squareRoot())) + let rows = Int(ceil(Double(count) / Double(cols))) + return (rows, cols) + } + } + // MARK: - Self-View PiP @ViewBuilder @@ -452,6 +518,144 @@ private struct VideoRendererView: View { } } +// MARK: - Participant Tile + +/// A single tile in the remote-participants grid. Video (cropped to fill) +/// inside a rounded rect with a soft shadow, a name pill bottom-left, and +/// a faint outer glow when the participant is speaking. Mirrors the +/// FaceTime-on-Mac aesthetic: clean cards, no hard borders. +private struct ParticipantTile: View { + let viewModel: any CallViewModelProtocol + let participant: CallParticipant + + private static let cornerRadius: CGFloat = 14 + /// Aspect used for camera-off tiles or before the first frame arrives. + private static let placeholderAspect: CGFloat = 16.0 / 9.0 + + var body: some View { + let shape = RoundedRectangle(cornerRadius: Self.cornerRadius, style: .continuous) + // Re-evaluate when video tracks change so we pick up the real + // dimensions after the first frame (RoomDelegate bumps + // videoTrackRevision on streamState transitions). + let _ = viewModel.videoTrackRevision + let aspect: CGFloat = { + if participant.isCameraEnabled, + let live = viewModel.videoAspectRatio(for: participant.id) { + return live + } + return Self.placeholderAspect + }() + + ZStack(alignment: .bottomLeading) { + // Card background — neutral so video looks at home. + shape.fill( + LinearGradient( + colors: [Color(white: 0.18), Color(white: 0.10)], + startPoint: .top, + endPoint: .bottom + ) + ) + + if participant.isCameraEnabled { + VideoRendererView(viewModel: viewModel, participantID: participant.id) { + placeholder + } + .clipShape(shape) + } else { + placeholder + } + + nameLabel + .padding(10) + } + // Tile sizes itself to the source video aspect, centered in the + // grid cell. Surrounding cell area is transparent so the + // background gradient shows through (no harsh letterbox). + // Modifier order matters: shadow + overlay must apply to the + // aspect-fitted shape, then the outer frame centers it in the cell. + .aspectRatio(aspect, contentMode: .fit) + .shadow(color: .black.opacity(0.35), radius: 8, y: 2) + .overlay(speakingGlow.allowsHitTesting(false)) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + + // MARK: Subviews + + @ViewBuilder + private var placeholder: some View { + VStack(spacing: 8) { + Image(systemName: "person.fill") + .font(.system(size: 44)) + .foregroundStyle(.white.opacity(0.35)) + Text(Self.displayName(for: participant)) + .font(.callout.weight(.medium)) + .foregroundStyle(.white.opacity(0.6)) + .lineLimit(1) + .padding(.horizontal, 12) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + + @ViewBuilder + private var nameLabel: some View { + // Always show mic state next to the name. Filled mic icon when on, + // slashed (red-tinted) when muted — mirrors FaceTime / Zoom badges. + // Solid dark capsule for guaranteed contrast over any video frame — + // .ultraThinMaterial blends into bright frames and the name vanishes. + HStack(spacing: 6) { + Image(systemName: participant.isMicrophoneEnabled ? "mic.fill" : "mic.slash.fill") + .font(.caption.weight(.semibold)) + .foregroundStyle(participant.isMicrophoneEnabled ? .white : .red) + Text(Self.displayName(for: participant)) + .font(.caption.weight(.semibold)) + .foregroundStyle(.white) + .lineLimit(1) + .truncationMode(.tail) + } + .padding(.horizontal, 10) + .padding(.vertical, 5) + .background(Color.black.opacity(0.55), in: Capsule()) + .overlay( + Capsule().strokeBorder(Color.white.opacity(0.12), lineWidth: 0.5) + ) + .shadow(color: .black.opacity(0.4), radius: 3, y: 1) + } + + /// Pulls a friendly name out of the participant: `displayName` if the + /// SFU/JWT supplied one, otherwise the localpart of the Matrix user ID + /// (`@andrew:matrix.example.com:DEVICE` → `andrew`). Falls back to the + /// raw id if neither pattern matches. + static func displayName(for p: CallParticipant) -> String { + if let dn = p.displayName, !dn.isEmpty { return dn } + let id = p.id + // LiveKit identity layout used by Element Call: + // `@::` — strip server + device. + if id.hasPrefix("@") { + let body = id.dropFirst() + if let colon = body.firstIndex(of: ":") { + let localpart = body[.. CGFloat? { + let isLocal = room.localParticipant.identity?.stringValue == participantID + let participant: Participant? = isLocal + ? room.localParticipant + : room.remoteParticipants.values.first { $0.identity?.stringValue == participantID } + + guard let publication = participant?.videoTracks.first, + !publication.isMuted, + let track = publication.track as? VideoTrack else { + return nil + } + if let remotePub = publication as? RemoteTrackPublication, !remotePub.isSubscribed { + return nil + } + guard let dim = track.dimensions, dim.height > 0 else { return nil } + return CGFloat(dim.width) / CGFloat(dim.height) + } + public func makeVideoView(for participantID: String) -> AnyView? { let isLocal = room.localParticipant.identity?.stringValue == participantID let participant: Participant? = isLocal @@ -590,11 +608,31 @@ public final class CallViewModel: CallViewModelProtocol { /// the main actor so that `CallViewModel`'s `@Observable` state is always mutated /// safely. The class is `@unchecked Sendable` because `viewModel` is a weak reference /// that is only read inside `Task { @MainActor in … }` blocks. - private final class Delegate: RoomDelegate, @unchecked Sendable { + /// + /// Also conforms to ``TrackDelegate`` so it can observe per-track + /// dimension changes (e.g. a remote rotating their camera, simulcast + /// layer changes). LiveKit's `RoomDelegate` does not surface those. + private final class Delegate: NSObject, RoomDelegate, TrackDelegate, @unchecked Sendable { weak var viewModel: CallViewModel? init(viewModel: CallViewModel) { self.viewModel = viewModel + super.init() + } + + /// Bumps `videoTrackRevision` whenever a track's dimensions change, + /// so SwiftUI tiles re-read `videoAspectRatio(for:)`. + func track(_ track: VideoTrack, didUpdateDimensions dimensions: Dimensions?) { + Task { @MainActor [weak viewModel] in + viewModel?.videoTrackRevision += 1 + } + } + + /// Attaches `self` as a `TrackDelegate` on a publication's underlying + /// video track if present. Multicast — safe to call repeatedly. + func observeDimensions(of publication: TrackPublication?) { + guard let videoTrack = publication?.track as? VideoTrack else { return } + videoTrack.add(delegate: self) } func room(_ room: LiveKit.Room, didUpdateConnectionState connectionState: LiveKit.ConnectionState, from oldValue: LiveKit.ConnectionState) { @@ -631,6 +669,7 @@ public final class CallViewModel: CallViewModelProtocol { } func room(_ room: LiveKit.Room, participant: RemoteParticipant, didSubscribeTrack publication: RemoteTrackPublication) { + observeDimensions(of: publication) Task { @MainActor [weak viewModel] in let identityStr = participant.identity?.stringValue ?? "(none)" let kind = publication.kind.rawValue @@ -654,6 +693,7 @@ public final class CallViewModel: CallViewModelProtocol { } func room(_ room: LiveKit.Room, localParticipant: LocalParticipant, didPublishTrack publication: LocalTrackPublication) { + observeDimensions(of: publication) Task { @MainActor [weak viewModel] in viewModel?.videoTrackRevision += 1 } @@ -664,5 +704,13 @@ public final class CallViewModel: CallViewModelProtocol { viewModel?.syncParticipants(trackChanged: true) } } + + // First-frame indicator: dimensions become valid here, so bump + // videoTrackRevision so aspect-ratio observers re-read. + func room(_ room: LiveKit.Room, participant: RemoteParticipant, trackPublication: RemoteTrackPublication, didUpdateStreamState streamState: StreamState) { + Task { @MainActor [weak viewModel] in + viewModel?.videoTrackRevision += 1 + } + } } } From 3ce460f8bdd6bae1bbf897ea5213ce65e32f9553 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 13:24:24 -0400 Subject: [PATCH 15/24] Show camera-off placeholder immediately on remote mute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire up the missing LiveKit RoomDelegate callbacks so toggling a remote's camera or mic flips the tile state instantly instead of waiting for an unrelated sync to fire: - didUpdateIsMuted: refresh participants so isCameraEnabled / isMicrophoneEnabled flip and the tile re-evaluates makeVideoView (returns nil for muted tracks → placeholder appears). - didUnpublishTrack / didUnsubscribeTrack (remote): same path. - didUnpublishTrack (local): bump videoTrackRevision so the self PiP swaps to the off state. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Call/CallViewModel.swift | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 3224bd8..8e75079 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -712,5 +712,37 @@ public final class CallViewModel: CallViewModelProtocol { viewModel?.videoTrackRevision += 1 } } + + // A peer toggled their camera/mic. We need to refresh the participant + // snapshot (so `isCameraEnabled` / `isMicrophoneEnabled` flip) AND + // bump videoTrackRevision so the tile body re-evaluates and + // `makeVideoView` returns nil for the muted track — which surfaces + // the placeholder immediately instead of waiting for the next + // unrelated sync. + func room(_ room: LiveKit.Room, participant: Participant, trackPublication: TrackPublication, didUpdateIsMuted isMuted: Bool) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants(trackChanged: true) + } + } + + // Track-removed events behave the same way for our UI: refresh + // participant state and bump the revision so the placeholder shows. + func room(_ room: LiveKit.Room, participant: RemoteParticipant, didUnpublishTrack publication: RemoteTrackPublication) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants(trackChanged: true) + } + } + + func room(_ room: LiveKit.Room, participant: RemoteParticipant, didUnsubscribeTrack publication: RemoteTrackPublication) { + Task { @MainActor [weak viewModel] in + viewModel?.syncParticipants(trackChanged: true) + } + } + + func room(_ room: LiveKit.Room, participant: LocalParticipant, didUnpublishTrack publication: LocalTrackPublication) { + Task { @MainActor [weak viewModel] in + viewModel?.videoTrackRevision += 1 + } + } } } From aeccc30561b4b396a6b373dc7945795825f0bc92 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 13:28:33 -0400 Subject: [PATCH 16/24] Auto-close call window on clean disconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the "Call Ended" + Dismiss-button overlay for normal endings. The .disconnected case now renders Color.clear with a .task that fires onDismiss() the moment the branch mounts, so the window closes immediately. Background cleanup (removeCallMemberEvent, LiveKit teardown) continues in the existing disconnect() task. The endedOverlay is kept for .failed only — errors still need a UI so users can read what went wrong before dismissing. Co-Authored-By: Claude Opus 4.7 --- Relay/Views/CallView.swift | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Relay/Views/CallView.swift b/Relay/Views/CallView.swift index b231a35..f9cddc6 100644 --- a/Relay/Views/CallView.swift +++ b/Relay/Views/CallView.swift @@ -63,13 +63,15 @@ struct CallView: View { connectedView case .disconnected: - endedOverlay( - title: "Call Ended", - systemImage: "phone.down.fill", - isError: false - ) + // Clean ending — close the window immediately. No overlay, + // no "Dismiss" button. Background cleanup (leave event, + // LiveKit teardown) continues in disconnect()'s task. + Color.clear + .task { onDismiss() } case .failed(let message): + // Errors still show the overlay so the user can read what + // went wrong before dismissing. endedOverlay( title: "Call Failed", systemImage: "exclamationmark.triangle.fill", @@ -298,8 +300,8 @@ struct CallView: View { // End call Button { - // Only disconnect — the endedOverlay auto-dismiss - // will call onDismiss() after a brief delay. + // Disconnect — the .disconnected case in `body` calls + // onDismiss() immediately so the window closes. Task { await viewModel.disconnect() } } label: { Image(systemName: "phone.down.fill") @@ -386,7 +388,10 @@ struct CallView: View { } } - // MARK: - Ended/Failed Overlay + // MARK: - Failed Overlay + // + // Clean endings auto-close via `.disconnected` in `body`. This overlay + // is only used for failures so the user sees the error before dismissing. @ViewBuilder private func endedOverlay(title: String, systemImage: String, isError: Bool, detail: String? = nil) -> some View { @@ -411,13 +416,6 @@ struct CallView: View { .padding(.top, 4) Spacer() } - .task { - // Auto-dismiss after a few seconds for clean endings. - if !isError { - try? await Task.sleep(for: .seconds(2)) - onDismiss() - } - } } // MARK: - Join Form (manual entry fallback) From a54214b88da9038aac482bec2076b395767cfec2 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 13:43:24 -0400 Subject: [PATCH 17/24] Plug secret leaks in [RTC] development logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of the [RTC]-prefixed logs added during LiveKit / Element Call interop work surfaced several places that wrote sensitive data to the system log at .public privacy. Critical: - CallWidgetBridge.recvLoop logged the raw JSON of every widget driver message at .public — including outbound and inbound send_to_device payloads of type io.element.call.encryption_keys whose `keys.key` field carries the raw 16-byte AES key. Replaced with a byte-count-only debug log; action + type are still logged separately one line later for traceability. Defensive: - m.call.member event body, state key, and existing-call-member content dropped to .debug and marked .private. Routing data and per-call membership UUIDs aren't secrets but don't belong in Console output either. - LiveKitLogBridge now forwards all SDK log content as .private — the SDK can surface JWTs, signaling URLs, or handshake details at its own discretion. - All error.localizedDescription interpolations in [RTC] logs now carry privacy: .private. SDK error strings can embed request URLs, tokens, or response bodies. Already safe and left alone: - AES keys are only ever logged as sha256[0..8] fingerprints. - OpenID and LiveKit JWTs are never logged. - Matrix user/device IDs intentionally remain .public — they're observable on the homeserver, not secret. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Call/CallEncryptionService.swift | 9 ++++++--- RelayKit/Call/CallViewModel.swift | 12 ++++++------ RelayKit/Call/CallWidgetBridge.swift | 11 ++++++++--- RelayKit/Call/LiveKitLogBridge.swift | 12 ++++++++---- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/RelayKit/Call/CallEncryptionService.swift b/RelayKit/Call/CallEncryptionService.swift index b3fc6e1..d3e359b 100644 --- a/RelayKit/Call/CallEncryptionService.swift +++ b/RelayKit/Call/CallEncryptionService.swift @@ -113,8 +113,10 @@ struct CallEncryptionService { let jsonData = try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) let jsonString = String(data: jsonData, encoding: .utf8) ?? "{}" - logger.info("[RTC]Call member event body: \(jsonString)") - logger.info("[RTC]Call member state key: \(stateKey)") + // Body + state key contain device IDs and per-call membership UUIDs; + // not raw secrets but routing data we don't need leaking to Console. + logger.debug("[RTC]Call member event body: \(jsonString, privacy: .private)") + logger.debug("[RTC]Call member state key: \(stateKey, privacy: .private)") _ = try await sdkRoom.sendStateEventRaw( eventType: Self.callMemberEventType, @@ -165,7 +167,8 @@ struct CallEncryptionService { if let content = event["content"], let contentData = try? JSONSerialization.data(withJSONObject: content, options: [.sortedKeys]), let contentStr = String(data: contentData, encoding: .utf8) { - logger.info("[RTC]Existing call member [key=\(stateKey)]: \(contentStr)") + // .private — call routing data + device IDs, not for Console. + logger.debug("[RTC]Existing call member [key=\(stateKey, privacy: .private)]: \(contentStr, privacy: .private)") } } } diff --git a/RelayKit/Call/CallViewModel.swift b/RelayKit/Call/CallViewModel.swift index 8e75079..6bb2cb0 100644 --- a/RelayKit/Call/CallViewModel.swift +++ b/RelayKit/Call/CallViewModel.swift @@ -249,7 +249,7 @@ public final class CallViewModel: CallViewModelProtocol { bridge.start() self.widgetBridge = bridge } catch { - logger.error("[RTC]Failed to create CallWidgetBridge: \(error.localizedDescription)") + logger.error("[RTC]Failed to create CallWidgetBridge: \(error.localizedDescription, privacy: .private)") } } @@ -321,7 +321,7 @@ public final class CallViewModel: CallViewModelProtocol { membershipId: membershipId ) } catch { - logger.warning("[RTC]Call membership event failed: \(error.localizedDescription)") + logger.warning("[RTC]Call membership event failed: \(error.localizedDescription, privacy: .private)") } // 2. Start the membership heartbeat. matrix-js-sdk's @@ -352,7 +352,7 @@ public final class CallViewModel: CallViewModelProtocol { toMembers: targets ) } catch { - logger.warning("[RTC]Widget-bridge key distribution failed: \(error.localizedDescription)") + logger.warning("[RTC]Widget-bridge key distribution failed: \(error.localizedDescription, privacy: .private)") } } } @@ -367,7 +367,7 @@ public final class CallViewModel: CallViewModelProtocol { state = .connected videoTrackRevision += 1 } catch { - logger.error("[RTC]Connect failed: \(error.localizedDescription)") + logger.error("[RTC]Connect failed: \(error.localizedDescription, privacy: .private)") state = .failed(error.localizedDescription) throw error } @@ -434,7 +434,7 @@ public final class CallViewModel: CallViewModelProtocol { ) log.debug("[RTC]Heartbeat refreshed call.member state event") } catch { - log.warning("[RTC]Heartbeat refresh failed: \(error.localizedDescription)") + log.warning("[RTC]Heartbeat refresh failed: \(error.localizedDescription, privacy: .private)") } } } @@ -559,7 +559,7 @@ public final class CallViewModel: CallViewModelProtocol { ) logger.info("[RTC]Redistributed key to \(participantIdentity, privacy: .private)") } catch { - logger.warning("[RTC]Key redistribution failed for \(participantIdentity, privacy: .private): \(error.localizedDescription)") + logger.warning("[RTC]Key redistribution failed for \(participantIdentity, privacy: .private): \(error.localizedDescription, privacy: .private)") } } } diff --git a/RelayKit/Call/CallWidgetBridge.swift b/RelayKit/Call/CallWidgetBridge.swift index 7c14e30..96f0d80 100644 --- a/RelayKit/Call/CallWidgetBridge.swift +++ b/RelayKit/Call/CallWidgetBridge.swift @@ -204,7 +204,7 @@ public final class CallWidgetBridge: @unchecked Sendable { try await self?.sendRequest(action: "content_loaded", data: [:]) logger.info("[RTC]Widget content_loaded acknowledged by driver") } catch { - logger.warning("[RTC]content_loaded failed: \(error.localizedDescription)") + logger.warning("[RTC]content_loaded failed: \(error.localizedDescription, privacy: .private)") } } @@ -402,11 +402,16 @@ public final class CallWidgetBridge: @unchecked Sendable { break } - logger.info("[RTC]widget recv: \(raw, privacy: .public)") + // SECURITY: never log the raw widget JSON. Outbound and inbound + // `send_to_device` payloads of type `io.element.call.encryption_keys` + // carry raw AES keys in the `keys.key` field — those would land + // unredacted in the system log. Action / type only; full bodies + // are .private so they're stripped from non-debug Console output. + logger.debug("[RTC]widget recv (\(raw.count) bytes)") guard let data = raw.data(using: .utf8), let msg = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - logger.warning("[RTC]Non-JSON message from widget driver: \(raw, privacy: .public)") + logger.warning("[RTC]Non-JSON message from widget driver: \(raw, privacy: .private)") continue } diff --git a/RelayKit/Call/LiveKitLogBridge.swift b/RelayKit/Call/LiveKitLogBridge.swift index ce51878..4de7926 100644 --- a/RelayKit/Call/LiveKitLogBridge.swift +++ b/RelayKit/Call/LiveKitLogBridge.swift @@ -46,15 +46,19 @@ struct LiveKitLogBridge: LiveKit.Logger { return "[RTC] \(typeName).\(function) \(message().description)\(meta)" }() + // SECURITY: LiveKit SDK log content is at the SDK's discretion and + // can include connection JWTs, signaling URLs, peer identities, etc. + // Mark as .private so the Console redacts it on release; developers + // can still see the messages by enabling unredacted logging in Xcode. switch level { case .debug: - Self.osLogger.debug("\(rendered, privacy: .public)") + Self.osLogger.debug("\(rendered, privacy: .private)") case .info: - Self.osLogger.info("\(rendered, privacy: .public)") + Self.osLogger.info("\(rendered, privacy: .private)") case .warning: - Self.osLogger.warning("\(rendered, privacy: .public)") + Self.osLogger.warning("\(rendered, privacy: .private)") case .error: - Self.osLogger.error("\(rendered, privacy: .public)") + Self.osLogger.error("\(rendered, privacy: .private)") } } } From 07dd527ea27cc942b39ccdcd298884e74b1d7855 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 13:56:14 -0400 Subject: [PATCH 18/24] Restore RelayKit/Widget/WidgetProxy.swift placeholder Inadvertently dropped in bbf9580 ("Fix LiveKit E2EE interop with Element Call") during incidental cleanup. The stub is unreferenced by any build target on either branch, but restoring it keeps this branch's tree consistent with upstream/main and avoids a cosmetic deletion in the PR diff. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Widget/WidgetProxy.swift | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 RelayKit/Widget/WidgetProxy.swift diff --git a/RelayKit/Widget/WidgetProxy.swift b/RelayKit/Widget/WidgetProxy.swift new file mode 100644 index 0000000..996c8b4 --- /dev/null +++ b/RelayKit/Widget/WidgetProxy.swift @@ -0,0 +1,28 @@ +// Copyright 2026 Link Dupont +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// WidgetProxy.swift +// RelayKit +// +// SPDX-License-Identifier: Apache-2.0 + +/// Manages Matrix widget lifecycle and communication. +/// +/// Provides a bridge for embedding Matrix widgets in SwiftUI views. +/// The widget API is evolving; this proxy will be expanded as the +/// SDK stabilizes. +public final class WidgetProxy: @unchecked Sendable { + /// Creates a widget proxy. + public init() {} +} From 4764a10639f4105178dced564e1560b36d50d2a5 Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Sat, 25 Apr 2026 14:01:27 -0400 Subject: [PATCH 19/24] Render call.member events as "started a call" in the surgical map path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-item TimelineMessageMapper.mapItem path was bypassing describeStateEvent and falling through to the generic stateEventDescription, which surfaces "Room settings were updated" for any custom state event — including org.matrix.msc3401.call.member. Route .state through describeStateEvent like the bulk and rebuild paths already do, so MatrixRTC call membership renders as "X started a call" with .callEvent kind, and the noisy io.element.call.encryption_keys events are filtered out. Marks describeStateEvent nonisolated (it's pure) so mapItem can call it from its nonisolated context. Co-Authored-By: Claude Opus 4.7 --- RelayKit/Services/TimelineMessageMapper.swift | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/RelayKit/Services/TimelineMessageMapper.swift b/RelayKit/Services/TimelineMessageMapper.swift index 9b3c4a1..6558b89 100644 --- a/RelayKit/Services/TimelineMessageMapper.swift +++ b/RelayKit/Services/TimelineMessageMapper.swift @@ -464,9 +464,23 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len prevAvatarUrl: prevAvatarUrl ) msgKind = .profileChange - case .state(_, let content): - msgBody = Self.stateEventDescription(content) - msgKind = .stateEvent + case .state(let stateKey, let content): + // Use describeStateEvent so call membership events render as + // "X started a call" with .callEvent kind, and so the noisy + // io.element.call.encryption_keys events are filtered out — + // matching the bulk-mapping and rebuild paths. + let (body, kind) = Self.describeStateEvent( + content, + stateKey: stateKey, + senderDisplayName: { + if case .ready(let name, _, _) = event.senderProfile { return name } + return nil + }(), + senderId: event.sender + ) + guard let body else { return nil } + msgBody = body + msgKind = kind default: return nil } @@ -978,7 +992,7 @@ struct TimelineMessageMapper: Sendable { // swiftlint:disable:this type_body_len /// Routes a state event to the appropriate description and message kind. /// /// Returns `nil` body for events that should be hidden (e.g. encryption key exchange). - static func describeStateEvent( + nonisolated static func describeStateEvent( _ state: OtherState, stateKey: String, senderDisplayName: String?, From c89614876105250b32129a06e9ada5b2794661c7 Mon Sep 17 00:00:00 2001 From: Link Dupont Date: Tue, 28 Apr 2026 19:06:12 -0400 Subject: [PATCH 20/24] Resolve package graph --- .../xcshareddata/swiftpm/Package.resolved | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 2330cd9..bc82ea0 100644 --- a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,12 +1,12 @@ { - "originHash" : "0b3fc90c7698ae92e7b431d3f3402e432621245a4f88792e53c71e7d7dc7a510", + "originHash" : "511add025e69bc3f3ba18f303ad317ba3b8652d9929dea52c11fb543b1f34c77", "pins" : [ { "identity" : "client-sdk-swift", "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "revision" : "4e930e856e3b076c2aacce98c77cc81fd2db498b", + "revision" : "f70911172031fe40042266ed3bc35171224c2ef0", "version" : "2.13.0" } }, @@ -28,15 +28,6 @@ "version" : "26.4.1" } }, - { - "identity" : "swift-collections", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-collections.git", - "state" : { - "revision" : "8c0c0a8b49e080e54e5e328cc552821ff07cd341", - "version" : "1.2.1" - } - }, { "identity" : "swift-protobuf", "kind" : "remoteSourceControl", @@ -51,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/webrtc-xcframework.git", "state" : { - "revision" : "e2a0ab3be155475ad60f845813f2088847e584f7", - "version" : "144.7559.03" + "revision" : "f6017e189972b86f90e0ec406b1629828bc75748", + "version" : "144.7559.3" } } ], From 4e7a0a7b33fcff397b6cd05952704eab96baa2a4 Mon Sep 17 00:00:00 2001 From: Link Dupont Date: Tue, 28 Apr 2026 19:12:15 -0400 Subject: [PATCH 21/24] Restore Secrets.xcconfig as base configuration for all build configs The pbxproj conflict resolution during rebase dropped the PBXFileReference, group entry, and all six baseConfigurationReference entries for Secrets.xcconfig. Without these, DEVELOPMENT_TEAM and GIPHY_API_KEY are not loaded as build settings and the Generate Secrets build phase cannot read them from the environment. Assisted-By: Claude (OpenCode) --- Relay.xcodeproj/project.pbxproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Relay.xcodeproj/project.pbxproj b/Relay.xcodeproj/project.pbxproj index 2de8e3b..ada5f82 100644 --- a/Relay.xcodeproj/project.pbxproj +++ b/Relay.xcodeproj/project.pbxproj @@ -51,6 +51,7 @@ 3B4AFD892F638A35001F0EA1 /* Relay.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Relay.app; sourceTree = BUILT_PRODUCTS_DIR; }; 3BAE76A72F6A43BA000EC1E6 /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 3BRK00002FC00000001F0EA1 /* RelayKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = RelayKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Secrets.xcconfig; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ @@ -117,6 +118,7 @@ 3B4AFD802F638A35001F0EA1 = { isa = PBXGroup; children = ( + 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */, 3BAE76A72F6A43BA000EC1E6 /* README.md */, 3B4AFD8B2F638A35001F0EA1 /* Relay */, 3BRK00112FC00011001F0EA1 /* RelayKit */, @@ -397,6 +399,7 @@ }; 3B4AFDAA2F638A36001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; @@ -464,6 +467,7 @@ }; 3B4AFDAB2F638A36001F0EA1 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; @@ -523,6 +527,7 @@ }; 3B4AFDAD2F638A36001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIconDev; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColorDev; @@ -572,6 +577,7 @@ }; 3B4AFDAE2F638A36001F0EA1 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; @@ -621,6 +627,7 @@ }; 3BRK00212FC00021001F0EA1 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { CODE_SIGN_IDENTITY = ""; CODE_SIGN_STYLE = Automatic; @@ -654,6 +661,7 @@ }; 3BRK00222FC00022001F0EA1 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 3BSEC0012FEF0001001F0EA1 /* Secrets.xcconfig */; buildSettings = { CODE_SIGN_IDENTITY = ""; CODE_SIGN_STYLE = Automatic; From d28365ae5c334b8a924320377e16e105c3379524 Mon Sep 17 00:00:00 2001 From: Link Dupont Date: Thu, 30 Apr 2026 08:28:57 -0400 Subject: [PATCH 22/24] Restore call button in toolbar The toolbar conflict resolution during rebase kept main's structure but dropped the call button ToolbarItem and startCallButton helper that the LiveKit branch added. Assisted-By: Claude (OpenCode) --- Relay/Views/MainView.swift | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Relay/Views/MainView.swift b/Relay/Views/MainView.swift index cf1afcb..4cd3271 100644 --- a/Relay/Views/MainView.swift +++ b/Relay/Views/MainView.swift @@ -300,6 +300,11 @@ struct MainView: View { // swiftlint:disable:this type_body_length } if !appActions.showRoomDirectory && previewingInvite == nil { + if let selectedRoomId, currentRoom != nil { + ToolbarItem(placement: .primaryAction) { + startCallButton(roomId: selectedRoomId) + } + } ToolbarItem(placement: .primaryAction) { showInspectorButton } @@ -307,6 +312,16 @@ struct MainView: View { // swiftlint:disable:this type_body_length } + private func startCallButton(roomId: String) -> some View { + Button { + startCall(roomId: roomId) + } label: { + Label("Start Call", systemImage: "phone.fill") + } + .help("Start Call") + .disabled(callManager.hasActiveCall) + } + private var toolbarTitleCapsule: some View { HStack(spacing: 0) { if let currentRoom { From 83b804826859521066f0f0f08181094b6478b487 Mon Sep 17 00:00:00 2001 From: Link Dupont Date: Thu, 30 Apr 2026 08:54:26 -0400 Subject: [PATCH 23/24] Fix .well-known RTC lookup for delegated homeservers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchWellKnownSFUURL was querying .well-known on the delegated homeserver URL (e.g. fedora.ems.host) which does not serve .well-known. Matrix requires .well-known to be fetched from the server name domain (e.g. fedora.im), which is the part after ":" in the user ID. Extract the server name from the user ID and use it for the .well-known lookup so servers with delegation (like fedora.im → fedora.ems.host) correctly discover the rtc_foci LiveKit SFU URL. Assisted-By: Claude (OpenCode) --- RelayKit/Call/LiveKitCredentialService.swift | 10 +++++----- RelayKit/Services/MatrixService.swift | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/RelayKit/Call/LiveKitCredentialService.swift b/RelayKit/Call/LiveKitCredentialService.swift index a575e3b..c9aaee9 100644 --- a/RelayKit/Call/LiveKitCredentialService.swift +++ b/RelayKit/Call/LiveKitCredentialService.swift @@ -41,6 +41,10 @@ struct LiveKitCredentialService { let accessToken: String let userID: String let deviceID: String + /// The Matrix server name (e.g. `fedora.im`) extracted from the user ID. + /// Used for `.well-known` lookups, which must query the server name domain, + /// not the delegated homeserver URL (e.g. `fedora.ems.host`). + let serverName: String // MARK: - Public Entry Point @@ -91,11 +95,7 @@ struct LiveKitCredentialService { } private func fetchWellKnownSFUURL() async throws -> String { - guard let serverURL = URL(string: homeserver), let host = serverURL.host else { - throw LiveKitCredentialError.invalidURL - } - let scheme = serverURL.scheme ?? "https" - guard let url = URL(string: "\(scheme)://\(host)/.well-known/matrix/client") else { + guard let url = URL(string: "https://\(serverName)/.well-known/matrix/client") else { throw LiveKitCredentialError.invalidURL } diff --git a/RelayKit/Services/MatrixService.swift b/RelayKit/Services/MatrixService.swift index def4952..e63f64d 100644 --- a/RelayKit/Services/MatrixService.swift +++ b/RelayKit/Services/MatrixService.swift @@ -1495,11 +1495,15 @@ public final class MatrixService: MatrixServiceProtocol { throw LiveKitCredentialError.serverError } let session = try client.session() + // Extract the server name from the user ID (e.g. "@user:fedora.im" → "fedora.im"). + // .well-known must be queried on the server name domain, not the delegated homeserver. + let serverName = client.userID.split(separator: ":").dropFirst().joined(separator: ":") let service = LiveKitCredentialService( homeserver: client.homeserver, accessToken: session.accessToken, userID: client.userID, - deviceID: client.deviceID + deviceID: client.deviceID, + serverName: serverName ) let result = try await service.credentials(for: roomId) return (livekitURL: result.url, token: result.token, sfuServiceURL: result.sfuServiceURL) From 10d6fe1b90a78446a49f80b910b416fe87287f6f Mon Sep 17 00:00:00 2001 From: Andrew Hunter Date: Wed, 29 Apr 2026 19:03:41 -0400 Subject: [PATCH 24/24] Add Livekit as a framework dependancy --- Relay.xcodeproj/project.pbxproj | 16 ++++++++++++++++ .../xcshareddata/swiftpm/Package.resolved | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Relay.xcodeproj/project.pbxproj b/Relay.xcodeproj/project.pbxproj index ada5f82..2727f8d 100644 --- a/Relay.xcodeproj/project.pbxproj +++ b/Relay.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ 3BRK00012FC00001001F0EA1 /* RelayKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3BRK00002FC00000001F0EA1 /* RelayKit.framework */; }; 3BRK00022FC00002001F0EA1 /* RelayKit.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 3BRK00002FC00000001F0EA1 /* RelayKit.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; DF41154C2F82F7A30028241B /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = 3BLK00022FD10002001F0EA1 /* LiveKit */; }; + DFDAFCB22FA2C56400A27353 /* LiveKit in Frameworks */ = {isa = PBXBuildFile; productRef = DFDAFCB12FA2C56400A27353 /* LiveKit */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -97,6 +98,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + DFDAFCB22FA2C56400A27353 /* LiveKit in Frameworks */, 3BRK00012FC00001001F0EA1 /* RelayKit.framework in Frameworks */, 3BRI00032FC10003001F0EA1 /* RelayInterface in Frameworks */, ); @@ -123,6 +125,7 @@ 3B4AFD8B2F638A35001F0EA1 /* Relay */, 3BRK00112FC00011001F0EA1 /* RelayKit */, 3B11E0892F9A3F600051F7B3 /* RelayTests */, + DFDAFCAE2FA2C4F700A27353 /* Frameworks */, 3B4AFD8A2F638A35001F0EA1 /* Products */, ); sourceTree = ""; @@ -137,6 +140,13 @@ name = Products; sourceTree = ""; }; + DFDAFCAE2FA2C4F700A27353 /* Frameworks */ = { + isa = PBXGroup; + children = ( + ); + name = Frameworks; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ @@ -194,6 +204,7 @@ name = Relay; packageProductDependencies = ( 3BRI00042FC10004001F0EA1 /* RelayInterface */, + DFDAFCB12FA2C56400A27353 /* LiveKit */, ); productName = Relay; productReference = 3B4AFD892F638A35001F0EA1 /* Relay.app */; @@ -779,6 +790,11 @@ isa = XCSwiftPackageProductDependency; productName = RelayInterface; }; + DFDAFCB12FA2C56400A27353 /* LiveKit */ = { + isa = XCSwiftPackageProductDependency; + package = 3BLK00032FD10003001F0EA1 /* XCRemoteSwiftPackageReference "client-sdk-swift" */; + productName = LiveKit; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 3B4AFD812F638A35001F0EA1 /* Project object */; diff --git a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index bc82ea0..f2b2069 100644 --- a/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Relay.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,12 +1,12 @@ { - "originHash" : "511add025e69bc3f3ba18f303ad317ba3b8652d9929dea52c11fb543b1f34c77", + "originHash" : "beed1a9e4d2e17ac1e82e835ac2cc90809f53ecb8556bc36ab04903d7e3cf291", "pins" : [ { "identity" : "client-sdk-swift", "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift", "state" : { - "revision" : "f70911172031fe40042266ed3bc35171224c2ef0", + "revision" : "4e930e856e3b076c2aacce98c77cc81fd2db498b", "version" : "2.13.0" } }, @@ -42,7 +42,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/webrtc-xcframework.git", "state" : { - "revision" : "f6017e189972b86f90e0ec406b1629828bc75748", + "revision" : "e2a0ab3be155475ad60f845813f2088847e584f7", "version" : "144.7559.3" } }