diff --git a/app/index.html b/app/index.html index 087d6acf..32219720 100644 --- a/app/index.html +++ b/app/index.html @@ -12,19 +12,19 @@ - + - - + + - - - + + + diff --git a/app/package.json b/app/package.json index ad40dc81..bfeb31b6 100644 --- a/app/package.json +++ b/app/package.json @@ -27,6 +27,7 @@ "@tauri-apps/plugin-opener": "^2.5.0", "@tauri-apps/plugin-process": "^2.3.0", "@tauri-apps/plugin-updater": "^2.5.0", + "@types/semver": "^7.7.1", "comlink": "^4.4.2", "dompurify": "^3.2.6", "jszip": "^3.10.1", diff --git a/app/public/image/icon-default-128px.png b/app/public/image/icon-default-128px.png new file mode 100644 index 00000000..62789db4 Binary files /dev/null and b/app/public/image/icon-default-128px.png differ diff --git a/app/public/image/icon-default-256px.png b/app/public/image/icon-default-256px.png deleted file mode 100644 index f601332d..00000000 Binary files a/app/public/image/icon-default-256px.png and /dev/null differ diff --git a/app/src/components/profile.tsx b/app/src/components/profile.tsx index 131cb388..bbdae37b 100644 --- a/app/src/components/profile.tsx +++ b/app/src/components/profile.tsx @@ -4,7 +4,6 @@ import solid from "@kixelated/signals/solid"; import { createSignal, JSX, onCleanup, Show } from "solid-js"; import * as Api from "../api"; import { Camera, Microphone } from "../controls"; -import { Broadcast } from "../room/broadcast"; import { Canvas } from "../room/canvas"; import { Local } from "../room/local"; import { Sound } from "../room/sound"; @@ -118,7 +117,6 @@ export default function Profile(props: { local: Local }): JSX.Element { */ class LocalPreview { canvas: Canvas; - broadcast: Broadcast; sound: Sound; space: Space; @@ -137,26 +135,16 @@ class LocalPreview { profile: true, }); - // Create a broadcast wrapper for rendering - this.broadcast = new Broadcast(camera, this.canvas, this.sound, { - visible: true, - position: { - x: 0, - y: 0, - z: 0, - s: 1, - }, - }); - - this.space.add("local", this.broadcast); + const broadcast = this.space.add("local", camera); + this.signals.cleanup(() => this.space.remove("local")); this.signals.effect((effect: Effect) => { - const position = effect.get(this.broadcast.position); + const position = effect.get(broadcast.position); if (position.x === 0 && position.y === 0 && position.s === 1) return; // Reset the position after 2 seconds. effect.timer(() => { - this.broadcast.position.set({ + broadcast.position.set({ x: 0, y: 0, z: 0, @@ -167,10 +155,9 @@ class LocalPreview { } close() { + this.signals.close(); this.space.close(); this.canvas.close(); this.sound.close(); - this.broadcast.close(); // NOTE: Doesn't close the source broadcast. - this.signals.close(); } } diff --git a/app/src/controls.tsx b/app/src/controls.tsx index 9081962f..1c5d6da3 100644 --- a/app/src/controls.tsx +++ b/app/src/controls.tsx @@ -266,7 +266,7 @@ export function Camera(props: { local: Local; room?: Room }): JSX.Element { const toggle = () => { props.local.webcam.enabled.update((prev: boolean) => !prev); }; - const media = solid(props.local.webcam.stream); + const media = solid(props.local.webcam.source); const [showMenu, setShowMenu] = createSignal(false); const [deviceChangeIndicator, setDeviceChangeIndicator] = createSignal(false); @@ -448,7 +448,7 @@ function Screen(props: { local: Local; room: Room }): JSX.Element { const toggle = () => { props.local.screen.enabled.update((prev: boolean) => !prev); }; - const media = solid(props.local.screen.stream); + const media = solid(props.local.screen.source); return ( diff --git a/app/src/room/audio.ts b/app/src/room/audio.ts index 491470a2..2daeab84 100644 --- a/app/src/room/audio.ts +++ b/app/src/room/audio.ts @@ -141,8 +141,8 @@ export class Audio { ctx.translate(bounds.position.x, bounds.position.y); - const RADIUS = 12 * this.broadcast.scale; - const PADDING = 12 * this.broadcast.scale; + const RADIUS = 12 * this.broadcast.zoom.peek(); + const PADDING = 12 * this.broadcast.zoom.peek(); // Background outline ctx.beginPath(); @@ -166,7 +166,7 @@ export class Audio { if (!analyserBuffer) return; // undefined in potato mode const bounds = this.broadcast.bounds.peek(); - const scale = this.broadcast.scale; + const scale = this.broadcast.zoom.peek(); ctx.save(); ctx.translate(bounds.position.x, bounds.position.y); @@ -217,7 +217,7 @@ export class Audio { // Add an additional border if we're speaking, ramping up/down the alpha if (this.#speakingAlpha > 0) { ctx.strokeStyle = `hsla(${hue}, 80%, 45%, ${this.#speakingAlpha})`; - ctx.lineWidth = 6 * this.broadcast.scale; + ctx.lineWidth = 6 * scale; ctx.stroke(); } diff --git a/app/src/room/broadcast.ts b/app/src/room/broadcast.ts index 78577c02..32dcffbc 100644 --- a/app/src/room/broadcast.ts +++ b/app/src/room/broadcast.ts @@ -1,6 +1,6 @@ -import { type Catalog, Publish, Watch } from "@kixelated/hang"; +import { Publish, Watch } from "@kixelated/hang"; import { Effect, Signal } from "@kixelated/signals"; -import { Audio, type AudioProps } from "./audio"; +import { Audio } from "./audio"; import { Canvas } from "./canvas"; import { Captions } from "./captions"; import { Chat } from "./chat"; @@ -19,12 +19,6 @@ export type ChatMessage = { expires: DOMHighResTimeStamp; }; -export type BroadcastProps = { - audio?: AudioProps; - position?: Catalog.Position; - visible?: boolean; -}; - // Catalog.Position but all fields are required. type Position = { x: number; @@ -33,6 +27,13 @@ type Position = { s: number; }; +export interface BroadcastProps { + source: T; + canvas: Canvas; + sound: Sound; + scale: Signal; +} + export class Broadcast { source: T; canvas: Canvas; @@ -46,7 +47,6 @@ export class Broadcast { message = new Signal(undefined); bounds: Signal; // 0 to canvas - scale = 1.0; // 1 is 100% velocity = Vector.create(0, 0); // in pixels per ? // Replaced by position @@ -64,31 +64,35 @@ export class Broadcast { meme = new Signal(undefined); memeName = new Signal(undefined); + scale: Signal; // room scale, 1 is 100% + zoom = new Signal(1.0); // local zoom, 1 is 100% + // Show a locator arrow for 8 seconds to show our position on join. #locatorStart?: DOMHighResTimeStamp; signals = new Effect(); - constructor(source: T, canvas: Canvas, sound: Sound, props?: BroadcastProps) { - this.source = source; - this.canvas = canvas; - this.visible = new Signal(props?.visible ?? true); + constructor(props: BroadcastProps) { + this.source = props.source; + this.canvas = props.canvas; + this.visible = new Signal(true); // TODO + this.scale = props.scale; // Unless provided, start them at the center of the screen with a tiiiiny bit of variance to break ties. const start = () => (Math.random() - 0.5) / 100; const position = { - x: props?.position?.x ?? start(), - y: props?.position?.y ?? start(), - z: props?.position?.z ?? 0, - s: props?.position?.s ?? 1, + x: start(), + y: start(), + z: 0, + s: 1, }; this.position = new Signal(position); this.video = new Video(this); - this.audio = new Audio(this, sound, props?.audio); - this.chat = new Chat(this, canvas); - this.captions = new Captions(this, canvas); + this.audio = new Audio(this, props.sound); + this.chat = new Chat(this, props.canvas); + this.captions = new Captions(this, props.canvas); const viewport = this.canvas.viewport.peek(); @@ -101,36 +105,38 @@ export class Broadcast { // Normalize to find the closest edge of the screen. startPosition = startPosition.normalize().mult(viewport.length()).add(viewport.div(2)); - this.bounds = new Signal(new Bounds(startPosition, this.video.targetSize)); - - // Load the broadcaster's position from the network. - this.signals.effect((effect) => { - if (!effect.get(this.visible)) { - // Change the target position to somewhere outside the screen. - this.position.update((prev) => { - const offscreen = Vector.create(prev.x, prev.y).normalize().mult(2); - return { ...prev, x: offscreen.x, y: offscreen.y }; - }); - - return; - } + this.bounds = new Signal(new Bounds(startPosition, this.video.targetSize.peek())); - // Update the target position from the network. - const location = effect.get(this.source.location.window.position); - if (!location) return; + this.signals.effect(this.#runLocation.bind(this)); + this.signals.effect(this.#runChat.bind(this)); + this.signals.effect(this.#runTarget.bind(this)); + } + // Load the broadcaster's position from the network. + #runLocation(effect: Effect) { + if (!effect.get(this.visible)) { + // Change the target position to somewhere outside the screen. this.position.update((prev) => { - return { - ...prev, - x: location.x ?? prev.x, - y: location.y ?? prev.y, - z: location.z ?? prev.z, - s: location.s ?? prev.s, - }; + const offscreen = Vector.create(prev.x, prev.y).normalize().mult(2); + return { ...prev, x: offscreen.x, y: offscreen.y }; }); - }); - this.signals.effect(this.#runChat.bind(this)); + return; + } + + // Update the target position from the network. + const location = effect.get(this.source.location.window.position); + if (!location) return; + + this.position.update((prev) => { + return { + ...prev, + x: location.x ?? prev.x, + y: location.y ?? prev.y, + z: location.z ?? prev.z, + s: location.s ?? prev.s, + }; + }); } #runChat(effect: Effect) { @@ -165,8 +171,29 @@ export class Broadcast { }); } + // Decides the simulcast size to use based on the number of pixels. + #runTarget(effect: Effect) { + if (!(this.source instanceof Watch.Broadcast)) return; + + const catalog = effect.get(this.source.video.catalog); + if (!catalog) return; + + for (const rendition of catalog) { + if (!rendition.config.displayAspectHeight || !rendition.config.displayAspectWidth) continue; + + const pixels = rendition.config.displayAspectHeight * rendition.config.displayAspectWidth; + const scale = effect.get(this.scale); + const zoom = effect.get(this.zoom); + + const scaled = pixels * scale * zoom; + effect.set(this.source.video.target, { pixels: scaled }); + + return; + } + } + // TODO Also make scale a signal - tick(scale: number) { + tick() { this.video.tick(); const bounds = this.bounds.peek(); @@ -213,8 +240,12 @@ export class Broadcast { } // Apply everything now. - const targetSize = this.video.targetSize.mult(this.scale * scale); - this.scale += (targetPosition.s - this.scale) * 0.1; + const targetSize = this.video.targetSize.peek().mult(this.zoom.peek() * this.scale.peek()); + + const dz = (targetPosition.s - this.zoom.peek()) * 0.1; + if (Math.abs(dz) >= 0.002) { + this.zoom.update((prev) => prev + dz); + } // Apply the velocity and size. const dx = this.velocity.x / 50; @@ -275,9 +306,9 @@ export class Broadcast { ctx.globalAlpha *= alpha; // Calculate arrow position and animation - const arrowSize = 12 * this.scale; + const arrowSize = 12 * this.zoom.peek(); const pulseScale = 1 + Math.sin(now / 500) * 0.1; // Subtle pulsing effect - const offset = 10 * this.scale; + const offset = 10 * this.zoom.peek(); const gap = 2 * (arrowSize + offset); @@ -294,14 +325,14 @@ export class Broadcast { ctx.closePath(); // Style the arrow - ctx.lineWidth = 4 * this.scale; + ctx.lineWidth = 4 * this.zoom.peek(); ctx.strokeStyle = "#000"; // Gold color ctx.fillStyle = "#FFD700"; ctx.stroke(); ctx.fill(); // Draw "YOU" text - const fontSize = Math.round(32 * this.scale); // round to avoid busting font caches + const fontSize = Math.round(32 * this.zoom.peek()); // round to avoid busting font caches ctx.font = `bold ${fontSize}px Arial`; ctx.textAlign = "center"; ctx.textBaseline = "middle"; diff --git a/app/src/room/fake.ts b/app/src/room/fake.ts index 5878d59a..582585f4 100644 --- a/app/src/room/fake.ts +++ b/app/src/room/fake.ts @@ -1,6 +1,6 @@ import { Catalog } from "@kixelated/hang"; +import { u53 } from "@kixelated/hang/catalog"; import { Effect, Signal } from "@kixelated/signals"; -import { Broadcast } from "./broadcast"; import { Canvas } from "./canvas"; import { Sound } from "./sound"; import { Space } from "./space"; @@ -53,9 +53,8 @@ export class FakeBroadcast { }; video = { - media: new Signal(undefined), frame: new Signal(undefined), - flip: new Signal(undefined), + catalog: new Signal(undefined), detection: { enabled: new Signal(false), objects: new Signal(undefined), @@ -118,6 +117,20 @@ export class FakeBroadcast { this.#video = video; this.video.frame.set(video); + video.onloadedmetadata = () => { + this.video.catalog.set([ + { + track: "video", + config: { + codec: "fake", + // Required for the correct display size. + displayAspectWidth: u53(video.videoWidth), + displayAspectHeight: u53(video.videoHeight), + }, + }, + ]); + }; + const source = new MediaElementAudioSourceNode(this.sound.context, { mediaElement: video }); this.audio.root.set(source); } @@ -126,8 +139,6 @@ export class FakeBroadcast { this.#video?.pause(); this.#video = undefined; - this.video.frame.set(undefined); - this.audio.root.update((prev) => { prev?.disconnect(); return undefined; @@ -153,13 +164,12 @@ export class FakeRoom { } add(path: string, broadcast: FakeBroadcast) { - this.space.add(path, new Broadcast(broadcast, this.space.canvas, this.sound)); + this.space.add(path, broadcast); } remove(path: string) { this.space.remove(path).then((broadcast) => { broadcast.close(); - broadcast.source.close(); }); } diff --git a/app/src/room/index.ts b/app/src/room/index.ts index b326d1c5..7fc55066 100644 --- a/app/src/room/index.ts +++ b/app/src/room/index.ts @@ -2,7 +2,6 @@ import { Publish, Watch } from "@kixelated/hang"; import * as Moq from "@kixelated/moq"; import { Effect } from "@kixelated/signals"; import Settings from "../settings"; -import { Broadcast } from "./broadcast"; import type { Canvas } from "./canvas"; import { Local } from "./local"; import { Space } from "./space"; @@ -71,17 +70,10 @@ export class Room { if (local) { if (update.active) { - const broadcast = new Broadcast(local, this.space.canvas, this.space.sound, { - // Wait until we get an announcement before rendering ourselves as online. - visible: false, - }); - - this.space.add(update.path, broadcast); + this.space.add(update.path, local); } else { - this.space.remove(update.path).then((broadcast) => { - broadcast.close(); - // We don't close local sources so we can toggle them. - }); + // NOTE: We don't close local sources so we can toggle them. + this.space.remove(update.path); } continue; } @@ -89,10 +81,7 @@ export class Room { if (update.active) { this.#addRemote(update.path); } else { - this.space.remove(update.path).then((broadcast) => { - broadcast.close(); - broadcast.source.close(); - }); + this.space.remove(update.path).then((broadcast) => broadcast.close()); } } } @@ -117,7 +106,6 @@ export class Room { // Download the preview track to receive high-level information about the broadcaster. preview: { enabled: true }, audio: { - enabled: this.space.sound.suspended, // Download the speaking indicator. speaking: { enabled: true }, captions: { enabled: Settings.captions.render }, @@ -131,13 +119,13 @@ export class Room { }, }); - const broadcast = new Broadcast(watch, this.space.canvas, this.space.sound, { - visible: true, + watch.signals.effect((effect) => { + watch.audio.enabled.set(!effect.get(this.space.sound.suspended)); }); // Request the position we should use from this remote broadcast. - broadcast.signals.effect((effect) => { - const positions = effect.get(broadcast.source.location.peers.positions); + watch.signals.effect((effect) => { + const positions = effect.get(watch.location.peers.positions); if (!positions) return; // Check if our local handles are in the positions. @@ -157,7 +145,7 @@ export class Room { } }); - this.space.add(path, broadcast); + this.space.add(path, watch); } close() { diff --git a/app/src/room/local.ts b/app/src/room/local.ts index b02437e0..96c44d61 100644 --- a/app/src/room/local.ts +++ b/app/src/room/local.ts @@ -80,14 +80,26 @@ export class Local { avatar: this.avatar, }, video: { - enabled: Settings.camera.enabled, - source: this.webcam.stream, - flip: true, // TODO setting? + source: this.webcam.source, + hd: { + enabled: Settings.camera.enabled, + config: { + maxPixels: 640 * 640, + flip: true, + }, + }, + sd: { + enabled: Settings.camera.enabled, + config: { + maxPixels: 320 * 320, + flip: true, + }, + }, }, audio: { enabled: Settings.microphone.enabled, volume: Settings.microphone.gain, - source: this.microphone.stream, + source: this.microphone.source, speaking: { // TODO Figure out an efficient way to run models on mobile. enabled: !Tauri.MOBILE ? Settings.microphone.enabled : undefined, @@ -127,7 +139,7 @@ export class Local { frameRate: { ideal: 60 }, resizeMode: "none", width: { max: 1920 }, - height: { max: 1080 }, + height: { max: 1920 }, }, audio: { channelCount: { ideal: 2, max: 2 }, @@ -145,7 +157,21 @@ export class Local { enabled: this.screen.enabled, }, video: { - enabled: this.screen.enabled, + hd: { + enabled: this.screen.enabled, + config: { + maxPixels: 1920 * 1920, + bitrateScale: 0.08, + }, + }, + // TODO only enable for large enough screen + sd: { + enabled: this.screen.enabled, + config: { + maxPixels: 960 * 960, + bitrateScale: 0.06, + }, + }, }, location: { window: { @@ -160,11 +186,11 @@ export class Local { }); this.#signals.effect((effect) => { - const stream = effect.get(this.screen.stream); - if (!stream) return; + const source = effect.get(this.screen.source); + if (!source) return; - effect.set(this.share.audio.source, stream.audio); - effect.set(this.share.video.source, stream.video); + effect.set(this.share.audio.source, source.audio); + effect.set(this.share.video.source, source.video); effect.set(this.share.enabled, true, false); // only enable once there is a stream }); diff --git a/app/src/room/space.ts b/app/src/room/space.ts index 5f0d7325..33489ecd 100644 --- a/app/src/room/space.ts +++ b/app/src/room/space.ts @@ -1,6 +1,6 @@ import { Publish, Watch } from "@kixelated/hang"; import { Effect, Signal } from "@kixelated/signals"; -import { Broadcast } from "./broadcast"; +import { Broadcast, BroadcastSource } from "./broadcast"; import type { Canvas } from "./canvas"; import { Vector } from "./geometry"; import type { Sound } from "./sound"; @@ -25,7 +25,8 @@ export class Space { #hovering: Broadcast | undefined = undefined; #dragging?: Broadcast; - #scale = 1.0; + + #scale = new Signal(1.0); #maxZ = 0; @@ -53,6 +54,8 @@ export class Space { this.#signals.event(canvas.element, "touchend", this.#onTouchEnd.bind(this), { passive: false }); this.#signals.event(canvas.element, "touchcancel", this.#onTouchCancel.bind(this), { passive: false }); + this.#signals.effect(this.#runScale.bind(this)); + // This is a bit of a hack, but register our render method. this.canvas.onRender = this.#tick.bind(this); this.#signals.cleanup(() => { @@ -379,7 +382,9 @@ export class Space { return undefined; } - add(id: string, broadcast: Broadcast) { + add(id: string, source: BroadcastSource): Broadcast { + const broadcast = new Broadcast({ source, canvas: this.canvas, sound: this.sound, scale: this.#scale }); + // Put new broadcasts on top of the stack. // NOTE: This is not sent over the network. broadcast.position.update((prev) => ({ @@ -455,9 +460,11 @@ export class Space { this.sound.tts.left(name); }); + + return broadcast; } - async remove(path: string): Promise { + async remove(path: string): Promise { const broadcast = this.lookup.get(path); if (!broadcast) { throw new Error(`broadcast not found: ${path}`); @@ -476,7 +483,9 @@ export class Space { await new Promise((resolve) => setTimeout(resolve, 1000)); this.#rip.splice(this.#rip.indexOf(broadcast), 1); - return broadcast; + broadcast.close(); + + return broadcast.source; } clear(): Broadcast[] { @@ -487,16 +496,14 @@ export class Space { } #tick(ctx: CanvasRenderingContext2D, now: DOMHighResTimeStamp) { - this.#tickScale(); - for (const broadcast of this.#rip) { - broadcast.tick(this.#scale); + broadcast.tick(); } const broadcasts = this.ordered.peek(); for (const broadcast of broadcasts) { - broadcast.tick(this.#scale); + broadcast.tick(); } // Check for collisions. @@ -611,24 +618,26 @@ export class Space { ctx.restore(); } - #tickScale() { - const broadcasts = this.ordered.peek(); + #runScale(effect: Effect) { + const broadcasts = effect.get(this.ordered); if (broadcasts.length === 0) { // Avoid division by zero. return; } - const canvasArea = this.canvas.viewport.peek().area(); + const canvasArea = effect.get(this.canvas.viewport).area(); let broadcastArea = 0; for (const broadcast of broadcasts) { - broadcastArea += broadcast.video.targetSize.x * broadcast.video.targetSize.y; + const size = effect.get(broadcast.video.targetSize); + broadcastArea += size.x * size.y; } const fillRatio = broadcastArea / canvasArea; const targetFill = 0.25; - this.#scale = Math.min(Math.sqrt(targetFill / fillRatio), 1); + const scale = Math.min(Math.sqrt(targetFill / fillRatio), 1); + this.#scale.set(scale); } close() { diff --git a/app/src/room/tts/index.ts b/app/src/room/tts/index.ts index 4e091569..8bb6c1a7 100644 --- a/app/src/room/tts/index.ts +++ b/app/src/room/tts/index.ts @@ -242,7 +242,6 @@ export class TTS { } close() { - this.context.close(); this.#signals.close(); } } diff --git a/app/src/room/video.ts b/app/src/room/video.ts index e2111d64..f4ea7271 100644 --- a/app/src/room/video.ts +++ b/app/src/room/video.ts @@ -1,6 +1,8 @@ import { Publish, Watch } from "@kixelated/hang"; +import { Effect, Signal } from "@kixelated/signals"; import * as Api from "../api"; import type { Broadcast } from "./broadcast"; +import { FakeBroadcast } from "./fake"; import { Vector } from "./geometry"; import { MEME_AUDIO, MEME_AUDIO_LOOKUP, MEME_VIDEO, MEME_VIDEO_LOOKUP, type MemeVideoName } from "./meme"; @@ -17,11 +19,14 @@ export class Video { // 1 when a video frame is fully rendered, 0 when their avatar is fully rendered. avatarTransition = 0; - // The current video frame, for transitioning back to the avatar. - frame?: VideoFrame | HTMLVideoElement; + // The size of the avatar in pixels. + avatarSize = new Signal(undefined); + + // The current video frame. + frame?: CanvasImageSource; // The desired size of the video in pixels. - targetSize: Vector; // in pixels + targetSize = new Signal(Vector.create(128, 128)); // The opacity from 0 to 1, where 0 is offline and 1 is online. online = 0; @@ -31,67 +36,74 @@ export class Video { constructor(broadcast: Broadcast) { this.broadcast = broadcast; - this.targetSize = Vector.create(128, 128); + this.broadcast.signals.effect(this.#runAvatar.bind(this)); + this.broadcast.signals.effect(this.#runTargetSize.bind(this)); + this.broadcast.signals.effect(this.#runFrame.bind(this)); + } - // Set a random default avatar while the user details are loading. - // TODO Only start a broadcast after receiving the catalog to avoid this. - this.avatar.src = Api.randomAvatar(); + #runAvatar(effect: Effect) { + let avatar = effect.get(this.broadcast.source.user.avatar); + if (!avatar) { + // Don't unset the avatar if it's already set. + if (this.avatar) return; - // This doesn't use a memo because we intentionally prevent going back to the default avatar. - this.broadcast.signals.effect((effect) => { - const avatar = effect.get(this.broadcast.source.user.avatar); - if (!avatar) return; // don't unset + // Set a random default avatar while the user details are loading. + avatar = Api.randomAvatar(); + } - // TODO only set the avatar if it successfully loads - const newAvatar = new Image(); - newAvatar.src = avatar; + // TODO only set the avatar if it successfully loads + const newAvatar = new Image(); + newAvatar.src = avatar; - const load = () => { - this.avatar = newAvatar; - }; + const load = () => { + this.avatar = newAvatar; + this.avatarSize.set(Vector.create(newAvatar.width, newAvatar.height)); + }; - effect.event(newAvatar, "load", load); - }); + effect.event(newAvatar, "load", load); } - tick() { - const next = this.broadcast.source.video.frame.peek(); - if (next) { - this.avatarTransition = Math.min(this.avatarTransition + 0.05, 1); - - let width: number; - let height: number; + #runTargetSize(effect: Effect) { + const catalog = effect.get(this.broadcast.source.video.catalog); - if (next instanceof HTMLVideoElement) { - width = next.videoWidth; - height = next.videoHeight; - } else { - width = next.displayWidth; - height = next.displayHeight; + if (catalog) { + for (const rendition of catalog) { + if (rendition.config.displayAspectHeight && rendition.config.displayAspectWidth) { + this.targetSize.set( + Vector.create(rendition.config.displayAspectWidth, rendition.config.displayAspectHeight), + ); + return; + } } + } - this.targetSize = Vector.create(width, height); + const avatar = effect.get(this.avatarSize); + if (avatar) { + // If the avatar is larger than 256x256, then shrink it to match the target area. + const ratio = Math.sqrt(avatar.x * avatar.y) / 256; + this.targetSize.set(avatar.div(ratio)); + return; + } - if (this.frame instanceof VideoFrame) this.frame.close(); - this.frame = next instanceof HTMLVideoElement ? next : next.clone(); - } else { - this.avatarTransition = Math.max(this.avatarTransition - 0.05, 0); - // TODO do this once, not on every frame. - if (this.avatar.complete) { - this.targetSize = Vector.create(this.avatar.width, this.avatar.height); + this.targetSize.set(Vector.create(128, 128)); + } - // If the avatar is larger than 256x256, then shrink it to match the target area. - const ratio = Math.sqrt(this.targetSize.x * this.targetSize.y) / 256; - if (ratio > 1) { - this.targetSize = this.targetSize.div(ratio); - } - } + #runFrame(effect: Effect) { + if (this.broadcast.source instanceof FakeBroadcast) { + // TODO FakeBroadcast should return a VideoFrame instead of a HTMLVideoElement. + this.frame = effect.get(this.broadcast.source.video.frame); + } else { + const frame = effect.get(this.broadcast.source.video.frame)?.clone(); + effect.cleanup(() => frame?.close()); + this.frame = frame; + } + } - // Deallocate the frame once we're done with it. - if (this.avatarTransition === 0 && this.frame instanceof VideoFrame) { - this.frame.close(); - this.frame = undefined; - } + tick() { + if (this.frame) { + this.avatarTransition = Math.min(this.avatarTransition + 0.05, 1); + } else { + this.avatarTransition = Math.max(this.avatarTransition - 0.05, 0); } if (this.broadcast.visible.peek()) { @@ -118,7 +130,7 @@ export class Video { ctx.save(); const bounds = this.broadcast.bounds.peek(); - const scale = this.broadcast.scale; + const scale = this.broadcast.zoom.peek(); ctx.translate(bounds.position.x, bounds.position.y); ctx.globalAlpha *= this.online; @@ -166,12 +178,12 @@ export class Video { ctx.save(); ctx.globalAlpha *= this.avatarTransition; - // Apply horizontal flip only for Publish.Broadcast - // Watch.Broadcast already handles flipping internally with WebCodecs - const flip = this.broadcast.source.video.flip?.peek(); - const shouldFlip = flip && this.broadcast.source instanceof Publish.Broadcast; + // Apply horizontal flip when rendering the preview. + const flip = + this.broadcast.source instanceof Publish.Broadcast && + this.broadcast.source.video.hd.config.peek()?.flip; - if (shouldFlip) { + if (flip) { ctx.save(); ctx.scale(-1, 1); ctx.translate(-bounds.size.x, 0); diff --git a/bun.lock b/bun.lock index 425829d6..ae6afeeb 100644 --- a/bun.lock +++ b/bun.lock @@ -46,6 +46,7 @@ "@tauri-apps/plugin-opener": "^2.5.0", "@tauri-apps/plugin-process": "^2.3.0", "@tauri-apps/plugin-updater": "^2.5.0", + "@types/semver": "^7.7.1", "comlink": "^4.4.2", "dompurify": "^3.2.6", "jszip": "^3.10.1", @@ -101,8 +102,9 @@ }, "moq/js/moq": { "name": "@kixelated/moq", - "version": "0.9.0", + "version": "0.9.1", "dependencies": { + "@kixelated/signals": "workspace:*", "@kixelated/web-transport-ws": "^0.1.2", "async-mutex": "^0.5.0", }, @@ -118,7 +120,6 @@ "vite-plugin-html": "^3.2.2", }, "peerDependencies": { - "@kixelated/signals": "workspace:*", "zod": "^4.1.0", }, }, @@ -585,6 +586,8 @@ "@types/react": ["@types/react@19.1.12", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w=="], + "@types/semver": ["@types/semver@7.7.1", "", {}, "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA=="], + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], "@types/uuid": ["@types/uuid@10.0.0", "", {}, "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ=="], diff --git a/moq b/moq index 154508a9..0a6101da 160000 --- a/moq +++ b/moq @@ -1 +1 @@ -Subproject commit 154508a96c1f2c3cbfbdec17c4a901ff496f34d1 +Subproject commit 0a6101daae9e363a99f9507c683502aa684d811c