Deform mesh (#21)

* Deform the mesh on drag/zoom.

* PR stuff.

* Util folder.

* Subdivie based on zoom.

Author: kixelated

app/package.json

@@ -54,6 +54,7 @@
 		"tsx": "^4.20.5",
 		"typescript": "~5.8.3",
 		"vite": "^6.3.5",
+		"vite-plugin-glsl": "^1.5.4",
 		"vite-plugin-solid": "^2.11.8",
 		"vite-plugin-static-copy": "^2.1.0",
 		"vite-plugin-wrangler": "^0.1.1",

app/src/components/meme-selector.tsx

@@ -117,7 +117,12 @@ export function MemeSelector(props: MemeSelectorProps): JSX.Element {
 		setPlayingAudioMeme(null);
 
 		if (type === "audio") {
-			const audio = new Audio(new URL(`/meme/${MEME_AUDIO[memeName as keyof typeof MEME_AUDIO].file}`, import.meta.env.VITE_APP_URL).toString());
+			const audio = new Audio(
+				new URL(
+					`/meme/${MEME_AUDIO[memeName as keyof typeof MEME_AUDIO].file}`,
+					import.meta.env.VITE_APP_URL,
+				).toString(),
+			);
 			audio.volume = 0.5; // Lower volume for preview
 			audio.play();
 			setPreviewAudio(audio);
@@ -131,7 +136,10 @@ export function MemeSelector(props: MemeSelectorProps): JSX.Element {
 		} else {
 			// For video, play with sound
 			const video = document.createElement("video");
-			video.src = new URL(`/meme/${MEME_VIDEO[memeName as keyof typeof MEME_VIDEO].file}`, import.meta.env.VITE_APP_URL).toString();
+			video.src = new URL(
+				`/meme/${MEME_VIDEO[memeName as keyof typeof MEME_VIDEO].file}`,
+				import.meta.env.VITE_APP_URL,
+			).toString();
 			video.volume = 0.5;
 			video.style.display = "none";
 			document.body.appendChild(video);
@@ -310,7 +318,10 @@ export function MemeSelector(props: MemeSelectorProps): JSX.Element {
 									<div class="group relative bg-white/10 hover:bg-white/20 rounded overflow-hidden transition-colors cursor-pointer aspect-video basis-42 flex-grow">
 										{/* Thumbnail background */}
 										<img
-											src={new URL(`/meme/${thumbnailName}`, import.meta.env.VITE_APP_URL).toString()}
+											src={new URL(
+												`/meme/${thumbnailName}`,
+												import.meta.env.VITE_APP_URL,
+											).toString()}
 											alt={meme}
 											class="absolute inset-0 w-full h-full opacity-30"
 											style={{
@@ -321,7 +332,10 @@ export function MemeSelector(props: MemeSelectorProps): JSX.Element {
 										{/* Video preview when playing */}
 										<Show when={isPlaying()}>
 											<video
-												src={new URL(`/meme/${memeData.file}`, import.meta.env.VITE_APP_URL).toString()}
+												src={new URL(
+													`/meme/${memeData.file}`,
+													import.meta.env.VITE_APP_URL,
+												).toString()}
 												autoplay
 												muted
 												class="absolute inset-0 w-full h-full opacity-70"

app/src/room/broadcast.ts

@@ -6,6 +6,7 @@ import { Captions } from "./captions";
 import { Chat } from "./chat";
 import { FakeBroadcast } from "./fake";
 import { Bounds, Vector } from "./geometry";
+import { MeshBuffer } from "./gl/mesh";
 import { Meme } from "./meme";
 import { Name } from "./name";
 import { Sound } from "./sound";
@@ -52,6 +53,22 @@ export class Broadcast<T extends BroadcastSource = BroadcastSource> {
 	bounds: Signal<Bounds>; // 0 to canvas
 	velocity = Vector.create(0, 0); // in pixels per ?
 
+	// Drag point in normalized coordinates (0-1) relative to the broadcast
+	dragPoint = new Signal<Vector>(Vector.create(0.5, 0.5));
+
+	// Deformation velocity for the drag effect (decays independently from physics velocity)
+	deformVelocity = Vector.create(0, 0);
+
+	// Zoom deformation for scaling effect (positive = expanding, negative = contracting)
+	// Only applies during user-initiated zooming (mouse wheel or pinch)
+	zoomDeform = 0;
+
+	// Zoom center point in normalized coordinates (0-1) relative to the broadcast
+	zoomCenter = new Signal<Vector>(Vector.create(0.5, 0.5));
+
+	// Shared mesh buffer for all renderers
+	mesh: MeshBuffer;
+
 	// Replaced by position
 	//targetPosition = Vector.create(0, 0); // -0.5 to 0.5, sent over the network
 	//targetScale = 1.0; // 1 is 100%
@@ -83,6 +100,9 @@ export class Broadcast<T extends BroadcastSource = BroadcastSource> {
 		this.visible = new Signal(true); // TODO
 		this.scale = props.scale;
 
+		// Create shared mesh buffer
+		this.mesh = new MeshBuffer(props.canvas);
+
 		// Unless provided, start them at the center of the screen with a tiiiiny bit of variance to break ties.
 		const start = () => (Math.random() - 0.5) / 100;
 		const position = {
@@ -200,6 +220,9 @@ export class Broadcast<T extends BroadcastSource = BroadcastSource> {
 		this.audio.tick();
 		this.video.tick(now);
 
+		// Update mesh based on deformation velocity and zoom deformation
+		this.mesh.update(this.deformVelocity, this.zoomDeform);
+
 		// Update opacity based on online status
 		const fadeTime = 300; // ms
 		const elapsed = now - this.#onlineTransition;
@@ -279,6 +302,21 @@ export class Broadcast<T extends BroadcastSource = BroadcastSource> {
 
 		// Slow down the velocity for the next frame.
 		this.velocity = this.velocity.mult(0.5);
+
+		// Decay the deformation velocity smoothly over time (faster decay than physics velocity)
+		if (this.deformVelocity.length() > 0.01) {
+			this.deformVelocity = this.deformVelocity.mult(0.85);
+		} else {
+			this.deformVelocity = Vector.create(0, 0);
+		}
+
+		// Decay zoom deformation (set by user interaction in Space)
+		// Slower decay than drag to keep mesh subdivided during zoom animation
+		if (Math.abs(this.zoomDeform) > 0.01) {
+			this.zoomDeform *= 0.95;
+		} else {
+			this.zoomDeform = 0;
+		}
 	}
 
 	// Returns true if the broadcaster is locked to a position.
@@ -303,6 +341,7 @@ export class Broadcast<T extends BroadcastSource = BroadcastSource> {
 		this.chat.close();
 		this.captions.close();
 		this.name.close();
+		this.mesh.close();
 
 		// NOTE: Don't close the source broadcast; we need it for the local preview.
 		// this.source.close();

app/src/room/fake.ts

@@ -126,7 +126,7 @@ export class FakeBroadcast {
 		video.onloadedmetadata = () => {
 			this.video.catalog.set([
 				{
-					track: "video",
+					track: { name: "video", priority: 0 },
 					config: {
 						codec: "fake",
 						// Required for the correct display size.
@@ -155,7 +155,7 @@ export class FakeBroadcast {
 
 			this.video.catalog.set([
 				{
-					track: "image",
+					track: { name: "image", priority: 0 },
 					config: {
 						codec: "fake",
 						displayAspectWidth: u53(image.width),

app/src/room/gl/outline.frag app/src/room/gl/audio.fragapp/src/room/gl/outline.frag renamed to app/src/room/gl/audio.frag

@@ -14,11 +14,7 @@ uniform float u_finalAlpha; // Pre-computed final alpha (0.3 + volume * 0.4)
 
 out vec4 fragColor;
 
-// Signed distance function for rounded rectangle
-float roundedBoxSDF(vec2 center, vec2 size, float radius) {
-	vec2 q = abs(center) - size + radius;
-	return min(max(q.x, q.y), 0.0) + length(max(q, 0.0)) - radius;
-}
+#include "./util/sdf.glsl"
 
 void main() {
 	if (u_opacity <= 0.01) {

app/src/room/gl/outline.ts app/src/room/gl/audio.tsapp/src/room/gl/outline.ts renamed to app/src/room/gl/audio.ts

@@ -1,16 +1,15 @@
 import type { Broadcast } from "../broadcast";
 import { Canvas } from "../canvas";
+import audioFragSource from "./audio.frag";
+import audioVertSource from "./audio.vert";
 import type { Camera } from "./camera";
-import outlineFragSource from "./outline.frag?raw";
-import outlineVertSource from "./outline.vert?raw";
+import type { MeshBuffer } from "./mesh";
 import { Attribute, Shader, Uniform1f, Uniform2f, Uniform3f, Uniform4f, UniformMatrix4fv } from "./shader";
 
-export class OutlineRenderer {
+export class AudioRenderer {
 	#canvas: Canvas;
 	#program: Shader;
-	#vao: WebGLVertexArrayObject;
-	#positionBuffer: WebGLBuffer;
-	#indexBuffer: WebGLBuffer;
+	#vaos = new Map<MeshBuffer, WebGLVertexArrayObject>();
 
 	// Typed uniforms
 	#u_projection: UniformMatrix4fv;
@@ -24,13 +23,18 @@ export class OutlineRenderer {
 	#u_color: Uniform3f;
 	#u_time: Uniform1f;
 	#u_finalAlpha: Uniform1f;
+	#u_dragPoint: Uniform2f;
+	#u_velocity: Uniform2f;
+	#u_dragStrength: Uniform1f;
+	#u_zoomDeform: Uniform1f;
+	#u_zoomCenter: Uniform2f;
 
 	// Typed attributes
 	#a_position: Attribute;
 
 	constructor(canvas: Canvas) {
 		this.#canvas = canvas;
-		this.#program = new Shader(canvas.gl, outlineVertSource, outlineFragSource);
+		this.#program = new Shader(canvas.gl, audioVertSource, audioFragSource);
 
 		// Initialize typed uniforms
 		this.#u_projection = this.#program.createUniformMatrix4fv("u_projection");
@@ -44,63 +48,51 @@ export class OutlineRenderer {
 		this.#u_color = this.#program.createUniform3f("u_color");
 		this.#u_time = this.#program.createUniform1f("u_time");
 		this.#u_finalAlpha = this.#program.createUniform1f("u_finalAlpha");
+		this.#u_dragPoint = this.#program.createUniform2f("u_dragPoint");
+		this.#u_velocity = this.#program.createUniform2f("u_velocity");
+		this.#u_dragStrength = this.#program.createUniform1f("u_dragStrength");
+		this.#u_zoomDeform = this.#program.createUniform1f("u_zoomDeform");
+		this.#u_zoomCenter = this.#program.createUniform2f("u_zoomCenter");
 
 		// Initialize typed attributes
 		this.#a_position = this.#program.createAttribute("a_position");
-
-		const vao = this.#canvas.gl.createVertexArray();
-		if (!vao) throw new Error("Failed to create VAO");
-		this.#vao = vao;
-
-		const positionBuffer = this.#canvas.gl.createBuffer();
-		if (!positionBuffer) throw new Error("Failed to create position buffer");
-		this.#positionBuffer = positionBuffer;
-
-		const indexBuffer = this.#canvas.gl.createBuffer();
-		if (!indexBuffer) throw new Error("Failed to create index buffer");
-		this.#indexBuffer = indexBuffer;
-
-		this.#setupBuffers();
 	}
 
-	#setupBuffers() {
-		const gl = this.#canvas.gl;
-
-		// Quad vertices (0-1 range, will be scaled by bounds)
-		const positions = new Float32Array([
-			0,
-			0, // Top-left
-			1,
-			0, // Top-right
-			1,
-			1, // Bottom-right
-			0,
-			1, // Bottom-left
-		]);
+	#getOrCreateVAO(mesh: MeshBuffer): WebGLVertexArrayObject {
+		let vao = this.#vaos.get(mesh);
+		if (vao) return vao;
 
-		// Indices for two triangles
-		const indices = new Uint16Array([0, 1, 2, 0, 2, 3]);
+		const gl = this.#canvas.gl;
+		vao = gl.createVertexArray();
+		if (!vao) throw new Error("Failed to create VAO");
 
-		gl.bindVertexArray(this.#vao);
+		gl.bindVertexArray(vao);
 
 		// Position attribute
-		gl.bindBuffer(gl.ARRAY_BUFFER, this.#positionBuffer);
-		gl.bufferData(gl.ARRAY_BUFFER, positions, gl.STATIC_DRAW);
+		gl.bindBuffer(gl.ARRAY_BUFFER, mesh.positionBuffer);
 		gl.enableVertexAttribArray(this.#a_position.location);
 		gl.vertexAttribPointer(this.#a_position.location, 2, gl.FLOAT, false, 0, 0);
 
 		// Index buffer
-		gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.#indexBuffer);
-		gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, indices, gl.STATIC_DRAW);
+		gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, mesh.indexBuffer);
 
 		gl.bindVertexArray(null);
+
+		this.#vaos.set(mesh, vao);
+		return vao;
 	}
 
 	render(broadcast: Broadcast, camera: Camera, maxZ: number, now: DOMHighResTimeStamp) {
 		const gl = this.#canvas.gl;
 		const bounds = broadcast.bounds.peek();
 		const scale = broadcast.zoom.peek();
 		const volume = broadcast.audio.volume;
+		const dragPoint = broadcast.dragPoint.peek();
+		const deformVelocity = broadcast.deformVelocity;
+		const zoomCenter = broadcast.zoomCenter.peek();
+
+		// Get or create VAO for this broadcast's mesh
+		const vao = this.#getOrCreateVAO(broadcast.mesh);
 
 		this.#program.use();
 
@@ -180,17 +172,27 @@ export class OutlineRenderer {
 
 		this.#u_color.set(r, g, b);
 
+		// Set drag deformation uniforms (using deformVelocity which decays separately)
+		this.#u_dragPoint.set(dragPoint.x, dragPoint.y);
+		this.#u_velocity.set(deformVelocity.x, deformVelocity.y);
+		this.#u_dragStrength.set(0.5); // Halved for subtler effect
+
+		// Set zoom deformation uniforms
+		this.#u_zoomDeform.set(broadcast.zoomDeform);
+		this.#u_zoomCenter.set(zoomCenter.x, zoomCenter.y);
+
 		// Draw
-		gl.bindVertexArray(this.#vao);
-		gl.drawElements(gl.TRIANGLES, 6, gl.UNSIGNED_SHORT, 0);
+		gl.bindVertexArray(vao);
+		gl.drawElements(gl.TRIANGLES, broadcast.mesh.indexCount, gl.UNSIGNED_SHORT, 0);
 		gl.bindVertexArray(null);
 	}
 
 	close() {
 		const gl = this.#canvas.gl;
-		gl.deleteVertexArray(this.#vao);
-		gl.deleteBuffer(this.#positionBuffer);
-		gl.deleteBuffer(this.#indexBuffer);
+		for (const vao of this.#vaos.values()) {
+			gl.deleteVertexArray(vao);
+		}
+		this.#vaos.clear();
 		this.#program.cleanup();
 	}
 }

app/src/room/gl/audio.vert

@@ -0,0 +1,35 @@
+#version 300 es
+precision highp float;
+
+in vec2 a_position;
+
+uniform mat4 u_projection;
+uniform vec4 u_bounds; // x, y, width, height
+uniform float u_depth;
+uniform vec2 u_dragPoint; // Normalized drag point (0-1) relative to broadcast
+uniform vec2 u_velocity; // Current velocity vector
+uniform float u_dragStrength; // Strength multiplier for drag effect
+uniform float u_zoomDeform; // Zoom deformation (positive = expanding, negative = contracting)
+uniform vec2 u_zoomCenter; // Normalized zoom center (0-1) relative to broadcast
+
+out vec2 v_pos; // Position within the quad (0-1)
+
+#include "./deformation.glsl"
+
+void main() {
+	// Apply deformation using shared function
+	vec2 pos = applyDeformation(
+		a_position,
+		u_dragPoint,
+		u_velocity,
+		u_dragStrength,
+		u_zoomDeform,
+		u_zoomCenter,
+		u_bounds
+	);
+
+	// Apply projection
+	gl_Position = u_projection * vec4(pos, u_depth, 1.0);
+
+	v_pos = a_position;
+}

app/src/room/gl/background.frag

@@ -14,28 +14,13 @@ const float WOBBLE_SPEED = 0.0004;
 
 const float SEGMENT_WIDTH = 120.0; // Pixels per segment
 
+#include "./util/color.glsl"
+
 // Hash function for deterministic randomness
 float hash(float n) {
 	return fract(sin(n) * 43758.5453123);
 }
 
-// Convert HSL to RGB
-vec3 hsl2rgb(float h, float s, float l) {
-	float c = (1.0 - abs(2.0 * l - 1.0)) * s;
-	float x = c * (1.0 - abs(mod(h / 60.0, 2.0) - 1.0));
-	float m = l - c / 2.0;
-
-	vec3 rgb;
-	if (h < 60.0) rgb = vec3(c, x, 0.0);
-	else if (h < 120.0) rgb = vec3(x, c, 0.0);
-	else if (h < 180.0) rgb = vec3(0.0, c, x);
-	else if (h < 240.0) rgb = vec3(0.0, x, c);
-	else if (h < 300.0) rgb = vec3(x, 0.0, c);
-	else rgb = vec3(c, 0.0, x);
-
-	return rgb + m;
-}
-
 void main() {
 	// Work in simple horizontal line space - rotation happens in vertex shader
 	vec2 pos = v_pixel;