Allow direct image uploads for LemonSlice avatars

.changeset/lemonslice-direct-image-upload.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-lemonslice': patch
+---
+
+Allow direct image uploads for LemonSlice avatars.

plugins/lemonslice/etc/agents-plugin-lemonslice.api.md

@@ -0,0 +1,53 @@
+## API Report File for "@livekit/agents-plugin-lemonslice"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+import { APIConnectOptions } from '@livekit/agents';
+import type { Room } from '@livekit/rtc-node';
+import { voice } from '@livekit/agents';
+
+// @public
+export type AgentImage = Uint8Array | ArrayBuffer | Blob;
+
+// @public
+export class AvatarSession extends voice.AvatarSession {
+    constructor(options?: AvatarSessionOptions);
+    // (undocumented)
+    get avatarIdentity(): string;
+    // (undocumented)
+    get provider(): string;
+    start(agentSession: voice.AgentSession, room: Room, options?: StartOptions): Promise<string>;
+}
+
+// @public
+export interface AvatarSessionOptions {
+    agentId?: string | null;
+    agentImage?: AgentImage | null;
+    agentImageUrl?: string | null;
+    agentPrompt?: string | null;
+    apiKey?: string;
+    apiUrl?: string;
+    avatarParticipantIdentity?: string;
+    avatarParticipantName?: string;
+    connOptions?: APIConnectOptions;
+    extraPayload?: Record<string, unknown> | null;
+    idleTimeout?: number | null;
+}
+
+// @public
+export class LemonSliceException extends Error {
+    constructor(message: string);
+}
+
+// @public
+export interface StartOptions {
+    livekitApiKey?: string;
+    livekitApiSecret?: string;
+    livekitUrl?: string;
+}
+
+// (No @packageDocumentation comment for this package)
+
+```

plugins/lemonslice/src/avatar.ts

@@ -22,8 +22,17 @@ const SAMPLE_RATE = 16000;
 const AVATAR_AGENT_IDENTITY = 'lemonslice-avatar-agent';
 const AVATAR_AGENT_NAME = 'lemonslice-avatar-agent';
 
+/**
+ * Image bytes accepted for direct LemonSlice avatar uploads.
+ *
+ * @public
+ */
+export type AgentImage = Uint8Array | ArrayBuffer | Blob;
+
 /**
  * Exception thrown when there are errors with the LemonSlice API.
+ *
+ * @public
  */
 export class LemonSliceException extends Error {
   constructor(message: string) {
@@ -34,18 +43,25 @@ export class LemonSliceException extends Error {
 
 /**
  * Options for configuring an AvatarSession.
+ *
+ * @public
  */
 export interface AvatarSessionOptions {
   /**
    * The ID of the LemonSlice agent to add to the session.
-   * Either agentId or agentImageUrl must be provided.
+   * Exactly one of agentId, agentImageUrl, or agentImage must be provided.
    */
   agentId?: string | null;
   /**
    * The URL of the image to use as the agent's avatar.
-   * Either agentId or agentImageUrl must be provided.
+   * Exactly one of agentId, agentImageUrl, or agentImage must be provided.
    */
   agentImageUrl?: string | null;
+  /**
+   * Image bytes to upload and use as the agent's avatar.
+   * Exactly one of agentId, agentImageUrl, or agentImage must be provided.
+   */
+  agentImage?: AgentImage | null;
   /**
    * A prompt that subtly influences the avatar's movements and expressions.
    */
@@ -83,6 +99,8 @@ export interface AvatarSessionOptions {
 
 /**
  * Options for starting an avatar session.
+ *
+ * @public
  */
 export interface StartOptions {
   /**
@@ -121,10 +139,13 @@ export interface StartOptions {
  * });
  * await avatar.start(agentSession, room);
  * ```
+ *
+ * @public
  */
 export class AvatarSession extends voice.AvatarSession {
   private agentId: string | null;
   private agentImageUrl: string | null;
+  private agentImage: AgentImage | null;
   private agentPrompt: string | null;
   private idleTimeout: number | null;
   private extraPayload: Record<string, unknown> | null;
@@ -140,18 +161,24 @@ export class AvatarSession extends voice.AvatarSession {
    * Creates a new AvatarSession.
    *
    * @param options - Configuration options for the avatar session
-   * @throws LemonSliceException if invalid agentId or agentImageUrl is provided, or if LemonSlice API key is not set
+   * @throws LemonSliceException if invalid agentId, agentImageUrl, or agentImage is provided, or if LemonSlice API key is not set
    */
   constructor(options: AvatarSessionOptions = {}) {
     super();
     this.agentId = options.agentId ?? null;
     this.agentImageUrl = options.agentImageUrl ?? null;
+    this.agentImage = options.agentImage ?? null;
 
-    if (!this.agentId && !this.agentImageUrl) {
-      throw new LemonSliceException('Missing agentId or agentImageUrl');
+    const givenSources = [this.agentId, this.agentImageUrl, this.agentImage].filter(
+      (source) => source !== null,
+    );
+    if (givenSources.length === 0) {
+      throw new LemonSliceException('Missing one of agentId, agentImageUrl or agentImage');
     }
-    if (this.agentId && this.agentImageUrl) {
-      throw new LemonSliceException('Only one of agentId or agentImageUrl can be provided');
+    if (givenSources.length > 1) {
+      throw new LemonSliceException(
+        'Only one of agentId, agentImageUrl or agentImage can be provided',
+      );
     }
 
     this.agentPrompt = options.agentPrompt ?? null;
@@ -266,23 +293,23 @@ export class AvatarSession extends voice.AvatarSession {
   private async startAgent(livekitUrl: string, livekitToken: string): Promise<string> {
     for (let i = 0; i <= this.connOptions.maxRetry; i++) {
       try {
-        const payload: Record<string, any> = {
+        const payload: Record<string, unknown> = {
           transport_type: 'livekit',
           properties: {
             livekit_url: livekitUrl,
             livekit_token: livekitToken,
           },
         };
 
-        if (this.agentId) {
+        if (this.agentId !== null) {
           payload.agent_id = this.agentId;
         }
 
-        if (this.agentImageUrl) {
+        if (this.agentImageUrl !== null) {
           payload.agent_image_url = this.agentImageUrl;
         }
 
-        if (this.agentPrompt) {
+        if (this.agentPrompt !== null) {
           payload.agent_prompt = this.agentPrompt;
         }
 
@@ -294,13 +321,33 @@ export class AvatarSession extends voice.AvatarSession {
           Object.assign(payload, this.extraPayload);
         }
 
+        const headers: Record<string, string> = {
+          'X-API-Key': this.apiKey,
+        };
+        let body: BodyInit;
+
+        if (this.agentImage !== null) {
+          const formData = new FormData();
+          formData.append(
+            'payload',
+            new Blob([JSON.stringify(payload)], { type: 'application/json' }),
+          );
+          formData.append(
+            'image',
+            new File([imageBlobPart(this.agentImage)], 'image.png', {
+              type: 'image/png',
+            }),
+          );
+          body = formData;
+        } else {
+          headers['Content-Type'] = 'application/json';
+          body = JSON.stringify(payload);
+        }
+
         const response = await fetch(this.apiUrl, {
           method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'X-API-Key': this.apiKey,
-          },
-          body: JSON.stringify(payload),
+          headers,
+          body,
           signal: AbortSignal.timeout(this.connOptions.timeoutMs),
         });
 
@@ -336,3 +383,13 @@ export class AvatarSession extends voice.AvatarSession {
     });
   }
 }
+
+function imageBlobPart(image: AgentImage): BlobPart {
+  if (image instanceof Blob || image instanceof ArrayBuffer) {
+    return image;
+  }
+
+  const bytes = new Uint8Array(image.byteLength);
+  bytes.set(image);
+  return bytes;
+}