diff --git a/.changeset/lemonslice-direct-image-upload.md b/.changeset/lemonslice-direct-image-upload.md new file mode 100644 index 000000000..7efb11af3 --- /dev/null +++ b/.changeset/lemonslice-direct-image-upload.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-lemonslice': patch +--- + +Allow direct image uploads for LemonSlice avatars. diff --git a/plugins/lemonslice/etc/agents-plugin-lemonslice.api.md b/plugins/lemonslice/etc/agents-plugin-lemonslice.api.md new file mode 100644 index 000000000..234504e58 --- /dev/null +++ b/plugins/lemonslice/etc/agents-plugin-lemonslice.api.md @@ -0,0 +1,53 @@ +## API Report File for "@livekit/agents-plugin-lemonslice" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { APIConnectOptions } from '@livekit/agents'; +import type { Room } from '@livekit/rtc-node'; +import { voice } from '@livekit/agents'; + +// @public +export type AgentImage = Uint8Array | ArrayBuffer | Blob; + +// @public +export class AvatarSession extends voice.AvatarSession { + constructor(options?: AvatarSessionOptions); + // (undocumented) + get avatarIdentity(): string; + // (undocumented) + get provider(): string; + start(agentSession: voice.AgentSession, room: Room, options?: StartOptions): Promise; +} + +// @public +export interface AvatarSessionOptions { + agentId?: string | null; + agentImage?: AgentImage | null; + agentImageUrl?: string | null; + agentPrompt?: string | null; + apiKey?: string; + apiUrl?: string; + avatarParticipantIdentity?: string; + avatarParticipantName?: string; + connOptions?: APIConnectOptions; + extraPayload?: Record | null; + idleTimeout?: number | null; +} + +// @public +export class LemonSliceException extends Error { + constructor(message: string); +} + +// @public +export interface StartOptions { + livekitApiKey?: string; + livekitApiSecret?: string; + livekitUrl?: string; +} + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/lemonslice/src/avatar.ts b/plugins/lemonslice/src/avatar.ts index e4d6a970b..39da92f1a 100644 --- a/plugins/lemonslice/src/avatar.ts +++ b/plugins/lemonslice/src/avatar.ts @@ -22,8 +22,17 @@ const SAMPLE_RATE = 16000; const AVATAR_AGENT_IDENTITY = 'lemonslice-avatar-agent'; const AVATAR_AGENT_NAME = 'lemonslice-avatar-agent'; +/** + * Image bytes accepted for direct LemonSlice avatar uploads. + * + * @public + */ +export type AgentImage = Uint8Array | ArrayBuffer | Blob; + /** * Exception thrown when there are errors with the LemonSlice API. + * + * @public */ export class LemonSliceException extends Error { constructor(message: string) { @@ -34,18 +43,25 @@ export class LemonSliceException extends Error { /** * Options for configuring an AvatarSession. + * + * @public */ export interface AvatarSessionOptions { /** * The ID of the LemonSlice agent to add to the session. - * Either agentId or agentImageUrl must be provided. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. */ agentId?: string | null; /** * The URL of the image to use as the agent's avatar. - * Either agentId or agentImageUrl must be provided. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. */ agentImageUrl?: string | null; + /** + * Image bytes to upload and use as the agent's avatar. + * Exactly one of agentId, agentImageUrl, or agentImage must be provided. + */ + agentImage?: AgentImage | null; /** * A prompt that subtly influences the avatar's movements and expressions. */ @@ -83,6 +99,8 @@ export interface AvatarSessionOptions { /** * Options for starting an avatar session. + * + * @public */ export interface StartOptions { /** @@ -121,10 +139,13 @@ export interface StartOptions { * }); * await avatar.start(agentSession, room); * ``` + * + * @public */ export class AvatarSession extends voice.AvatarSession { private agentId: string | null; private agentImageUrl: string | null; + private agentImage: AgentImage | null; private agentPrompt: string | null; private idleTimeout: number | null; private extraPayload: Record | null; @@ -140,18 +161,24 @@ export class AvatarSession extends voice.AvatarSession { * Creates a new AvatarSession. * * @param options - Configuration options for the avatar session - * @throws LemonSliceException if invalid agentId or agentImageUrl is provided, or if LemonSlice API key is not set + * @throws LemonSliceException if invalid agentId, agentImageUrl, or agentImage is provided, or if LemonSlice API key is not set */ constructor(options: AvatarSessionOptions = {}) { super(); this.agentId = options.agentId ?? null; this.agentImageUrl = options.agentImageUrl ?? null; + this.agentImage = options.agentImage ?? null; - if (!this.agentId && !this.agentImageUrl) { - throw new LemonSliceException('Missing agentId or agentImageUrl'); + const givenSources = [this.agentId, this.agentImageUrl, this.agentImage].filter( + (source) => source !== null, + ); + if (givenSources.length === 0) { + throw new LemonSliceException('Missing one of agentId, agentImageUrl or agentImage'); } - if (this.agentId && this.agentImageUrl) { - throw new LemonSliceException('Only one of agentId or agentImageUrl can be provided'); + if (givenSources.length > 1) { + throw new LemonSliceException( + 'Only one of agentId, agentImageUrl or agentImage can be provided', + ); } this.agentPrompt = options.agentPrompt ?? null; @@ -266,7 +293,7 @@ export class AvatarSession extends voice.AvatarSession { private async startAgent(livekitUrl: string, livekitToken: string): Promise { for (let i = 0; i <= this.connOptions.maxRetry; i++) { try { - const payload: Record = { + const payload: Record = { transport_type: 'livekit', properties: { livekit_url: livekitUrl, @@ -274,15 +301,15 @@ export class AvatarSession extends voice.AvatarSession { }, }; - if (this.agentId) { + if (this.agentId !== null) { payload.agent_id = this.agentId; } - if (this.agentImageUrl) { + if (this.agentImageUrl !== null) { payload.agent_image_url = this.agentImageUrl; } - if (this.agentPrompt) { + if (this.agentPrompt !== null) { payload.agent_prompt = this.agentPrompt; } @@ -294,13 +321,33 @@ export class AvatarSession extends voice.AvatarSession { Object.assign(payload, this.extraPayload); } + const headers: Record = { + 'X-API-Key': this.apiKey, + }; + let body: BodyInit; + + if (this.agentImage !== null) { + const formData = new FormData(); + formData.append( + 'payload', + new Blob([JSON.stringify(payload)], { type: 'application/json' }), + ); + formData.append( + 'image', + new File([imageBlobPart(this.agentImage)], 'image.png', { + type: 'image/png', + }), + ); + body = formData; + } else { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify(payload); + } + const response = await fetch(this.apiUrl, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-API-Key': this.apiKey, - }, - body: JSON.stringify(payload), + headers, + body, signal: AbortSignal.timeout(this.connOptions.timeoutMs), }); @@ -336,3 +383,13 @@ export class AvatarSession extends voice.AvatarSession { }); } } + +function imageBlobPart(image: AgentImage): BlobPart { + if (image instanceof Blob || image instanceof ArrayBuffer) { + return image; + } + + const bytes = new Uint8Array(image.byteLength); + bytes.set(image); + return bytes; +}