Skip to content

Commit e1385d9

Browse files
hackall360hackall360
authored
Merge pull request #27 from Unity-Lab-AI/codex/evaluate-website-functionality-and-interactivity
Handle multimodal pollinations responses
2 parents db0b3c3 + d8e4ab5 commit e1385d9

3 files changed

Lines changed: 195 additions & 130 deletions

File tree

‎js/chat/chat-core.js‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -537,11 +537,31 @@ document.addEventListener("DOMContentLoaded", () => {
537537
// Use polliLib OpenAI-compatible chat endpoint
538538
const data = await (window.polliLib?.chat?.({ model, messages }) ?? Promise.reject(new Error('polliLib not loaded')));
539539
loadingDiv.remove();
540-
const aiContentRaw = data?.choices?.[0]?.message?.content || "";
541-
let aiContent = aiContentRaw;
542-
543-
const memRegex = /\[memory\]([\s\S]*?)\[\/memory\]/gi;
544-
let m;
540+
541+
const messageObj = data?.choices?.[0]?.message || {};
542+
const imageUrls = [];
543+
const audioUrls = [];
544+
let aiContent = "";
545+
546+
if (Array.isArray(messageObj.content)) {
547+
for (const part of messageObj.content) {
548+
if (!part) continue;
549+
if (typeof part === 'string') {
550+
aiContent += part;
551+
} else if (part.type === 'text' && part.text) {
552+
aiContent += part.text;
553+
} else if (part.type === 'image_url' && part.image_url?.url) {
554+
imageUrls.push(part.image_url.url);
555+
} else if (part.type === 'audio' && part.audio?.url) {
556+
audioUrls.push(part.audio.url);
557+
}
558+
}
559+
} else {
560+
aiContent = messageObj.content || "";
561+
}
562+
563+
const memRegex = /\[memory\]([\s\S]*?)\[\/memory\]/gi;
564+
let m;
545565
while ((m = memRegex.exec(aiContent)) !== null) Memory.addMemoryEntry(m[1].trim());
546566
aiContent = aiContent.replace(memRegex, "").trim();
547567

@@ -572,12 +592,12 @@ document.addEventListener("DOMContentLoaded", () => {
572592
}
573593
}
574594

575-
window.addNewMessage({ role: "ai", content: aiContent });
576-
if (autoSpeakEnabled) {
577-
const sentences = aiContent.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0);
578-
speakSentences(sentences);
579-
} else {
580-
stopSpeaking();
595+
window.addNewMessage({ role: "ai", content: aiContent, imageUrls, audioUrls });
596+
if (autoSpeakEnabled) {
597+
const sentences = aiContent.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0);
598+
speakSentences(sentences);
599+
} else {
600+
stopSpeaking();
581601
}
582602
if (callback) callback();
583603
} catch (err) {

‎js/chat/chat-init.js‎

Lines changed: 84 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ document.addEventListener("DOMContentLoaded", () => {
1717
if (!window.Prism) return;
1818
chatBox.querySelectorAll("pre code").forEach(block => Prism.highlightElement(block));
1919
};
20-
const appendMessage = ({ role, content, index, imageUrls = [] }) => {
20+
const appendMessage = ({ role, content, index, imageUrls = [], audioUrls = [] }) => {
2121
const container = document.createElement("div");
2222
container.classList.add("message");
2323
container.dataset.index = index;
@@ -63,15 +63,21 @@ document.addEventListener("DOMContentLoaded", () => {
6363
bubbleContent.appendChild(textNode);
6464
}
6565
}
66-
if (imageUrls.length > 0) {
67-
imageUrls.forEach(url => {
68-
const imageContainer = createImageElement(url, index);
69-
bubbleContent.appendChild(imageContainer);
70-
});
71-
}
72-
} else {
73-
bubbleContent.textContent = content;
74-
}
66+
if (imageUrls.length > 0) {
67+
imageUrls.forEach(url => {
68+
const imageContainer = createImageElement(url, index);
69+
bubbleContent.appendChild(imageContainer);
70+
});
71+
}
72+
if (audioUrls.length > 0) {
73+
audioUrls.forEach(url => {
74+
const audioEl = createAudioElement(url);
75+
bubbleContent.appendChild(audioEl);
76+
});
77+
}
78+
} else {
79+
bubbleContent.textContent = content;
80+
}
7581
container.appendChild(bubbleContent);
7682
const actionsDiv = document.createElement("div");
7783
actionsDiv.className = "message-actions";
@@ -281,8 +287,8 @@ document.addEventListener("DOMContentLoaded", () => {
281287
window.open(img.src, "_blank");
282288
showToast("Image opened in new tab");
283289
};
284-
const createImageElement = (url, msgIndex) => {
285-
const imageId = `img-${msgIndex}-${Date.now()}`;
290+
const createImageElement = (url, msgIndex) => {
291+
const imageId = `img-${msgIndex}-${Date.now()}`;
286292
localStorage.setItem(`imageId_${msgIndex}`, imageId);
287293
const imageContainer = document.createElement("div");
288294
imageContainer.className = "ai-image-container";
@@ -315,11 +321,19 @@ document.addEventListener("DOMContentLoaded", () => {
315321
imageContainer.appendChild(img);
316322
const imgButtonContainer = document.createElement("div");
317323
imgButtonContainer.className = "image-button-container";
318-
imgButtonContainer.dataset.imageId = imageId;
319-
imageContainer.appendChild(imgButtonContainer);
320-
return imageContainer;
321-
};
322-
const attachImageButtonListeners = (img, imageId) => {
324+
imgButtonContainer.dataset.imageId = imageId;
325+
imageContainer.appendChild(imgButtonContainer);
326+
return imageContainer;
327+
};
328+
const createAudioElement = (url) => {
329+
const audio = document.createElement("audio");
330+
audio.controls = true;
331+
audio.src = url;
332+
audio.className = "ai-generated-audio";
333+
audio.style.display = "block";
334+
return audio;
335+
};
336+
const attachImageButtonListeners = (img, imageId) => {
323337
const imgButtonContainer = document.querySelector(`.image-button-container[data-image-id="${imageId}"]`);
324338
if (!imgButtonContainer) {
325339
console.warn(`No image button container found for image ID: ${imageId}`);
@@ -372,58 +386,64 @@ document.addEventListener("DOMContentLoaded", () => {
372386
});
373387
imgButtonContainer.appendChild(openImgBtn);
374388
};
375-
const renderStoredMessages = messages => {
376-
console.log("Rendering stored messages...");
377-
chatBox.innerHTML = "";
378-
messages.forEach((msg, idx) => {
379-
console.log(`Appending message at index ${idx}: ${msg.role}`);
380-
if (!window.polliClient || !window.polliClient.imageBase) return appendMessage({ role: msg.role, content: msg.content, index: idx, imageUrls: [] });
381-
const base = window.polliClient.imageBase;
382-
const escaped = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
383-
const imgRegex = new RegExp(`(${escaped}\/prompt\/[^ ]+)`, 'g');
384-
const imgMatches = msg.content.match(imgRegex) || [];
385-
appendMessage({
386-
role: msg.role,
387-
content: msg.content,
388-
index: idx,
389-
imageUrls: imgMatches
390-
});
391-
});
392-
messages.forEach((msg, idx) => {
393-
const storedImageId = localStorage.getItem(`imageId_${idx}`);
394-
if (storedImageId) {
395-
const img = chatBox.querySelector(`img[data-image-id="${storedImageId}"]`);
396-
if (img) {
397-
console.log(`Re-attaching image button listeners for stored image ID: ${storedImageId}`);
398-
attachImageButtonListeners(img, storedImageId);
399-
} else {
400-
console.warn(`Image with ID ${storedImageId} not found in DOM`);
401-
}
402-
}
403-
});
404-
highlightAllCodeBlocks();
405-
};
406-
window.addNewMessage = ({ role, content }) => {
407-
const currentSession = Storage.getCurrentSession();
408-
currentSession.messages.push({ role, content });
409-
Storage.updateSessionMessages(currentSession.id, currentSession.messages);
389+
const renderStoredMessages = messages => {
390+
console.log("Rendering stored messages...");
391+
chatBox.innerHTML = "";
392+
messages.forEach((msg, idx) => {
393+
console.log(`Appending message at index ${idx}: ${msg.role}`);
394+
const storedImages = Array.isArray(msg.imageUrls) ? msg.imageUrls : [];
395+
const storedAudio = Array.isArray(msg.audioUrls) ? msg.audioUrls : [];
396+
let imgMatches = storedImages;
397+
if (imgMatches.length === 0 && msg.content && window.polliClient && window.polliClient.imageBase) {
398+
const base = window.polliClient.imageBase;
399+
const escaped = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
400+
const imgRegex = new RegExp(`(${escaped}\/prompt\/[^ ]+)`, 'g');
401+
imgMatches = msg.content.match(imgRegex) || [];
402+
}
403+
appendMessage({
404+
role: msg.role,
405+
content: msg.content,
406+
index: idx,
407+
imageUrls: imgMatches,
408+
audioUrls: storedAudio
409+
});
410+
});
411+
messages.forEach((msg, idx) => {
412+
const storedImageId = localStorage.getItem(`imageId_${idx}`);
413+
if (storedImageId) {
414+
const img = chatBox.querySelector(`img[data-image-id="${storedImageId}"]`);
415+
if (img) {
416+
console.log(`Re-attaching image button listeners for stored image ID: ${storedImageId}`);
417+
attachImageButtonListeners(img, storedImageId);
418+
} else {
419+
console.warn(`Image with ID ${storedImageId} not found in DOM`);
420+
}
421+
}
422+
});
423+
highlightAllCodeBlocks();
424+
};
425+
window.addNewMessage = ({ role, content, imageUrls = [], audioUrls = [] }) => {
426+
const currentSession = Storage.getCurrentSession();
427+
currentSession.messages.push({ role, content, imageUrls, audioUrls });
428+
Storage.updateSessionMessages(currentSession.id, currentSession.messages);
410429
if (!window.polliClient || !window.polliClient.imageBase) {
411-
appendMessage({ role, content, index: currentSession.messages.length - 1, imageUrls: [] });
430+
appendMessage({ role, content, index: currentSession.messages.length - 1, imageUrls, audioUrls });
412431
if (role === "ai") checkAndUpdateSessionTitle();
413432
return;
414433
}
415434
const base = window.polliClient.imageBase;
416-
const escaped = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
417-
const imgRegex = new RegExp(`(${escaped}\/prompt\/[^ ]+)`, 'g');
418-
const imgMatches = content.match(imgRegex) || [];
419-
appendMessage({
420-
role,
421-
content,
422-
index: currentSession.messages.length - 1,
423-
imageUrls: imgMatches
424-
});
425-
if (role === "ai") checkAndUpdateSessionTitle();
426-
};
435+
const escaped = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
436+
const imgRegex = new RegExp(`(${escaped}\/prompt\/[^ ]+)`, 'g');
437+
const imgMatches = imageUrls.length > 0 ? imageUrls : (content.match(imgRegex) || []);
438+
appendMessage({
439+
role,
440+
content,
441+
index: currentSession.messages.length - 1,
442+
imageUrls: imgMatches,
443+
audioUrls
444+
});
445+
if (role === "ai") checkAndUpdateSessionTitle();
446+
};
427447
const editMessage = msgIndex => {
428448
const currentSession = Storage.getCurrentSession();
429449
const oldMessage = currentSession.messages[msgIndex];

0 commit comments

Comments
 (0)