diff --git a/Cargo.toml b/Cargo.toml index 2b65f8d..cd7da48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "echokit" -version = "0.2.2" +version = "0.3.0" authors = ["csh <458761603@qq.com>"] edition = "2021" resolver = "2" @@ -26,7 +26,7 @@ experimental = ["esp-idf-svc/experimental"] boards = ["voice_interrupt"] _no_default = [] -box = ["_no_default", "voice_interrupt"] +box = ["_no_default", "voice_interrupt", "custom_ui"] cube = ["_no_default", "voice_interrupt"] cube2 = ["_no_default", "voice_interrupt"] nfc_cube2 = ["cube2", "mfrc522", "exio"] @@ -38,6 +38,7 @@ extra_server = [] i2c = [] voice_interrupt = [] +custom_ui = [] [dependencies] log = "0.4" @@ -53,14 +54,17 @@ serde_json = "1.0" rmp-serde = "1" esp32-nimble = "0.11.1" -# embedded-websocket = { version = "0.9.4" } + +# UI libraries embedded-graphics = "0.8.1" embedded-text = "0.7.2" -# async-io = "2.4.0" - u8g2-fonts = { version = "0.6.0", features = ["embedded_graphics_textstyle"] } -tinygif = "0.0.4" -# futures-lite = "2.6.0" +image = { version = "0.25.6", default-features = false, features = [ + "png", + "gif", + "webp", +] } + futures-util = { version = "0.3.31", features = ["sink"] } # futures-sink = "0.3.31" diff --git a/assets/96x96.png b/assets/96x96.png new file mode 100755 index 0000000..9e44b4c Binary files /dev/null and b/assets/96x96.png differ diff --git a/assets/avatar.gif b/assets/avatar.gif new file mode 100755 index 0000000..14b2043 Binary files /dev/null and b/assets/avatar.gif differ diff --git a/assets/lm_320x240.png b/assets/lm_320x240.png new file mode 100755 index 0000000..45eace3 Binary files /dev/null and b/assets/lm_320x240.png differ diff --git a/assets/xx.gif b/assets/xx.gif new file mode 100644 index 0000000..f759970 Binary files /dev/null and b/assets/xx.gif differ diff --git a/components/hal_driver/lcd.c b/components/hal_driver/lcd.c index 0aa2b7d..0ccab51 100644 --- a/components/hal_driver/lcd.c +++ b/components/hal_driver/lcd.c @@ -24,6 +24,8 @@ static const char *TAG = "LCD"; esp_lcd_panel_handle_t panel_handle = NULL; /* LCD句柄 */ +uint16_t *lcd_dma_buffer = NULL; + uint32_t g_back_color = 0xFFFF; lcd_obj_t lcd_dev; @@ -129,23 +131,23 @@ void lcd_color_fill(uint16_t sx, uint16_t sy, uint16_t ex, uint16_t ey, uint16_t uint16_t height = ey - sy; uint32_t buf_index = 0; - uint16_t *buffer = heap_caps_malloc(width * sizeof(uint16_t), MALLOC_CAP_INTERNAL); + // uint16_t *buffer = heap_caps_malloc(width * sizeof(uint16_t), MALLOC_CAP_INTERNAL); for (uint16_t y_index = 0; y_index < height; y_index++) { for (uint16_t x_index = 0; x_index < width; x_index++) { - buffer[x_index] = color[buf_index]; + lcd_dma_buffer[x_index] = color[buf_index]; buf_index++; } for (uint16_t i = 0; i < width; i += 80) { - esp_lcd_panel_draw_bitmap(panel_handle, sx + i, sy + y_index, sx + i + 80, sy + y_index + 1, &buffer[i]); + esp_lcd_panel_draw_bitmap(panel_handle, sx + i, sy + y_index, sx + i + 80, sy + y_index + 1, &lcd_dma_buffer[i]); } } /* 释放内存 */ - heap_caps_free(buffer); + // heap_caps_free(buffer); } /** @@ -284,7 +286,8 @@ void lcd_init(lcd_cfg_t lcd_config) }, .bus_width = 8, .max_transfer_bytes = lcd_dev.pwidth * lcd_dev.pheight * sizeof(uint16_t), - .psram_trans_align = 64, + // .psram_trans_align = 64, + .dma_burst_size = 64, .sram_trans_align = 4, }; ESP_ERROR_CHECK(esp_lcd_new_i80_bus(&bus_config, &i80_bus)); /* 新建80并口总线 */ @@ -293,7 +296,7 @@ void lcd_init(lcd_cfg_t lcd_config) /* 80并口配置 */ .cs_gpio_num = lcd_dev.cs, .pclk_hz = (10 * 1000 * 1000), - .trans_queue_depth = 10, + .trans_queue_depth = 15, .dc_levels = { .dc_idle_level = 0, .dc_cmd_level = 0, @@ -327,4 +330,6 @@ void lcd_init(lcd_cfg_t lcd_config) ESP_ERROR_CHECK(esp_lcd_panel_disp_on_off(panel_handle, true)); /* 启动屏幕 */ lcd_clear(WHITE); /* 默认填充白色 */ LCD_BL(1); /* 打开背光 */ + + lcd_dma_buffer = esp_lcd_i80_alloc_draw_buffer(io_handle, lcd_dev.pwidth * sizeof(uint16_t), MALLOC_CAP_DMA); } diff --git a/components/hal_driver/lcd.h b/components/hal_driver/lcd.h index 4c4c1f2..c619f0d 100644 --- a/components/hal_driver/lcd.h +++ b/components/hal_driver/lcd.h @@ -104,6 +104,7 @@ typedef struct _lcd_config_t /* 导出相关变量 */ extern lcd_obj_t lcd_dev; extern esp_lcd_panel_handle_t panel_handle; /* LCD句柄 */ +extern uint16_t *lcd_dma_buffer; /* lcd相关函数 */ void lcd_init(lcd_cfg_t lcd_config); /* 初始化lcd */ void lcd_clear(uint16_t color); /* 清除屏幕 */ diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 7a44a6c..395f0e4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -46,7 +46,7 @@ CONFIG_BT_ENABLED=y CONFIG_BT_BLE_ENABLED=y CONFIG_BT_BLUEDROID_ENABLED=n CONFIG_BT_NIMBLE_ENABLED=y -CONFIG_BT_NIMBLE_HOST_TASK_STACK_SIZE=7000 +#CONFIG_BT_NIMBLE_HOST_TASK_STACK_SIZE=7000 #CONFIG_BT_NIMBLE_NVS_PERSIST=y diff --git a/setup/index.html b/setup/index.html index ff002cb..bc1dbf6 100644 --- a/setup/index.html +++ b/setup/index.html @@ -92,6 +92,75 @@

Background Image

+ + +
+ +
+ 🛠️ Advanced Settings +
+
+
+ +
+
+

AFE Linear Gain

+ + +
+
+ + +
+
+

AGC Target Level

+ + +
+
+ + +
+
+

AGC Compression + Gain

+ + +
+
+
+
+
@@ -125,6 +194,9 @@

⚠️ Device Reset Required

const SERVER_URL_ID = "cef520a9-bcb5-4fc6-87f7-82804eee2b20"; const BACKGROUND_IMAGE_ID = "d1f3b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d"; const RESET_ID = "f0e1d2c3-b4a5-6789-0abc-def123456789"; + const AFE_LINEAR_GAIN_ID = "a1b2c3d4-e5f6-4789-0abc-def123456789"; + const AGC_TARGET_LEVEL_ID = "b2c3d4e5-f6a7-4890-1bcd-ef2345678901"; + const AGC_COMPRESSION_GAIN_ID = "c3d4e5f6-a7b8-4901-2cde-f34567890123"; // global variables let device = null; @@ -156,6 +228,22 @@

⚠️ Device Reset Required

const toastMessage = document.getElementById('toastMessage'); const resetNotSupportedModal = document.getElementById('resetNotSupportedModal'); + // AFE related DOM elements + const afeLinearGainRange = document.getElementById('afeLinearGainRange'); + const afeLinearGainValue = document.getElementById('afeLinearGainValue'); + const saveAfeLinearGainButton = document.getElementById('saveAfeLinearGainButton'); + const afeLinearGainTitle = document.getElementById('afeLinearGainTitle'); + + const agcTargetLevelRange = document.getElementById('agcTargetLevelRange'); + const agcTargetLevelValue = document.getElementById('agcTargetLevelValue'); + const saveAgcTargetLevelButton = document.getElementById('saveAgcTargetLevelButton'); + const agcTargetLevelTitle = document.getElementById('agcTargetLevelTitle'); + + const agcCompressionGainRange = document.getElementById('agcCompressionGainRange'); + const agcCompressionGainValue = document.getElementById('agcCompressionGainValue'); + const saveAgcCompressionGainButton = document.getElementById('saveAgcCompressionGainButton'); + const agcCompressionGainTitle = document.getElementById('agcCompressionGainTitle'); + // Track modified fields const modifiedFields = { ssid: false, @@ -175,7 +263,9 @@

⚠️ Device Reset Required

// Clear field modification mark function clearFieldModification(fieldName, titleElement) { modifiedFields[fieldName] = false; - titleElement.textContent = titleElement.textContent.replace(' *', ''); + if (titleElement) { + titleElement.textContent = titleElement.textContent.replace(' *', ''); + } updateSaveButtonState(); } @@ -335,6 +425,11 @@

⚠️ Device Reset Required

backgroundImage.disabled = false; clearBgButton.disabled = false; controlPanel.classList.remove('opacity-50', 'pointer-events-none'); + + // Enable AFE controls + afeLinearGainRange.disabled = false; + agcTargetLevelRange.disabled = false; + agcCompressionGainRange.disabled = false; } // Disable all controls @@ -348,6 +443,14 @@

⚠️ Device Reset Required

writeBgButton.disabled = true; clearBgButton.disabled = true; controlPanel.classList.add('opacity-50', 'pointer-events-none'); + + // Disable AFE controls + afeLinearGainRange.disabled = true; + agcTargetLevelRange.disabled = true; + agcCompressionGainRange.disabled = true; + saveAfeLinearGainButton.disabled = true; + saveAgcTargetLevelButton.disabled = true; + saveAgcCompressionGainButton.disabled = true; } // Reads Characteristic @@ -388,6 +491,11 @@

⚠️ Device Reset Required

await readCharacteristic(PASS_ID, passInput); await readCharacteristic(SERVER_URL_ID, serverUrlInput); + // Load AFE parameters + await readAfeLinearGain(); + await readAgcTargetLevel(); + await readAgcCompressionGain(); + // Clear all modification marks clearFieldModification('ssid', ssidTitle); clearFieldModification('pass', passTitle); @@ -614,6 +722,173 @@

⚠️ Device Reset Required

showNotification('Message', 'Background image cleared'); }); + // Read AFE Linear Gain (string format f32) + async function readAfeLinearGain() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AFE_LINEAR_GAIN_ID); + const value = await characteristic.readValue(); + const decoder = new TextDecoder(); + const stringValue = decoder.decode(value); + const gain = parseFloat(stringValue); + if (!isNaN(gain)) { + afeLinearGainRange.value = Math.round(gain * 10); + afeLinearGainValue.textContent = gain.toFixed(1); + // Reset title (remove not supported label) + afeLinearGainTitle.textContent = 'AFE Linear Gain'; + saveAfeLinearGainButton.disabled = true; + } + return true; + } catch (error) { + console.error('Failed to read AFE Linear Gain:', error); + // Backward compatibility: disable this control + afeLinearGainRange.disabled = true; + saveAfeLinearGainButton.disabled = true; + afeLinearGainTitle.textContent = 'AFE Linear Gain (Not Supported)'; + return false; + } + } + + // Read AGC Target Level (i32, 4 bytes little endian) + async function readAgcTargetLevel() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AGC_TARGET_LEVEL_ID); + const value = await characteristic.readValue(); + // value is already DataView, use directly + const level = value.getInt32(0, true); + agcTargetLevelRange.value = level; + agcTargetLevelValue.textContent = level; + // Reset title (remove not supported label) + agcTargetLevelTitle.textContent = 'AGC Target Level'; + saveAgcTargetLevelButton.disabled = true; + return true; + } catch (error) { + console.error('Failed to read AGC Target Level:', error); + // Backward compatibility: disable this control + agcTargetLevelRange.disabled = true; + saveAgcTargetLevelButton.disabled = true; + agcTargetLevelTitle.textContent = 'AGC Target Level (Not Supported)'; + return false; + } + } + + // Read AGC Compression Gain (i32, 4 bytes little endian) + async function readAgcCompressionGain() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AGC_COMPRESSION_GAIN_ID); + const value = await characteristic.readValue(); + // value is already DataView, use directly + const gain = value.getInt32(0, true); + agcCompressionGainRange.value = gain; + agcCompressionGainValue.textContent = gain; + // Reset title (remove not supported label) + agcCompressionGainTitle.textContent = 'AGC Compression Gain'; + saveAgcCompressionGainButton.disabled = true; + return true; + } catch (error) { + console.error('Failed to read AGC Compression Gain:', error); + // Backward compatibility: disable this control + agcCompressionGainRange.disabled = true; + saveAgcCompressionGainButton.disabled = true; + agcCompressionGainTitle.textContent = 'AGC Compression Gain (Not Supported)'; + return false; + } + } + + // AFE Linear Gain save function + async function saveAfeLinearGain() { + if (!isConnected || !service) { + showNotification('Error', 'Device not connected', true); + return; + } + try { + const gain = parseFloat((afeLinearGainRange.value / 10).toFixed(1)); + const characteristic = await service.getCharacteristic(AFE_LINEAR_GAIN_ID); + const encoder = new TextEncoder(); + const data = encoder.encode(gain.toString()); + await characteristic.writeValue(data); + showNotification('Success', `AFE Linear Gain set to ${gain}`); + clearFieldModification('afeLinearGain', afeLinearGainTitle); + saveAfeLinearGainButton.disabled = true; + } catch (error) { + console.error('Failed to save AFE Linear Gain:', error); + showNotification('Error', 'Failed to save AFE Linear Gain: ' + error.message, true); + } + } + + // AGC Target Level save function + async function saveAgcTargetLevel() { + if (!isConnected || !service) { + showNotification('Error', 'Device not connected', true); + return; + } + try { + const level = parseInt(agcTargetLevelRange.value); + const characteristic = await service.getCharacteristic(AGC_TARGET_LEVEL_ID); + const data = new ArrayBuffer(4); + const view = new DataView(data); + view.setInt32(0, level, true); + await characteristic.writeValue(data); + showNotification('Success', `AGC Target Level set to ${level}`); + clearFieldModification('agcTargetLevel', agcTargetLevelTitle); + saveAgcTargetLevelButton.disabled = true; + } catch (error) { + console.error('Failed to save AGC Target Level:', error); + showNotification('Error', 'Failed to save AGC Target Level: ' + error.message, true); + } + } + + // AGC Compression Gain save function + async function saveAgcCompressionGain() { + if (!isConnected || !service) { + showNotification('Error', 'Device not connected', true); + return; + } + try { + const gain = parseInt(agcCompressionGainRange.value); + const characteristic = await service.getCharacteristic(AGC_COMPRESSION_GAIN_ID); + const data = new ArrayBuffer(4); + const view = new DataView(data); + view.setInt32(0, gain, true); + await characteristic.writeValue(data); + showNotification('Success', `AGC Compression Gain set to ${gain}`); + clearFieldModification('agcCompressionGain', agcCompressionGainTitle); + saveAgcCompressionGainButton.disabled = true; + } catch (error) { + console.error('Failed to save AGC Compression Gain:', error); + showNotification('Error', 'Failed to save AGC Compression Gain: ' + error.message, true); + } + } + + // AFE Linear Gain slider event + afeLinearGainRange.addEventListener('input', () => { + const gain = (afeLinearGainRange.value / 10).toFixed(1); + afeLinearGainValue.textContent = gain; + markFieldAsModified('afeLinearGain', afeLinearGainTitle); + saveAfeLinearGainButton.disabled = false; + }); + + // AGC Target Level slider event + agcTargetLevelRange.addEventListener('input', () => { + agcTargetLevelValue.textContent = agcTargetLevelRange.value; + markFieldAsModified('agcTargetLevel', agcTargetLevelTitle); + saveAgcTargetLevelButton.disabled = false; + }); + + // AGC Compression Gain slider event + agcCompressionGainRange.addEventListener('input', () => { + agcCompressionGainValue.textContent = agcCompressionGainRange.value; + markFieldAsModified('agcCompressionGain', agcCompressionGainTitle); + saveAgcCompressionGainButton.disabled = false; + }); + + // AFE save button events + saveAfeLinearGainButton.addEventListener('click', saveAfeLinearGain); + saveAgcTargetLevelButton.addEventListener('click', saveAgcTargetLevel); + saveAgcCompressionGainButton.addEventListener('click', saveAgcCompressionGain); + if (!navigator.bluetooth) { showNotification('Error', 'Your browser does not support the Web Bluetooth API. Please use Chrome or Edge', true); connectButton.disabled = true; @@ -624,4 +899,4 @@

⚠️ Device Reset Required

- + \ No newline at end of file diff --git a/setup/index_zh.html b/setup/index_zh.html index 4c35a27..8b4e199 100644 --- a/setup/index_zh.html +++ b/setup/index_zh.html @@ -89,6 +89,71 @@

背景图片设置

+ + +
+ +
+ 🛠️ 高级设置 +
+
+
+ +
+
+

AFE 线性增益

+ + +
+
+ + +
+
+

AGC 目标电平

+ + +
+
+ + +
+
+

AGC 压缩增益

+ + +
+
+
+
+
@@ -122,6 +187,9 @@

⚠️ 需要重启设备

const SERVER_URL_ID = "cef520a9-bcb5-4fc6-87f7-82804eee2b20"; const BACKGROUND_IMAGE_ID = "d1f3b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d"; const RESET_ID = "f0e1d2c3-b4a5-6789-0abc-def123456789"; + const AFE_LINEAR_GAIN_ID = "a1b2c3d4-e5f6-4789-0abc-def123456789"; + const AGC_TARGET_LEVEL_ID = "b2c3d4e5-f6a7-4890-1bcd-ef2345678901"; + const AGC_COMPRESSION_GAIN_ID = "c3d4e5f6-a7b8-4901-2cde-f34567890123"; // 全局变量 let device = null; @@ -153,6 +221,22 @@

⚠️ 需要重启设备

const toastMessage = document.getElementById('toastMessage'); const resetNotSupportedModal = document.getElementById('resetNotSupportedModal'); + // AFE 相关 DOM 元素 + const afeLinearGainRange = document.getElementById('afeLinearGainRange'); + const afeLinearGainValue = document.getElementById('afeLinearGainValue'); + const saveAfeLinearGainButton = document.getElementById('saveAfeLinearGainButton'); + const afeLinearGainTitle = document.getElementById('afeLinearGainTitle'); + + const agcTargetLevelRange = document.getElementById('agcTargetLevelRange'); + const agcTargetLevelValue = document.getElementById('agcTargetLevelValue'); + const saveAgcTargetLevelButton = document.getElementById('saveAgcTargetLevelButton'); + const agcTargetLevelTitle = document.getElementById('agcTargetLevelTitle'); + + const agcCompressionGainRange = document.getElementById('agcCompressionGainRange'); + const agcCompressionGainValue = document.getElementById('agcCompressionGainValue'); + const saveAgcCompressionGainButton = document.getElementById('saveAgcCompressionGainButton'); + const agcCompressionGainTitle = document.getElementById('agcCompressionGainTitle'); + // 跟踪哪些字段被修改 const modifiedFields = { ssid: false, @@ -172,7 +256,9 @@

⚠️ 需要重启设备

// 清除字段修改标记 function clearFieldModification(fieldName, titleElement) { modifiedFields[fieldName] = false; - titleElement.textContent = titleElement.textContent.replace(' *', ''); + if (titleElement) { + titleElement.textContent = titleElement.textContent.replace(' *', ''); + } updateSaveButtonState(); } @@ -332,6 +418,11 @@

⚠️ 需要重启设备

backgroundImage.disabled = false; clearBgButton.disabled = false; controlPanel.classList.remove('opacity-50', 'pointer-events-none'); + + // 启用 AFE 控件 + afeLinearGainRange.disabled = false; + agcTargetLevelRange.disabled = false; + agcCompressionGainRange.disabled = false; } // 禁用所有控件 @@ -345,6 +436,14 @@

⚠️ 需要重启设备

writeBgButton.disabled = true; clearBgButton.disabled = true; controlPanel.classList.add('opacity-50', 'pointer-events-none'); + + // 禁用 AFE 控件 + afeLinearGainRange.disabled = true; + agcTargetLevelRange.disabled = true; + agcCompressionGainRange.disabled = true; + saveAfeLinearGainButton.disabled = true; + saveAgcTargetLevelButton.disabled = true; + saveAgcCompressionGainButton.disabled = true; } // 读取Characteristic值 @@ -372,6 +471,81 @@

⚠️ 需要重启设备

} } + // 读取 AFE Linear Gain (字符串格式的 f32) + async function readAfeLinearGain() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AFE_LINEAR_GAIN_ID); + const value = await characteristic.readValue(); + const decoder = new TextDecoder(); + const stringValue = decoder.decode(value); + const gain = parseFloat(stringValue); + if (!isNaN(gain)) { + afeLinearGainRange.value = Math.round(gain * 10); + afeLinearGainValue.textContent = gain.toFixed(1); + // 重置标题（移除不支持标识） + afeLinearGainTitle.textContent = 'AFE 线性增益'; + saveAfeLinearGainButton.disabled = true; + } + return true; + } catch (error) { + console.error('读取 AFE Linear Gain 失败:', error); + // 兼容旧版本：禁用该控件 + afeLinearGainRange.disabled = true; + saveAfeLinearGainButton.disabled = true; + afeLinearGainTitle.textContent = 'AFE 线性增益 (不支持)'; + return false; + } + } + + // 读取 AGC Target Level (i32, 4 bytes little endian) + async function readAgcTargetLevel() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AGC_TARGET_LEVEL_ID); + const value = await characteristic.readValue(); + // value 已经是 DataView，直接使用 + const level = value.getInt32(0, true); + agcTargetLevelRange.value = level; + agcTargetLevelValue.textContent = level; + // 重置标题（移除不支持标识） + agcTargetLevelTitle.textContent = 'AGC 目标电平'; + saveAgcTargetLevelButton.disabled = true; + return true; + } catch (error) { + console.error('读取 AGC Target Level 失败:', error); + // 兼容旧版本：禁用该控件 + agcTargetLevelRange.disabled = true; + saveAgcTargetLevelButton.disabled = true; + agcTargetLevelTitle.textContent = 'AGC 目标电平 (不支持)'; + return false; + } + } + + // 读取 AGC Compression Gain (i32, 4 bytes little endian) + async function readAgcCompressionGain() { + if (!isConnected || !service) return false; + try { + const characteristic = await service.getCharacteristic(AGC_COMPRESSION_GAIN_ID); + const value = await characteristic.readValue(); + // value 已经是 DataView，直接使用 + const gain = value.getInt32(0, true); + agcCompressionGainRange.value = gain; + agcCompressionGainValue.textContent = gain; + // 重置标题（移除不支持标识） + agcCompressionGainTitle.textContent = 'AGC 压缩增益'; + saveAgcCompressionGainButton.disabled = true; + return true; + } catch (error) { + console.error('读取 AGC Compression Gain 失败:', error); + // 兼容旧版本：禁用该控件 + agcCompressionGainRange.disabled = true; + saveAgcCompressionGainButton.disabled = true; + agcCompressionGainTitle.textContent = 'AGC 压缩增益 (不支持)'; + return false; + } + } + // 加载所有配置 async function loadAllConfiguration() { if (!isConnected || !service) { @@ -387,6 +561,11 @@

⚠️ 需要重启设备

await readCharacteristic(PASS_ID, passInput); await readCharacteristic(SERVER_URL_ID, serverUrlInput); + // 加载 AFE 参数 + await readAfeLinearGain(); + await readAgcTargetLevel(); + await readAgcCompressionGain(); + // 清除所有修改标记 clearFieldModification('ssid', ssidTitle); clearFieldModification('pass', passTitle); @@ -617,6 +796,98 @@

⚠️ 需要重启设备

showNotification('信息', '背景图片已清除'); }); + // AFE Linear Gain 保存函数 + async function saveAfeLinearGain() { + if (!isConnected || !service) { + showNotification('错误', '设备未连接', true); + return; + } + try { + const gain = parseFloat((afeLinearGainRange.value / 10).toFixed(1)); + const characteristic = await service.getCharacteristic(AFE_LINEAR_GAIN_ID); + const encoder = new TextEncoder(); + const data = encoder.encode(gain.toString()); + await characteristic.writeValue(data); + showNotification('成功', `AFE 线性增益已设置为 ${gain}`); + clearFieldModification('afeLinearGain', afeLinearGainTitle); + saveAfeLinearGainButton.disabled = true; + } catch (error) { + console.error('保存 AFE Linear Gain 失败:', error); + showNotification('错误', '保存 AFE 线性增益失败: ' + error.message, true); + } + } + + // AGC Target Level 保存函数 + async function saveAgcTargetLevel() { + if (!isConnected || !service) { + showNotification('错误', '设备未连接', true); + return; + } + try { + const level = parseInt(agcTargetLevelRange.value); + const characteristic = await service.getCharacteristic(AGC_TARGET_LEVEL_ID); + const data = new ArrayBuffer(4); + const view = new DataView(data); + view.setInt32(0, level, true); + await characteristic.writeValue(data); + showNotification('成功', `AGC 目标电平已设置为 ${level}`); + clearFieldModification('agcTargetLevel', agcTargetLevelTitle); + saveAgcTargetLevelButton.disabled = true; + } catch (error) { + console.error('保存 AGC Target Level 失败:', error); + showNotification('错误', '保存 AGC 目标电平失败: ' + error.message, true); + } + } + + // AGC Compression Gain 保存函数 + async function saveAgcCompressionGain() { + if (!isConnected || !service) { + showNotification('错误', '设备未连接', true); + return; + } + try { + const gain = parseInt(agcCompressionGainRange.value); + const characteristic = await service.getCharacteristic(AGC_COMPRESSION_GAIN_ID); + const data = new ArrayBuffer(4); + const view = new DataView(data); + view.setInt32(0, gain, true); + await characteristic.writeValue(data); + showNotification('成功', `AGC 压缩增益已设置为 ${gain}`); + clearFieldModification('agcCompressionGain', agcCompressionGainTitle); + saveAgcCompressionGainButton.disabled = true; + } catch (error) { + console.error('保存 AGC Compression Gain 失败:', error); + showNotification('错误', '保存 AGC 压缩增益失败: ' + error.message, true); + } + } + + // AFE Linear Gain 滑块事件 + afeLinearGainRange.addEventListener('input', () => { + const gain = (afeLinearGainRange.value / 10).toFixed(1); + afeLinearGainValue.textContent = gain; + markFieldAsModified('afeLinearGain', afeLinearGainTitle); + saveAfeLinearGainButton.disabled = false; + }); + + // AGC Target Level 滑块事件 + agcTargetLevelRange.addEventListener('input', () => { + agcTargetLevelValue.textContent = agcTargetLevelRange.value; + markFieldAsModified('agcTargetLevel', agcTargetLevelTitle); + saveAgcTargetLevelButton.disabled = false; + }); + + // AGC Compression Gain 滑块事件 + agcCompressionGainRange.addEventListener('input', () => { + agcCompressionGainValue.textContent = agcCompressionGainRange.value; + markFieldAsModified('agcCompressionGain', agcCompressionGainTitle); + saveAgcCompressionGainButton.disabled = false; + }); + + // AFE 保存按钮事件 + saveAfeLinearGainButton.addEventListener('click', saveAfeLinearGain); + saveAgcTargetLevelButton.addEventListener('click', saveAgcTargetLevel); + saveAgcCompressionGainButton.addEventListener('click', saveAgcCompressionGain); + // 检查浏览器是否支持Web Bluetooth API if (!navigator.bluetooth) { showNotification('错误', '您的浏览器不支持Web Bluetooth API，请使用Chrome或Edge浏览器', true); diff --git a/src/app.rs b/src/app.rs index 95059c1..3e55c4e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -5,6 +5,7 @@ use tokio::sync::mpsc; use crate::{ audio::{self, AudioEvent, EventRx}, protocol::{self, ServerEvent}, + ui::DisplayTargetDrive, ws::Server, }; @@ -14,6 +15,7 @@ pub enum Event { ServerEvent(ServerEvent), MicAudioChunk(Vec), MicAudioEnd, + Vowel(u8), MicInterruptWaitTimeout, #[cfg_attr(not(feature = "extra_server"), allow(unused))] ServerUrl(String), @@ -22,7 +24,6 @@ pub enum Event { #[allow(unused)] impl Event { pub const IDLE: &'static str = "idle"; - pub const GAIA: &'static str = "gaia"; pub const NO: &'static str = "no"; pub const YES: &'static str = "yes"; pub const NOISE: &'static str = "noise"; @@ -85,6 +86,9 @@ async fn select_evt( Event::MicInterruptWaitTimeout => { log::info!("[Select] Received MicInterruptWaitTimeout"); } + Event::Vowel(v) => { + log::debug!("[Select] Received Vowel: {}", v); + } Event::ServerUrl(url) => { log::info!("[Select] Received ServerUrl: {}", url); } @@ -156,11 +160,12 @@ const SPEED_LIMIT: f64 = 1.0; const INTERNAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); const NORMAL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60 * 5); -pub async fn main_work<'d>( +pub async fn main_work<'d, const N: usize>( mut server: Server, player_tx: audio::PlayerTx, mut evt_rx: EventRx, - backgroud_buffer: Option<&'d [u8]>, + framebuffer: &mut crate::boards::ui::DisplayBuffer, + gui: &mut crate::boards::ui::ChatUI, ) -> anyhow::Result<()> { #[derive(PartialEq, Eq)] enum State { @@ -170,10 +175,10 @@ pub async fn main_work<'d>( Idle, } - let mut gui = crate::ui::UI::new(backgroud_buffer, crate::boards::flush_display)?; - - gui.state = "Idle".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Idle".to_string()); + gui.set_text("".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; let mut state = State::Idle; @@ -200,15 +205,22 @@ pub async fn main_work<'d>( while let Some(evt) = select_evt(&mut evt_rx, &mut server, ¬ify, wait_notify, timeout).await { match evt { - Event::Event(Event::GAIA | Event::K0) => { + Event::Event(Event::K0) => { log::info!("Received event: k0"); if state == State::Listening { state = State::Idle; - gui.state = "Idle".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Idle".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; server.close().await?; } else { + gui.set_state("Connecting...".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; + + server.reconnect_with_retry(3).await?; + let hello_notify = Arc::new(tokio::sync::Notify::new()); player_tx .send(AudioEvent::Hello(hello_notify.clone())) @@ -216,8 +228,6 @@ pub async fn main_work<'d>( log::info!("Waiting for hello response"); let _ = hello_notify.notified().await; - server.reconnect_with_retry(3).await?; - start_submit = false; submit_audio = 0.0; audio_buffer = Vec::with_capacity(8192); @@ -225,8 +235,9 @@ pub async fn main_work<'d>( log::info!("Hello response received"); state = State::Listening; - gui.state = "Listening...".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Ready".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } Event::Event(Event::K0_) => { @@ -234,8 +245,9 @@ pub async fn main_work<'d>( { allow_interrupt = !allow_interrupt; log::info!("Set allow_interrupt to {}", allow_interrupt); - gui.state = format!("Interrupt: {}", allow_interrupt); - gui.display_flush().unwrap(); + gui.set_state(format!("Interrupt: {}", allow_interrupt)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } Event::Event(Event::VOL_UP) => { @@ -247,8 +259,9 @@ pub async fn main_work<'d>( .send(AudioEvent::VolSet(vol)) .map_err(|e| anyhow::anyhow!("Error sending volume set: {e:?}"))?; log::info!("Volume set to {}", vol); - gui.state = format!("Volume: {}", vol); - gui.display_flush().unwrap(); + gui.set_state(format!("Volume: {}", vol)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::Event(Event::VOL_DOWN) => { vol -= 1; @@ -259,8 +272,9 @@ pub async fn main_work<'d>( .send(AudioEvent::VolSet(vol)) .map_err(|e| anyhow::anyhow!("Error sending volume set: {e:?}"))?; log::info!("Volume set to {}", vol); - gui.state = format!("Volume: {}", vol); - gui.display_flush().unwrap(); + gui.set_state(format!("Volume: {}", vol)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::Event(Event::VOL_SWITCH) => { vol -= 1; @@ -271,16 +285,18 @@ pub async fn main_work<'d>( .send(AudioEvent::VolSet(vol)) .map_err(|e| anyhow::anyhow!("Error sending volume set: {e:?}"))?; log::info!("Volume set to {}", vol); - gui.state = format!("Volume: {}", vol); - gui.display_flush().unwrap(); + gui.set_state(format!("Volume: {}", vol)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::Event(Event::YES | Event::K1) => {} Event::Event(Event::IDLE) => { log::info!("Received idle event"); if state == State::Listening { state = State::Idle; - gui.state = "Idle".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Idle".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; server.close().await?; } } @@ -291,6 +307,11 @@ pub async fn main_work<'d>( Event::Event(evt) => { log::info!("Received event: {:?}", evt); } + Event::Vowel(v) => { + gui.set_avatar_index(v as usize); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; + } Event::MicAudioChunk(data) if state == State::Listening => { submit_audio += data.len() as f32 / 16000.0; audio_buffer.extend_from_slice(&data); @@ -306,6 +327,10 @@ pub async fn main_work<'d>( start_submit = true; server.send_client_audio_chunk_i16(audio_buffer).await?; audio_buffer = Vec::with_capacity(8192); + + gui.set_state("Listening...".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } Event::MicAudioChunk(data) if state == State::Speaking && allow_interrupt => { @@ -319,8 +344,9 @@ pub async fn main_work<'d>( if submit_audio > 0.6 { state = State::Listening; - gui.state = "Listening...".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Listening...".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; server.reconnect_with_retry(3).await?; @@ -377,8 +403,9 @@ pub async fn main_work<'d>( start_submit = false; wait_notify = false; state = State::Waiting; - gui.state = "Waiting...".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Waiting...".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } Event::MicInterruptWaitTimeout => { @@ -405,20 +432,23 @@ pub async fn main_work<'d>( start_submit = false; wait_notify = false; state = State::Waiting; - gui.state = "Waiting...".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Waiting...".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::ServerEvent(ServerEvent::ASR { text }) => { log::info!("Received ASR: {:?}", text); state = State::Speaking; - gui.state = "ASR".to_string(); - gui.text = text.trim().to_string(); - gui.display_flush().unwrap(); + gui.set_state("ASR".to_string()); + gui.set_asr(text.trim().to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::ServerEvent(ServerEvent::Action { action }) => { log::info!("Received action"); - gui.state = format!("Action: {}", action); - gui.display_flush().unwrap(); + gui.set_state(format!("Action: {}", action)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } Event::ServerEvent(ServerEvent::StartAudio { text }) => { start_audio = true; @@ -427,42 +457,15 @@ pub async fn main_work<'d>( continue; } log::info!("Received audio start: {:?}", text); - gui.state = format!("[{:.2}x]|Speaking...", speed); - gui.text = text.trim().to_string(); - gui.display_flush().unwrap(); + gui.set_state(format!("[{:.2}x]|Speaking...", speed)); + gui.set_text(text.trim().to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; player_tx .send(AudioEvent::StartSpeech) .map_err(|e| anyhow::anyhow!("Error sending start: {e:?}"))?; } - Event::ServerEvent(ServerEvent::AudioChunk { data }) => { - log::debug!("Received audio chunk"); - if state != State::Speaking { - log::debug!("Received audio chunk while not speaking"); - continue; - } - - if need_compute { - if start_audio { - metrics.reset(); - start_audio = false; - } - metrics.add_data(data.len()); - } - - if speed < SPEED_LIMIT { - if let Err(e) = player_tx.send(AudioEvent::SpeechChunk(data)) { - log::error!("Error sending audio chunk: {:?}", e); - gui.state = "Error on audio chunk".to_string(); - gui.display_flush().unwrap(); - } - } else { - let data_ = unsafe { - std::slice::from_raw_parts(data.as_ptr() as *const i16, data.len() / 2) - }; - recv_audio_buffer.extend_from_slice(data_); - } - } - Event::ServerEvent(ServerEvent::AudioChunki16 { data }) => { + Event::ServerEvent(ServerEvent::AudioChunki16 { data, vowel }) => { log::debug!("Received audio chunk"); if state != State::Speaking { log::debug!("Received audio chunk while not speaking"); @@ -478,10 +481,12 @@ pub async fn main_work<'d>( } if speed < SPEED_LIMIT { - if let Err(e) = player_tx.send(AudioEvent::SpeechChunki16(data)) { + if let Err(e) = player_tx.send(AudioEvent::SpeechChunki16WithVowel(data, vowel)) + { log::error!("Error sending audio chunk: {:?}", e); - gui.state = "Error on audio chunk".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Error on audio chunk".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } else { recv_audio_buffer.extend_from_slice(&data); @@ -500,16 +505,18 @@ pub async fn main_work<'d>( if recv_audio_buffer.len() > 0 { if let Err(e) = player_tx.send(AudioEvent::SpeechChunki16(recv_audio_buffer)) { log::error!("Error sending audio chunk: {:?}", e); - gui.state = "Error on audio chunk".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Error on audio chunk".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } recv_audio_buffer = Vec::with_capacity(8192); } if let Err(e) = player_tx.send(AudioEvent::EndSpeech(notify.clone())) { log::error!("Error sending audio chunk: {:?}", e); - gui.state = "Error on audio chunk".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Error on audio chunk".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } if need_compute { @@ -527,8 +534,9 @@ pub async fn main_work<'d>( Event::ServerEvent(ServerEvent::EndResponse) => { log::info!("Received request end"); state = State::Listening; - gui.state = "Listening...".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Ready".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; recv_audio_buffer.clear(); } Event::ServerEvent(ServerEvent::HelloStart) => { @@ -546,8 +554,9 @@ pub async fn main_work<'d>( if !init_hello { if let Err(_) = player_tx.send(AudioEvent::SetHello(hello_wav)) { log::error!("Error sending hello end"); - gui.state = "Error on hello end".to_string(); - gui.display_flush().unwrap(); + gui.set_state("Error on hello end".to_string()); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } hello_wav = Vec::with_capacity(1024 * 30); init_hello = true; @@ -555,15 +564,24 @@ pub async fn main_work<'d>( } Event::ServerEvent(ServerEvent::StartVideo | ServerEvent::EndVideo) => {} + Event::ServerEvent(ServerEvent::AudioChunk { .. }) => { + log::warn!("Received deprecated AudioChunk, please use AudioChunki16 instead"); + } + Event::ServerEvent(ServerEvent::AudioChunkWithVowel { .. }) => { + log::warn!( + "Received deprecated AudioChunkWithVowel, please use AudioChunki16 instead" + ); + } Event::ServerUrl(url) => { log::info!("Received ServerUrl: {}", url); if url != server.url { init_hello = false; server = Server::new(server.id, url).await?; state = State::Idle; - gui.state = "Idle".to_string(); - gui.text = format!("Server URL updated:\n{}", server.url); - gui.display_flush().unwrap(); + gui.set_state("Idle".to_string()); + gui.set_text(format!("Server URL updated:\n{}", server.url)); + gui.render_to_target(framebuffer)?; + framebuffer.flush()?; } } } diff --git a/src/audio.rs b/src/audio.rs index 3e6a3c4..51db947 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -7,6 +7,10 @@ use esp_idf_svc::sys::esp_sr; const SAMPLE_RATE: u32 = 16000; +pub static mut AFE_LINEAR_GAIN: f32 = 1.0; +pub static mut AGC_TARGET_LEVEL_DBFS: i32 = 3; +pub static mut AGC_COMPRESSION_GAIN_DB: i32 = 9; + unsafe fn afe_init() -> ( *mut esp_sr::esp_afe_sr_iface_t, *mut esp_sr::esp_afe_sr_data_t, @@ -29,7 +33,9 @@ unsafe fn afe_init() -> ( afe_config.vad_mode = esp_sr::vad_mode_t_VAD_MODE_4; afe_config.agc_init = true; - afe_config.afe_linear_gain = 2.0; + afe_config.afe_linear_gain = AFE_LINEAR_GAIN; + afe_config.agc_target_level_dbfs = AGC_TARGET_LEVEL_DBFS; + afe_config.agc_compression_gain_db = AGC_COMPRESSION_GAIN_DB; afe_config.aec_init = true; afe_config.aec_mode = esp_sr::aec_mode_t_AEC_MODE_VOIP_HIGH_PERF; @@ -131,6 +137,12 @@ impl AFE { result.vad_cache, result.vad_cache_size as usize / 2, ); + // log::info!( + // "Using vad cache len: {} {}ms", + // data_.len(), + // data_.len() * 1000 / SAMPLE_RATE as usize + // ); + // 352ms data.extend_from_slice(data_); } if data_size > 0 { @@ -224,13 +236,14 @@ pub enum AudioEvent { StopSpeech, StartSpeech, ClearSpeech, - SpeechChunk(Vec), SpeechChunki16(Vec), + SpeechChunki16WithVowel(Vec, u8), EndSpeech(Arc), VolSet(u8), } pub enum SendBufferItem { + Vowel(u8), Audio(Vec), EndSpeech(Arc), } @@ -340,14 +353,19 @@ impl SendBuffer { } } + pub fn push_vowel(&mut self, vowel: u8) { + self.cache.push_back(SendBufferItem::Vowel(vowel)); + } + pub fn push_back_end_speech(&mut self, notify: Arc) { self.cache.push_back(SendBufferItem::EndSpeech(notify)); } - pub fn get_chunk(&mut self) -> Option> { + pub fn get_chunk(&mut self) -> Option { loop { match self.cache.pop_front() { - Some(SendBufferItem::Audio(v)) => return Some(v), + Some(SendBufferItem::Vowel(v)) => return Some(SendBufferItem::Vowel(v)), + Some(SendBufferItem::Audio(v)) => return Some(SendBufferItem::Audio(v)), Some(SendBufferItem::EndSpeech(notify)) => { let _ = notify.notify_one(); continue; @@ -412,6 +430,7 @@ const CHUNK_SIZE: usize = 256; fn audio_task_run( rx: &mut tokio::sync::mpsc::UnboundedReceiver, + tx: EventTx, fn_read: &mut dyn FnMut(&mut [i16]) -> Result, fn_write: &mut dyn FnMut(&[i16]) -> Result, afe_handle: Arc, @@ -479,13 +498,15 @@ fn audio_task_run( AudioEvent::ClearSpeech => { send_buffer.clear(); } - AudioEvent::SpeechChunk(items) => { - send_buffer.push_u8(&items); + AudioEvent::SpeechChunki16WithVowel(items, vowel) => { + send_buffer.push_vowel(vowel); + send_buffer.push_i16(&items); } AudioEvent::SpeechChunki16(items) => { send_buffer.push_i16(&items); } AudioEvent::EndSpeech(sender) => { + send_buffer.push_vowel(0); send_buffer.push_back_end_speech(sender); } AudioEvent::VolSet(vol) => { @@ -503,7 +524,20 @@ fn audio_task_run( } } let play_data_ = if allow_speech { - send_buffer.get_chunk() + loop { + break match send_buffer.get_chunk() { + Some(SendBufferItem::Audio(v)) => Some(v), + Some(SendBufferItem::Vowel(v)) => { + tx.blocking_send(crate::app::Event::Vowel(v)) + .map_err(|_| anyhow::anyhow!("Failed to send vowel event"))?; + continue; + } + Some(SendBufferItem::EndSpeech(_)) => { + unreachable!("EndSpeech should be handled in get_chunk") + } + None => None, + }; + } } else { None }; @@ -615,6 +649,7 @@ impl BoxAudioWorker { let afe_handle = Arc::new(AFE::new()); let afe_handle_ = afe_handle.clone(); crate::log_heap(); + let tx_ = tx.clone(); let _afe_r = std::thread::Builder::new().stack_size(8 * 1024).spawn(|| { let r = afe_worker(afe_handle_, tx); @@ -623,7 +658,7 @@ impl BoxAudioWorker { } })?; - audio_task_run(&mut rx, &mut fn_read, &mut fn_write, afe_handle) + audio_task_run(&mut rx, tx_, &mut fn_read, &mut fn_write, afe_handle) } } @@ -706,10 +741,15 @@ impl BoardsAudioWorker { let afe_handle = Arc::new(AFE::new()); let afe_handle_ = afe_handle.clone(); - let _afe_r = std::thread::Builder::new() - .stack_size(8 * 1024) - .spawn(|| afe_worker(afe_handle_, tx))?; + let tx_ = tx.clone(); + + let _afe_r = std::thread::Builder::new().stack_size(8 * 1024).spawn(|| { + let r = afe_worker(afe_handle_, tx); + if let Err(e) = r { + log::error!("AFE worker error: {:?}", e); + } + })?; - audio_task_run(&mut rx, &mut fn_read, &mut fn_write, afe_handle) + audio_task_run(&mut rx, tx_, &mut fn_read, &mut fn_write, afe_handle) } } diff --git a/src/boards/atom_box.rs b/src/boards/atom_box.rs index 70e4212..2c3aedc 100644 --- a/src/boards/atom_box.rs +++ b/src/boards/atom_box.rs @@ -9,8 +9,7 @@ pub const AFE_AEC_OFFSET: usize = 512; pub fn afe_config(afe_config: &mut esp_idf_svc::sys::esp_sr::afe_config_t) { afe_config.agc_init = true; afe_config.agc_mode = esp_idf_svc::sys::esp_sr::afe_agc_mode_t_AFE_AGC_MODE_WEBRTC; - afe_config.afe_linear_gain = 1.0; - afe_config.ns_init = false; + afe_config.ns_init = true; } pub fn audio_init(_i2c: I2C0, _sda: Gpio48, _scl: Gpio45) { @@ -150,21 +149,474 @@ pub fn lcd_init( Ok(()) } -pub fn flush_display(color_data: &[u8], x_start: i32, y_start: i32, x_end: i32, y_end: i32) -> i32 { - debug_assert_eq!( - x_end - x_start, - DISPLAY_WIDTH as i32, - "x_end - x_start must be equal to DISPLAY_WIDTH" - ); - unsafe { - esp_idf_svc::sys::hal_driver::lcd_color_fill( - x_start as u16, - y_start as u16, - x_end as u16, - y_end as u16, - color_data.as_ptr() as _, - ); - 0 +pub mod ui { + use super::*; + + use embedded_graphics::{ + framebuffer::{buffer_size, Framebuffer}, + image::GetPixel, + pixelcolor::raw::{LittleEndian, RawU16}, + prelude::*, + primitives::{PrimitiveStyleBuilder, Rectangle}, + text::{Alignment, Text}, + Drawable, + }; + use u8g2_fonts::U8g2TextStyle; + + use crate::ui::{ColorFormat, DisplayTargetDrive, DynamicImage, ImageArea}; + + type FrameBufferChunk8x12 = Framebuffer< + ColorFormat, + RawU16, + LittleEndian, + 8, + 12, + { buffer_size::(8, 12) }, + >; + + pub type DisplayBuffer = BoxFrameBuffer; + + type FrameMask = [u8; (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12)]; + + pub struct BoxFrameBuffer { + buffers: Vec, //[FrameBufferChunk8x12; (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12)], + background_buffers: Vec, //[FrameBufferChunk8x12; (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12)], + diff_indexs: Vec, + resume_indexs: Vec, + draw_mask: FrameMask, + } + + impl Dimensions for BoxFrameBuffer { + fn bounding_box(&self) -> Rectangle { + Rectangle::new( + Point::new(0, 0), + Size::new(DISPLAY_WIDTH as u32, DISPLAY_HEIGHT as u32), + ) + } + } + + impl DrawTarget for BoxFrameBuffer { + type Color = ColorFormat; + type Error = core::convert::Infallible; + + fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> + where + I: IntoIterator>, + { + for embedded_graphics::Pixel(coord, color) in pixels { + if coord.x < 0 + || coord.x >= DISPLAY_WIDTH as i32 + || coord.y < 0 + || coord.y >= DISPLAY_HEIGHT as i32 + { + continue; + } + + let x = coord.x as usize; + let y = coord.y as usize; + + let chunk_x = x / 8; + let chunk_y = y / 12; + let chunk_index = chunk_y * (DISPLAY_WIDTH / 8) + chunk_x; + + let local_x = x % 8; + let local_y = y % 12; + + if self.draw_mask[chunk_index] == 0 { + self.diff_indexs.push(chunk_index); + self.draw_mask[chunk_index] = 1; + } + + self.buffers[chunk_index].set_pixel( + embedded_graphics::prelude::Point::new(local_x as i32, local_y as i32), + color, + ); + } + + Ok(()) + } + } + + impl GetPixel for BoxFrameBuffer { + type Color = ColorFormat; + + fn pixel(&self, point: Point) -> Option { + if point.x < 0 + || point.x >= DISPLAY_WIDTH as i32 + || point.y < 0 + || point.y >= DISPLAY_HEIGHT as i32 + { + return None; + } + + let x = point.x as usize; + let y = point.y as usize; + + let chunk_x = x / 8; + let chunk_y = y / 12; + let chunk_index = chunk_y * (DISPLAY_WIDTH / 8) + chunk_x; + + let local_x = x % 8; + let local_y = y % 12; + + self.buffers[chunk_index].pixel(embedded_graphics::prelude::Point::new( + local_x as i32, + local_y as i32, + )) + } + } + + impl DisplayTargetDrive for BoxFrameBuffer { + fn new(color: ColorFormat) -> Self { + let mut s = Self { + buffers: vec![Framebuffer::new(); (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12)], + background_buffers: vec![ + Framebuffer::new(); + (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12) + ], + diff_indexs: Vec::new(), + resume_indexs: Vec::new(), + draw_mask: [0; (DISPLAY_WIDTH / 8) * (DISPLAY_HEIGHT / 12)], + }; + + for buffer in s.buffers.iter_mut() { + buffer.clear(color).unwrap(); + } + + for buffer in s.background_buffers.iter_mut() { + buffer.clear(color).unwrap(); + } + + s + } + + fn flush(&mut self) -> anyhow::Result<()> { + unsafe { + let panel_handle = std::mem::transmute(esp_idf_svc::sys::hal_driver::panel_handle); + + for i in self.diff_indexs.iter().chain(self.resume_indexs.iter()) { + let i = *i; + let x_start = ((i % (DISPLAY_WIDTH / 8)) * 8) as i32; + let y_start = ((i / (DISPLAY_WIDTH / 8)) * 12) as i32; + let x_end = x_start + 8; + let y_end = y_start + 12; + + // DEBUG + // self.buffers[i].clear(ColorFormat::CSS_GOLD).unwrap(); + + let color_data = self.buffers[i].data(); + let size = color_data.len(); + + let lcd_dma: *mut u8 = esp_idf_svc::sys::hal_driver::lcd_dma_buffer as *mut u8; + lcd_dma.copy_from(color_data.as_ptr() as *const u8, size); + + let e = esp_idf_svc::sys::esp_lcd_panel_draw_bitmap( + panel_handle, + x_start, + y_start, + x_end, + y_end, + lcd_dma as *const _, + ); + if e != 0 { + log::warn!("flush_display error: {}", e); + } + + if self.draw_mask[i] != 0 { + self.draw_mask[i] = 0; + self.buffers[i].clone_from(&self.background_buffers[i]); + } + } + + self.diff_indexs.clear(); + self.resume_indexs.clear(); + } + Ok(()) + } + + fn fix_background(&mut self) -> anyhow::Result<()> { + self.background_buffers.clone_from(&self.buffers); + Ok(()) + } + } + + impl BoxFrameBuffer { + fn resume_chunks(&mut self, chunks: &[usize]) { + for &i in chunks { + if self.draw_mask[i] == 0 { + self.resume_indexs.push(i); + } + } + } + } + + pub struct ChatUI { + state_text: String, + state_text_updated: bool, + state_chunks: Vec, + + asr_text: String, + asr_text_updated: bool, + asr_text_chunks: Vec, + + content: String, + content_updated: bool, + content_chunks: Vec, + + avatar: DynamicImage, + avatar_updated: bool, + avatar_chunks: Vec, + } + + impl ChatUI { + pub fn new(avatar: DynamicImage) -> Self { + Self { + state_text: String::new(), + state_text_updated: false, + state_chunks: Vec::new(), + + asr_text: String::new(), + asr_text_updated: false, + asr_text_chunks: Vec::new(), + + content: String::new(), + content_updated: false, + content_chunks: Vec::new(), + + avatar: avatar, + avatar_updated: true, + avatar_chunks: Vec::new(), + } + } + + pub fn set_state(&mut self, text: String) { + if self.state_text != text { + self.state_text = text; + self.state_text_updated = true; + } + } + + pub fn set_asr(&mut self, text: String) { + if self.asr_text != text { + self.asr_text = text; + self.asr_text_updated = true; + } + } + + pub fn set_text(&mut self, text: String) { + if self.content != text { + self.content = text; + self.content_updated = true; + } + } + + pub fn set_avatar_index(&mut self, index: usize) { + self.avatar.set_index(index); + self.avatar_updated = true; + } + + pub fn clear_update_flags(&mut self) { + self.state_text_updated = false; + self.asr_text_updated = false; + self.content_updated = false; + self.avatar_updated = false; + } + + pub fn render_to_target(&mut self, target: &mut BoxFrameBuffer) -> anyhow::Result<()> { + let bounding_box = target.bounding_box(); + let (state_area_box, asr_area_box, content_area_box) = Self::layout(bounding_box); + + let mut start_i = 0; + + if self.state_text_updated { + Text::with_alignment( + &self.state_text, + state_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_LIGHT_CYAN, + ), + Alignment::Center, + ) + .draw(target)?; + target.resume_chunks(&self.state_chunks); + self.state_chunks = target.diff_indexs.clone(); + start_i = self.state_chunks.len(); + } + + if self.asr_text_updated { + Text::with_alignment( + &self.asr_text, + asr_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_WHEAT, + ), + Alignment::Center, + ) + .draw(target)?; + target.resume_chunks(&self.asr_text_chunks); + self.asr_text_chunks = target.diff_indexs[start_i..].to_vec(); + start_i += self.asr_text_chunks.len(); + } + + if self.content_updated { + let textbox_style = embedded_text::style::TextBoxStyleBuilder::new() + .height_mode(embedded_text::style::HeightMode::FitToText) + .alignment(embedded_text::alignment::HorizontalAlignment::Center) + .line_height(embedded_graphics::text::LineHeight::Percent(120)) + .paragraph_spacing(16) + .build(); + + embedded_text::TextBox::with_textbox_style( + &self.content, + content_area_box, + crate::ui::MyTextStyle( + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy16_t_gb2312, + ColorFormat::CSS_WHEAT, + ), + 3, + ), + textbox_style, + ) + .draw(target)?; + target.resume_chunks(&self.content_chunks); + self.content_chunks = target.diff_indexs[start_i..].to_vec(); + start_i += self.content_chunks.len(); + } + + if self.avatar_updated { + self.avatar.render(target)?; + target.resume_chunks(&self.avatar_chunks); + self.avatar_chunks = target.diff_indexs[start_i..].to_vec(); + } + + self.clear_update_flags(); + + Ok(()) + } + + pub fn layout(bounding_box: Rectangle) -> (Rectangle, Rectangle, Rectangle) { + let state_area_box = Rectangle::new( + bounding_box.top_left + Point::new(96, 0), + Size::new(bounding_box.size.width - 96, 32), + ); + + let asr_area_box = Rectangle::new( + bounding_box.top_left + Point::new(96, 32), + Size::new(bounding_box.size.width - 96, 64), + ); + + let content_area_box = Rectangle::new( + bounding_box.top_left + Point::new(0, 32 + 64), + Size::new(bounding_box.size.width, bounding_box.size.height - 32 - 64), + ); + + (state_area_box, asr_area_box, content_area_box) + } + } + + pub fn new_chat_ui(target: &mut BoxFrameBuffer) -> anyhow::Result> { + let bounding_box = target.bounding_box(); + let avatar_area_box = Rectangle::new(bounding_box.top_left, Size::new(96, 96)); + + let (state_area_box, asr_area_box, content_area_box) = ChatUI::::layout(bounding_box); + let state_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_DARK_BLUE) + .stroke_width(1) + .fill_color(ColorFormat::CSS_DARK_BLUE) + .build(); + + let pixels = crate::ui::get_background_pixels(target, state_area_box, state_style, 0.5); + target.draw_iter(pixels)?; + + let asr_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_DARK_CYAN) + .stroke_width(1) + .fill_color(ColorFormat::CSS_DARK_CYAN) + .build(); + + let pixels = crate::ui::get_background_pixels(target, asr_area_box, asr_style, 0.15); + target.draw_iter(pixels)?; + + let content_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_BLACK) + .stroke_width(5) + .fill_color(ColorFormat::CSS_BLACK) + .build(); + let pixels = + crate::ui::get_background_pixels(target, content_area_box, content_style, 0.25); + target.draw_iter(pixels)?; + + target.background_buffers.clone_from(&target.buffers); + + target.flush()?; + + let avatar = DynamicImage::new_from_gif(avatar_area_box, crate::ui::AVATAR_GIF)?; + Ok(ChatUI::new(avatar)) + } + + pub struct ConfiguresUI { + qr_area: ImageArea, + info: String, + } + + impl ConfiguresUI { + pub fn new( + bounding_box: Rectangle, + qr_content: &str, + info: String, + ) -> anyhow::Result { + let height = bounding_box.size.height; + let qr_area_box = Rectangle::new( + bounding_box.top_left + Point::new(0, height as i32 / 3), + Size::new(bounding_box.size.width, 2 * height / 3), + ); + + Ok(Self { + qr_area: ImageArea::new_from_qr_code(qr_area_box, qr_content)?, + info, + }) + } + + pub fn set_info(&mut self, info: String) { + self.info = info; + } + } + + impl Drawable for ConfiguresUI { + type Color = ColorFormat; + + type Output = (); + + fn draw(&self, target: &mut D) -> Result + where + D: DrawTarget, + { + let info_area_box = Rectangle::new( + target.bounding_box().top_left, + Size::new( + target.bounding_box().size.width, + target.bounding_box().size.height / 3, + ), + ); + + Text::with_alignment( + &self.info, + info_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_WHEAT, + ), + Alignment::Center, + ) + .draw(target)?; + + target.draw_iter(self.qr_area.image_data.iter().cloned())?; + + Ok(()) + } } } diff --git a/src/boards/base.rs b/src/boards/base.rs index def7b9e..8284f74 100644 --- a/src/boards/base.rs +++ b/src/boards/base.rs @@ -13,9 +13,6 @@ pub const AFE_AEC_OFFSET: usize = 256; pub fn afe_config(afe_config: &mut esp_idf_svc::sys::esp_sr::afe_config_t) { afe_config.agc_init = true; afe_config.agc_mode = esp_idf_svc::sys::esp_sr::afe_agc_mode_t_AFE_AGC_MODE_WEBRTC; - afe_config.agc_target_level_dbfs = 1; - afe_config.agc_compression_gain_db = 15; - afe_config.afe_linear_gain = 1.0; } pub fn start_audio_workers( diff --git a/src/boards/cube.rs b/src/boards/cube.rs index 76b6a54..7390f12 100644 --- a/src/boards/cube.rs +++ b/src/boards/cube.rs @@ -13,9 +13,6 @@ pub const AFE_AEC_OFFSET: usize = 256; pub fn afe_config(afe_config: &mut esp_idf_svc::sys::esp_sr::afe_config_t) { afe_config.agc_init = true; afe_config.agc_mode = esp_idf_svc::sys::esp_sr::afe_agc_mode_t_AFE_AGC_MODE_WEBRTC; - afe_config.agc_target_level_dbfs = 1; - afe_config.agc_compression_gain_db = 15; - afe_config.afe_linear_gain = 1.0; } pub fn start_audio_workers( diff --git a/src/boards/cube2.rs b/src/boards/cube2.rs index 63880b9..d74e02b 100644 --- a/src/boards/cube2.rs +++ b/src/boards/cube2.rs @@ -13,9 +13,6 @@ pub const AFE_AEC_OFFSET: usize = 256; pub fn afe_config(afe_config: &mut esp_idf_svc::sys::esp_sr::afe_config_t) { afe_config.agc_init = true; afe_config.agc_mode = esp_idf_svc::sys::esp_sr::afe_agc_mode_t_AFE_AGC_MODE_WEBRTC; - afe_config.agc_target_level_dbfs = 1; - afe_config.agc_compression_gain_db = 25; - afe_config.afe_linear_gain = 1.0; } pub fn start_audio_workers( diff --git a/src/boards/mod.rs b/src/boards/mod.rs index d252448..d9cf167 100644 --- a/src/boards/mod.rs +++ b/src/boards/mod.rs @@ -217,3 +217,383 @@ pub fn set_backlight<'d>( ledc_driver.set_duty(duty)?; Ok(()) } + +#[cfg(not(feature = "custom_ui"))] +pub mod ui { + use super::*; + + use embedded_graphics::{ + framebuffer::{buffer_size, Framebuffer}, + image::GetPixel, + pixelcolor::raw::{LittleEndian, RawU16}, + prelude::*, + primitives::{PrimitiveStyleBuilder, Rectangle}, + text::{Alignment, Text}, + Drawable, + }; + use u8g2_fonts::U8g2TextStyle; + + use crate::ui::{ColorFormat, DisplayTargetDrive, DynamicImage, ImageArea}; + + pub type DisplayBuffer = FrameBuffer; + + type Framebuffer_ = Framebuffer< + ColorFormat, + RawU16, + LittleEndian, + DISPLAY_WIDTH, + DISPLAY_HEIGHT, + { buffer_size::(DISPLAY_WIDTH, DISPLAY_HEIGHT) }, + >; + + struct PixelsTarget<'a> { + pixels: &'a mut Vec>, + bounding_box: Rectangle, + } + + impl Dimensions for PixelsTarget<'_> { + fn bounding_box(&self) -> Rectangle { + self.bounding_box + } + } + + impl DrawTarget for PixelsTarget<'_> { + type Color = ColorFormat; + type Error = core::convert::Infallible; + + fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> + where + I: IntoIterator>, + { + self.pixels.extend(pixels); + Ok(()) + } + } + + pub struct FrameBuffer { + buffers: Box, + background_buffers: Box, + } + + impl Dimensions for FrameBuffer { + fn bounding_box(&self) -> Rectangle { + Rectangle::new( + Point::new(0, 0), + Size::new(DISPLAY_WIDTH as u32, DISPLAY_HEIGHT as u32), + ) + } + } + + impl DrawTarget for FrameBuffer { + type Color = ColorFormat; + type Error = core::convert::Infallible; + + fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> + where + I: IntoIterator>, + { + self.buffers.draw_iter(pixels)?; + Ok(()) + } + } + + impl GetPixel for FrameBuffer { + type Color = ColorFormat; + + fn pixel(&self, point: Point) -> Option { + self.buffers.pixel(point) + } + } + + impl DisplayTargetDrive for FrameBuffer { + fn new(color: ColorFormat) -> Self { + let mut s = Self { + buffers: Box::new(Framebuffer::new()), + background_buffers: Box::new(Framebuffer::new()), + }; + + s.buffers.clear(color).unwrap(); + s.background_buffers.clear(color).unwrap(); + + s + } + fn flush(&mut self) -> anyhow::Result<()> { + let bounding_box = self.bounding_box(); + let x_start = bounding_box.top_left.x as i32; + let y_start = bounding_box.top_left.y as i32; + let x_end = bounding_box.top_left.x + bounding_box.size.width as i32; + let y_end = bounding_box.top_left.y + bounding_box.size.height as i32; + + let e = flush_display(self.buffers.data(), x_start, y_start, x_end, y_end); + if e != 0 { + return Err(anyhow::anyhow!("Failed to flush display: error code {}", e)); + } + + self.buffers.clone_from(&self.background_buffers); + + Ok(()) + } + + fn fix_background(&mut self) -> anyhow::Result<()> { + self.background_buffers.clone_from(&self.buffers); + Ok(()) + } + } + + const AVATAR_SIZE: u32 = 96; + pub struct ChatUI { + state_text: String, + state_text_pixels: Vec>, + + asr_text: String, + asr_text_pixels: Vec>, + + content: String, + content_pixels: Vec>, + + avatar: DynamicImage, + } + + impl ChatUI { + pub fn new(avatar: DynamicImage) -> Self { + Self { + state_text: String::new(), + state_text_pixels: Vec::with_capacity(DISPLAY_WIDTH * 32), + asr_text: String::new(), + asr_text_pixels: Vec::with_capacity(DISPLAY_WIDTH * 32), + content: String::new(), + content_pixels: Vec::with_capacity(DISPLAY_WIDTH * DISPLAY_HEIGHT / 4), + avatar: avatar, + } + } + + pub fn set_state(&mut self, text: String) { + if self.state_text != text { + self.state_text = text; + self.state_text_pixels.clear(); + } + } + + pub fn set_asr(&mut self, text: String) { + if self.asr_text != text { + self.asr_text = text; + self.asr_text_pixels.clear(); + } + } + + pub fn set_text(&mut self, text: String) { + if self.content != text { + self.content = text; + self.content_pixels.clear(); + } + } + + pub fn set_avatar_index(&mut self, index: usize) { + self.avatar.set_index(index); + } + + pub fn render_to_target(&mut self, target: &mut FrameBuffer) -> anyhow::Result<()> { + let bounding_box = target.bounding_box(); + + self.avatar.render(target)?; + + let (state_area_box, asr_area_box, content_area_box) = Self::layout(bounding_box); + + if self.state_text_pixels.is_empty() { + let mut pixel_target = PixelsTarget { + pixels: &mut self.state_text_pixels, + bounding_box, + }; + Text::with_alignment( + &self.state_text, + state_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_LIGHT_CYAN, + ), + Alignment::Center, + ) + .draw(&mut pixel_target)?; + } + target.draw_iter(self.state_text_pixels.iter().cloned())?; + + if self.asr_text_pixels.is_empty() { + let mut pixel_target = PixelsTarget { + pixels: &mut self.asr_text_pixels, + bounding_box, + }; + Text::with_alignment( + &self.asr_text, + asr_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_WHEAT, + ), + Alignment::Center, + ) + .draw(&mut pixel_target)?; + } + target.draw_iter(self.asr_text_pixels.iter().cloned())?; + + if self.content_pixels.is_empty() { + let mut pixel_target = PixelsTarget { + pixels: &mut self.content_pixels, + bounding_box, + }; + let textbox_style = embedded_text::style::TextBoxStyleBuilder::new() + .height_mode(embedded_text::style::HeightMode::FitToText) + .alignment(embedded_text::alignment::HorizontalAlignment::Center) + .line_height(embedded_graphics::text::LineHeight::Percent(120)) + .paragraph_spacing(16) + .build(); + + embedded_text::TextBox::with_textbox_style( + &self.content, + content_area_box, + crate::ui::MyTextStyle( + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy16_t_gb2312, + ColorFormat::CSS_WHEAT, + ), + 3, + ), + textbox_style, + ) + .draw(&mut pixel_target)?; + } + target.draw_iter(self.content_pixels.iter().cloned())?; + + Ok(()) + } + + pub fn layout(bounding_box: Rectangle) -> (Rectangle, Rectangle, Rectangle) { + let state_area_box = Rectangle::new( + bounding_box.top_left, + Size::new(bounding_box.size.width, 32), + ); + + let asr_area_box = Rectangle::new( + bounding_box.top_left + Point::new(0, 32), + Size::new(bounding_box.size.width, 32), + ); + + let content_height = bounding_box.size.height - 64; + + let content_area_box = Rectangle::new( + bounding_box.top_left + Point::new(0, 64), + Size::new(bounding_box.size.width, content_height), + ); + + (state_area_box, asr_area_box, content_area_box) + } + } + + pub fn new_chat_ui(target: &mut FrameBuffer) -> anyhow::Result> { + let bounding_box = target.bounding_box(); + + let header_area_box = Rectangle::new( + bounding_box.center() + - Point { + x: AVATAR_SIZE as i32 / 2, + y: AVATAR_SIZE as i32 / 2, + }, + Size::new(AVATAR_SIZE, AVATAR_SIZE), + ); + + let (state_area_box, asr_area_box, content_area_box) = ChatUI::::layout(bounding_box); + let state_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_DARK_BLUE) + .stroke_width(1) + .fill_color(ColorFormat::CSS_DARK_BLUE) + .build(); + + let pixels = crate::ui::get_background_pixels(target, state_area_box, state_style, 0.5); + target.draw_iter(pixels)?; + + let asr_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_DARK_CYAN) + .stroke_width(1) + .fill_color(ColorFormat::CSS_DARK_CYAN) + .build(); + + let pixels = crate::ui::get_background_pixels(target, asr_area_box, asr_style, 0.15); + target.draw_iter(pixels)?; + + let content_style = PrimitiveStyleBuilder::new() + .stroke_color(ColorFormat::CSS_BLACK) + .stroke_width(5) + .fill_color(ColorFormat::CSS_BLACK) + .build(); + let pixels = + crate::ui::get_background_pixels(target, content_area_box, content_style, 0.25); + target.draw_iter(pixels)?; + + target.background_buffers.clone_from(&target.buffers); + + let avatar = DynamicImage::new_from_gif(header_area_box, crate::ui::AVATAR_GIF)?; + + Ok(ChatUI::new(avatar)) + } + + pub struct ConfiguresUI { + qr_area: ImageArea, + info: String, + } + + impl ConfiguresUI { + pub fn new( + bounding_box: Rectangle, + qr_content: &str, + info: String, + ) -> anyhow::Result { + let height = bounding_box.size.height; + let qr_area_box = Rectangle::new( + bounding_box.top_left + Point::new(0, height as i32 / 3), + Size::new(bounding_box.size.width, 2 * height / 3), + ); + + Ok(Self { + qr_area: ImageArea::new_from_qr_code(qr_area_box, qr_content)?, + info, + }) + } + + pub fn set_info(&mut self, info: String) { + self.info = info; + } + } + + impl Drawable for ConfiguresUI { + type Color = ColorFormat; + + type Output = (); + + fn draw(&self, target: &mut D) -> Result + where + D: DrawTarget, + { + let info_area_box = Rectangle::new( + target.bounding_box().top_left, + Size::new( + target.bounding_box().size.width, + target.bounding_box().size.height / 3, + ), + ); + + Text::with_alignment( + &self.info, + info_area_box.center(), + U8g2TextStyle::new( + u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, + ColorFormat::CSS_WHEAT, + ), + Alignment::Center, + ) + .draw(target)?; + + target.draw_iter(self.qr_area.image_data.iter().cloned())?; + + Ok(()) + } + } +} diff --git a/src/bt.rs b/src/bt.rs index 2d0dd26..eddf59e 100644 --- a/src/bt.rs +++ b/src/bt.rs @@ -8,13 +8,16 @@ const PASS_ID: BleUuid = uuid128!("a987ab18-a940-421a-a1d7-b94ee22bccbe"); const SERVER_URL_ID: BleUuid = uuid128!("cef520a9-bcb5-4fc6-87f7-82804eee2b20"); const BACKGROUND_GIF_ID: BleUuid = uuid128!("d1f3b2c4-5e6f-4a7b-8c9d-0e1f2a3b4c5d"); const RESET_ID: BleUuid = uuid128!("f0e1d2c3-b4a5-6789-0abc-def123456789"); +const AFE_LINEAR_GAIN_ID: BleUuid = uuid128!("a1b2c3d4-e5f6-4789-0abc-def123456789"); +const AGC_TARGET_LEVEL_ID: BleUuid = uuid128!("b2c3d4e5-f6a7-4890-1bcd-ef2345678901"); +const AGC_COMPRESSION_GAIN_ID: BleUuid = uuid128!("c3d4e5f6-a7b8-4901-2cde-f34567890123"); pub fn bt( + device_id: &str, setting: Arc>, evt_tx: tokio::sync::mpsc::Sender, -) -> anyhow::Result { +) -> anyhow::Result<()> { let ble_device = esp32_nimble::BLEDevice::take(); - let ble_addr = ble_device.get_addr()?.to_string(); let ble_advertising = ble_device.get_advertising(); let server = ble_device.get_server(); @@ -103,6 +106,7 @@ pub fn bt( let setting = setting.clone(); let setting_ = setting.clone(); let setting_gif = setting.clone(); + let setting_afe = setting.clone(); // Extra clone for AFE characteristics let server_url_characteristic = service.lock().create_characteristic( SERVER_URL_ID, @@ -166,11 +170,112 @@ pub fn bt( } }); + // AFE linear gain characteristic + let setting1 = setting_afe.clone(); + let setting2 = setting_afe.clone(); + let afe_linear_gain_characteristic = service.lock().create_characteristic( + AFE_LINEAR_GAIN_ID, + NimbleProperties::READ | NimbleProperties::WRITE, + ); + afe_linear_gain_characteristic + .lock() + .on_read(move |c, _| { + log::info!("Read from AFE linear gain characteristic"); + let setting = setting1.lock().unwrap(); + let afe_line_gain_str = format!("{}", setting.0.afe_linear_gain); + c.set_value(afe_line_gain_str.as_bytes()); + }) + .on_write(move |args| { + let data = args.recv_data(); + let gain = String::from_utf8(data.to_vec()) + .map(|s| s.parse::().ok()) + .ok() + .flatten(); + + if let Some(gain) = gain { + log::info!("New AFE linear gain: {}", gain); + let mut setting = setting2.lock().unwrap(); + if let Err(e) = setting.1.set_blob("afe_linear_gain", &gain.to_le_bytes()) { + log::error!("Failed to save AFE linear gain to NVS: {:?}", e); + args.reject(); + } else { + setting.0.afe_linear_gain = gain; + } + } else { + log::error!("Failed to parse new AFE linear gain from bytes."); + args.reject(); + } + }); + + // AGC target level characteristic + let setting1 = setting_afe.clone(); + let setting2 = setting_afe.clone(); + let agc_target_level_characteristic = service.lock().create_characteristic( + AGC_TARGET_LEVEL_ID, + NimbleProperties::READ | NimbleProperties::WRITE, + ); + agc_target_level_characteristic + .lock() + .on_read(move |c, _| { + log::info!("Read from AGC target level characteristic"); + let setting = setting1.lock().unwrap(); + c.set_value(setting.0.agc_target_level_dbfs.to_le_bytes().as_ref()); + }) + .on_write(move |args| { + let data = args.recv_data(); + if data.len() == 4 { + let level = i32::from_le_bytes([data[0], data[1], data[2], data[3]]); + log::info!("New AGC target level: {}", level); + let mut setting = setting2.lock().unwrap(); + if let Err(e) = setting.1.set_i32("agc_tl_dbfs", level) { + log::error!("Failed to save AGC target level to NVS: {:?}", e); + args.reject(); + } else { + setting.0.agc_target_level_dbfs = level; + } + } else { + log::error!("Failed to parse new AGC target level from bytes."); + args.reject(); + } + }); + + // AGC compression gain characteristic + let setting1 = setting_afe.clone(); + let setting2 = setting_afe.clone(); + let agc_compression_gain_characteristic = service.lock().create_characteristic( + AGC_COMPRESSION_GAIN_ID, + NimbleProperties::READ | NimbleProperties::WRITE, + ); + agc_compression_gain_characteristic + .lock() + .on_read(move |c, _| { + log::info!("Read from AGC compression gain characteristic"); + let setting = setting1.lock().unwrap(); + c.set_value(setting.0.agc_compression_gain_db.to_le_bytes().as_ref()); + }) + .on_write(move |args| { + let data = args.recv_data(); + if data.len() == 4 { + let gain = i32::from_le_bytes([data[0], data[1], data[2], data[3]]); + log::info!("New AGC compression gain: {}", gain); + let mut setting = setting2.lock().unwrap(); + if let Err(e) = setting.1.set_i32("agc_cg_db", gain) { + log::error!("Failed to save AGC compression gain to NVS: {:?}", e); + args.reject(); + } else { + setting.0.agc_compression_gain_db = gain; + } + } else { + log::error!("Failed to parse new AGC compression gain from bytes."); + args.reject(); + } + }); + ble_advertising.lock().set_data( BLEAdvertisementData::new() - .name(&format!("EchoKit-{}", ble_addr)) + .name(&format!("EchoKit-{}", device_id)) .add_service_uuid(SERVICE_ID), )?; ble_advertising.lock().start()?; - Ok(ble_addr) + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 0ae9d0d..831e860 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,13 @@ use std::sync::{Arc, Mutex}; +use embedded_graphics::{ + prelude::{Dimensions, RgbColor}, + Drawable, +}; use esp_idf_svc::eventloop::EspSystemEventLoop; +use crate::ui::DisplayTargetDrive; + mod app; mod audio; mod bt; @@ -21,6 +27,121 @@ struct Setting { pass: String, server_url: String, background_gif: (Vec, bool), // (data, ended) + state: u8, // if 1, enter setup mode + // AFE parameters + afe_linear_gain: f32, + agc_target_level_dbfs: i32, + agc_compression_gain_db: i32, +} + +impl Setting { + fn load_from_nvs(nvs: &esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result { + let mut str_buf = [0; 128]; + + let ssid = nvs + .get_str("ssid", &mut str_buf) + .map_err(|e| log::error!("Failed to get ssid: {:?}", e)) + .ok() + .flatten() + .unwrap_or_default() + .to_string(); + + let pass = nvs + .get_str("pass", &mut str_buf) + .map_err(|e| log::error!("Failed to get pass: {:?}", e)) + .ok() + .flatten() + .unwrap_or_default() + .to_string(); + + static DEFAULT_SERVER_URL: Option<&str> = std::option_env!("DEFAULT_SERVER_URL"); + log::info!("DEFAULT_SERVER_URL: {:?}", DEFAULT_SERVER_URL); + + let server_url = nvs + .get_str("server_url", &mut str_buf) + .map_err(|e| log::error!("Failed to get server_url: {:?}", e)) + .ok() + .flatten() + .or(DEFAULT_SERVER_URL) + .unwrap_or_default() + .to_string(); + + let background_gif = if nvs.contains("background_gif")? { + let background_gif_size = nvs + .blob_len("background_gif") + .map_err(|e| log::error!("Failed to get background_gif size: {:?}", e)) + .ok() + .flatten() + .unwrap_or(1024 * 1024); + + let mut gif_buf = vec![0; background_gif_size]; + let gif_buf_ = nvs + .get_blob("background_gif", &mut gif_buf)? + .unwrap_or(ui::DEFAULT_BACKGROUND); + + if gif_buf_.len() != background_gif_size { + log::warn!( + "Background GIF size mismatch: expected {}, got {}", + background_gif_size, + gif_buf_.len() + ); + gif_buf_.to_vec() + } else { + gif_buf + } + } else { + ui::DEFAULT_BACKGROUND.to_vec() + }; + + let state = nvs.get_u8("state")?.unwrap_or(0); + + let mut afe_linear_gain_buf = [0u8; 4]; + let afe_linear_gain = nvs + .get_blob("afe_linear_gain", &mut afe_linear_gain_buf) + .map_err(|e| { + log::error!("Failed to get afe_linear_gain: {:?}", e); + }) + .ok() + .flatten() + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .unwrap_or(unsafe { audio::AFE_LINEAR_GAIN }); + + let agc_target_level_dbfs = nvs + .get_i32("agc_tl_dbfs") + .map_err(|e| { + log::error!("Failed to get agc_target_level_dbfs: {:?}", e); + }) + .ok() + .flatten() + .unwrap_or(unsafe { audio::AGC_TARGET_LEVEL_DBFS }); + + let agc_compression_gain_db = nvs + .get_i32("agc_cg_db") + .map_err(|e| { + log::error!("Failed to get agc_compression_gain_db: {:?}", e); + }) + .ok() + .flatten() + .unwrap_or(unsafe { audio::AGC_COMPRESSION_GAIN_DB }); + + Ok(Setting { + ssid, + pass, + server_url, + background_gif: (background_gif.to_vec(), false), + state, + afe_linear_gain, + agc_target_level_dbfs, + agc_compression_gain_db, + }) + } + + fn need_init(&self) -> bool { + self.state == 1 + || self.ssid.is_empty() + || self.pass.is_empty() + || self.server_url.is_empty() + } } fn main() -> anyhow::Result<()> { @@ -32,55 +153,13 @@ fn main() -> anyhow::Result<()> { let partition = esp_idf_svc::nvs::EspDefaultNvsPartition::take()?; let nvs = esp_idf_svc::nvs::EspDefaultNvs::new(partition, "setting", true)?; - let state = nvs.get_u8("state").ok().flatten().unwrap_or(0); - - let mut str_buf = [0; 128]; - let ssid = nvs - .get_str("ssid", &mut str_buf) - .map_err(|e| log::error!("Failed to get ssid: {:?}", e)) - .ok() - .flatten() - .unwrap_or_default() - .to_string(); - - let pass = nvs - .get_str("pass", &mut str_buf) - .map_err(|e| log::error!("Failed to get pass: {:?}", e)) - .ok() - .flatten() - .unwrap_or_default() - .to_string(); - - static DEFAULT_SERVER_URL: Option<&str> = std::option_env!("DEFAULT_SERVER_URL"); - log::info!("DEFAULT_SERVER_URL: {:?}", DEFAULT_SERVER_URL); - - let mut server_url = nvs - .get_str("server_url", &mut str_buf) - .map_err(|e| log::error!("Failed to get server_url: {:?}", e)) - .ok() - .flatten() - .or(DEFAULT_SERVER_URL) - .unwrap_or_default() - .to_string(); - - // 1MB buffer for GIF - let has_bg = nvs.contains("background_gif").unwrap_or(false); - let mut gif_buf = if has_bg { - vec![0; 1024 * 1024] - } else { - Vec::new() - }; - - let background_gif = nvs - .get_blob("background_gif", &mut gif_buf)? - .unwrap_or(ui::DEFAULT_BACKGROUND); - - log::info!("SSID: {:?}", ssid); - log::info!("PASS: {:?}", pass); - log::info!("Server URL: {:?}", server_url); - + let mut setting = Setting::load_from_nvs(&nvs)?; nvs.set_u8("state", 0).unwrap(); + log::info!("SSID: {:?}", setting.ssid); + log::info!("PASS: {:?}", setting.pass); + log::info!("Server URL: {:?}", setting.server_url); + log_heap(); let (evt_tx, mut evt_rx) = tokio::sync::mpsc::channel(64); @@ -88,66 +167,73 @@ fn main() -> anyhow::Result<()> { crate::start_hal!(peripherals, evt_tx); - let _ = ui::backgroud(&background_gif, boards::flush_display); + let mut framebuffer = Box::new(boards::ui::DisplayBuffer::new(ui::ColorFormat::WHITE)); + framebuffer.flush()?; + + crate::ui::display_gif(framebuffer.as_mut(), &setting.background_gif.0).unwrap(); // Configures the button let mut button = esp_idf_svc::hal::gpio::PinDriver::input(peripherals.pins.gpio0)?; button.set_pull(esp_idf_svc::hal::gpio::Pull::Up)?; - button.set_interrupt_type(esp_idf_svc::hal::gpio::InterruptType::PosEdge)?; + button.set_interrupt_type(esp_idf_svc::hal::gpio::InterruptType::AnyEdge)?; let b = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; - let mut gui = ui::UI::new(None, boards::flush_display).unwrap(); - log_heap(); + let mut chat_ui = boards::ui::new_chat_ui::<6>(framebuffer.as_mut())?; + #[cfg(feature = "extra_server")] { - gui.state = "Initializing...".to_string(); - gui.text = "Loading Server URL...".to_string(); - gui.display_flush().unwrap(); + chat_ui.set_state("Initializing...".to_string()); + chat_ui.set_text("Loading Server URL...".to_string()); + + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; while let Some(event) = evt_rx.blocking_recv() { if let app::Event::ServerUrl(url) = event { log::info!("Received ServerUrl event: {}", url); if !url.is_empty() { - server_url = url; + setting.server_url = url; } break; } } std::thread::sleep(std::time::Duration::from_millis(500)); - gui.text = format!("Server URL: {}\nContinuing...", server_url); - gui.display_flush().unwrap(); + chat_ui.set_text(format!("Server URL: {}\nContinuing...", setting.server_url)); + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; std::thread::sleep(std::time::Duration::from_millis(2000)); } - let need_init = { - button.is_low() || state == 1 || ssid.is_empty() || pass.is_empty() || server_url.is_empty() - }; + let need_init = button.is_low() || setting.need_init(); + if need_init { - gif_buf.clear(); - let setting = Arc::new(Mutex::new(( - Setting { - ssid, - pass, - server_url, - background_gif: (gif_buf, false), // 1MB - }, - nvs, - ))); - - let ble_addr = bt::bt(setting.clone(), evt_tx).unwrap(); + // let mut config_ui = ui::new_config_ui(start_ui, "https://echokit.dev/setup/")?; + + let esp_wifi = esp_idf_svc::wifi::EspWifi::new(peripherals.modem, sysloop, None)?; + let mac = esp_wifi.sta_netif().get_mac()?; + let dev_id = format!( + "{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] + ); + + setting.background_gif.0.clear(); + let setting = Arc::new(Mutex::new((setting, nvs))); + + bt::bt(&dev_id, setting.clone(), evt_tx).unwrap(); log_heap(); let version = env!("CARGO_PKG_VERSION"); - gui.state = "Please setup device by bt".to_string(); - gui.text = format!("Goto https://echokit.dev/setup/ to set up the device.\nDevice Name: EchoKit-{}\nVersion: {}", ble_addr, version); - gui.display_qrcode("https://echokit.dev/setup/").unwrap(); + let mut config_ui = boards::ui::ConfiguresUI::new(framebuffer.bounding_box(), "https://echokit.dev/setup/", format!("Goto https://echokit.dev/setup/ to set up the device.\nDevice Name: EchoKit-{}\nVersion: {}", dev_id, version)).unwrap(); + + config_ui.draw(framebuffer.as_mut())?; + framebuffer.flush()?; #[cfg(feature = "boards")] { @@ -179,17 +265,19 @@ fn main() -> anyhow::Result<()> { { let mut setting = setting.lock().unwrap(); if setting.0.background_gif.1 { - gui.text = "Testing background GIF...".to_string(); - gui.display_flush().unwrap(); + config_ui.set_info("Testing background GIF...".to_string()); + config_ui.draw(framebuffer.as_mut())?; + framebuffer.flush()?; let mut new_gif = Vec::new(); std::mem::swap(&mut setting.0.background_gif.0, &mut new_gif); - let _ = ui::backgroud(&new_gif, boards::flush_display); + crate::ui::display_gif(framebuffer.as_mut(), &new_gif).unwrap(); log::info!("Background GIF set from NVS"); - gui.text = "Background GIF set OK".to_string(); - gui.display_flush().unwrap(); + config_ui.set_info("Background GIF set OK".to_string()); + config_ui.draw(framebuffer.as_mut())?; + framebuffer.flush()?; setting .1 @@ -203,15 +291,28 @@ fn main() -> anyhow::Result<()> { unsafe { esp_idf_svc::sys::esp_restart() } } - gui.state = "Connecting to wifi...".to_string(); - gui.text.clear(); - gui.display_flush().unwrap(); + unsafe { + audio::AFE_LINEAR_GAIN = setting.afe_linear_gain; + audio::AGC_TARGET_LEVEL_DBFS = setting.agc_target_level_dbfs; + audio::AGC_COMPRESSION_GAIN_DB = setting.agc_compression_gain_db; + } + + chat_ui.set_state("Connecting to wifi...".to_string()); + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; - let _wifi = network::wifi(&ssid, &pass, peripherals.modem, sysloop.clone()); + let _wifi = network::wifi( + &setting.ssid, + &setting.pass, + peripherals.modem, + sysloop.clone(), + ); if _wifi.is_err() { - gui.state = "Failed to connect to wifi".to_string(); - gui.text = "Press K0 to open settings".to_string(); - gui.display_flush().unwrap(); + chat_ui.set_state("Failed to connect to wifi".to_string()); + chat_ui.set_text("Press K0 to open settings".to_string()); + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; + b.block_on(button.wait_for_falling_edge()).unwrap(); nvs.set_u8("state", 1).unwrap(); unsafe { esp_idf_svc::sys::esp_restart() } @@ -226,18 +327,23 @@ fn main() -> anyhow::Result<()> { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); - gui.state = "Connecting to server...".to_string(); - gui.text.clear(); - gui.display_flush().unwrap(); + chat_ui.set_state("Connecting to server...".to_string()); + chat_ui.set_text("".to_string()); + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; log_heap(); - gui.state = "Failed to connect to server".to_string(); - gui.text = format!("Please check your server URL: {server_url}\nPress K0 to open settings"); - let server = b.block_on(ws::Server::new(dev_id, server_url)); + chat_ui.set_state("Failed to connect to server".to_string()); + chat_ui.set_text(format!( + "Please check your server URL: {}\nPress K0 to open settings", + setting.server_url + )); + let server = b.block_on(ws::Server::new(dev_id, setting.server_url)); if server.is_err() { log::info!("Failed to connect to server: {:?}", server.err()); - gui.display_flush().unwrap(); + chat_ui.render_to_target(framebuffer.as_mut())?; + framebuffer.flush()?; b.block_on(button.wait_for_falling_edge()).unwrap(); nvs.set_u8("state", 1).unwrap(); unsafe { esp_idf_svc::sys::esp_restart() } @@ -247,7 +353,7 @@ fn main() -> anyhow::Result<()> { crate::start_audio_workers!(peripherals, rx1, evt_tx.clone(), &b); - let ws_task = app::main_work(server, tx1, evt_rx, Some(background_gif)); + let ws_task = app::main_work(server, tx1, evt_rx, &mut framebuffer, &mut chat_ui); b.spawn(async move { loop { diff --git a/src/protocol.rs b/src/protocol.rs index d249e8a..4c24d3b 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -11,7 +11,8 @@ pub enum ServerEvent { Action { action: String }, StartAudio { text: String }, AudioChunk { data: Vec }, - AudioChunki16 { data: Vec }, + AudioChunkWithVowel { data: Vec, vowel: u8 }, + AudioChunki16 { data: Vec, vowel: u8 }, EndAudio, StartVideo, EndVideo, diff --git a/src/ui.rs b/src/ui.rs index 71b1664..1800911 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -1,68 +1,23 @@ use embedded_graphics::{ - framebuffer::{buffer_size, Framebuffer}, image::GetPixel, - pixelcolor::{ - raw::{LittleEndian, RawU16}, - Rgb565, - }, + pixelcolor::Rgb565, prelude::*, - primitives::{PrimitiveStyleBuilder, Rectangle}, - text::{ - renderer::{CharacterStyle, TextRenderer}, - Alignment, Text, - }, + primitives::{PrimitiveStyle, Rectangle}, + text::renderer::{CharacterStyle, TextRenderer}, }; -use embedded_text::TextBox; use u8g2_fonts::U8g2TextStyle; pub type ColorFormat = Rgb565; pub const DEFAULT_BACKGROUND: &[u8] = include_bytes!("../assets/echokit.gif"); +// pub const DEFAULT_BACKGROUND: &[u8] = include_bytes!("../assets/ht.gif"); -use crate::boards::{DISPLAY_HEIGHT, DISPLAY_WIDTH}; - -pub type FlushDisplayFn = - fn(color_data: &[u8], x_start: i32, y_start: i32, x_end: i32, y_end: i32) -> i32; - -pub fn backgroud(gif: &[u8], f: FlushDisplayFn) -> Result<(), std::convert::Infallible> { - let image = tinygif::Gif::::from_slice(gif).unwrap(); - - // Create a new framebuffer - let mut display = Box::new(Framebuffer::< - ColorFormat, - _, - LittleEndian, - DISPLAY_WIDTH, - DISPLAY_HEIGHT, - { buffer_size::(DISPLAY_WIDTH, DISPLAY_HEIGHT) }, - >::new()); - - display.clear(ColorFormat::WHITE)?; - - for frame in image.frames() { - if !frame.is_transparent { - display.clear(ColorFormat::WHITE)?; - } - frame.draw(display.as_mut())?; - f( - display.data(), - 0, - 0, - DISPLAY_WIDTH as _, - DISPLAY_HEIGHT as _, - ); - let delay_ms = frame.delay_centis * 10; - std::thread::sleep(std::time::Duration::from_millis(delay_ms as u64)); - } - - Ok(()) -} - -const ALPHA: f32 = 0.5; +// pub const LM_PNG: &[u8] = include_bytes!("../assets/lm_320x240.png"); +pub const AVATAR_GIF: &[u8] = include_bytes!("../assets/avatar.gif"); // TextRenderer + CharacterStyle #[derive(Debug, Clone)] -struct MyTextStyle(U8g2TextStyle, i32); +pub struct MyTextStyle(pub U8g2TextStyle, pub i32); impl TextRenderer for MyTextStyle { type Color = ColorFormat; @@ -136,31 +91,106 @@ impl CharacterStyle for MyTextStyle { } } -pub struct UI { - pub state: String, - state_area: Rectangle, - state_background: Vec>, - pub text: String, - text_area: Rectangle, - text_background: Vec>, - - display: Box< - Framebuffer< - ColorFormat, - RawU16, - LittleEndian, - DISPLAY_WIDTH, - DISPLAY_HEIGHT, - { buffer_size::(DISPLAY_WIDTH, DISPLAY_HEIGHT) }, - >, - >, - - flush_fn: FlushDisplayFn, +pub trait DisplayTargetDrive: + DrawTarget + GetPixel +{ + fn new(color: ColorFormat) -> Self; + fn flush(&mut self) -> anyhow::Result<()>; + fn fix_background(&mut self) -> anyhow::Result<()>; } -const COLOR_WIDTH: u32 = 2; +pub fn display_gif( + display_target: &mut D, + gif: &[u8], +) -> anyhow::Result<()> { + use image::AnimationDecoder; + let img_gif = image::codecs::gif::GifDecoder::new(std::io::Cursor::new(gif))?; + + let mut frames = img_gif.into_frames(); + let mut ff = frames.next(); + + loop { + if ff.is_none() { + break; + } -fn alpha_mix(source: ColorFormat, target: ColorFormat, alpha: f32) -> ColorFormat { + let frame = ff.unwrap()?; + + let delay = frame.delay(); + + let img = frame.into_buffer(); + let pixels = img.enumerate_pixels().map(|(x, y, p)| { + let (x, y) = if p[3] == 0 { + (-1, -1) + } else { + (x as i32, y as i32) + }; + + Pixel( + Point { x, y }, + ColorFormat::new( + p[0] / (u8::MAX / ColorFormat::MAX_R), + p[1] / (u8::MAX / ColorFormat::MAX_G), + p[2] / (u8::MAX / ColorFormat::MAX_B), + ), + ) + }); + + display_target + .draw_iter(pixels) + .map_err(|_| anyhow::anyhow!("Failed to draw GIF frame"))?; + + let now = std::time::Instant::now(); + ff = frames.next(); + if ff.is_none() { + display_target.fix_background()?; + } + + display_target.flush()?; + + let delay = std::time::Duration::from(delay); + + std::thread::sleep(std::time::Instant::now() - (now + delay)); + } + + Ok(()) +} + +pub fn display_png( + display_target: &mut D, + png: &[u8], + timeout: std::time::Duration, +) -> anyhow::Result<()> { + let img_reader = + image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png); + + let img = img_reader.decode().unwrap().to_rgb8(); + + let p = img.enumerate_pixels().map(|(x, y, p)| { + Pixel( + Point::new(x as i32, y as i32), + ColorFormat::new( + p[0] / (u8::MAX / ColorFormat::MAX_R), + p[1] / (u8::MAX / ColorFormat::MAX_G), + p[2] / (u8::MAX / ColorFormat::MAX_B), + ), + ) + }); + + display_target + .draw_iter(p) + .map_err(|_| anyhow::anyhow!("Failed to draw PNG image"))?; + + display_target.fix_background()?; + + display_target.flush()?; + + std::thread::sleep(timeout); + + Ok(()) +} + +pub fn alpha_mix(source: ColorFormat, target: ColorFormat, alpha: f32) -> ColorFormat { ColorFormat::new( ((1. - alpha) * source.r() as f32 + alpha * target.r() as f32) as u8, ((1. - alpha) * source.g() as f32 + alpha * target.g() as f32) as u8, @@ -168,35 +198,6 @@ fn alpha_mix(source: ColorFormat, target: ColorFormat, alpha: f32) -> ColorForma ) } -fn flush_area( - data: &[u8], - size: Size, - area: Rectangle, - flash_fn: FlushDisplayFn, -) -> i32 { - let start_y = area.top_left.y as u32; - let end_y = start_y + area.size.height; - - let start_index = start_y * size.width * COLOR_WIDTH; - let data_len = area.size.height * size.width * COLOR_WIDTH; - if let Some(area_data) = data.get(start_index as usize..(start_index + data_len) as usize) { - flash_fn( - area_data, - 0, - start_y as i32, - size.width as i32, - end_y as i32, - ) - } else { - log::warn!("flush_area error: data out of bounds"); - log::warn!( - "start_index: {start_index}, area_len: {data_len}, data_len: {}", - data.len() - ); - -1 - } -} - #[derive(Debug, Clone, Copy)] pub struct QrPixel(ColorFormat); @@ -251,225 +252,159 @@ impl qrcode::render::Canvas for QrCanvas { } } -impl UI { - pub fn new(backgroud_gif: Option<&[u8]>, flush_fn: FlushDisplayFn) -> anyhow::Result { - let mut display = Box::new(Framebuffer::< - ColorFormat, - _, - LittleEndian, - DISPLAY_WIDTH, - DISPLAY_HEIGHT, - { buffer_size::(DISPLAY_WIDTH, DISPLAY_HEIGHT) }, - >::new()); - - display.clear(ColorFormat::WHITE).unwrap(); - - let state_area = Rectangle::new( - display.bounding_box().top_left + Point::new(0, 0), - Size::new(DISPLAY_WIDTH as u32, 32), - ); - let text_area = Rectangle::new( - display.bounding_box().top_left + Point::new(0, 32), - Size::new(DISPLAY_WIDTH as u32, DISPLAY_HEIGHT as u32 - 32), - ); - - if let Some(gif) = backgroud_gif { - let image = tinygif::Gif::::from_slice(gif) - .map_err(|e| anyhow::anyhow!("Failed to parse GIF: {:?}", e))?; - for frame in image.frames() { - frame.draw(display.as_mut()).unwrap(); +pub fn get_background_pixels>( + display: &T, + area: Rectangle, + background_style: PrimitiveStyle, + alpha: f32, +) -> Vec> { + area.into_styled(background_style) + .pixels() + .map(|p| { + if let Some(color) = display.pixel(p.0) { + Pixel(p.0, alpha_mix(color, p.1, alpha)) + } else { + p } - } - - let img = display.as_image(); + }) + .collect() +} - let state_pixels: Vec> = state_area - .into_styled( - PrimitiveStyleBuilder::new() - .stroke_color(ColorFormat::CSS_DARK_BLUE) - .stroke_width(1) - .fill_color(ColorFormat::CSS_DARK_BLUE) - .build(), - ) - .pixels() - .map(|p| { - if let Some(color) = img.pixel(p.0) { - Pixel(p.0, alpha_mix(color, p.1, ALPHA)) - } else { - p - } - }) - .collect(); +pub struct ImageArea { + pub image_data: Vec>, +} - let box_pixels: Vec> = text_area - .into_styled( - PrimitiveStyleBuilder::new() - .stroke_color(ColorFormat::CSS_BLACK) - .stroke_width(5) - .fill_color(ColorFormat::CSS_BLACK) - .build(), - ) - .pixels() - .map(|p| { - if let Some(color) = img.pixel(p.0) { - Pixel(p.0, alpha_mix(color, p.1, ALPHA)) - } else { - p - } - }) - .collect(); +impl ImageArea { + pub fn new_from_color(area: Rectangle, color: ColorFormat) -> anyhow::Result { + let pixels: Vec> = + area.points().map(|point| Pixel(point, color)).collect(); - Ok(Self { - state: String::new(), - state_background: state_pixels, - text: String::new(), - text_background: box_pixels, - display, - state_area, - text_area, - flush_fn, - }) + Ok(Self { image_data: pixels }) } - pub fn display_flush(&mut self) -> anyhow::Result<()> { - self.state_background - .iter() - .cloned() - .draw(self.display.as_mut())?; - self.text_background - .iter() - .cloned() - .draw(self.display.as_mut())?; - - Text::with_alignment( - &self.state, - self.state_area.center(), - U8g2TextStyle::new( - u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, - ColorFormat::CSS_LIGHT_CYAN, - ), - Alignment::Center, - ) - .draw(self.display.as_mut())?; - - let textbox_style = embedded_text::style::TextBoxStyleBuilder::new() - .height_mode(embedded_text::style::HeightMode::FitToText) - .alignment(embedded_text::alignment::HorizontalAlignment::Center) - .line_height(embedded_graphics::text::LineHeight::Percent(120)) - .paragraph_spacing(16) - .build(); - let text_box = TextBox::with_textbox_style( - &self.text, - self.text_area, - MyTextStyle( - U8g2TextStyle::new( - u8g2_fonts::fonts::u8g2_font_wqy16_t_gb2312, - ColorFormat::CSS_WHEAT, - ), - 3, - ), - textbox_style, + pub fn new_from_png(area: Rectangle, png_data: &[u8]) -> anyhow::Result { + let ht = image::ImageReader::with_format( + std::io::Cursor::new(png_data), + image::ImageFormat::Png, ); - text_box.draw(self.display.as_mut())?; - - for i in 0..5 { - let e = flush_area::( - self.display.data(), - self.display.size(), - Rectangle::new( - self.state_area.top_left, - Size::new( - self.text_area.size.width, - self.text_area.size.height + self.state_area.size.height, - ), - ), - self.flush_fn, - ); - if e == 0 { - break; + let img = ht.decode().unwrap().to_rgb8(); + + let mut pixels = Vec::with_capacity((area.size.width * area.size.height) as usize); + + for (x, y, p) in img.enumerate_pixels() { + if x >= area.size.width || y >= area.size.height { + continue; } - log::warn!("flush_display error: {} retry {i}", e); + pixels.push(Pixel( + Point::new(area.top_left.x + x as i32, area.top_left.y + y as i32), + ColorFormat::new( + p[0] / (u8::MAX / ColorFormat::MAX_R), + p[1] / (u8::MAX / ColorFormat::MAX_G), + p[2] / (u8::MAX / ColorFormat::MAX_B), + ), + )); } - Ok(()) + + Ok(Self { image_data: pixels }) } - pub fn display_qrcode(&mut self, qr_context: &str) -> anyhow::Result<()> { - let code = qrcode::QrCode::new(qr_context).unwrap(); + pub fn new_from_qr_code(area: Rectangle, qr_content: &str) -> anyhow::Result { + let code = qrcode::QrCode::new(qr_content).unwrap(); let ((width, height), code_pixel) = code .render::() .quiet_zone(true) .module_dimensions(4, 4) .build(); - self.state_background - .iter() - .cloned() - .draw(self.display.as_mut())?; - self.text_background - .iter() - .cloned() - .draw(self.display.as_mut())?; - - self.display - .cropped(&Rectangle::new( - self.text_area.top_left - + Point::new( - ((self.text_area.size.width - width) / 2) as i32, - (self.text_area.size.height - height) as i32, + let offset_x = if area.size.width > width { + (area.size.width - width) / 2 + } else { + 0 + }; + let offset_y = if area.size.height > height { + (area.size.height - height) / 2 + } else { + 0 + }; + + let pixels: Vec> = code_pixel + .into_iter() + .map(|p| { + Pixel( + Point::new( + p.0.x + area.top_left.x + offset_x as i32, + p.0.y + area.top_left.y + offset_y as i32, ), - Size::new(width, height), - )) - .draw_iter(code_pixel)?; - - Text::with_alignment( - &self.state, - self.state_area.center(), - U8g2TextStyle::new( - u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, - ColorFormat::CSS_LIGHT_CYAN, - ), - Alignment::Center, - ) - .draw(self.display.as_mut())?; + p.1, + ) + }) + .collect(); - let textbox_style = embedded_text::style::TextBoxStyleBuilder::new() - .height_mode(embedded_text::style::HeightMode::FitToText) - .alignment(embedded_text::alignment::HorizontalAlignment::Center) - .line_height(embedded_graphics::text::LineHeight::Percent(120)) - .paragraph_spacing(12) - .build(); - let text_box = TextBox::with_textbox_style( - &self.text, - self.text_area, - MyTextStyle( - U8g2TextStyle::new( - u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312a, - ColorFormat::CSS_WHEAT, - ), - 3, - ), - textbox_style, - ); - text_box.draw(self.display.as_mut())?; - - for i in 0..5 { - let e = flush_area::( - self.display.data(), - self.display.size(), - Rectangle::new( - self.state_area.top_left, - Size::new( - self.text_area.size.width, - self.text_area.size.height + self.state_area.size.height, + Ok(Self { + // area: Rectangle { + // top_left: area.top_left + Point::new(offset_x as i32, offset_y as i32), + // size: Size::new(width, height), + // }, + image_data: pixels, + }) + } +} + +pub struct DynamicImage { + pub display_index: usize, + pub image_data: Vec>>, +} + +impl DynamicImage { + pub fn new_from_gif(area: Rectangle, gif_data: &[u8]) -> anyhow::Result { + use image::AnimationDecoder; + let img_gif = image::codecs::gif::GifDecoder::new(std::io::Cursor::new(gif_data))?; + + let frames = img_gif.into_frames(); + let mut image_data: Vec>> = Vec::new(); + for ff in frames.take(N) { + let frame = ff?; + + let img = frame.into_buffer(); + let mut pixels = Vec::with_capacity((area.size.width * area.size.height) as usize); + + for (x, y, p) in img.enumerate_pixels() { + if x >= area.size.width || y >= area.size.height || p[3] == 0 { + continue; + } + pixels.push(Pixel( + Point::new(area.top_left.x + x as i32, area.top_left.y + y as i32), + ColorFormat::new( + p[0] / (u8::MAX / ColorFormat::MAX_R), + p[1] / (u8::MAX / ColorFormat::MAX_G), + p[2] / (u8::MAX / ColorFormat::MAX_B), ), - ), - self.flush_fn, - ); - if e == 0 { - break; + )); } - log::warn!("flush_display error: {} retry {i}", e); + + image_data.push(pixels); + } + + Ok(Self { + display_index: 0, + image_data, + }) + } + + pub fn set_index(&mut self, index: usize) { + let new_idx = index % N; + if new_idx == self.display_index { + self.display_index = 0; + } else { + self.display_index = index % N; } + } + + pub fn render>( + &self, + display: &mut D, + ) -> Result<(), D::Error> { + display.draw_iter(self.image_data[self.display_index].iter().cloned())?; Ok(()) } } diff --git a/src/ws.rs b/src/ws.rs index 922d8bf..06c9f00 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -61,7 +61,28 @@ async fn ws_manager( .iter() .cloned() .collect::>(); - let server_event = ServerEvent::AudioChunki16 { data }; + let server_event = + ServerEvent::AudioChunki16 { data, vowel: 0 }; + tx.send(server_event).await.map_err(|_| { + anyhow::anyhow!( + "Failed to send opus audio chunk to channel", + ) + })?; + } + Err(e) => { + log::warn!("Failed to decode opus audio chunk: {}", e); + continue; + } + } + } + Ok(ServerEvent::AudioChunkWithVowel { data, vowel }) => { + match opus_decoder.decode(&data, &mut opus_buffer, false) { + Ok(decoded_samples) => { + let data = opus_buffer[..decoded_samples] + .iter() + .cloned() + .collect::>(); + let server_event = ServerEvent::AudioChunki16 { data, vowel }; tx.send(server_event).await.map_err(|_| { anyhow::anyhow!( "Failed to send opus audio chunk to channel", @@ -171,9 +192,9 @@ pub struct Server { impl Server { pub async fn new(id: String, url: String) -> anyhow::Result { let u = if url.ends_with("/") { - format!("{}{}?opus=true", url, id) + format!("{}{}?opus=true&vowel=true", url, id) } else { - format!("{}/{}?opus=true", url, id) + format!("{}/{}?opus=true&vowel=true", url, id) }; let (ws, _resp) = tokio_websockets::ClientBuilder::new() @@ -205,9 +226,15 @@ impl Server { pub async fn reconnect(&mut self) -> anyhow::Result<()> { let u = if self.url.ends_with("/") { - format!("{}{}?reconnect=true&opus=true", self.url, self.id) + format!( + "{}{}?reconnect=true&opus=true&vowel=true", + self.url, self.id + ) } else { - format!("{}/{}?reconnect=true&opus=true", self.url, self.id) + format!( + "{}/{}?reconnect=true&opus=true&vowel=true", + self.url, self.id + ) }; let (ws, _resp) = tokio_websockets::ClientBuilder::new()