Parameter server issue with ESP32c6

[Geibinger]

Parameter server issue with ESP32c6

• Hardware description: ESP32-C6-DevKitM-1
• RTOS: FreeRTOS
• Installation type: micro_ros_espidf_components
• Version or commit hash: 031fac2

Steps to reproduce the issue

After updating the Parameter example to the new API (see this pull request), I tried it on the typical NodeMCU-ESP32. It worked without issues. However, when trying the same code on the RISC-V based ESP32-C6-DevKitM-1, I get the problem that can be seen in the "Actual behavior" output, meaning that the executor spin results in a Guru Meditation Error.

Here is the code used:

#include <stdio.h>
#include <string.h>
#include <unistd.h>

#include "esp_log.h"
#include "esp_system.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"

#include <rcl/error_handling.h>
#include <rcl/rcl.h>
#include <rclc/executor.h>
#include <rclc/rclc.h>
#include <rclc_parameter/rclc_parameter.h>
#include <uros_network_interfaces.h>
#include <uxr/client/config.h>

#include <rmw_microros/rmw_microros.h>
#include <uxr/client/config.h>

#define RCCHECK(fn) { rcl_ret_t temp_rc = fn; if((temp_rc != RCL_RET_OK)){printf("Failed status on line %d: %d. Aborting.\n",__LINE__,(int)temp_rc);vTaskDelete(NULL);}}
#define RCSOFTCHECK(fn) { rcl_ret_t temp_rc = fn; if((temp_rc != RCL_RET_OK)){printf("Failed status on line %d: %d. Continuing.\n",__LINE__,(int)temp_rc);}}

rclc_parameter_server_t param_server;

void timer_callback(rcl_timer_t * timer, int64_t last_call_time)
{
    (void) timer;
    (void) last_call_time;

    int64_t value;
    rclc_parameter_get_int(&param_server, "param2", &value);
    value++;
    rclc_parameter_set_int(&param_server, "param2", value);
}

bool on_parameter_changed(const Parameter *old_param, const Parameter *new_param, void *context)
    {
    printf("Parameter %s modified.", new_param->name.data);
    switch (new_param->value.type)
    {
    case RCLC_PARAMETER_BOOL:
        printf(" New value: %d (bool)", new_param->value.bool_value);
        break;
    case RCLC_PARAMETER_INT:
        printf(" New value: %lld (int)", new_param->value.integer_value);
        break;
    case RCLC_PARAMETER_DOUBLE:
        printf(" New value: %f (double)", new_param->value.double_value);
        break;
    default:
        break;
    }
    printf("\n");
    return true;
}

void micro_ros_task(void * arg)
{

 	rcl_ret_t rc;
	rcl_allocator_t allocator = rcl_get_default_allocator();
	rclc_support_t support;

	rcl_init_options_t init_options = rcl_get_zero_initialized_init_options();
	RCCHECK(rcl_init_options_init(&init_options, allocator));
	rmw_init_options_t* rmw_options = rcl_init_options_get_rmw_init_options(&init_options);

	// Static Agent IP and port can be used instead of autodisvery.
	RCCHECK(rmw_uros_options_set_udp_address(CONFIG_MICRO_ROS_AGENT_IP, CONFIG_MICRO_ROS_AGENT_PORT, rmw_options));

	// create init_options
	RCCHECK(rclc_support_init_with_options(&support, 0, NULL, &init_options, &allocator));

    // Create node
    rcl_node_t node;
    rclc_node_init_default(&node, "esp32_param_node", "", &support);

    // Create parameter service
    rclc_parameter_server_init_default(&param_server, &node);

    // create timer,
    rcl_timer_t timer;
    rclc_timer_init_default2(
        &timer,
        &support,
        RCL_MS_TO_NS(1000),
        timer_callback,
        true);

    // Create executor
    rclc_executor_t executor;
    rclc_executor_init(&executor, &support.context, RCLC_EXECUTOR_PARAMETER_SERVER_HANDLES + 1, &allocator);
    rclc_executor_add_parameter_server(&executor, &param_server, on_parameter_changed);
    rclc_executor_add_timer(&executor, &timer);

    // Add parameters
    rclc_add_parameter(&param_server, "param1", RCLC_PARAMETER_BOOL);
    rclc_add_parameter(&param_server, "param2", RCLC_PARAMETER_INT);
    rclc_add_parameter(&param_server, "param3", RCLC_PARAMETER_DOUBLE);

    rclc_parameter_set_bool(&param_server, "param1", false);
    rclc_parameter_set_int(&param_server, "param2", 10);
    rclc_parameter_set_double(&param_server, "param3", 0.01);

    bool param1;
    int64_t param2;
    double param3;

    rclc_parameter_get_bool(&param_server, "param1", &param1);
    rclc_parameter_get_int(&param_server, "param2", &param2);
    rclc_parameter_get_double(&param_server, "param3", &param3);

	while(1){
		rclc_executor_spin_some(&executor, RCL_MS_TO_NS(100));
		usleep(100000);
	}

    // clean up
    rc = rclc_executor_fini(&executor);
    rc += rclc_parameter_server_fini(&param_server, &node);
    rc += rcl_node_fini(&node);

    if (rc != RCL_RET_OK) {
        printf("Error while cleaning up!\n");
    }

  	vTaskDelete(NULL);
}

void app_main(void)
{
#ifdef UCLIENT_PROFILE_UDP
    // Start the networking if required
    ESP_ERROR_CHECK(uros_network_interface_initialize());
#endif  // UCLIENT_PROFILE_UDP

    //pin micro-ros task in APP_CPU to make PRO_CPU to deal with wifi:
    xTaskCreate(micro_ros_task,
            "uros_task",
            CONFIG_MICRO_ROS_APP_STACK,
            NULL,
            CONFIG_MICRO_ROS_APP_TASK_PRIO,
            NULL);
}

with this config:

{
    "names": {
        "rmw_microxrcedds": {
            "cmake-args": [
                "-DRMW_UXRCE_MAX_NODES=1",
                "-DRMW_UXRCE_MAX_PUBLISHERS=10",
                "-DRMW_UXRCE_MAX_SUBSCRIPTIONS=1",
                "-DRMW_UXRCE_MAX_SERVICES=10",
                "-DRMW_UXRCE_MAX_CLIENTS=1",
                "-DRMW_UXRCE_MAX_HISTORY=8"
            ]
        }
    }
}

Expected behavior

Working parameter server, like when set up with ESP32 Node MCU.

Actual behavior

Rebooting...
ESP-ROM:esp32c6-20220919
Build:Sep 19 2022
rst:0xc (SW_CPU),boot:0xc (SPI_FAST_FLASH_BOOT)
Saved PC:0x4001975a
--- 0x4001975a: software_reset_cpu in ROM

SPIWP:0xee
mode:DIO, clock div:2
load:0x40875720,len:0x16b8
load:0x4086c110,len:0xe84
load:0x4086e610,len:0x3058
entry 0x4086c11a
I (26) boot: ESP-IDF v5.4-727-g5cbd2a3877 2nd stage bootloader
I (27) boot: compile time Apr 11 2025 17:26:12
I (28) boot: chip revision: v0.0
I (29) boot: efuse block revision: v0.1
I (32) boot.esp32c6: SPI Speed      : 80MHz
I (35) boot.esp32c6: SPI Mode       : DIO
I (39) boot.esp32c6: SPI Flash Size : 2MB
I (43) boot: Enabling RNG early entropy source...
I (48) boot: Partition Table:
I (50) boot: ## Label            Usage          Type ST Offset   Length
I (56) boot:  0 nvs              WiFi data        01 02 00009000 00006000
I (63) boot:  1 phy_init         RF data          01 01 0000f000 00001000
I (69) boot:  2 factory          factory app      00 00 00010000 00100000
I (76) boot: End of partition table
I (79) esp_image: segment 0: paddr=00010020 vaddr=420c0020 size=1fe08h (130568) map
I (134) esp_image: segment 1: paddr=0002fe30 vaddr=40800000 size=001e8h (   488) load
I (136) esp_image: segment 2: paddr=00030020 vaddr=42000020 size=b8c70h (756848) map
I (413) esp_image: segment 3: paddr=000e8c98 vaddr=408001e8 size=192ach (103084) load
I (457) esp_image: segment 4: paddr=00101f4c vaddr=408194a0 size=0b318h ( 45848) load
I (493) boot: Loaded app from partition at offset 0x10000
I (493) boot: Disabling RNG early entropy source...
I (504) cpu_start: Unicore app
I (512) cpu_start: Pro cpu start user code
I (513) cpu_start: cpu freq: 160000000 Hz
I (513) app_init: Application information:
I (513) app_init: Project name:     int32_pub_sub
I (517) app_init: App version:      5.0.1-8-g031fac2-dirty
I (522) app_init: Compile time:     Apr 11 2025 17:26:09
I (527) app_init: ELF file SHA256:  97634b4f2...
I (532) app_init: ESP-IDF:          v5.4-727-g5cbd2a3877
I (537) efuse_init: Min chip rev:     v0.0
I (540) efuse_init: Max chip rev:     v0.99 
I (544) efuse_init: Chip rev:         v0.0
I (548) heap_init: Initializing. RAM available for dynamic allocation:
I (555) heap_init: At 408324D0 len 0004A140 (296 KiB): RAM
I (560) heap_init: At 4087C610 len 00002F54 (11 KiB): RAM
I (565) heap_init: At 50000000 len 00003FE8 (15 KiB): RTCRAM
I (571) spi_flash: detected chip: generic
I (574) spi_flash: flash io: dio
W (577) spi_flash: Detected size(4096k) larger than the size in the binary image header(2048k). Using the size in the binary image header.
I (589) sleep_gpio: Configure to isolate all GPIO pins in sleep state
I (595) sleep_gpio: Enable automatic switching of GPIO sleep configuration
I (602) coexist: coex firmware version: e727207
I (620) coexist: coexist rom version 5b8dcfa
I (621) main_task: Started on CPU0
I (621) main_task: Calling app_main()
I (631) wifi_station_netif: ESP_WIFI_MODE_STA
I (631) pp: pp rom version: 5b8dcfa
I (631) net80211: net80211 rom version: 5b8dcfa
I (641) wifi:wifi driver task: 4083af30, prio:23, stack:6656, core=0
I (641) wifi:wifi firmware version: 79fa3f41ba
I (641) wifi:wifi certification version: v7.0
I (641) wifi:config NVS flash: enabled
I (651) wifi:config nano formatting: disabled
I (651) wifi:mac_version:HAL_MAC_ESP32AX_761,ut_version:N, band mode:0x1
I (661) wifi:Init data frame dynamic rx buffer num: 32
I (661) wifi:Init static rx mgmt buffer num: 5
I (671) wifi:Init management short buffer num: 32
I (671) wifi:Init dynamic tx buffer num: 32
I (671) wifi:Init static tx FG buffer num: 2
I (681) wifi:Init static rx buffer size: 1700 (rxctrl:92, csi:512)
I (681) wifi:Init static rx buffer num: 10
I (691) wifi:Init dynamic rx buffer num: 32
I (691) wifi_init: rx ba win: 6
I (691) wifi_init: accept mbox: 6
I (701) wifi_init: tcpip mbox: 32
I (701) wifi_init: udp mbox: 6
I (701) wifi_init: tcp mbox: 6
I (711) wifi_init: tcp tx win: 5760
I (711) wifi_init: tcp rx win: 5760
I (711) wifi_init: tcp mss: 1440
I (721) wifi_init: WiFi IRAM OP enabled
I (721) wifi_init: WiFi RX IRAM OP enabled
I (721) wifi_init: WiFi SLP IRAM OP enabled
W (731) wifi:Password length matches WPA2 standards, authmode threshold changes from OPEN to WPA2
I (741) phy_init: phy_version 330,4af1121,Feb  6 2025,14:39:13
I (781) phy_init: Saving new calibration data due to checksum failure or outdated calibration data, mode(0)
W (811) wifi:ACK_TAB0   :0x   90a0b, QAM16:0x9 (24Mbps), QPSK:0xa (12Mbps), BPSK:0xb (6Mbps)
W (811) wifi:CTS_TAB0   :0x   90a0b, QAM16:0x9 (24Mbps), QPSK:0xa (12Mbps), BPSK:0xb (6Mbps)
W (811) wifi:(agc)0x600a7128:0xd2022800, min.avgNF:0xce->0xd2(dB), RCalCount:0x22, min.RRssi:0x800(-128.00)
W (821) wifi:(TB)WDEV_PWR_TB_MCS0:19
W (821) wifi:(TB)WDEV_PWR_TB_MCS1:19
W (831) wifi:(TB)WDEV_PWR_TB_MCS2:19
W (831) wifi:(TB)WDEV_PWR_TB_MCS3:19
W (831) wifi:(TB)WDEV_PWR_TB_MCS4:19
W (841) wifi:(TB)WDEV_PWR_TB_MCS5:19
W (841) wifi:(TB)WDEV_PWR_TB_MCS6:18
W (841) wifi:(TB)WDEV_PWR_TB_MCS7:18
W (851) wifi:(TB)WDEV_PWR_TB_MCS8:17
W (851) wifi:(TB)WDEV_PWR_TB_MCS9:15
W (851) wifi:(TB)WDEV_PWR_TB_MCS10:15
W (861) wifi:(TB)WDEV_PWR_TB_MCS11:15
I (861) wifi:11ax coex: WDEVAX_PTI0(0x55777555), WDEVAX_PTI1(0x00003377).

I (871) wifi:mode : sta (54:32:04:07:28:3c)
I (871) wifi:enable tsf
I (881) wifi_station_netif: wifi_init_sta finished.
I (881) wifi:new:<1,0>, old:<1,0>, ap:<255,255>, sta:<1,0>, prof:1, snd_ch_cfg:0x0
I (891) wifi:(connect)dot11_authmode:0x3, pairwise_cipher:0x3, group_cipher:0x3
I (891) wifi:state: init -> auth (0xb0)
I (921) wifi:state: auth -> assoc (0x0)
I (941) wifi:(assoc)RESP, Extended Capabilities length:1, operating_mode_notification:0
I (941) wifi:(assoc)RESP, Extended Capabilities, MBSSID:0, TWT Responder:0, OBSS Narrow Bandwidth RU In OFDMA Tolerance:0
I (951) wifi:state: assoc -> run (0x10)
I (951) wifi:(trc)phytype:CBW20-SGI, snr:75, maxRate:144, highestRateIdx:0
W (961) wifi:(trc)band:2G, phymode:3, highestRateIdx:0, lowestRateIdx:11, dataSchedTableSize:14
I (971) wifi:(trc)band:2G, rate(S-MCS7, rateIdx:0), ampdu(rate:S-MCS7, schedIdx(0, stop:8)), snr:75, ampduState:wait operational
I (981) wifi:ifidx:0, rssi:-21, nf:-96, phytype(0x3, CBW20-SGI), phymode(0x3, 11bgn), max_rate:144, he:0, vht:0, ht:1
I (991) wifi:(ht)max.RxAMPDULenExponent:3(65535 bytes), MMSS:4(2 us)
I (1021) wifi:connected with It Burns When IP, aid = 6, channel 1, BW20, bssid = ee:6d:56:02:6f:3b
I (1021) wifi:security: WPA2-PSK, phy:11bgn, rssi:-21, cipher(pairwise:0x3, group:0x3), pmf:0, 
I (1031) wifi:pm start, type: 1, twt_start:0

I (1031) wifi:pm start, type:1, aid:0x6, trans-BSSID:ee:6d:56:02:6f:3b, BSSID[5]:0x3b, mbssid(max-indicator:0, index:0), he:0
I (1041) wifi:dp: 1, bi: 102400, li: 3, scale listen interval from 307200 us to 307200 us
I (1051) wifi:set rx beacon pti, rx_bcn_pti: 10, bcn_timeout: 25000, mt_pti: 10, mt_time: 10000
I (1061) wifi:[ADDBA]TX addba request, tid:0, dialogtoken:1, bufsize:64, A-MSDU:0(not supported), policy:1(IMR), ssn:0(0x0)
I (1071) wifi:[ADDBA]TX addba request, tid:7, dialogtoken:2, bufsize:64, A-MSDU:0(not supported), policy:1(IMR), ssn:0(0x20)
I (1081) wifi:[ADDBA]TX addba request, tid:5, dialogtoken:3, bufsize:64, A-MSDU:0(not supported), policy:1(IMR), ssn:0(0x0)
I (1091) wifi:dp: 2, bi: 102400, li: 4, scale listen interval from 307200 us to 409600 us
I (1101) wifi:AP's beacon interval = 102400 us, DTIM period = 2
I (1111) wifi:[ADDBA]RX addba response, status:0, tid:0/tb:1(0xa1), bufsize:64, batimeout:0, txa_wnd:64
I (1111) wifi:[ADDBA]RX addba response, status:0, tid:7/tb:1(0xa1), bufsize:64, batimeout:0, txa_wnd:64
I (1121) wifi:[ADDBA]RX addba response, status:0, tid:5/tb:1(0xa1), bufsize:64, batimeout:0, txa_wnd:64
I (2191) esp_netif_handlers: sta ip: 192.168.249.245, mask: 255.255.255.0, gw: 192.168.249.123
I (2191) wifi_station_netif: got ip:192.168.249.245
I (2191) wifi_station_netif: connected to ap SSID:**** password:****
I (2201) main_task: Returned from app_main()
W (2211) wifi:<ba-add>idx:0, ifx:0, tid:0, TAHI:0x1003b6f, TALO:0x2566dee, (ssn:4, win:64, cur_ssn:4), CONF:0xc0000005
Parameter param1 modified. New value: 0 (bool)
Parameter param2 modified. New value: 10 (int)
Parameter param3 modified. New value: 0.010000 (double)
Guru Meditation Error: Core  0 panic'ed (Illegal instruction). Exception was unhandled.

Core  0 register dump:
--- Stack dump detected
MEPC    : 0x420104ba  RA      : 0x420104aa  SP      : 0x40846dd0  GP      : 0x40819ca4  
--- 0x420104ba: rclc_parameter_get_double at ??:?
0x420104aa: rclc_parameter_get_double at ??:?

TP      : 0x40846f50  T0      : 0x7f7fffff  T1      : 0x7f7f7f7f  T2      : 0xffffffff  
S0/FP   : 0x40846e00  S1      : 0x420c3000  A0      : 0x00000001  A1      : 0x40835108  
A2      : 0x0000336d  A3      : 0x00000003  A4      : 0x00000003  A5      : 0x40834b7c  
A6      : 0xf4240000  A7      : 0xe6f50000  S2      : 0x420c3000  S3      : 0x420c3000  
S4      : 0x00000000  S5      : 0x00000000  S6      : 0x00000000  S7      : 0x00000000  
S8      : 0x00000000  S9      : 0x00000000  S10     : 0x00000000  S11     : 0x00000000  
T3      : 0x00000000  T4      : 0x00000413  T5      : 0x00000001  T6      : 0x3f847ae1  
MSTATUS : 0x00001881  MTVEC   : 0x40800001  MCAUSE  : 0x00000002  MTVAL   : 0x0000339c  
--- 0x40800001: _vector_table at /opt/esp/idf/components/riscv/vectors_intc.S:54

MHARTID : 0x00000000  


--- Backtrace:


0x420104ba in rclc_parameter_get_double ()
#0  0x420104ba in rclc_parameter_get_double ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)



ELF file SHA256: 97634b4f2

Additional information

I see that the esp32c6 hasn't been supported for long. If you could point me to where the difference lies for esp32 and esp32c6, I might be able to tackle the issue myself. If you have ideas, let me know. I will take a deeper look and post updates here.

[Geibinger]

Also note that the rclc_parameter_get_double error might be misleading. If I comment out all the double parameter code, I get:

W (7448) wifi:<ba-add>idx:0, ifx:0, tid:0, TAHI:0x1003b6f, TALO:0x2566dee, (ssn:3, win:64, cur_ssn:3), CONF:0xc0000005
Parameter param1 modified. New value: 0 (bool)
Parameter param2 modified. New value: 10 (int)
Guru Meditation Error: Core  0 panic'ed (Illegal instruction). Exception was unhandled.

--- Stack dump detected
Core  0 register dump:
MEPC    : 0x420241be  RA      : 0x420240f2  SP      : 0x40846c60  GP      : 0x40819ca4  
--- 0x420241be: rmw_wait at ??:?
0x420240f2: rmw_wait at ??:?

TP      : 0x40846ef0  T0      : 0x40030dca  T1      : 0x0000000c  T2      : 0xffffffff  
--- 0x40030dca: memset in ROM

S0/FP   : 0x4082adc0  S1      : 0x40834e58  A0      : 0x00000006  A1      : 0x00001000  
A2      : 0x00001000  A3      : 0x00000001  A4      : 0xffffffff  A5      : 0x00000001  
A6      : 0x00000000  A7      : 0xf4240000  S2      : 0x40834e4c  S3      : 0x05f5e100  
S4      : 0x40834e64  S5      : 0x00000064  S6      : 0x40834e70  S7      : 0x00000007  
S8      : 0x00000000  S9      : 0x00000000  S10     : 0x00000000  S11     : 0x00000000  
T3      : 0x00000008  T4      : 0x00000000  T5      : 0x42010872  T6      : 0x4201087a  
--- 0x42010872: __default_deallocate at allocator.c:?
0x4201087a: __default_allocate at allocator.c:?

MSTATUS : 0x00001881  MTVEC   : 0x40800001  MCAUSE  : 0x00000002  MTVAL   : 0xd00af7d3  
--- 0x40800001: _vector_table at /opt/esp/idf/components/riscv/vectors_intc.S:54

MHARTID : 0x00000000  


--- Backtrace:


0x420241be in rmw_wait ()
#0  0x420241be in rmw_wait ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)

[Geibinger]

Ok, after some headbanging, I found the culprit. It seems to be this line in the rmw_microxrcedds source code. I assume (int32_t)((float)timeout.i32 / (float)available_contexts) somehow clashes with the compilation on RISC-V targets (possibly because the FPU is not enabled when executing the line).

int32_t per_session_timeout = (timeout.i32 == UXR_TIMEOUT_INF) ? UXR_TIMEOUT_INF : (int32_t)((float)timeout.i32 / (float)available_contexts);

After changing it to:

int32_t per_session_timeout = (timeout.i32 == UXR_TIMEOUT_INF) ? UXR_TIMEOUT_INF : (timeout.i32 / available_contexts);

It works without issues. However, I don't know if this is the best solution (probably not).

How should I proceed? Should the Issue be moved to rmw_microxrcedds?