From a21db285d70347c28bcd342ea230cc8d3e44e492 Mon Sep 17 00:00:00 2001
From: Diego Tavares <dtavares@imageworks.com>
Date: Fri, 8 May 2026 10:58:07 -0700
Subject: [PATCH] [cuebot] Fix channel closing issue
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The application os not properly closing the grpc channel at shutdown, leaving this SEVERE warning:

```
Channel ManagedChannelImpl{logId=2787, target=elk0815:8444} was not shutdown properly
```

Explanation:

`RqdClientGrpc` (cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java:71-89) holds a
Guava `LoadingCache<String, ManagedChannel>`. The `removalListener` calls `conn.shutdown()` **on
cache eviction** — but never on application shutdown. The bean has no `destroy-method`
(applicationContext-service.xml:39, no destroy hook), and `RqdClientGrpc` has no `shutdown()` method
at all. So when cuebot stops:

- The cache is GC'd
- Each `ManagedChannel` is finalized **without** `shutdown()` having been called
- gRPC logs `SEVERE: Channel ... was not shutdown properly!!!` per leaked channel, with the stack
  capturing where the channel was first allocated (the `RuntimeException: ManagedChannel allocation
  site` — that's a diagnostic stack, not a real exception)

The first truncated stack in your paste is another such allocation-site diagnostic. The actual
runtime call (`launchFrame`) **succeeded** — the frame was dispatched. This is purely a
resource-cleanup hygiene warning, not a functional failure.
---
 .../java/com/imageworks/spcue/rqd/RqdClientGrpc.java |  7 +++++++
 .../conf/spring/applicationContext-service.xml       | 12 ++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java
index 5d2677598..9c7f4d6a9 100644
--- a/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java
+++ b/cuebot/src/main/java/com/imageworks/spcue/rqd/RqdClientGrpc.java
@@ -216,4 +216,11 @@ public void launchFrame(final RunFrame frame, final VirtualProc proc) {
     public void setTestMode(boolean testMode) {
         this.testMode = testMode;
     }
+
+    public void shutdown() {
+        if (channelCache != null) {
+            logger.info("Shutting down RqdClientGrpc channel cache");
+            channelCache.invalidateAll();
+        }
+    }
 }
diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml
index 5a42d4b8c..0c5fad6b6 100644
--- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml
+++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml
@@ -36,7 +36,7 @@
   <!-- Non-Transactional Service Domain -->
   <!-- ##################################################################################### -->
 
-  <bean id="rqdClient" class="com.imageworks.spcue.rqd.RqdClientGrpc">
+  <bean id="rqdClient" class="com.imageworks.spcue.rqd.RqdClientGrpc" destroy-method="shutdown">
     <constructor-arg index="0" type="int">
       <value>${grpc.rqd_server_port}</value>
     </constructor-arg>
@@ -66,7 +66,7 @@
     <property name="queueCapacity" value="500" />
   </bean>
 
-  <bean id="bookingQueue" class="com.imageworks.spcue.dispatcher.BookingQueue" lazy-init="true" destroy-method="shutdown" depends-on="cueDataSource">
+  <bean id="bookingQueue" class="com.imageworks.spcue.dispatcher.BookingQueue" lazy-init="true" destroy-method="shutdown" depends-on="cueDataSource,rqdClient">
     <constructor-arg index="0" type="int">
       <value>${booking_queue.threadpool.health_threshold}</value>
     </constructor-arg>
@@ -85,7 +85,7 @@
     <property name="shutdownDrainMs" value="${healthy_threadpool.shutdown_drain_ms:60000}"/>
   </bean>
 
-  <bean id="dispatchQueue" class="com.imageworks.spcue.dispatcher.DispatchQueue" destroy-method="shutdown" depends-on="cueDataSource">
+  <bean id="dispatchQueue" class="com.imageworks.spcue.dispatcher.DispatchQueue" destroy-method="shutdown" depends-on="cueDataSource,rqdClient">
     <constructor-arg index="0" type="java.lang.String">
       <value>DispatchQueue</value>
     </constructor-arg>
@@ -107,7 +107,7 @@
     <property name="shutdownDrainMs" value="${healthy_threadpool.shutdown_drain_ms:60000}"/>
   </bean>
 
-  <bean id="manageQueue" class="com.imageworks.spcue.dispatcher.DispatchQueue"   destroy-method="shutdown" depends-on="cueDataSource">
+  <bean id="manageQueue" class="com.imageworks.spcue.dispatcher.DispatchQueue"   destroy-method="shutdown" depends-on="cueDataSource,rqdClient">
     <constructor-arg index="0" type="java.lang.String">
       <value>ManageQueue</value>
     </constructor-arg>
@@ -128,7 +128,7 @@
     </constructor-arg>
     <property name="shutdownDrainMs" value="${healthy_threadpool.shutdown_drain_ms:60000}"/>
   </bean>
-  <bean id="reportQueue" class="com.imageworks.spcue.dispatcher.HostReportQueue" destroy-method="shutdown" depends-on="cueDataSource">
+  <bean id="reportQueue" class="com.imageworks.spcue.dispatcher.HostReportQueue" destroy-method="shutdown" depends-on="cueDataSource,rqdClient">
     <constructor-arg index="0" type="int">
       <value>${report_queue.threadPoolSizeInitial}</value>
     </constructor-arg>
@@ -140,7 +140,7 @@
     </constructor-arg>
     <property name="shutdownDrainMs" value="${host_report_queue.shutdown_drain_ms:60000}"/>
   </bean>
-  <bean id="killQueue" class="com.imageworks.spcue.dispatcher.HostReportQueue" destroy-method="shutdown" depends-on="cueDataSource">
+  <bean id="killQueue" class="com.imageworks.spcue.dispatcher.HostReportQueue" destroy-method="shutdown" depends-on="cueDataSource,rqdClient">
     <constructor-arg index="0" type="int">
       <value>${kill_queue.threadPoolSizeInitial}</value>
     </constructor-arg>