fix(cli): bound upgrade watch with timeout and progress (RND-568)

packages/cli/src/commands/compute/app/upgrade.ts

@@ -1,5 +1,10 @@
 import { Command, Args, Flags } from "@oclif/core";
-import { getEnvironmentConfig, UserApiClient, isMainnet } from "@layr-labs/ecloud-sdk";
+import {
+  getEnvironmentConfig,
+  UserApiClient,
+  isMainnet,
+  WatchTimeoutError,
+} from "@layr-labs/ecloud-sdk";
 import { withTelemetry } from "../../../telemetry";
 import { commonFlags, applyTxOverrides } from "../../../flags";
 import { createBuildClient, createComputeClient } from "../../../client";
@@ -130,6 +135,11 @@ export default class AppUpgrade extends Command {
       description: "Skip all confirmation prompts",
       default: false,
     }),
+    "watch-timeout": Flags.integer({
+      description:
+        "Maximum seconds to wait for the upgrade to complete before returning a recovery hint (default: 600)",
+      env: "ECLOUD_WATCH_TIMEOUT_SECONDS",
+    }),
   };
 
   async run() {
@@ -407,7 +417,32 @@ export default class AppUpgrade extends Command {
       const res = await compute.app.executeUpgrade(prepared, finalTx);
 
       // 12. Watch until upgrade completes
-      await compute.app.watchUpgrade(res.appId);
+      try {
+        await compute.app.watchUpgrade(res.appId, { timeoutSeconds: flags["watch-timeout"] });
+      } catch (err: any) {
+        if (err instanceof WatchTimeoutError) {
+          this.log("");
+          this.log(
+            chalk.yellow(
+              `Timed out after ${err.elapsedSeconds}s waiting for upgrade to complete (last status: ${err.lastStatus ?? "unknown"}).`,
+            ),
+          );
+          this.log(chalk.gray("The on-chain transaction was submitted; the orchestrator may"));
+          this.log(chalk.gray("still be processing. To check the current status, run:"));
+          this.log("");
+          this.log(`  ${chalk.cyan(`ecloud compute app info ${res.appId}`)}`);
+          this.log("");
+          this.log(chalk.gray(`appId:  ${res.appId}`));
+          this.log(chalk.gray(`txHash: ${res.txHash}`));
+          this.log(
+            chalk.gray(
+              `(override the watch deadline with ECLOUD_WATCH_TIMEOUT_SECONDS, currently ${err.timeoutSeconds}s)`,
+            ),
+          );
+          this.exit(1);
+        }
+        throw err;
+      }
 
       try {
         const cwd = process.env.INIT_CWD || process.cwd();

packages/sdk/src/client/common/contract/watcher.ts

@@ -122,20 +122,83 @@ export interface WatchUntilUpgradeCompleteOptions {
   publicClient: PublicClient;
   environmentConfig: EnvironmentConfig;
   appId: Address;
+  /**
+   * Maximum time (in seconds) to wait for the upgrade to complete before
+   * throwing a {@link WatchTimeoutError}. If unspecified, defaults to
+   * the value of the `ECLOUD_WATCH_TIMEOUT_SECONDS` env var, falling back to
+   * {@link WATCH_DEFAULT_TIMEOUT_SECONDS}.
+   */
+  timeoutSeconds?: number;
 }
 
 const APP_STATUS_STOPPED = "Stopped";
 
+/** Default upgrade watch timeout: 10 minutes. */
+export const WATCH_DEFAULT_TIMEOUT_SECONDS = 10 * 60;
+
+/**
+ * Error thrown when {@link watchUntilUpgradeComplete} exceeds its deadline
+ * without observing a terminal status (Stopped or Running) for the app.
+ *
+ * Callers (e.g. the CLI) can catch this and surface a recovery hint pointing
+ * the user at `ecloud compute app info <appId>`.
+ */
+export class WatchTimeoutError extends Error {
+  public readonly appId: Address;
+  public readonly lastStatus: string | undefined;
+  public readonly elapsedSeconds: number;
+  public readonly timeoutSeconds: number;
+
+  constructor(params: {
+    appId: Address;
+    lastStatus: string | undefined;
+    elapsedSeconds: number;
+    timeoutSeconds: number;
+  }) {
+    super(
+      `Timed out after ${params.elapsedSeconds}s waiting for upgrade to complete (last status: ${
+        params.lastStatus ?? "unknown"
+      })`,
+    );
+    this.name = "WatchTimeoutError";
+    this.appId = params.appId;
+    this.lastStatus = params.lastStatus;
+    this.elapsedSeconds = params.elapsedSeconds;
+    this.timeoutSeconds = params.timeoutSeconds;
+  }
+}
+
+/**
+ * Resolve the upgrade watch timeout from explicit option, env var, or default.
+ */
+function resolveWatchTimeoutSeconds(explicit?: number): number {
+  if (typeof explicit === "number" && Number.isFinite(explicit) && explicit > 0) {
+    return explicit;
+  }
+  const fromEnv = process.env.ECLOUD_WATCH_TIMEOUT_SECONDS;
+  if (fromEnv) {
+    const parsed = Number.parseInt(fromEnv, 10);
+    if (Number.isFinite(parsed) && parsed > 0) {
+      return parsed;
+    }
+  }
+  return WATCH_DEFAULT_TIMEOUT_SECONDS;
+}
+
 /**
  * Watch app until upgrade completes
  * For upgrades, we watch until the app reaches Stopped status (upgrade complete)
- * or Running status (if it was running before upgrade)
+ * or Running status (if it was running before upgrade).
+ *
+ * Throws {@link WatchTimeoutError} if the timeout elapses before a
+ * terminal status is observed.
  */
 export async function watchUntilUpgradeComplete(
   options: WatchUntilUpgradeCompleteOptions,
   logger: Logger,
 ): Promise<void> {
   const { walletClient, publicClient, environmentConfig, appId } = options;
+  const timeoutSeconds = resolveWatchTimeoutSeconds(options.timeoutSeconds);
 
   // Create UserAPI client
   const userApiClient = new UserApiClient(environmentConfig, walletClient, publicClient);
@@ -193,7 +256,21 @@ export async function watchUntilUpgradeComplete(
   };
 
   // Main watch loop
+  const startTime = Date.now();
+  const deadline = startTime + timeoutSeconds * 1000;
+  let lastLoggedStatus: string | undefined;
+  let lastObservedStatus: string | undefined;
   while (true) {
+    if (Date.now() >= deadline) {
+      const elapsedSeconds = Math.round((Date.now() - startTime) / 1000);
+      throw new WatchTimeoutError({
+        appId,
+        lastStatus: lastObservedStatus,
+        elapsedSeconds,
+        timeoutSeconds,
+      });
+    }
+
     try {
       // Fetch app info
       const info = await userApiClient.getInfos([appId], 1);
@@ -205,6 +282,14 @@ export async function watchUntilUpgradeComplete(
       const appInfo = info[0];
       const currentStatus = appInfo.status;
       const currentIP = appInfo.ip || "";
+      lastObservedStatus = currentStatus;
+
+      // Log status changes and elapsed time on a new line per transition
+      const elapsed = Math.round((Date.now() - startTime) / 1000);
+      if (currentStatus !== lastLoggedStatus) {
+        logger.info(`Status: ${currentStatus} (${elapsed}s)`);
+        lastLoggedStatus = currentStatus;
+      }
 
       // Check stop condition
       if (stopCondition(currentStatus, currentIP)) {
@@ -214,6 +299,14 @@ export async function watchUntilUpgradeComplete(
       // Wait before next poll
       await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000);
     } catch (error: any) {
+      // Re-throw timeout errors and terminal failures from stopCondition.
+      if (error instanceof WatchTimeoutError) {
+        throw error;
+      }
+      // Heuristic: stopCondition throws plain Error for "Failed" state — let it propagate.
+      if (typeof error?.message === "string" && error.message.includes("Failed")) {
+        throw error;
+      }
       logger.warn(`Failed to fetch app info: ${error.message}`);
       await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000);
     }

packages/sdk/src/client/index.ts

@@ -47,7 +47,9 @@ export {
   executeUpgrade,
   watchUpgrade,
   type PrepareUpgradeResult,
+  type WatchUpgradeOptions,
 } from "./modules/compute/app/upgrade";
+export { WatchTimeoutError, WATCH_DEFAULT_TIMEOUT_SECONDS } from "./common/contract/watcher";
 
 // Export compute module for standalone use
 export {

packages/sdk/src/client/modules/compute/app/index.ts

@@ -23,6 +23,7 @@ import {
   prepareUpgradeFromVerifiableBuild as prepareUpgradeFromVerifiableBuildFn,
   executeUpgrade as executeUpgradeFn,
   watchUpgrade as watchUpgradeFn,
+  type WatchUpgradeOptions,
 } from "./upgrade";
 import { createApp, CreateAppOpts } from "./create";
 import { logs, LogsOptions } from "./logs";
@@ -143,7 +144,7 @@ export interface AppModule {
     gasEstimate: GasEstimate;
   }>;
   executeUpgrade: (prepared: PreparedUpgrade, gas?: GasEstimate) => Promise<ExecuteUpgradeResult>;
-  watchUpgrade: (appId: AppId) => Promise<void>;
+  watchUpgrade: (appId: AppId, opts?: WatchUpgradeOptions) => Promise<void>;
 
   // Profile management
   setProfile: (appId: AppId, profile: AppProfile) => Promise<AppProfileResponse>;
@@ -383,8 +384,16 @@ export function createAppModule(ctx: AppModuleConfig): AppModule {
       };
     },
 
-    async watchUpgrade(appId) {
-      return watchUpgradeFn(appId, walletClient, publicClient, environment, logger, skipTelemetry);
+    async watchUpgrade(appId, opts) {
+      return watchUpgradeFn(
+        appId,
+        walletClient,
+        publicClient,
+        environment,
+        logger,
+        skipTelemetry,
+        opts,
+      );
     },
 
     // Profile management

packages/sdk/src/client/modules/compute/app/upgrade.ts

@@ -543,13 +543,18 @@ export async function executeUpgrade(options: ExecuteUpgradeOptions): Promise<Up
  * Call this after executeUpgrade to wait for the upgrade to finish.
  * Can be called separately to allow for intermediate operations.
  */
+export interface WatchUpgradeOptions {
+  timeoutSeconds?: number;
+}
+
 export async function watchUpgrade(
   appId: string,
   walletClient: WalletClient,
   publicClient: PublicClient,
   environmentConfig: EnvironmentConfig,
   logger: Logger = defaultLogger,
   skipTelemetry?: boolean,
+  opts?: WatchUpgradeOptions,
 ): Promise<void> {
   return withSDKTelemetry(
     {
@@ -567,6 +572,7 @@ export async function watchUpgrade(
           publicClient,
           environmentConfig,
           appId: appId as Address,
+          timeoutSeconds: opts?.timeoutSeconds,
         },
         logger,
       );