diff --git a/packages/cli/src/commands/compute/app/upgrade.ts b/packages/cli/src/commands/compute/app/upgrade.ts index 90a3ad0..cc944ec 100644 --- a/packages/cli/src/commands/compute/app/upgrade.ts +++ b/packages/cli/src/commands/compute/app/upgrade.ts @@ -1,5 +1,10 @@ import { Command, Args, Flags } from "@oclif/core"; -import { getEnvironmentConfig, UserApiClient, isMainnet } from "@layr-labs/ecloud-sdk"; +import { + getEnvironmentConfig, + UserApiClient, + isMainnet, + WatchTimeoutError, +} from "@layr-labs/ecloud-sdk"; import { withTelemetry } from "../../../telemetry"; import { commonFlags, applyTxOverrides } from "../../../flags"; import { createBuildClient, createComputeClient } from "../../../client"; @@ -130,6 +135,11 @@ export default class AppUpgrade extends Command { description: "Skip all confirmation prompts", default: false, }), + "watch-timeout": Flags.integer({ + description: + "Maximum seconds to wait for the upgrade to complete before returning a recovery hint (default: 600)", + env: "ECLOUD_WATCH_TIMEOUT_SECONDS", + }), }; async run() { @@ -407,7 +417,32 @@ export default class AppUpgrade extends Command { const res = await compute.app.executeUpgrade(prepared, finalTx); // 12. Watch until upgrade completes - await compute.app.watchUpgrade(res.appId); + try { + await compute.app.watchUpgrade(res.appId, { timeoutSeconds: flags["watch-timeout"] }); + } catch (err: any) { + if (err instanceof WatchTimeoutError) { + this.log(""); + this.log( + chalk.yellow( + `Timed out after ${err.elapsedSeconds}s waiting for upgrade to complete (last status: ${err.lastStatus ?? "unknown"}).`, + ), + ); + this.log(chalk.gray("The on-chain transaction was submitted; the orchestrator may")); + this.log(chalk.gray("still be processing. To check the current status, run:")); + this.log(""); + this.log(` ${chalk.cyan(`ecloud compute app info ${res.appId}`)}`); + this.log(""); + this.log(chalk.gray(`appId: ${res.appId}`)); + this.log(chalk.gray(`txHash: ${res.txHash}`)); + this.log( + chalk.gray( + `(override the watch deadline with ECLOUD_WATCH_TIMEOUT_SECONDS, currently ${err.timeoutSeconds}s)`, + ), + ); + this.exit(1); + } + throw err; + } try { const cwd = process.env.INIT_CWD || process.cwd(); diff --git a/packages/sdk/src/client/common/contract/watcher.ts b/packages/sdk/src/client/common/contract/watcher.ts index 3482ff0..2a139b9 100644 --- a/packages/sdk/src/client/common/contract/watcher.ts +++ b/packages/sdk/src/client/common/contract/watcher.ts @@ -122,20 +122,83 @@ export interface WatchUntilUpgradeCompleteOptions { publicClient: PublicClient; environmentConfig: EnvironmentConfig; appId: Address; + /** + * Maximum time (in seconds) to wait for the upgrade to complete before + * throwing a {@link WatchTimeoutError}. If unspecified, defaults to + * the value of the `ECLOUD_WATCH_TIMEOUT_SECONDS` env var, falling back to + * {@link WATCH_DEFAULT_TIMEOUT_SECONDS}. + */ + timeoutSeconds?: number; } const APP_STATUS_STOPPED = "Stopped"; +/** Default upgrade watch timeout: 10 minutes. */ +export const WATCH_DEFAULT_TIMEOUT_SECONDS = 10 * 60; + +/** + * Error thrown when {@link watchUntilUpgradeComplete} exceeds its deadline + * without observing a terminal status (Stopped or Running) for the app. + * + * Callers (e.g. the CLI) can catch this and surface a recovery hint pointing + * the user at `ecloud compute app info `. + */ +export class WatchTimeoutError extends Error { + public readonly appId: Address; + public readonly lastStatus: string | undefined; + public readonly elapsedSeconds: number; + public readonly timeoutSeconds: number; + + constructor(params: { + appId: Address; + lastStatus: string | undefined; + elapsedSeconds: number; + timeoutSeconds: number; + }) { + super( + `Timed out after ${params.elapsedSeconds}s waiting for upgrade to complete (last status: ${ + params.lastStatus ?? "unknown" + })`, + ); + this.name = "WatchTimeoutError"; + this.appId = params.appId; + this.lastStatus = params.lastStatus; + this.elapsedSeconds = params.elapsedSeconds; + this.timeoutSeconds = params.timeoutSeconds; + } +} + +/** + * Resolve the upgrade watch timeout from explicit option, env var, or default. + */ +function resolveWatchTimeoutSeconds(explicit?: number): number { + if (typeof explicit === "number" && Number.isFinite(explicit) && explicit > 0) { + return explicit; + } + const fromEnv = process.env.ECLOUD_WATCH_TIMEOUT_SECONDS; + if (fromEnv) { + const parsed = Number.parseInt(fromEnv, 10); + if (Number.isFinite(parsed) && parsed > 0) { + return parsed; + } + } + return WATCH_DEFAULT_TIMEOUT_SECONDS; +} + /** * Watch app until upgrade completes * For upgrades, we watch until the app reaches Stopped status (upgrade complete) - * or Running status (if it was running before upgrade) + * or Running status (if it was running before upgrade). + * + * Throws {@link WatchTimeoutError} if the timeout elapses before a + * terminal status is observed. */ export async function watchUntilUpgradeComplete( options: WatchUntilUpgradeCompleteOptions, logger: Logger, ): Promise { const { walletClient, publicClient, environmentConfig, appId } = options; + const timeoutSeconds = resolveWatchTimeoutSeconds(options.timeoutSeconds); // Create UserAPI client const userApiClient = new UserApiClient(environmentConfig, walletClient, publicClient); @@ -193,7 +256,21 @@ export async function watchUntilUpgradeComplete( }; // Main watch loop + const startTime = Date.now(); + const deadline = startTime + timeoutSeconds * 1000; + let lastLoggedStatus: string | undefined; + let lastObservedStatus: string | undefined; while (true) { + if (Date.now() >= deadline) { + const elapsedSeconds = Math.round((Date.now() - startTime) / 1000); + throw new WatchTimeoutError({ + appId, + lastStatus: lastObservedStatus, + elapsedSeconds, + timeoutSeconds, + }); + } + try { // Fetch app info const info = await userApiClient.getInfos([appId], 1); @@ -205,6 +282,14 @@ export async function watchUntilUpgradeComplete( const appInfo = info[0]; const currentStatus = appInfo.status; const currentIP = appInfo.ip || ""; + lastObservedStatus = currentStatus; + + // Log status changes and elapsed time on a new line per transition + const elapsed = Math.round((Date.now() - startTime) / 1000); + if (currentStatus !== lastLoggedStatus) { + logger.info(`Status: ${currentStatus} (${elapsed}s)`); + lastLoggedStatus = currentStatus; + } // Check stop condition if (stopCondition(currentStatus, currentIP)) { @@ -214,6 +299,14 @@ export async function watchUntilUpgradeComplete( // Wait before next poll await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000); } catch (error: any) { + // Re-throw timeout errors and terminal failures from stopCondition. + if (error instanceof WatchTimeoutError) { + throw error; + } + // Heuristic: stopCondition throws plain Error for "Failed" state — let it propagate. + if (typeof error?.message === "string" && error.message.includes("Failed")) { + throw error; + } logger.warn(`Failed to fetch app info: ${error.message}`); await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000); } diff --git a/packages/sdk/src/client/index.ts b/packages/sdk/src/client/index.ts index 5f2e987..c75237c 100644 --- a/packages/sdk/src/client/index.ts +++ b/packages/sdk/src/client/index.ts @@ -47,7 +47,9 @@ export { executeUpgrade, watchUpgrade, type PrepareUpgradeResult, + type WatchUpgradeOptions, } from "./modules/compute/app/upgrade"; +export { WatchTimeoutError, WATCH_DEFAULT_TIMEOUT_SECONDS } from "./common/contract/watcher"; // Export compute module for standalone use export { diff --git a/packages/sdk/src/client/modules/compute/app/index.ts b/packages/sdk/src/client/modules/compute/app/index.ts index ccf6921..d318e87 100644 --- a/packages/sdk/src/client/modules/compute/app/index.ts +++ b/packages/sdk/src/client/modules/compute/app/index.ts @@ -23,6 +23,7 @@ import { prepareUpgradeFromVerifiableBuild as prepareUpgradeFromVerifiableBuildFn, executeUpgrade as executeUpgradeFn, watchUpgrade as watchUpgradeFn, + type WatchUpgradeOptions, } from "./upgrade"; import { createApp, CreateAppOpts } from "./create"; import { logs, LogsOptions } from "./logs"; @@ -143,7 +144,7 @@ export interface AppModule { gasEstimate: GasEstimate; }>; executeUpgrade: (prepared: PreparedUpgrade, gas?: GasEstimate) => Promise; - watchUpgrade: (appId: AppId) => Promise; + watchUpgrade: (appId: AppId, opts?: WatchUpgradeOptions) => Promise; // Profile management setProfile: (appId: AppId, profile: AppProfile) => Promise; @@ -383,8 +384,16 @@ export function createAppModule(ctx: AppModuleConfig): AppModule { }; }, - async watchUpgrade(appId) { - return watchUpgradeFn(appId, walletClient, publicClient, environment, logger, skipTelemetry); + async watchUpgrade(appId, opts) { + return watchUpgradeFn( + appId, + walletClient, + publicClient, + environment, + logger, + skipTelemetry, + opts, + ); }, // Profile management diff --git a/packages/sdk/src/client/modules/compute/app/upgrade.ts b/packages/sdk/src/client/modules/compute/app/upgrade.ts index 4c857e9..bf58707 100644 --- a/packages/sdk/src/client/modules/compute/app/upgrade.ts +++ b/packages/sdk/src/client/modules/compute/app/upgrade.ts @@ -543,6 +543,10 @@ export async function executeUpgrade(options: ExecuteUpgradeOptions): Promise { return withSDKTelemetry( { @@ -567,6 +572,7 @@ export async function watchUpgrade( publicClient, environmentConfig, appId: appId as Address, + timeoutSeconds: opts?.timeoutSeconds, }, logger, );