diff --git a/packages/cli/src/commands/compute/app/deploy.ts b/packages/cli/src/commands/compute/app/deploy.ts index 202b17b..0fb1530 100644 --- a/packages/cli/src/commands/compute/app/deploy.ts +++ b/packages/cli/src/commands/compute/app/deploy.ts @@ -1,5 +1,10 @@ import { Command, Flags } from "@oclif/core"; -import { getEnvironmentConfig, UserApiClient, isMainnet } from "@layr-labs/ecloud-sdk"; +import { + getEnvironmentConfig, + UserApiClient, + isMainnet, + WatchTimeoutError, +} from "@layr-labs/ecloud-sdk"; import { withTelemetry } from "../../../telemetry"; import { commonFlags, applyTxOverrides } from "../../../flags"; import { createComputeClient } from "../../../client"; @@ -147,6 +152,11 @@ export default class AppDeploy extends Command { description: "Skip all confirmation prompts", default: false, }), + "watch-timeout": Flags.integer({ + description: + "Maximum seconds to wait for the app to start before returning a recovery hint (default: 600)", + env: "ECLOUD_WATCH_TIMEOUT_SECONDS", + }), }; async run() { @@ -533,7 +543,41 @@ export default class AppDeploy extends Command { } // 12. Watch until app is running - const ipAddress = await compute.app.watchDeployment(res.appId); + let ipAddress: string | undefined; + try { + ipAddress = await compute.app.watchDeployment(res.appId, { + timeoutSeconds: flags["watch-timeout"], + }); + } catch (watchErr: any) { + if (watchErr instanceof WatchTimeoutError) { + this.log( + `\n${chalk.yellow("⚠")} ${chalk.yellow( + `Deployment did not reach Running within ${watchErr.elapsedSeconds}s (last status: ${watchErr.lastStatus ?? "unknown"}).`, + )}`, + ); + this.log( + chalk.gray( + `The deploy transaction succeeded, but the orchestrator hasn't reported the app as Running yet.`, + ), + ); + this.log(chalk.gray(` appId: ${res.appId}`)); + if (res.txHash) { + this.log(chalk.gray(` txHash: ${res.txHash}`)); + } + this.log( + chalk.gray( + `Check progress later with: ${chalk.cyan(`ecloud compute app info ${res.appId}`)}`, + ), + ); + this.log( + chalk.gray( + `Override the watch timeout with the ${chalk.cyan("ECLOUD_WATCH_TIMEOUT_SECONDS")} environment variable.`, + ), + ); + this.exit(1); + } + throw watchErr; + } try { const cwd = process.env.INIT_CWD || process.cwd(); diff --git a/packages/sdk/src/client/common/contract/watcher.ts b/packages/sdk/src/client/common/contract/watcher.ts index 3482ff0..c6c7fea 100644 --- a/packages/sdk/src/client/common/contract/watcher.ts +++ b/packages/sdk/src/client/common/contract/watcher.ts @@ -17,13 +17,70 @@ export interface WatchUntilRunningOptions { publicClient: PublicClient; environmentConfig: EnvironmentConfig; appId: Address; + /** + * Maximum seconds to wait before throwing {@link WatchTimeoutError}. + * Precedence: explicit value > `ECLOUD_WATCH_TIMEOUT_SECONDS` env var > 600s default. + */ + timeoutSeconds?: number; } const WATCH_POLL_INTERVAL_SECONDS = 5; +const WATCH_HEARTBEAT_INTERVAL_SECONDS = 30; +export const WATCH_DEFAULT_TIMEOUT_SECONDS = 10 * 60; const APP_STATUS_RUNNING = "Running"; const APP_STATUS_FAILED = "Failed"; // const APP_STATUS_DEPLOYING = 'Deploying'; +/** + * Typed error thrown when watch loops exceed their timeout budget. + * + * Callers (e.g. the CLI) can catch this specifically to surface a + * troubleshooting hint without treating it as a generic failure. + */ +export class WatchTimeoutError extends Error { + public readonly appId: string; + public readonly elapsedSeconds: number; + public readonly lastStatus: string | undefined; + public readonly timeoutSeconds: number; + + constructor(args: { + appId: string; + elapsedSeconds: number; + lastStatus: string | undefined; + timeoutSeconds: number; + message?: string; + }) { + super( + args.message ?? + `Timed out after ${args.elapsedSeconds}s waiting for app ${args.appId} (last status: ${args.lastStatus ?? "unknown"})`, + ); + this.name = "WatchTimeoutError"; + this.appId = args.appId; + this.elapsedSeconds = args.elapsedSeconds; + this.lastStatus = args.lastStatus; + this.timeoutSeconds = args.timeoutSeconds; + } +} + +/** + * Resolve the watch timeout in seconds, honoring the + * ECLOUD_WATCH_TIMEOUT_SECONDS environment override. + */ +function resolveWatchTimeoutSeconds(explicit?: number): number { + if (typeof explicit === "number" && Number.isFinite(explicit) && explicit > 0) { + return Math.floor(explicit); + } + const raw = process.env.ECLOUD_WATCH_TIMEOUT_SECONDS; + if (raw === undefined || raw === "") { + return WATCH_DEFAULT_TIMEOUT_SECONDS; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + return WATCH_DEFAULT_TIMEOUT_SECONDS; + } + return Math.floor(parsed); +} + /** * Watch app until it reaches Running status with IP address */ @@ -79,8 +136,23 @@ export async function watchUntilRunning( // Main watch loop const startTime = Date.now(); + const timeoutSeconds = resolveWatchTimeoutSeconds(options.timeoutSeconds); let lastLoggedStatus: string | undefined; + let lastHeartbeatAt = startTime; while (true) { + const elapsedMs = Date.now() - startTime; + const elapsed = Math.round(elapsedMs / 1000); + + // Bound the loop: surface a typed timeout so callers can hint the user. + if (elapsed >= timeoutSeconds) { + throw new WatchTimeoutError({ + appId, + elapsedSeconds: elapsed, + lastStatus: lastLoggedStatus, + timeoutSeconds, + }); + } + try { // Fetch app info const info = await userApiClient.getInfos([appId], 1); @@ -93,11 +165,16 @@ export async function watchUntilRunning( const currentStatus = appInfo.status; const currentIP = appInfo.ip || ""; - // Log status changes and elapsed time - const elapsed = Math.round((Date.now() - startTime) / 1000); + // Log status transitions, plus a periodic heartbeat so non-TTY + // stdout (where carriage-return updates are invisible) still shows + // progress when the status string is unchanged for a long time. if (currentStatus !== lastLoggedStatus) { logger.info(`Status: ${currentStatus} (${elapsed}s)`); lastLoggedStatus = currentStatus; + lastHeartbeatAt = Date.now(); + } else if (Date.now() - lastHeartbeatAt >= WATCH_HEARTBEAT_INTERVAL_SECONDS * 1000) { + logger.info(`Status: ${currentStatus} (${elapsed}s)`); + lastHeartbeatAt = Date.now(); } // Check stop condition @@ -108,6 +185,10 @@ export async function watchUntilRunning( // Wait before next poll await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000); } catch (error: any) { + // Re-throw typed terminal errors so the caller can react to them. + if (error instanceof WatchTimeoutError) { + throw error; + } logger.warn(`Failed to fetch app info: ${error.message}`); await sleep(WATCH_POLL_INTERVAL_SECONDS * 1000); } diff --git a/packages/sdk/src/client/index.ts b/packages/sdk/src/client/index.ts index 5f2e987..0e5dd8d 100644 --- a/packages/sdk/src/client/index.ts +++ b/packages/sdk/src/client/index.ts @@ -39,6 +39,7 @@ export { executeDeploy, watchDeployment, type PrepareDeployResult, + type WatchDeploymentOptions, } from "./modules/compute/app/deploy"; export { SDKUpgradeOptions, @@ -112,6 +113,9 @@ export { type EstimateGasOptions, } from "./common/contract/caller"; +// Export watcher errors so callers can react to typed terminal failures. +export { WatchTimeoutError } from "./common/contract/watcher"; + // Export batch gas estimation and delegation check export { estimateBatchGas, diff --git a/packages/sdk/src/client/modules/compute/app/deploy.ts b/packages/sdk/src/client/modules/compute/app/deploy.ts index 2b5ea5d..27ebcf0 100644 --- a/packages/sdk/src/client/modules/compute/app/deploy.ts +++ b/packages/sdk/src/client/modules/compute/app/deploy.ts @@ -711,6 +711,10 @@ export async function executeDeploy(options: ExecuteDeployOptions): Promise { return withSDKTelemetry( { @@ -735,6 +740,7 @@ export async function watchDeployment( publicClient, environmentConfig, appId: appId as Address, + timeoutSeconds: opts?.timeoutSeconds, }, logger, ); diff --git a/packages/sdk/src/client/modules/compute/app/index.ts b/packages/sdk/src/client/modules/compute/app/index.ts index ccf6921..59014ca 100644 --- a/packages/sdk/src/client/modules/compute/app/index.ts +++ b/packages/sdk/src/client/modules/compute/app/index.ts @@ -16,6 +16,7 @@ import { prepareDeployFromVerifiableBuild as prepareDeployFromVerifiableBuildFn, executeDeploy as executeDeployFn, watchDeployment as watchDeploymentFn, + type WatchDeploymentOptions, } from "./deploy"; import { upgrade as upgradeApp, @@ -125,7 +126,10 @@ export interface AppModule { gasEstimate: GasEstimate; }>; executeDeploy: (prepared: PreparedDeploy, gas?: GasEstimate) => Promise; - watchDeployment: (appId: AppId) => Promise; + watchDeployment: ( + appId: AppId, + opts?: WatchDeploymentOptions, + ) => Promise; // Granular upgrade control prepareUpgrade: ( @@ -314,7 +318,7 @@ export function createAppModule(ctx: AppModuleConfig): AppModule { }; }, - async watchDeployment(appId) { + async watchDeployment(appId, opts) { return watchDeploymentFn( appId, walletClient, @@ -322,6 +326,7 @@ export function createAppModule(ctx: AppModuleConfig): AppModule { environment, logger, skipTelemetry, + opts, ); },