diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 0d5d42b6d3ef..6c3a50ef4271 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -161,6 +161,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { private proofVerifier: ClientProtocolCircuitVerifier, private telemetry: TelemetryClient = getTelemetryClient(), private log = createLogger('node'), + private blobClient?: BlobClientInterface, ) { this.metrics = new NodeMetrics(telemetry, 'AztecNodeService'); this.tracer = telemetry.getTracer('AztecNodeService'); @@ -522,6 +523,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { proofVerifier, telemetry, log, + blobClient, ); } @@ -781,6 +783,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { await tryStop(this.p2pClient); await tryStop(this.worldStateSynchronizer); await tryStop(this.blockSource); + await tryStop(this.blobClient); await tryStop(this.telemetry); this.log.info(`Stopped Aztec Node`); } diff --git a/yarn-project/blob-client/README.md b/yarn-project/blob-client/README.md index 52a90513f97f..84ee2cbf7988 100644 --- a/yarn-project/blob-client/README.md +++ b/yarn-project/blob-client/README.md @@ -34,6 +34,15 @@ Beacon node URLs for fetching recent blobs directly from L1. **Archive API URL** (`BLOB_SINK_ARCHIVE_API_URL`): Blobscan or similar archive API for historical blob data. +### File Store Connectivity Testing + +All file stores (S3, GCS, HTTP, local) test connectivity by checking if a well-known healthcheck file (`.healthcheck`) exists. This approach was chosen because: + +1. **HTTP compatibility**: For HTTP-based file stores, requesting a known file is the only reliable way to verify connectivity +2. **Uniform behavior**: Using the same healthcheck mechanism across all store types ensures consistent behavior and simplifies testing + +When uploading is enabled, the sequencer uploads the healthcheck file on startup and then periodically re-uploads it (by default every 60 minutes) to ensure it remains available. This guards against accidental deletion, storage pruning, or other failures that might remove the file. + ### Example Usage ```typescript diff --git a/yarn-project/blob-client/src/client/config.ts b/yarn-project/blob-client/src/client/config.ts index 746019d363b1..afdf97e757f0 100644 --- a/yarn-project/blob-client/src/client/config.ts +++ b/yarn-project/blob-client/src/client/config.ts @@ -50,6 +50,11 @@ export interface BlobClientConfig extends BlobArchiveApiConfig { * URL for uploading blobs to filestore (s3://, gs://, file://) */ blobFileStoreUploadUrl?: string; + + /** + * Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour) + */ + blobHealthcheckUploadIntervalMinutes?: number; } export const blobClientConfigMapping: ConfigMappingsType = { @@ -98,6 +103,11 @@ export const blobClientConfigMapping: ConfigMappingsType = { env: 'BLOB_FILE_STORE_UPLOAD_URL', description: 'URL for uploading blobs to filestore (s3://, gs://, file://)', }, + blobHealthcheckUploadIntervalMinutes: { + env: 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES', + description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)', + parseEnv: (val: string | undefined) => (val ? +val : undefined), + }, ...blobArchiveApiConfigMappings, }; diff --git a/yarn-project/blob-client/src/client/factory.ts b/yarn-project/blob-client/src/client/factory.ts index 9a9e37b15bb5..2dfbe00e255d 100644 --- a/yarn-project/blob-client/src/client/factory.ts +++ b/yarn-project/blob-client/src/client/factory.ts @@ -58,10 +58,11 @@ export interface BlobClientWithFileStoresConfig extends BlobClientConfig { * 2. Creating read-only FileStore clients * 3. Creating a writable FileStore client for uploads * 4. Creating the BlobClient with these dependencies + * 5. Starting the client (uploads initial healthcheck file if upload client is configured) * * @param config - Configuration containing blob client settings and chain metadata * @param logger - Optional logger for the blob client - * @returns A BlobClientInterface configured with file store support + * @returns A BlobClientInterface configured with file store support, already started */ export async function createBlobClientWithFileStores( config: BlobClientWithFileStoresConfig, @@ -80,9 +81,13 @@ export async function createBlobClientWithFileStores( createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log), ]); - return createBlobClient(config, { + const client = createBlobClient(config, { logger: log, fileStoreClients, fileStoreUploadClient, }); + + await client.start?.(); + + return client; } diff --git a/yarn-project/blob-client/src/client/http.ts b/yarn-project/blob-client/src/client/http.ts index 62aa28f2ed4e..a9fa8090062b 100644 --- a/yarn-project/blob-client/src/client/http.ts +++ b/yarn-project/blob-client/src/client/http.ts @@ -9,6 +9,7 @@ import { type RpcBlock, createPublicClient, fallback, http } from 'viem'; import { createBlobArchiveClient } from '../archive/factory.js'; import type { BlobArchiveClient } from '../archive/interface.js'; import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js'; +import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js'; import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js'; import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js'; @@ -21,6 +22,7 @@ export class HttpBlobClient implements BlobClientInterface { protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined; private disabled = false; + private healthcheckUploadIntervalId?: NodeJS.Timeout; constructor( config?: BlobClientConfig, @@ -545,6 +547,45 @@ export class HttpBlobClient implements BlobClientInterface { public getArchiveClient(): BlobArchiveClient | undefined { return this.archiveClient; } + + /** + * Start the blob client. + * Uploads the initial healthcheck file (awaited) and starts periodic uploads. + */ + public async start(): Promise { + if (!this.fileStoreUploadClient) { + return; + } + + await this.fileStoreUploadClient.uploadHealthcheck(); + this.log.debug('Initial healthcheck file uploaded'); + + this.startPeriodicHealthcheckUpload(); + } + + /** + * Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted. + */ + private startPeriodicHealthcheckUpload(): void { + const intervalMs = + (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000; + + this.healthcheckUploadIntervalId = setInterval(() => { + void this.fileStoreUploadClient!.uploadHealthcheck().catch(err => { + this.log.warn('Failed to upload periodic healthcheck file', err); + }); + }, intervalMs); + } + + /** + * Stop the blob client, clearing any periodic tasks. + */ + public stop(): void { + if (this.healthcheckUploadIntervalId) { + clearInterval(this.healthcheckUploadIntervalId); + this.healthcheckUploadIntervalId = undefined; + } + } } function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] { diff --git a/yarn-project/blob-client/src/client/interface.ts b/yarn-project/blob-client/src/client/interface.ts index 16f60a8846c1..400f9baa7edd 100644 --- a/yarn-project/blob-client/src/client/interface.ts +++ b/yarn-project/blob-client/src/client/interface.ts @@ -18,6 +18,10 @@ export interface BlobClientInterface { sendBlobsToFilestore(blobs: Blob[]): Promise; /** Fetches the given blob sidecars by block hash and blob hashes. */ getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise; + /** Starts the blob client (e.g., uploads healthcheck file if not exists). */ + start?(): Promise; /** Tests all configured blob sources and logs whether they are reachable or not. */ testSources(): Promise; + /** Stops the blob client, clearing any periodic tasks. */ + stop?(): void; } diff --git a/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts b/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts index 87b2ca8a2789..fe30df599709 100644 --- a/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts +++ b/yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts @@ -212,15 +212,18 @@ describe('FileStoreBlobClient', () => { }); describe('testConnection', () => { - it('should return true when store is accessible', async () => { + it('should return true when healthcheck file exists', async () => { + await client.uploadHealthcheck(); expect(await client.testConnection()).toBe(true); }); - // Note: The current implementation of testConnection() always returns true - // as noted in the TODO comment in the source. This test documents the current - // behavior. When proper connectivity testing is implemented, this test should - // be updated to verify that testConnection returns false for failing stores. - it('should return true even when store might fail (current implementation limitation)', async () => { + it('should return false when healthcheck file does not exist', async () => { + const freshStore = new MockFileStore(); + const freshClient = new FileStoreBlobClient(freshStore, basePath); + expect(await freshClient.testConnection()).toBe(false); + }); + + it('should return false when store throws error', async () => { const failingStore: ReadOnlyFileStore = { read: () => Promise.reject(new Error('fail')), download: () => Promise.reject(new Error('fail')), @@ -228,8 +231,7 @@ describe('FileStoreBlobClient', () => { }; const failingClient = new FileStoreBlobClient(failingStore, basePath); - // Current implementation always returns true - expect(await failingClient.testConnection()).toBe(true); + expect(await failingClient.testConnection()).toBe(false); }); }); diff --git a/yarn-project/blob-client/src/filestore/filestore_blob_client.ts b/yarn-project/blob-client/src/filestore/filestore_blob_client.ts index 14e202eb3529..20320e504bae 100644 --- a/yarn-project/blob-client/src/filestore/filestore_blob_client.ts +++ b/yarn-project/blob-client/src/filestore/filestore_blob_client.ts @@ -3,6 +3,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store'; import { inboundTransform, outboundTransform } from '../encoding/index.js'; +import { HEALTHCHECK_CONTENT, HEALTHCHECK_FILENAME } from './healthcheck.js'; /** * A blob client that uses a FileStore (S3/GCS/local) as the data source. @@ -27,6 +28,14 @@ export class FileStoreBlobClient { return `${this.basePath}/blobs/${versionedBlobHash}.data`; } + /** + * Get the path for the healthcheck file. + * Format: basePath/.healthcheck + */ + private healthcheckPath(): string { + return `${this.basePath}/${HEALTHCHECK_FILENAME}`; + } + /** * Fetch blobs by their versioned hashes. * @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings) @@ -104,12 +113,31 @@ export class FileStoreBlobClient { } /** - * Test if the filestore connection is working. + * Test if the filestore connection is working by checking for healthcheck file. + * The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start(). + * This provides a uniform connection test across all store types (S3/GCS/Local/HTTP). + */ + async testConnection(): Promise { + try { + return await this.store.exists(this.healthcheckPath()); + } catch (err: any) { + this.log.warn(`Connection test failed: ${err?.message ?? String(err)}`); + return false; + } + } + + /** + * Upload the healthcheck file if it doesn't already exist. + * This enables read-only clients (HTTP) to verify connectivity. */ - testConnection(): Promise { - // This implementation will be improved in a separate PR - // Currently underlying filestore implementations do not expose an easy way to test connectivitiy - return Promise.resolve(true); + async uploadHealthcheck(): Promise { + if (!this.isWritable()) { + this.log.trace('Cannot upload healthcheck: store is read-only'); + return; + } + const path = this.healthcheckPath(); + await (this.store as FileStore).save(path, Buffer.from(HEALTHCHECK_CONTENT)); + this.log.debug(`Uploaded healthcheck file to ${path}`); } /** diff --git a/yarn-project/blob-client/src/filestore/healthcheck.ts b/yarn-project/blob-client/src/filestore/healthcheck.ts new file mode 100644 index 000000000000..3f6e2d74f348 --- /dev/null +++ b/yarn-project/blob-client/src/filestore/healthcheck.ts @@ -0,0 +1,5 @@ +/** Constants for healthcheck file used to test file store connectivity. */ + +export const HEALTHCHECK_FILENAME = '.healthcheck'; +export const HEALTHCHECK_CONTENT = 'ok'; +export const DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES = 60; // 1 hour diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 7af74dd8e956..78347ec39a70 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -26,6 +26,7 @@ export type EnvVar = | 'BLOB_SINK_URL' | 'BLOB_FILE_STORE_URLS' | 'BLOB_FILE_STORE_UPLOAD_URL' + | 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES' | 'BOT_DA_GAS_LIMIT' | 'BOT_FEE_PAYMENT_METHOD' | 'BOT_BASE_FEE_PADDING' diff --git a/yarn-project/sequencer-client/src/client/sequencer-client.ts b/yarn-project/sequencer-client/src/client/sequencer-client.ts index 0ae213eb54fa..05908a8d5ea5 100644 --- a/yarn-project/sequencer-client/src/client/sequencer-client.ts +++ b/yarn-project/sequencer-client/src/client/sequencer-client.ts @@ -60,7 +60,7 @@ export class SequencerClient { l1ToL2MessageSource: L1ToL2MessageSource; telemetry: TelemetryClient; publisherFactory?: SequencerPublisherFactory; - blobClient?: BlobClientInterface; + blobClient: BlobClientInterface; dateProvider: DateProvider; epochCache?: EpochCache; l1TxUtils: L1TxUtilsWithBlobs[]; diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts index dd0a2473603c..2942b7ec3be1 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher-factory.ts @@ -33,7 +33,7 @@ export class SequencerPublisherFactory { private deps: { telemetry: TelemetryClient; publisherManager: PublisherManager; - blobClient?: BlobClientInterface; + blobClient: BlobClientInterface; dateProvider: DateProvider; epochCache: EpochCache; rollupContract: RollupContract; diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index 262e76544dc3..77ae8dea1040 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -1,4 +1,4 @@ -import { type BlobClientInterface, createBlobClient } from '@aztec/blob-client/client'; +import type { BlobClientInterface } from '@aztec/blob-client/client'; import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib'; import type { EpochCache } from '@aztec/epoch-cache'; import type { L1ContractsConfig } from '@aztec/ethereum/config'; @@ -145,7 +145,7 @@ export class SequencerPublisher { private config: TxSenderConfig & PublisherConfig & Pick, deps: { telemetry?: TelemetryClient; - blobClient?: BlobClientInterface; + blobClient: BlobClientInterface; l1TxUtils: L1TxUtilsWithBlobs; rollupContract: RollupContract; slashingProposerContract: EmpireSlashingProposerContract | TallySlashingProposerContract | undefined; @@ -163,8 +163,7 @@ export class SequencerPublisher { this.epochCache = deps.epochCache; this.lastActions = deps.lastActions; - this.blobClient = - deps.blobClient ?? createBlobClient(config, { logger: createLogger('sequencer:blob-client:client') }); + this.blobClient = deps.blobClient; const telemetry = deps.telemetry ?? getTelemetryClient(); this.metrics = deps.metrics ?? new SequencerPublisherMetrics(telemetry, 'SequencerPublisher');