feat: testConnection for filestores

yarn-project/archiver/src/archiver/archiver.ts

@@ -63,7 +63,7 @@ import {
   getTimestampRangeForEpoch,
 } from '@aztec/stdlib/epoch-helpers';
 import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client';
-import type { L2LogsSource } from '@aztec/stdlib/interfaces/server';
+import { type L2LogsSource, tryStop } from '@aztec/stdlib/interfaces/server';
 import {
   ContractClassLog,
   type LogFilter,
@@ -1180,6 +1180,7 @@ export class Archiver
   public async stop(): Promise<void> {
     this.log.debug('Stopping...');
     await this.runningPromise.stop();
+    await tryStop(this.blobClient);
 
     this.log.info('Stopped.');
     return Promise.resolve();

yarn-project/blob-client/README.md

@@ -34,6 +34,15 @@ Beacon node URLs for fetching recent blobs directly from L1.
 **Archive API URL** (`BLOB_SINK_ARCHIVE_API_URL`):
 Blobscan or similar archive API for historical blob data.
 
+### File Store Connectivity Testing
+
+All file stores (S3, GCS, HTTP, local) test connectivity by checking if a well-known healthcheck file (`.healthcheck`) exists. This approach was chosen because:
+
+1. **HTTP compatibility**: For HTTP-based file stores, requesting a known file is the only reliable way to verify connectivity
+2. **Uniform behavior**: Using the same healthcheck mechanism across all store types ensures consistent behavior and simplifies testing
+
+When uploading is enabled, the sequencer uploads the healthcheck file on startup and then periodically re-uploads it (by default every 60 minutes) to ensure it remains available. This guards against accidental deletion, storage pruning, or other failures that might remove the file.
+
 ### Example Usage
 
 ```typescript

yarn-project/blob-client/src/client/config.ts

@@ -50,6 +50,11 @@ export interface BlobClientConfig extends BlobArchiveApiConfig {
    * URL for uploading blobs to filestore (s3://, gs://, file://)
    */
   blobFileStoreUploadUrl?: string;
+
+  /**
+   * Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)
+   */
+  blobHealthcheckUploadIntervalMinutes?: number;
 }
 
 export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
@@ -98,6 +103,11 @@ export const blobClientConfigMapping: ConfigMappingsType<BlobClientConfig> = {
     env: 'BLOB_FILE_STORE_UPLOAD_URL',
     description: 'URL for uploading blobs to filestore (s3://, gs://, file://)',
   },
+  blobHealthcheckUploadIntervalMinutes: {
+    env: 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES',
+    description: 'Interval in minutes for uploading healthcheck file to file store (default: 60 = 1 hour)',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+  },
   ...blobArchiveApiConfigMappings,
 };
 

yarn-project/blob-client/src/client/factory.ts

@@ -58,10 +58,11 @@ export interface BlobClientWithFileStoresConfig extends BlobClientConfig {
  * 2. Creating read-only FileStore clients
  * 3. Creating a writable FileStore client for uploads
  * 4. Creating the BlobClient with these dependencies
+ * 5. Starting the client (uploads initial healthcheck file if upload client is configured)
  *
  * @param config - Configuration containing blob client settings and chain metadata
  * @param logger - Optional logger for the blob client
- * @returns A BlobClientInterface configured with file store support
+ * @returns A BlobClientInterface configured with file store support, already started
  */
 export async function createBlobClientWithFileStores(
   config: BlobClientWithFileStoresConfig,
@@ -80,9 +81,13 @@ export async function createBlobClientWithFileStores(
     createWritableFileStoreBlobClient(config.blobFileStoreUploadUrl, fileStoreMetadata, log),
   ]);
 
-  return createBlobClient(config, {
+  const client = createBlobClient(config, {
     logger: log,
     fileStoreClients,
     fileStoreUploadClient,
   });
+
+  await client.start?.();
+
+  return client;
 }

yarn-project/blob-client/src/client/http.ts

@@ -9,6 +9,7 @@ import { type RpcBlock, createPublicClient, fallback, http } from 'viem';
 import { createBlobArchiveClient } from '../archive/factory.js';
 import type { BlobArchiveClient } from '../archive/interface.js';
 import type { FileStoreBlobClient } from '../filestore/filestore_blob_client.js';
+import { DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES } from '../filestore/healthcheck.js';
 import { type BlobClientConfig, getBlobClientConfigFromEnv } from './config.js';
 import type { BlobClientInterface, GetBlobSidecarOptions } from './interface.js';
 
@@ -21,6 +22,7 @@ export class HttpBlobClient implements BlobClientInterface {
   protected readonly fileStoreUploadClient: FileStoreBlobClient | undefined;
 
   private disabled = false;
+  private healthcheckUploadIntervalId?: NodeJS.Timeout;
 
   constructor(
     config?: BlobClientConfig,
@@ -545,6 +547,45 @@ export class HttpBlobClient implements BlobClientInterface {
   public getArchiveClient(): BlobArchiveClient | undefined {
     return this.archiveClient;
   }
+
+  /**
+   * Start the blob client.
+   * Uploads the initial healthcheck file (awaited) and starts periodic uploads.
+   */
+  public async start(): Promise<void> {
+    if (!this.fileStoreUploadClient) {
+      return;
+    }
+
+    await this.fileStoreUploadClient.uploadHealthcheck();
+    this.log.debug('Initial healthcheck file uploaded');
+
+    this.startPeriodicHealthcheckUpload();
+  }
+
+  /**
+   * Start periodic healthcheck upload to the file store to ensure it remains available even if accidentally deleted.
+   */
+  private startPeriodicHealthcheckUpload(): void {
+    const intervalMs =
+      (this.config.blobHealthcheckUploadIntervalMinutes ?? DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES) * 60 * 1000;
+
+    this.healthcheckUploadIntervalId = setInterval(() => {
+      void this.fileStoreUploadClient!.uploadHealthcheck().catch(err => {
+        this.log.warn('Failed to upload periodic healthcheck file', err);
+      });
+    }, intervalMs);
+  }
+
+  /**
+   * Stop the blob client, clearing any periodic tasks.
+   */
+  public stop(): void {
+    if (this.healthcheckUploadIntervalId) {
+      clearInterval(this.healthcheckUploadIntervalId);
+      this.healthcheckUploadIntervalId = undefined;
+    }
+  }
 }
 
 function parseBlobJsonsFromResponse(response: any, logger: Logger): BlobJson[] {

yarn-project/blob-client/src/client/interface.ts

@@ -18,6 +18,10 @@ export interface BlobClientInterface {
   sendBlobsToFilestore(blobs: Blob[]): Promise<boolean>;
   /** Fetches the given blob sidecars by block hash and blob hashes. */
   getBlobSidecar(blockId: string, blobHashes?: Buffer[], opts?: GetBlobSidecarOptions): Promise<Blob[]>;
+  /** Starts the blob client (e.g., uploads healthcheck file if not exists). */
+  start?(): Promise<void>;
   /** Tests all configured blob sources and logs whether they are reachable or not. */
   testSources(): Promise<void>;
+  /** Stops the blob client, clearing any periodic tasks. */
+  stop?(): void;
 }

yarn-project/blob-client/src/filestore/filestore_blob_client.test.ts

@@ -212,24 +212,26 @@ describe('FileStoreBlobClient', () => {
   });
 
   describe('testConnection', () => {
-    it('should return true when store is accessible', async () => {
+    it('should return true when healthcheck file exists', async () => {
+      await client.uploadHealthcheck();
       expect(await client.testConnection()).toBe(true);
     });
 
-    // Note: The current implementation of testConnection() always returns true
-    // as noted in the TODO comment in the source. This test documents the current
-    // behavior. When proper connectivity testing is implemented, this test should
-    // be updated to verify that testConnection returns false for failing stores.
-    it('should return true even when store might fail (current implementation limitation)', async () => {
+    it('should return false when healthcheck file does not exist', async () => {
+      const freshStore = new MockFileStore();
+      const freshClient = new FileStoreBlobClient(freshStore, basePath);
+      expect(await freshClient.testConnection()).toBe(false);
+    });
+
+    it('should return false when store throws error', async () => {
       const failingStore: ReadOnlyFileStore = {
         read: () => Promise.reject(new Error('fail')),
         download: () => Promise.reject(new Error('fail')),
         exists: () => Promise.reject(new Error('fail')),
       };
 
       const failingClient = new FileStoreBlobClient(failingStore, basePath);
-      // Current implementation always returns true
-      expect(await failingClient.testConnection()).toBe(true);
+      expect(await failingClient.testConnection()).toBe(false);
     });
   });
 

yarn-project/blob-client/src/filestore/filestore_blob_client.ts

@@ -3,6 +3,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
 import type { FileStore, ReadOnlyFileStore } from '@aztec/stdlib/file-store';
 
 import { inboundTransform, outboundTransform } from '../encoding/index.js';
+import { HEALTHCHECK_CONTENT, HEALTHCHECK_FILENAME } from './healthcheck.js';
 
 /**
  * A blob client that uses a FileStore (S3/GCS/local) as the data source.
@@ -27,6 +28,14 @@ export class FileStoreBlobClient {
     return `${this.basePath}/blobs/${versionedBlobHash}.data`;
   }
 
+  /**
+   * Get the path for the healthcheck file.
+   * Format: basePath/.healthcheck
+   */
+  private healthcheckPath(): string {
+    return `${this.basePath}/${HEALTHCHECK_FILENAME}`;
+  }
+
   /**
    * Fetch blobs by their versioned hashes.
    * @param blobHashes - Array of versioned blob hashes (0x-prefixed hex strings)
@@ -104,12 +113,35 @@ export class FileStoreBlobClient {
   }
 
   /**
-   * Test if the filestore connection is working.
+   * Test if the filestore connection is working by checking for healthcheck file.
+   * The healthcheck file is uploaded periodically by writable clients via HttpBlobClient.start().
+   * This provides a uniform connection test across all store types (S3/GCS/Local/HTTP).
+   */
+  async testConnection(): Promise<boolean> {
+    try {
+      return await this.store.exists(this.healthcheckPath());
+    } catch (err: any) {
+      this.log.warn(`Connection test failed: ${err?.message ?? String(err)}`);
+      return false;
+    }
+  }
+
+  /**
+   * Upload the healthcheck file if it doesn't already exist.
+   * This enables read-only clients (HTTP) to verify connectivity.
    */
-  testConnection(): Promise<boolean> {
-    // This implementation will be improved in a separate PR
-    // Currently underlying filestore implementations do not expose an easy way to test connectivitiy
-    return Promise.resolve(true);
+  async uploadHealthcheck(): Promise<void> {
+    if (!this.isWritable()) {
+      this.log.trace('Cannot upload healthcheck: store is read-only');
+      return;
+    }
+    const path = this.healthcheckPath();
+    if (await this.store.exists(path)) {
+      this.log.trace(`Healthcheck file already exists at ${path}, skipping upload`);
+      return;
+    }
+    await (this.store as FileStore).save(path, Buffer.from(HEALTHCHECK_CONTENT));
+    this.log.debug(`Uploaded healthcheck file to ${path}`);
   }
 
   /**

yarn-project/blob-client/src/filestore/healthcheck.ts

@@ -0,0 +1,5 @@
+/** Constants for healthcheck file used to test file store connectivity. */
+
+export const HEALTHCHECK_FILENAME = '.healthcheck';
+export const HEALTHCHECK_CONTENT = 'ok';
+export const DEFAULT_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES = 60; // 1 hour

yarn-project/foundation/src/config/env_var.ts

@@ -26,6 +26,7 @@ export type EnvVar =
   | 'BLOB_SINK_URL'
   | 'BLOB_FILE_STORE_URLS'
   | 'BLOB_FILE_STORE_UPLOAD_URL'
+  | 'BLOB_HEALTHCHECK_UPLOAD_INTERVAL_MINUTES'
   | 'BOT_DA_GAS_LIMIT'
   | 'BOT_FEE_PAYMENT_METHOD'
   | 'BOT_BASE_FEE_PADDING'

yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts

@@ -37,6 +37,7 @@ import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts';
 import { type ProposerSlashAction, encodeSlashConsensusVotes } from '@aztec/slasher';
 import { CommitteeAttestationsAndSigners, type ValidateBlockResult } from '@aztec/stdlib/block';
 import type { Checkpoint } from '@aztec/stdlib/checkpoint';
+import { tryStop } from '@aztec/stdlib/interfaces/server';
 import { SlashFactoryContract } from '@aztec/stdlib/l1-contracts';
 import type { CheckpointHeader } from '@aztec/stdlib/rollup';
 import type { L1PublishCheckpointStats } from '@aztec/stdlib/stats';
@@ -1035,6 +1036,11 @@ export class SequencerPublisher {
     this.l1TxUtils.restart();
   }
 
+  /** Stops the publisher, cleaning up any periodic tasks. */
+  public async stop(): Promise<void> {
+    await tryStop(this.blobClient);
+  }
+
   private async prepareProposeTx(
     encodedData: L1ProcessArgs,
     timestamp: bigint,

yarn-project/sequencer-client/src/sequencer/sequencer.ts

@@ -156,6 +156,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
     this.setState(SequencerState.STOPPING, undefined, { force: true });
     this.publisher?.interrupt();
     await this.runningPromise?.stop();
+    await this.publisher?.stop();
     this.setState(SequencerState.STOPPED, undefined, { force: true });
     this.log.info('Stopped sequencer');
   }