From c29ac82f5bc780350a5544721080eeb8e0ef8f28 Mon Sep 17 00:00:00 2001 From: randyquaye Date: Thu, 5 Feb 2026 15:54:54 +0000 Subject: [PATCH 1/2] chore: explicit provider aggregation path for testnet --- atp-indexer/bootstrap.sh | 14 +- atp-indexer/package.json | 1 + .../scripts/aggregate-providers-core.ts | 164 +++++++++++++++++ .../scripts/aggregate-providers-testnet.ts | 12 ++ atp-indexer/scripts/aggregate-providers.ts | 169 +----------------- providers-testnet/_example.json | 10 ++ 6 files changed, 203 insertions(+), 167 deletions(-) create mode 100644 atp-indexer/scripts/aggregate-providers-core.ts create mode 100644 atp-indexer/scripts/aggregate-providers-testnet.ts create mode 100644 providers-testnet/_example.json diff --git a/atp-indexer/bootstrap.sh b/atp-indexer/bootstrap.sh index 1b2e9395d..cc2264d16 100755 --- a/atp-indexer/bootstrap.sh +++ b/atp-indexer/bootstrap.sh @@ -223,7 +223,13 @@ function build() { # Aggregate provider metadata log_step "Aggregating provider metadata" - yarn bootstrap + + # call separate provider bootstrap script if environment is testnet + if [ "$ENVIRONMENT" = "testnet" ]; then + yarn bootstrap-testnet + else + yarn bootstrap + fi # Build TypeScript log_step "Building TypeScript" @@ -325,7 +331,11 @@ function deploy() { fi yarn install --frozen-lockfile - yarn bootstrap + if [ "$infra_environment" = "testnet" ]; then + yarn bootstrap-testnet + else + yarn bootstrap + fi # Load contract addresses from environment variables or JSON file # For CI/CD, these should be passed as environment variables diff --git a/atp-indexer/package.json b/atp-indexer/package.json index 8dfa461f4..b4728b242 100644 --- a/atp-indexer/package.json +++ b/atp-indexer/package.json @@ -10,6 +10,7 @@ "build": "ponder codegen", "codegen": "ponder codegen", "bootstrap": "tsx scripts/aggregate-providers.ts", + "bootstrap-testnet": "tsx scripts/aggregate-providers-testnet.ts", "compare": "tsx scripts/compare-databases.ts", "test": "jest", "test:watch": "jest --watch", diff --git a/atp-indexer/scripts/aggregate-providers-core.ts b/atp-indexer/scripts/aggregate-providers-core.ts new file mode 100644 index 000000000..0209d21cd --- /dev/null +++ b/atp-indexer/scripts/aggregate-providers-core.ts @@ -0,0 +1,164 @@ +import { readFileSync, writeFileSync, readdirSync, mkdirSync } from 'fs'; +import { join } from 'path'; + +export interface ProviderMetadata { + providerId: number; + providerName: string; + providerDescription: string; + providerEmail: string; + providerWebsite: string; + providerLogoUrl: string; + discordUsername: string; + providerSelfStake?: string[]; +} + +/** + * Validate and normalize provider metadata + * Only providerId is required, other fields will fallback to empty strings if invalid + */ +export function normalizeProvider(metadata: any, filename: string): ProviderMetadata | null { + // ProviderId is the only required field + if (typeof metadata.providerId !== 'number' || metadata.providerId <= 0) { + console.warn(`⚠️ ${filename}: invalid or missing providerId - skipping`); + return null; + } + + const warnings: string[] = []; + + // Helper to validate URL + const validateUrl = (url: string): boolean => { + try { + new URL(url); + return true; + } catch { + return false; + } + }; + + // Normalize each field + const providerName = typeof metadata.providerName === 'string' && metadata.providerName.trim() + ? metadata.providerName.trim() + : ''; + + const providerDescription = typeof metadata.providerDescription === 'string' && metadata.providerDescription.trim() + ? metadata.providerDescription.trim() + : ''; + + const providerEmail = typeof metadata.providerEmail === 'string' && metadata.providerEmail.trim() + ? metadata.providerEmail.trim() + : ''; + + const providerWebsite = typeof metadata.providerWebsite === 'string' && validateUrl(metadata.providerWebsite) + ? metadata.providerWebsite + : ''; + + const providerLogoUrl = typeof metadata.providerLogoUrl === 'string' && validateUrl(metadata.providerLogoUrl) + ? metadata.providerLogoUrl + : ''; + + const discordUsername = typeof metadata.discordUsername === 'string' && metadata.discordUsername.trim() + ? metadata.discordUsername.trim() + : ''; + + // Validate providerSelfStake (optional array of attester addresses) + const providerSelfStake = Array.isArray(metadata.providerSelfStake) && metadata.providerSelfStake.length > 0 + ? metadata.providerSelfStake.filter((addr: any) => typeof addr === 'string' && addr.trim().length > 0) + : undefined; + + // Collect warnings for missing/invalid fields + if (!providerName) warnings.push('providerName'); + if (!providerDescription) warnings.push('providerDescription'); + if (!providerEmail) warnings.push('providerEmail'); + if (!providerWebsite) warnings.push('providerWebsite'); + if (!providerLogoUrl) warnings.push('providerLogoUrl'); + if (!discordUsername) warnings.push('discordUsername'); + + if (warnings.length > 0) { + console.warn(`⚠️ ${filename}: missing or invalid fields: ${warnings.join(', ')}`); + } + + const result: ProviderMetadata = { + providerId: metadata.providerId, + providerName, + providerDescription, + providerEmail, + providerWebsite, + providerLogoUrl, + discordUsername + }; + + // Only add providerSelfStake if it has valid entries + if (providerSelfStake && providerSelfStake.length > 0) { + result.providerSelfStake = providerSelfStake; + } + + return result; +} + +/** + * Aggregate all provider metadata files from a directory into a single JSON file + */ +export function aggregateProvidersFromDir(providersDir: string, outputFile: string) { + const providerMap = new Map(); + let skippedCount = 0; + let duplicateCount = 0; + + try { + const files = readdirSync(providersDir).sort(); // Sort to ensure consistent ordering + + for (const file of files) { + if (!file.endsWith('.json') || file.startsWith('_')) { + continue; + } + + try { + const filePath = join(providersDir, file); + const content = readFileSync(filePath, 'utf-8'); + const metadata = JSON.parse(content); + + const normalized = normalizeProvider(metadata, file); + if (!normalized) { + skippedCount++; + continue; + } + + // Check for duplicate provider ID + if (providerMap.has(normalized.providerId)) { + const existing = providerMap.get(normalized.providerId)!; + console.warn(`⚠️ ${file}: duplicate providerId ${normalized.providerId} (keeping ${existing.filename})`); + duplicateCount++; + continue; + } + + providerMap.set(normalized.providerId, { provider: normalized, filename: file }); + } catch (error) { + console.warn(`⚠️ ${file}: failed to parse JSON - ${error}`); + skippedCount++; + } + } + + // Convert map to array and sort by providerId + const providers = Array.from(providerMap.values()) + .map(entry => entry.provider) + .sort((a, b) => a.providerId - b.providerId); + + // Create output directory if it doesn't exist + const outputDir = outputFile.substring(0, outputFile.lastIndexOf('/')); + mkdirSync(outputDir, { recursive: true }); + + // Write aggregated file + writeFileSync(outputFile, JSON.stringify(providers, null, 2), 'utf-8'); + + console.log(`✓ Aggregated ${providers.length} provider metadata file(s) to ${outputFile}`); + if (duplicateCount > 0) { + console.log(`⚠️ Skipped ${duplicateCount} duplicate provider ID(s)`); + } + if (skippedCount > 0) { + console.log(`⚠️ Skipped ${skippedCount} invalid file(s)`); + } + } catch (error) { + console.error('❌ Failed to aggregate provider metadata:', error); + process.exit(1); + } +} + diff --git a/atp-indexer/scripts/aggregate-providers-testnet.ts b/atp-indexer/scripts/aggregate-providers-testnet.ts new file mode 100644 index 000000000..0fd4312f6 --- /dev/null +++ b/atp-indexer/scripts/aggregate-providers-testnet.ts @@ -0,0 +1,12 @@ +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { aggregateProvidersFromDir } from './aggregate-providers-core'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const providersDir = join(__dirname, '../../providers-testnet'); +const outputFile = join(__dirname, '../src/api/data/providers.json'); + +aggregateProvidersFromDir(providersDir, outputFile); + diff --git a/atp-indexer/scripts/aggregate-providers.ts b/atp-indexer/scripts/aggregate-providers.ts index 7605509b4..916329fd1 100644 --- a/atp-indexer/scripts/aggregate-providers.ts +++ b/atp-indexer/scripts/aggregate-providers.ts @@ -1,172 +1,11 @@ -import { readFileSync, writeFileSync, readdirSync, mkdirSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; +import { aggregateProvidersFromDir } from './aggregate-providers-core'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); -interface ProviderMetadata { - providerId: number; - providerName: string; - providerDescription: string; - providerEmail: string; - providerWebsite: string; - providerLogoUrl: string; - discordUsername: string; - providerSelfStake?: string[]; -} +const providersDir = join(__dirname, '../../providers'); +const outputFile = join(__dirname, '../src/api/data/providers.json'); -/** - * Validate and normalize provider metadata - * Only providerId is required, other fields will fallback to empty strings if invalid - */ -function normalizeProvider(metadata: any, filename: string): ProviderMetadata | null { - // ProviderId is the only required field - if (typeof metadata.providerId !== 'number' || metadata.providerId <= 0) { - console.warn(`⚠️ ${filename}: invalid or missing providerId - skipping`); - return null; - } - - const warnings: string[] = []; - - // Helper to validate URL - const validateUrl = (url: string): boolean => { - try { - new URL(url); - return true; - } catch { - return false; - } - }; - - // Normalize each field - const providerName = typeof metadata.providerName === 'string' && metadata.providerName.trim() - ? metadata.providerName.trim() - : ''; - - const providerDescription = typeof metadata.providerDescription === 'string' && metadata.providerDescription.trim() - ? metadata.providerDescription.trim() - : ''; - - const providerEmail = typeof metadata.providerEmail === 'string' && metadata.providerEmail.trim() - ? metadata.providerEmail.trim() - : ''; - - const providerWebsite = typeof metadata.providerWebsite === 'string' && validateUrl(metadata.providerWebsite) - ? metadata.providerWebsite - : ''; - - const providerLogoUrl = typeof metadata.providerLogoUrl === 'string' && validateUrl(metadata.providerLogoUrl) - ? metadata.providerLogoUrl - : ''; - - const discordUsername = typeof metadata.discordUsername === 'string' && metadata.discordUsername.trim() - ? metadata.discordUsername.trim() - : ''; - - // Validate providerSelfStake (optional array of attester addresses) - const providerSelfStake = Array.isArray(metadata.providerSelfStake) && metadata.providerSelfStake.length > 0 - ? metadata.providerSelfStake.filter((addr: any) => typeof addr === 'string' && addr.trim().length > 0) - : undefined; - - // Collect warnings for missing/invalid fields - if (!providerName) warnings.push('providerName'); - if (!providerDescription) warnings.push('providerDescription'); - if (!providerEmail) warnings.push('providerEmail'); - if (!providerWebsite) warnings.push('providerWebsite'); - if (!providerLogoUrl) warnings.push('providerLogoUrl'); - if (!discordUsername) warnings.push('discordUsername'); - - if (warnings.length > 0) { - console.warn(`⚠️ ${filename}: missing or invalid fields: ${warnings.join(', ')}`); - } - - const result: ProviderMetadata = { - providerId: metadata.providerId, - providerName, - providerDescription, - providerEmail, - providerWebsite, - providerLogoUrl, - discordUsername - }; - - // Only add providerSelfStake if it has valid entries - if (providerSelfStake && providerSelfStake.length > 0) { - result.providerSelfStake = providerSelfStake; - } - - return result; -} - -/** - * Aggregate all provider metadata files into a single JSON file - */ -function aggregateProviders() { - const providersDir = join(__dirname, '../../providers'); - const outputDir = join(__dirname, '../src/api/data'); - const outputFile = join(outputDir, 'providers.json'); - - const providerMap = new Map(); - let skippedCount = 0; - let duplicateCount = 0; - - try { - const files = readdirSync(providersDir).sort(); // Sort to ensure consistent ordering - - for (const file of files) { - if (!file.endsWith('.json') || file.startsWith('_')) { - continue; - } - - try { - const filePath = join(providersDir, file); - const content = readFileSync(filePath, 'utf-8'); - const metadata = JSON.parse(content); - - const normalized = normalizeProvider(metadata, file); - if (!normalized) { - skippedCount++; - continue; - } - - // Check for duplicate provider ID - if (providerMap.has(normalized.providerId)) { - const existing = providerMap.get(normalized.providerId)!; - console.warn(`⚠️ ${file}: duplicate providerId ${normalized.providerId} (keeping ${existing.filename})`); - duplicateCount++; - continue; - } - - providerMap.set(normalized.providerId, { provider: normalized, filename: file }); - } catch (error) { - console.warn(`⚠️ ${file}: failed to parse JSON - ${error}`); - skippedCount++; - } - } - - // Convert map to array and sort by providerId - const providers = Array.from(providerMap.values()) - .map(entry => entry.provider) - .sort((a, b) => a.providerId - b.providerId); - - // Create output directory if it doesn't exist - mkdirSync(outputDir, { recursive: true }); - - // Write aggregated file - writeFileSync(outputFile, JSON.stringify(providers, null, 2), 'utf-8'); - - console.log(`✓ Aggregated ${providers.length} provider metadata file(s) to ${outputFile}`); - if (duplicateCount > 0) { - console.log(`⚠️ Skipped ${duplicateCount} duplicate provider ID(s)`); - } - if (skippedCount > 0) { - console.log(`⚠️ Skipped ${skippedCount} invalid file(s)`); - } - } catch (error) { - console.error('❌ Failed to aggregate provider metadata:', error); - process.exit(1); - } -} - -aggregateProviders(); +aggregateProvidersFromDir(providersDir, outputFile); diff --git a/providers-testnet/_example.json b/providers-testnet/_example.json new file mode 100644 index 000000000..f356992d5 --- /dev/null +++ b/providers-testnet/_example.json @@ -0,0 +1,10 @@ +{ + "providerId": 0, + "providerName": "", + "providerDescription": "", + "providerEmail": "", + "providerWebsite": "", + "providerLogoUrl": "", + "discordUsername": "", + "providerSelfStake": ["0x..."] +} \ No newline at end of file From 3eb2e9cff89b3d9d85bcf627f92a93c6b0f1d617 Mon Sep 17 00:00:00 2001 From: randyquaye Date: Thu, 5 Feb 2026 16:09:10 +0000 Subject: [PATCH 2/2] fix co-pilot nits --- atp-indexer/bootstrap.sh | 2 +- atp-indexer/scripts/aggregate-providers-core.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/atp-indexer/bootstrap.sh b/atp-indexer/bootstrap.sh index cc2264d16..9c73dcc6b 100755 --- a/atp-indexer/bootstrap.sh +++ b/atp-indexer/bootstrap.sh @@ -332,7 +332,7 @@ function deploy() { yarn install --frozen-lockfile if [ "$infra_environment" = "testnet" ]; then - yarn bootstrap-testnet + yarn bootstrap-testnet else yarn bootstrap fi diff --git a/atp-indexer/scripts/aggregate-providers-core.ts b/atp-indexer/scripts/aggregate-providers-core.ts index 0209d21cd..2c0ce0863 100644 --- a/atp-indexer/scripts/aggregate-providers-core.ts +++ b/atp-indexer/scripts/aggregate-providers-core.ts @@ -1,5 +1,5 @@ import { readFileSync, writeFileSync, readdirSync, mkdirSync } from 'fs'; -import { join } from 'path'; +import { join, dirname } from 'path'; export interface ProviderMetadata { providerId: number; @@ -142,8 +142,8 @@ export function aggregateProvidersFromDir(providersDir: string, outputFile: stri .map(entry => entry.provider) .sort((a, b) => a.providerId - b.providerId); - // Create output directory if it doesn't exist - const outputDir = outputFile.substring(0, outputFile.lastIndexOf('/')); + // Create output directory if it doesn't exist (cross-platform) + const outputDir = dirname(outputFile); mkdirSync(outputDir, { recursive: true }); // Write aggregated file