diff --git a/src/@types/index.ts b/src/@types/index.ts index fcc880a5..4d940118 100644 --- a/src/@types/index.ts +++ b/src/@types/index.ts @@ -934,6 +934,7 @@ export interface NetworkParameters { minTollUsdStr: string defaultTollUsdStr: string goldenTicketServerUrl: string + goldenTicketRetryInterval: number dao: { proposalFeeUsdStr: string voteThresholdUsdStr: string diff --git a/src/config/index.ts b/src/config/index.ts index 03bc91de..34ea0293 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -104,6 +104,7 @@ export const INITIAL_PARAMETERS: NetworkParameters = { minTollUsdStr: '0.2', defaultTollUsdStr: '0.2', goldenTicketServerUrl: 'http://localhost:3456/golden/ticket', + goldenTicketRetryInterval: 10 * ONE_MINUTE, messageRetentionDays: 7, messageMaxLength: 500, dao: { diff --git a/src/index.ts b/src/index.ts index 74d84323..f5f97199 100644 --- a/src/index.ts +++ b/src/index.ts @@ -77,6 +77,7 @@ const daoPreCrackTxTypes = new Set([ let isReadyToJoinLatestValue = false let mustUseAdminCert = false +let nextGoldenTicketRetryAt = 0 const shardusSetup = (): void => { // SDK SETUP FUNCTIONS @@ -1565,27 +1566,41 @@ const shardusSetup = (): void => { isReadyToJoinLatestValue = false mustUseAdminCert = false + const hasExpiredGoldenTicket = adminCert && adminCert.certExp <= dapp.shardusGetTime() && adminCert?.goldenTicket === true // query admin cert from the golden ticket server - if (!isRequestedAdminCert && networkAccount && utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion)) { + if ( + (!isRequestedAdminCert || hasExpiredGoldenTicket) && + networkAccount && + utils.isEqualOrNewerVersion('2.4.3', networkAccount.current.activeVersion) && + dapp.shardusGetTime() >= nextGoldenTicketRetryAt + ) { try { - markRequestedAdminCert() // one-time request only - - const goldenTicket = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp) - if (goldenTicket) { - setAdminCertificate(goldenTicket) + const goldenTicketResult = await tryAndFetchGoldenTicket(publicKey, networkAccount, dapp) + if (goldenTicketResult.ticket) { + setAdminCertificate(goldenTicketResult.ticket) + markRequestedAdminCert() /* prettier-ignore */ - if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicket)}`) + if (LiberdusFlags.VerboseLogs) console.log(`fetched golden ticket: ${Utils.safeStringify(goldenTicketResult.ticket)}`) nestedCountersInstance.countEvent('liberdus-staking', 'fetched golden ticket from server') console.log(`Admin certificate is set to `, adminCert) + } else if (goldenTicketResult.terminal) { + markRequestedAdminCert() + /* prettier-ignore */ + if (LiberdusFlags.VerboseLogs) console.log(`terminal golden ticket fetch error: ${goldenTicketResult.error}`) + nestedCountersInstance.countEvent('liberdus-staking', `terminal golden ticket fetch error: ${goldenTicketResult.error}`) } else { + const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE + nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval /* prettier-ignore */ - if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server`) - nestedCountersInstance.countEvent('liberdus-staking', 'no golden ticket available from server') + if (LiberdusFlags.VerboseLogs) console.log(`no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`) + nestedCountersInstance.countEvent('liberdus-staking', `no golden ticket available from server, retrying in ${goldenTicketRetryInterval}ms`) } } catch (e) { + const goldenTicketRetryInterval = networkAccount.current.goldenTicketRetryInterval || 10 * configs.ONE_MINUTE + nextGoldenTicketRetryAt = dapp.shardusGetTime() + goldenTicketRetryInterval /* prettier-ignore */ - if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}`) // non fatal + if (logFlags.error) console.log(`Error fetching golden ticket: ${e.message}; retrying in ${goldenTicketRetryInterval}ms`) // non fatal } } console.log('is AdminCert set to ', adminCert) diff --git a/src/transactions/admin_certificate.ts b/src/transactions/admin_certificate.ts index 1885fb00..da39633a 100644 --- a/src/transactions/admin_certificate.ts +++ b/src/transactions/admin_certificate.ts @@ -22,6 +22,13 @@ export interface AdminCertResponse { cached?: boolean } +export interface GoldenTicketFetchResult { + ticket?: AdminCert + error?: string + retryable: boolean + terminal: boolean +} + export type PutAdminCertRequest = AdminCert export interface PutAdminCertResult { @@ -32,6 +39,27 @@ export interface PutAdminCertResult { export let adminCert: AdminCert = null export let isRequestedAdminCert: boolean = false +export function isTerminalGoldenTicketError(error?: string): boolean { + if (!error) return false + const normalizedError = error.toLowerCase() + return ( + normalizedError.includes('public key not registered') || + normalizedError.includes('inactive') || + normalizedError.includes('signature owner does not match registered public key') || + normalizedError.includes('invalid signature') || + normalizedError.includes('signature validation failed') + ) +} + +function createGoldenTicketFetchResult(error?: string): GoldenTicketFetchResult { + const terminal = isTerminalGoldenTicketError(error) + return { + error, + retryable: !terminal, + terminal, + } +} + function validatePutAdminCertRequest(req: PutAdminCertRequest, shardus: Shardus): PutAdminCertResult { const publicKey = shardus.crypto.getPublicKey() @@ -79,7 +107,7 @@ export async function putAdminCertificateHandler(req: Request, shardus: Shardus) return { success: true } } -export async function tryAndFetchGoldenTicket(publicKey: string, network: NetworkAccount, dapp: Shardus): Promise { +export async function tryAndFetchGoldenTicket(publicKey: string, network: NetworkAccount, dapp: Shardus): Promise { try { if (LiberdusFlags.VerboseLogs) console.log('Fetching golden ticket from', network.current.goldenTicketServerUrl, 'for publicKey', publicKey, 'node') const goldenTicketRequest: any = { @@ -93,15 +121,23 @@ export async function tryAndFetchGoldenTicket(publicKey: string, network: Networ if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket request', signedGoldenTicketRequest) const response = await shardusPost(network.current.goldenTicketServerUrl, signedGoldenTicketRequest, { timeout: 5000 }) if (LiberdusFlags.VerboseLogs) console.log('Golden Ticket response', response.data) - if (response.data && response.data.success) { - return response.data.ticket + if (response.data && response.data.success && response.data.ticket) { + return { + ticket: response.data.ticket, + retryable: false, + terminal: false, + } } else { - console.error('No golden ticket received') - return null + const error = response.data?.error || 'No golden ticket received' + console.error(error) + return createGoldenTicketFetchResult(error) } } catch (error) { - console.error(`Error fetching golden ticket from - ${(error as Error).message}`) - return null + const axiosError = error as { message?: string; response?: { data?: AdminCertResponse; status?: number } } + const responseError = axiosError.response?.data?.error + const errorMessage = responseError || axiosError.message || 'Unknown Golden Ticket fetch error' + console.error(`Error fetching golden ticket from - ${errorMessage}`) + return createGoldenTicketFetchResult(errorMessage) } } export function setAdminCertificate(cert: AdminCert): void { diff --git a/src/transactions/apply_change_network_param.ts b/src/transactions/apply_change_network_param.ts index d5888d11..4a588262 100644 --- a/src/transactions/apply_change_network_param.ts +++ b/src/transactions/apply_change_network_param.ts @@ -79,6 +79,9 @@ export function backfillNetworkAccount(network: NetworkAccount): void { if (network.current.goldenTicketServerUrl === undefined || network.current.goldenTicketServerUrl === null) { network.current.goldenTicketServerUrl = 'http://localhost:3456/golden/ticket' } + if (network.current.goldenTicketRetryInterval === undefined || network.current.goldenTicketRetryInterval === null) { + network.current.goldenTicketRetryInterval = 10 * config.ONE_MINUTE + } if (network.current.messageRetentionDays === undefined || network.current.messageRetentionDays === null) { network.current.messageRetentionDays = 7 } diff --git a/test/admin_certificate.test.ts b/test/admin_certificate.test.ts new file mode 100644 index 00000000..df3f1649 --- /dev/null +++ b/test/admin_certificate.test.ts @@ -0,0 +1,104 @@ +jest.mock('../src/utils/request', () => ({ + shardusPost: jest.fn(), +})) + +import { Shardus } from '@shardus/core' +import { shardusPost } from '../src/utils/request' +import { AdminCert, isTerminalGoldenTicketError, tryAndFetchGoldenTicket } from '../src/transactions/admin_certificate' +import { NetworkAccount } from '../src/@types' + +const mockShardusPost = shardusPost as jest.Mock + +const network = { + current: { + goldenTicketServerUrl: 'http://localhost:3456/golden/ticket', + }, +} as NetworkAccount + +const dapp = { + shardusGetTime: jest.fn(() => 123456), + signAsNode: jest.fn((request) => ({ ...request, sign: { owner: request.publicKey, sig: 'signature' } })), +} as unknown as Shardus + +const ticket: AdminCert = { + nominee: 'public-key', + certCreation: 123456, + certExp: 223456, + goldenTicket: true, + sign: { + owner: 'server', + sig: 'signature', + }, +} + +describe('Golden Ticket fetch', () => { + beforeEach(() => { + mockShardusPost.mockReset() + jest.spyOn(console, 'error').mockImplementation(() => undefined) + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + it('returns a ticket on a successful Golden Ticket response', async () => { + mockShardusPost.mockResolvedValue({ data: { success: true, ticket } }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result).toEqual({ ticket, retryable: false, terminal: false }) + }) + + it('classifies timestamp errors from HTTP responses as retryable', async () => { + mockShardusPost.mockRejectedValue({ + message: 'Request failed with status code 409', + response: { + status: 409, + data: { + success: false, + error: 'Timestamp out of acceptable range. Difference: 10000ms, Tolerance: 5000ms', + }, + }, + }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(true) + expect(result.terminal).toBe(false) + expect(result.error).toContain('Timestamp out of acceptable range') + }) + + it('classifies unregistered public keys as terminal', async () => { + mockShardusPost.mockRejectedValue({ + message: 'Request failed with status code 401', + response: { + status: 401, + data: { + success: false, + error: 'Public key not registered or inactive', + }, + }, + }) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(false) + expect(result.terminal).toBe(true) + expect(result.error).toBe('Public key not registered or inactive') + }) + + it('classifies network failures as retryable', async () => { + mockShardusPost.mockRejectedValue(new Error('connect ECONNREFUSED 127.0.0.1:3456')) + + const result = await tryAndFetchGoldenTicket('public-key', network, dapp) + + expect(result.retryable).toBe(true) + expect(result.terminal).toBe(false) + expect(result.error).toContain('ECONNREFUSED') + }) + + it('identifies terminal Golden Ticket validation errors', () => { + expect(isTerminalGoldenTicketError('Public key not registered or inactive')).toBe(true) + expect(isTerminalGoldenTicketError('Timestamp out of acceptable range')).toBe(false) + }) +})