From 445ca57f660d23df82a0c876245405ad222473c1 Mon Sep 17 00:00:00 2001 From: benesjan <13470840+benesjan@users.noreply.github.com> Date: Fri, 2 Jan 2026 16:53:26 +0000 Subject: [PATCH] feat: integrating new log sync Fixes #17775 In this PR I integrate the new recipient log sync algo and the old one gets dropped. The file structure is still a bit messy but that I will handle in a followup PR to make this PR not insane to review. I Introduced `SenderAddressBook` that was originally part of `RecipientTaggingDataProvider`. I decided to do this separation because I want the new log sync algo to not be aware of sender addresses etc. - I wanted it to deal only with `DirectionalAppTaggingSecret`s as it feels like a nice separation. --- .../pending_note_hashes_contract/src/main.nr | 60 ++- .../src/bench/bench_build_block.test.ts | 4 +- yarn-project/end-to-end/src/bench/utils.ts | 18 +- .../e2e_pending_note_hashes_contract.test.ts | 19 +- .../block_synchronizer.test.ts | 15 +- .../block_synchronizer/block_synchronizer.ts | 8 - .../contract_function_simulator.ts | 7 +- .../oracle/oracle_version_is_checked.test.ts | 11 +- .../oracle/private_execution.test.ts | 24 +- .../oracle/private_execution_oracle.ts | 9 +- .../oracle/utility_execution.test.ts | 45 +- .../oracle/utility_execution_oracle.ts | 8 +- yarn-project/pxe/src/logs/log_service.test.ts | 413 +----------------- yarn-project/pxe/src/logs/log_service.ts | 217 +++------ yarn-project/pxe/src/pxe.test.ts | 8 + yarn-project/pxe/src/pxe.ts | 14 +- .../storage/tagging_data_provider/index.ts | 3 +- .../recipient_tagging_data_provider.ts | 86 ---- .../sender_address_book.ts | 42 ++ yarn-project/pxe/src/tagging/constants.ts | 3 - yarn-project/pxe/src/tagging/index.ts | 2 - ...ate_logs_for_sender_recipient_pair.test.ts | 6 +- ..._private_logs_for_sender_recipient_pair.ts | 4 +- ....ts => recipient_tagging_data_provider.ts} | 4 +- .../sync/sync_sender_tagging_indexes.ts | 14 +- yarn-project/pxe/src/tagging/utils.ts | 32 -- .../stdlib/src/logs/tx_scoped_l2_log.ts | 2 +- .../oracle/txe_oracle_top_level_context.ts | 5 +- yarn-project/txe/src/txe_session.test.ts | 1 + yarn-project/txe/src/txe_session.ts | 9 +- 30 files changed, 302 insertions(+), 791 deletions(-) delete mode 100644 yarn-project/pxe/src/storage/tagging_data_provider/recipient_tagging_data_provider.ts create mode 100644 yarn-project/pxe/src/storage/tagging_data_provider/sender_address_book.ts delete mode 100644 yarn-project/pxe/src/tagging/constants.ts rename yarn-project/pxe/src/tagging/recipient_sync/{new_recipient_tagging_data_provider.ts => recipient_tagging_data_provider.ts} (94%) delete mode 100644 yarn-project/pxe/src/tagging/utils.ts diff --git a/noir-projects/noir-contracts/contracts/test/pending_note_hashes_contract/src/main.nr b/noir-projects/noir-contracts/contracts/test/pending_note_hashes_contract/src/main.nr index ee335c0f1c8d..dcfce2042d72 100644 --- a/noir-projects/noir-contracts/contracts/test/pending_note_hashes_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/test/pending_note_hashes_contract/src/main.nr @@ -294,50 +294,66 @@ pub contract PendingNoteHashes { //} #[external("private")] - fn test_recursively_create_notes( - owner: AztecAddress, - sender: AztecAddress, - how_many_recursions: u64, - ) { - self.internal.create_max_notes(owner); - - self.call_self.recursively_destroy_and_create_notes(owner, sender, how_many_recursions); + fn test_recursively_create_notes(recipients: [AztecAddress; 10], how_many_recursions: u64) { + let initial_offset: u64 = 0; + self.internal.create_max_notes(recipients, initial_offset); + + let max_notes = self.internal.max_notes_per_call() as u64; + self.call_self.recursively_destroy_and_create_notes( + recipients, + how_many_recursions, + max_notes, + ); } #[external("private")] fn recursively_destroy_and_create_notes( - owner: AztecAddress, - sender: AztecAddress, + recipients: [AztecAddress; 10], executions_left: u64, + current_offset: u64, ) { assert(executions_left > 0); - self.internal.destroy_max_notes(owner); - self.internal.create_max_notes(owner); + self.internal.destroy_max_notes(recipients); + self.internal.create_max_notes(recipients, current_offset); let executions_left = executions_left - 1; if executions_left > 0 { - self.call_self.recursively_destroy_and_create_notes(owner, sender, executions_left); + let max_notes = self.internal.max_notes_per_call() as u64; + self.call_self.recursively_destroy_and_create_notes( + recipients, + executions_left, + current_offset + max_notes, + ); } } #[internal("private")] - fn create_max_notes(owner: AztecAddress) { - let owner_balance = self.storage.balances.at(owner); - + fn create_max_notes(recipients: [AztecAddress; 10], offset: u64) { + // Distribute notes across recipients using global offset to ensure + // no recipient receives more than 10 notes (UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN) for i in 0..self.internal.max_notes_per_call() { + let global_index = offset + i as u64; + let recipient_index = (global_index % 10) as u32; + let recipient = recipients[recipient_index]; + let recipient_balance = self.storage.balances.at(recipient); + let note = FieldNote { value: i as Field }; - owner_balance.insert(note).deliver(MessageDelivery.CONSTRAINED_ONCHAIN); + recipient_balance.insert(note).deliver(MessageDelivery.CONSTRAINED_ONCHAIN); } } #[internal("private")] - fn destroy_max_notes(owner: AztecAddress) { - let owner_balance = self.storage.balances.at(owner); - // Note that we're relying on PXE actually returning the notes, we're not constraining that any specific - // number of notes are deleted. - let _ = owner_balance.pop_notes(NoteGetterOptions::new()); + fn destroy_max_notes(recipients: [AztecAddress; 10]) { + // Pop notes from all recipients + for i in 0..10 { + let recipient = recipients[i]; + let recipient_balance = self.storage.balances.at(recipient); + // Note that we're relying on PXE actually returning the notes, we're not constraining that any specific + // number of notes are deleted. + let _ = recipient_balance.pop_notes(NoteGetterOptions::new()); + } } #[internal("private")] diff --git a/yarn-project/end-to-end/src/bench/bench_build_block.test.ts b/yarn-project/end-to-end/src/bench/bench_build_block.test.ts index 6494507fef0c..e07f11eed7f9 100644 --- a/yarn-project/end-to-end/src/bench/bench_build_block.test.ts +++ b/yarn-project/end-to-end/src/bench/bench_build_block.test.ts @@ -35,14 +35,14 @@ describe('benchmarks/build_block', () => { const TX_COUNT = 32; it(`builds a block with ${TX_COUNT} standard txs`, async () => { sequencer.updateConfig({ minTxsPerBlock: TX_COUNT }); - const sentTxs = sendTxs(TX_COUNT, context, contract); + const sentTxs = await sendTxs(TX_COUNT, context, contract); await waitTxs(sentTxs, context); }); const TX_COUNT_HEAVY_COMPUTE = 8; it(`builds a block with ${TX_COUNT_HEAVY_COMPUTE} compute-heavy txs`, async () => { sequencer.updateConfig({ minTxsPerBlock: TX_COUNT_HEAVY_COMPUTE }); - const sentTxs = sendTxs(TX_COUNT_HEAVY_COMPUTE, context, contract, /*heavyPublicComput=*/ true); + const sentTxs = await sendTxs(TX_COUNT_HEAVY_COMPUTE, context, contract, /*heavyPublicComput=*/ true); await waitTxs(sentTxs, context); }); }); diff --git a/yarn-project/end-to-end/src/bench/utils.ts b/yarn-project/end-to-end/src/bench/utils.ts index 2fa4a0759941..37c0bff594f1 100644 --- a/yarn-project/end-to-end/src/bench/utils.ts +++ b/yarn-project/end-to-end/src/bench/utils.ts @@ -1,4 +1,5 @@ import type { AztecNodeService } from '@aztec/aztec-node'; +import { AztecAddress } from '@aztec/aztec.js/addresses'; import { BatchCall, type SentTx, type WaitOpts } from '@aztec/aztec.js/contracts'; import { mean, stdDev, times } from '@aztec/foundation/collection'; import { BenchmarkingContract } from '@aztec/noir-test-contracts.js/Benchmarking'; @@ -103,19 +104,22 @@ function getMetricValues(points: BenchmarkDataPoint[]) { * @param heavyPublicCompute - Whether the transactions include heavy public compute (like a big sha256). * @returns A BatchCall instance. */ -function makeCall( +async function makeCall( index: number, context: EndToEndContext, contract: BenchmarkingContract, heavyPublicCompute: boolean, ) { - const [owner] = context.accounts; if (heavyPublicCompute) { return new BatchCall(context.wallet, [contract.methods.sha256_hash_1024(randomBytesAsBigInts(1024))]); } else { + // We use random address for the new note owner because we can emit at most UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN + // logs for a given sender-recipient-contract tuple. + const ownerOfNewNote = await AztecAddress.random(); + const [ownerOfBalance] = context.accounts; return new BatchCall(context.wallet, [ - contract.methods.create_note(owner, index + 1), - contract.methods.increment_balance(owner, index + 1), + contract.methods.create_note(ownerOfNewNote, index + 1), + contract.methods.increment_balance(ownerOfBalance, index + 1), ]); } } @@ -129,13 +133,13 @@ function makeCall( * @param heavyPublicCompute - Whether the transactions include heavy public compute (like a big sha256). * @returns Array of sent txs. */ -export function sendTxs( +export async function sendTxs( txCount: number, context: EndToEndContext, contract: BenchmarkingContract, heavyPublicCompute: boolean = false, -): SentTx[] { - const calls = times(txCount, index => makeCall(index, context, contract, heavyPublicCompute)); +): Promise { + const calls = await Promise.all(times(txCount, index => makeCall(index, context, contract, heavyPublicCompute))); context.logger.info(`Creating ${txCount} txs`); const [from] = context.accounts; context.logger.info(`Sending ${txCount} txs`); diff --git a/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts b/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts index 5ecbc10e4505..ff203c43ca40 100644 --- a/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts @@ -1,5 +1,5 @@ -import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; +import { AztecAddress } from '@aztec/aztec.js/addresses'; +import { Fr, GrumpkinScalar } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; import { @@ -289,12 +289,23 @@ describe('e2e_pending_note_hashes_contract', () => { it('Should handle overflowing the kernel data structures in nested calls', async () => { // This test verifies that a transaction can emit more notes than MAX_NOTE_HASHES_PER_TX without failing, since // the notes are nullified and will be squashed by the kernel reset circuit. - const sender = owner; + const notesPerIteration = Math.min(MAX_NOTE_HASHES_PER_CALL, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL); const minToNeedReset = Math.min(MAX_NOTE_HASHES_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_TX) + 1; const deployedContract = await deployContract(); + + // We use 10 different recipients to send private logs to in order to avoid exceeding + // UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN logs emitted for any single sender-recipient pair. + const recipients = ( + await Promise.all( + Array.from({ length: 10 }, () => + wallet.createSchnorrAccount(Fr.random(), Fr.random(), GrumpkinScalar.random()), + ), + ) + ).map(a => a.address); + await deployedContract.methods - .test_recursively_create_notes(owner, sender, Math.ceil(minToNeedReset / notesPerIteration)) + .test_recursively_create_notes(recipients, Math.ceil(minToNeedReset / notesPerIteration)) .send({ from: owner }) .wait(); }); diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts index ba68a92e6bc9..83a86b48984b 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts @@ -11,7 +11,6 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js'; import { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js'; -import { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js'; import { BlockSynchronizer } from './block_synchronizer.js'; describe('BlockSynchronizer', () => { @@ -19,7 +18,6 @@ describe('BlockSynchronizer', () => { let tipsStore: L2TipsKVStore; let anchorBlockDataProvider: AnchorBlockDataProvider; let noteDataProvider: NoteDataProvider; - let recipientTaggingDataProvider: RecipientTaggingDataProvider; let aztecNode: MockProxy; let blockStream: MockProxy; @@ -36,14 +34,7 @@ describe('BlockSynchronizer', () => { tipsStore = new L2TipsKVStore(store, 'pxe'); anchorBlockDataProvider = new AnchorBlockDataProvider(store); noteDataProvider = await NoteDataProvider.create(store); - recipientTaggingDataProvider = new RecipientTaggingDataProvider(store); - synchronizer = new TestSynchronizer( - aztecNode, - anchorBlockDataProvider, - noteDataProvider, - recipientTaggingDataProvider, - tipsStore, - ); + synchronizer = new TestSynchronizer(aztecNode, anchorBlockDataProvider, noteDataProvider, tipsStore); }); it('sets header from latest block', async () => { @@ -58,9 +49,6 @@ describe('BlockSynchronizer', () => { const rollbackNotesAndNullifiers = jest .spyOn(noteDataProvider, 'rollbackNotesAndNullifiers') .mockImplementation(() => Promise.resolve()); - const resetNoteSyncData = jest - .spyOn(recipientTaggingDataProvider, 'resetNoteSyncData') - .mockImplementation(() => Promise.resolve()); aztecNode.getBlockHeader.mockImplementation(async blockNumber => (await L2Block.random(BlockNumber(blockNumber as number))).getBlockHeader(), ); @@ -72,6 +60,5 @@ describe('BlockSynchronizer', () => { await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', block: { number: BlockNumber(3), hash: '0x3' } }); expect(rollbackNotesAndNullifiers).toHaveBeenCalledWith(3, 4); - expect(resetNoteSyncData).toHaveBeenCalled(); }); }); diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts index 3649511b59b0..c6d1f5892799 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts @@ -7,7 +7,6 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { PXEConfig } from '../config/index.js'; import type { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js'; import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js'; /** * The BlockSynchronizer class orchestrates synchronization between PXE and Aztec node, maintaining an up-to-date @@ -23,7 +22,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { private node: AztecNode, private anchorBlockDataProvider: AnchorBlockDataProvider, private noteDataProvider: NoteDataProvider, - private recipientTaggingDataProvider: RecipientTaggingDataProvider, private l2TipsStore: L2TipsKVStore, config: Partial> = {}, loggerOrSuffix?: string | Logger, @@ -64,12 +62,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted. const lastSynchedBlockNumber = (await this.anchorBlockDataProvider.getBlockHeader()).getBlockNumber(); await this.noteDataProvider.rollbackNotesAndNullifiers(event.block.number, lastSynchedBlockNumber); - // Remove all note tagging indexes to force a full resync. This is suboptimal, but unless we track the - // block number in which each index is used it's all we can do. - // Note: This is now unnecessary for the sender tagging data provider because the new algorithm handles reorgs. - // TODO(#17775): Once this issue is implemented we will have the index-block number mapping, so we can - // implement this more intelligently. - await this.recipientTaggingDataProvider.resetNoteSyncData(); // Update the header to the last block. const newHeader = await this.node.getBlockHeader(event.block.number); if (!newHeader) { diff --git a/yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts b/yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts index 62fb61d86dbb..8eb077f4adaf 100644 --- a/yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts +++ b/yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts @@ -77,8 +77,9 @@ import type { CapsuleDataProvider } from '../storage/capsule_data_provider/capsu import type { ContractDataProvider } from '../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import type { SenderAddressBook } from '../storage/tagging_data_provider/sender_address_book.js'; import type { SenderTaggingDataProvider } from '../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from '../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { ExecutionNoteCache } from './execution_note_cache.js'; import { ExecutionTaggingIndexCache } from './execution_tagging_index_cache.js'; import { HashedValuesCache } from './hashed_values_cache.js'; @@ -103,6 +104,7 @@ export class ContractFunctionSimulator { private anchorBlockDataProvider: AnchorBlockDataProvider, private senderTaggingDataProvider: SenderTaggingDataProvider, private recipientTaggingDataProvider: RecipientTaggingDataProvider, + private senderAddressBook: SenderAddressBook, private capsuleDataProvider: CapsuleDataProvider, private privateEventDataProvider: PrivateEventDataProvider, private simulator: CircuitSimulator, @@ -183,6 +185,7 @@ export class ContractFunctionSimulator { this.anchorBlockDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, 0, // totalPublicArgsCount @@ -278,8 +281,8 @@ export class ContractFunctionSimulator { this.addressDataProvider, this.aztecNode, this.anchorBlockDataProvider, - this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, undefined, diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts index 6259860fc432..116d972d4e90 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts @@ -18,8 +18,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js'; import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { ContractFunctionSimulator } from '../contract_function_simulator.js'; import { UtilityExecutionOracle } from './utility_execution_oracle.js'; @@ -34,6 +35,7 @@ describe('Oracle Version Check test suite', () => { let anchorBlockDataProvider: ReturnType>; let senderTaggingDataProvider: ReturnType>; let recipientTaggingDataProvider: ReturnType>; + let senderAddressBook: ReturnType>; let capsuleDataProvider: ReturnType>; let privateEventDataProvider: ReturnType>; let acirSimulator: ContractFunctionSimulator; @@ -52,6 +54,7 @@ describe('Oracle Version Check test suite', () => { anchorBlockDataProvider = mock(); senderTaggingDataProvider = mock(); recipientTaggingDataProvider = mock(); + senderAddressBook = mock(); capsuleDataProvider = mock(); privateEventDataProvider = mock(); utilityAssertCompatibleOracleVersionSpy = jest.spyOn( @@ -69,10 +72,7 @@ describe('Oracle Version Check test suite', () => { senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined); senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]); senderTaggingDataProvider.storePendingIndexes.mockResolvedValue(); - recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]); - recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets => - Promise.resolve(secrets.map(() => undefined)), - ); + noteDataProvider.getNotes.mockResolvedValue([]); keyStore.getAccounts.mockResolvedValue([]); @@ -100,6 +100,7 @@ describe('Oracle Version Check test suite', () => { anchorBlockDataProvider, senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, simulator, diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts index d1817edd5e88..e8fc3370d83a 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts @@ -70,8 +70,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js'; import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { ContractFunctionSimulator } from '../contract_function_simulator.js'; jest.setTimeout(60_000); @@ -116,12 +117,12 @@ describe('Private Execution test suite', () => { let keyStore: MockProxy; let senderTaggingDataProvider: MockProxy; let recipientTaggingDataProvider: MockProxy; + let senderAddressBook: MockProxy; let aztecNode: MockProxy; let anchorBlockDataProvider: MockProxy; let capsuleDataProvider: MockProxy; let privateEventDataProvider: MockProxy; let acirSimulator: ContractFunctionSimulator; - let anchorBlockHeader = BlockHeader.empty(); let logger: Logger; @@ -310,6 +311,7 @@ describe('Private Execution test suite', () => { anchorBlockDataProvider = mock(); capsuleDataProvider = mock(); privateEventDataProvider = mock(); + senderAddressBook = mock(); contracts = {}; anchorBlockHeader = makeBlockHeader(); anchorBlockDataProvider.getBlockHeader.mockImplementation(() => Promise.resolve(anchorBlockHeader)); @@ -320,15 +322,24 @@ describe('Private Execution test suite', () => { senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined); senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]); senderTaggingDataProvider.storePendingIndexes.mockResolvedValue(); - recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]); - recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets => - Promise.resolve(new Array(secrets?.length ?? 0).fill(undefined)), - ); + + senderAddressBook.getSenders.mockResolvedValue([]); // Mock aztec node methods - the return array needs to have the same length as the number of tags // on the input. aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => Promise.resolve(tags.map(() => []))); + // Mock getL2Tips and getBlockHeader for loadPrivateLogsForSenderRecipientPair + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: anchorBlockHeader.globalVariables.blockNumber }, + } as any); + aztecNode.getBlockHeader.mockImplementation((blockNumber: BlockNumber | 'latest') => { + if (blockNumber === 'latest') { + return Promise.resolve(anchorBlockHeader); + } + return Promise.resolve(anchorBlockHeader); + }); + // TODO: refactor. Maybe it's worth stubbing a key store // and cleaning up the mess that is setting up keys. // Also: having owner, recipient, and sender for tags @@ -434,6 +445,7 @@ describe('Private Execution test suite', () => { anchorBlockDataProvider, senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, simulator, diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts index 36899c72c6aa..876f516bceb2 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution_oracle.ts @@ -36,8 +36,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js'; import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { syncSenderTaggingIndexes } from '../../tagging/sync/sync_sender_tagging_indexes.js'; import type { ExecutionNoteCache } from '../execution_note_cache.js'; import { ExecutionTaggingIndexCache } from '../execution_tagging_index_cache.js'; @@ -95,8 +96,9 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP addressDataProvider: AddressDataProvider, aztecNode: AztecNode, anchorBlockDataProvider: AnchorBlockDataProvider, - senderTaggingDataProvider: SenderTaggingDataProvider, + private readonly senderTaggingDataProvider: SenderTaggingDataProvider, recipientTaggingDataProvider: RecipientTaggingDataProvider, + senderAddressBook: SenderAddressBook, capsuleDataProvider: CapsuleDataProvider, privateEventDataProvider: PrivateEventDataProvider, private totalPublicCalldataCount: number = 0, @@ -117,8 +119,8 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP addressDataProvider, aztecNode, anchorBlockDataProvider, - senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, log, @@ -580,6 +582,7 @@ export class PrivateExecutionOracle extends UtilityExecutionOracle implements IP this.anchorBlockDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, this.totalPublicCalldataCount, diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts index 698f50f307ca..623e24918e10 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts @@ -1,5 +1,6 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; +import { GrumpkinScalar } from '@aztec/foundation/curves/grumpkin'; import type { KeyStore } from '@aztec/key-store'; import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; import { WASMSimulator } from '@aztec/simulator/client'; @@ -8,6 +9,7 @@ import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { L2BlockHash } from '@aztec/stdlib/block'; import { CompleteAddress, type ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { deriveKeys } from '@aztec/stdlib/keys'; import { Note, NoteDao } from '@aztec/stdlib/note'; import { BlockHeader, GlobalVariables, TxHash } from '@aztec/stdlib/tx'; @@ -20,8 +22,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js'; import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { ContractFunctionSimulator } from '../contract_function_simulator.js'; import { UtilityExecutionOracle } from './utility_execution_oracle.js'; @@ -36,6 +39,7 @@ describe('Utility Execution test suite', () => { let anchorBlockDataProvider: ReturnType>; let senderTaggingDataProvider: ReturnType>; let recipientTaggingDataProvider: ReturnType>; + let senderAddressBook: ReturnType>; let capsuleDataProvider: ReturnType>; let privateEventDataProvider: ReturnType>; let acirSimulator: ContractFunctionSimulator; @@ -57,6 +61,7 @@ describe('Utility Execution test suite', () => { anchorBlockDataProvider = mock(); senderTaggingDataProvider = mock(); recipientTaggingDataProvider = mock(); + senderAddressBook = mock(); capsuleDataProvider = mock(); privateEventDataProvider = mock(); const capsuleArrays = new Map(); @@ -66,10 +71,20 @@ describe('Utility Execution test suite', () => { senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined); senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]); senderTaggingDataProvider.storePendingIndexes.mockResolvedValue(); - recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]); - recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets => - Promise.resolve(secrets.map(() => undefined)), - ); + senderAddressBook.getSenders.mockResolvedValue([]); + + // Mock getL2Tips and getBlockHeader for loadPrivateLogsForSenderRecipientPair + aztecNode.getL2Tips.mockResolvedValue({ + finalized: { number: anchorBlockHeader.globalVariables.blockNumber }, + } as any); + aztecNode.getBlockHeader.mockImplementation((blockNumber: BlockNumber | 'latest') => { + if (blockNumber === 'latest') { + return Promise.resolve(anchorBlockHeader); + } + return Promise.resolve(anchorBlockHeader); + }); + aztecNode.getPrivateLogsByTags.mockImplementation((tags: any[]) => Promise.resolve(tags.map(() => []))); + capsuleDataProvider.setCapsuleArray.mockImplementation((address, slot, content) => { capsuleArrays.set(`${address.toString()}:${slot.toString()}`, content); return Promise.resolve(); @@ -86,15 +101,31 @@ describe('Utility Execution test suite', () => { anchorBlockDataProvider, senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, simulator, ); - ownerCompleteAddress = await CompleteAddress.fromSecretKeyAndPartialAddress(ownerSecretKey, Fr.random()); + const ownerPartialAddress = Fr.random(); + ownerCompleteAddress = await CompleteAddress.fromSecretKeyAndPartialAddress(ownerSecretKey, ownerPartialAddress); owner = ownerCompleteAddress.address; + + // Derive keys to get the incoming viewing secret key + const { masterIncomingViewingSecretKey: ownerIvskM } = await deriveKeys(ownerSecretKey); + keyStore.getAccounts.mockResolvedValue([owner]); + // Mock getMasterIncomingViewingSecretKey to return a valid scalar + // This is needed when LogService tries to compute directional app tagging secrets + keyStore.getMasterIncomingViewingSecretKey.mockImplementation((address: AztecAddress) => { + if (address.equals(owner)) { + return Promise.resolve(ownerIvskM); + } + // Return a default value for any other address + return Promise.resolve(GrumpkinScalar.random()); + }); + addressDataProvider.getCompleteAddress.mockImplementation((account: AztecAddress) => { if (account.equals(owner)) { return Promise.resolve(ownerCompleteAddress); @@ -187,8 +218,8 @@ describe('Utility Execution test suite', () => { addressDataProvider, aztecNode, anchorBlockDataProvider, - senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, ); diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts index 3d381ba9d76f..ee4785f45333 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution_oracle.ts @@ -27,8 +27,8 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js'; import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js'; import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js'; -import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js'; -import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js'; +import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js'; +import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { TreeMembershipService } from '../../tree_membership/tree_membership_service.js'; import { EventValidationRequest } from '../noir-structs/event_validation_request.js'; import { LogRetrievalRequest } from '../noir-structs/log_retrieval_request.js'; @@ -60,8 +60,8 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra protected readonly addressDataProvider: AddressDataProvider, protected readonly aztecNode: AztecNode, protected readonly anchorBlockDataProvider: AnchorBlockDataProvider, - protected readonly senderTaggingDataProvider: SenderTaggingDataProvider, protected readonly recipientTaggingDataProvider: RecipientTaggingDataProvider, + protected readonly senderAddressBook: SenderAddressBook, protected readonly capsuleDataProvider: CapsuleDataProvider, protected readonly privateEventDataProvider: PrivateEventDataProvider, protected log = createLogger('simulator:client_view_context'), @@ -351,6 +351,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra this.keyStore, this.capsuleDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.addressDataProvider, ); @@ -447,6 +448,7 @@ export class UtilityExecutionOracle implements IMiscOracle, IUtilityExecutionOra this.keyStore, this.capsuleDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.addressDataProvider, ); diff --git a/yarn-project/pxe/src/logs/log_service.test.ts b/yarn-project/pxe/src/logs/log_service.test.ts index 573220886d98..2443333b0e0e 100644 --- a/yarn-project/pxe/src/logs/log_service.test.ts +++ b/yarn-project/pxe/src/logs/log_service.test.ts @@ -1,16 +1,13 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; -import { timesParallel } from '@aztec/foundation/collection'; import { randomInt } from '@aztec/foundation/crypto/random'; -import { type Fq, Fr } from '@aztec/foundation/curves/bn254'; +import { Fr } from '@aztec/foundation/curves/bn254'; import { KeyStore } from '@aztec/key-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { L2BlockHash, randomDataInBlock } from '@aztec/stdlib/block'; -import { CompleteAddress } from '@aztec/stdlib/contract'; +import { L2BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; -import { computeAddress, deriveKeys } from '@aztec/stdlib/keys'; -import { DirectionalAppTaggingSecret, PrivateLog, PublicLog, SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs'; -import { BlockHeader, GlobalVariables, TxEffect, TxHash, randomIndexedTxEffect } from '@aztec/stdlib/tx'; +import { PublicLog, Tag, TxScopedL2Log } from '@aztec/stdlib/logs'; +import { TxHash, randomIndexedTxEffect } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -18,420 +15,46 @@ import { LogRetrievalRequest } from '../contract_function_simulator/noir-structs import { AddressDataProvider } from '../storage/address_data_provider/address_data_provider.js'; import { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js'; import { CapsuleDataProvider } from '../storage/capsule_data_provider/capsule_data_provider.js'; -import { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js'; -import { WINDOW_HALF_SIZE } from '../tagging/constants.js'; +import { SenderAddressBook } from '../storage/tagging_data_provider/sender_address_book.js'; +import { RecipientTaggingDataProvider } from '../tagging/recipient_sync/recipient_tagging_data_provider.js'; import { LogService } from './log_service.js'; -async function computeSiloedTagForIndex( - sender: { completeAddress: CompleteAddress; ivsk: Fq }, - recipient: AztecAddress, - contractAddress: AztecAddress, - index: number, -) { - const secret = await DirectionalAppTaggingSecret.compute( - sender.completeAddress, - sender.ivsk, - recipient, - contractAddress, - recipient, - ); - const tag = await Tag.compute({ secret, index }); - return SiloedTag.compute(tag, contractAddress); -} - describe('LogService', () => { let contractAddress: AztecAddress; - let recipient: CompleteAddress; - let keyStore: KeyStore; let aztecNode: MockProxy; - let recipientTaggingDataProvider: RecipientTaggingDataProvider; let anchorBlockDataProvider: AnchorBlockDataProvider; + let keyStore: KeyStore; let capsuleDataProvider: CapsuleDataProvider; + let recipientTaggingDataProvider: RecipientTaggingDataProvider; let addressDataProvider: AddressDataProvider; - + let senderAddressBook: SenderAddressBook; let logService: LogService; - describe('sync tagged logs', () => { - const NUM_SENDERS = 10; - - let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[]; - - // The block number of the first log to be emitted. - const MIN_BLOCK_NUMBER_OF_A_LOG = BlockNumber(1); - // The block number of the last log to be emitted. - const MAX_BLOCK_NUMBER_OF_A_LOG = BlockNumber(3); - - const setSyncedBlockNumber = (blockNumber: BlockNumber) => { - return anchorBlockDataProvider.setHeader( - BlockHeader.empty({ - globalVariables: GlobalVariables.empty({ blockNumber }), - }), - ); - }; - - async function generateMockLogs(tagIndex: number) { - const logs: { [k: string]: TxScopedL2Log[] } = {}; - - // Add a random note from every address in the address book for our account with index tagIndex - // Compute the tag as sender (knowledge of preaddress and ivsk) - for (const sender of senders) { - const tag = await computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex); - const log = new TxScopedL2Log( - TxHash.random(), - 0, - 0, - MIN_BLOCK_NUMBER_OF_A_LOG, - L2BlockHash.random(), - 0n, - PrivateLog.random(tag.value), - ); - logs[tag.toString()] = [log]; - } - // Accumulated logs intended for recipient: NUM_SENDERS - - // Add a random note from the first sender in the address book, repeating the tag - // Compute the tag as sender (knowledge of preaddress and ivsk) - const firstSender = senders[0]; - const tag = await computeSiloedTagForIndex(firstSender, recipient.address, contractAddress, tagIndex); - const log = new TxScopedL2Log( - TxHash.random(), - 1, - 0, - BlockNumber.ZERO, - L2BlockHash.random(), - 0n, - PrivateLog.random(tag.value), - ); - logs[tag.toString()].push(log); - // Accumulated logs intended for recipient: NUM_SENDERS + 1 - - // Add a random note from half the address book for our account with index tagIndex + 1 - // Compute the tag as sender (knowledge of preaddress and ivsk) - for (let i = NUM_SENDERS / 2; i < NUM_SENDERS; i++) { - const sender = senders[i]; - const tag = await computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex + 1); - const blockNumber = BlockNumber(2); - const log = new TxScopedL2Log( - TxHash.random(), - 0, - 0, - blockNumber, - L2BlockHash.random(), - 0n, - PrivateLog.random(tag.value), - ); - logs[tag.toString()] = [log]; - } - // Accumulated logs intended for recipient: NUM_SENDERS + 1 + NUM_SENDERS / 2 - - // Add a random note from every address in the address book for a random recipient with index tagIndex - // Compute the tag as sender (knowledge of preaddress and ivsk) - for (const sender of senders) { - const keys = await deriveKeys(Fr.random()); - const partialAddress = Fr.random(); - const randomRecipient = await computeAddress(keys.publicKeys, partialAddress); - const tag = await computeSiloedTagForIndex(sender, randomRecipient, contractAddress, tagIndex); - const log = new TxScopedL2Log( - TxHash.random(), - 0, - 0, - MAX_BLOCK_NUMBER_OF_A_LOG, - L2BlockHash.random(), - 0n, - PrivateLog.random(tag.value), - ); - logs[tag.toString()] = [log]; - } - // Accumulated logs intended for recipient: NUM_SENDERS + 1 + NUM_SENDERS / 2 - - // Set up the getPrivateLogsByTags mock - aztecNode.getPrivateLogsByTags.mockImplementation(tags => { - return Promise.resolve(tags.map(tag => logs[tag.toString()] ?? [])); - }); - } - - // Set to a random value in this test we don't care about Noir loading the logs from the capsule array. - const PENDING_TAGGED_LOG_ARRAY_BASE_SLOT = Fr.random(); + describe('bulkRetrieveLogs', () => { + const tag = new Tag(Fr.random()); beforeEach(async () => { // Set up contract address contractAddress = await AztecAddress.random(); + anchorBlockDataProvider = new AnchorBlockDataProvider(await openTmpStore('test')); + keyStore = new KeyStore(await openTmpStore('test')); + capsuleDataProvider = new CapsuleDataProvider(await openTmpStore('test')); + recipientTaggingDataProvider = new RecipientTaggingDataProvider(await openTmpStore('test')); + senderAddressBook = new SenderAddressBook(await openTmpStore('test')); + addressDataProvider = new AddressDataProvider(await openTmpStore('test')); aztecNode = mock(); - const store = await openTmpStore('test'); - - keyStore = new KeyStore(store); - recipientTaggingDataProvider = new RecipientTaggingDataProvider(store); - anchorBlockDataProvider = new AnchorBlockDataProvider(store); - capsuleDataProvider = new CapsuleDataProvider(store); - addressDataProvider = new AddressDataProvider(store); - await setSyncedBlockNumber(MAX_BLOCK_NUMBER_OF_A_LOG); - - // Set up recipient account - recipient = await keyStore.addAccount(new Fr(69), Fr.random()); - await addressDataProvider.addCompleteAddress(recipient); - // Set up the address book - senders = await timesParallel(NUM_SENDERS, async index => { - const keys = await deriveKeys(new Fr(index)); - const partialAddress = Fr.random(); - const address = await computeAddress(keys.publicKeys, partialAddress); - const completeAddress = await CompleteAddress.create(address, keys.publicKeys, partialAddress); - return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey, secretKey: new Fr(index) }; - }); - for (const sender of senders) { - await recipientTaggingDataProvider.addSenderAddress(sender.completeAddress.address); - } - aztecNode.getPrivateLogsByTags.mockReset(); - aztecNode.getTxEffect.mockResolvedValue({ - ...randomDataInBlock(await TxEffect.random({ numNullifiers: 1 })), - txIndexInBlock: 0, - }); - logService = new LogService( aztecNode, anchorBlockDataProvider, keyStore, capsuleDataProvider, recipientTaggingDataProvider, + senderAddressBook, addressDataProvider, ); - }); - - it('should sync tagged logs', async () => { - const tagIndex = 0; - await generateMockLogs(tagIndex); - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // We expect to have all logs intended for the recipient synced (and hence stored in the capsule for later - // processing), one per sender + 1 with a duplicated tag for the first sender + half of the logs for the second - // index - await expectPendingTaggedLogArrayLengthToBe(contractAddress, NUM_SENDERS + 1 + NUM_SENDERS / 2); - - // Recompute the secrets (as recipient) to ensure indexes are updated - const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); - const secrets = await Promise.all( - senders.map(sender => - DirectionalAppTaggingSecret.compute( - recipient, - ivsk, - sender.completeAddress.address, - contractAddress, - recipient.address, - ), - ), - ); - - // First sender should have 2 logs, but keep index 0 since they were built using the same tag - // Next 4 senders should also have index 0 = offset + 0 - // Last 5 senders should have index 1 = offset + 1 - const indexes = await recipientTaggingDataProvider.getLastUsedIndexes(secrets); - - expect(indexes).toHaveLength(NUM_SENDERS); - expect(indexes).toEqual([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]); - - // We should have called the node 2 times: - // 2 times: first time during initial request, second time after pushing the edge of the window once - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(2); - }); - - it('should sync tagged logs with a sender index offset', async () => { - const tagIndex = 5; - await generateMockLogs(tagIndex); - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first - // one + half of the logs for the second index - await expectPendingTaggedLogArrayLengthToBe(contractAddress, NUM_SENDERS + 1 + NUM_SENDERS / 2); - - // Recompute the secrets (as recipient) to ensure indexes are updated - const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); - const secrets = await Promise.all( - senders.map(sender => - DirectionalAppTaggingSecret.compute( - recipient, - ivsk, - sender.completeAddress.address, - contractAddress, - recipient.address, - ), - ), - ); - - // First sender should have 2 logs, but keep index 5 since they were built using the same tag - // Next 4 senders should also have index 5 = offset - // Last 5 senders should have index 6 = offset + 1 - const indexes = await recipientTaggingDataProvider.getLastUsedIndexes(secrets); - - expect(indexes).toHaveLength(NUM_SENDERS); - expect(indexes).toEqual([5, 5, 5, 5, 5, 6, 6, 6, 6, 6]); - - // We should have called the node 2 times: - // 2 times: first time during initial request, second time after pushing the edge of the window once - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(2); - }); - - it("should sync tagged logs for which indexes are not updated if they're inside the window", async () => { - const tagIndex = 1; - await generateMockLogs(tagIndex); - - // Recompute the secrets (as recipient) to update indexes - const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); - const secrets = await Promise.all( - senders.map(sender => - DirectionalAppTaggingSecret.compute( - recipient, - ivsk, - sender.completeAddress.address, - contractAddress, - recipient.address, - ), - ), - ); - - // Set last used indexes to 1 (so next scan starts at 2) - await recipientTaggingDataProvider.setLastUsedIndexes(secrets.map(secret => ({ secret, index: 1 }))); - - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - // Even if our index as recipient is higher than what the sender sent, we should be able to find the logs - // since the window starts at Math.max(0, 2 - window_size) = 0 - await expectPendingTaggedLogArrayLengthToBe(contractAddress, NUM_SENDERS + 1 + NUM_SENDERS / 2); - - // First sender should have 2 logs, but keep index 1 since they were built using the same tag - // Next 4 senders should also have index 1 = tagIndex - // Last 5 senders should have index 2 = tagIndex + 1 - const indexes = await recipientTaggingDataProvider.getLastUsedIndexes(secrets); - - expect(indexes).toHaveLength(NUM_SENDERS); - expect(indexes).toEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - - // We should have called the node 2 times: - // first time during initial request, second time after pushing the edge of the window once - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(2); - }); - - it("should not sync tagged logs for which indexes are not updated if they're outside the window", async () => { - const tagIndex = 0; - await generateMockLogs(tagIndex); - - // Recompute the secrets (as recipient) to update indexes - const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); - const secrets = await Promise.all( - senders.map(sender => - DirectionalAppTaggingSecret.compute( - recipient, - ivsk, - sender.completeAddress.address, - contractAddress, - recipient.address, - ), - ), - ); - - // We set the last used indexes to WINDOW_HALF_SIZE so that next scan starts at WINDOW_HALF_SIZE + 1, - // which is outside the window, and for this reason no updates should be triggered. - const index = WINDOW_HALF_SIZE + 1; - await recipientTaggingDataProvider.setLastUsedIndexes(secrets.map(secret => ({ secret, index }))); - - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // Only half of the logs should be synced since we start from index 1 = (11 - window_size), the other half should - // be skipped - await expectPendingTaggedLogArrayLengthToBe(contractAddress, NUM_SENDERS / 2); - - // Indexes should remain where we set them (window_size) - const indexes = await recipientTaggingDataProvider.getLastUsedIndexes(secrets); - - expect(indexes).toHaveLength(NUM_SENDERS); - expect(indexes).toEqual([index, index, index, index, index, index, index, index, index, index]); - - // We should have called the node once and that is only for the first window - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(1); - }); - - it('should sync tagged logs from scratch after a DB wipe', async () => { - const tagIndex = 0; - await generateMockLogs(tagIndex); - - // Recompute the secrets (as recipient) to update indexes - const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); - const secrets = await Promise.all( - senders.map(sender => - DirectionalAppTaggingSecret.compute( - recipient, - ivsk, - sender.completeAddress.address, - contractAddress, - recipient.address, - ), - ), - ); - - await recipientTaggingDataProvider.setLastUsedIndexes( - secrets.map(secret => ({ secret, index: WINDOW_HALF_SIZE + 2 })), - ); - - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // No logs should be synced (and hence no capsules stored) since we start from index 2 = 12 - window_size - await expectPendingTaggedLogArrayLengthToBe(contractAddress, 0); - - // Since no logs were synced, window edge hash not been pushed and for this reason we should have called - // the node only once for the initial window - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(1); - - aztecNode.getPrivateLogsByTags.mockClear(); - - // Wipe the database - await recipientTaggingDataProvider.resetNoteSyncData(); - - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // First sender should have 2 logs, but keep index 0 since they were built using the same tag - // Next 4 senders should also have index 0 = offset - // Last 5 senders should have index 1 = offset + 1 - const indexes = await recipientTaggingDataProvider.getLastUsedIndexes(secrets); - - expect(indexes).toHaveLength(NUM_SENDERS); - expect(indexes).toEqual([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]); - - // We should have called the node 2 times: - // first time during initial request, second time after pushing the edge of the window once - expect(aztecNode.getPrivateLogsByTags.mock.calls.length).toBe(2); - }); - - it('should not sync tagged logs with a blockNumber larger than the block number to which PXE is synced', async () => { - // We set the block number to which PXE is synced to a block number in which only the first batch of logs was - // emitted and then we check that we receive logs only from this batch. - await setSyncedBlockNumber(MIN_BLOCK_NUMBER_OF_A_LOG); - - const tagIndex = 0; - await generateMockLogs(tagIndex); - await logService.syncTaggedLogs(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - - // Only NUM_SENDERS + 1 logs should be synched, since the rest have blockNumber > 1 - await expectPendingTaggedLogArrayLengthToBe(contractAddress, NUM_SENDERS + 1); - }); - - const expectPendingTaggedLogArrayLengthToBe = async (contractAddress: AztecAddress, expectedLength: number) => { - // Capsule array length is stored in the array base slot. - const capsule = await capsuleDataProvider.loadCapsule(contractAddress, PENDING_TAGGED_LOG_ARRAY_BASE_SLOT); - if (expectedLength === 0 && capsule === null) { - // If expected length is 0 we are fine with the capsule not existing since the array might not have been - // initialized yet. - return; - } - expect(capsule).toBeDefined(); - expect(capsule!.length).toBe(1); - expect(capsule![0].toNumber()).toBe(expectedLength); - }; - }); - - describe('bulkRetrieveLogs', () => { - const tag = new Tag(Fr.random()); - - beforeEach(() => { aztecNode.getPrivateLogsByTags.mockReset(); aztecNode.getPublicLogsByTagsFromContract.mockReset(); aztecNode.getTxEffect.mockReset(); diff --git a/yarn-project/pxe/src/logs/log_service.ts b/yarn-project/pxe/src/logs/log_service.ts index c37785c72ffa..b23d610622d6 100644 --- a/yarn-project/pxe/src/logs/log_service.ts +++ b/yarn-project/pxe/src/logs/log_service.ts @@ -1,6 +1,7 @@ import type { Fr } from '@aztec/foundation/curves/bn254'; +import { createLogger } from '@aztec/foundation/log'; import type { KeyStore } from '@aztec/key-store'; -import type { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { CompleteAddress } from '@aztec/stdlib/contract'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { @@ -18,17 +19,20 @@ import { LogRetrievalResponse } from '../contract_function_simulator/noir-struct import { AddressDataProvider } from '../storage/address_data_provider/address_data_provider.js'; import { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js'; import { CapsuleDataProvider } from '../storage/capsule_data_provider/capsule_data_provider.js'; -import { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js'; -import { WINDOW_HALF_SIZE } from '../tagging/constants.js'; -import { getInitialIndexesMap, getPreTagsForTheWindow } from '../tagging/utils.js'; +import type { SenderAddressBook } from '../storage/tagging_data_provider/sender_address_book.js'; +import { loadPrivateLogsForSenderRecipientPair } from '../tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.js'; +import type { RecipientTaggingDataProvider } from '../tagging/recipient_sync/recipient_tagging_data_provider.js'; export class LogService { + private log = createLogger('log_service'); + constructor( private readonly aztecNode: AztecNode, private readonly anchorBlockDataProvider: AnchorBlockDataProvider, private readonly keyStore: KeyStore, private readonly capsuleDataProvider: CapsuleDataProvider, private readonly recipientTaggingDataProvider: RecipientTaggingDataProvider, + private readonly senderAddressBook: SenderAddressBook, private readonly addressDataProvider: AddressDataProvider, ) {} @@ -132,192 +136,77 @@ export class LogService { ); } - // TODO(#17775): Replace this implementation of this function with one implementing an approach similar - // to syncSenderTaggingIndexes. Not done yet due to re-prioritization to devex and this doesn't directly affect - // devex. public async syncTaggedLogs( contractAddress: AztecAddress, pendingTaggedLogArrayBaseSlot: Fr, scopes?: AztecAddress[], ) { - const maxBlockNumber = (await this.anchorBlockDataProvider.getBlockHeader()).getBlockNumber(); - - // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. - // However it is impossible at the moment due to the language not supporting nested slices. - // This nesting is necessary because for a given set of tags we don't - // know how many logs we will get back. Furthermore, these logs are of undetermined - // length, since we don't really know the note they correspond to until we decrypt them. - const recipients = scopes ? scopes : await this.keyStore.getAccounts(); - for (const recipient of recipients) { - // Get all the secrets for the recipient and sender pairs (#9365) - const indexedSecrets = await this.getLastUsedTaggingIndexesForSenders(contractAddress, recipient); - - // We fetch logs for a window of indexes in a range: - // . - // - // We use this window approach because it could happen that a sender might have messed up and inadvertently - // incremented their index without us getting any logs (for example, in case of a revert). If we stopped looking - // for logs the first time we don't receive any logs for a tag, we might never receive anything from that sender again. - // Also there's a possibility that we have advanced our index, but the sender has reused it, so we might have missed - // some logs. For these reasons, we have to look both back and ahead of the stored index. - let secretsAndWindows = indexedSecrets.map(indexedSecret => { - if (indexedSecret.index === undefined) { - return { - secret: indexedSecret.secret, - leftMostIndex: 0, - rightMostIndex: WINDOW_HALF_SIZE, - }; - } else { - return { - secret: indexedSecret.secret, - leftMostIndex: Math.max(0, indexedSecret.index - WINDOW_HALF_SIZE), - rightMostIndex: indexedSecret.index + WINDOW_HALF_SIZE, - }; - } - }); - - // As we iterate we store the largest index we have seen for a given secret to later on store it in the db. - const newLargestIndexMapToStore: { [k: string]: number } = {}; - - // The initial/unmodified indexes of the secrets stored in a key-value map where key is the directional app - // tagging secret. - const initialIndexesMap = getInitialIndexesMap(indexedSecrets); - - while (secretsAndWindows.length > 0) { - const preTagsForTheWholeWindow = getPreTagsForTheWindow(secretsAndWindows); - const tagsForTheWholeWindow = await Promise.all( - preTagsForTheWholeWindow.map(async preTag => { - return SiloedTag.compute(await Tag.compute(preTag), contractAddress); - }), + this.log.verbose('Searching for tagged logs', { contract: contractAddress }); + + // We only load logs from block up to and including the anchor block number + const anchorBlockNumber = (await this.anchorBlockDataProvider.getBlockHeader()).getBlockNumber(); + + // Determine recipients: use scopes if provided, otherwise get all accounts + const recipients = scopes && scopes.length > 0 ? scopes : await this.keyStore.getAccounts(); + + // For each recipient, fetch secrets, load logs, and store them. + // We run these per-recipient tasks in parallel so that logs are loaded for all recipients concurrently. + await Promise.all( + recipients.map(async recipient => { + // Get all secrets for this recipient (one per sender) + const secrets = await this.#getSecretsForSenders(contractAddress, recipient); + + // Load logs for all sender-recipient pairs in parallel + const logArrays = await Promise.all( + secrets.map(secret => + loadPrivateLogsForSenderRecipientPair( + secret, + contractAddress, + this.aztecNode, + this.recipientTaggingDataProvider, + anchorBlockNumber, + ), + ), ); - // We store the new largest indexes we find in the iteration in the following map to later on construct - // a new set of secrets and windows to fetch logs for. - const newLargestIndexMapForIteration: { [k: string]: number } = {}; - - // Fetch the private logs for the tags and iterate over them - const logsByTags = await this.aztecNode.getPrivateLogsByTags(tagsForTheWholeWindow); + // Flatten all logs from all secrets + const allLogs = logArrays.flat(); - for (let logIndex = 0; logIndex < logsByTags.length; logIndex++) { - const logsByTag = logsByTags[logIndex]; - if (logsByTag.length > 0) { - // We filter out the logs that are newer than the anchor block number of the tx currently being constructed - const filteredLogsByBlockNumber = logsByTag.filter(l => l.blockNumber <= maxBlockNumber); - - // We store the logs in capsules (to later be obtained in Noir) - await this.#storePendingTaggedLogs( - contractAddress, - pendingTaggedLogArrayBaseSlot, - recipient, - filteredLogsByBlockNumber, - ); - - // We retrieve the pre-tag corresponding to the log as I need that to evaluate whether - // a new largest index have been found. - const preTagCorrespondingToLog = preTagsForTheWholeWindow[logIndex]; - const initialIndex = initialIndexesMap[preTagCorrespondingToLog.secret.toString()]; - - if ( - preTagCorrespondingToLog.index >= initialIndex && - (newLargestIndexMapForIteration[preTagCorrespondingToLog.secret.toString()] === undefined || - preTagCorrespondingToLog.index >= - newLargestIndexMapForIteration[preTagCorrespondingToLog.secret.toString()]) - ) { - // We have found a new largest index so we store it for later processing (storing it in the db + fetching - // the difference of the window sets of current and the next iteration) - newLargestIndexMapForIteration[preTagCorrespondingToLog.secret.toString()] = - preTagCorrespondingToLog.index + 1; - } - } + // Store the logs for this recipient + if (allLogs.length > 0) { + await this.#storePendingTaggedLogs(contractAddress, pendingTaggedLogArrayBaseSlot, recipient, allLogs); } - - // Now based on the new largest indexes we found, we will construct a new secrets and windows set to fetch logs - // for. Note that it's very unlikely that a new log from the current window would appear between the iterations - // so we fetch the logs only for the difference of the window sets. - const newSecretsAndWindows = []; - for (const [directionalAppTaggingSecret, newIndex] of Object.entries(newLargestIndexMapForIteration)) { - const maybeIndexedSecret = indexedSecrets.find( - indexedSecret => indexedSecret.secret.toString() === directionalAppTaggingSecret, - ); - if (maybeIndexedSecret) { - newSecretsAndWindows.push({ - secret: maybeIndexedSecret.secret, - // We set the left most index to the new index to avoid fetching the same logs again - leftMostIndex: newIndex, - rightMostIndex: newIndex + WINDOW_HALF_SIZE, - }); - - // We store the new largest index in the map to later store it in the db. - newLargestIndexMapToStore[directionalAppTaggingSecret] = newIndex; - } else { - throw new Error( - `Secret not found for directionalAppTaggingSecret ${directionalAppTaggingSecret}. This is a bug as it should never happen!`, - ); - } - } - - // Now we set the new secrets and windows and proceed to the next iteration. - secretsAndWindows = newSecretsAndWindows; - } - - // At this point we have processed all the logs for the recipient so we store the last used indexes in the db. - // newLargestIndexMapToStore contains "next" indexes to look for (one past the last found), so subtract 1 to get - // last used. - await this.recipientTaggingDataProvider.setLastUsedIndexes( - Object.entries(newLargestIndexMapToStore).map(([directionalAppTaggingSecret, index]) => ({ - secret: DirectionalAppTaggingSecret.fromString(directionalAppTaggingSecret), - index: index - 1, - })), - ); - } + }), + ); } - /** - * Returns the last used tagging indexes along with the directional app tagging secrets for a given recipient and all - * the senders in the address book. - * This method should be exposed as an oracle call to allow aztec.nr to perform the orchestration - * of the syncTaggedLogs and processTaggedLogs methods. However, it is not possible to do so at the moment, - * so we're keeping it private for now. - * @param contractAddress - The contract address to silo the secret for - * @param recipient - The address receiving the notes - * @returns A list of directional app tagging secrets along with the last used tagging indexes. If the corresponding - * secret was never used, the index is undefined. - * TODO(#17775): The naming here is broken as the function name does not reflect the return type. Make sure this gets - * fixed when implementing the linked issue. - */ - protected async getLastUsedTaggingIndexesForSenders( + async #getSecretsForSenders( contractAddress: AztecAddress, recipient: AztecAddress, - ): Promise<{ secret: DirectionalAppTaggingSecret; index: number | undefined }[]> { + ): Promise { const recipientCompleteAddress = await this.#getCompleteAddress(recipient); const recipientIvsk = await this.keyStore.getMasterIncomingViewingSecretKey(recipient); // We implicitly add all PXE accounts as senders, this helps us decrypt tags on notes that we send to ourselves // (recipient = us, sender = us) - const senders = [ - ...(await this.recipientTaggingDataProvider.getSenderAddresses()), - ...(await this.keyStore.getAccounts()), - ].filter((address, index, self) => index === self.findIndex(otherAddress => otherAddress.equals(address))); - const secrets = await Promise.all( - senders.map(contact => { + const allSenders = [...(await this.senderAddressBook.getSenders()), ...(await this.keyStore.getAccounts())]; + + // We deduplicate the senders by adding them to a set and then converting the set back to an array + const deduplicatedSenders = Array.from(new Set(allSenders.map(sender => sender.toString()))).map(sender => + AztecAddress.fromString(sender), + ); + + return Promise.all( + deduplicatedSenders.map(sender => { return DirectionalAppTaggingSecret.compute( recipientCompleteAddress, recipientIvsk, - contact, + sender, contractAddress, recipient, ); }), ); - const indexes = await this.recipientTaggingDataProvider.getLastUsedIndexes(secrets); - if (indexes.length !== secrets.length) { - throw new Error('Indexes and directional app tagging secrets have different lengths'); - } - - return secrets.map((secret, i) => ({ - secret, - index: indexes[i], - })); } async #storePendingTaggedLogs( diff --git a/yarn-project/pxe/src/pxe.test.ts b/yarn-project/pxe/src/pxe.test.ts index 39978449f91a..a4a6f9ce99f8 100644 --- a/yarn-project/pxe/src/pxe.test.ts +++ b/yarn-project/pxe/src/pxe.test.ts @@ -1,4 +1,5 @@ import { BBBundlePrivateKernelProver } from '@aztec/bb-prover/client/bundle'; +import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; import { BlockNumber } from '@aztec/foundation/branded-types'; import { omit } from '@aztec/foundation/collection'; @@ -175,6 +176,13 @@ describe('PXE', () => { }); node.getBlockHeader.mockResolvedValue(blockHeader); + // Mock getL2Tips which is needed for syncing tagged logs + node.getL2Tips.mockResolvedValue({ + latest: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + proven: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + finalized: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + }); + // This is read when PXE tries to resolve the // class id of a contract instance node.getPublicStorageAt.mockResolvedValue(Fr.ZERO); diff --git a/yarn-project/pxe/src/pxe.ts b/yarn-project/pxe/src/pxe.ts index 7da0489074b8..56aa263986dd 100644 --- a/yarn-project/pxe/src/pxe.ts +++ b/yarn-project/pxe/src/pxe.ts @@ -74,8 +74,9 @@ import { CapsuleDataProvider } from './storage/capsule_data_provider/capsule_dat import { ContractDataProvider } from './storage/contract_data_provider/contract_data_provider.js'; import { NoteDataProvider } from './storage/note_data_provider/note_data_provider.js'; import { PrivateEventDataProvider } from './storage/private_event_data_provider/private_event_data_provider.js'; -import { RecipientTaggingDataProvider } from './storage/tagging_data_provider/recipient_tagging_data_provider.js'; +import { SenderAddressBook } from './storage/tagging_data_provider/sender_address_book.js'; import { SenderTaggingDataProvider } from './storage/tagging_data_provider/sender_tagging_data_provider.js'; +import { RecipientTaggingDataProvider } from './tagging/recipient_sync/recipient_tagging_data_provider.js'; export type PackedPrivateEvent = InTx & { packedEvent: Fr[]; @@ -96,6 +97,7 @@ export class PXE { private capsuleDataProvider: CapsuleDataProvider, private anchorBlockDataProvider: AnchorBlockDataProvider, private senderTaggingDataProvider: SenderTaggingDataProvider, + private senderAddressBook: SenderAddressBook, private recipientTaggingDataProvider: RecipientTaggingDataProvider, private addressDataProvider: AddressDataProvider, private privateEventDataProvider: PrivateEventDataProvider, @@ -136,6 +138,7 @@ export class PXE { const noteDataProvider = await NoteDataProvider.create(store); const anchorBlockDataProvider = new AnchorBlockDataProvider(store); const senderTaggingDataProvider = new SenderTaggingDataProvider(store); + const senderAddressBook = new SenderAddressBook(store); const recipientTaggingDataProvider = new RecipientTaggingDataProvider(store); const capsuleDataProvider = new CapsuleDataProvider(store); const keyStore = new KeyStore(store); @@ -144,7 +147,6 @@ export class PXE { node, anchorBlockDataProvider, noteDataProvider, - recipientTaggingDataProvider, tipsStore, config, loggerOrSuffix, @@ -163,6 +165,7 @@ export class PXE { capsuleDataProvider, anchorBlockDataProvider, senderTaggingDataProvider, + senderAddressBook, recipientTaggingDataProvider, addressDataProvider, privateEventDataProvider, @@ -202,6 +205,7 @@ export class PXE { this.anchorBlockDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, this.simulator, @@ -491,7 +495,7 @@ export class PXE { return sender; } - const wasAdded = await this.recipientTaggingDataProvider.addSenderAddress(sender); + const wasAdded = await this.senderAddressBook.addSender(sender); if (wasAdded) { this.log.info(`Added sender:\n ${sender.toString()}`); @@ -507,7 +511,7 @@ export class PXE { * @returns Senders registered in this PXE. */ public getSenders(): Promise { - return this.recipientTaggingDataProvider.getSenderAddresses(); + return this.senderAddressBook.getSenders(); } /** @@ -515,7 +519,7 @@ export class PXE { * @param sender - The address of the sender to remove. */ public async removeSender(sender: AztecAddress): Promise { - const wasRemoved = await this.recipientTaggingDataProvider.removeSenderAddress(sender); + const wasRemoved = await this.senderAddressBook.removeSender(sender); if (wasRemoved) { this.log.info(`Removed sender:\n ${sender.toString()}`); diff --git a/yarn-project/pxe/src/storage/tagging_data_provider/index.ts b/yarn-project/pxe/src/storage/tagging_data_provider/index.ts index 22ad7b18f2dc..69c3c6cfd361 100644 --- a/yarn-project/pxe/src/storage/tagging_data_provider/index.ts +++ b/yarn-project/pxe/src/storage/tagging_data_provider/index.ts @@ -1,2 +1,3 @@ export { SenderTaggingDataProvider } from './sender_tagging_data_provider.js'; -export { RecipientTaggingDataProvider } from './recipient_tagging_data_provider.js'; +export { SenderAddressBook } from './sender_address_book.js'; +export { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js'; diff --git a/yarn-project/pxe/src/storage/tagging_data_provider/recipient_tagging_data_provider.ts b/yarn-project/pxe/src/storage/tagging_data_provider/recipient_tagging_data_provider.ts deleted file mode 100644 index ac291a718dcb..000000000000 --- a/yarn-project/pxe/src/storage/tagging_data_provider/recipient_tagging_data_provider.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { toArray } from '@aztec/foundation/iterable'; -import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; -import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; - -/** - * Data provider of tagging data used when syncing the logs as a recipient. The sender counterpart of this class is - * called SenderTaggingDataProvider. We have the providers separate for the sender and recipient because - * the algorithms are completely disjoint and there is not data reuse between the 2. - */ -export class RecipientTaggingDataProvider { - #store: AztecAsyncKVStore; - #addressBook: AztecAsyncMap; - - // Stores the last used index for each directional app tagging secret. - #lastUsedIndexes: AztecAsyncMap; - - constructor(store: AztecAsyncKVStore) { - this.#store = store; - - this.#addressBook = this.#store.openMap('address_book'); - this.#lastUsedIndexes = this.#store.openMap('last_used_indexes'); - } - - /** - * Sets the last used indexes when looking for logs. - * @param preTags - The pre-tags containing the directional app tagging secrets and the indexes that are to be - * updated in the db. - * @throws If any two pre-tags contain the same directional app tagging secret - */ - setLastUsedIndexes(preTags: PreTag[]) { - // Non-unique secrets would indicate a bug in the caller function. - const secretsSet = new Set(preTags.map(preTag => preTag.secret.toString())); - if (secretsSet.size !== preTags.length) { - throw new Error(`Duplicate secrets found when setting last used indexes`); - } - - return Promise.all(preTags.map(({ secret, index }) => this.#lastUsedIndexes.set(secret.toString(), index))); - } - - /** - * Returns the last used indexes when looking for logs. - * @param secrets - The directional app tagging secrets to obtain the indexes for. - * @returns The last used indexes for the given directional app tagging secrets, or undefined if have never yet found - * a log for a given secret. - */ - getLastUsedIndexes(secrets: DirectionalAppTaggingSecret[]): Promise<(number | undefined)[]> { - return Promise.all(secrets.map(secret => this.#lastUsedIndexes.getAsync(secret.toString()))); - } - - resetNoteSyncData(): Promise { - return this.#store.transactionAsync(async () => { - const keys = await toArray(this.#lastUsedIndexes.keysAsync()); - await Promise.all(keys.map(secret => this.#lastUsedIndexes.delete(secret))); - }); - } - - // It might seem weird that the following 3 methods are in RecipientTaggingDataProvider and not - // in SenderTaggingDataProvider but that is because this data is truly only used for the purposes of syncing logs - // as a recipient. When sending logs or when syncing sender tagging indexes we only receive directional app tagging - // secret from Aztec.nr via an oracle and we don't need to access sender addresses. - - async addSenderAddress(address: AztecAddress): Promise { - if (await this.#addressBook.hasAsync(address.toString())) { - return false; - } - - await this.#addressBook.set(address.toString(), true); - - return true; - } - - async getSenderAddresses(): Promise { - return (await toArray(this.#addressBook.keysAsync())).map(AztecAddress.fromString); - } - - async removeSenderAddress(address: AztecAddress): Promise { - if (!(await this.#addressBook.hasAsync(address.toString()))) { - return false; - } - - await this.#addressBook.delete(address.toString()); - - return true; - } -} diff --git a/yarn-project/pxe/src/storage/tagging_data_provider/sender_address_book.ts b/yarn-project/pxe/src/storage/tagging_data_provider/sender_address_book.ts new file mode 100644 index 000000000000..a62e0537c8d1 --- /dev/null +++ b/yarn-project/pxe/src/storage/tagging_data_provider/sender_address_book.ts @@ -0,0 +1,42 @@ +import { toArray } from '@aztec/foundation/iterable'; +import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; + +/** + * Stores sender addresses. During recipient log synchronization, these senders are used, along with a given recipient, + * to derive directional app tagging secrets that are then used to sync the logs. + */ +export class SenderAddressBook { + #store: AztecAsyncKVStore; + #addressBook: AztecAsyncMap; + + constructor(store: AztecAsyncKVStore) { + this.#store = store; + + this.#addressBook = this.#store.openMap('address_book'); + } + + async addSender(address: AztecAddress): Promise { + if (await this.#addressBook.hasAsync(address.toString())) { + return false; + } + + await this.#addressBook.set(address.toString(), true); + + return true; + } + + async getSenders(): Promise { + return (await toArray(this.#addressBook.keysAsync())).map(AztecAddress.fromString); + } + + async removeSender(address: AztecAddress): Promise { + if (!(await this.#addressBook.hasAsync(address.toString()))) { + return false; + } + + await this.#addressBook.delete(address.toString()); + + return true; + } +} diff --git a/yarn-project/pxe/src/tagging/constants.ts b/yarn-project/pxe/src/tagging/constants.ts deleted file mode 100644 index ba547f894e48..000000000000 --- a/yarn-project/pxe/src/tagging/constants.ts +++ /dev/null @@ -1,3 +0,0 @@ -// Half the size of the window we slide over the tagging indexes. -// TODO(#17775): Move this to the recipient log sync function once it's implemented in this directory. -export const WINDOW_HALF_SIZE = 10; diff --git a/yarn-project/pxe/src/tagging/index.ts b/yarn-project/pxe/src/tagging/index.ts index 82b8329348f7..c480c5ee3257 100644 --- a/yarn-project/pxe/src/tagging/index.ts +++ b/yarn-project/pxe/src/tagging/index.ts @@ -1,4 +1,2 @@ -export * from './constants.js'; -export * from './utils.js'; export { DirectionalAppTaggingSecret, Tag, SiloedTag } from '@aztec/stdlib/logs'; export { type PreTag } from '@aztec/stdlib/logs'; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts index 1479ec721fc2..dec3c567d3d3 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts @@ -13,7 +13,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../sync/sync_sender_tagging_indexes.js'; import { loadPrivateLogsForSenderRecipientPair } from './load_private_logs_for_sender_recipient_pair.js'; -import { NewRecipientTaggingDataProvider } from './new_recipient_tagging_data_provider.js'; +import { RecipientTaggingDataProvider } from './recipient_tagging_data_provider.js'; // In this test suite we don't care about the anchor block behavior as that is sufficiently tested by // the loadLogsForRange test suite, so we use a high block number to ensure it occurs after all logs. @@ -24,7 +24,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { let app: AztecAddress; let aztecNode: MockProxy; - let taggingDataProvider: NewRecipientTaggingDataProvider; + let taggingDataProvider: RecipientTaggingDataProvider; const currentTimestamp = BigInt(Math.floor(Date.now() / 1000)); @@ -56,7 +56,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { aztecNode.getPrivateLogsByTags.mockReset(); aztecNode.getL2Tips.mockReset(); aztecNode.getBlockHeader.mockReset(); - taggingDataProvider = new NewRecipientTaggingDataProvider(await openTmpStore('test')); + taggingDataProvider = new RecipientTaggingDataProvider(await openTmpStore('test')); }); it('returns empty array when no logs found', async () => { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts index 70ab480cd5d6..2eff435e0975 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts @@ -4,7 +4,7 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { DirectionalAppTaggingSecret, TxScopedL2Log } from '@aztec/stdlib/logs'; import { UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN } from '../sync/sync_sender_tagging_indexes.js'; -import type { NewRecipientTaggingDataProvider } from './new_recipient_tagging_data_provider.js'; +import type { RecipientTaggingDataProvider } from './recipient_tagging_data_provider.js'; import { findHighestIndexes } from './utils/find_highest_indexes.js'; import { loadLogsForRange } from './utils/load_logs_for_range.js'; @@ -19,7 +19,7 @@ export async function loadPrivateLogsForSenderRecipientPair( secret: DirectionalAppTaggingSecret, app: AztecAddress, aztecNode: AztecNode, - taggingDataProvider: NewRecipientTaggingDataProvider, + taggingDataProvider: RecipientTaggingDataProvider, anchorBlockNumber: BlockNumber, ): Promise { // # Explanation of how the algorithm works diff --git a/yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts b/yarn-project/pxe/src/tagging/recipient_sync/recipient_tagging_data_provider.ts similarity index 94% rename from yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts rename to yarn-project/pxe/src/tagging/recipient_sync/recipient_tagging_data_provider.ts index fb14342e0368..355a6d880ba3 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/new_recipient_tagging_data_provider.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/recipient_tagging_data_provider.ts @@ -9,9 +9,9 @@ import type { DirectionalAppTaggingSecret } from '@aztec/stdlib/logs'; * @dev Chain reorgs do not need to be handled here because both the finalized and aged indexes refer to finalized * blocks, which by definition cannot be affected by reorgs. * - * TODO(benesjan): Rename as to RecipientTaggingDataProvider and relocate once the old tagging sync is purged. + * TODO(benesjan): Relocate to yarn-project/pxe/src/storage/tagging_data_provider */ -export class NewRecipientTaggingDataProvider { +export class RecipientTaggingDataProvider { #store: AztecAsyncKVStore; #highestAgedIndex: AztecAsyncMap; diff --git a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts index 758f81b03dcb..a0b47dfb89f8 100644 --- a/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sync/sync_sender_tagging_indexes.ts @@ -12,16 +12,10 @@ import { loadAndStoreNewTaggingIndexes } from './utils/load_and_store_new_taggin // MAX_PRIVATE_LOGS_PER_TX indexes consumed in case the logs are squashed. This happens when the log contains a note // and the note is nullified in the same tx. // -// Rationale for value 95: -// - The `e2e_pending_note_hashes_contract` test's "Should handle overflowing the kernel data structures in nested -// calls" test case hits 95 tagging indexes emitted in a single transaction. This test creates and nullifies many -// notes recursively to test kernel reset circuit behavior, which causes logs to be squashed but still consume -// tagging indexes during the sync process. Since this is testing MAX_PRIVATE_LOGS_PER_TX overflow we can be -// reasonably certain that this value is large enough for standard use cases. -// - This value is below MAX_RPC_LEN (100) which is the limit for array parameters in the JSON RPC schema for -// `getLogsByTags`. Any test that would perform sync over JSON RPC (not by having access to the Aztec node instance -// directly) would error out if that maximum was hit (docs_examples.test.ts is an example of this). -export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 95; +// Having a large window significantly slowed down `e2e_l1_with_wall_time` test as there we perform sync for more than +// 1000 secrets. For this reason we set it to a relatively low value of 20. 20 should be sufficient for all the use +// cases. +export const UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN = 20; /** * Syncs tagging indexes. This function needs to be called whenever a private log is being sent. diff --git a/yarn-project/pxe/src/tagging/utils.ts b/yarn-project/pxe/src/tagging/utils.ts deleted file mode 100644 index 1b3b7f3eb22f..000000000000 --- a/yarn-project/pxe/src/tagging/utils.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { DirectionalAppTaggingSecret, PreTag } from '@aztec/stdlib/logs'; - -// TODO(#17775): If this does not get dropped when implementing the linked issue make this return tags instead. This -// will move some complexity from syncTaggedLogs to here. -export function getPreTagsForTheWindow( - secretsAndWindows: { secret: DirectionalAppTaggingSecret; leftMostIndex: number; rightMostIndex: number }[], -): PreTag[] { - const secrets = []; - for (const secretAndWindow of secretsAndWindows) { - for (let i = secretAndWindow.leftMostIndex; i <= secretAndWindow.rightMostIndex; i++) { - secrets.push({ secret: secretAndWindow.secret, index: i }); - } - } - return secrets; -} - -/** - * Creates a map from directional app tagging secret to initial index. - * @param preTags - The pre-tags to get the initial indexes map from. - * @returns The map from directional app tagging secret to initial index. - */ -export function getInitialIndexesMap(preTags: { secret: DirectionalAppTaggingSecret; index: number | undefined }[]): { - [k: string]: number; -} { - const initialIndexes: { [k: string]: number } = {}; - - for (const preTag of preTags) { - initialIndexes[preTag.secret.toString()] = preTag.index ?? 0; - } - - return initialIndexes; -} diff --git a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts b/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts index 41877c08eb5d..bee03d8472d2 100644 --- a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts +++ b/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts @@ -10,7 +10,7 @@ import type { UInt64 } from '../types/shared.js'; import { PrivateLog } from './private_log.js'; import { PublicLog } from './public_log.js'; -// TODO(#14460): Split to private and public versions instead of having this weird mix. +// TODO(F-231): Drop this and return the PrivateLogWithTxData and PublicLogWithTxData from Aztec node instead. export class TxScopedL2Log { constructor( /* diff --git a/yarn-project/txe/src/oracle/txe_oracle_top_level_context.ts b/yarn-project/txe/src/oracle/txe_oracle_top_level_context.ts index acd836212871..dec5408aa81d 100644 --- a/yarn-project/txe/src/oracle/txe_oracle_top_level_context.ts +++ b/yarn-project/txe/src/oracle/txe_oracle_top_level_context.ts @@ -19,6 +19,7 @@ import { ORACLE_VERSION, PrivateEventDataProvider, RecipientTaggingDataProvider, + SenderAddressBook, SenderTaggingDataProvider, enrichPublicSimulationError, } from '@aztec/pxe/server'; @@ -102,6 +103,7 @@ export class TXEOracleTopLevelContext implements IMiscOracle, ITxeExecutionOracl private accountDataProvider: TXEAccountDataProvider, private senderTaggingDataProvider: SenderTaggingDataProvider, private recipientTaggingDataProvider: RecipientTaggingDataProvider, + private senderAddressBook: SenderAddressBook, private capsuleDataProvider: CapsuleDataProvider, private privateEventDataProvider: PrivateEventDataProvider, private nextBlockTimestamp: bigint, @@ -333,6 +335,7 @@ export class TXEOracleTopLevelContext implements IMiscOracle, ITxeExecutionOracl this.stateMachine.anchorBlockDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, 0, @@ -657,8 +660,8 @@ export class TXEOracleTopLevelContext implements IMiscOracle, ITxeExecutionOracl this.addressDataProvider, this.stateMachine.node, this.stateMachine.anchorBlockDataProvider, - this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, ); diff --git a/yarn-project/txe/src/txe_session.test.ts b/yarn-project/txe/src/txe_session.test.ts index bf2a0d0cf32c..bef32ac68bb1 100644 --- a/yarn-project/txe/src/txe_session.test.ts +++ b/yarn-project/txe/src/txe_session.test.ts @@ -18,6 +18,7 @@ describe('TXESession.processFunction', () => { {} as any, // accountDataProvider {} as any, // senderTaggingDataProvider {} as any, // recipientTaggingDataProvider + {} as any, // senderAddressBook {} as any, // capsuleDataProvider {} as any, // privateEventDataProvider new Fr(1), // chainId diff --git a/yarn-project/txe/src/txe_session.ts b/yarn-project/txe/src/txe_session.ts index 97e92bd1e717..88199d10e39c 100644 --- a/yarn-project/txe/src/txe_session.ts +++ b/yarn-project/txe/src/txe_session.ts @@ -11,6 +11,7 @@ import { NoteService, PrivateEventDataProvider, RecipientTaggingDataProvider, + SenderAddressBook, SenderTaggingDataProvider, } from '@aztec/pxe/server'; import { @@ -127,6 +128,7 @@ export class TXESession implements TXESessionStateHandler { private accountDataProvider: TXEAccountDataProvider, private senderTaggingDataProvider: SenderTaggingDataProvider, private recipientTaggingDataProvider: RecipientTaggingDataProvider, + private senderAddressBook: SenderAddressBook, private capsuleDataProvider: CapsuleDataProvider, private privateEventDataProvider: PrivateEventDataProvider, private chainId: Fr, @@ -143,6 +145,7 @@ export class TXESession implements TXESessionStateHandler { const noteDataProvider = await NoteDataProvider.create(store); const senderTaggingDataProvider = new SenderTaggingDataProvider(store); const recipientTaggingDataProvider = new RecipientTaggingDataProvider(store); + const senderAddressBook = new SenderAddressBook(store); const capsuleDataProvider = new CapsuleDataProvider(store); const keyStore = new KeyStore(store); const accountDataProvider = new TXEAccountDataProvider(store); @@ -168,6 +171,7 @@ export class TXESession implements TXESessionStateHandler { accountDataProvider, senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, nextBlockTimestamp, @@ -188,6 +192,7 @@ export class TXESession implements TXESessionStateHandler { accountDataProvider, senderTaggingDataProvider, recipientTaggingDataProvider, + senderAddressBook, capsuleDataProvider, privateEventDataProvider, version, @@ -258,6 +263,7 @@ export class TXESession implements TXESessionStateHandler { this.accountDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, this.nextBlockTimestamp, @@ -323,6 +329,7 @@ export class TXESession implements TXESessionStateHandler { this.stateMachine.anchorBlockDataProvider, this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, ); @@ -389,8 +396,8 @@ export class TXESession implements TXESessionStateHandler { this.addressDataProvider, this.stateMachine.node, this.stateMachine.anchorBlockDataProvider, - this.senderTaggingDataProvider, this.recipientTaggingDataProvider, + this.senderAddressBook, this.capsuleDataProvider, this.privateEventDataProvider, );