diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 656cec0d..95686406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,13 @@ jobs: fetch-depth: 0 - name: Guard hidden bidirectional Unicode run: | - ! rg -nP '[\x{202A}-\x{202E}\x{2066}-\x{2069}]' .github crates + ! rg -nP '[\x{202A}-\x{202E}\x{2066}-\x{2069}]' .github crates scripts wikibrowser workers extensions \ + --glob '!**/node_modules/**' \ + --glob '!**/target/**' \ + --glob '!**/.next/**' \ + --glob '!**/dist/**' \ + --glob '!**/out/**' \ + --glob '!**/coverage/**' - id: paths name: Detect changed areas shell: bash @@ -61,19 +67,23 @@ jobs: fi } - public_api_pattern='^(crates/vfs_canister/vfs\.did|crates/vfs_types/|crates/vfs_canister/src/lib\.rs)' - deploy_pattern='^scripts/(deploy|local|mainnet|smoke)/' - canister_pattern="^(Cargo\.toml|Cargo\.lock|crates/(vfs_canister|vfs_runtime|vfs_store|vfs_types|wiki_domain)/|docs/DB_LIFECYCLE\.md|scripts/build-vfs-canister\.sh)|$deploy_pattern" + set_output canister '^(Cargo\.lock|crates/(vfs_canister|vfs_runtime|vfs_types|vfs_store|wiki_domain)/|docs/DB_LIFECYCLE\.md|scripts/build-vfs-canister\.sh)' + set_output rust_all '^(Cargo\.toml|crates/(vfs_canister|vfs_runtime|vfs_types|vfs_cli_app|vfs_cli_core|vfs_client|vfs_store|wiki_domain|ic_sqlite_vfs_probe)/|scripts/(kinic_vfs_cli_release_version|package_kinic_vfs_cli)\.sh)' + set_output cli '^(crates/(vfs_cli_app|vfs_cli_core|vfs_client)/|docs/(CLI|PUBLIC_SMOKE)\.md|scripts/(local|smoke|mainnet|kinic_vfs_cli_release_version|package_kinic_vfs_cli))' + set_output wikibrowser '^(wikibrowser/|crates/vfs_canister/vfs\.did)' + set_output extension '^(extensions/wiki-clipper/|crates/vfs_canister/vfs\.did)' + set_output wiki_generator '^workers/wiki-generator/' + set_output skill_registry_web '^(skill-registry-web/|wikibrowser/app/skills/|wikibrowser/scripts/check-skill-registry\.mjs|skills/)' - printf '%s\n' 'crates/vfs_store/src/lib.rs' | grep -Eq "$canister_pattern" - - set_output canister "$canister_pattern" - set_output rust_all "^(Cargo\.toml|crates/(vfs_cli_app|vfs_cli_core|vfs_client|vfs_store|vfs_types|wiki_domain|ic_sqlite_vfs_probe)/|crates/vfs_canister/(src/lib\.rs|vfs\.did)|scripts/(kinic_vfs_cli_release_version|package_kinic_vfs_cli)\.sh)" - set_output cli "^(crates/(vfs_cli_app|vfs_cli_core|vfs_client|vfs_types)/|crates/vfs_canister/(src/lib\.rs|vfs\.did)|docs/(CLI|PUBLIC_SMOKE)\.md|scripts/(deploy|local|smoke|mainnet|kinic_vfs_cli_release_version|package_kinic_vfs_cli))" - set_output wikibrowser "^wikibrowser/|$public_api_pattern" - set_output extension "^extensions/wiki-clipper/|$public_api_pattern" - set_output wiki_generator "^workers/wiki-generator/|$public_api_pattern" - set_output skill_registry_web "^(skill-registry-web/|wikibrowser/app/skills/|wikibrowser/scripts/check-skill-registry\.mjs|skills/)|$public_api_pattern" + regression-groups-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + - name: Check regression groups + run: node scripts/check-regression-groups.mjs wikibrowser-check: needs: changes @@ -94,6 +104,8 @@ jobs: cache-dependency-path: wikibrowser/pnpm-lock.yaml - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Run audit + run: pnpm audit --audit-level moderate - name: Run tests run: pnpm test - name: Run lint @@ -124,6 +136,8 @@ jobs: cache-dependency-path: workers/wiki-generator/pnpm-lock.yaml - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Run audit + run: pnpm audit --audit-level moderate - name: Run typecheck run: pnpm typecheck - name: Run tests @@ -148,8 +162,12 @@ jobs: cache-dependency-path: skill-registry-web/pnpm-lock.yaml - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Run audit + run: pnpm audit --audit-level moderate - name: Run tests run: pnpm test + - name: Run lint + run: pnpm lint - name: Run typecheck run: pnpm typecheck - name: Build diff --git a/Cargo.lock b/Cargo.lock index 82b30c6f..86014c95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1389,6 +1389,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ic-cdk-timers" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6852b9c1d4a82ff50fc7318599298aee8bfb082bd7e9fe7e5c1420692b2170f7" +dependencies = [ + "ic-cdk-executor", + "ic0", + "slotmap", +] + [[package]] name = "ic-certification" version = "3.2.0" @@ -2259,7 +2270,7 @@ dependencies = [ "bit-vec", "bitflags", "num-traits", - "rand 0.9.2", + "rand 0.9.3", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -2333,7 +2344,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.3", "ring", "rustc-hash", "rustls", @@ -2393,9 +2404,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3488,6 +3499,7 @@ dependencies = [ "canbench-rs", "candid", "ic-cdk 0.20.1", + "ic-cdk-timers", "ic-http-certification", "ic-sqlite-vfs", "ic-stable-structures", diff --git a/crates/vfs_canister/Cargo.toml b/crates/vfs_canister/Cargo.toml index 2973e335..eae48484 100644 --- a/crates/vfs_canister/Cargo.toml +++ b/crates/vfs_canister/Cargo.toml @@ -13,6 +13,7 @@ canbench-rs = ["dep:canbench-rs", "dep:serde_json"] canbench-rs = { version = "0.4.1", optional = true } candid = "0.10.26" ic-cdk = "0.20.0" +ic-cdk-timers = "1.0.0" ic-http-certification = "3.2.0" ic-stable-structures = "0.7.2" serde = { version = "1.0.228", features = ["derive"] } diff --git a/crates/vfs_canister/src/lib.rs b/crates/vfs_canister/src/lib.rs index 6e87ad54..d2ff10b7 100644 --- a/crates/vfs_canister/src/lib.rs +++ b/crates/vfs_canister/src/lib.rs @@ -8,6 +8,8 @@ use std::collections::BTreeMap; use std::fs::create_dir_all; #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; +#[cfg(target_arch = "wasm32")] +use std::time::Duration; #[cfg(any(target_arch = "wasm32", test))] use candid::utils::decode_args; @@ -15,6 +17,8 @@ use candid::{CandidType, Decode, Deserialize, Nat, Principal, export_service}; #[cfg(not(test))] use ic_cdk::call::Call; use ic_cdk::{init, post_upgrade, query, update}; +#[cfg(target_arch = "wasm32")] +use ic_cdk_timers::set_timer_interval; use ic_http_certification::{ CERTIFICATE_EXPRESSION_HEADER_NAME, DefaultCelBuilder, DefaultResponseCertification, HttpCertification, HttpCertificationPath, HttpCertificationTree, HttpCertificationTreeEntry, @@ -26,29 +30,31 @@ use ic_sqlite_vfs::{Db, DbHandle}; use ic_stable_structures::DefaultMemoryImpl; #[cfg(target_arch = "wasm32")] use ic_stable_structures::memory_manager::{MemoryId, MemoryManager}; +#[cfg(target_arch = "wasm32")] +use vfs_runtime::STORAGE_BILLING_INTERVAL_MS; use vfs_runtime::{ - CreditsPendingLedgerDetailsInput, DatabaseCreditPurchaseWithLedgerDetails, DatabaseMeta, - RequiredRole, VfsService, + CyclesPendingLedgerDetailsInput, DatabaseCyclesPurchaseWithLedgerDetails, DatabaseMeta, + RequiredRole, VfsService, cycles_for_payment_amount_e8s, }; use vfs_types::{ AppendNodeRequest, CanisterHealth, CanonicalRole, ChildNode, CreateDatabaseRequest, - CreateDatabaseResult, CreditsConfig, CreditsConfigUpdate, CreditsPurchaseResult, - DatabaseArchiveChunk, DatabaseArchiveInfo, DatabaseCreditEntryPage, - DatabaseCreditPendingOperation, DatabaseCreditPendingOperationPage, - DatabaseCreditPurchasePreview, DatabaseCreditPurchaseRequest, DatabaseMember, + CreateDatabaseResult, CyclesBillingConfig, CyclesBillingConfigUpdate, CyclesPurchaseResult, + DatabaseArchiveChunk, DatabaseArchiveInfo, DatabaseCycleEntryPage, + DatabaseCyclesPendingPurchase, DatabaseCyclesPurchaseRequest, DatabaseMember, DatabaseRestoreChunkRequest, DatabaseRole, DatabaseSummary, DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, IncomingLinksRequest, - IndexSqlJsonQueryResult, LinkEdge, ListChildrenRequest, ListNodesRequest, MemoryCapability, - MemoryManifest, MemoryRoot, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, - MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, NodeContextRequest, NodeEntry, - OpsAnswerSessionCheckRequest, OpsAnswerSessionCheckResult, OpsAnswerSessionRequest, - OutgoingLinksRequest, QueryContext, QueryContextRequest, RenameDatabaseRequest, SearchNodeHit, - SearchNodePathsRequest, SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, - SourceRunSessionCheckRequest, Status, UrlIngestTriggerSessionCheckRequest, - UrlIngestTriggerSessionRequest, WriteNodeRequest, WriteNodeResult, WriteNodesRequest, - WriteSourceForGenerationRequest, WriteSourceForGenerationResult, + IndexSqlJsonQueryResult, KINIC_DECIMALS, KINIC_LEDGER_FEE_E8S, LinkEdge, ListChildrenRequest, + ListNodesRequest, MemoryCapability, MemoryManifest, MemoryRoot, MkdirNodeRequest, + MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, + Node, NodeContext, NodeContextRequest, NodeEntry, OpsAnswerSessionCheckRequest, + OpsAnswerSessionCheckResult, OpsAnswerSessionRequest, OutgoingLinksRequest, QueryContext, + QueryContextRequest, RenameDatabaseRequest, SearchNodeHit, SearchNodePathsRequest, + SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, SourceRunSessionCheckRequest, + Status, UrlIngestTriggerSessionCheckRequest, UrlIngestTriggerSessionRequest, WriteNodeRequest, + WriteNodeResult, WriteNodesRequest, WriteSourceForGenerationRequest, + WriteSourceForGenerationResult, kinic_base_units_per_token, }; #[cfg(not(target_arch = "wasm32"))] @@ -92,6 +98,7 @@ thread_local! { #[derive(Clone, Debug)] enum LedgerTransferFromOutcome { Completed(u64), + BadFee { expected_fee_e8s: u64 }, LedgerErr(String), Ambiguous(String), } @@ -126,48 +133,6 @@ struct TransferFromArg { created_at_time: Option, } -#[allow(dead_code)] -#[derive(Clone, Debug, CandidType)] -struct GetTransactionsRequest { - start: Nat, - length: Nat, -} - -#[allow(dead_code)] -#[derive(Clone, Debug, CandidType, Deserialize)] -struct GetTransactionsResponse { - transactions: Vec, -} - -#[allow(dead_code)] -#[derive(Clone, Debug, CandidType, Deserialize)] -struct LedgerTransaction { - kind: String, - transfer: Option, -} - -#[allow(dead_code)] -#[derive(Clone, Debug, CandidType, Deserialize)] -struct LedgerTransfer { - from: IcrcAccount, - to: IcrcAccount, - amount: Nat, - fee: Option, - memo: Option>, - created_at_time: Option, - spender: Option, -} - -#[derive(Clone, Debug)] -struct ExpectedLedgerTransfer { - from: IcrcAccount, - to: IcrcAccount, - amount_e8s: u64, - ledger_fee_e8s: u64, - memo: Vec, - created_at_time_ns: u64, -} - #[allow(dead_code)] #[derive(Clone, Debug, CandidType, Deserialize)] struct Icrc21ConsentMessageRequest { @@ -274,16 +239,19 @@ enum TransferFromError { } #[init] -fn init_hook(config: CreditsConfig) { +fn init_hook(config: CyclesBillingConfig) { initialize_or_trap(Some(config)); certify_http_responses(); + schedule_storage_billing_timer(); } #[post_upgrade] fn post_upgrade_hook() { - let config = post_upgrade_credits_config_arg().unwrap_or_else(|error| ic_cdk::trap(&error)); + let config = + post_upgrade_cycles_billing_config_arg().unwrap_or_else(|error| ic_cdk::trap(&error)); initialize_upgrade_or_trap(config); certify_http_responses(); + schedule_storage_billing_timer(); } #[query] @@ -436,14 +404,6 @@ fn list_databases() -> Result, String> { with_service(|service| service.list_database_summaries_for_caller(&caller_text())) } -#[query] -fn preview_database_credit_purchase( - database_id: String, - credits: u64, -) -> Result { - with_service(|service| service.preview_database_credit_purchase(&database_id, credits)) -} - #[query] fn icrc10_supported_standards() -> Vec { vec![Icrc10SupportedStandard { @@ -456,26 +416,30 @@ fn icrc10_supported_standards() -> Vec { fn icrc21_canister_call_consent_message( request: Icrc21ConsentMessageRequest, ) -> Icrc21ConsentMessageResponse { - if request.method != "purchase_database_credits" { + if request.method != "purchase_database_cycles" { return icrc21_unsupported(format!("unsupported canister call: {}", request.method)); } - let purchase = match Decode!(&request.arg, DatabaseCreditPurchaseRequest) { + let purchase = match Decode!(&request.arg, DatabaseCyclesPurchaseRequest) { Ok(decoded) => decoded, Err(error) => { return icrc21_unavailable(format!( - "purchase_database_credits argument decode failed: {error}" + "purchase_database_cycles argument decode failed: {error}" )); } }; - let preview = match with_service(|service| { - service.preview_database_credit_purchase(&purchase.database_id, purchase.credits) + let cycles = match with_service(|service| { + let config = service.cycles_billing_config()?; + let cycles = cycles_for_payment_amount_e8s(purchase.payment_amount_e8s, &config)?; + service.validate_database_cycles_purchase_with_minimum( + &purchase.database_id, + purchase.payment_amount_e8s, + purchase.min_expected_cycles, + )?; + Ok(cycles) }) { - Ok(preview) => preview, + Ok(cycles) => cycles, Err(error) => return icrc21_unsupported(error), }; - if let Err(error) = validate_credit_purchase_expectations(&purchase, &preview) { - return icrc21_unsupported(error); - } let language = if request.user_preferences.metadata.language.trim().is_empty() { "en".to_string() } else { @@ -487,43 +451,38 @@ fn icrc21_canister_call_consent_message( utc_offset_minutes: request.user_preferences.metadata.utc_offset_minutes, }, consent_message: Icrc21ConsentMessage::GenericDisplayMessage(format!( - "# Purchase Kinic database credits\n\nDatabase: `{database_id}`\n\nCredits: `{credits}`\n\nPayment: `{payment}` KINIC\n\nLedger transfer fee in allowance: `{fee}` KINIC\n\nSpender canister: `{spender}`", + "# Purchase Kinic database cycles\n\nDatabase: `{database_id}`\n\nCycles: `{cycles}`\n\nPayment: `{payment}` KINIC\n\nLedger transfer fee in allowance: `{fee}` KINIC\n\nSpender canister: `{spender}`", database_id = purchase.database_id, - credits = purchase.credits, - payment = format_e8s(preview.payment_amount_e8s), - fee = format_e8s(preview.ledger_fee_e8s), + cycles = format_cycles(cycles), + payment = format_e8s(purchase.payment_amount_e8s), + fee = format_e8s(KINIC_LEDGER_FEE_E8S), spender = canister_principal().to_text() )), }) } #[update] -async fn purchase_database_credits( - request: DatabaseCreditPurchaseRequest, -) -> Result { +async fn purchase_database_cycles( + request: DatabaseCyclesPurchaseRequest, +) -> Result { require_authenticated_caller()?; let caller = caller_text(); let now = now_millis(); - let config = with_service(|service| service.credits_config())?; + let config = with_service(|service| service.cycles_billing_config())?; let ledger = Principal::from_text(&config.kinic_ledger_canister_id) .map_err(|error| format!("invalid KINIC ledger canister id: {error}"))?; - let preview = with_service(|service| { - service.preview_database_credit_purchase(&request.database_id, request.credits) - })?; - validate_credit_purchase_expectations(&request, &preview)?; - let ledger_fee_e8s = preview.ledger_fee_e8s; - let payment_amount_e8s = preview.payment_amount_e8s; + let ledger_fee_e8s = KINIC_LEDGER_FEE_E8S; + let payment_amount_e8s = request.payment_amount_e8s; let ledger_created_at_time_ns = now_nanos(); let canister_owner = canister_principal().to_text(); - let operation_id = match with_service(|service| { - service.begin_database_credit_purchase_with_ledger_details( - DatabaseCreditPurchaseWithLedgerDetails { + let purchase_start = match with_service(|service| { + service.begin_database_cycles_purchase_with_ledger_details( + DatabaseCyclesPurchaseWithLedgerDetails { database_id: &request.database_id, caller: &caller, - credits: request.credits, - expected_payment_amount_e8s: request.expected_payment_amount_e8s, - expected_config_version: request.expected_config_version, - ledger: CreditsPendingLedgerDetailsInput { + payment_amount_e8s: request.payment_amount_e8s, + min_expected_cycles: request.min_expected_cycles, + ledger: CyclesPendingLedgerDetailsInput { from_owner: &caller, from_subaccount: None, to_owner: &canister_owner, @@ -535,9 +494,11 @@ async fn purchase_database_credits( }, ) }) { - Ok(operation_id) => operation_id, + Ok(purchase_start) => purchase_start, Err(error) => return Err(error), }; + let operation_id = purchase_start.operation_id; + let amount_cycles = purchase_start.amount_cycles; match ledger_transfer_from( ledger, IcrcAccount { @@ -556,114 +517,116 @@ async fn purchase_database_credits( .await { LedgerTransferFromOutcome::Completed(block_index) => { - with_service(|service| { - service.mark_database_credit_purchase_completed( + if let Err(error) = with_service(|service| { + service.complete_database_cycles_purchase_ledger_transfer( operation_id, &request.database_id, &caller, - request.credits, + amount_cycles, + block_index, ) - }) - .map_err(|error| credit_purchase_local_apply_error(operation_id, block_index, error))?; - activate_pending_database_after_credit_purchase_ledger_success( + }) { + return Err(cycles_purchase_local_apply_error( + operation_id, + block_index, + error, + )); + } + if let Err(error) = activate_pending_database_after_cycles_purchase_ledger_success( &request.database_id, now, - ) - .map_err(|error| credit_purchase_local_apply_error(operation_id, block_index, error))?; + ) { + return Err(cycles_purchase_local_apply_error( + operation_id, + block_index, + error, + )); + } #[cfg(test)] - if TEST_CREDIT_DATABASE_PURCHASE_APPLY_FAIL_ONCE.with(|flag| flag.replace(false)) { - return Err(credit_purchase_local_apply_error( + if TEST_DATABASE_CYCLES_PURCHASE_APPLY_FAIL_ONCE.with(|flag| flag.replace(false)) { + return Err(cycles_purchase_local_apply_error( operation_id, block_index, - "test credit purchase apply failure".to_string(), + "test cycle purchase apply failure".to_string(), )); } - let balance = with_service(|service| { - service.credit_database_purchase( + let balance = match with_service(|service| { + service.apply_database_cycles_purchase( operation_id, &request.database_id, &caller, - request.credits, + amount_cycles, block_index, now, ) - }) - .map_err(|error| credit_purchase_local_apply_error(operation_id, block_index, error))?; - Ok(CreditsPurchaseResult { + }) { + Ok(balance) => balance, + Err(error) => { + return Err(cycles_purchase_local_apply_error( + operation_id, + block_index, + error, + )); + } + }; + Ok(CyclesPurchaseResult { block_index, - balance_credits: balance, + amount_cycles, + balance_cycles: balance, }) } + LedgerTransferFromOutcome::BadFee { expected_fee_e8s } => { + let _ = with_service(|service| { + service.cancel_database_cycles_purchase( + operation_id, + &request.database_id, + &caller, + amount_cycles, + ) + }); + Err(format!( + "icrc2_transfer_from failed: BadFee expected fee {expected_fee_e8s}; re-approve with the current ledger fee and retry" + )) + } LedgerTransferFromOutcome::LedgerErr(error) => { let _ = with_service(|service| { - service.cancel_database_credit_purchase( + service.cancel_database_cycles_purchase( operation_id, &request.database_id, &caller, - request.credits, + amount_cycles, ) }); Err(error) } LedgerTransferFromOutcome::Ambiguous(error) => { - match with_service(|service| { - service.mark_database_credit_purchase_ambiguous( + if let Err(mark_error) = with_service(|service| { + service.mark_database_cycles_purchase_ambiguous( operation_id, &request.database_id, &caller, - request.credits, - now, + amount_cycles, ) }) { - Ok(_) => Err(format!( - "credit purchase pending operation {operation_id}; manual repair required: {error}" - )), - Err(mark_error) => Err(format!( - "credit purchase pending operation {operation_id}; ledger result ambiguous; mark ambiguous failed: {mark_error}; original ledger error: {error}" - )), + return Err(format!( + "icrc2_transfer_from result ambiguous for operation_id {operation_id}; failed to mark operation ambiguous; billing authority review required: {mark_error}; original ledger ambiguity: {error}" + )); } + Err(format!( + "icrc2_transfer_from result ambiguous for operation_id {operation_id}; billing authority review required: {error}" + )) } } } -fn validate_credit_purchase_expectations( - request: &DatabaseCreditPurchaseRequest, - preview: &DatabaseCreditPurchasePreview, -) -> Result<(), String> { - if request.expected_config_version != preview.config_version { - return Err(format!( - "credits config changed: expected version {}, current version {}", - request.expected_config_version, preview.config_version - )); - } - if request.expected_payment_amount_e8s != preview.payment_amount_e8s { - return Err(format!( - "credit purchase payment amount changed: expected {}, current {}", - request.expected_payment_amount_e8s, preview.payment_amount_e8s - )); - } - Ok(()) -} - -#[query] -fn list_database_credit_entries( - database_id: String, - cursor: Option, - limit: u32, -) -> Result { - with_service(|service| { - service.list_database_credit_entries(&database_id, &caller_text(), cursor, limit) - }) -} - #[query] -fn list_database_credit_pending_operations( +fn list_database_cycle_entries( database_id: String, cursor: Option, limit: u32, -) -> Result { +) -> Result { with_service(|service| { - service.list_database_credit_pending_operations(&database_id, &caller_text(), cursor, limit) + service.list_database_cycle_entries(&database_id, &caller_text(), cursor, limit) }) } @@ -674,48 +637,19 @@ fn query_index_sql_json(sql: String, limit: u32) -> Result Result { - require_authenticated_caller()?; - let config = with_service(|service| service.credits_config())?; - let ledger = Principal::from_text(&config.kinic_ledger_canister_id) - .map_err(|error| format!("invalid KINIC ledger canister id: {error}"))?; - let operation = with_service(|service| { - service.get_database_credit_pending_operation_for_complete(&database_id, operation_id) - })?; - let expected = expected_credit_purchase_transfer(&operation)?; - validate_ledger_transfer_block(ledger, ledger_block_index, expected).await?; +fn settle_database_storage_charges() -> Result<(), String> { + require_controller_caller()?; with_service(|service| { - service.mark_database_credit_purchase_repair_completed(&database_id, operation_id) - })?; - let now = now_millis(); - activate_pending_database_after_credit_purchase_ledger_success(&database_id, now).map_err( - |error| credit_purchase_local_apply_error(operation_id, ledger_block_index, error), - )?; - let balance = with_service(|service| { - service.repair_database_credit_purchase_complete( - &database_id, - operation_id, - ledger_block_index, - now, - ) - }) - .map_err(|error| credit_purchase_local_apply_error(operation_id, ledger_block_index, error))?; - Ok(CreditsPurchaseResult { - block_index: ledger_block_index, - balance_credits: balance, + service.settle_database_storage_charges(&canister_principal().to_text(), now_millis()) }) } -fn activate_pending_database_after_credit_purchase_ledger_success( +fn activate_pending_database_after_cycles_purchase_ledger_success( database_id: &str, now: i64, ) -> Result<(), String> { let activation = with_service(|service| { - service.activate_pending_database_for_credit_purchase(database_id, now) + service.activate_pending_database_for_cycles_purchase(database_id, now) })?; if let Some(meta) = &activation { if let Err(error) = mount_database_file(meta) { @@ -731,42 +665,38 @@ fn activate_pending_database_after_credit_purchase_ledger_success( Ok(()) } -fn credit_purchase_local_apply_error(operation_id: u64, block_index: u64, cause: String) -> String { +fn cycles_purchase_local_apply_error(operation_id: u64, block_index: u64, cause: String) -> String { format!( - "credit purchase payment completed but local credit application failed; pending operation {operation_id} can be completed with verified ledger block {block_index}: {cause}" + "cycles purchase payment completed at ledger block {block_index} but local cycles application failed; pending operation {operation_id} remains completed for billing authority review: {cause}" ) } -#[update] -fn repair_database_credit_purchase_cancel( - database_id: String, - operation_id: u64, -) -> Result<(), String> { - require_authenticated_caller()?; - let caller = caller_text(); - with_service(|service| { - service.repair_database_credit_purchase_cancel( - &database_id, - operation_id, - &caller, - now_millis(), - ) - }) +#[query] +fn get_cycles_billing_config() -> Result { + with_service(|service| service.cycles_billing_config()) } #[query] -fn get_credits_config() -> Result { - with_service(|service| service.credits_config()) +fn list_database_cycles_pending_purchases( + database_id: String, +) -> Result, String> { + with_service(|service| { + service.list_database_cycles_pending_purchases(&database_id, &caller_text()) + }) } #[update] -fn update_credits_config(payload: Vec) -> Result<(), String> { +fn update_cycles_billing_config(update: CyclesBillingConfigUpdate) -> Result<(), String> { require_authenticated_caller()?; - let update = Decode!(&payload, CreditsConfigUpdate) - .map_err(|error| format!("invalid credits config payload: {error}"))?; - with_unmetered_update("update_credits_config", None, |service, caller, _now| { - service.update_credits_config(update, caller).map(|_| ()) - }) + with_unmetered_update( + "update_cycles_billing_config", + None, + |service, caller, _now| { + service + .update_cycles_billing_config(update, caller) + .map(|_| ()) + }, + ) } #[update] @@ -973,8 +903,8 @@ fn check_url_ingest_trigger_session( } #[query] -fn check_database_write_credits(database_id: String) -> Result<(), String> { - with_service(|service| service.check_database_write_credits(&database_id, &caller_text())) +fn check_database_write_cycles(database_id: String) -> Result<(), String> { + with_service(|service| service.check_database_write_cycles(&database_id, &caller_text())) } #[update] @@ -1126,15 +1056,30 @@ fn fetch_updates(request: FetchUpdatesRequest) -> Result) { +fn initialize_or_trap(config: Option) { initialize_service_with_config(config).unwrap_or_else(|error| ic_cdk::trap(&error)); } -fn initialize_upgrade_or_trap(config: Option) { +fn initialize_upgrade_or_trap(config: Option) { initialize_service_for_upgrade(config).unwrap_or_else(|error| ic_cdk::trap(&error)); } -fn initialize_service_with_config(config: Option) -> Result<(), String> { +fn schedule_storage_billing_timer() { + #[cfg(target_arch = "wasm32")] + { + let interval_ms = u64::try_from(STORAGE_BILLING_INTERVAL_MS).unwrap_or(24 * 60 * 60 * 1000); + set_timer_interval(Duration::from_millis(interval_ms), || async { + if let Err(error) = with_service(|service| { + service + .settle_database_storage_charges(&canister_principal().to_text(), now_millis()) + }) { + ic_cdk::println!("storage billing settle failed: {error}"); + } + }); + } +} + +fn initialize_service_with_config(config: Option) -> Result<(), String> { initialize_sqlite_storage()?; #[cfg(not(target_arch = "wasm32"))] let service = VfsService::new(PathBuf::from(INDEX_DB_PATH), PathBuf::from(DATABASES_DIR)); @@ -1152,7 +1097,7 @@ fn initialize_service_with_config(config: Option) -> Result<(), S Ok(()) } -fn initialize_service_for_upgrade(config: Option) -> Result<(), String> { +fn initialize_service_for_upgrade(config: Option) -> Result<(), String> { initialize_sqlite_storage()?; #[cfg(not(target_arch = "wasm32"))] let service = VfsService::new(PathBuf::from(INDEX_DB_PATH), PathBuf::from(DATABASES_DIR)); @@ -1167,26 +1112,28 @@ fn initialize_service_for_upgrade(config: Option) -> Result<(), S } #[cfg(any(target_arch = "wasm32", test))] -fn parse_upgrade_credits_config_arg(bytes: &[u8]) -> Result, String> { +fn parse_upgrade_cycles_billing_config_arg( + bytes: &[u8], +) -> Result, String> { if bytes.is_empty() || bytes == b"DIDL\0\0" { return Ok(None); } - if let Ok((config,)) = decode_args::<(CreditsConfig,)>(bytes) { + if let Ok((config,)) = decode_args::<(CyclesBillingConfig,)>(bytes) { return Ok(Some(config)); } - if let Ok((config,)) = decode_args::<(Option,)>(bytes) { + if let Ok((config,)) = decode_args::<(Option,)>(bytes) { return Ok(config); } Err( - "post_upgrade credits config arg must be empty, CreditsConfig, or opt CreditsConfig" + "post_upgrade cycles config arg must be empty, CyclesBillingConfig, or opt CyclesBillingConfig" .to_string(), ) } -fn post_upgrade_credits_config_arg() -> Result, String> { +fn post_upgrade_cycles_billing_config_arg() -> Result, String> { #[cfg(target_arch = "wasm32")] { - parse_upgrade_credits_config_arg(&ic_cdk::api::msg_arg_data()) + parse_upgrade_cycles_billing_config_arg(&ic_cdk::api::msg_arg_data()) } #[cfg(not(target_arch = "wasm32"))] { @@ -1242,12 +1189,13 @@ fn unmount_database_file(_db_file_name: &str) {} #[cfg(test)] thread_local! { static TEST_MOUNT_DATABASE_FILE_FAIL_ONCE: RefCell = const { RefCell::new(false) }; - static TEST_LEDGER_TRANSFER_FROM_OUTCOME: RefCell> = const { RefCell::new(None) }; - static TEST_LEDGER_TRANSACTIONS: RefCell> = const { RefCell::new(Vec::new()) }; + static TEST_LEDGER_TRANSFER_FROM_OUTCOMES: RefCell> = const { RefCell::new(Vec::new()) }; + static TEST_LEDGER_TRANSFER_FEES: RefCell> = const { RefCell::new(Vec::new()) }; static TEST_LAST_LEDGER_MEMO: RefCell>> = const { RefCell::new(None) }; static TEST_LAST_LEDGER_FROM: RefCell> = const { RefCell::new(None) }; static TEST_CALLER_PRINCIPAL: RefCell> = const { RefCell::new(None) }; - static TEST_CREDIT_DATABASE_PURCHASE_APPLY_FAIL_ONCE: RefCell = const { RefCell::new(false) }; + static TEST_DATABASE_CYCLES_PURCHASE_APPLY_FAIL_ONCE: RefCell = const { RefCell::new(false) }; + static TEST_CYCLE_BALANCES: RefCell> = const { RefCell::new(Vec::new()) }; } #[cfg(test)] @@ -1256,27 +1204,33 @@ fn fail_next_mount_database_file_for_test() { } #[cfg(test)] -fn fail_next_credit_database_purchase_apply_for_test() { - TEST_CREDIT_DATABASE_PURCHASE_APPLY_FAIL_ONCE.with(|flag| flag.replace(true)); +fn fail_next_apply_database_cycles_purchase_apply_for_test() { + TEST_DATABASE_CYCLES_PURCHASE_APPLY_FAIL_ONCE.with(|flag| flag.replace(true)); } #[cfg(test)] fn set_next_ledger_transfer_from_outcome_for_test(outcome: LedgerTransferFromOutcome) { - TEST_LEDGER_TRANSFER_FROM_OUTCOME.with(|slot| { - slot.replace(Some(outcome)); + TEST_LEDGER_TRANSFER_FROM_OUTCOMES.with(|slot| { + slot.replace(vec![outcome]); }); } #[cfg(test)] -fn set_ledger_transaction_for_test(block_index: u64, transaction: LedgerTransaction) { - TEST_LEDGER_TRANSACTIONS.with(|slot| { - slot.borrow_mut().push((block_index, transaction)); +fn set_cycle_balances_for_test(balances: Vec) { + TEST_CYCLE_BALANCES.with(|slot| { + slot.replace(balances); }); } #[cfg(test)] fn clear_ledger_transactions_for_test() { - TEST_LEDGER_TRANSACTIONS.with(|slot| { + TEST_LEDGER_TRANSFER_FROM_OUTCOMES.with(|slot| { + slot.borrow_mut().clear(); + }); + TEST_LEDGER_TRANSFER_FEES.with(|slot| { + slot.borrow_mut().clear(); + }); + TEST_CYCLE_BALANCES.with(|slot| { slot.borrow_mut().clear(); }); } @@ -1312,6 +1266,11 @@ fn last_ledger_from_for_test() -> Option { TEST_LAST_LEDGER_FROM.with(|slot| slot.borrow().clone()) } +#[cfg(test)] +fn ledger_transfer_fees_for_test() -> Vec { + TEST_LEDGER_TRANSFER_FEES.with(|slot| slot.borrow().clone()) +} + #[cfg(test)] fn clear_last_ledger_memo_for_test() { TEST_LAST_LEDGER_MEMO.with(|slot| { @@ -1342,18 +1301,23 @@ fn icrc21_unavailable(description: String) -> Icrc21ConsentMessageResponse { } fn format_e8s(amount_e8s: u64) -> String { - let whole = amount_e8s / 100_000_000; - let fractional = amount_e8s % 100_000_000; + let units_per_token = kinic_base_units_per_token(); + let whole = amount_e8s / units_per_token; + let fractional = amount_e8s % units_per_token; if fractional == 0 { return whole.to_string(); } - let mut fraction = format!("{fractional:08}"); + let mut fraction = format!("{fractional:0width$}", width = usize::from(KINIC_DECIMALS)); while fraction.ends_with('0') { fraction.pop(); } format!("{whole}.{fraction}") } +fn format_cycles(cycles: u64) -> String { + cycles.to_string() +} + fn caller_text() -> String { #[cfg(test)] { @@ -1436,7 +1400,14 @@ fn now_millis() -> i64 { fn cycle_balance() -> u128 { #[cfg(test)] { - 1_000_000_000_000 + TEST_CYCLE_BALANCES.with(|slot| { + let mut balances = slot.borrow_mut(); + if balances.is_empty() { + 1_000_000_000_000 + } else { + balances.remove(0) + } + }) } #[cfg(not(test))] { @@ -1455,124 +1426,6 @@ fn now_nanos() -> u64 { } } -fn expected_credit_purchase_transfer( - operation: &DatabaseCreditPendingOperation, -) -> Result { - if operation.kind != "credit_purchase" { - return Err("pending credit operation kind mismatch".to_string()); - } - expected_ledger_transfer(operation, "credit_purchase") -} - -fn expected_ledger_transfer( - operation: &DatabaseCreditPendingOperation, - memo_kind: &str, -) -> Result { - let from_owner = pending_principal(operation.from_owner.as_deref(), "from_owner")?; - let to_owner = pending_principal(operation.to_owner.as_deref(), "to_owner")?; - let amount_e8s = operation.payment_amount_e8s; - let ledger_fee_e8s = operation - .ledger_fee_e8s - .ok_or_else(|| "pending operation missing ledger_fee_e8s".to_string())?; - let created_at_time_ns = operation - .ledger_created_at_time_ns - .ok_or_else(|| "pending operation missing ledger_created_at_time_ns".to_string())?; - Ok(ExpectedLedgerTransfer { - from: IcrcAccount { - owner: from_owner, - subaccount: operation.from_subaccount.clone(), - }, - to: IcrcAccount { - owner: to_owner, - subaccount: operation.to_subaccount.clone(), - }, - amount_e8s, - ledger_fee_e8s, - memo: credit_operation_memo(memo_kind, operation.operation_id), - created_at_time_ns, - }) -} - -fn pending_principal(value: Option<&str>, field: &str) -> Result { - let value = value.ok_or_else(|| format!("pending operation missing {field}"))?; - Principal::from_text(value).map_err(|error| format!("invalid pending {field}: {error}")) -} - -async fn validate_ledger_transfer_block( - ledger: Principal, - block_index: u64, - expected: ExpectedLedgerTransfer, -) -> Result<(), String> { - let transaction = ledger_transaction(ledger, block_index).await?; - if transaction.kind != "transfer" { - return Err(format!( - "ledger transaction kind mismatch: {}", - transaction.kind - )); - } - let transfer = transaction - .transfer - .ok_or_else(|| "ledger transaction missing transfer".to_string())?; - if transfer.from != expected.from { - return Err("ledger transaction from account mismatch".to_string()); - } - if transfer.to != expected.to { - return Err("ledger transaction to account mismatch".to_string()); - } - if nat_to_u64(&transfer.amount)? != expected.amount_e8s { - return Err("ledger transaction amount mismatch".to_string()); - } - let fee = transfer - .fee - .as_ref() - .ok_or_else(|| "ledger transaction missing fee".to_string())?; - if nat_to_u64(fee)? != expected.ledger_fee_e8s { - return Err("ledger transaction fee mismatch".to_string()); - } - if transfer.memo.as_deref() != Some(expected.memo.as_slice()) { - return Err("ledger transaction memo mismatch".to_string()); - } - if transfer.created_at_time != Some(expected.created_at_time_ns) { - return Err("ledger transaction created_at_time mismatch".to_string()); - } - Ok(()) -} - -async fn ledger_transaction( - ledger: Principal, - block_index: u64, -) -> Result { - #[cfg(test)] - { - let _ = ledger; - TEST_LEDGER_TRANSACTIONS.with(|slot| { - slot.borrow() - .iter() - .find(|(index, _)| *index == block_index) - .map(|(_, transaction)| transaction.clone()) - .ok_or_else(|| format!("test ledger transaction not found: {block_index}")) - }) - } - #[cfg(not(test))] - { - let response = Call::bounded_wait(ledger, "get_transactions") - .with_arg(GetTransactionsRequest { - start: Nat::from(block_index), - length: Nat::from(1_u64), - }) - .await - .map_err(|error| format!("get_transactions call failed: {error:?}"))?; - let response: GetTransactionsResponse = response - .candid() - .map_err(|error| format!("get_transactions decode failed: {error}"))?; - response - .transactions - .into_iter() - .next() - .ok_or_else(|| format!("ledger transaction not found: {block_index}")) - } -} - async fn ledger_transfer_from( ledger: Principal, from: IcrcAccount, @@ -1582,17 +1435,22 @@ async fn ledger_transfer_from( operation_id: u64, created_at_time_ns: u64, ) -> LedgerTransferFromOutcome { - let memo = credit_operation_memo("credit_purchase", operation_id); + let memo = cycles_purchase_memo(operation_id); #[cfg(test)] { record_test_ledger_from(&from); - let _ = (ledger, to, amount_e8s, ledger_fee_e8s, created_at_time_ns); + let _ = (ledger, to, amount_e8s, created_at_time_ns); record_test_ledger_memo(&memo); - TEST_LEDGER_TRANSFER_FROM_OUTCOME.with(|outcome| { - outcome - .borrow_mut() - .take() - .unwrap_or(LedgerTransferFromOutcome::Completed(1)) + TEST_LEDGER_TRANSFER_FEES.with(|fees| { + fees.borrow_mut().push(ledger_fee_e8s); + }); + TEST_LEDGER_TRANSFER_FROM_OUTCOMES.with(|outcomes| { + let mut outcomes = outcomes.borrow_mut(); + if outcomes.is_empty() { + LedgerTransferFromOutcome::Completed(1) + } else { + outcomes.remove(0) + } }) } #[cfg(not(test))] @@ -1638,6 +1496,10 @@ async fn ledger_transfer_from( fn transfer_from_error_outcome(error: TransferFromError) -> LedgerTransferFromOutcome { match error { + TransferFromError::BadFee { expected_fee } => match nat_to_u64(&expected_fee) { + Ok(expected_fee_e8s) => LedgerTransferFromOutcome::BadFee { expected_fee_e8s }, + Err(error) => LedgerTransferFromOutcome::Ambiguous(error), + }, TransferFromError::Duplicate { duplicate_of } => match nat_to_u64(&duplicate_of) { Ok(block_index) => LedgerTransferFromOutcome::Completed(block_index), Err(error) => LedgerTransferFromOutcome::Ambiguous(error), @@ -1656,8 +1518,8 @@ fn nat_to_u64(value: &Nat) -> Result { .map_err(|_| "nat exceeds u64".to_string()) } -fn credit_operation_memo(kind: &str, operation_id: u64) -> Vec { - format!("kinic:vfs:{kind}:{operation_id}").into_bytes() +fn cycles_purchase_memo(operation_id: u64) -> Vec { + format!("kvfs:cp:{operation_id}").into_bytes() } fn with_unmetered_update(method: &str, database_id: Option, f: F) -> Result @@ -1725,7 +1587,7 @@ fn with_authorized_metered_update( f: F, ) -> Result where - A: FnOnce(&VfsService, &str) -> Result, + A: FnOnce(&VfsService, &str) -> Result, F: FnOnce(&VfsService, &str, i64) -> Result, { let caller = caller_text(); @@ -1736,14 +1598,14 @@ where let service = borrowed .as_ref() .ok_or_else(|| "wiki service is not initialized".to_string())?; - let credits_config = authorize(service, &caller)?; + let cycles_billing_config = authorize(service, &caller)?; let result = f(service, &caller, now); let after_cycles = cycle_balance(); let cycles_delta = before_cycles.saturating_sub(after_cycles); if result.is_ok() && let Some(database_id) = database_id.as_deref() && let Err(error) = service.charge_database_update( - &credits_config, + &cycles_billing_config, database_id, &caller, method, @@ -1751,7 +1613,7 @@ where now, ) { - ic_cdk::trap(format!("credits charge failed after update: {error}")); + ic_cdk::trap(format!("cycles charge failed after update: {error}")); } result }) diff --git a/crates/vfs_canister/src/tests.rs b/crates/vfs_canister/src/tests.rs index 2ef3323b..e50fff37 100644 --- a/crates/vfs_canister/src/tests.rs +++ b/crates/vfs_canister/src/tests.rs @@ -7,12 +7,10 @@ use std::task::{Context, Poll, Waker}; use candid::{Encode, Nat, Principal}; use sha2::{Digest, Sha256}; use tempfile::tempdir; -use vfs_runtime::{ - CreditsPendingLedgerDetailsInput, DatabaseCreditPurchaseWithLedgerDetails, VfsService, -}; +use vfs_runtime::{DEFAULT_LLM_WRITER_PRINCIPAL, VfsService}; use vfs_types::{ - AppendNodeRequest, CreateDatabaseRequest, CreditsConfig, CreditsConfigUpdate, - DatabaseCreditPurchaseRequest, DatabaseRestoreChunkRequest, DatabaseRole, DatabaseStatus, + AppendNodeRequest, CreateDatabaseRequest, CyclesBillingConfig, CyclesBillingConfigUpdate, + DatabaseCyclesPurchaseRequest, DatabaseRestoreChunkRequest, DatabaseRole, DatabaseStatus, DeleteDatabaseRequest, DeleteNodeRequest, EditNodeRequest, ExportSnapshotRequest, FetchUpdatesRequest, GlobNodeType, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, IncomingLinksRequest, KINIC_LEDGER_FEE_E8S, ListChildrenRequest, @@ -24,24 +22,24 @@ use vfs_types::{ use super::{ Icrc21ConsentMessage, Icrc21ConsentMessageMetadata, Icrc21ConsentMessageRequest, - Icrc21ConsentMessageResponse, Icrc21ConsentMessageSpec, IcrcAccount, LedgerTransaction, - LedgerTransfer, LedgerTransferFromOutcome, SERVICE, TransferFromError, append_node, - begin_database_archive, begin_database_restore, cancel_database_archive, - check_database_write_credits, clear_last_ledger_memo_for_test, + Icrc21ConsentMessageResponse, Icrc21ConsentMessageSpec, IcrcAccount, LedgerTransferFromOutcome, + SERVICE, TransferFromError, append_node, begin_database_archive, begin_database_restore, + cancel_database_archive, check_database_write_cycles, clear_last_ledger_memo_for_test, clear_ledger_transactions_for_test, create_database, delete_node, edit_node, export_snapshot, - fail_next_credit_database_purchase_apply_for_test, fail_next_mount_database_file_for_test, - fetch_updates, finalize_database_archive, finalize_database_restore, glob_nodes, - grant_database_access, graph_links, graph_neighborhood, icrc21_canister_call_consent_message, - incoming_links, last_ledger_from_for_test, last_ledger_memo_for_test, list_children, - list_database_credit_entries, list_database_credit_pending_operations, list_database_members, - list_databases, list_nodes, memory_manifest, mkdir_node, move_node, multi_edit_node, - outgoing_links, parse_upgrade_credits_config_arg, preview_database_credit_purchase, - purchase_database_credits, query_context, query_index_sql_json, read_database_archive_chunk, - read_node, read_node_context, rename_database, repair_database_credit_purchase_cancel, - repair_database_credit_purchase_complete, revoke_database_access, search_node_paths, - search_nodes, set_ledger_transaction_for_test, set_next_ledger_transfer_from_outcome_for_test, - set_test_caller_principal_for_test, source_evidence, status, transfer_from_error_outcome, - write_database_restore_chunk, write_node, write_nodes, + fail_next_apply_database_cycles_purchase_apply_for_test, + fail_next_mount_database_file_for_test, fetch_updates, finalize_database_archive, + finalize_database_restore, get_cycles_billing_config, glob_nodes, grant_database_access, + graph_links, graph_neighborhood, icrc21_canister_call_consent_message, incoming_links, + last_ledger_from_for_test, last_ledger_memo_for_test, ledger_transfer_fees_for_test, + list_children, list_database_cycle_entries, list_database_cycles_pending_purchases, + list_database_members, list_databases, list_nodes, memory_manifest, mkdir_node, move_node, + multi_edit_node, outgoing_links, parse_upgrade_cycles_billing_config_arg, + purchase_database_cycles, query_context, query_index_sql_json, read_database_archive_chunk, + read_node, read_node_context, rename_database, revoke_database_access, search_node_paths, + search_nodes, set_cycle_balances_for_test, set_next_ledger_transfer_from_outcome_for_test, + set_test_caller_principal_for_test, settle_database_storage_charges, source_evidence, status, + transfer_from_error_outcome, update_cycles_billing_config, write_database_restore_chunk, + write_node, write_nodes, }; fn install_test_service() { @@ -56,24 +54,26 @@ fn install_test_service() { .create_database("default", "2vxsx-fae", 1_700_000_000_000) .expect("default database should create"); service - .begin_database_credit_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) + .begin_database_cycles_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( + let cycles = cycles_for_test_payment(&service, 1_000_000); + service.complete_database_cycles_purchase_ledger_transfer( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, + 1, )?; - service.credit_database_purchase( + service.apply_database_cycles_purchase( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, 1, 1_700_000_000_001, ) }) - .expect("default database should have write credits available"); + .expect("default database should have write cycles available"); SERVICE.with(|slot| *slot.borrow_mut() = Some(service)); } @@ -97,6 +97,48 @@ fn database_status_and_mount(database_id: &str) -> (DatabaseStatus, Option) }) } +fn database_exists(database_id: &str) -> bool { + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .list_database_infos() + .expect("database infos should load") + .into_iter() + .any(|info| info.database_id == database_id) + }) +} + +fn pending_cycle_purchase_state(database_id: &str) -> String { + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .query_index_sql_json( + &format!( + "SELECT json_object('status', operation_status, 'block', ledger_block_index) FROM database_cycle_pending_operations WHERE database_id = '{}' AND kind = 'cycles_purchase' LIMIT 1", + database_id + ), + 1, + ) + .expect("pending operation should query") + .rows + .into_iter() + .next() + .expect("pending operation should exist") + }) +} + +fn cycles_for_test_payment(service: &VfsService, payment_amount_e8s: u64) -> u64 { + super::cycles_for_payment_amount_e8s( + payment_amount_e8s, + &service + .cycles_billing_config() + .expect("cycles config should load"), + ) + .expect("cycles amount should compute") +} + fn install_empty_test_service() { let dir = tempdir().expect("tempdir should create"); let root = dir.keep(); @@ -118,14 +160,14 @@ fn block_on_ready(future: impl Future) -> T { } } -fn credit_purchase_request(database_id: &str, credits: u64) -> DatabaseCreditPurchaseRequest { - let preview = preview_database_credit_purchase(database_id.to_string(), credits) - .expect("credit purchase preview should load"); - DatabaseCreditPurchaseRequest { +fn cycles_purchase_request( + database_id: &str, + payment_amount_e8s: u64, +) -> DatabaseCyclesPurchaseRequest { + DatabaseCyclesPurchaseRequest { database_id: database_id.to_string(), - credits, - expected_payment_amount_e8s: preview.payment_amount_e8s, - expected_config_version: preview.config_version, + payment_amount_e8s, + min_expected_cycles: 1, } } @@ -143,71 +185,26 @@ fn consent_request(method: &str, arg: Vec) -> Icrc21ConsentMessageRequest { } } -fn ledger_transfer_transaction( - from: IcrcAccount, - to: IcrcAccount, - amount_e8s: u64, - ledger_fee_e8s: u64, - memo: Vec, - created_at_time: u64, -) -> LedgerTransaction { - LedgerTransaction { - kind: "transfer".to_string(), - transfer: Some(LedgerTransfer { - from, - to, - amount: Nat::from(amount_e8s), - fee: Some(Nat::from(ledger_fee_e8s)), - memo: Some(memo), - created_at_time: Some(created_at_time), - spender: None, - }), - } -} - -fn pending_credit_purchase_transaction( - operation: &vfs_types::DatabaseCreditPendingOperation, -) -> LedgerTransaction { - ledger_transfer_transaction( - IcrcAccount { - owner: Principal::from_text(operation.from_owner.as_ref().expect("from owner")) - .expect("from owner should parse"), - subaccount: operation.from_subaccount.clone(), - }, - IcrcAccount { - owner: Principal::from_text(operation.to_owner.as_ref().expect("to owner")) - .expect("to owner should parse"), - subaccount: operation.to_subaccount.clone(), - }, - operation.payment_amount_e8s, - operation.ledger_fee_e8s.expect("ledger fee"), - format!("kinic:vfs:credit_purchase:{}", operation.operation_id).into_bytes(), - operation - .ledger_created_at_time_ns - .expect("ledger created_at"), - ) -} - -fn explicit_credits_config() -> CreditsConfig { - CreditsConfig { +fn explicit_cycles_billing_config() -> CyclesBillingConfig { + CyclesBillingConfig { kinic_ledger_canister_id: "aaaaa-aa".to_string(), - sns_governance_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), - credits_per_kinic: 1_000, - min_update_credits: 1, + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), + cycles_per_kinic: 1_000, + min_update_cycles: 1_000_000, } } #[test] -fn credits_config_rejects_anonymous_principals() { +fn cycles_billing_config_rejects_anonymous_principals() { let dir = tempdir().expect("tempdir should create"); let root = dir.keep(); let service = VfsService::new(root.join("index.sqlite3"), root.join("databases")); - let mut config = explicit_credits_config(); - config.sns_governance_id = Principal::anonymous().to_text(); + let mut config = explicit_cycles_billing_config(); + config.billing_authority_id = Principal::anonymous().to_text(); let error = service .run_index_migrations_with_config(config) - .expect_err("anonymous governance should reject"); + .expect_err("anonymous billing authority should reject"); assert!(error.contains("principal must not be anonymous")); } @@ -218,7 +215,7 @@ fn controller_can_query_index_sql_json() { set_test_caller_principal_for_test(Principal::management_canister()); let result = query_index_sql_json( - "SELECT json_object('credit_purchase_credits', COALESCE(SUM(amount_credits), 0)) FROM database_credit_ledger WHERE kind = 'credit_purchase' LIMIT 1".to_string(), + "SELECT json_object('cycles_purchase_cycles', COALESCE(SUM(amount_cycles), 0)) FROM database_cycle_ledger WHERE kind = 'cycles_purchase' LIMIT 1".to_string(), 10, ) .expect("controller should query index SQL"); @@ -227,7 +224,7 @@ fn controller_can_query_index_sql_json() { assert_eq!(result.row_count, 1); assert_eq!( result.rows, - vec![r#"{"credit_purchase_credits":1000000}"#.to_string()] + vec![r#"{"cycles_purchase_cycles":2345000000}"#.to_string()] ); } @@ -244,6 +241,18 @@ fn index_sql_json_rejects_non_controller_callers() { assert!(error.contains("caller is not a canister controller")); } +#[test] +fn settle_database_storage_charges_rejects_non_controller_callers() { + install_test_service(); + let non_controller = Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai") + .expect("valid non-controller principal"); + set_test_caller_principal_for_test(non_controller); + + let error = settle_database_storage_charges().expect_err("non-controller should reject"); + + assert!(error.contains("caller is not a canister controller")); +} + #[test] fn index_sql_json_rejects_anonymous_callers() { install_test_service(); @@ -261,12 +270,12 @@ fn index_sql_json_rejects_mutating_and_multi_statement_sql() { set_test_caller_principal_for_test(Principal::management_canister()); for sql in [ - "UPDATE database_credit_accounts SET balance_credits = 0", - "DELETE FROM database_credit_ledger", - "INSERT INTO database_credit_ledger (database_id) VALUES ('x')", + "UPDATE database_cycle_accounts SET balance_cycles = 0", + "DELETE FROM database_cycle_ledger", + "INSERT INTO database_cycle_ledger (database_id) VALUES ('x')", "CREATE TABLE x (id INTEGER)", - "DROP TABLE database_credit_ledger", - "PRAGMA table_info(database_credit_ledger)", + "DROP TABLE database_cycle_ledger", + "PRAGMA table_info(database_cycle_ledger)", "ATTACH DATABASE 'x' AS x", "SELECT json_object('ok', 1); SELECT json_object('ok', 2)", ] { @@ -289,29 +298,21 @@ fn index_sql_json_requires_text_json_first_column() { assert!(error.contains("one non-null TEXT JSON column")); } -fn fund_database(database_id: &str, amount_credits: u64, ledger_block_index: u64) { +fn fund_database(database_id: &str, payment_amount_e8s: u64, ledger_block_index: u64) { let principal = Principal::management_canister().to_text(); SERVICE.with(|slot| { let service = slot.borrow(); let service = service.as_ref().expect("service should be installed"); let operation_id = service - .begin_database_credit_purchase( + .begin_database_cycles_purchase( database_id, &principal, - amount_credits, + payment_amount_e8s, 1_700_000_000_000, ) - .expect("database credit purchase should begin"); - service - .mark_database_credit_purchase_completed( - operation_id, - database_id, - &principal, - amount_credits, - ) - .expect("database credit purchase should be marked completed"); + .expect("database cycle purchase should begin"); if service - .activate_pending_database_for_credit_purchase(database_id, 1_700_000_000_000) + .activate_pending_database_for_cycles_purchase(database_id, 1_700_000_000_000) .expect("pending database activation should prepare") .is_some() { @@ -319,12 +320,28 @@ fn fund_database(database_id: &str, amount_credits: u64, ledger_block_index: u64 .run_pending_database_migrations(database_id) .expect("pending database migrations should run"); } + let cycles = super::cycles_for_payment_amount_e8s( + payment_amount_e8s, + &service + .cycles_billing_config() + .expect("cycles config should load"), + ) + .expect("cycles amount should compute"); service - .credit_database_purchase( + .complete_database_cycles_purchase_ledger_transfer( operation_id, database_id, &principal, - amount_credits, + cycles, + ledger_block_index, + ) + .expect("ledger transfer should be marked complete"); + service + .apply_database_cycles_purchase( + operation_id, + database_id, + &principal, + cycles, ledger_block_index, 1_700_000_000_000, ) @@ -332,8 +349,9 @@ fn fund_database(database_id: &str, amount_credits: u64, ledger_block_index: u64 }); } -fn test_governance_principal() -> Principal { - Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai").expect("governance principal should parse") +fn test_billing_authority_principal() -> Principal { + Principal::from_text("rrkah-fqaaa-aaaaa-aaaaq-cai") + .expect("billing authority principal should parse") } struct AuthenticatedCallerGuard; @@ -357,32 +375,48 @@ impl Drop for AuthenticatedCallerGuard { } #[test] -fn post_upgrade_credits_config_arg_accepts_no_arg() { +fn update_cycles_billing_config_accepts_record_argument() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install_principal(test_billing_authority_principal()); + + update_cycles_billing_config(CyclesBillingConfigUpdate { + cycles_per_kinic: 469_000_000_000, + min_update_cycles: 2_000_000, + }) + .expect("cycles config update should accept record argument"); + + let config = get_cycles_billing_config().expect("cycles config should load"); + assert_eq!(config.cycles_per_kinic, 469_000_000_000); + assert_eq!(config.min_update_cycles, 2_000_000); +} + +#[test] +fn post_upgrade_cycles_billing_config_arg_accepts_no_arg() { let bytes = Encode!().expect("empty candid args should encode"); - let parsed = - parse_upgrade_credits_config_arg(&bytes).expect("empty post-upgrade arg should parse"); + let parsed = parse_upgrade_cycles_billing_config_arg(&bytes) + .expect("empty post-upgrade arg should parse"); assert_eq!(parsed, None); } #[test] -fn post_upgrade_credits_config_arg_accepts_bare_config() { - let config = explicit_credits_config(); - let bytes = Encode!(&config).expect("credits config should encode"); +fn post_upgrade_cycles_billing_config_arg_accepts_bare_config() { + let config = explicit_cycles_billing_config(); + let bytes = Encode!(&config).expect("cycles config should encode"); - let parsed = - parse_upgrade_credits_config_arg(&bytes).expect("bare post-upgrade config should parse"); + let parsed = parse_upgrade_cycles_billing_config_arg(&bytes) + .expect("bare post-upgrade config should parse"); assert_eq!(parsed, Some(config)); } #[test] -fn post_upgrade_credits_config_arg_accepts_optional_config() { - let config = explicit_credits_config(); - let bytes = Encode!(&Some(config.clone())).expect("optional credits config should encode"); +fn post_upgrade_cycles_billing_config_arg_accepts_optional_config() { + let config = explicit_cycles_billing_config(); + let bytes = Encode!(&Some(config.clone())).expect("optional cycles config should encode"); - let parsed = parse_upgrade_credits_config_arg(&bytes) + let parsed = parse_upgrade_cycles_billing_config_arg(&bytes) .expect("optional post-upgrade config should parse"); assert_eq!(parsed, Some(config)); @@ -401,7 +435,21 @@ fn transfer_from_duplicate_outcome_is_completed() { } #[test] -fn purchase_database_credits_credits_completed_transfer_from() { +fn transfer_from_bad_fee_outcome_is_typed() { + let outcome = transfer_from_error_outcome(TransferFromError::BadFee { + expected_fee: Nat::from(99_u64), + }); + + match outcome { + LedgerTransferFromOutcome::BadFee { expected_fee_e8s } => { + assert_eq!(expected_fee_e8s, 99); + } + other => panic!("bad fee should be typed, got {other:?}"), + } +} + +#[test] +fn purchase_database_cycles_cycles_completed_transfer_from() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { @@ -410,54 +458,266 @@ fn purchase_database_credits_credits_completed_transfer_from() { .expect("database should create"); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); - let result = block_on_ready(purchase_database_credits(credit_purchase_request( + let result = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 500, ))) - .expect("completed transfer-from should credit database"); + .expect("completed transfer-from should cycle database"); assert_eq!(result.block_index, 42); - assert_eq!(result.balance_credits, 500); + assert_eq!(result.amount_cycles, 1_172_500); + assert_eq!(result.balance_cycles, 1_172_500); assert_eq!( database_status_and_mount(&database.database_id).0, DatabaseStatus::Active ); - let entries = list_database_credit_entries(database.database_id.clone(), None, 10) + let entries = list_database_cycle_entries(database.database_id.clone(), None, 10) .expect("database ledger should load") .entries; assert_eq!(entries.len(), 1); - assert_eq!(entries[0].kind, "credit_purchase"); + assert_eq!(entries[0].kind.as_str(), "cycles_purchase"); assert_eq!(entries[0].ledger_block_index, Some(42)); } #[test] -fn preview_database_credit_purchase_rejects_invalid_target_before_approve() { +fn purchase_database_cycles_rejects_anonymous_before_ledger_call() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Anonymous purchase".to_string(), + }) + .expect("database should create"); + drop(_caller); + clear_last_ledger_memo_for_test(); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); + + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 500, + ))) + .expect_err("anonymous caller should reject before ledger transfer"); + + assert!(error.contains("anonymous caller not allowed")); + assert_eq!(last_ledger_memo_for_test(), None); + assert_eq!( + database_status_and_mount(&database.database_id), + (DatabaseStatus::Pending, None) + ); +} + +#[test] +fn purchase_database_cycles_treats_duplicate_as_completed_transfer() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { - name: "Preview".to_string(), + name: "Duplicate ledger".to_string(), }) .expect("database should create"); + set_next_ledger_transfer_from_outcome_for_test(transfer_from_error_outcome( + TransferFromError::Duplicate { + duplicate_of: Nat::from(77_u64), + }, + )); - let preview = preview_database_credit_purchase(database.database_id.clone(), 500) - .expect("preview should accept"); - assert_eq!(preview.payment_amount_e8s, 50_000_000); - assert_eq!(preview.ledger_fee_e8s, KINIC_LEDGER_FEE_E8S); - assert_eq!(preview.credits_per_kinic, 1_000); - assert_eq!(preview.config_version, 1); - let zero = preview_database_credit_purchase(database.database_id.clone(), 0) - .expect_err("zero amount should reject"); - assert!(zero.contains("credit purchase credits must be positive")); - let overflow = preview_database_credit_purchase(database.database_id.clone(), i64::MAX as u64) - .expect_err("payment amount overflow should reject before approve"); - assert!(overflow.contains("credit purchase payment amount overflow")); - let missing = preview_database_credit_purchase("missing".to_string(), 500) - .expect_err("missing database should reject"); + let result = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 500, + ))) + .expect("duplicate transfer-from should cycle database"); + + assert_eq!(result.block_index, 77); + assert_eq!( + database_status_and_mount(&database.database_id).0, + DatabaseStatus::Active + ); + let entries = list_database_cycle_entries(database.database_id, None, 10) + .expect("database ledger should load") + .entries; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].kind, "cycles_purchase"); + assert_eq!(entries[0].ledger_block_index, Some(77)); +} + +#[test] +fn list_database_cycle_entries_paginates_with_clamped_limits() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Cycle history pages".to_string(), + }) + .expect("database should create"); + for index in 0..105 { + fund_database(&database.database_id, 500, index + 1); + } + + let minimum_page = list_database_cycle_entries(database.database_id.clone(), None, 0) + .expect("minimum page should load"); + assert_eq!(minimum_page.entries.len(), 1); + assert_eq!(minimum_page.entries[0].entry_id, 1); + assert_eq!(minimum_page.next_cursor, Some(1)); + + let first_page = list_database_cycle_entries(database.database_id.clone(), None, 200) + .expect("first clamped page should load"); + assert_eq!(first_page.entries.len(), 100); + assert_eq!(first_page.entries[0].entry_id, 1); + assert_eq!(first_page.entries[99].entry_id, 100); + assert_eq!(first_page.next_cursor, Some(100)); + + let second_page = + list_database_cycle_entries(database.database_id, first_page.next_cursor, 200) + .expect("second clamped page should load"); + assert_eq!(second_page.entries.len(), 5); + assert_eq!(second_page.entries[0].entry_id, 101); + assert_eq!(second_page.entries[4].entry_id, 105); + assert_eq!(second_page.next_cursor, None); +} + +#[test] +fn purchase_database_cycles_rejects_bad_fee_without_credit() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Bad fee".to_string(), + }) + .expect("database should create"); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::BadFee { + expected_fee_e8s: KINIC_LEDGER_FEE_E8S + 1, + }); + + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 500, + ))) + .expect_err("BadFee should reject and leave no credit"); + + assert!(error.contains("BadFee expected fee")); + assert!(error.contains("re-approve with the current ledger fee")); + assert_eq!(ledger_transfer_fees_for_test(), vec![KINIC_LEDGER_FEE_E8S]); + assert!( + list_database_cycle_entries(database.database_id.clone(), None, 10) + .expect("ledger should load") + .entries + .is_empty() + ); + assert_eq!( + database_status_and_mount(&database.database_id), + (DatabaseStatus::Pending, None) + ); +} + +#[test] +fn purchase_database_cycles_rejects_invalid_target_before_ledger_call() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Purchase validation".to_string(), + }) + .expect("database should create"); + + clear_last_ledger_memo_for_test(); + let zero = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 0, + ))) + .expect_err("zero amount should reject"); + assert!(zero.contains("cycles purchase payment amount must be positive")); + + let overflow = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + i64::MAX as u64, + ))) + .expect_err("payment amount overflow should reject before approve"); + assert!(overflow.contains("cycles purchase amount exceeds u64")); + + let missing = block_on_ready(purchase_database_cycles(cycles_purchase_request( + "missing", 500, + ))) + .expect_err("missing database should reject"); assert!(missing.contains("database not found")); + assert_eq!(last_ledger_memo_for_test(), None); +} + +#[test] +fn purchase_database_cycles_rejects_archive_restore_statuses() { + install_test_service(); + let _owner = AuthenticatedCallerGuard::install_principal(Principal::anonymous()); + + let archive = begin_database_archive("default".to_string()).expect("archive should begin"); + clear_last_ledger_memo_for_test(); + let archiving = { + let _caller = AuthenticatedCallerGuard::install(); + block_on_ready(purchase_database_cycles(cycles_purchase_request( + "default", 500, + ))) + .expect_err("archiving database should reject purchase") + }; + assert!(archiving.contains("database is archiving")); + assert_eq!(last_ledger_memo_for_test(), None); + + let bytes = read_database_archive_chunk( + "default".to_string(), + 0, + archive + .size_bytes + .try_into() + .expect("test archive should fit in one chunk"), + ) + .expect("archive chunk should read") + .bytes; + let snapshot_hash = sha256_bytes(&bytes); + finalize_database_archive("default".to_string(), snapshot_hash.clone()) + .expect("archive should finalize"); + let archived = { + let _caller = AuthenticatedCallerGuard::install(); + block_on_ready(purchase_database_cycles(cycles_purchase_request( + "default", 500, + ))) + .expect_err("archived database should reject purchase") + }; + assert!(archived.contains("database is archived")); + + begin_database_restore("default".to_string(), snapshot_hash, archive.size_bytes) + .expect("restore should begin"); + let restoring = { + let _caller = AuthenticatedCallerGuard::install(); + block_on_ready(purchase_database_cycles(cycles_purchase_request( + "default", 500, + ))) + .expect_err("restoring database should reject purchase") + }; + assert!(restoring.contains("database is restoring")); +} + +#[test] +fn begin_database_archive_rejects_pending_cycle_purchase() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + let database = create_database(CreateDatabaseRequest { + name: "Pending lifecycle".to_string(), + }) + .expect("database should create"); + fund_database(&database.database_id, 1_000_000, 41); + SERVICE.with(|slot| { + slot.borrow() + .as_ref() + .expect("service should be installed") + .begin_database_cycles_purchase( + &database.database_id, + &Principal::management_canister().to_text(), + 500, + 1_700_000_000_001, + ) + .expect("cycle purchase should begin") + }); + + let error = begin_database_archive(database.database_id) + .expect_err("archive should reject pending cycle operation"); + + assert!(error.contains("pending cycle operation")); } #[test] -fn purchase_database_credits_rejects_balance_overflow_before_ledger_call() { +fn purchase_database_cycles_rejects_balance_overflow_before_ledger_call() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { @@ -468,24 +728,23 @@ fn purchase_database_credits_rejects_balance_overflow_before_ledger_call() { slot.borrow() .as_ref() .expect("service should be installed") - .update_credits_config( - CreditsConfigUpdate { - credits_per_kinic: 100_000_000, - min_update_credits: 1, + .update_cycles_billing_config( + CyclesBillingConfigUpdate { + cycles_per_kinic: 100_000_000, + min_update_cycles: 1, }, - &test_governance_principal().to_text(), + &test_billing_authority_principal().to_text(), ) - .expect("credits config should update"); + .expect("cycles config should update"); }); fund_database(&database.database_id, i64::MAX as u64, 41); clear_last_ledger_memo_for_test(); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); - let error = block_on_ready(purchase_database_credits(DatabaseCreditPurchaseRequest { + let error = block_on_ready(purchase_database_cycles(DatabaseCyclesPurchaseRequest { database_id: database.database_id, - credits: 1, - expected_payment_amount_e8s: 1, - expected_config_version: 1, + payment_amount_e8s: 1, + min_expected_cycles: 1, })) .expect_err("overflow should reject before ledger"); @@ -494,39 +753,45 @@ fn purchase_database_credits_rejects_balance_overflow_before_ledger_call() { } #[test] -fn purchase_database_credits_rejects_stale_preview_before_ledger_call() { +fn purchase_database_cycles_uses_current_config_amount() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { - name: "Stale preview".to_string(), + name: "Current config".to_string(), }) .expect("database should create"); - let request = credit_purchase_request(&database.database_id, 500); SERVICE.with(|slot| { slot.borrow() .as_ref() .expect("service should be installed") - .update_credits_config( - CreditsConfigUpdate { - credits_per_kinic: 2_000, - min_update_credits: 1, + .update_cycles_billing_config( + CyclesBillingConfigUpdate { + cycles_per_kinic: 234_500_000_000, + min_update_cycles: 2, }, - &test_governance_principal().to_text(), + &test_billing_authority_principal().to_text(), ) - .expect("credits config should update"); + .expect("cycles config should update"); }); clear_last_ledger_memo_for_test(); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); - let error = block_on_ready(purchase_database_credits(request)) - .expect_err("stale preview should reject before ledger"); + let result = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 500, + ))) + .expect("purchase should use current config"); - assert!(error.contains("credits config changed")); - assert_eq!(last_ledger_memo_for_test(), None); + assert_eq!(result.amount_cycles, 1_172_500); + assert_eq!(result.balance_cycles, 1_172_500); + let entries = list_database_cycle_entries(database.database_id, None, 10) + .expect("cycles history should load") + .entries; + assert_eq!(entries[0].amount_cycles, 1_172_500); } #[test] -fn purchase_database_credits_leaves_balance_on_ledger_reject() { +fn purchase_database_cycles_leaves_balance_on_ledger_reject() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { @@ -537,25 +802,25 @@ fn purchase_database_credits_leaves_balance_on_ledger_reject() { "icrc2_transfer_from failed: InsufficientAllowance".to_string(), )); - let error = block_on_ready(purchase_database_credits(credit_purchase_request( + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 500, ))) - .expect_err("ledger reject should not credit database"); + .expect_err("ledger reject should not cycle database"); assert!(error.contains("InsufficientAllowance")); assert_eq!( database_status_and_mount(&database.database_id), (DatabaseStatus::Pending, None) ); - let entries = list_database_credit_entries(database.database_id.clone(), None, 10) + let entries = list_database_cycle_entries(database.database_id.clone(), None, 10) .expect("database ledger should load") .entries; assert!(entries.is_empty()); } #[test] -fn purchase_database_credits_records_ambiguous_transfer_from() { +fn purchase_database_cycles_keeps_ambiguous_transfer_from_for_review() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { @@ -566,430 +831,180 @@ fn purchase_database_credits_records_ambiguous_transfer_from() { "icrc2_transfer_from decode failed".to_string(), )); - let error = block_on_ready(purchase_database_credits(credit_purchase_request( + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 500, ))) - .expect_err("ambiguous transfer-from should return pending error"); + .expect_err("ambiguous transfer-from should require review"); - assert!(error.contains("credit purchase pending operation")); - assert!(error.contains("manual repair required")); - let entries = list_database_credit_entries(database.database_id.clone(), None, 10) - .expect("database ledger should load") - .entries; - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].kind, "credit_purchase_ambiguous"); - assert_eq!(entries[0].amount_credits, 0); - assert_eq!(entries[0].balance_after_credits, 0); - assert_eq!(entries[0].ledger_block_index, None); + assert!(error.contains("result ambiguous")); + assert!(error.contains("operation_id 1")); + assert!(error.contains("billing authority review required")); + assert_eq!( + pending_cycle_purchase_state(&database.database_id), + r#"{"status":"ambiguous","block":null}"# + ); + assert!(database_exists(&database.database_id)); assert_eq!( database_status_and_mount(&database.database_id), (DatabaseStatus::Pending, None) ); + let duplicate = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 500, + ))) + .expect_err("ambiguous pending operation should block duplicate purchase"); + assert!( + duplicate.contains("database activation is pending") + || duplicate.contains("cycles purchase already pending") + ); +} + +#[test] +fn list_database_cycles_pending_purchases_allows_owner_authority_and_payer() { + install_empty_test_service(); + let owner = Principal::management_canister(); + let payer = Principal::from_text(DEFAULT_LLM_WRITER_PRINCIPAL).expect("payer should parse"); + let stranger = + Principal::from_text("bkyz2-fmaaa-aaaaa-qaaaq-cai").expect("stranger should parse"); + let database = { + let _owner = AuthenticatedCallerGuard::install_principal(owner); + create_database(CreateDatabaseRequest { + name: "Pending".to_string(), + }) + .expect("database should create") + }; + + let payer_guard = AuthenticatedCallerGuard::install_principal(payer); + set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( + "timeout".to_string(), + )); + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 50_000_000, + ))) + .expect_err("ambiguous transfer should keep pending operation"); + assert!(error.contains("billing authority review required")); + + let payer_view = list_database_cycles_pending_purchases(database.database_id.clone()) + .expect("payer should view own pending purchase"); + assert_eq!(payer_view.len(), 1); + assert_eq!(payer_view[0].status, "ambiguous"); + assert_eq!(payer_view[0].required_action, "billing_authority_review"); + + drop(payer_guard); + let owner_guard = AuthenticatedCallerGuard::install_principal(owner); + let owner_view = list_database_cycles_pending_purchases(database.database_id.clone()) + .expect("owner should view pending purchase"); + assert_eq!(owner_view, payer_view); + + drop(owner_guard); + let authority_guard = + AuthenticatedCallerGuard::install_principal(test_billing_authority_principal()); + let authority_view = list_database_cycles_pending_purchases(database.database_id.clone()) + .expect("billing authority should view pending purchase"); + assert_eq!(authority_view, payer_view); + + drop(authority_guard); + let _stranger = AuthenticatedCallerGuard::install_principal(stranger); + let error = list_database_cycles_pending_purchases(database.database_id) + .expect_err("unrelated caller should reject"); + assert!(error.contains("cannot view pending cycle purchases")); } #[test] -fn purchase_database_credits_mount_failure_keeps_pending_operation_for_repair() { +fn purchase_database_cycles_mount_failure_keeps_completed_pending_operation() { install_empty_test_service(); let _owner = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { - name: "Mount retry".to_string(), + name: "Mount review".to_string(), }) .expect("database should create"); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); fail_next_mount_database_file_for_test(); - let error = block_on_ready(purchase_database_credits(credit_purchase_request( + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 500, ))) - .expect_err("mount failure after ledger success should keep repair path"); + .expect_err("mount failure after ledger success should not credit"); assert!(error.contains("test mount failure")); - assert!(error.contains("credit purchase payment completed")); - assert!(error.contains("verified ledger block")); - assert!(error.contains("block 42")); + assert!(error.contains("remains completed for billing authority review")); assert_eq!( - database_status_and_mount(&database.database_id), - (DatabaseStatus::Pending, Some(11)) + pending_cycle_purchase_state(&database.database_id), + r#"{"status":"completed","block":42}"# + ); + assert!(database_exists(&database.database_id)); + assert_eq!( + database_status_and_mount(&database.database_id).0, + DatabaseStatus::Pending ); - let pending = list_database_credit_pending_operations(database.database_id.clone(), None, 10) - .expect("pending should load") - .entries; - assert_eq!(pending.len(), 1); - assert!(error.contains(&format!("pending operation {}", pending[0].operation_id))); - assert_eq!(pending[0].operation_status, "completed"); - assert_eq!(pending[0].credits, 500); - assert_eq!(pending[0].payment_amount_e8s, 50_000_000); - assert_eq!(pending[0].ledger_fee_e8s, Some(KINIC_LEDGER_FEE_E8S)); assert!( - list_database_credit_entries(database.database_id.clone(), None, 10) - .expect("ledger should load") + list_database_cycle_entries(database.database_id.clone(), None, 10) + .expect("database ledger should load") .entries .is_empty() ); - - set_ledger_transaction_for_test(42, pending_credit_purchase_transaction(&pending[0])); - let result = block_on_ready(repair_database_credit_purchase_complete( - database.database_id.clone(), - pending[0].operation_id, - 42, - )) - .expect("verified complete should retry mount and credit"); - - assert_eq!(result.balance_credits, 500); assert_eq!( - database_status_and_mount(&database.database_id).0, - DatabaseStatus::Active + list_database_cycles_pending_purchases(database.database_id) + .expect("owner should view completed pending purchase")[0] + .required_action, + "billing_authority_review" ); } #[test] -fn repair_complete_succeeds_after_activation_started_and_credit_apply_failed() { +fn purchase_database_cycles_apply_failure_keeps_completed_pending_for_review() { install_empty_test_service(); let _owner = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { - name: "Apply retry".to_string(), + name: "Apply review".to_string(), }) .expect("database should create"); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(44)); - fail_next_credit_database_purchase_apply_for_test(); + fail_next_apply_database_cycles_purchase_apply_for_test(); - let error = block_on_ready(purchase_database_credits(credit_purchase_request( + let error = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 600, ))) - .expect_err("credit apply failure after activation should keep repair path"); + .expect_err("cycle apply failure after activation should not credit"); - assert!(error.contains("test credit purchase apply failure")); + assert!(error.contains("test cycle purchase apply failure")); + assert!(error.contains("remains completed for billing authority review")); assert_eq!( - database_status_and_mount(&database.database_id), - (DatabaseStatus::Pending, Some(11)) - ); - let pending = list_database_credit_pending_operations(database.database_id.clone(), None, 10) - .expect("pending should load") - .entries; - assert_eq!(pending.len(), 1); - assert!( - list_database_credit_entries(database.database_id.clone(), None, 10) - .expect("ledger should load") - .entries - .is_empty() + pending_cycle_purchase_state(&database.database_id), + r#"{"status":"completed","block":44}"# ); - let cancel_error = repair_database_credit_purchase_cancel( - database.database_id.clone(), - pending[0].operation_id, - ) - .expect_err("completed payment should not be cancellable"); - assert!(cancel_error.contains("credit purchase operation is completed")); - - set_ledger_transaction_for_test(44, pending_credit_purchase_transaction(&pending[0])); - let result = block_on_ready(repair_database_credit_purchase_complete( - database.database_id.clone(), - pending[0].operation_id, - 44, - )) - .expect("repair complete should finish activation and credit"); - - assert_eq!(result.balance_credits, 600); + assert!(database_exists(&database.database_id)); assert_eq!( database_status_and_mount(&database.database_id).0, - DatabaseStatus::Active + DatabaseStatus::Pending ); -} - -#[test] -fn repair_cancel_rejects_in_flight_credit_purchase() { - install_empty_test_service(); - let _owner = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "In flight cancel".to_string(), - }) - .expect("database should create"); - let caller = Principal::management_canister().to_text(); - let operation_id = SERVICE.with(|slot| { - slot.borrow() - .as_ref() - .expect("service should be installed") - .begin_database_credit_purchase(&database.database_id, &caller, 500, 1_700_000_000_000) - .expect("credit purchase should begin") - }); - - let error = repair_database_credit_purchase_cancel(database.database_id, operation_id) - .expect_err("in-flight purchase cancel should reject"); - - assert!(error.contains("credit purchase operation is in_flight")); -} - -#[test] -fn repair_complete_accepts_verified_in_flight_credit_purchase() { - install_empty_test_service(); - let _owner = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "In flight complete".to_string(), - }) - .expect("database should create"); - let caller = Principal::management_canister().to_text(); - let canister_owner = Principal::management_canister().to_text(); - let preview = preview_database_credit_purchase(database.database_id.clone(), 500) - .expect("credit purchase preview should load"); - let operation_id = SERVICE.with(|slot| { - slot.borrow() - .as_ref() - .expect("service should be installed") - .begin_database_credit_purchase_with_ledger_details( - DatabaseCreditPurchaseWithLedgerDetails { - database_id: &database.database_id, - caller: &caller, - credits: 500, - expected_payment_amount_e8s: preview.payment_amount_e8s, - expected_config_version: preview.config_version, - ledger: CreditsPendingLedgerDetailsInput { - from_owner: &caller, - from_subaccount: None, - to_owner: &canister_owner, - to_subaccount: None, - ledger_fee_e8s: preview.ledger_fee_e8s, - ledger_created_at_time_ns: 1_700_000_000_000_000_000, - }, - now: 1_700_000_000_000, - }, - ) - .expect("credit purchase should begin") - }); - let pending = list_database_credit_pending_operations(database.database_id.clone(), None, 10) - .expect("pending should load") - .entries; - assert_eq!(pending.len(), 1); - assert_eq!(pending[0].operation_id, operation_id); - assert_eq!(pending[0].operation_status, "in_flight"); - assert_eq!(pending[0].credits, 500); - assert_eq!(pending[0].payment_amount_e8s, 50_000_000); - assert_eq!(pending[0].ledger_fee_e8s, Some(KINIC_LEDGER_FEE_E8S)); assert!( - list_database_credit_entries(database.database_id.clone(), None, 10) - .expect("ledger should load") + list_database_cycle_entries(database.database_id.clone(), None, 10) + .expect("database ledger should load") .entries .is_empty() ); - set_ledger_transaction_for_test(42, pending_credit_purchase_transaction(&pending[0])); - - let result = block_on_ready(repair_database_credit_purchase_complete( - database.database_id.clone(), - operation_id, - 42, - )) - .expect("verified in-flight purchase should complete"); - - assert_eq!(result.block_index, 42); - assert_eq!(result.balance_credits, 500); - assert_eq!( - database_status_and_mount(&database.database_id).0, - DatabaseStatus::Active - ); - let pending = list_database_credit_pending_operations(database.database_id.clone(), None, 10) - .expect("pending should load") - .entries; - assert!(pending.is_empty()); - let entries = list_database_credit_entries(database.database_id, None, 10) - .expect("ledger should load") - .entries; - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].kind, "credit_purchase_repair_complete"); - assert_eq!(entries[0].caller, caller); - assert_eq!(entries[0].ledger_block_index, Some(42)); -} - -#[test] -fn authenticated_caller_can_complete_verified_ambiguous_credit_purchase() { - install_empty_test_service(); - let database_id; - let operation_id; - let payer; - { - let _owner = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "Repair credit purchase".to_string(), - }) - .expect("database should create"); - database_id = database.database_id; - payer = Principal::management_canister().to_text(); - set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( - "icrc2_transfer_from decode failed".to_string(), - )); - let error = block_on_ready(purchase_database_credits(credit_purchase_request( - &database_id, - 500, - ))) - .expect_err("ambiguous transfer-from should return pending error"); - assert!(error.contains("credit purchase pending")); - - let pending = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("owner should list pending operations") - .entries; - assert_eq!(pending.len(), 1); - operation_id = pending[0].operation_id; - set_ledger_transaction_for_test(77, pending_credit_purchase_transaction(&pending[0])); - } - - let repair_caller = Principal::from_text("ryjl3-tyaaa-aaaaa-aaaba-cai") - .expect("non-governance principal should parse"); - { - let _authenticated = AuthenticatedCallerGuard::install_principal(repair_caller); - let result = block_on_ready(repair_database_credit_purchase_complete( - database_id.clone(), - operation_id, - 77, - )) - .expect("authenticated caller should complete verified credit purchase"); - assert_eq!(result.block_index, 77); - assert_eq!(result.balance_credits, 500); - assert_eq!( - database_status_and_mount(&database_id).0, - DatabaseStatus::Active - ); - } - - let _owner = AuthenticatedCallerGuard::install(); - let pending = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("owner should list pending operations") - .entries; - assert!(pending.is_empty()); - let entries = list_database_credit_entries(database_id.clone(), None, 10) - .expect("database ledger should load") - .entries; - assert_eq!(entries[0].kind, "credit_purchase_ambiguous"); - assert_eq!(entries[1].kind, "credit_purchase_repair_complete"); - assert_eq!(entries[0].amount_credits, 0); - assert_eq!(entries[1].amount_credits, 500); - assert_eq!(entries[1].caller, payer); - assert_ne!(entries[1].caller, repair_caller.to_text()); - assert_eq!(entries[1].ledger_block_index, Some(77)); - - let error = block_on_ready(repair_database_credit_purchase_complete( - database_id.clone(), - operation_id, - 77, - )) - .expect_err("second complete should reject missing pending operation"); - assert!(error.contains("pending credit operation not found")); - let entries = list_database_credit_entries(database_id, None, 10) - .expect("database ledger should load") - .entries; - assert_eq!(entries.len(), 2); -} - -#[test] -fn repair_credit_purchase_complete_rejects_mismatched_ledger_block() { - install_empty_test_service(); - let operation_id; - let database_id; - { - let _owner = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "Repair mismatch".to_string(), - }) - .expect("database should create"); - database_id = database.database_id; - set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( - "icrc2_transfer_from decode failed".to_string(), - )); - let _ = block_on_ready(purchase_database_credits(credit_purchase_request( - &database_id, - 500, - ))) - .expect_err("ambiguous credit purchase should stay pending"); - operation_id = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("pending should load") - .entries[0] - .operation_id; - } - - let _owner = AuthenticatedCallerGuard::install(); - let pending = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("pending should load") - .entries; - let mut transaction = pending_credit_purchase_transaction(&pending[0]); - transaction.transfer.as_mut().expect("transfer").amount = Nat::from(499_u64); - set_ledger_transaction_for_test(78, transaction); - - let error = block_on_ready(repair_database_credit_purchase_complete( - database_id.clone(), - operation_id, - 78, - )) - .expect_err("mismatched block should reject"); - assert!(error.contains("amount mismatch")); - let pending = list_database_credit_pending_operations(database_id, None, 10) - .expect("pending should remain") - .entries; - assert_eq!(pending.len(), 1); -} - -#[test] -fn repair_credit_purchase_cancel_removes_ambiguous_operation() { - install_empty_test_service(); - let operation_id; - let database_id; - { - let _owner = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "Cancel credit purchase".to_string(), - }) - .expect("database should create"); - database_id = database.database_id; - set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Ambiguous( - "icrc2_transfer_from call failed".to_string(), - )); - let _ = block_on_ready(purchase_database_credits(credit_purchase_request( - &database_id, - 500, - ))) - .expect_err("ambiguous credit purchase should stay pending"); - operation_id = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("pending should load") - .entries[0] - .operation_id; - - let third_party = Principal::from_text("ryjl3-tyaaa-aaaaa-aaaba-cai") - .expect("third party principal should parse"); - let _third_party = AuthenticatedCallerGuard::install_principal(third_party); - let error = repair_database_credit_purchase_cancel(database_id.clone(), operation_id) - .expect_err("third party cancel should reject"); - assert!(error.contains("not credit purchase payer or database owner")); - } + let duplicate = block_on_ready(purchase_database_cycles(cycles_purchase_request( + &database.database_id, + 600, + ))) + .expect_err("completed pending operation should block duplicate purchase"); + assert!(duplicate.contains("database activation is pending")); - let _owner = AuthenticatedCallerGuard::install(); - repair_database_credit_purchase_cancel(database_id.clone(), operation_id) - .expect("owner should cancel ambiguous credit purchase after verification"); - assert_eq!( - database_status_and_mount(&database_id), - (DatabaseStatus::Pending, None) - ); - let pending = list_database_credit_pending_operations(database_id.clone(), None, 10) - .expect("pending should load") - .entries; - assert!(pending.is_empty()); - let entries = list_database_credit_entries(database_id, None, 10) - .expect("ledger should load") - .entries; assert_eq!( - entries - .iter() - .map(|entry| entry.kind.as_str()) - .collect::>(), - vec![ - "credit_purchase_ambiguous", - "credit_purchase_repair_cancelled" - ] + list_database_cycles_pending_purchases(database.database_id) + .expect("owner should view completed pending purchase")[0] + .required_action, + "billing_authority_review" ); - assert_eq!(entries[0].amount_credits, 0); - assert_eq!(entries[1].amount_credits, 0); - assert_eq!(entries[1].balance_after_credits, 0); } #[test] -fn purchase_database_credits_allows_non_owner_payer() { +fn purchase_database_cycles_allows_non_owner_payer() { install_empty_test_service(); let database_id = { let _owner = AuthenticatedCallerGuard::install(); @@ -1005,14 +1020,15 @@ fn purchase_database_credits_allows_non_owner_payer() { clear_last_ledger_memo_for_test(); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(43)); - let result = block_on_ready(purchase_database_credits(credit_purchase_request( + let result = block_on_ready(purchase_database_cycles(cycles_purchase_request( &database_id, 700, ))) .expect("non-owner payer should fund DB"); assert_eq!(result.block_index, 43); - assert_eq!(result.balance_credits, 700); + assert_eq!(result.amount_cycles, 1_641_500); + assert_eq!(result.balance_cycles, 1_641_500); assert_eq!( last_ledger_from_for_test().expect("ledger from should be recorded"), IcrcAccount { @@ -1023,18 +1039,18 @@ fn purchase_database_credits_allows_non_owner_payer() { } #[test] -fn icrc21_purchase_database_credits_returns_consent_message() { +fn icrc21_purchase_database_cycles_returns_consent_message() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { name: "Consent".to_string(), }) .expect("database should create"); - let request = credit_purchase_request(&database.database_id, 500); + let request = cycles_purchase_request(&database.database_id, 50_000); let arg = Encode!(&request).expect("arg should encode"); let response = - icrc21_canister_call_consent_message(consent_request("purchase_database_credits", arg)); + icrc21_canister_call_consent_message(consent_request("purchase_database_cycles", arg)); let message = match response { Icrc21ConsentMessageResponse::Ok(info) => match info.consent_message { @@ -1045,37 +1061,32 @@ fn icrc21_purchase_database_credits_returns_consent_message() { } }; assert!(message.contains(&database.database_id)); - assert!(message.contains("Credits: `500`")); - assert!(message.contains("Payment: `0.5` KINIC")); - assert!(message.contains("Ledger transfer fee in allowance: `0.0001` KINIC")); + assert!(message.contains("Cycles: `117250000`")); + assert!(message.contains("Payment: `0.0005` KINIC")); + assert!(message.contains("Ledger transfer fee in allowance: `0.001` KINIC")); assert!(message.contains("Spender canister:")); } #[test] -fn icrc21_purchase_database_credits_rejects_stale_expected_amount() { +fn icrc21_purchase_database_cycles_rejects_missing_database() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); - let database = create_database(CreateDatabaseRequest { - name: "Consent mismatch".to_string(), - }) - .expect("database should create"); - let mut request = credit_purchase_request(&database.database_id, 500); - request.expected_payment_amount_e8s += 1; + let request = cycles_purchase_request("missing", 500); let arg = Encode!(&request).expect("arg should encode"); let response = - icrc21_canister_call_consent_message(consent_request("purchase_database_credits", arg)); + icrc21_canister_call_consent_message(consent_request("purchase_database_cycles", arg)); match response { Icrc21ConsentMessageResponse::Err(super::Icrc21Error::UnsupportedCanisterCall(info)) => { - assert!(info.description.contains("payment amount changed")); + assert!(info.description.contains("database not found")); } - other => panic!("stale consent should reject: {other:?}"), + other => panic!("missing database consent should reject: {other:?}"), } } #[test] -fn icrc21_rejects_unsupported_credit_consent_method() { +fn icrc21_rejects_unsupported_cycle_consent_method() { install_empty_test_service(); let response = icrc21_canister_call_consent_message(consent_request("write_node", Vec::new())); @@ -1086,10 +1097,10 @@ fn icrc21_rejects_unsupported_credit_consent_method() { } #[test] -fn icrc21_rejects_malformed_credit_consent_arg() { +fn icrc21_rejects_malformed_cycle_consent_arg() { install_empty_test_service(); let response = icrc21_canister_call_consent_message(consent_request( - "purchase_database_credits", + "purchase_database_cycles", Vec::new(), )); @@ -1100,7 +1111,7 @@ fn icrc21_rejects_malformed_credit_consent_arg() { } #[test] -fn purchase_database_credits_sends_operation_memo_to_ledger() { +fn purchase_database_cycles_sends_operation_memo_to_ledger() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); let database = create_database(CreateDatabaseRequest { @@ -1110,27 +1121,31 @@ fn purchase_database_credits_sends_operation_memo_to_ledger() { clear_last_ledger_memo_for_test(); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(43)); - block_on_ready(purchase_database_credits(credit_purchase_request( + block_on_ready(purchase_database_cycles(cycles_purchase_request( &database.database_id, 700, ))) - .expect("credit purchase should succeed"); + .expect("cycle purchase should succeed"); let memo = String::from_utf8(last_ledger_memo_for_test().expect("memo should be recorded")) .expect("memo should be utf8"); - assert!(memo.starts_with("kinic:vfs:credit_purchase:")); + let operation_id = memo + .strip_prefix("kvfs:cp:") + .expect("memo should use compact cycles purchase prefix") + .parse::() + .expect("memo should end with decimal operation id"); + assert!(operation_id > 0); } #[test] -fn purchase_database_credits_rejects_unknown_and_deleted_database() { +fn purchase_database_cycles_rejects_unknown_and_deleted_database() { install_empty_test_service(); let _caller = AuthenticatedCallerGuard::install(); - let missing = block_on_ready(purchase_database_credits(DatabaseCreditPurchaseRequest { + let missing = block_on_ready(purchase_database_cycles(DatabaseCyclesPurchaseRequest { database_id: "missing".to_string(), - credits: 500, - expected_payment_amount_e8s: 50_000_000, - expected_config_version: 1, + payment_amount_e8s: 50_000_000, + min_expected_cycles: 1, })) .expect_err("unknown database should reject"); assert!(missing.contains("database not found")); @@ -1143,19 +1158,18 @@ fn purchase_database_credits_rejects_unknown_and_deleted_database() { super::delete_database(delete_database_request(&database.database_id)) .expect("owner should delete"); - let deleted = block_on_ready(purchase_database_credits(DatabaseCreditPurchaseRequest { + let deleted = block_on_ready(purchase_database_cycles(DatabaseCyclesPurchaseRequest { database_id: database.database_id, - credits: 500, - expected_payment_amount_e8s: 50_000_000, - expected_config_version: 1, + payment_amount_e8s: 50_000_000, + min_expected_cycles: 1, })) .expect_err("deleted database should reject"); assert!(deleted.contains("database not found")); } fn database_charge_methods(database_id: &str) -> Vec { - list_database_credit_entries(database_id.to_string(), None, 20) - .expect("database credits ledger should load") + list_database_cycle_entries(database_id.to_string(), None, 20) + .expect("database cycles ledger should load") .entries .into_iter() .filter(|entry| entry.kind == "charge") @@ -1186,38 +1200,6 @@ fn install_suspended_default_service() { service .create_database("default", "2vxsx-fae", 1_700_000_000_000) .expect("default database should create"); - service - .begin_database_credit_purchase("default", "2vxsx-fae", 1, 1_700_000_000_001) - .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( - operation_id, - "default", - "2vxsx-fae", - 1, - )?; - service.credit_database_purchase( - operation_id, - "default", - "2vxsx-fae", - 1, - 1, - 1_700_000_000_001, - ) - }) - .expect("default database should become suspended"); - let config = service - .credits_config() - .expect("credits config should load"); - service - .charge_database_update( - &config, - "default", - "2vxsx-fae", - "test_suspend", - 1_000_000_000, - 1_700_000_000_002, - ) - .expect("default database should be charged to suspension"); SERVICE.with(|slot| *slot.borrow_mut() = Some(service)); } @@ -1232,19 +1214,21 @@ fn install_low_balance_default_service() { .create_database("default", "2vxsx-fae", 1_700_000_000_000) .expect("default database should create"); service - .begin_database_credit_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) + .begin_database_cycles_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( + let cycles = cycles_for_test_payment(&service, 1_000_000); + service.complete_database_cycles_purchase_ledger_transfer( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, + 1, )?; - service.credit_database_purchase( + service.apply_database_cycles_purchase( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, 1, 1_700_000_000_001, ) @@ -1254,18 +1238,18 @@ fn install_low_balance_default_service() { .grant_database_access( "default", "2vxsx-fae", - &test_governance_principal().to_text(), + &test_billing_authority_principal().to_text(), DatabaseRole::Writer, 1_700_000_000_002, ) .expect("writer should be granted before low-balance config"); service - .update_credits_config( - CreditsConfigUpdate { - credits_per_kinic: 1_000, - min_update_credits: 2_000_000, + .update_cycles_billing_config( + CyclesBillingConfigUpdate { + cycles_per_kinic: 1_000, + min_update_cycles: 2_000_000, }, - &test_governance_principal().to_text(), + &test_billing_authority_principal().to_text(), ) .expect("minimum balance should update"); SERVICE.with(|slot| *slot.borrow_mut() = Some(service)); @@ -1412,9 +1396,66 @@ fn write_node_and_write_nodes_skip_zero_charge_ledger() { } #[test] -fn write_nodes_rejects_low_database_credits_balance() { +fn write_node_overdrawn_charge_consumes_balance_and_suspends_database() { + install_test_service(); + let before_balance = list_databases() + .expect("database summaries should load") + .into_iter() + .find(|database| database.database_id == "default") + .and_then(|database| database.cycles_balance) + .expect("default database should have cycles balance"); + + set_cycle_balances_for_test(vec![1_000_000_000_000, 0]); + let written = write_node(WriteNodeRequest { + database_id: "default".to_string(), + path: "/Wiki/overdrawn.md".to_string(), + kind: NodeKind::File, + content: "overdrawn charge still writes".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .expect("overdrawn post-charge should not trap"); + + assert_eq!(written.node.path, "/Wiki/overdrawn.md"); + assert!( + read_node("default".to_string(), "/Wiki/overdrawn.md".to_string()) + .expect("written node should read") + .is_some() + ); + let summary = list_databases() + .expect("database summaries should load") + .into_iter() + .find(|database| database.database_id == "default") + .expect("default database summary should exist"); + assert_eq!(summary.cycles_balance, Some(0)); + assert_eq!(summary.cycles_suspended_at_ms, Some(1_700_000_000_000)); + + let entries = list_database_cycle_entries("default".to_string(), None, 20) + .expect("database cycles ledger should load") + .entries; + let charge = entries + .iter() + .find(|entry| entry.kind == "charge") + .expect("charge entry should exist"); + assert_eq!(charge.amount_cycles, -(before_balance as i64)); + assert_eq!(charge.balance_after_cycles, 0); + assert_eq!(charge.cycles_delta, Some(1_000_000_000_000)); + assert_eq!(charge.method.as_deref(), Some("write_node")); +} + +#[test] +fn write_nodes_rejects_low_database_cycles_balance() { install_unfunded_default_service(); + let single = write_node(WriteNodeRequest { + database_id: "default".to_string(), + path: "/Wiki/no-balance-single.md".to_string(), + kind: NodeKind::File, + content: "no balance single".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .expect_err("low balance database should reject single write"); let error = write_nodes(WriteNodesRequest { database_id: "default".to_string(), nodes: vec![WriteNodeItem { @@ -1427,7 +1468,21 @@ fn write_nodes_rejects_low_database_credits_balance() { }) .expect_err("low balance database should reject batch write"); - assert!(error.contains("database credits are suspended")); + assert!(single.contains("database cycles are suspended")); + assert!(error.contains("database cycles are suspended")); + assert!( + read_node( + "default".to_string(), + "/Wiki/no-balance-single.md".to_string() + ) + .expect("single path read should succeed") + .is_none() + ); + assert!( + read_node("default".to_string(), "/Wiki/no-balance.md".to_string()) + .expect("batch path read should succeed") + .is_none() + ); } #[test] @@ -1453,9 +1508,9 @@ fn suspended_database_rejects_metered_mutations() { let cancel = super::cancel_database_restore("default".to_string()) .expect_err("suspended database should reject restore cancel before runtime mutation"); - assert!(batch.contains("database credits are suspended")); - assert!(mkdir.contains("database credits are suspended")); - assert!(cancel.contains("database credits are suspended")); + assert!(batch.contains("database cycles are suspended")); + assert!(mkdir.contains("database cycles are suspended")); + assert!(cancel.contains("database cycles are suspended")); } #[test] @@ -1475,14 +1530,17 @@ fn suspended_database_allows_owner_management_operations() { fn low_balance_database_allows_owner_revoke_and_delete() { install_low_balance_default_service(); - revoke_database_access("default".to_string(), test_governance_principal().to_text()) - .expect("low-balance database owner should revoke"); + revoke_database_access( + "default".to_string(), + test_billing_authority_principal().to_text(), + ) + .expect("low-balance database owner should revoke"); super::delete_database(delete_database_request("default")) .expect("low-balance database owner should delete"); } #[test] -fn metered_update_checks_access_before_credits_state() { +fn metered_update_checks_access_before_cycles_state() { install_suspended_default_service(); let _caller = AuthenticatedCallerGuard::install(); @@ -1496,14 +1554,14 @@ fn metered_update_checks_access_before_credits_state() { expected_etag: None, }], }) - .expect_err("non-member should fail before credits state"); + .expect_err("non-member should fail before cycles state"); assert!(error.contains("principal has no access")); - assert!(!error.contains("database credits are suspended")); + assert!(!error.contains("database cycles are suspended")); } #[test] -fn check_database_write_credits_requires_authenticated_writer() { +fn check_database_write_cycles_requires_authenticated_writer() { install_empty_test_service(); let owner = Principal::management_canister(); let reader = @@ -1511,22 +1569,22 @@ fn check_database_write_credits_requires_authenticated_writer() { let database_id = { let _caller = AuthenticatedCallerGuard::install_principal(owner); create_database(CreateDatabaseRequest { - name: "Write credits check".to_string(), + name: "Write cycles check".to_string(), }) .expect("database should create") .database_id }; - let anonymous = check_database_write_credits(database_id.clone()) - .expect_err("anonymous caller should fail"); + let anonymous = + check_database_write_cycles(database_id.clone()).expect_err("anonymous caller should fail"); assert!(anonymous.contains("anonymous caller not allowed")); let suspended = { let _caller = AuthenticatedCallerGuard::install_principal(owner); - check_database_write_credits(database_id.clone()) + check_database_write_cycles(database_id.clone()) .expect_err("suspended database should fail") }; - assert!(suspended.contains("database credits are suspended")); + assert!(suspended.contains("database cycles are suspended")); fund_database(&database_id, 1_000_000, 91); SERVICE.with(|slot| { @@ -1545,12 +1603,12 @@ fn check_database_write_credits_requires_authenticated_writer() { let reader_error = { let _caller = AuthenticatedCallerGuard::install_principal(reader); - check_database_write_credits(database_id.clone()).expect_err("reader should fail") + check_database_write_cycles(database_id.clone()).expect_err("reader should fail") }; assert!(reader_error.contains("principal lacks required database role")); let _caller = AuthenticatedCallerGuard::install_principal(owner); - check_database_write_credits(database_id).expect("owner should pass write credits check"); + check_database_write_cycles(database_id).expect("owner should pass write cycles check"); } #[test] @@ -1565,17 +1623,19 @@ fn write_nodes_rejects_reader_role() { .create_database("public", "owner", 1) .expect("database should create"); service - .begin_database_credit_purchase("public", "owner", 1_000_000, 2) + .begin_database_cycles_purchase("public", "owner", 1_000_000, 2) .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( + let cycles = cycles_for_test_payment(&service, 1_000_000); + service.complete_database_cycles_purchase_ledger_transfer( operation_id, "public", "owner", - 1_000_000, + cycles, + 1, )?; - service.credit_database_purchase(operation_id, "public", "owner", 1_000_000, 1, 2) + service.apply_database_cycles_purchase(operation_id, "public", "owner", cycles, 1, 2) }) - .expect("database should have write credits available"); + .expect("database should have write cycles available"); service .grant_database_access("public", "owner", "2vxsx-fae", DatabaseRole::Reader, 3) .expect("anonymous reader should grant"); @@ -1640,32 +1700,13 @@ fn create_database_returns_result() { }) .expect_err("pending DB should reject reads"); assert!(pending_read.contains("database is pending")); - let pending_members = list_database_members(result.database_id.clone()) - .expect_err("pending DB should reject member listing"); - assert!(pending_members.contains("database is pending")); - let pending_rename = rename_database(RenameDatabaseRequest { - database_id: result.database_id.clone(), - name: "Renamed".to_string(), - }) - .expect_err("pending DB should reject rename"); - assert!(pending_rename.contains("database is pending")); - let pending_grant = grant_database_access( - result.database_id.clone(), - "aaaaa-aa".to_string(), - DatabaseRole::Reader, - ) - .expect_err("pending DB should reject grants"); - assert!(pending_grant.contains("database is pending")); - let pending_revoke = revoke_database_access(result.database_id.clone(), "aaaaa-aa".to_string()) - .expect_err("pending DB should reject revokes"); - assert!(pending_revoke.contains("database is pending")); set_next_ledger_transfer_from_outcome_for_test(LedgerTransferFromOutcome::Completed(42)); - block_on_ready(purchase_database_credits(credit_purchase_request( + block_on_ready(purchase_database_cycles(cycles_purchase_request( &result.database_id, 1_000_000, ))) - .expect("credit purchase should activate database"); + .expect("cycle purchase should activate database"); let status = status(result.database_id.clone()); assert_eq!(status.file_count, 0); assert_eq!(status.source_count, 0); @@ -1677,6 +1718,33 @@ fn create_database_returns_result() { assert!(children.is_empty()); } +#[test] +fn create_database_rejects_pending_database_limit() { + install_empty_test_service(); + let _caller = AuthenticatedCallerGuard::install(); + + for offset in 0..3 { + create_database(CreateDatabaseRequest { + name: format!("Pending {offset}"), + }) + .expect("pending database should create within limit"); + } + + let error = create_database(CreateDatabaseRequest { + name: "Pending 3".to_string(), + }) + .expect_err("fourth pending database should reject"); + assert!(error.contains("too many pending databases for caller")); + + let summaries = list_databases().expect("database summaries should load"); + assert_eq!(summaries.len(), 3); + assert!( + summaries + .iter() + .all(|summary| summary.status == DatabaseStatus::Pending) + ); +} + #[test] fn canister_rename_database_requires_owner() { install_test_service(); @@ -1705,6 +1773,35 @@ fn grant_database_access_rejects_invalid_principal() { assert!(error.contains("invalid principal")); } +#[test] +fn grant_database_access_rejects_member_limit() { + install_test_service(); + + for index in 0..30 { + grant_database_access( + "default".to_string(), + Principal::self_authenticating([index as u8]).to_text(), + DatabaseRole::Reader, + ) + .expect("member grant should fit limit"); + } + + let error = grant_database_access( + "default".to_string(), + Principal::self_authenticating([30]).to_text(), + DatabaseRole::Reader, + ) + .expect_err("member cap should reject new member"); + assert!(error.contains("too many database members")); + + grant_database_access( + "default".to_string(), + Principal::self_authenticating([0]).to_text(), + DatabaseRole::Writer, + ) + .expect("existing member role update should remain allowed"); +} + #[test] fn revoke_database_access_validates_and_canonicalizes_principal() { install_test_service(); @@ -2621,24 +2718,26 @@ fn cancel_database_archive_entrypoint_rejects_non_owner() { .create_database("default", "owner", 1_700_000_000_000) .expect("default database should create"); service - .begin_database_credit_purchase("default", "owner", 1_000_000, 1_700_000_000_001) + .begin_database_cycles_purchase("default", "owner", 1_000_000, 1_700_000_000_001) .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( + let cycles = cycles_for_test_payment(&service, 1_000_000); + service.complete_database_cycles_purchase_ledger_transfer( operation_id, "default", "owner", - 1_000_000, + cycles, + 1, )?; - service.credit_database_purchase( + service.apply_database_cycles_purchase( operation_id, "default", "owner", - 1_000_000, + cycles, 1, 1_700_000_000_001, ) }) - .expect("database should have write credits available"); + .expect("database should have write cycles available"); service .begin_database_archive("default", "owner", 1_700_000_000_002) .expect("archive should begin"); diff --git a/crates/vfs_canister/src/tests_sync_contract.rs b/crates/vfs_canister/src/tests_sync_contract.rs index 02b8a7db..5d423ac9 100644 --- a/crates/vfs_canister/src/tests_sync_contract.rs +++ b/crates/vfs_canister/src/tests_sync_contract.rs @@ -26,24 +26,25 @@ fn install_test_service() { .create_database("default", "2vxsx-fae", 1_700_000_000_000) .expect("default database should create"); service - .begin_database_credit_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) + .begin_database_cycles_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) .and_then(|operation_id| { - service.mark_database_credit_purchase_completed( + service.complete_database_cycles_purchase_ledger_transfer( operation_id, "default", "2vxsx-fae", - 1_000_000, + 2_345_000_000, + 1, )?; - service.credit_database_purchase( + service.apply_database_cycles_purchase( operation_id, "default", "2vxsx-fae", - 1_000_000, + 2_345_000_000, 1, 1_700_000_000_001, ) }) - .expect("default database should have write credits available"); + .expect("default database should have write cycles available"); SERVICE.with(|slot| *slot.borrow_mut() = Some(service)); } diff --git a/crates/vfs_canister/vfs.did b/crates/vfs_canister/vfs.did index 145c97c8..6fa24ffa 100644 --- a/crates/vfs_canister/vfs.did +++ b/crates/vfs_canister/vfs.did @@ -25,68 +25,56 @@ type ChildNode = record { }; type CreateDatabaseRequest = record { name : text }; type CreateDatabaseResult = record { name : text; database_id : text }; -type CreditsConfig = record { - credits_per_kinic : nat64; - min_update_credits : nat64; +type CyclesBillingConfig = record { + billing_authority_id : text; kinic_ledger_canister_id : text; - sns_governance_id : text; + cycles_per_kinic : nat64; + min_update_cycles : nat64; }; -type CreditsPurchaseResult = record { +type CyclesBillingConfigUpdate = record { + cycles_per_kinic : nat64; + min_update_cycles : nat64; +}; +type CyclesPurchaseResult = record { block_index : nat64; - balance_credits : nat64; + balance_cycles : nat64; + amount_cycles : nat64; }; type RenameDatabaseRequest = record { name : text; database_id : text }; type DatabaseArchiveChunk = record { bytes : blob }; type DatabaseArchiveInfo = record { size_bytes : nat64; database_id : text }; -type DatabaseCreditEntry = record { +type DatabaseCycleEntry = record { method : opt text; - credits_per_kinic : opt nat64; payment_amount_e8s : opt nat64; kind : text; - balance_after_credits : nat64; - amount_credits : int64; + balance_after_cycles : nat64; created_at_ms : int64; + cycles_per_kinic : opt nat64; ledger_block_index : opt nat64; database_id : text; + amount_cycles : int64; caller : text; cycles_delta : opt nat64; entry_id : nat64; }; -type DatabaseCreditEntryPage = record { - entries : vec DatabaseCreditEntry; +type DatabaseCycleEntryPage = record { + entries : vec DatabaseCycleEntry; next_cursor : opt nat64; }; -type DatabaseCreditPendingOperation = record { - credits : nat64; - operation_status : text; +type DatabaseCyclesPendingPurchase = record { + status : text; payment_amount_e8s : nat64; - to_owner : opt text; - to_subaccount : opt blob; - from_owner : opt text; - kind : text; operation_id : nat64; - from_subaccount : opt blob; created_at_ms : int64; - ledger_fee_e8s : opt nat64; - ledger_created_at_time_ns : opt nat64; + required_action : text; + ledger_block_index : opt nat64; database_id : text; - caller : text; -}; -type DatabaseCreditPendingOperationPage = record { - entries : vec DatabaseCreditPendingOperation; - next_cursor : opt nat64; + amount_cycles : nat64; }; -type DatabaseCreditPurchasePreview = record { - credits_per_kinic : nat64; +type DatabaseCyclesPurchaseRequest = record { payment_amount_e8s : nat64; - ledger_fee_e8s : nat64; - config_version : nat64; -}; -type DatabaseCreditPurchaseRequest = record { - credits : nat64; - expected_config_version : nat64; database_id : text; - expected_payment_amount_e8s : nat64; + min_expected_cycles : nat64; }; type DatabaseMember = record { "principal" : text; @@ -109,11 +97,11 @@ type DatabaseStatus = variant { }; type DatabaseSummary = record { status : DatabaseStatus; + cycles_balance : opt nat64; name : text; role : DatabaseRole; logical_size_bytes : nat64; - credits_suspended_at_ms : opt int64; - credits_balance : opt nat64; + cycles_suspended_at_ms : opt int64; database_id : text; archived_at_ms : opt int64; }; @@ -355,35 +343,31 @@ type Result_1 = variant { Ok; Err : text }; type Result_10 = variant { Ok : vec GlobNodeHit; Err : text }; type Result_11 = variant { Ok : vec LinkEdge; Err : text }; type Result_12 = variant { Ok : vec ChildNode; Err : text }; -type Result_13 = variant { Ok : DatabaseCreditEntryPage; Err : text }; -type Result_14 = variant { - Ok : DatabaseCreditPendingOperationPage; - Err : text; -}; +type Result_13 = variant { Ok : DatabaseCycleEntryPage; Err : text }; +type Result_14 = variant { Ok : vec DatabaseCyclesPendingPurchase; Err : text }; type Result_15 = variant { Ok : vec DatabaseMember; Err : text }; type Result_16 = variant { Ok : vec DatabaseSummary; Err : text }; type Result_17 = variant { Ok : vec NodeEntry; Err : text }; type Result_18 = variant { Ok : MkdirNodeResult; Err : text }; type Result_19 = variant { Ok : MoveNodeResult; Err : text }; type Result_2 = variant { Ok : DatabaseArchiveInfo; Err : text }; -type Result_20 = variant { Ok : DatabaseCreditPurchasePreview; Err : text }; -type Result_21 = variant { Ok : CreditsPurchaseResult; Err : text }; -type Result_22 = variant { Ok : QueryContext; Err : text }; -type Result_23 = variant { Ok : IndexSqlJsonQueryResult; Err : text }; -type Result_24 = variant { Ok : DatabaseArchiveChunk; Err : text }; -type Result_25 = variant { Ok : opt Node; Err : text }; -type Result_26 = variant { Ok : opt NodeContext; Err : text }; -type Result_27 = variant { Ok : vec SearchNodeHit; Err : text }; -type Result_28 = variant { Ok : SourceEvidence; Err : text }; -type Result_29 = variant { Ok : vec WriteNodeResult; Err : text }; +type Result_20 = variant { Ok : CyclesPurchaseResult; Err : text }; +type Result_21 = variant { Ok : QueryContext; Err : text }; +type Result_22 = variant { Ok : IndexSqlJsonQueryResult; Err : text }; +type Result_23 = variant { Ok : DatabaseArchiveChunk; Err : text }; +type Result_24 = variant { Ok : opt Node; Err : text }; +type Result_25 = variant { Ok : opt NodeContext; Err : text }; +type Result_26 = variant { Ok : vec SearchNodeHit; Err : text }; +type Result_27 = variant { Ok : SourceEvidence; Err : text }; +type Result_28 = variant { Ok : vec WriteNodeResult; Err : text }; +type Result_29 = variant { Ok : WriteSourceForGenerationResult; Err : text }; type Result_3 = variant { Ok : OpsAnswerSessionCheckResult; Err : text }; -type Result_30 = variant { Ok : WriteSourceForGenerationResult; Err : text }; type Result_4 = variant { Ok : CreateDatabaseResult; Err : text }; type Result_5 = variant { Ok : DeleteNodeResult; Err : text }; type Result_6 = variant { Ok : EditNodeResult; Err : text }; type Result_7 = variant { Ok : ExportSnapshotResponse; Err : text }; type Result_8 = variant { Ok : FetchUpdatesResponse; Err : text }; -type Result_9 = variant { Ok : CreditsConfig; Err : text }; +type Result_9 = variant { Ok : CyclesBillingConfig; Err : text }; type SearchNodeHit = record { preview : opt SearchPreview; kind : NodeKind; @@ -470,7 +454,7 @@ type WriteSourceForGenerationResult = record { session_nonce : text; write : WriteNodeResult; }; -service : (CreditsConfig) -> { +service : (CyclesBillingConfig) -> { append_node : (AppendNodeRequest) -> (Result); authorize_ops_answer_session : (OpsAnswerSessionRequest) -> (Result_1); authorize_url_ingest_trigger_session : (UrlIngestTriggerSessionRequest) -> ( @@ -481,7 +465,7 @@ service : (CreditsConfig) -> { cancel_database_archive : (text) -> (Result_1); cancel_database_restore : (text) -> (Result_1); canister_health : () -> (CanisterHealth) query; - check_database_write_credits : (text) -> (Result_1) query; + check_database_write_cycles : (text) -> (Result_1) query; check_ops_answer_session : (OpsAnswerSessionRequest) -> (Result_3) query; check_source_run_session : (SourceRunSessionCheckRequest) -> (Result_1) query; check_url_ingest_trigger_session : (UrlIngestTriggerSessionCheckRequest) -> ( @@ -495,7 +479,7 @@ service : (CreditsConfig) -> { fetch_updates : (FetchUpdatesRequest) -> (Result_8) query; finalize_database_archive : (text, blob) -> (Result_1); finalize_database_restore : (text) -> (Result_1); - get_credits_config : () -> (Result_9) query; + get_cycles_billing_config : () -> (Result_9) query; glob_nodes : (GlobNodesRequest) -> (Result_10) query; grant_database_access : (text, text, DatabaseRole) -> (Result_1); graph_links : (GraphLinksRequest) -> (Result_11) query; @@ -507,10 +491,8 @@ service : (CreditsConfig) -> { ); incoming_links : (IncomingLinksRequest) -> (Result_11) query; list_children : (ListChildrenRequest) -> (Result_12) query; - list_database_credit_entries : (text, opt nat64, nat32) -> (Result_13) query; - list_database_credit_pending_operations : (text, opt nat64, nat32) -> ( - Result_14, - ) query; + list_database_cycle_entries : (text, opt nat64, nat32) -> (Result_13) query; + list_database_cycles_pending_purchases : (text) -> (Result_14) query; list_database_members : (text) -> (Result_15) query; list_databases : () -> (Result_16) query; list_nodes : (ListNodesRequest) -> (Result_17) query; @@ -519,28 +501,24 @@ service : (CreditsConfig) -> { move_node : (MoveNodeRequest) -> (Result_19); multi_edit_node : (MultiEditNodeRequest) -> (Result_6); outgoing_links : (OutgoingLinksRequest) -> (Result_11) query; - preview_database_credit_purchase : (text, nat64) -> (Result_20) query; - purchase_database_credits : (DatabaseCreditPurchaseRequest) -> (Result_21); - query_context : (QueryContextRequest) -> (Result_22) query; - query_index_sql_json : (text, nat32) -> (Result_23) query; - read_database_archive_chunk : (text, nat64, nat32) -> (Result_24) query; - read_node : (text, text) -> (Result_25) query; - read_node_context : (NodeContextRequest) -> (Result_26) query; + purchase_database_cycles : (DatabaseCyclesPurchaseRequest) -> (Result_20); + query_context : (QueryContextRequest) -> (Result_21) query; + query_index_sql_json : (text, nat32) -> (Result_22) query; + read_database_archive_chunk : (text, nat64, nat32) -> (Result_23) query; + read_node : (text, text) -> (Result_24) query; + read_node_context : (NodeContextRequest) -> (Result_25) query; rename_database : (RenameDatabaseRequest) -> (Result_1); - repair_database_credit_purchase_cancel : (text, nat64) -> (Result_1); - repair_database_credit_purchase_complete : (text, nat64, nat64) -> ( - Result_21, - ); revoke_database_access : (text, text) -> (Result_1); - search_node_paths : (SearchNodePathsRequest) -> (Result_27) query; - search_nodes : (SearchNodesRequest) -> (Result_27) query; - source_evidence : (SourceEvidenceRequest) -> (Result_28) query; + search_node_paths : (SearchNodePathsRequest) -> (Result_26) query; + search_nodes : (SearchNodesRequest) -> (Result_26) query; + settle_database_storage_charges : () -> (Result_1); + source_evidence : (SourceEvidenceRequest) -> (Result_27) query; status : (text) -> (Status) query; - update_credits_config : (blob) -> (Result_1); + update_cycles_billing_config : (CyclesBillingConfigUpdate) -> (Result_1); write_database_restore_chunk : (DatabaseRestoreChunkRequest) -> (Result_1); write_node : (WriteNodeRequest) -> (Result); - write_nodes : (WriteNodesRequest) -> (Result_29); + write_nodes : (WriteNodesRequest) -> (Result_28); write_source_for_generation : (WriteSourceForGenerationRequest) -> ( - Result_30, + Result_29, ); } diff --git a/crates/vfs_cli_app/src/bin/local_canister_archive_restore_smoke.rs b/crates/vfs_cli_app/src/bin/local_canister_archive_restore_smoke.rs index 68b53af9..be79cf31 100644 --- a/crates/vfs_cli_app/src/bin/local_canister_archive_restore_smoke.rs +++ b/crates/vfs_cli_app/src/bin/local_canister_archive_restore_smoke.rs @@ -8,8 +8,9 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use vfs_client::{CanisterVfsClient, VfsApi}; use vfs_types::{ - DatabaseCreditPurchaseRequest, DatabaseRestoreChunkRequest, DatabaseStatus, MkdirNodeRequest, + DatabaseCyclesPurchaseRequest, DatabaseRestoreChunkRequest, DatabaseStatus, MkdirNodeRequest, NodeKind, OutgoingLinksRequest, SearchNodesRequest, SearchPreviewMode, WriteNodeRequest, + kinic_base_units_per_token, }; const PRIMARY_SOURCE_PATH: &str = "/Sources/raw/smoke/smoke.md"; @@ -19,7 +20,7 @@ const PRIMARY_QUERY: &str = "alpha canister"; const CJK_CONTENT_MARKER: &str = "検索精度改善"; const CJK_QUERY: &str = "検索精度改善"; const ISOLATION_CONTENT_MARKER: &str = "beta isolated db"; -const DEFAULT_SMOKE_CREDIT_PURCHASE_CREDITS: u64 = 1_000; +const DEFAULT_SMOKE_CYCLE_PURCHASE_E8S: u64 = 100_000_000; #[derive(Debug)] struct SmokeArgs { @@ -56,12 +57,12 @@ async fn main() -> Result<()> { .transpose() .context("ARCHIVE_CHUNK_SIZE must be a u32")? .unwrap_or(64 * 1024); - let credit_purchase_credits = env::var("SMOKE_CREDIT_PURCHASE_CREDITS") + let cycle_purchase_e8s = env::var("SMOKE_CYCLE_PURCHASE_E8S") .ok() .map(|value| value.parse::()) .transpose() - .context("SMOKE_CREDIT_PURCHASE_CREDITS must be a u64")? - .unwrap_or(DEFAULT_SMOKE_CREDIT_PURCHASE_CREDITS); + .context("SMOKE_CYCLE_PURCHASE_E8S must be a u64")? + .unwrap_or(DEFAULT_SMOKE_CYCLE_PURCHASE_E8S); let identity = vfs_cli_app::identity::load_default_identity(&canister_id, true).await?; let client = @@ -77,8 +78,7 @@ async fn main() -> Result<()> { return Ok(()); } let state = - run_create_restore_smoke(&client, &canister_id, chunk_size, credit_purchase_credits) - .await?; + run_create_restore_smoke(&client, &canister_id, chunk_size, cycle_purchase_e8s).await?; if let Some(path) = args.state_output { write_state(&path, &state)?; } @@ -129,15 +129,15 @@ async fn run_create_restore_smoke( client: &CanisterVfsClient, canister_id: &str, chunk_size: u32, - credit_purchase_credits: u64, + cycle_purchase_e8s: u64, ) -> Result { let database_id = client.create_database("Archive smoke").await?.database_id; let isolation_database_id = client .create_database("Archive smoke isolation") .await? .database_id; - activate_smoke_database(client, &database_id, credit_purchase_credits).await?; - activate_smoke_database(client, &isolation_database_id, credit_purchase_credits).await?; + activate_smoke_database(client, &database_id, cycle_purchase_e8s).await?; + activate_smoke_database(client, &isolation_database_id, cycle_purchase_e8s).await?; ensure_parent_folders(client, &database_id, PRIMARY_SOURCE_PATH).await?; ensure_parent_folders(client, &isolation_database_id, PRIMARY_SOURCE_PATH).await?; client @@ -282,24 +282,35 @@ async fn run_create_restore_smoke( async fn activate_smoke_database( client: &CanisterVfsClient, database_id: &str, - amount_credits: u64, + payment_amount_e8s: u64, ) -> Result<()> { - let preview = client - .preview_database_credit_purchase(database_id, amount_credits) - .await - .with_context(|| format!("failed to preview credits for smoke database {database_id}"))?; + let config = client.get_cycles_billing_config().await?; + let min_expected_cycles = + cycles_for_payment_amount_e8s(payment_amount_e8s, config.cycles_per_kinic)?; client - .purchase_database_credits(DatabaseCreditPurchaseRequest { + .purchase_database_cycles(DatabaseCyclesPurchaseRequest { database_id: database_id.to_string(), - credits: amount_credits, - expected_payment_amount_e8s: preview.payment_amount_e8s, - expected_config_version: preview.config_version, + payment_amount_e8s, + min_expected_cycles, }) .await - .with_context(|| format!("failed to purchase credits for smoke database {database_id}"))?; + .with_context(|| format!("failed to purchase cycles for smoke database {database_id}"))?; Ok(()) } +fn cycles_for_payment_amount_e8s(payment_amount_e8s: u64, cycles_per_kinic: u64) -> Result { + let cycles = u128::from(payment_amount_e8s) + .checked_mul(u128::from(cycles_per_kinic)) + .ok_or_else(|| anyhow!("cycles purchase amount overflow"))? + / u128::from(kinic_base_units_per_token()); + let cycles = + u64::try_from(cycles).map_err(|_| anyhow!("cycles purchase amount exceeds u64"))?; + if cycles == 0 { + return Err(anyhow!("cycles purchase amount is too small")); + } + Ok(cycles) +} + async fn ensure_parent_folders( client: &CanisterVfsClient, database_id: &str, diff --git a/crates/vfs_cli_app/src/bin/local_canister_post_upgrade_smoke.rs b/crates/vfs_cli_app/src/bin/local_canister_post_upgrade_smoke.rs index 52c4ce84..3d4c14d7 100644 --- a/crates/vfs_cli_app/src/bin/local_canister_post_upgrade_smoke.rs +++ b/crates/vfs_cli_app/src/bin/local_canister_post_upgrade_smoke.rs @@ -1,12 +1,14 @@ // Where: crates/vfs_cli_app/src/bin/local_canister_post_upgrade_smoke.rs -// What: Verify local wiki canister CreditsConfig and pending DB persistence across upgrade. -// Why: Fresh install requires CreditsConfig, and upgrade operators need a small state smoke. +// What: Verify local wiki canister cycles config and pending DB persistence across upgrade. +// Why: Fresh install requires cycles billing config, and upgrade operators need a small state smoke. use std::{env, fs}; use anyhow::{Context, Result, anyhow}; use serde::{Deserialize, Serialize}; use vfs_client::{CanisterVfsClient, VfsApi}; -use vfs_types::{CreditsConfig, DatabaseStatus}; +use vfs_types::{ + CyclesBillingConfig, DatabaseCyclesPurchaseRequest, DatabaseStatus, kinic_base_units_per_token, +}; #[derive(Debug)] struct SmokeArgs { @@ -18,21 +20,24 @@ struct SmokeArgs { struct SmokeState { canister_id: String, database_id: String, - expected_config: CreditsConfig, + active_database_id: String, + active_balance_cycles: u64, + active_ledger_entry_count: usize, + expected_config: CyclesBillingConfig, } #[tokio::main] async fn main() -> Result<()> { let args = parse_args()?; let replica_host = - env::var("REPLICA_HOST").unwrap_or_else(|_| "http://127.0.0.1:8001".to_string()); + env::var("REPLICA_HOST").unwrap_or_else(|_| "http://127.0.0.1:8011".to_string()); let canister_id = env::var("CANISTER_ID") .or_else(|_| env::var("VFS_CANISTER_ID")) .context("CANISTER_ID or VFS_CANISTER_ID is required")?; - let expected_config = expected_credits_config()?; + let expected_config = expected_cycles_config()?; let client = authenticated_client(&replica_host, &canister_id).await?; - assert_credits_config(&client, &expected_config).await?; + assert_cycles_config(&client, &expected_config).await?; if let Some(path) = args.verify_state { let state = read_state(&path)?; if state.canister_id != canister_id { @@ -44,13 +49,15 @@ async fn main() -> Result<()> { } if state.expected_config != expected_config { return Err(anyhow!( - "current CreditsConfig env differs from smoke state" + "current cycles config env differs from smoke state" )); } assert_pending_database(&client, &state.database_id).await?; + assert_active_database(&client, &state).await?; println!("local_canister_post_upgrade_smoke verify ok"); println!("canister_id={}", state.canister_id); println!("database_id={}", state.database_id); + println!("active_database_id={}", state.active_database_id); return Ok(()); } @@ -59,18 +66,32 @@ async fn main() -> Result<()> { .await? .database_id; assert_pending_database(&client, &database_id).await?; + let active_database_id = client + .create_database("Post-upgrade active smoke") + .await? + .database_id; + let active_balance_cycles = + activate_smoke_database(&client, &active_database_id, smoke_cycle_purchase_e8s()?).await?; + let active_ledger_entry_count = client + .list_database_cycle_entries(&active_database_id, None, 10) + .await? + .entries + .len(); + let state = SmokeState { + canister_id, + database_id: database_id.clone(), + active_database_id: active_database_id.clone(), + active_balance_cycles, + active_ledger_entry_count, + expected_config, + }; + assert_active_database(&client, &state).await?; if let Some(path) = args.state_output { - write_state( - &path, - &SmokeState { - canister_id, - database_id: database_id.clone(), - expected_config, - }, - )?; + write_state(&path, &state)?; } println!("local_canister_post_upgrade_smoke ok"); println!("database_id={database_id}"); + println!("active_database_id={active_database_id}"); Ok(()) } @@ -113,12 +134,12 @@ async fn authenticated_client(replica_host: &str, canister_id: &str) -> Result Result { - Ok(CreditsConfig { +fn expected_cycles_config() -> Result { + Ok(CyclesBillingConfig { kinic_ledger_canister_id: required_env("KINIC_LEDGER_CANISTER_ID")?, - sns_governance_id: required_env("SNS_GOVERNANCE_ID")?, - credits_per_kinic: env_u64("CREDITS_PER_KINIC", 1_000)?, - min_update_credits: env_u64("MIN_UPDATE_CREDITS", 1)?, + billing_authority_id: required_env("BILLING_AUTHORITY_ID")?, + cycles_per_kinic: env_u64("CYCLES_PER_KINIC", 234_500_000_000)?, + min_update_cycles: env_u64("MIN_UPDATE_CYCLES", 1_000_000)?, }) } @@ -146,11 +167,14 @@ fn env_u64(name: &str, default: u64) -> Result { }) } -async fn assert_credits_config(client: &CanisterVfsClient, expected: &CreditsConfig) -> Result<()> { - let actual = client.get_credits_config().await?; +async fn assert_cycles_config( + client: &CanisterVfsClient, + expected: &CyclesBillingConfig, +) -> Result<()> { + let actual = client.get_cycles_billing_config().await?; if &actual != expected { return Err(anyhow!( - "unexpected CreditsConfig: expected {:?}, got {:?}", + "unexpected cycles config: expected {:?}, got {:?}", expected, actual )); @@ -174,6 +198,86 @@ async fn assert_pending_database(client: &CanisterVfsClient, database_id: &str) Ok(()) } +async fn activate_smoke_database( + client: &CanisterVfsClient, + database_id: &str, + payment_amount_e8s: u64, +) -> Result { + let config = client.get_cycles_billing_config().await?; + let min_expected_cycles = cycles_for_payment_amount_e8s(payment_amount_e8s, &config)?; + let result = client + .purchase_database_cycles(DatabaseCyclesPurchaseRequest { + database_id: database_id.to_string(), + payment_amount_e8s, + min_expected_cycles, + }) + .await + .with_context(|| format!("failed to purchase cycles for smoke database {database_id}"))?; + Ok(result.balance_cycles) +} + +fn cycles_for_payment_amount_e8s( + payment_amount_e8s: u64, + config: &CyclesBillingConfig, +) -> Result { + let cycles = u128::from(payment_amount_e8s) + .checked_mul(u128::from(config.cycles_per_kinic)) + .ok_or_else(|| anyhow!("cycles purchase amount overflow"))? + / u128::from(kinic_base_units_per_token()); + let cycles = + u64::try_from(cycles).map_err(|_| anyhow!("cycles purchase amount exceeds u64"))?; + if cycles == 0 { + return Err(anyhow!("cycles purchase amount is too small")); + } + Ok(cycles) +} + +async fn assert_active_database(client: &CanisterVfsClient, state: &SmokeState) -> Result<()> { + let summary = client + .list_databases() + .await? + .into_iter() + .find(|database| database.database_id == state.active_database_id) + .ok_or_else(|| { + anyhow!( + "active smoke database missing: {}", + state.active_database_id + ) + })?; + if summary.status != DatabaseStatus::Active { + return Err(anyhow!( + "active smoke database should remain active, got {:?}", + summary.status + )); + } + if summary.cycles_balance != Some(state.active_balance_cycles) { + return Err(anyhow!( + "active smoke database balance changed: expected {:?}, got {:?}", + Some(state.active_balance_cycles), + summary.cycles_balance + )); + } + let entries = client + .list_database_cycle_entries(&state.active_database_id, None, 10) + .await? + .entries; + if entries.len() != state.active_ledger_entry_count { + return Err(anyhow!( + "active smoke ledger entry count changed: expected {}, got {}", + state.active_ledger_entry_count, + entries.len() + )); + } + if !entries.iter().any(|entry| entry.kind == "cycles_purchase") { + return Err(anyhow!("active smoke cycles purchase ledger entry missing")); + } + Ok(()) +} + +fn smoke_cycle_purchase_e8s() -> Result { + env_u64("SMOKE_CYCLE_PURCHASE_E8S", 100_000_000) +} + fn read_state(path: &str) -> Result { let text = fs::read_to_string(path).with_context(|| format!("failed to read {path}"))?; serde_json::from_str(&text).with_context(|| format!("failed to parse {path}")) diff --git a/crates/vfs_cli_app/src/cli.rs b/crates/vfs_cli_app/src/cli.rs index 1a56edf8..ed2589b0 100644 --- a/crates/vfs_cli_app/src/cli.rs +++ b/crates/vfs_cli_app/src/cli.rs @@ -5,7 +5,7 @@ use clap::{Parser, Subcommand}; use std::path::PathBuf; use vfs_cli::cli::VfsCommand; pub use vfs_cli::cli::{ - ConnectionArgs, DatabaseCommand, GlobNodeTypeArg, IdentityModeArg, NodeKindArg, + ConnectionArgs, CyclesCommand, DatabaseCommand, GlobNodeTypeArg, IdentityModeArg, NodeKindArg, SearchPreviewModeArg, }; use wiki_domain::WIKI_ROOT_PATH; @@ -24,6 +24,11 @@ pub struct Cli { #[derive(Subcommand, Debug, Clone)] pub enum Command { + #[command(about = "Show KINIC cycles configuration")] + Cycles { + #[command(subcommand)] + command: CyclesCommand, + }, #[command(about = "Manage database creation, workspace links, grants, archive, and restore")] Database { #[command(subcommand)] @@ -402,7 +407,7 @@ pub enum SkillCommand { #[arg(long)] json: bool, }, - #[command(about = "Mark a skill improvement proposal as approved")] + #[command(about = "Mark a skill improvement proposal as reviewed")] ApproveProposal { id: String, proposal_path: String, @@ -418,7 +423,7 @@ pub enum SkillCommand { #[arg(long)] json: bool, }, - #[command(about = "Apply an approved skill proposal when the base etag still matches")] + #[command(about = "Apply a reviewed skill proposal when the base etag still matches")] ApplyProposal { id: String, proposal_id: String, @@ -640,9 +645,12 @@ pub enum GitHubIngestCommand { impl Command { pub fn requires_identity(&self) -> bool { match self { + Self::Cycles { command: _ } => false, Self::Database { command } => matches!( command, DatabaseCommand::Create { .. } + | DatabaseCommand::PurchaseCycles { .. } + | DatabaseCommand::CyclesHistory { .. } | DatabaseCommand::Rename { .. } | DatabaseCommand::Grant { .. } | DatabaseCommand::GrantCurrentIdentity { .. } @@ -713,6 +721,7 @@ impl Command { | Self::SearchPathRemote { .. } | Self::Status { .. } => true, Self::Database { .. } + | Self::Cycles { .. } | Self::Identity { .. } | Self::Hermes { .. } | Self::Codex { .. } @@ -748,6 +757,9 @@ impl Command { pub fn as_vfs_command(&self) -> Option { match self { + Self::Cycles { command } => Some(VfsCommand::Cycles { + command: command.clone(), + }), Self::Database { command } => Some(VfsCommand::Database { command: command.clone(), }), @@ -954,20 +966,13 @@ impl Command { #[cfg(test)] mod tests { use super::{ - ClaudeCommand, Cli, CodexCommand, Command, DatabaseCommand, HermesCommand, IdentityModeArg, - NodeKindArg, SkillCommand, SkillImportCommand, SkillRunOutcomeArg, SkillStatusArg, + ClaudeCommand, Cli, CodexCommand, Command, CyclesCommand, DatabaseCommand, HermesCommand, + IdentityModeArg, NodeKindArg, SkillCommand, SkillImportCommand, SkillRunOutcomeArg, + SkillStatusArg, }; use clap::{CommandFactory, Parser}; use vfs_cli::cli::VfsCommand; - #[test] - fn main_cli_help_does_not_list_beam_bench() { - let mut command = Cli::command(); - let help = command.render_long_help().to_string(); - - assert!(!help.contains("beam-bench")); - } - #[test] fn main_cli_help_describes_agent_entrypoints() { let mut command = Cli::command(); @@ -1078,6 +1083,56 @@ mod tests { assert_eq!(name, "team-db"); assert!(Cli::try_parse_from(["kinic-vfs-cli", "database", "create"]).is_err()); + let cli = Cli::parse_from([ + "kinic-vfs-cli", + "database", + "purchase-cycles", + "db_alpha", + "1.25", + ]); + let Command::Database { + command: DatabaseCommand::PurchaseCycles { database_id, kinic }, + } = cli.command + else { + panic!("expected database cycle purchase command"); + }; + assert_eq!(database_id, "db_alpha"); + assert_eq!(kinic, "1.25"); + + let cli = Cli::parse_from([ + "kinic-vfs-cli", + "database", + "cycles", + "db_alpha", + "1.25", + "--browser-origin", + "http://127.0.0.1:3000", + ]); + let Command::Database { + command: + DatabaseCommand::Cycles { + database_id, + kinic, + browser_origin, + }, + } = cli.command + else { + panic!("expected database cycles command"); + }; + assert_eq!(database_id, "db_alpha"); + assert_eq!(kinic, "1.25"); + assert_eq!(browser_origin.as_deref(), Some("http://127.0.0.1:3000")); + + let cli = Cli::parse_from(["kinic-vfs-cli", "database", "cycles-history", "db_alpha"]); + let Command::Database { + command: DatabaseCommand::CyclesHistory { database_id, json }, + } = cli.command + else { + panic!("expected database cycles-history command"); + }; + assert_eq!(database_id, "db_alpha"); + assert!(!json); + let cli = Cli::parse_from(["kinic-vfs-cli", "database", "rename", "db_alpha", "Alpha"]); let Command::Database { command: DatabaseCommand::Rename { database_id, name }, @@ -1135,6 +1190,18 @@ mod tests { assert!(json); } + #[test] + fn main_cli_parses_cycles_commands() { + let cli = Cli::parse_from(["kinic-vfs-cli", "cycles", "config"]); + let Command::Cycles { + command: CyclesCommand::Config { json }, + } = cli.command + else { + panic!("expected cycles config command"); + }; + assert!(!json); + } + #[test] fn command_identity_requirement_keeps_reads_anonymous() { let read = Cli::parse_from(["kinic-vfs-cli", "read-node", "--path", "/Wiki/index.md"]); @@ -1193,6 +1260,27 @@ mod tests { let list = Cli::parse_from(["kinic-vfs-cli", "database", "list"]); assert!(!list.command.requires_identity()); assert!(list.command.prefers_identity_in_auto()); + + let cycles_config = Cli::parse_from(["kinic-vfs-cli", "cycles", "config"]); + assert!(!cycles_config.command.requires_identity()); + assert!(!cycles_config.command.probes_anonymous_database_read()); + + let database_cycles_purchase = Cli::parse_from([ + "kinic-vfs-cli", + "database", + "purchase-cycles", + "db_alpha", + "1.25", + ]); + assert!(database_cycles_purchase.command.requires_identity()); + + let database_cycles_history = + Cli::parse_from(["kinic-vfs-cli", "database", "cycles-history", "db_alpha"]); + assert!(database_cycles_history.command.requires_identity()); + + let database_cycles = + Cli::parse_from(["kinic-vfs-cli", "database", "cycles", "db_alpha", "1.25"]); + assert!(!database_cycles.command.requires_identity()); } #[test] @@ -1320,7 +1408,7 @@ mod tests { "kinic-vfs-cli", "--local", "--replica-host", - "http://127.0.0.1:8001", + "http://127.0.0.1:8011", "status", ]); assert!(parsed.is_err()); diff --git a/crates/vfs_cli_app/src/commands_fs_tests.rs b/crates/vfs_cli_app/src/commands_fs_tests.rs index e6d6289e..c20b221c 100644 --- a/crates/vfs_cli_app/src/commands_fs_tests.rs +++ b/crates/vfs_cli_app/src/commands_fs_tests.rs @@ -7,11 +7,12 @@ use tempfile::tempdir; use vfs_cli::connection::ResolvedConnection; use vfs_client::VfsApi; use vfs_types::{ - AppendNodeRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, - ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, - GlobNodeHit, GlobNodesRequest, ListNodesRequest, MkdirNodeRequest, MkdirNodeResult, - MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeEntry, - NodeKind, NodeMutationAck, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, Status, + AppendNodeRequest, CyclesBillingConfig, DatabaseRole, DatabaseStatus, DatabaseSummary, + DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, + ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, + GlobNodesRequest, ListNodesRequest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, + MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeEntry, NodeKind, + NodeMutationAck, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, Status, WriteNodeRequest, WriteNodeResult, }; @@ -57,6 +58,28 @@ impl VfsApi for MockClient { }) } + async fn get_cycles_billing_config(&self) -> Result { + Ok(CyclesBillingConfig { + kinic_ledger_canister_id: "ryjl3-tyaaa-aaaaa-aaaba-cai".to_string(), + billing_authority_id: "aaaaa-aa".to_string(), + cycles_per_kinic: 1_000, + min_update_cycles: 1, + }) + } + + async fn list_databases(&self) -> Result> { + Ok(vec![DatabaseSummary { + database_id: "default".to_string(), + name: "Default".to_string(), + status: DatabaseStatus::Active, + role: DatabaseRole::Owner, + logical_size_bytes: 0, + cycles_balance: Some(10), + cycles_suspended_at_ms: None, + archived_at_ms: None, + }]) + } + async fn read_node(&self, _database_id: &str, _path: &str) -> Result> { if self.nodes.is_empty() { return Ok(Some(Node { @@ -340,6 +363,49 @@ async fn write_node_rejects_non_canonical_source_paths() { assert!(writes.is_empty()); } +#[tokio::test] +async fn move_node_rejects_non_canonical_source_target() { + let client = MockClient { + nodes: vec![Node { + path: "/Sources/raw/web/abc.md".to_string(), + kind: NodeKind::Source, + content: "source".to_string(), + created_at: 1, + updated_at: 2, + etag: "etag-source".to_string(), + metadata_json: "{}".to_string(), + }], + ..MockClient::default() + }; + + let error = run_command( + &client, + Cli { + connection: ConnectionArgs { + database_id: Some("default".to_string()), + local: false, + replica_host: None, + canister_id: None, + identity_mode: IdentityModeArg::Auto, + allow_non_ii_identity: false, + }, + command: Command::MoveNode { + from_path: "/Sources/raw/web/abc.md".to_string(), + to_path: "/Sources/raw/web/wrong.txt".to_string(), + expected_etag: Some("etag-source".to_string()), + overwrite: false, + json: false, + }, + }, + &test_connection(), + ) + .await + .expect_err("non-canonical source target should fail"); + + assert!(error.to_string().contains("canonical form")); + assert!(client.moves.lock().expect("moves should lock").is_empty()); +} + #[tokio::test] async fn delete_node_autofills_folder_index_etag() { let client = MockClient { diff --git a/crates/vfs_cli_app/src/conversation_wiki.rs b/crates/vfs_cli_app/src/conversation_wiki.rs index 45cd8ca3..8d5d192d 100644 --- a/crates/vfs_cli_app/src/conversation_wiki.rs +++ b/crates/vfs_cli_app/src/conversation_wiki.rs @@ -6,7 +6,7 @@ use chrono::Utc; use serde::Serialize; use vfs_client::VfsApi; use vfs_types::{NodeKind, WriteNodeRequest}; -use wiki_domain::RAW_SOURCES_PREFIX; +use wiki_domain::{RAW_SOURCES_PREFIX, validate_canonical_source_path}; const CONVERSATION_WIKI_PREFIX: &str = "/Wiki/conversations"; @@ -85,6 +85,7 @@ fn parse_raw_conversation(source_path: &str, content: &str) -> Result Result { + validate_canonical_source_path(source_path).map_err(anyhow::Error::msg)?; let relative = source_path .strip_prefix(&format!("{RAW_SOURCES_PREFIX}/")) .ok_or_else(|| anyhow!("source path must be under {RAW_SOURCES_PREFIX}: {source_path}"))?; @@ -113,10 +114,17 @@ fn metadata_value(content: &str, key: &str) -> Option { content.lines().find_map(|line| { let trimmed = line.trim(); let value = trimmed.strip_prefix(&format!("- {key}:"))?.trim(); - Some(value.trim_matches('"').to_string()).filter(|value| !value.is_empty()) + Some(clean_metadata_value(value)).filter(|value| !value.is_empty()) }) } +fn clean_metadata_value(value: &str) -> String { + if value.starts_with('"') && value.ends_with('"') { + return serde_json::from_str::(value).unwrap_or_else(|_| value.to_string()); + } + value.to_string() +} + fn count_turns(content: &str) -> usize { content .lines() @@ -253,6 +261,25 @@ mod tests { assert_eq!(raw.message_count, 2); } + #[test] + fn parse_raw_conversation_unescapes_quoted_metadata() { + let raw = parse_raw_conversation( + "/Sources/raw/chatgpt/abc.md", + "# Raw Conversation Source\n\n## Metadata\n\n- provider: chatgpt\n- conversation_title: \"Project \\\"Alpha\\\"\"\n- message_count: 1\n", + ) + .expect("raw should parse"); + + assert_eq!(raw.title, "Project \"Alpha\""); + } + + #[test] + fn parse_raw_conversation_rejects_noncanonical_source_path() { + let error = parse_raw_conversation("/Sources/raw/../evil.md", RAW) + .expect_err("noncanonical raw path should fail"); + + assert!(error.to_string().contains("canonical form")); + } + #[test] fn generated_wiki_does_not_copy_transcript_body() { let raw = diff --git a/crates/vfs_cli_app/src/github_ingest.rs b/crates/vfs_cli_app/src/github_ingest.rs index 692f0898..3c4eeef9 100644 --- a/crates/vfs_cli_app/src/github_ingest.rs +++ b/crates/vfs_cli_app/src/github_ingest.rs @@ -235,6 +235,8 @@ fn print_result(json: bool, path: &str) -> Result<()> { fn valid_segment(segment: &str) -> bool { !segment.is_empty() + && segment != "." + && segment != ".." && segment .chars() .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.') diff --git a/crates/vfs_cli_app/src/hermes.rs b/crates/vfs_cli_app/src/hermes.rs index e131c3b8..490fe4a8 100644 --- a/crates/vfs_cli_app/src/hermes.rs +++ b/crates/vfs_cli_app/src/hermes.rs @@ -599,19 +599,38 @@ fn frontmatter_scalar(content: &str, key: &str) -> Option { if !content.starts_with("---\n") { return None; } - let end = content[4..].find("\n---")? + 4; + let rest = &content[4..]; + let end = frontmatter_end(rest)? + 4; for line in content[4..end].lines() { if line.starts_with(' ') || !line.contains(':') { continue; } let (field, value) = line.split_once(':')?; if field.trim() == key { - return Some(value.trim().trim_matches('"').to_string()); + return Some(clean_yaml_value(value)); } } None } +fn frontmatter_end(rest: &str) -> Option { + rest.find("\n---\n").or_else(|| { + rest.ends_with("\n---") + .then_some(rest.len() - "\n---".len()) + }) +} + +fn clean_yaml_value(value: &str) -> String { + let trimmed = value.trim(); + if trimmed.starts_with('"') && trimmed.ends_with('"') { + return serde_json::from_str::(trimmed).unwrap_or_else(|_| trimmed.to_string()); + } + if trimmed.starts_with('\'') && trimmed.ends_with('\'') { + return trimmed[1..trimmed.len() - 1].replace("''", "'"); + } + trimmed.to_string() +} + fn pending_json_files(pending_dir: &Path) -> Result> { if !pending_dir.exists() { return Ok(Vec::new()); @@ -763,6 +782,21 @@ mod tests { ); } + #[test] + fn frontmatter_scalar_requires_whole_line_terminator_and_unescapes_quotes() { + let content = + "---\nstatus: \"queued\\\"now\"\n---not-a-terminator\nignored: true\n---\n# Job\n"; + + assert_eq!( + frontmatter_scalar(content, "status"), + Some("queued\"now".to_string()) + ); + assert_eq!( + frontmatter_scalar(content, "ignored"), + Some("true".to_string()) + ); + } + #[test] fn install_plugin_writes_self_contained_payload() { let temp = TempDir::new().unwrap(); diff --git a/crates/vfs_cli_app/src/purge_url_ingest.rs b/crates/vfs_cli_app/src/purge_url_ingest.rs index 8aa33026..4aa1a9b9 100644 --- a/crates/vfs_cli_app/src/purge_url_ingest.rs +++ b/crates/vfs_cli_app/src/purge_url_ingest.rs @@ -351,12 +351,20 @@ fn parse_frontmatter(content: &str) -> Result> { let Some(rest) = content.strip_prefix("---\n") else { return Ok(None); }; - let Some((frontmatter, _body)) = rest.split_once("\n---") else { + let Some(end) = frontmatter_end(rest) else { return Ok(None); }; + let frontmatter = &rest[..end]; Ok(Some(serde_yaml::from_str(frontmatter)?)) } +fn frontmatter_end(rest: &str) -> Option { + rest.find("\n---\n").or_else(|| { + rest.ends_with("\n---") + .then_some(rest.len() - "\n---".len()) + }) +} + fn normalize_url(value: &str) -> Result { let mut parsed = reqwest::Url::parse(value).map_err(|error| anyhow!("invalid URL: {error}"))?; if parsed.scheme() != "http" && parsed.scheme() != "https" { @@ -481,6 +489,18 @@ mod tests { use async_trait::async_trait; use vfs_types::*; + #[test] + fn parse_frontmatter_requires_whole_line_terminator() { + let parsed = parse_frontmatter( + "---\nkind: kinic.url_ingest_request\nurl: https://example.com\nnote: ---not-a-terminator\nstatus: queued\n---\n# Request\n", + ) + .expect("frontmatter parse should not fail") + .expect("frontmatter should be present"); + + assert_eq!(parsed.kind.as_deref(), Some("kinic.url_ingest_request")); + assert_eq!(parsed.status.as_deref(), Some("queued")); + } + #[derive(Default)] struct PlanClient { entries: Vec, diff --git a/crates/vfs_cli_app/src/skill_registry.rs b/crates/vfs_cli_app/src/skill_registry.rs index 3e6b3f34..84356453 100644 --- a/crates/vfs_cli_app/src/skill_registry.rs +++ b/crates/vfs_cli_app/src/skill_registry.rs @@ -11,7 +11,8 @@ use anyhow::{Context, Result, anyhow}; use chrono::{DateTime, Utc}; use model::{ PRIVATE_ROOT, RUN_ROOT, SkillId, extract_frontmatter, manifest_for_source, normalize_manifest, - now_millis, now_rfc3339, parse_skill_source_frontmatter, print, run_base_path, + now_millis, now_rfc3339, parse_skill_source_frontmatter, print, + remove_root_frontmatter_fields_preserving_content, run_base_path, set_manifest_provenance_field, set_manifest_status_preserving_content, set_root_frontmatter_field_preserving_content, skill_base_path, }; @@ -26,8 +27,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; pub(crate) use vfs_cli::skill_kb::{find_skills, inspect_skill}; use vfs_client::VfsApi; use vfs_types::{ - DeleteNodeRequest, ListNodesRequest, MkdirNodeRequest, NodeEntryKind, NodeKind, WriteNodeItem, - WriteNodeRequest, WriteNodesRequest, + DeleteNodeRequest, ListNodesRequest, MkdirNodeRequest, Node, NodeEntryKind, NodeKind, + WriteNodeItem, WriteNodeRequest, WriteNodesRequest, }; const SKILL_PACKAGE_FILE_LIMIT_MAX: usize = 100; @@ -500,10 +501,16 @@ pub(crate) async fn set_skill_status( let timestamp = now_rfc3339(); match status { SkillStatusArg::Promoted => { + content = remove_root_frontmatter_fields_preserving_content( + &content, + &["deprecated_at", "deprecated_reason"], + )?; content = set_root_frontmatter_field_preserving_content(&content, "promoted_at", ×tamp)?; } SkillStatusArg::Deprecated => { + content = + remove_root_frontmatter_fields_preserving_content(&content, &["promoted_at"])?; if let Some(reason) = reason { content = set_root_frontmatter_field_preserving_content( &content, @@ -517,7 +524,12 @@ pub(crate) async fn set_skill_status( ×tamp, )?; } - SkillStatusArg::Draft | SkillStatusArg::Reviewed => {} + SkillStatusArg::Draft | SkillStatusArg::Reviewed => { + content = remove_root_frontmatter_fields_preserving_content( + &content, + &["deprecated_at", "deprecated_reason", "promoted_at"], + )?; + } } client .write_node(WriteNodeRequest { @@ -603,12 +615,20 @@ pub(crate) async fn propose_improvement( } let diff = std::fs::read_to_string(diff_file) .with_context(|| format!("failed to read {}", diff_file.display()))?; - let path_timestamp = now_millis(); + let proposal_id = now_millis().to_string(); let created_at = now_rfc3339(); - let proposal_path = format!( - "{}/improvement-proposals/{path_timestamp}.md", - skill_base_path(&skill_id) - ); + let base_path = skill_base_path(&skill_id); + let current_path = format!("{base_path}/SKILL.md"); + let current = client + .read_node(database_id, ¤t_path) + .await? + .ok_or_else(|| anyhow!("SKILL.md not found: {current_path}"))?; + let proposal_root = format!("{base_path}/proposals/{proposal_id}"); + let proposal_path = format!("{proposal_root}/proposal.md"); + let diff_path = format!("{proposal_root}/diff.md"); + let candidate_path = format!("{proposal_root}/candidate/SKILL.md"); + let metrics_path = format!("{proposal_root}/metrics.json"); + let status_path = format!("{proposal_root}/status.md"); let source_runs = runs .iter() .map(|run| format!(" - {run}")) @@ -619,22 +639,78 @@ pub(crate) async fn propose_improvement( .map(|run| format!("- [{run}]({run})")) .collect::>() .join("\n"); - let content = format!( - "---\nkind: kinic.skill_improvement_proposal\nschema_version: 1\nskill_id: {id}\nstatus: proposed\nsource_runs:\n{source_runs}\ncreated_at: {created_at}\ncreated_by: cli\n---\n# Skill Improvement Proposal\n\n## Summary\n\n{summary}\n\n## Evidence\n\n{evidence_links}\n\n## Proposed Diff\n\n{}\n", + let proposal_content = format!( + "---\nkind: kinic.skill_evolution_proposal\nschema_version: 1\nskill_id: {id}\nproposal_id: {proposal_id}\ncreated_at: {created_at}\ncreated_by: cli\nsource_runs:\n{source_runs}\n---\n# Skill Evolution Proposal\n\n## Summary\n\n{summary}\n\n## Evidence\n\n{evidence_links}\n\n## Diff\n\n[diff.md](diff.md)\n" + ); + let diff_content = format!( + "# Proposal Diff\n\n{}\n", markdown_code_block("diff", &diff) ); - ensure_parent_folders(client, database_id, &proposal_path).await?; + let metrics_content = serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "skill_id": id, + "proposal_id": proposal_id, + "base_etag": current.etag, + "created_at": created_at, + "source_runs": runs, + "candidate_score_gate": "manual_review_required", + "heading_consistency_gate": "manual_review_required", + "permission_gate": "manual_review_required" + }))?; + let status_content = proposal_status_content(&skill_id, &proposal_id, "proposed", None); + let paths = vec![ + proposal_path.clone(), + diff_path.clone(), + candidate_path.clone(), + metrics_path.clone(), + status_path.clone(), + ]; + ensure_parent_folders_for_paths(client, database_id, &paths).await?; client - .write_node(WriteNodeRequest { + .write_nodes(WriteNodesRequest { database_id: database_id.to_string(), - path: proposal_path.clone(), - kind: NodeKind::File, - content, - metadata_json: "{}".to_string(), - expected_etag: None, + nodes: vec![ + WriteNodeItem { + path: proposal_path.clone(), + kind: NodeKind::File, + content: proposal_content, + metadata_json: "{}".to_string(), + expected_etag: None, + }, + WriteNodeItem { + path: diff_path, + kind: NodeKind::File, + content: diff_content, + metadata_json: "{}".to_string(), + expected_etag: None, + }, + WriteNodeItem { + path: candidate_path, + kind: NodeKind::File, + content: current.content, + metadata_json: "{}".to_string(), + expected_etag: None, + }, + WriteNodeItem { + path: metrics_path, + kind: NodeKind::File, + content: metrics_content, + metadata_json: "{}".to_string(), + expected_etag: None, + }, + WriteNodeItem { + path: status_path, + kind: NodeKind::File, + content: status_content, + metadata_json: "{}".to_string(), + expected_etag: None, + }, + ], }) .await?; - Ok(json!({ "id": id, "proposal_path": proposal_path, "status": "proposed" })) + Ok( + json!({ "id": id, "proposal_id": proposal_id, "proposal_path": proposal_path, "proposal_root": proposal_root, "status": "proposed" }), + ) } pub(crate) async fn approve_proposal( @@ -643,25 +719,37 @@ pub(crate) async fn approve_proposal( id: &str, proposal_path: &str, ) -> Result { - validate_proposal_target(id, proposal_path)?; + let (proposal_root, proposal_id) = validate_proposal_target(id, proposal_path)?; + let proposal_md_path = format!("{proposal_root}/proposal.md"); let node = client - .read_node(database_id, proposal_path) + .read_node(database_id, &proposal_md_path) .await? - .ok_or_else(|| anyhow!("proposal not found: {proposal_path}"))?; - validate_proposal_frontmatter(id, &node.content)?; - let content = - set_root_frontmatter_field_preserving_content(&node.content, "status", "approved")?; + .ok_or_else(|| anyhow!("proposal not found: {proposal_md_path}"))?; + validate_proposal_frontmatter(id, &proposal_id, &node.content)?; + let status_path = format!("{proposal_root}/status.md"); + let current_status = read_required_proposal_status( + client, + database_id, + &status_path, + id, + &proposal_id, + &["proposed"], + ) + .await?; + let content = proposal_status_content(&SkillId::parse(id)?, &proposal_id, "reviewed", None); client .write_node(WriteNodeRequest { database_id: database_id.to_string(), - path: proposal_path.to_string(), + path: status_path.clone(), kind: NodeKind::File, content, - metadata_json: node.metadata_json, - expected_etag: Some(node.etag), + metadata_json: current_status.metadata_json, + expected_etag: Some(current_status.etag), }) .await?; - Ok(json!({ "id": id, "proposal_path": proposal_path, "status": "approved" })) + Ok( + json!({ "id": id, "proposal_id": proposal_id, "proposal_path": proposal_md_path, "status_path": status_path, "status": "reviewed" }), + ) } pub(crate) async fn install_skill_lockfile( @@ -861,6 +949,16 @@ pub(crate) async fn apply_evolution_proposal( .read_node(database_id, &metrics_path) .await? .ok_or_else(|| anyhow!("metrics.json not found: {metrics_path}"))?; + let status_path = format!("{base_path}/proposals/{proposal_id}/status.md"); + let status_node = read_required_proposal_status( + client, + database_id, + &status_path, + &skill_id.to_string(), + proposal_id, + &["proposed", "reviewed"], + ) + .await?; let metrics_json: serde_json::Value = serde_json::from_str(&metrics.content) .with_context(|| format!("invalid metrics JSON: {metrics_path}"))?; let base_etag = metrics_json @@ -869,18 +967,16 @@ pub(crate) async fn apply_evolution_proposal( .ok_or_else(|| anyhow!("metrics.json must contain base_etag"))?; let gate_failure = proposal_gate_failure(&metrics_json); if let Some(gate_failure) = gate_failure { - let status_path = format!("{base_path}/proposals/{proposal_id}/status.md"); let content = proposal_status_content(&skill_id, proposal_id, "gate_failed", Some(gate_failure)); - ensure_parent_folders(client, database_id, &status_path).await?; client .write_node(WriteNodeRequest { database_id: database_id.to_string(), path: status_path.clone(), kind: NodeKind::File, content, - metadata_json: "{}".to_string(), - expected_etag: None, + metadata_json: status_node.metadata_json, + expected_etag: Some(status_node.etag), }) .await?; return Ok( @@ -888,18 +984,16 @@ pub(crate) async fn apply_evolution_proposal( ); } if base_etag != current.etag { - let status_path = format!("{base_path}/proposals/{proposal_id}/status.md"); let content = proposal_status_content(&skill_id, proposal_id, "conflict", Some(¤t.etag)); - ensure_parent_folders(client, database_id, &status_path).await?; client .write_node(WriteNodeRequest { database_id: database_id.to_string(), path: status_path.clone(), kind: NodeKind::File, content, - metadata_json: "{}".to_string(), - expected_etag: None, + metadata_json: status_node.metadata_json, + expected_etag: Some(status_node.etag), }) .await?; return Ok( @@ -942,17 +1036,15 @@ pub(crate) async fn apply_evolution_proposal( sync_error = Some(error.to_string()); } } - let status_path = format!("{base_path}/proposals/{proposal_id}/status.md"); let content = proposal_status_content(&skill_id, proposal_id, status, sync_error.as_deref()); - ensure_parent_folders(client, database_id, &status_path).await?; client .write_node(WriteNodeRequest { database_id: database_id.to_string(), path: status_path.clone(), kind: NodeKind::File, content, - metadata_json: "{}".to_string(), - expected_etag: None, + metadata_json: status_node.metadata_json, + expected_etag: Some(status_node.etag), }) .await?; Ok( @@ -1561,12 +1653,23 @@ fn frontmatter_scalar(content: &str, key: &str) -> Option { continue; }; if field.trim() == key { - return Some(value.trim().trim_matches('"').to_string()); + return Some(clean_yaml_value(value)); } } None } +fn clean_yaml_value(value: &str) -> String { + let trimmed = value.trim(); + if trimmed.starts_with('"') && trimmed.ends_with('"') { + return serde_json::from_str::(trimmed).unwrap_or_else(|_| trimmed.to_string()); + } + if trimmed.starts_with('\'') && trimmed.ends_with('\'') { + return trimmed[1..trimmed.len() - 1].replace("''", "'"); + } + trimmed.to_string() +} + fn is_claim_expired(value: &str) -> bool { DateTime::parse_from_rfc3339(value) .map(|expires| expires.with_timezone(&Utc) <= Utc::now()) @@ -1592,6 +1695,16 @@ mod skill_evolve_jobs_tests { ); } + #[test] + fn frontmatter_scalar_unescapes_json_quoted_value() { + let content = "---\nstatus: \"queued\\\"now\"\n---\n# Job\n"; + + assert_eq!( + frontmatter_scalar(content, "status"), + Some("queued\"now".to_string()) + ); + } + #[test] fn truncate_error_removes_newlines_and_caps_length() { let value = truncate_error(&format!("a\n{}", "b".repeat(1000))); @@ -1606,25 +1719,43 @@ struct ProposalFrontmatter { kind: String, schema_version: u32, skill_id: String, + proposal_id: String, +} + +#[derive(Deserialize)] +struct ProposalStatusFrontmatter { + kind: String, + schema_version: u32, + skill_id: String, + proposal_id: String, status: String, + recorded_at: String, } -fn validate_proposal_target(id: &str, proposal_path: &str) -> Result<()> { +fn validate_proposal_target(id: &str, proposal_path: &str) -> Result<(String, String)> { let skill_id = SkillId::parse(id)?; - let private_prefix = format!("{}/{}/improvement-proposals/", PRIVATE_ROOT, skill_id); - if proposal_path.starts_with(&private_prefix) { - return Ok(()); + let private_prefix = format!("{}/{}/proposals/", PRIVATE_ROOT, skill_id); + let Some(rest) = proposal_path.strip_prefix(&private_prefix) else { + return Err(anyhow!("proposal path must belong to skill {id} proposals")); + }; + let proposal_id = rest + .strip_suffix("/proposal.md") + .unwrap_or(rest) + .trim_end_matches('/'); + if valid_id_segment(proposal_id) { + return Ok(( + format!("{private_prefix}{proposal_id}"), + proposal_id.to_string(), + )); } - Err(anyhow!( - "proposal path must belong to skill {id} improvement-proposals" - )) + Err(anyhow!("proposal id must use a single path-safe name")) } -fn validate_proposal_frontmatter(id: &str, content: &str) -> Result<()> { +fn validate_proposal_frontmatter(id: &str, proposal_id: &str, content: &str) -> Result<()> { let frontmatter: ProposalFrontmatter = serde_yaml::from_str(extract_frontmatter(content)?)?; - if frontmatter.kind != "kinic.skill_improvement_proposal" { + if frontmatter.kind != "kinic.skill_evolution_proposal" { return Err(anyhow!( - "proposal kind must be kinic.skill_improvement_proposal" + "proposal kind must be kinic.skill_evolution_proposal" )); } if frontmatter.schema_version != 1 { @@ -1633,12 +1764,69 @@ fn validate_proposal_frontmatter(id: &str, content: &str) -> Result<()> { if frontmatter.skill_id != id { return Err(anyhow!("proposal skill_id must match id")); } - if frontmatter.status != "proposed" { - return Err(anyhow!("proposal status must be proposed")); + if frontmatter.proposal_id != proposal_id { + return Err(anyhow!("proposal proposal_id must match path")); } Ok(()) } +fn validate_proposal_status_frontmatter_one_of( + id: &str, + proposal_id: &str, + content: &str, + expected_statuses: &[&str], +) -> Result<()> { + let frontmatter: ProposalStatusFrontmatter = + serde_yaml::from_str(extract_frontmatter(content)?)?; + if frontmatter.kind != "kinic.skill_evolution_proposal_status" { + return Err(anyhow!( + "proposal status kind must be kinic.skill_evolution_proposal_status" + )); + } + if frontmatter.schema_version != 1 { + return Err(anyhow!("proposal status schema_version must be 1")); + } + if frontmatter.skill_id != id { + return Err(anyhow!("proposal status skill_id must match id")); + } + if frontmatter.proposal_id != proposal_id { + return Err(anyhow!("proposal status proposal_id must match path")); + } + if !expected_statuses + .iter() + .any(|expected| frontmatter.status == *expected) + { + return Err(anyhow!( + "proposal status must be one of {}", + expected_statuses.join(", ") + )); + } + DateTime::parse_from_rfc3339(&frontmatter.recorded_at) + .map(|_| ()) + .map_err(|_| anyhow!("proposal status recorded_at must be RFC3339")) +} + +async fn read_required_proposal_status( + client: &impl VfsApi, + database_id: &str, + status_path: &str, + id: &str, + proposal_id: &str, + expected_statuses: &[&str], +) -> Result { + let status = client + .read_node(database_id, status_path) + .await? + .ok_or_else(|| anyhow!("proposal status not found: {status_path}"))?; + validate_proposal_status_frontmatter_one_of( + id, + proposal_id, + &status.content, + expected_statuses, + )?; + Ok(status) +} + async fn ensure_parent_folders(client: &impl VfsApi, database_id: &str, path: &str) -> Result<()> { ensure_parent_folders_for_paths(client, database_id, &[path.to_string()]).await } @@ -1769,14 +1957,34 @@ fn referenced_markdown_files(source_dir: &Path, skill: &str) -> Result Vec { let mut targets = Vec::new(); - let mut rest = content; - while let Some(start) = rest.find("](") { - rest = &rest[start + 2..]; - let Some(end) = rest.find(')') else { - break; - }; - targets.push(rest[..end].to_string()); - rest = &rest[end + 1..]; + let bytes = content.as_bytes(); + let mut index = 0; + while let Some(start) = content[index..].find("](").map(|found| index + found + 2) { + let mut cursor = start; + if bytes.get(cursor) == Some(&b'<') + && let Some(close) = content[cursor + 1..] + .find('>') + .map(|found| cursor + 1 + found) + && bytes.get(close + 1) == Some(&b')') + { + targets.push(content[cursor..=close].to_string()); + index = close + 2; + continue; + } + let mut depth = 0_usize; + while cursor < content.len() { + match bytes[cursor] { + b'(' => depth += 1, + b')' if depth == 0 => break, + b')' => depth -= 1, + _ => {} + } + cursor += 1; + } + if cursor < content.len() { + targets.push(content[start..cursor].to_string()); + } + index = cursor.saturating_add(1); } targets } @@ -1811,7 +2019,11 @@ pub(crate) fn markdown_target_package_key(raw_target: &str) -> Option { } fn clean_markdown_link_target(raw_target: &str) -> Option { - let target = raw_target.split_whitespace().next()?.trim(); + let target = markdown_destination_without_title(raw_target.trim()); + let target = target + .strip_prefix('<') + .and_then(|inner| inner.strip_suffix('>')) + .unwrap_or(target); let target = target.split(['#', '?']).next()?.trim(); if target.is_empty() || target.starts_with('#') @@ -1824,6 +2036,73 @@ fn clean_markdown_link_target(raw_target: &str) -> Option { Some(target.to_string()) } +fn markdown_destination_without_title(target: &str) -> &str { + let target = target.trim(); + if let Some(inner) = target.strip_prefix('<') + && let Some(close) = inner.find('>') + { + let destination = &inner[..close]; + let suffix = inner[close + 1..].trim(); + if suffix.is_empty() || is_markdown_title_suffix(suffix) { + return destination; + } + } + strip_quoted_markdown_title(target, b'"') + .or_else(|| strip_quoted_markdown_title(target, b'\'')) + .or_else(|| strip_parenthesized_markdown_title(target)) + .unwrap_or(target) +} + +fn strip_quoted_markdown_title(target: &str, quote: u8) -> Option<&str> { + let bytes = target.as_bytes(); + if bytes.last().copied() != Some(quote) { + return None; + } + for index in (0..bytes.len().saturating_sub(1)).rev() { + if bytes[index] == quote && index > 0 && bytes[index - 1].is_ascii_whitespace() { + let destination = target[..index - 1].trim_end(); + if is_markdown_destination_candidate(destination) { + return Some(destination); + } + } + } + None +} + +fn strip_parenthesized_markdown_title(target: &str) -> Option<&str> { + if !target.ends_with(')') { + return None; + } + let title_start = target.rfind(" (")?; + let destination = target[..title_start].trim_end(); + if is_markdown_destination_candidate(destination) { + return Some(destination); + } + None +} + +fn is_markdown_title_suffix(value: &str) -> bool { + (value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\'')) + || (value.starts_with('(') && value.ends_with(')')) +} + +fn is_markdown_destination_candidate(value: &str) -> bool { + let target = value + .strip_prefix('<') + .and_then(|inner| inner.strip_suffix('>')) + .unwrap_or(value) + .split(['#', '?']) + .next() + .unwrap_or("") + .trim(); + !target.is_empty() + && !target.starts_with('#') + && !target.starts_with('/') + && !target.contains("://") + && target.ends_with(".md") +} + fn path_to_package_key(path: &Path) -> Option { let mut parts = Vec::new(); for component in path.components() { @@ -1871,10 +2150,11 @@ fn resolve_run_id(run_id_override: Option<&str>, evidence_run_id: Option<&str>) } fn valid_id_segment(value: &str) -> bool { - !value.is_empty() - && value - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + let mut chars = value.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_alphanumeric()) + && value.len() <= 128 + && !value.contains("..") + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) } fn markdown_code_block(info: &str, content: &str) -> String { @@ -1905,15 +2185,11 @@ fn is_runtime_export_file(relative_path: &str) -> bool { if matches!(relative_path, "manifest.md" | "provenance.md" | "evals.md") { return false; } - !relative_path.starts_with("proposals/") - && !relative_path.starts_with("versions/") - && !relative_path.starts_with("improvement-proposals/") + !relative_path.starts_with("proposals/") && !relative_path.starts_with("versions/") } fn is_github_export_file(relative_path: &str) -> bool { - !relative_path.starts_with("proposals/") - && !relative_path.starts_with("versions/") - && !relative_path.starts_with("improvement-proposals/") + !relative_path.starts_with("proposals/") && !relative_path.starts_with("versions/") } async fn github_export_files( diff --git a/crates/vfs_cli_app/src/skill_registry/model.rs b/crates/vfs_cli_app/src/skill_registry/model.rs index a2676511..3e1b094e 100644 --- a/crates/vfs_cli_app/src/skill_registry/model.rs +++ b/crates/vfs_cli_app/src/skill_registry/model.rs @@ -237,6 +237,43 @@ pub(super) fn set_root_frontmatter_field_preserving_content( )) } +pub(super) fn remove_root_frontmatter_fields_preserving_content( + content: &str, + keys: &[&str], +) -> Result { + let frontmatter = extract_frontmatter(content)?; + let frontmatter_start = "---\n".len(); + let frontmatter_end = frontmatter_start + frontmatter.len(); + let mut updated = String::new(); + let mut skipping_removed_block = false; + for line in frontmatter.split_inclusive('\n') { + let trimmed = line.trim_start(); + let is_root_key = + !line.starts_with(' ') && !line.starts_with('\t') && trimmed.contains(':'); + if is_root_key + && keys + .iter() + .any(|key| trimmed.starts_with(&format!("{key}:"))) + { + skipping_removed_block = true; + continue; + } + if skipping_removed_block { + if line.starts_with(' ') || line.starts_with('\t') || line.trim().is_empty() { + continue; + } + skipping_removed_block = false; + } + updated.push_str(line); + } + Ok(format!( + "{}{}{}", + &content[..frontmatter_start], + updated, + &content[frontmatter_end..] + )) +} + pub(super) fn set_manifest_provenance_field( content: &str, key: &str, @@ -289,14 +326,21 @@ pub(super) fn extract_frontmatter(content: &str) -> Result<&str> { .strip_prefix("---\n") .ok_or_else(|| anyhow!("manifest must start with YAML frontmatter"))?; let end = rest - .find("\n---") + .lines() + .scan(0_usize, |offset, line| { + let start = *offset; + *offset += line.len() + 1; + Some((start, line)) + }) + .find_map(|(offset, line)| (line == "---").then_some(offset.saturating_sub(1))) .ok_or_else(|| anyhow!("manifest frontmatter is not closed"))?; Ok(&rest[..end]) } fn valid_segment(value: &str) -> bool { - !value.is_empty() - && value - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + let mut chars = value.chars(); + matches!(chars.next(), Some(ch) if ch.is_ascii_alphanumeric()) + && value.len() <= 128 + && !value.contains("..") + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) } diff --git a/crates/vfs_cli_app/src/skill_registry_tests.rs b/crates/vfs_cli_app/src/skill_registry_tests.rs index a7264ae9..2900b532 100644 --- a/crates/vfs_cli_app/src/skill_registry_tests.rs +++ b/crates/vfs_cli_app/src/skill_registry_tests.rs @@ -661,6 +661,17 @@ async fn skill_record_run_evidence_export_correction_and_apply_proposal() { }) .await .expect("metrics"); + client + .write_node(WriteNodeRequest { + database_id: "team-db".to_string(), + path: "/Wiki/skills/legal-review/proposals/p1/status.md".to_string(), + kind: NodeKind::File, + content: proposal_status_content("legal-review", "p1", "reviewed"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .expect("status"); let applied = apply_evolution_proposal(&client, "team-db", "legal-review", "p1", None, None) .await .expect("apply"); @@ -915,8 +926,9 @@ async fn proposal_generation_handles_backticks_in_diff_or_evidence() { ) .await .expect("proposal"); + let diff_path = format!("{}/diff.md", proposal["proposal_root"].as_str().unwrap()); let proposal_content = client - .read_node("team-db", proposal["proposal_path"].as_str().unwrap()) + .read_node("team-db", &diff_path) .await .unwrap() .unwrap() @@ -1056,6 +1068,17 @@ async fn skill_apply_proposal_rejects_failed_gates() { }) .await .unwrap(); + client + .write_node(WriteNodeRequest { + database_id: "team-db".to_string(), + path: "/Wiki/skills/legal-review/proposals/p1/status.md".to_string(), + kind: NodeKind::File, + content: proposal_status_content("legal-review", "p1", "reviewed"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .unwrap(); let applied = apply_evolution_proposal(&client, "team-db", "legal-review", "p1", None, None) .await @@ -1610,6 +1633,24 @@ async fn skill_upsert_uses_write_nodes_for_package_files() { ); } +#[tokio::test] +async fn skill_upsert_rejects_noncanonical_skill_ids_before_writing() { + let client = SkillMockClient::default(); + let temp = tempfile::tempdir().expect("tempdir"); + write(temp.path(), "SKILL.md", "# Legal Review\n\nReview."); + + let overlong = "a".repeat(129); + for invalid_id in ["legal..review", "_legal-review", overlong.as_str()] { + let error = upsert_skill(&client, "default", temp.path(), invalid_id, false) + .await + .expect_err("invalid skill id should fail before writes"); + assert!(error.to_string().contains("single path-safe name")); + } + + assert_eq!(client.write_batches.load(Ordering::SeqCst), 0); + assert_eq!(client.writes.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn skill_upsert_rejects_package_over_batch_limit_before_writing() { let client = SkillMockClient::default(); @@ -1929,6 +1970,66 @@ async fn skill_set_status_records_promoted_at_as_rfc3339() { assert_rfc3339_field(&manifest, "promoted_at"); } +#[tokio::test] +async fn skill_set_status_removes_stale_status_metadata() { + let client = SkillMockClient::default(); + let temp = tempfile::tempdir().expect("tempdir"); + write(temp.path(), "SKILL.md", "# Legal Review\n\nredlines"); + write(temp.path(), "manifest.md", &manifest("reviewed")); + upsert_skill(&client, "default", temp.path(), "legal-review", false) + .await + .expect("upsert"); + + set_skill_status( + &client, + "default", + "legal-review", + SkillStatusArg::Promoted, + None, + ) + .await + .expect("set promoted"); + set_skill_status( + &client, + "default", + "legal-review", + SkillStatusArg::Deprecated, + Some("retired"), + ) + .await + .expect("set deprecated"); + + let deprecated = client + .read_node("default", "/Wiki/skills/legal-review/manifest.md") + .await + .expect("read manifest") + .expect("manifest exists") + .content; + assert!(!deprecated.contains("promoted_at:")); + assert_rfc3339_field(&deprecated, "deprecated_at"); + assert!(deprecated.contains("deprecated_reason: retired")); + + set_skill_status( + &client, + "default", + "legal-review", + SkillStatusArg::Reviewed, + None, + ) + .await + .expect("set reviewed"); + + let reviewed = client + .read_node("default", "/Wiki/skills/legal-review/manifest.md") + .await + .expect("read manifest") + .expect("manifest exists") + .content; + assert!(!reviewed.contains("promoted_at:")); + assert!(!reviewed.contains("deprecated_at:")); + assert!(!reviewed.contains("deprecated_reason:")); +} + #[test] fn skill_markdown_targets_normalize_package_local_paths() { assert_eq!( @@ -1939,6 +2040,30 @@ fn skill_markdown_targets_normalize_package_local_paths() { markdown_target_package_key("./docs/usage.md#setup").as_deref(), Some("docs/usage.md") ); + assert_eq!( + markdown_target_package_key("docs/Project Plan.md").as_deref(), + Some("docs/Project Plan.md") + ); + assert_eq!( + markdown_target_package_key("").as_deref(), + Some("docs/Project Plan.md") + ); + assert_eq!( + markdown_target_package_key("docs/Project (Alpha).md").as_deref(), + Some("docs/Project (Alpha).md") + ); + assert_eq!( + markdown_target_package_key("docs/usage.md \"Usage\"").as_deref(), + Some("docs/usage.md") + ); + assert_eq!( + markdown_target_package_key(" 'Project plan'").as_deref(), + Some("docs/Project Plan.md") + ); + assert_eq!( + markdown_target_package_key("docs/usage.md (Usage)").as_deref(), + Some("docs/usage.md") + ); assert_eq!(markdown_target_package_key("../outside.md"), None); assert_eq!(markdown_target_package_key("/Wiki/skills/x.md"), None); assert_eq!( @@ -1949,7 +2074,7 @@ fn skill_markdown_targets_normalize_package_local_paths() { } #[tokio::test] -async fn skill_improvement_proposal_is_recorded_and_approved_without_editing_skill() { +async fn skill_evolution_proposal_is_recorded_and_reviewed_without_editing_skill() { let client = SkillMockClient::default(); let temp = tempfile::tempdir().expect("tempdir"); write(temp.path(), "SKILL.md", "# Legal Review\n\nredlines"); @@ -1976,15 +2101,15 @@ async fn skill_improvement_proposal_is_recorded_and_approved_without_editing_ski "/Wiki", "/Wiki/skills", "/Wiki/skills/legal-review", - "/Wiki/skills/legal-review/improvement-proposals", + "/Wiki/skills/legal-review/proposals", ], ); let proposal_path = proposal["proposal_path"].as_str().unwrap(); - let proposal_name = proposal_path - .strip_prefix("/Wiki/skills/legal-review/improvement-proposals/") + let proposal_root = proposal["proposal_root"].as_str().unwrap(); + let proposal_name = proposal_root + .strip_prefix("/Wiki/skills/legal-review/proposals/") .expect("proposal path prefix") - .strip_suffix(".md") - .expect("proposal extension"); + .trim_end_matches('/'); assert!(proposal_name.chars().all(|ch| ch.is_ascii_digit())); let skill_before = client .read_node("default", "/Wiki/skills/legal-review/SKILL.md") @@ -2002,14 +2127,21 @@ async fn skill_improvement_proposal_is_recorded_and_approved_without_editing_ski .unwrap() .unwrap() .content; + let status_content = client + .read_node("default", &format!("{proposal_root}/status.md")) + .await + .unwrap() + .unwrap() + .content; assert_rfc3339_field(&proposal_content, "created_at"); + assert_rfc3339_field(&status_content, "recorded_at"); let skill_after = client .read_node("default", "/Wiki/skills/legal-review/SKILL.md") .await .unwrap() .unwrap() .content; - assert!(proposal_content.contains("status: approved")); + assert!(status_content.contains("status: reviewed")); assert_eq!(skill_before, skill_after); } @@ -2019,9 +2151,9 @@ async fn skill_approve_proposal_rejects_wrong_path_and_frontmatter() { client .write_node(WriteNodeRequest { database_id: "default".to_string(), - path: "/Wiki/skills/other/improvement-proposals/1.md".to_string(), + path: "/Wiki/skills/other/proposals/1/proposal.md".to_string(), kind: NodeKind::File, - content: proposal_content("legal-review", "proposed"), + content: proposal_content("legal-review", "1"), metadata_json: "{}".to_string(), expected_etag: None, }) @@ -2032,7 +2164,7 @@ async fn skill_approve_proposal_rejects_wrong_path_and_frontmatter() { &client, "default", "legal-review", - "/Wiki/skills/other/improvement-proposals/1.md" + "/Wiki/skills/other/proposals/1" ) .await .is_err() @@ -2041,9 +2173,9 @@ async fn skill_approve_proposal_rejects_wrong_path_and_frontmatter() { client .write_node(WriteNodeRequest { database_id: "default".to_string(), - path: "/Wiki/skills/legal-review/improvement-proposals/1.md".to_string(), + path: "/Wiki/skills/legal-review/proposals/1/proposal.md".to_string(), kind: NodeKind::File, - content: proposal_content("other", "proposed"), + content: proposal_content("other", "1"), metadata_json: "{}".to_string(), expected_etag: None, }) @@ -2054,7 +2186,7 @@ async fn skill_approve_proposal_rejects_wrong_path_and_frontmatter() { &client, "default", "legal-review", - "/Wiki/skills/legal-review/improvement-proposals/1.md" + "/Wiki/skills/legal-review/proposals/1" ) .await .is_err() @@ -2063,31 +2195,63 @@ async fn skill_approve_proposal_rejects_wrong_path_and_frontmatter() { client .write_node(WriteNodeRequest { database_id: "default".to_string(), - path: "/Wiki/skills/legal-review/improvement-proposals/2.md".to_string(), + path: "/Wiki/skills/legal-review/proposals/2/proposal.md".to_string(), + kind: NodeKind::File, + content: proposal_content("legal-review", "2"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .expect("seed proposal"); + client + .write_node(WriteNodeRequest { + database_id: "default".to_string(), + path: "/Wiki/skills/legal-review/proposals/2/status.md".to_string(), kind: NodeKind::File, - content: proposal_content("legal-review", "approved"), + content: proposal_status_content("legal-review", "2", "reviewed"), metadata_json: "{}".to_string(), expected_etag: None, }) .await - .expect("seed approved"); + .expect("seed reviewed"); assert!( approve_proposal( &client, "default", "legal-review", - "/Wiki/skills/legal-review/improvement-proposals/2.md" + "/Wiki/skills/legal-review/proposals/2" ) .await .is_err() ); + client + .write_node(WriteNodeRequest { + database_id: "default".to_string(), + path: "/Wiki/skills/legal-review/proposals/3/proposal.md".to_string(), + kind: NodeKind::File, + content: proposal_content("legal-review", "3"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .expect("seed proposal without status"); + let error = approve_proposal( + &client, + "default", + "legal-review", + "/Wiki/skills/legal-review/proposals/3", + ) + .await + .expect_err("status is required"); + assert!(error.to_string().contains("proposal status not found")); + client .write_node(WriteNodeRequest { database_id: "default".to_string(), path: "/Wiki/skills/legal-review/SKILL.md".to_string(), kind: NodeKind::File, - content: proposal_content("legal-review", "proposed"), + content: proposal_content("legal-review", "3"), metadata_json: "{}".to_string(), expected_etag: None, }) @@ -2180,6 +2344,17 @@ async fn write_apply_proposal_fixture( }) .await .expect("write metrics"); + client + .write_node(WriteNodeRequest { + database_id: "team-db".to_string(), + path: "/Wiki/skills/legal-review/proposals/p1/status.md".to_string(), + kind: NodeKind::File, + content: proposal_status_content("legal-review", "p1", "reviewed"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .expect("write status"); } fn assert_rfc3339_field(content: &str, key: &str) { @@ -2221,9 +2396,15 @@ async fn write_skill_file(client: &SkillMockClient, database_id: &str, path: &st .expect("write skill file"); } -fn proposal_content(skill_id: &str, status: &str) -> String { +fn proposal_content(skill_id: &str, proposal_id: &str) -> String { + format!( + "---\nkind: kinic.skill_evolution_proposal\nschema_version: 1\nskill_id: {skill_id}\nproposal_id: {proposal_id}\ncreated_at: 2026-05-08T00:00:00Z\n---\n# Proposal\n" + ) +} + +fn proposal_status_content(skill_id: &str, proposal_id: &str, status: &str) -> String { format!( - "---\nkind: kinic.skill_improvement_proposal\nschema_version: 1\nskill_id: {skill_id}\nstatus: {status}\ncreated_at: 2026-05-08T00:00:00Z\n---\n# Proposal\n" + "---\nkind: kinic.skill_evolution_proposal_status\nschema_version: 1\nskill_id: {skill_id}\nproposal_id: {proposal_id}\nstatus: {status}\nrecorded_at: 2026-05-08T00:00:00Z\n---\n# Proposal Status\n" ) } @@ -2448,6 +2629,17 @@ async fn write_pbt_proposal( }) .await .expect("metrics should write"); + client + .write_node(WriteNodeRequest { + database_id: "team-db".to_string(), + path: format!("{base_path}/status.md"), + kind: NodeKind::File, + content: proposal_status_content(id, proposal_id, "reviewed"), + metadata_json: "{}".to_string(), + expected_etag: None, + }) + .await + .expect("status should write"); } async fn assert_single_run_plus_corrections(client: &SkillMockClient, id: &str, run_id: &str) { diff --git a/crates/vfs_cli_core/src/cli.rs b/crates/vfs_cli_core/src/cli.rs index c44ce3d1..3550c209 100644 --- a/crates/vfs_cli_core/src/cli.rs +++ b/crates/vfs_cli_core/src/cli.rs @@ -56,6 +56,10 @@ pub struct ConnectionArgs { #[derive(Subcommand, Debug, Clone)] pub enum VfsCommand { + Cycles { + #[command(subcommand)] + command: CyclesCommand, + }, Database { #[command(subcommand)] command: DatabaseCommand, @@ -265,6 +269,27 @@ pub enum DatabaseCommand { #[arg(long)] json: bool, }, + #[command(about = "Purchase non-refundable database cycles with KINIC")] + PurchaseCycles { database_id: String, kinic: String }, + #[command(about = "List cycles ledger entries for one database")] + CyclesHistory { + database_id: String, + #[arg(long)] + json: bool, + }, + #[command(about = "List pending cycles purchases for one database")] + CyclesPending { + database_id: String, + #[arg(long)] + json: bool, + }, + #[command(about = "Open the browser cycles purchase page for one database")] + Cycles { + database_id: String, + kinic: String, + #[arg(long)] + browser_origin: Option, + }, #[command(about = "Save a workspace database link so commands can omit --database-id")] Link { database_id: String }, #[command(about = "Show the currently linked workspace database")] @@ -308,7 +333,7 @@ pub enum DatabaseCommand { #[arg(long)] json: bool, }, - #[command(about = "Restore one archived or deleted database from a snapshot")] + #[command(about = "Restore one archived database from a snapshot")] ArchiveRestore { database_id: String, #[arg(long)] @@ -324,6 +349,15 @@ pub enum DatabaseCommand { RestoreCancel { database_id: String }, } +#[derive(Subcommand, Debug, Clone)] +pub enum CyclesCommand { + #[command(about = "Show canister cycles configuration")] + Config { + #[arg(long)] + json: bool, + }, +} + #[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] pub enum NodeKindArg { File, diff --git a/crates/vfs_cli_core/src/commands.rs b/crates/vfs_cli_core/src/commands.rs index 16cba83f..2271e6c9 100644 --- a/crates/vfs_cli_core/src/commands.rs +++ b/crates/vfs_cli_core/src/commands.rs @@ -5,39 +5,57 @@ use std::borrow::Cow; use std::fs::{self, File}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; -use anyhow::{Result, anyhow}; +use anyhow::{Context, Result, anyhow}; use serde::Deserialize; use sha2::{Digest, Sha256}; use vfs_client::VfsApi; use vfs_types::{ - AppendNodeRequest, DatabaseRestoreChunkRequest, DeleteNodeRequest, DeleteNodeResult, + AppendNodeRequest, CyclesBillingConfig, DatabaseCyclesPurchaseRequest, + DatabaseRestoreChunkRequest, DatabaseSummary, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, - IncomingLinksRequest, LinkEdge, ListChildrenRequest, ListNodesRequest, MkdirNodeRequest, - MoveNodeRequest, MultiEdit, MultiEditNodeRequest, NodeContextRequest, NodeEntryKind, NodeKind, - OutgoingLinksRequest, SearchNodePathsRequest, SearchNodesRequest, WriteNodeItem, - WriteNodeRequest, WriteNodesRequest, + IncomingLinksRequest, KINIC_DECIMALS, KINIC_LEDGER_FEE_E8S, LinkEdge, ListChildrenRequest, + ListNodesRequest, MkdirNodeRequest, MoveNodeRequest, MultiEdit, MultiEditNodeRequest, + NodeContextRequest, NodeEntryKind, NodeKind, OutgoingLinksRequest, SearchNodePathsRequest, + SearchNodesRequest, WriteNodeItem, WriteNodeRequest, WriteNodesRequest, + kinic_base_units_per_token, }; use wiki_domain::validate_source_path_for_kind; -use crate::cli::{DatabaseCommand, VfsCommand}; +use crate::cli::{CyclesCommand, DatabaseCommand, VfsCommand}; use crate::connection::{ ResolvedConnection, ResolvedConnectionPreview, link_workspace_database, unlink_workspace_database, workspace_config_path, }; +const DEFAULT_BROWSER_ORIGIN: &str = "https://wiki.kinic.xyz"; + pub async fn run_vfs_command( client: &impl VfsApi, connection: &ResolvedConnection, command: VfsCommand, ) -> Result<()> { - if let VfsCommand::Database { command } = command { - run_database_command(client, connection, command).await?; - return Ok(()); - } let database_id = connection.database_id.as_deref(); + let command = match command { + VfsCommand::Cycles { command } => { + run_cycles_command(client, command).await?; + return Ok(()); + } + VfsCommand::Database { command } => { + run_database_command(client, connection, command).await?; + return Ok(()); + } + command => command, + }; let database_id = require_database_id(database_id)?; + if command_requires_write_cycles_available(&command) { + require_write_cycles_available(client, database_id).await?; + } match command { + VfsCommand::Cycles { .. } => { + unreachable!("cycles command handled before db requirement") + } VfsCommand::Database { .. } => { unreachable!("database command handled before db requirement") } @@ -456,6 +474,59 @@ pub async fn run_vfs_command( Ok(()) } +fn command_requires_write_cycles_available(command: &VfsCommand) -> bool { + matches!( + command, + VfsCommand::WriteNode { .. } + | VfsCommand::AppendNode { .. } + | VfsCommand::EditNode { .. } + | VfsCommand::DeleteNode { .. } + | VfsCommand::DeleteTree { .. } + | VfsCommand::MkdirNode { .. } + | VfsCommand::MoveNode { .. } + | VfsCommand::MultiEditNode { .. } + ) +} + +async fn require_write_cycles_available(client: &impl VfsApi, database_id: &str) -> Result<()> { + let config = client + .get_cycles_billing_config() + .await + .context("cycles config unavailable")?; + let databases = client + .list_databases() + .await + .context("database list unavailable for cycles check")?; + let database = databases + .iter() + .find(|database| database.database_id == database_id) + .ok_or_else(|| anyhow!("database cycles state unavailable: {database_id}"))?; + if let Some(reason) = database_cycles_disabled_reason(database, &config) { + return Err(anyhow!("{reason}")); + } + Ok(()) +} + +fn database_cycles_disabled_reason( + database: &DatabaseSummary, + config: &CyclesBillingConfig, +) -> Option { + let balance = database.cycles_balance.unwrap_or(0); + if database.cycles_suspended_at_ms.is_some() { + return Some(format!( + "database cycles are suspended: {}", + database.database_id + )); + } + if balance < config.min_update_cycles { + return Some(format!( + "database cycles balance is below minimum: {} balance_cycles={} min_update_cycles={}", + database.database_id, balance, config.min_update_cycles + )); + } + None +} + fn print_links(links: Vec, json: bool) -> Result<()> { if json { println!("{}", serde_json::to_string_pretty(&links)?); @@ -587,16 +658,95 @@ async fn run_database_command( } else { for database in databases { println!( - "{}\t{}\t{:?}\t{:?}\t{}", + "{}\t{}\t{:?}\t{:?}\t{}\t{}\t{}", database.database_id, database.name, database.role, database.status, - database.logical_size_bytes + database.logical_size_bytes, + database.cycles_balance.unwrap_or(0), + database + .cycles_suspended_at_ms + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()) + ); + } + } + } + DatabaseCommand::PurchaseCycles { database_id, kinic } => { + let payment_amount_e8s = parse_kinic_amount_e8s(&kinic)?; + let config = client.get_cycles_billing_config().await?; + let min_expected_cycles = cycles_for_payment_amount_e8s(payment_amount_e8s, &config)?; + let result = client + .purchase_database_cycles(DatabaseCyclesPurchaseRequest { + database_id: database_id.clone(), + payment_amount_e8s, + min_expected_cycles, + }) + .await?; + println!( + "{database_id}\t{}\t{}\t{}", + result.block_index, result.amount_cycles, result.balance_cycles + ); + } + DatabaseCommand::CyclesHistory { database_id, json } => { + let page = client + .list_database_cycle_entries(&database_id, None, 100) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&page)?); + } else { + for entry in page.entries { + println!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + entry.entry_id, + entry.kind, + entry.amount_cycles, + entry.balance_after_cycles, + entry.caller, + entry.method.unwrap_or_else(|| "-".to_string()), + entry + .ledger_block_index + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()), + entry.created_at_ms ); } } } + DatabaseCommand::CyclesPending { database_id, json } => { + let pending = client + .list_database_cycles_pending_purchases(&database_id) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&pending)?); + } else { + for purchase in pending { + println!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}", + purchase.operation_id, + purchase.status, + purchase.amount_cycles, + purchase.payment_amount_e8s, + purchase + .ledger_block_index + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()), + purchase.required_action, + purchase.created_at_ms + ); + } + } + } + DatabaseCommand::Cycles { + database_id, + kinic, + browser_origin, + } => { + let url = database_cycles_url(browser_origin.as_deref(), &database_id, &kinic)?; + open_browser_url(&url)?; + println!("{url}"); + } DatabaseCommand::Link { database_id } => { let path = link_workspace_database(connection, &database_id)?; println!("{}", path.display()); @@ -678,6 +828,187 @@ async fn run_database_command( Ok(()) } +fn database_cycles_url( + browser_origin: Option<&str>, + database_id: &str, + kinic: &str, +) -> Result { + let kinic = kinic.trim(); + parse_kinic_amount_e8s(kinic)?; + let origin = browser_origin + .map(str::to_string) + .or_else(|| std::env::var("KINIC_WIKI_BROWSER_ORIGIN").ok()) + .unwrap_or_else(|| DEFAULT_BROWSER_ORIGIN.to_string()); + let origin = origin.trim_end_matches('/'); + if origin.is_empty() { + return Err(anyhow!("browser origin must not be empty")); + } + Ok(format!( + "{origin}/cycles?databaseId={}&kinic={}", + query_encode(database_id), + query_encode(kinic) + )) +} + +fn parse_kinic_amount_e8s(value: &str) -> Result { + let trimmed = value.trim(); + if trimmed.is_empty() { + return Err(anyhow!("KINIC amount must not be empty")); + } + let (whole, fractional) = match trimmed.split_once('.') { + Some((whole, fractional)) => (whole, Some(fractional)), + None => (trimmed, None), + }; + if whole.is_empty() || !whole.chars().all(|character| character.is_ascii_digit()) { + return Err(anyhow!( + "KINIC amount must be a positive decimal with up to {} fractional digits", + KINIC_DECIMALS + )); + } + let fractional = fractional.unwrap_or(""); + if fractional.is_empty() && trimmed.contains('.') { + return Err(anyhow!( + "KINIC amount must be a positive decimal with up to {} fractional digits", + KINIC_DECIMALS + )); + } + if fractional.len() > usize::from(KINIC_DECIMALS) + || !fractional + .chars() + .all(|character| character.is_ascii_digit()) + { + return Err(anyhow!( + "KINIC amount must be a positive decimal with up to {} fractional digits", + KINIC_DECIMALS + )); + } + let whole = whole + .parse::() + .map_err(|_| anyhow!("KINIC amount exceeds u64 e8s limit"))?; + let fractional_e8s = if fractional.is_empty() { + 0 + } else { + let padded = format!("{fractional:0() + .map_err(|_| anyhow!("KINIC amount exceeds u64 e8s limit"))? + }; + let amount = whole + .checked_mul(u128::from(kinic_base_units_per_token())) + .and_then(|amount| amount.checked_add(fractional_e8s)) + .ok_or_else(|| anyhow!("KINIC amount exceeds u64 e8s limit"))?; + if amount == 0 { + return Err(anyhow!("KINIC amount must be positive")); + } + u64::try_from(amount).map_err(|_| anyhow!("KINIC amount exceeds u64 e8s limit")) +} + +fn query_encode(value: &str) -> String { + let mut encoded = String::new(); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + encoded.push(char::from(byte)); + } + _ => encoded.push_str(&format!("%{byte:02X}")), + } + } + encoded +} + +fn open_browser_url(url: &str) -> Result<()> { + let status = if cfg!(target_os = "macos") { + ProcessCommand::new("open").arg(url).status() + } else if cfg!(target_os = "windows") { + ProcessCommand::new("rundll32") + .arg("url.dll,FileProtocolHandler") + .arg(url) + .status() + } else { + ProcessCommand::new("xdg-open").arg(url).status() + }; + let status = status.map_err(|error| anyhow!("failed to open browser: {error}"))?; + if !status.success() { + return Err(anyhow!("failed to open browser: exit status {status}")); + } + Ok(()) +} + +async fn run_cycles_command(client: &impl VfsApi, command: CyclesCommand) -> Result<()> { + match command { + CyclesCommand::Config { json } => { + let config = client.get_cycles_billing_config().await?; + if json { + println!( + "{}", + serde_json::to_string_pretty(&CyclesBillingConfigOutput::new( + config, + KINIC_LEDGER_FEE_E8S + ))? + ); + } else { + for line in cycles_config_lines(&config, KINIC_LEDGER_FEE_E8S) { + println!("{line}"); + } + } + } + } + Ok(()) +} + +#[derive(Debug, serde::Serialize)] +struct CyclesBillingConfigOutput { + kinic_ledger_canister_id: String, + billing_authority_id: String, + cycles_per_kinic: u64, + min_update_cycles: u64, + ledger_fee_e8s: u64, +} + +impl CyclesBillingConfigOutput { + fn new(config: CyclesBillingConfig, ledger_fee_e8s: u64) -> Self { + Self { + kinic_ledger_canister_id: config.kinic_ledger_canister_id, + billing_authority_id: config.billing_authority_id, + cycles_per_kinic: config.cycles_per_kinic, + min_update_cycles: config.min_update_cycles, + ledger_fee_e8s, + } + } +} + +fn cycles_config_lines(config: &CyclesBillingConfig, ledger_fee_e8s: u64) -> Vec { + vec![ + format!( + "kinic_ledger_canister_id\t{}", + config.kinic_ledger_canister_id + ), + format!("billing_authority_id\t{}", config.billing_authority_id), + format!("cycles_per_kinic\t{}", config.cycles_per_kinic), + format!("min_update_cycles\t{}", config.min_update_cycles), + format!("ledger_fee_e8s\t{ledger_fee_e8s}"), + ] +} + +fn cycles_for_payment_amount_e8s( + payment_amount_e8s: u64, + config: &CyclesBillingConfig, +) -> Result { + if payment_amount_e8s == 0 { + return Err(anyhow!("cycles purchase payment amount must be positive")); + } + let cycles = u128::from(payment_amount_e8s) + .checked_mul(u128::from(config.cycles_per_kinic)) + .ok_or_else(|| anyhow!("cycles purchase amount overflow"))? + / u128::from(kinic_base_units_per_token()); + let cycles = + u64::try_from(cycles).map_err(|_| anyhow!("cycles purchase amount exceeds u64"))?; + if cycles == 0 { + return Err(anyhow!("cycles purchase amount is too small")); + } + Ok(cycles) +} + #[derive(Debug, serde::Serialize)] struct ArchiveCommandResult { database_id: String, @@ -1071,10 +1402,10 @@ fn default_metadata_json() -> String { #[cfg(test)] mod tests { - use super::run_vfs_command; - use crate::cli::{NodeKindArg, VfsCommand}; + use super::{command_requires_write_cycles_available, run_vfs_command}; + use crate::cli::{CyclesCommand, NodeKindArg, VfsCommand}; use crate::connection::ResolvedConnection; - use anyhow::Result; + use anyhow::{Result, anyhow}; use async_trait::async_trait; use std::path::PathBuf; use std::sync::Mutex; @@ -1090,6 +1421,12 @@ mod tests { entries: Vec, created: Mutex, database_lists: Mutex, + database_cycle_purchases: Mutex>, + database_cycles_history: Mutex>, + database_cycles_pending: Mutex>, + database_summaries: Mutex>, + cycles_configs: Mutex, + fail_cycles_config: Mutex, writes: Mutex>, write_batches: Mutex>, deletes: Mutex>, @@ -1155,6 +1492,23 @@ mod tests { } } + fn database_summary( + database_id: &str, + balance_cycles: Option, + suspended_at_ms: Option, + ) -> DatabaseSummary { + DatabaseSummary { + database_id: database_id.to_string(), + name: database_id.to_string(), + status: DatabaseStatus::Active, + role: DatabaseRole::Owner, + logical_size_bytes: 42, + cycles_balance: balance_cycles, + cycles_suspended_at_ms: suspended_at_ms, + archived_at_ms: None, + } + } + #[async_trait] impl VfsApi for MockClient { async fn status(&self, _database_id: &str) -> Result { @@ -1168,20 +1522,95 @@ mod tests { name: name.to_string(), }) } + async fn purchase_database_cycles( + &self, + request: DatabaseCyclesPurchaseRequest, + ) -> Result { + self.database_cycle_purchases.lock().unwrap().push(request); + Ok(CyclesPurchaseResult { + block_index: 7, + amount_cycles: 1_250, + balance_cycles: 1_250, + }) + } + async fn list_database_cycle_entries( + &self, + database_id: &str, + _cursor: Option, + _limit: u32, + ) -> Result { + self.database_cycles_history + .lock() + .unwrap() + .push(database_id.to_string()); + Ok(DatabaseCycleEntryPage { + entries: vec![DatabaseCycleEntry { + entry_id: 1, + database_id: database_id.to_string(), + kind: "cycles_purchase".to_string(), + amount_cycles: 500_000, + balance_after_cycles: 500_000, + payment_amount_e8s: Some(50_000_000_000), + caller: "caller".to_string(), + method: Some("purchase_database_cycles".to_string()), + cycles_delta: None, + cycles_per_kinic: None, + ledger_block_index: Some(7), + created_at_ms: 1, + }], + next_cursor: None, + }) + } + async fn list_database_cycles_pending_purchases( + &self, + database_id: &str, + ) -> Result> { + self.database_cycles_pending + .lock() + .unwrap() + .push(database_id.to_string()); + Ok(vec![DatabaseCyclesPendingPurchase { + operation_id: 9, + database_id: database_id.to_string(), + status: "ambiguous".to_string(), + amount_cycles: 1_250, + payment_amount_e8s: 125_000_000, + ledger_block_index: None, + created_at_ms: 3, + required_action: "billing_authority_review".to_string(), + }]) + } + async fn get_cycles_billing_config(&self) -> Result { + let mut configs = self.cycles_configs.lock().unwrap(); + *configs += 1; + if *self.fail_cycles_config.lock().unwrap() { + return Err(anyhow!("cycles config unavailable")); + } + Ok(CyclesBillingConfig { + kinic_ledger_canister_id: "ryjl3-tyaaa-aaaaa-aaaba-cai".to_string(), + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), + cycles_per_kinic: 1_000, + min_update_cycles: 1, + }) + } async fn rename_database(&self, _database_id: &str, _name: &str) -> Result<()> { Ok(()) } async fn list_databases(&self) -> Result> { let mut lists = self.database_lists.lock().unwrap(); *lists += 1; + let summaries = self.database_summaries.lock().unwrap(); + if !summaries.is_empty() { + return Ok(summaries.clone()); + } Ok(vec![DatabaseSummary { database_id: "alpha".to_string(), name: "Alpha".to_string(), status: DatabaseStatus::Active, role: DatabaseRole::Owner, logical_size_bytes: 42, - credits_balance: Some(1_000_000), - credits_suspended_at_ms: None, + cycles_balance: Some(1_000_000), + cycles_suspended_at_ms: None, archived_at_ms: None, }]) } @@ -1414,6 +1843,202 @@ mod tests { assert_eq!(client.writes.lock().unwrap()[0].kind, NodeKind::Source); } + #[tokio::test] + async fn write_node_rejects_low_cycles_balance_before_write() { + let dir = tempdir().expect("temp dir should exist"); + let input = PathBuf::from(dir.path()).join("source.md"); + std::fs::write(&input, "# Source").expect("input should write"); + let client = MockClient { + database_summaries: Mutex::new(vec![database_summary("alpha", Some(0), None)]), + ..MockClient::default() + }; + + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::WriteNode { + path: "/Sources/raw/source/source.md".to_string(), + kind: NodeKindArg::Source, + input, + metadata_json: "{}".to_string(), + expected_etag: None, + json: true, + }, + ) + .await + .expect_err("low balance should reject"); + + assert!(error.to_string().contains("balance is below minimum")); + assert!(client.writes.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn mkdir_node_rejects_suspended_cycles_before_write() { + let client = MockClient { + database_summaries: Mutex::new(vec![database_summary("alpha", Some(20_000), Some(1))]), + ..MockClient::default() + }; + + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::MkdirNode { + path: "/Wiki/new".to_string(), + json: false, + }, + ) + .await + .expect_err("suspended cycles should reject"); + + assert!(error.to_string().contains("cycles are suspended")); + } + + #[tokio::test] + async fn mutating_commands_reject_missing_cycles_config_before_write() { + let dir = tempdir().expect("temp dir should exist"); + let input = PathBuf::from(dir.path()).join("source.md"); + std::fs::write(&input, "# Source").expect("input should write"); + let client = MockClient { + fail_cycles_config: Mutex::new(true), + ..MockClient::default() + }; + + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::WriteNode { + path: "/Sources/raw/source/source.md".to_string(), + kind: NodeKindArg::Source, + input, + metadata_json: "{}".to_string(), + expected_etag: None, + json: false, + }, + ) + .await + .expect_err("missing config should reject"); + + assert!(error.to_string().contains("cycles config unavailable")); + assert!(client.writes.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn mutating_commands_reject_missing_database_summary_before_write() { + let dir = tempdir().expect("temp dir should exist"); + let input = PathBuf::from(dir.path()).join("source.md"); + std::fs::write(&input, "# Source").expect("input should write"); + let client = MockClient { + database_summaries: Mutex::new(vec![database_summary("other", Some(20_000), None)]), + ..MockClient::default() + }; + + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::WriteNode { + path: "/Sources/raw/source/source.md".to_string(), + kind: NodeKindArg::Source, + input, + metadata_json: "{}".to_string(), + expected_etag: None, + json: false, + }, + ) + .await + .expect_err("missing summary should reject"); + + assert!(error.to_string().contains("cycles state unavailable")); + assert!(client.writes.lock().unwrap().is_empty()); + } + + #[test] + fn cycles_gate_covers_content_mutation_commands_only() { + assert!(command_requires_write_cycles_available( + &VfsCommand::WriteNode { + path: "/Wiki/a.md".to_string(), + kind: NodeKindArg::File, + input: PathBuf::from("a.md"), + metadata_json: "{}".to_string(), + expected_etag: None, + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::AppendNode { + path: "/Wiki/a.md".to_string(), + input: PathBuf::from("a.md"), + kind: None, + metadata_json: None, + expected_etag: None, + separator: None, + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::EditNode { + path: "/Wiki/a.md".to_string(), + old_text: "a".to_string(), + new_text: "b".to_string(), + expected_etag: None, + replace_all: false, + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::DeleteNode { + path: "/Wiki/a.md".to_string(), + expected_etag: None, + expected_folder_index_etag: None, + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::DeleteTree { + path: "/Wiki/a".to_string(), + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::MkdirNode { + path: "/Wiki/a".to_string(), + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::MoveNode { + from_path: "/Wiki/a.md".to_string(), + to_path: "/Wiki/b.md".to_string(), + expected_etag: None, + overwrite: false, + json: false, + } + )); + assert!(command_requires_write_cycles_available( + &VfsCommand::MultiEditNode { + path: "/Wiki/a.md".to_string(), + edits_file: PathBuf::from("edits.json"), + expected_etag: None, + json: false, + } + )); + assert!(!command_requires_write_cycles_available( + &VfsCommand::ReadNode { + path: "/Wiki/a.md".to_string(), + metadata_only: false, + fields: None, + json: false, + } + )); + assert!(!command_requires_write_cycles_available( + &VfsCommand::Database { + command: super::DatabaseCommand::PurchaseCycles { + database_id: "alpha".to_string(), + kinic: "1".to_string(), + }, + } + )); + } + #[tokio::test] async fn write_nodes_dispatches_one_batch() { let dir = tempdir().expect("temp dir should exist"); @@ -1679,6 +2304,168 @@ mod tests { assert_eq!(*client.created.lock().unwrap(), 1); } + #[tokio::test] + async fn database_cycles_purchase_calls_client() { + let client = MockClient::default(); + run_vfs_command( + &client, + &test_connection(), + VfsCommand::Database { + command: super::DatabaseCommand::PurchaseCycles { + database_id: "db_alpha".to_string(), + kinic: "1.25".to_string(), + }, + }, + ) + .await + .expect("database cycle purchase should succeed"); + assert_eq!( + *client.database_cycle_purchases.lock().unwrap(), + vec![DatabaseCyclesPurchaseRequest { + database_id: "db_alpha".to_string(), + payment_amount_e8s: 125_000_000, + min_expected_cycles: 1_250, + }] + ); + } + + #[tokio::test] + async fn database_cycles_purchase_requires_cycles_quote() { + let client = MockClient { + fail_cycles_config: Mutex::new(true), + ..MockClient::default() + }; + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::Database { + command: super::DatabaseCommand::PurchaseCycles { + database_id: "db_alpha".to_string(), + kinic: "1.25".to_string(), + }, + }, + ) + .await + .expect_err("database cycle purchase should require quote config"); + assert!(error.to_string().contains("cycles config unavailable")); + assert!(client.database_cycle_purchases.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn database_cycles_purchase_rejects_invalid_kinic_amounts() { + for kinic in ["0", "0.000000001", "abc", "184467440737.09551616"] { + let client = MockClient::default(); + let error = run_vfs_command( + &client, + &test_connection(), + VfsCommand::Database { + command: super::DatabaseCommand::PurchaseCycles { + database_id: "db_alpha".to_string(), + kinic: kinic.to_string(), + }, + }, + ) + .await + .expect_err("invalid KINIC amount should reject"); + assert!(error.to_string().contains("KINIC amount")); + assert!(client.database_cycle_purchases.lock().unwrap().is_empty()); + } + } + + #[tokio::test] + async fn database_cycles_history_calls_client() { + let client = MockClient::default(); + run_vfs_command( + &client, + &test_connection(), + VfsCommand::Database { + command: super::DatabaseCommand::CyclesHistory { + database_id: "db_alpha".to_string(), + json: false, + }, + }, + ) + .await + .expect("database cycles-history should succeed"); + assert_eq!( + *client.database_cycles_history.lock().unwrap(), + vec!["db_alpha".to_string()] + ); + } + + #[tokio::test] + async fn database_cycles_pending_calls_client() { + let client = MockClient::default(); + run_vfs_command( + &client, + &test_connection(), + VfsCommand::Database { + command: super::DatabaseCommand::CyclesPending { + database_id: "db_alpha".to_string(), + json: false, + }, + }, + ) + .await + .expect("database cycles-pending should succeed"); + assert_eq!( + *client.database_cycles_pending.lock().unwrap(), + vec!["db_alpha".to_string()] + ); + } + + #[test] + fn database_cycles_url_uses_browser_origin() { + let url = super::database_cycles_url(Some("http://127.0.0.1:3000/"), "db alpha", "1.25") + .expect("url should build"); + + assert_eq!( + url, + "http://127.0.0.1:3000/cycles?databaseId=db%20alpha&kinic=1.25" + ); + } + + #[test] + fn database_cycles_url_rejects_invalid_kinic_amount() { + for kinic in ["0", "0.000000001", "abc", "184467440737.09551616"] { + let error = + super::database_cycles_url(Some("http://127.0.0.1:3000/"), "db_alpha", kinic) + .expect_err("invalid KINIC amount should reject"); + assert!(error.to_string().contains("KINIC amount")); + } + } + + #[tokio::test] + async fn cycles_config_json_calls_client() { + let client = MockClient::default(); + run_vfs_command( + &client, + &test_connection(), + VfsCommand::Cycles { + command: CyclesCommand::Config { json: true }, + }, + ) + .await + .expect("cycles config should succeed"); + assert_eq!(*client.cycles_configs.lock().unwrap(), 1); + } + + #[test] + fn cycles_config_text_includes_billing_authority_principal() { + let lines = super::cycles_config_lines( + &CyclesBillingConfig { + kinic_ledger_canister_id: "ryjl3-tyaaa-aaaaa-aaaba-cai".to_string(), + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), + cycles_per_kinic: 1_000, + min_update_cycles: 1, + }, + KINIC_LEDGER_FEE_E8S, + ); + + assert!(lines.contains(&"billing_authority_id\trrkah-fqaaa-aaaaa-aaaaq-cai".to_string())); + assert!(lines.contains(&"ledger_fee_e8s\t100000".to_string())); + } + #[tokio::test] async fn database_rename_parses_and_calls_client() { let client = MockClient::default(); diff --git a/crates/vfs_cli_core/src/skill_kb.rs b/crates/vfs_cli_core/src/skill_kb.rs index 9c9c0085..57feed59 100644 --- a/crates/vfs_cli_core/src/skill_kb.rs +++ b/crates/vfs_cli_core/src/skill_kb.rs @@ -439,16 +439,26 @@ fn now_millis() -> i64 { fn extract_frontmatter(content: &str) -> Option<&str> { let rest = content.strip_prefix("---\n")?; - let end = rest.find("\n---")?; + let end = frontmatter_end(rest)?; Some(&rest[..end]) } +fn frontmatter_end(rest: &str) -> Option { + rest.find("\n---\n").or_else(|| { + rest.ends_with("\n---") + .then_some(rest.len() - "\n---".len()) + }) +} + fn clean_yaml_value(value: &str) -> String { - value - .trim() - .trim_matches('"') - .trim_matches('\'') - .to_string() + let trimmed = value.trim(); + if trimmed.starts_with('"') && trimmed.ends_with('"') { + return serde_json::from_str::(trimmed).unwrap_or_else(|_| trimmed.to_string()); + } + if trimmed.starts_with('\'') && trimmed.ends_with('\'') { + return trimmed[1..trimmed.len() - 1].replace("''", "'"); + } + trimmed.to_string() } fn non_empty(value: String) -> Option { @@ -474,10 +484,11 @@ fn skill_base_path(id: &str) -> String { } fn validate_skill_id(id: &str) -> Result<()> { - if id.is_empty() - || !id - .chars() - .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + let mut chars = id.chars(); + if !matches!(chars.next(), Some(ch) if ch.is_ascii_alphanumeric()) + || id.len() > 128 + || id.contains("..") + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) { anyhow::bail!("skill id must use a single path-safe name"); } @@ -497,3 +508,28 @@ fn validate_package_file(file: &str) -> Result { } Ok(file.to_string()) } + +#[cfg(test)] +mod tests { + use super::{extract_frontmatter, parse_run_frontmatter}; + + #[test] + fn extract_frontmatter_requires_whole_line_terminator() { + let content = "---\nskill_id: legal\n---not-a-terminator\noutcome: success\n---\n# Run\n"; + + assert_eq!( + extract_frontmatter(content), + Some("skill_id: legal\n---not-a-terminator\noutcome: success") + ); + } + + #[test] + fn run_frontmatter_unescapes_json_quoted_scalars() { + let run = parse_run_frontmatter( + "---\nskill_id: \"legal-\\\"review\\\"\"\noutcome: success\n---\n# Run\n", + ) + .expect("run frontmatter should parse"); + + assert_eq!(run.skill_id.as_deref(), Some("legal-\"review\"")); + } +} diff --git a/crates/vfs_client/src/lib.rs b/crates/vfs_client/src/lib.rs index 6c3243d7..db26029a 100644 --- a/crates/vfs_client/src/lib.rs +++ b/crates/vfs_client/src/lib.rs @@ -12,19 +12,18 @@ use ic_agent::{ use k256::{SecretKey, pkcs8::DecodePrivateKey}; use vfs_types::{ AppendNodeRequest, CanisterHealth, ChildNode, CreateDatabaseRequest, CreateDatabaseResult, - CreditsConfig, CreditsPurchaseResult, DatabaseArchiveChunk, DatabaseArchiveInfo, - DatabaseCreditEntryPage, DatabaseCreditPendingOperationPage, DatabaseCreditPurchasePreview, - DatabaseCreditPurchaseRequest, DatabaseMember, DatabaseRestoreChunkRequest, DatabaseRole, - DatabaseSummary, DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, - EditNodeResult, ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, - FetchUpdatesResponse, GlobNodeHit, GlobNodesRequest, GraphLinksRequest, - GraphNeighborhoodRequest, IncomingLinksRequest, LinkEdge, ListChildrenRequest, - ListNodesRequest, MemoryManifest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, - MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, - NodeContextRequest, NodeEntry, OutgoingLinksRequest, QueryContext, QueryContextRequest, - RenameDatabaseRequest, SearchNodeHit, SearchNodePathsRequest, SearchNodesRequest, - SourceEvidence, SourceEvidenceRequest, Status, WriteNodeRequest, WriteNodeResult, - WriteNodesRequest, + CyclesBillingConfig, CyclesPurchaseResult, DatabaseArchiveChunk, DatabaseArchiveInfo, + DatabaseCycleEntryPage, DatabaseCyclesPendingPurchase, DatabaseCyclesPurchaseRequest, + DatabaseMember, DatabaseRestoreChunkRequest, DatabaseRole, DatabaseSummary, + DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, + ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, + GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, + IncomingLinksRequest, LinkEdge, ListChildrenRequest, ListNodesRequest, MemoryManifest, + MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, + MultiEditNodeResult, Node, NodeContext, NodeContextRequest, NodeEntry, OutgoingLinksRequest, + QueryContext, QueryContextRequest, RenameDatabaseRequest, SearchNodeHit, + SearchNodePathsRequest, SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, Status, + WriteNodeRequest, WriteNodeResult, WriteNodesRequest, }; #[async_trait] @@ -40,76 +39,46 @@ pub trait VfsApi: Sync { async fn memory_manifest(&self) -> Result { Err(anyhow!("memory_manifest is not implemented by this client")) } - async fn get_credits_config(&self) -> Result { - Err(anyhow!( - "get_credits_config is not implemented by this client" - )) - } async fn create_database(&self, _name: &str) -> Result { Err(anyhow!("create_database is not implemented by this client")) } async fn rename_database(&self, _database_id: &str, _name: &str) -> Result<()> { Err(anyhow!("rename_database is not implemented by this client")) } - async fn purchase_database_credits( + async fn purchase_database_cycles( &self, - _request: DatabaseCreditPurchaseRequest, - ) -> Result { + _request: DatabaseCyclesPurchaseRequest, + ) -> Result { Err(anyhow!( - "purchase_database_credits is not implemented by this client" + "purchase_database_cycles is not implemented by this client" )) } - async fn preview_database_credit_purchase( - &self, - _database_id: &str, - _amount_credits: u64, - ) -> Result { - Err(anyhow!( - "preview_database_credit_purchase is not implemented by this client" - )) - } - async fn check_database_write_credits(&self, _database_id: &str) -> Result<()> { - Err(anyhow!( - "check_database_write_credits is not implemented by this client" - )) - } - async fn list_database_credit_entries( - &self, - _database_id: &str, - _cursor: Option, - _limit: u32, - ) -> Result { + async fn check_database_write_cycles(&self, _database_id: &str) -> Result<()> { Err(anyhow!( - "list_database_credit_entries is not implemented by this client" + "check_database_write_cycles is not implemented by this client" )) } - async fn list_database_credit_pending_operations( + async fn list_database_cycle_entries( &self, _database_id: &str, _cursor: Option, _limit: u32, - ) -> Result { + ) -> Result { Err(anyhow!( - "list_database_credit_pending_operations is not implemented by this client" + "list_database_cycle_entries is not implemented by this client" )) } - async fn repair_database_credit_purchase_complete( + async fn list_database_cycles_pending_purchases( &self, _database_id: &str, - _operation_id: u64, - _ledger_block_index: u64, - ) -> Result { + ) -> Result> { Err(anyhow!( - "repair_database_credit_purchase_complete is not implemented by this client" + "list_database_cycles_pending_purchases is not implemented by this client" )) } - async fn repair_database_credit_purchase_cancel( - &self, - _database_id: &str, - _operation_id: u64, - ) -> Result<()> { + async fn get_cycles_billing_config(&self) -> Result { Err(anyhow!( - "repair_database_credit_purchase_cancel is not implemented by this client" + "get_cycles_billing_config is not implemented by this client" )) } async fn grant_database_access( @@ -447,11 +416,6 @@ impl VfsApi for CanisterVfsClient { self.query("memory_manifest", &()).await } - async fn get_credits_config(&self) -> Result { - let result: Result = self.query("get_credits_config", &()).await?; - result.map_err(|error| anyhow!(error)) - } - async fn create_database(&self, name: &str) -> Result { let result: Result = self .update( @@ -477,46 +441,31 @@ impl VfsApi for CanisterVfsClient { result.map_err(|error| anyhow!(error)) } - async fn purchase_database_credits( + async fn purchase_database_cycles( &self, - request: DatabaseCreditPurchaseRequest, - ) -> Result { - let result: Result = - self.update("purchase_database_credits", &request).await?; + request: DatabaseCyclesPurchaseRequest, + ) -> Result { + let result: Result = + self.update("purchase_database_cycles", &request).await?; result.map_err(|error| anyhow!(error)) } - async fn preview_database_credit_purchase( - &self, - database_id: &str, - amount_credits: u64, - ) -> Result { - let result: Result = self - .query2( - "preview_database_credit_purchase", - &database_id.to_string(), - &amount_credits, - ) - .await?; - result.map_err(|error| anyhow!(error)) - } - - async fn check_database_write_credits(&self, database_id: &str) -> Result<()> { + async fn check_database_write_cycles(&self, database_id: &str) -> Result<()> { let result: Result<(), String> = self - .query("check_database_write_credits", &database_id.to_string()) + .query("check_database_write_cycles", &database_id.to_string()) .await?; result.map_err(|error| anyhow!(error)) } - async fn list_database_credit_entries( + async fn list_database_cycle_entries( &self, database_id: &str, cursor: Option, limit: u32, - ) -> Result { - let result: Result = self + ) -> Result { + let result: Result = self .query3( - "list_database_credit_entries", + "list_database_cycle_entries", &database_id.to_string(), &cursor, &limit, @@ -525,50 +474,20 @@ impl VfsApi for CanisterVfsClient { result.map_err(|error| anyhow!(error)) } - async fn list_database_credit_pending_operations( - &self, - database_id: &str, - cursor: Option, - limit: u32, - ) -> Result { - let result: Result = self - .query3( - "list_database_credit_pending_operations", - &database_id.to_string(), - &cursor, - &limit, - ) - .await?; + async fn get_cycles_billing_config(&self) -> Result { + let result: Result = + self.query("get_cycles_billing_config", &()).await?; result.map_err(|error| anyhow!(error)) } - async fn repair_database_credit_purchase_complete( + async fn list_database_cycles_pending_purchases( &self, database_id: &str, - operation_id: u64, - ledger_block_index: u64, - ) -> Result { - let result: Result = self - .update3( - "repair_database_credit_purchase_complete", - &database_id.to_string(), - &operation_id, - &ledger_block_index, - ) - .await?; - result.map_err(|error| anyhow!(error)) - } - - async fn repair_database_credit_purchase_cancel( - &self, - database_id: &str, - operation_id: u64, - ) -> Result<()> { - let result: Result<(), String> = self - .update2( - "repair_database_credit_purchase_cancel", + ) -> Result> { + let result: Result, String> = self + .query( + "list_database_cycles_pending_purchases", &database_id.to_string(), - &operation_id, ) .await?; result.map_err(|error| anyhow!(error)) diff --git a/crates/vfs_runtime/migrations/index_db/011_to_latest.sql b/crates/vfs_runtime/migrations/index_db/011_to_latest.sql index b8991616..fc21b218 100644 --- a/crates/vfs_runtime/migrations/index_db/011_to_latest.sql +++ b/crates/vfs_runtime/migrations/index_db/011_to_latest.sql @@ -1,36 +1,37 @@ -CREATE TABLE database_credit_accounts ( +CREATE TABLE database_cycle_accounts ( database_id TEXT PRIMARY KEY, - balance_credits INTEGER NOT NULL, + balance_cycles INTEGER NOT NULL, suspended_at_ms INTEGER, + storage_charged_at_ms INTEGER, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, FOREIGN KEY (database_id) REFERENCES databases(database_id) ); -CREATE TABLE database_credit_ledger ( +CREATE TABLE database_cycle_ledger ( entry_id INTEGER PRIMARY KEY AUTOINCREMENT, database_id TEXT NOT NULL, kind TEXT NOT NULL, - amount_credits INTEGER NOT NULL, - balance_after_credits INTEGER NOT NULL, + amount_cycles INTEGER NOT NULL, + balance_after_cycles INTEGER NOT NULL, payment_amount_e8s INTEGER, caller TEXT NOT NULL, method TEXT, cycles_delta INTEGER, - credits_per_kinic INTEGER, + cycles_per_kinic INTEGER, ledger_block_index INTEGER, created_at_ms INTEGER NOT NULL ); -CREATE INDEX database_credit_ledger_database_idx - ON database_credit_ledger(database_id, entry_id); +CREATE INDEX database_cycle_ledger_database_idx + ON database_cycle_ledger(database_id, entry_id); -CREATE TABLE database_credit_pending_operations ( +CREATE TABLE database_cycle_pending_operations ( operation_id INTEGER PRIMARY KEY AUTOINCREMENT, database_id TEXT NOT NULL, kind TEXT NOT NULL, caller TEXT NOT NULL, - credits INTEGER NOT NULL, + cycles INTEGER NOT NULL, payment_amount_e8s INTEGER NOT NULL, from_owner TEXT, from_subaccount BLOB, @@ -39,14 +40,54 @@ CREATE TABLE database_credit_pending_operations ( ledger_fee_e8s INTEGER, ledger_created_at_time_ns INTEGER, operation_status TEXT NOT NULL, + ledger_block_index INTEGER, created_at_ms INTEGER NOT NULL, FOREIGN KEY (database_id) REFERENCES databases(database_id) ); -CREATE INDEX database_credit_pending_operations_database_idx - ON database_credit_pending_operations(database_id); +CREATE INDEX database_cycle_pending_operations_database_idx + ON database_cycle_pending_operations(database_id); -CREATE TABLE credits_config ( +CREATE TABLE cycles_billing_config ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); + +INSERT INTO database_cycle_accounts + (database_id, balance_cycles, suspended_at_ms, storage_charged_at_ms, + created_at_ms, updated_at_ms) + SELECT database_id, 0, 0, NULL, 0, 0 FROM databases; + +UPDATE databases + SET status = 'active' + WHERE status = 'hot'; + +DELETE FROM database_cycle_pending_operations + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM database_cycle_ledger + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM database_cycle_accounts + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM database_members + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM database_restore_chunks + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM database_restore_sessions + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM url_ingest_trigger_sessions + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM ops_answer_sessions + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM source_run_sessions + WHERE database_id IN (SELECT database_id FROM databases WHERE status = 'deleted'); + +DELETE FROM databases + WHERE status = 'deleted'; diff --git a/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql b/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql index ecbfddf0..c8912093 100644 --- a/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql +++ b/crates/vfs_runtime/migrations/index_db/fresh_index_schema.sql @@ -104,39 +104,40 @@ CREATE TABLE database_restore_sessions ( FOREIGN KEY (database_id) REFERENCES databases(database_id) ); -CREATE TABLE database_credit_accounts ( +CREATE TABLE database_cycle_accounts ( database_id TEXT PRIMARY KEY, - balance_credits INTEGER NOT NULL, + balance_cycles INTEGER NOT NULL, suspended_at_ms INTEGER, + storage_charged_at_ms INTEGER, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL, FOREIGN KEY (database_id) REFERENCES databases(database_id) ); -CREATE TABLE database_credit_ledger ( +CREATE TABLE database_cycle_ledger ( entry_id INTEGER PRIMARY KEY AUTOINCREMENT, database_id TEXT NOT NULL, kind TEXT NOT NULL, - amount_credits INTEGER NOT NULL, - balance_after_credits INTEGER NOT NULL, + amount_cycles INTEGER NOT NULL, + balance_after_cycles INTEGER NOT NULL, payment_amount_e8s INTEGER, caller TEXT NOT NULL, method TEXT, cycles_delta INTEGER, - credits_per_kinic INTEGER, + cycles_per_kinic INTEGER, ledger_block_index INTEGER, created_at_ms INTEGER NOT NULL ); -CREATE INDEX database_credit_ledger_database_idx - ON database_credit_ledger(database_id, entry_id); +CREATE INDEX database_cycle_ledger_database_idx + ON database_cycle_ledger(database_id, entry_id); -CREATE TABLE database_credit_pending_operations ( +CREATE TABLE database_cycle_pending_operations ( operation_id INTEGER PRIMARY KEY AUTOINCREMENT, database_id TEXT NOT NULL, kind TEXT NOT NULL, caller TEXT NOT NULL, - credits INTEGER NOT NULL, + cycles INTEGER NOT NULL, payment_amount_e8s INTEGER NOT NULL, from_owner TEXT, from_subaccount BLOB, @@ -145,14 +146,15 @@ CREATE TABLE database_credit_pending_operations ( ledger_fee_e8s INTEGER, ledger_created_at_time_ns INTEGER, operation_status TEXT NOT NULL, + ledger_block_index INTEGER, created_at_ms INTEGER NOT NULL, FOREIGN KEY (database_id) REFERENCES databases(database_id) ); -CREATE INDEX database_credit_pending_operations_database_idx - ON database_credit_pending_operations(database_id); +CREATE INDEX database_cycle_pending_operations_database_idx + ON database_cycle_pending_operations(database_id); -CREATE TABLE credits_config ( +CREATE TABLE cycles_billing_config ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); diff --git a/crates/vfs_runtime/src/lib.rs b/crates/vfs_runtime/src/lib.rs index f96e790a..6ef75447 100644 --- a/crates/vfs_runtime/src/lib.rs +++ b/crates/vfs_runtime/src/lib.rs @@ -18,14 +18,13 @@ use ic_sqlite_vfs::{Db, DbError, DbHandle}; use sha2::{Digest, Sha256}; use vfs_store::FsStore; use vfs_types::{ - AppendNodeRequest, ChildNode, CreditsConfig, CreditsConfigUpdate, DatabaseArchiveInfo, - DatabaseCreditEntry, DatabaseCreditEntryPage, DatabaseCreditPendingOperation, - DatabaseCreditPendingOperationPage, DatabaseCreditPurchasePreview, DatabaseInfo, - DatabaseMember, DatabaseRole, DatabaseStatus, DatabaseSummary, DeleteDatabaseRequest, - DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, ExportSnapshotRequest, - ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, GlobNodeHit, - GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, IncomingLinksRequest, - IndexSqlJsonQueryResult, KINIC_LEDGER_FEE_E8S, LinkEdge, ListChildrenRequest, ListNodesRequest, + AppendNodeRequest, ChildNode, CyclesBillingConfig, CyclesBillingConfigUpdate, + DatabaseArchiveInfo, DatabaseCycleEntry, DatabaseCycleEntryPage, DatabaseCyclesPendingPurchase, + DatabaseInfo, DatabaseMember, DatabaseRole, DatabaseStatus, DatabaseSummary, + DeleteDatabaseRequest, DeleteNodeRequest, DeleteNodeResult, EditNodeRequest, EditNodeResult, + ExportSnapshotRequest, ExportSnapshotResponse, FetchUpdatesRequest, FetchUpdatesResponse, + GlobNodeHit, GlobNodesRequest, GraphLinksRequest, GraphNeighborhoodRequest, + IncomingLinksRequest, IndexSqlJsonQueryResult, LinkEdge, ListChildrenRequest, ListNodesRequest, MkdirNodeRequest, MkdirNodeResult, MoveNodeRequest, MoveNodeResult, MultiEditNodeRequest, MultiEditNodeResult, Node, NodeContext, NodeContextRequest, NodeEntry, NodeKind, OpsAnswerSessionCheckRequest, OpsAnswerSessionCheckResult, OpsAnswerSessionRequest, @@ -33,7 +32,7 @@ use vfs_types::{ SearchNodesRequest, SourceEvidence, SourceEvidenceRequest, SourceRunSessionCheckRequest, Status, UrlIngestTriggerSessionCheckRequest, UrlIngestTriggerSessionRequest, WriteNodeRequest, WriteNodeResult, WriteNodesRequest, WriteSourceForGenerationRequest, - WriteSourceForGenerationResult, + WriteSourceForGenerationResult, kinic_base_units_per_token, }; use wiki_domain::{RAW_SOURCES_PREFIX, validate_source_path_for_kind}; @@ -50,21 +49,26 @@ const INDEX_SCHEMA_VERSION_RESTORE_CHUNK_BYTES: &str = "database_index:009_resto const INDEX_SCHEMA_VERSION_DATABASE_NAME_BREAKING: &str = "database_index:010_database_name_breaking"; const INDEX_SCHEMA_VERSION_SOURCE_RUN_SESSIONS: &str = "database_index:011_source_run_sessions"; -const INDEX_SCHEMA_VERSION_BILLING_INITIAL: &str = "database_index:012_credits_initial"; -const INDEX_SCHEMA_VERSION_BILLING_PENDING: &str = "database_index:013_credits_pending"; +const INDEX_SCHEMA_VERSION_BILLING_INITIAL: &str = "database_index:012_cycles_initial"; +const INDEX_SCHEMA_VERSION_BILLING_PENDING: &str = "database_index:013_cycles_pending"; const INDEX_SCHEMA_VERSION_BILLING_LEDGER_BLOCK_INDEX: &str = - "database_index:014_credits_ledger_block_index"; + "database_index:014_cycles_ledger_block_index"; const INDEX_SCHEMA_VERSION_BILLING_PENDING_LEDGER_DETAILS: &str = - "database_index:015_credits_pending_ledger_details"; + "database_index:015_cycles_pending_ledger_details"; const INDEX_SCHEMA_VERSION_ACTIVE_STATUS: &str = "database_index:016_active_status"; const INDEX_SCHEMA_VERSION_HARD_DELETE_DATABASES: &str = "database_index:017_hard_delete_databases"; -const INDEX_SCHEMA_VERSION_CREDIT_LEDGER_ONLY: &str = "database_index:018_credit_ledger_only"; -const INDEX_SCHEMA_VERSION_FIXED_CYCLES_PER_CREDIT: &str = - "database_index:019_fixed_cycles_per_credit"; -const INDEX_SCHEMA_VERSION_CREDITS_CONFIG_VERSION: &str = - "database_index:020_credits_config_version"; -const INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS: &str = - "database_index:021_credit_pending_operation_status"; +const INDEX_SCHEMA_VERSION_CYCLES_LEDGER_ONLY: &str = "database_index:018_cycles_ledger_only"; +const INDEX_SCHEMA_VERSION_FIXED_CYCLES_ACCOUNTING: &str = + "database_index:019_fixed_cycles_accounting"; +const INDEX_SCHEMA_VERSION_CYCLES_BILLING_CONFIG_VERSION: &str = + "database_index:020_cycles_billing_config_version"; +const INDEX_SCHEMA_VERSION_CYCLES_PENDING_OPERATION_STATUS: &str = + "database_index:021_cycles_pending_operation_status"; +const INDEX_SCHEMA_VERSION_CYCLES: &str = "database_index:022_cycles"; +const INDEX_SCHEMA_VERSION_STORAGE_BILLING: &str = "database_index:023_storage_billing"; +const INDEX_SCHEMA_VERSION_DIRECT_CYCLES: &str = concat!("database_index:024_", "direct_cycles"); +const INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX: &str = + "database_index:025_cycles_pending_ledger_block_index"; const PENDING_DATABASE_MOUNT_ID: u16 = 0; const DATABASE_SCHEMA_VERSION: &str = "vfs_store:current"; const MIN_DATABASE_MOUNT_ID: u16 = 11; @@ -75,25 +79,28 @@ pub const MAX_DATABASE_SIZE_BYTES: u64 = i64::MAX as u64; const URL_INGEST_TRIGGER_SESSION_TTL_MS: i64 = 30 * 60 * 1000; const OPS_ANSWER_SESSION_TTL_MS: i64 = 30 * 60 * 1000; const SOURCE_RUN_SESSION_TTL_MS: i64 = URL_INGEST_TRIGGER_SESSION_TTL_MS; +const MAX_PENDING_DATABASES_PER_CALLER: i64 = 3; +const PENDING_DATABASE_TTL_MS: i64 = 24 * 60 * 60 * 1000; +const MAX_DATABASE_MEMBERS_PER_DATABASE: i64 = 32; const SHA256_DIGEST_BYTES: usize = 32; const GENERATED_DATABASE_ID_PREFIX: &str = "db_"; const GENERATED_DATABASE_ID_HASH_CHARS: usize = 12; const FRESH_INDEX_SCHEMA_SQL: &str = include_str!("../migrations/index_db/fresh_index_schema.sql"); const INDEX_011_TO_LATEST_SQL: &str = include_str!("../migrations/index_db/011_to_latest.sql"); -pub const KINIC_E8S_PER_TOKEN: u64 = 100_000_000; -pub const DEFAULT_CREDITS_PER_KINIC: u64 = 1_000; -pub const CYCLES_PER_CREDIT: u128 = 1_000_000_000; -pub const DEFAULT_MIN_UPDATE_CREDITS: u64 = 1; -const DEFAULT_CREDITS_CONFIG_VERSION: u64 = 1; +pub const DEFAULT_CYCLES_PER_KINIC: u64 = 234_500_000_000; +pub const DEFAULT_MIN_UPDATE_CYCLES: u64 = 1_000_000; +pub const STORAGE_BILLING_INTERVAL_MS: i64 = 24 * 60 * 60 * 1000; +pub const STORAGE_CYCLES_PER_GIB_SECOND: u128 = 127_000; +const GIB_BYTES: u128 = 1024 * 1024 * 1024; const MAX_DATABASE_NAME_CHARS: usize = 80; const FNV1A64_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; const FNV1A64_PRIME: u64 = 0x0000_0100_0000_01b3; pub const DEFAULT_LLM_WRITER_PRINCIPAL: &str = "ckurn-x74ln-nemlm-42vfv-gej7r-4cc3e-v22e5-otcod-jndlh-pbst4-3qe"; const ANONYMOUS_PRINCIPAL: &str = "2vxsx-fae"; -const CREDIT_OPERATION_STATUS_IN_FLIGHT: &str = "in_flight"; -const CREDIT_OPERATION_STATUS_AMBIGUOUS: &str = "ambiguous"; -const CREDIT_OPERATION_STATUS_COMPLETED: &str = "completed"; +const CYCLES_OPERATION_STATUS_IN_FLIGHT: &str = "in_flight"; +const CYCLES_OPERATION_STATUS_COMPLETED: &str = "completed"; +const CYCLES_OPERATION_STATUS_AMBIGUOUS: &str = "ambiguous"; #[derive(Clone, Debug, PartialEq, Eq)] pub struct DatabaseMeta { @@ -128,7 +135,7 @@ pub enum RequiredRole { Owner, } -pub struct CreditsPendingLedgerDetailsInput<'a> { +pub struct CyclesPendingLedgerDetailsInput<'a> { pub from_owner: &'a str, pub from_subaccount: Option<&'a [u8]>, pub to_owner: &'a str, @@ -137,16 +144,21 @@ pub struct CreditsPendingLedgerDetailsInput<'a> { pub ledger_created_at_time_ns: u64, } -pub struct DatabaseCreditPurchaseWithLedgerDetails<'a> { +pub struct DatabaseCyclesPurchaseWithLedgerDetails<'a> { pub database_id: &'a str, pub caller: &'a str, - pub credits: u64, - pub expected_payment_amount_e8s: u64, - pub expected_config_version: u64, - pub ledger: CreditsPendingLedgerDetailsInput<'a>, + pub payment_amount_e8s: u64, + pub min_expected_cycles: u64, + pub ledger: CyclesPendingLedgerDetailsInput<'a>, pub now: i64, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DatabaseCyclesPurchaseStart { + pub operation_id: u64, + pub amount_cycles: u64, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct RestoreChunk { offset: u64, @@ -178,10 +190,13 @@ impl VfsService { } pub fn run_index_migrations(&self) -> Result<(), String> { - self.run_index_migrations_with_config(default_credits_config()) + self.run_index_migrations_with_config(default_cycles_billing_config()) } - pub fn run_index_migrations_with_config(&self, config: CreditsConfig) -> Result<(), String> { + pub fn run_index_migrations_with_config( + &self, + config: CyclesBillingConfig, + ) -> Result<(), String> { #[cfg(not(target_arch = "wasm32"))] { let mut conn = self.open_index()?; @@ -195,7 +210,7 @@ impl VfsService { pub fn run_index_migrations_for_upgrade( &self, - config: Option, + config: Option, ) -> Result<(), String> { #[cfg(not(target_arch = "wasm32"))] { @@ -242,6 +257,33 @@ impl VfsService { }) } + pub fn settle_database_storage_charges(&self, caller: &str, now: i64) -> Result<(), String> { + let measured = self + .read_index(load_active_databases_for_storage_billing)? + .into_iter() + .map(|meta| { + let size = self.database_size(&meta)?; + Ok((meta.database_id, size)) + }) + .collect::, String>>()?; + self.write_index(|tx| { + let config = load_cycles_billing_config(tx)?; + for (database_id, size_bytes) in measured { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: &database_id, + caller, + size_bytes, + now, + config: &config, + }, + )?; + } + Ok(()) + }) + } + pub fn list_database_summaries_for_caller( &self, caller: &str, @@ -249,63 +291,29 @@ impl VfsService { self.read_index(|conn| load_database_summaries_for_caller(conn, caller)) } - pub fn credits_config(&self) -> Result { - self.read_index(load_credits_config) - } - - pub fn preview_database_credit_purchase( - &self, - database_id: &str, - credits: u64, - ) -> Result { - let credits = credits_to_i64(credits)?; - self.read_index(|conn| { - let config = load_credits_config(conn)?; - let config_version = load_credits_config_version(conn)?; - let payment_amount_e8s = payment_amount_e8s_for_credits(credits as u64, &config)?; - amount_to_i64(payment_amount_e8s)?; - validate_database_credit_purchase_for_conn(conn, database_id, credits)?; - Ok(DatabaseCreditPurchasePreview { - payment_amount_e8s, - ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, - credits_per_kinic: config.credits_per_kinic, - config_version, - }) - }) + pub fn cycles_billing_config(&self) -> Result { + self.read_index(load_cycles_billing_config) } - pub fn update_credits_config( + pub fn update_cycles_billing_config( &self, - update: CreditsConfigUpdate, + update: CyclesBillingConfigUpdate, caller: &str, - ) -> Result { - let current = self.credits_config()?; - let current_version = self.read_index(load_credits_config_version)?; - if caller != current.sns_governance_id { - return Err("caller is not SNS governance".to_string()); + ) -> Result { + let current = self.cycles_billing_config()?; + if caller != current.billing_authority_id { + return Err("caller is not billing authority".to_string()); } - let config_changed = current.credits_per_kinic != update.credits_per_kinic - || current.min_update_credits != update.min_update_credits; - let next = CreditsConfig { + let next = CyclesBillingConfig { kinic_ledger_canister_id: current.kinic_ledger_canister_id, - sns_governance_id: current.sns_governance_id, - credits_per_kinic: update.credits_per_kinic, - min_update_credits: update.min_update_credits, - }; - validate_credits_config(&next)?; - let next_version = if config_changed { - current_version - .checked_add(1) - .ok_or_else(|| "credits config version overflow".to_string())? - } else { - current_version + billing_authority_id: current.billing_authority_id, + cycles_per_kinic: update.cycles_per_kinic, + min_update_cycles: update.min_update_cycles, }; + validate_cycles_billing_config(&next)?; self.write_index(|tx| { - set_credits_config_value(tx, "credits_per_kinic", next.credits_per_kinic)?; - set_credits_config_value(tx, "min_update_credits", next.min_update_credits)?; - if config_changed { - set_credits_config_value(tx, "config_version", next_version)?; - } + set_cycles_billing_config_value(tx, "cycles_per_kinic", next.cycles_per_kinic)?; + set_cycles_billing_config_value(tx, "min_update_cycles", next.min_update_cycles)?; Ok(()) })?; Ok(next) @@ -356,6 +364,11 @@ impl VfsService { ) -> Result { let name = normalize_database_name(name)?; self.write_index(|tx| { + purge_expired_unstarted_pending_databases(tx, caller, now)?; + let pending_count = pending_database_count_for_caller(tx, caller)?; + if pending_count >= MAX_PENDING_DATABASES_PER_CALLER { + return Err("too many pending databases for caller".to_string()); + } let mut selected_database_id = None; for attempt in 0_u32..100 { let database_id = @@ -421,7 +434,7 @@ impl VfsService { caller: &str, now: i64, mount_id: u16, - initial_credits_balance: i64, + initial_cycles_balance: i64, ) -> Result { let db_file_name = self.database_file_name(database_id, mount_id)?; tx.execute( @@ -441,18 +454,19 @@ impl VfsService { .map_err(|error| error.to_string())?; record_mount_history(tx, database_id, mount_id, "create", now)?; insert_initial_database_members(tx, database_id, caller, now)?; - let suspended_at_ms = if initial_credits_balance == 0 { + let suspended_at_ms = if initial_cycles_balance == 0 { Some(now) } else { None }; tx.execute( - "INSERT INTO database_credit_accounts - (database_id, balance_credits, suspended_at_ms, created_at_ms, updated_at_ms) - VALUES (?1, ?2, ?3, ?4, ?4)", + "INSERT INTO database_cycle_accounts + (database_id, balance_cycles, suspended_at_ms, storage_charged_at_ms, + created_at_ms, updated_at_ms) + VALUES (?1, ?2, ?3, ?4, ?4, ?4)", params![ database_id, - initial_credits_balance, + initial_cycles_balance, crate::sqlite::nullable_integer_value(suspended_at_ms), now ], @@ -492,9 +506,10 @@ impl VfsService { .map_err(|error| error.to_string())?; insert_initial_database_members(tx, database_id, caller, now)?; tx.execute( - "INSERT INTO database_credit_accounts - (database_id, balance_credits, suspended_at_ms, created_at_ms, updated_at_ms) - VALUES (?1, 0, ?2, ?2, ?2)", + "INSERT INTO database_cycle_accounts + (database_id, balance_cycles, suspended_at_ms, storage_charged_at_ms, + created_at_ms, updated_at_ms) + VALUES (?1, 0, ?2, NULL, ?2, ?2)", params![database_id, now], ) .map_err(|error| error.to_string())?; @@ -521,17 +536,17 @@ impl VfsService { .optional() .map_err(|error| error.to_string())?; tx.execute( - "DELETE FROM database_credit_ledger WHERE database_id = ?1", + "DELETE FROM database_cycle_ledger WHERE database_id = ?1", params![database_id], ) .map_err(|error| error.to_string())?; tx.execute( - "DELETE FROM database_credit_pending_operations WHERE database_id = ?1", + "DELETE FROM database_cycle_pending_operations WHERE database_id = ?1", params![database_id], ) .map_err(|error| error.to_string())?; tx.execute( - "DELETE FROM database_credit_accounts WHERE database_id = ?1", + "DELETE FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], ) .map_err(|error| error.to_string())?; @@ -569,7 +584,7 @@ impl VfsService { Ok(()) } - pub fn activate_pending_database_for_credit_purchase( + pub fn activate_pending_database_for_cycles_purchase( &self, database_id: &str, now: i64, @@ -606,31 +621,44 @@ impl VfsService { }) } - pub fn validate_database_credit_purchase( + pub fn validate_database_cycles_purchase( + &self, + database_id: &str, + payment_amount_e8s: u64, + ) -> Result<(), String> { + self.validate_database_cycles_purchase_with_minimum(database_id, payment_amount_e8s, 0) + } + + pub fn validate_database_cycles_purchase_with_minimum( &self, database_id: &str, - credits: u64, + payment_amount_e8s: u64, + min_expected_cycles: u64, ) -> Result<(), String> { - self.preview_database_credit_purchase(database_id, credits) - .map(|_| ()) + amount_to_i64(payment_amount_e8s)?; + self.read_index(|conn| { + let config = load_cycles_billing_config(conn)?; + let cycles = cycles_for_payment_amount_e8s(payment_amount_e8s, &config)?; + validate_cycles_purchase_minimum(cycles, min_expected_cycles)?; + let cycles_i64 = cycles_to_i64(cycles)?; + validate_database_cycles_purchase_for_conn(conn, database_id, cycles_i64) + }) } - pub fn begin_database_credit_purchase( + pub fn begin_database_cycles_purchase( &self, database_id: &str, caller: &str, - credits: u64, + payment_amount_e8s: u64, now: i64, ) -> Result { - let preview = self.preview_database_credit_purchase(database_id, credits)?; - self.begin_database_credit_purchase_with_ledger_details( - DatabaseCreditPurchaseWithLedgerDetails { + self.begin_database_cycles_purchase_with_ledger_details( + DatabaseCyclesPurchaseWithLedgerDetails { database_id, caller, - credits, - expected_payment_amount_e8s: preview.payment_amount_e8s, - expected_config_version: preview.config_version, - ledger: CreditsPendingLedgerDetailsInput { + payment_amount_e8s, + min_expected_cycles: 0, + ledger: CyclesPendingLedgerDetailsInput { from_owner: caller, from_subaccount: None, to_owner: "canister", @@ -641,43 +669,33 @@ impl VfsService { now, }, ) + .map(|start| start.operation_id) } - pub fn begin_database_credit_purchase_with_ledger_details( + pub fn begin_database_cycles_purchase_with_ledger_details( &self, - request: DatabaseCreditPurchaseWithLedgerDetails<'_>, - ) -> Result { - let credits = credits_to_i64(request.credits)?; + request: DatabaseCyclesPurchaseWithLedgerDetails<'_>, + ) -> Result { + let payment_amount_e8s = amount_to_i64(request.payment_amount_e8s)?; let ledger_fee = amount_to_i64(request.ledger.ledger_fee_e8s)?; let ledger_created_at_time = i64::try_from(request.ledger.ledger_created_at_time_ns) .map_err(|_| "ledger created_at_time exceeds i64".to_string())?; self.write_index(|tx| { - let config = load_credits_config(tx)?; - let config_version = load_credits_config_version(tx)?; - let payment_amount_e8s_u64 = payment_amount_e8s_for_credits(request.credits, &config)?; - if request.expected_config_version != config_version { - return Err(format!( - "credits config changed: expected version {}, current version {}", - request.expected_config_version, config_version - )); - } - if request.expected_payment_amount_e8s != payment_amount_e8s_u64 { - return Err(format!( - "credit purchase payment amount changed: expected {}, current {}", - request.expected_payment_amount_e8s, payment_amount_e8s_u64 - )); - } - let payment_amount_e8s = amount_to_i64(payment_amount_e8s_u64)?; - validate_database_credit_purchase_for_conn(tx, request.database_id, credits)?; - insert_pending_credits_operation( + let config = load_cycles_billing_config(tx)?; + let cycles_u64 = cycles_for_payment_amount_e8s(request.payment_amount_e8s, &config)?; + validate_cycles_purchase_minimum(cycles_u64, request.min_expected_cycles)?; + let cycles = cycles_to_i64(cycles_u64)?; + validate_database_cycles_purchase_for_conn(tx, request.database_id, cycles)?; + ensure_no_pending_cycles_purchase_for_caller(tx, request.database_id, request.caller)?; + let operation_id = insert_pending_cycles_operation( tx, - PendingCreditsOperationInsert { + PendingCyclesOperationInsert { database_id: request.database_id, - kind: "credit_purchase", + kind: "cycles_purchase", caller: request.caller, - credits, + cycles, payment_amount_e8s, - ledger: PendingCreditsLedgerDetails { + ledger: PendingCyclesLedgerDetails { from_owner: request.ledger.from_owner, from_subaccount: request.ledger.from_subaccount, to_owner: request.ledger.to_owner, @@ -685,190 +703,229 @@ impl VfsService { ledger_fee_e8s: ledger_fee, ledger_created_at_time_ns: ledger_created_at_time, }, - operation_status: CREDIT_OPERATION_STATUS_IN_FLIGHT, + operation_status: CYCLES_OPERATION_STATUS_IN_FLIGHT, now: request.now, }, - ) + )?; + Ok(DatabaseCyclesPurchaseStart { + operation_id, + amount_cycles: cycles_u64, + }) }) } - pub fn credit_database_purchase( + + pub fn apply_database_cycles_purchase( &self, operation_id: u64, database_id: &str, caller: &str, - credits: u64, - ledger_block_index: u64, + cycles: u64, + _ledger_block_index: u64, now: i64, ) -> Result { - let credits = credits_to_i64(credits)?; - let config = self.credits_config()?; + let cycles_i64 = cycles_to_i64(cycles)?; + let config = self.cycles_billing_config()?; self.write_index(|tx| { - let operation = load_required_pending_credits_operation( + let operation = load_required_pending_cycles_operation( tx, - PendingCreditsOperationMatch { + PendingCyclesOperationMatch { operation_id, database_id, - kind: "credit_purchase", + kind: "cycles_purchase", caller, - credits, + cycles: cycles_i64, }, )?; require_pending_operation_status( &operation, - &[CREDIT_OPERATION_STATUS_COMPLETED], - "apply completed credit purchase", + &[CYCLES_OPERATION_STATUS_COMPLETED], + "apply cycle purchase", )?; + let ledger_block_index = operation + .ledger_block_index + .ok_or_else(|| "completed cycle purchase missing ledger block index".to_string())?; load_database_status(tx, database_id)?; complete_pending_database_activation(tx, database_id, now)?; let db_balance = database_balance_for_update(tx, database_id)?; - let next_database = checked_balance_add(db_balance, credits)?; - update_database_credits_balance(tx, database_id, next_database, &config, now)?; + let next_database = checked_balance_add(db_balance, cycles_i64)?; + update_database_cycles_balance(tx, database_id, next_database, &config, now)?; insert_database_ledger( tx, DatabaseLedgerInsert { database_id, - kind: "credit_purchase", - amount_credits: credits, - balance_after_credits: next_database, + kind: "cycles_purchase", + amount_cycles: cycles_i64, + balance_after_cycles: next_database, payment_amount_e8s: Some(operation.payment_amount_e8s), caller, - method: Some("purchase_database_credits"), + method: Some("purchase_database_cycles"), cycles_delta: None, config: None, - ledger_block_index: Some(ledger_block_index), + ledger_block_index: Some( + u64::try_from(ledger_block_index).map_err(|error| error.to_string())?, + ), now, }, )?; - delete_pending_credits_operation(tx, operation_id)?; + delete_pending_cycles_operation(tx, operation_id)?; Ok(next_database as u64) }) } - pub fn mark_database_credit_purchase_ambiguous( + pub fn complete_database_cycles_purchase_ledger_transfer( &self, operation_id: u64, database_id: &str, caller: &str, - credits: u64, - now: i64, - ) -> Result { - let credits = credits_to_i64(credits)?; + cycles: u64, + ledger_block_index: u64, + ) -> Result<(), String> { + let cycles_i64 = cycles_to_i64(cycles)?; + let ledger_block_index = i64::try_from(ledger_block_index) + .map_err(|_| "ledger block index exceeds i64".to_string())?; self.write_index(|tx| { - let operation = load_required_pending_credits_operation( + let operation = load_required_pending_cycles_operation( tx, - PendingCreditsOperationMatch { + PendingCyclesOperationMatch { operation_id, database_id, - kind: "credit_purchase", + kind: "cycles_purchase", caller, - credits, + cycles: cycles_i64, }, )?; require_pending_operation_status( &operation, - &[CREDIT_OPERATION_STATUS_IN_FLIGHT], - "mark credit purchase ambiguous", + &[CYCLES_OPERATION_STATUS_IN_FLIGHT], + "complete cycle purchase ledger transfer", )?; - load_database_status(tx, database_id)?; - let balance = database_balance_for_update(tx, database_id)?; - insert_database_ledger( + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + tx.execute( + "UPDATE database_cycle_pending_operations + SET operation_status = ?2, + ledger_block_index = ?3 + WHERE operation_id = ?1", + params![ + operation_id, + CYCLES_OPERATION_STATUS_COMPLETED, + ledger_block_index + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) + }) + } + + pub fn mark_database_cycles_purchase_ambiguous( + &self, + operation_id: u64, + database_id: &str, + caller: &str, + cycles: u64, + ) -> Result<(), String> { + let cycles_i64 = cycles_to_i64(cycles)?; + self.write_index(|tx| { + let operation = load_required_pending_cycles_operation( tx, - DatabaseLedgerInsert { + PendingCyclesOperationMatch { + operation_id, database_id, - kind: "credit_purchase_ambiguous", - amount_credits: 0, - balance_after_credits: balance, - payment_amount_e8s: Some(operation.payment_amount_e8s), + kind: "cycles_purchase", caller, - method: Some("purchase_database_credits"), - cycles_delta: None, - config: None, - ledger_block_index: None, - now, + cycles: cycles_i64, }, )?; - update_pending_credits_operation_status( - tx, - operation_id, - CREDIT_OPERATION_STATUS_AMBIGUOUS, + require_pending_operation_status( + &operation, + &[CYCLES_OPERATION_STATUS_IN_FLIGHT], + "mark cycle purchase ambiguous", )?; - Ok(balance as u64) + let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; + tx.execute( + "UPDATE database_cycle_pending_operations + SET operation_status = ?2 + WHERE operation_id = ?1", + params![operation_id, CYCLES_OPERATION_STATUS_AMBIGUOUS], + ) + .map_err(|error| error.to_string())?; + Ok(()) }) } - pub fn mark_database_credit_purchase_completed( + pub fn cleanup_database_cycles_purchase_after_no_credit( &self, operation_id: u64, database_id: &str, caller: &str, - credits: u64, + cycles: u64, ) -> Result<(), String> { - let credits = credits_to_i64(credits)?; - self.write_index(|tx| { - let operation = load_required_pending_credits_operation( + let cycles_i64 = cycles_to_i64(cycles)?; + let status = self.write_index(|tx| { + let operation = load_required_pending_cycles_operation( tx, - PendingCreditsOperationMatch { + PendingCyclesOperationMatch { operation_id, database_id, - kind: "credit_purchase", + kind: "cycles_purchase", caller, - credits, + cycles: cycles_i64, }, )?; require_pending_operation_status( &operation, - &[CREDIT_OPERATION_STATUS_IN_FLIGHT], - "mark credit purchase completed", + &[CYCLES_OPERATION_STATUS_IN_FLIGHT], + "cleanup cycle purchase", )?; - update_pending_credits_operation_status( - tx, - operation_id, - CREDIT_OPERATION_STATUS_COMPLETED, - ) - }) + load_database_status(tx, database_id) + })?; + if status == DatabaseStatus::Pending { + self.discard_database_reservation(database_id) + } else { + self.cancel_database_cycles_purchase(operation_id, database_id, caller, cycles) + } } - pub fn cancel_database_credit_purchase( + pub fn cancel_database_cycles_purchase( &self, operation_id: u64, database_id: &str, caller: &str, - credits: u64, + cycles: u64, ) -> Result<(), String> { - let credits = credits_to_i64(credits)?; + let cycles_i64 = cycles_to_i64(cycles)?; self.write_index(|tx| { - let operation = load_required_pending_credits_operation( + let operation = load_required_pending_cycles_operation( tx, - PendingCreditsOperationMatch { + PendingCyclesOperationMatch { operation_id, database_id, - kind: "credit_purchase", + kind: "cycles_purchase", caller, - credits, + cycles: cycles_i64, }, )?; require_pending_operation_status( &operation, - &[CREDIT_OPERATION_STATUS_IN_FLIGHT], - "cancel credit purchase", + &[CYCLES_OPERATION_STATUS_IN_FLIGHT], + "cancel cycle purchase", )?; - delete_pending_credits_operation(tx, operation_id) + delete_pending_cycles_operation(tx, operation_id) }) } - pub fn list_database_credit_entries( + pub fn list_database_cycle_entries( &self, database_id: &str, caller: &str, cursor: Option, limit: u32, - ) -> Result { - let config = self.credits_config()?; + ) -> Result { + let config = self.cycles_billing_config()?; let limit = page_limit(limit); let after = i64::try_from(cursor.unwrap_or(0)).map_err(|error| error.to_string())?; self.read_index(|conn| { let _status = load_database_status(conn, database_id)?; - let show_principal = if caller == config.sns_governance_id { + let show_principal = if caller == config.billing_authority_id { true } else { let role = load_member_role(conn, database_id, caller)? @@ -882,10 +939,10 @@ impl VfsService { }; let mut stmt = conn .prepare( - "SELECT entry_id, database_id, kind, amount_credits, balance_after_credits, - payment_amount_e8s, caller, method, cycles_delta, credits_per_kinic, + "SELECT entry_id, database_id, kind, amount_cycles, balance_after_cycles, + payment_amount_e8s, caller, method, cycles_delta, cycles_per_kinic, ledger_block_index, created_at_ms - FROM database_credit_ledger + FROM database_cycle_ledger WHERE database_id = ?1 AND entry_id > ?2 ORDER BY entry_id ASC LIMIT ?3", @@ -894,7 +951,7 @@ impl VfsService { let mut entries = crate::sqlite::query_map( &mut stmt, params![database_id, after, i64::from(limit) + 1], - map_database_credits_entry, + map_database_cycles_entry, ) .map_err(|error| error.to_string())?; if !show_principal { @@ -908,231 +965,44 @@ impl VfsService { } else { None }; - Ok(DatabaseCreditEntryPage { + Ok(DatabaseCycleEntryPage { entries, next_cursor, }) }) } - pub fn list_database_credit_pending_operations( + pub fn list_database_cycles_pending_purchases( &self, database_id: &str, caller: &str, - cursor: Option, - limit: u32, - ) -> Result { - let config = self.credits_config()?; - let limit = page_limit(limit); - let after = i64::try_from(cursor.unwrap_or(0)).map_err(|error| error.to_string())?; + ) -> Result, String> { + let config = self.cycles_billing_config()?; self.read_index(|conn| { load_database_status(conn, database_id)?; - if caller != config.sns_governance_id { - let role = load_member_role(conn, database_id, caller)? - .ok_or_else(|| format!("principal has no access to database: {database_id}"))?; - if role != DatabaseRole::Owner { + let role = load_member_role(conn, database_id, caller)?; + let show_all = + caller == config.billing_authority_id || role == Some(DatabaseRole::Owner); + let mut purchases = load_database_cycles_pending_purchase_statuses(conn, database_id)?; + if !show_all { + purchases.retain(|purchase| purchase.caller == caller); + if purchases.is_empty() { return Err(format!( - "principal lacks required database role: {database_id}" + "principal cannot view pending cycle purchases: {database_id}" )); } } - let mut stmt = conn - .prepare( - "SELECT operation_id, database_id, kind, caller, credits, payment_amount_e8s, - from_owner, from_subaccount, to_owner, to_subaccount, ledger_fee_e8s, - ledger_created_at_time_ns, operation_status, created_at_ms - FROM database_credit_pending_operations - WHERE database_id = ?1 AND operation_id > ?2 - ORDER BY operation_id ASC - LIMIT ?3", - ) - .map_err(|error| error.to_string())?; - let mut entries = crate::sqlite::query_map( - &mut stmt, - params![database_id, after, i64::from(limit) + 1], - map_pending_credits_operation, - ) - .map_err(|error| error.to_string())?; - let next_cursor = if entries.len() > limit as usize { - entries.pop(); - entries.last().map(|entry| entry.operation_id) - } else { - None - }; - let entries = entries + purchases .into_iter() - .map(pending_credits_operation_to_public) - .collect::, _>>()?; - Ok(DatabaseCreditPendingOperationPage { - entries, - next_cursor, - }) - }) - } - - pub fn get_database_credit_pending_operation_for_complete( - &self, - database_id: &str, - operation_id: u64, - ) -> Result { - self.write_index(|tx| { - let operation = load_pending_credits_operation(tx, operation_id)?; - if operation.database_id != database_id { - return Err("pending credit operation mismatch".to_string()); - } - require_pending_operation_status( - &operation, - &[ - CREDIT_OPERATION_STATUS_IN_FLIGHT, - CREDIT_OPERATION_STATUS_AMBIGUOUS, - CREDIT_OPERATION_STATUS_COMPLETED, - ], - "complete credit purchase repair", - )?; - pending_credits_operation_to_public(operation) - }) - } - - pub fn mark_database_credit_purchase_repair_completed( - &self, - database_id: &str, - operation_id: u64, - ) -> Result<(), String> { - self.write_index(|tx| { - let operation = load_pending_credits_operation(tx, operation_id)?; - require_pending_database_kind(&operation, database_id, "credit_purchase")?; - require_pending_operation_status( - &operation, - &[ - CREDIT_OPERATION_STATUS_IN_FLIGHT, - CREDIT_OPERATION_STATUS_AMBIGUOUS, - CREDIT_OPERATION_STATUS_COMPLETED, - ], - "complete credit purchase repair", - )?; - if operation.operation_status != CREDIT_OPERATION_STATUS_COMPLETED { - update_pending_credits_operation_status( - tx, - operation_id, - CREDIT_OPERATION_STATUS_COMPLETED, - )?; - } - Ok(()) - }) - } - - pub fn repair_database_credit_purchase_complete( - &self, - database_id: &str, - operation_id: u64, - ledger_block_index: u64, - now: i64, - ) -> Result { - let config = self.credits_config()?; - self.write_index(|tx| { - let operation = load_pending_credits_operation(tx, operation_id)?; - require_pending_database_kind(&operation, database_id, "credit_purchase")?; - require_pending_operation_status( - &operation, - &[ - CREDIT_OPERATION_STATUS_AMBIGUOUS, - CREDIT_OPERATION_STATUS_COMPLETED, - ], - "complete credit purchase repair", - )?; - load_database_status(tx, database_id)?; - complete_pending_database_activation(tx, database_id, now)?; - let balance = database_balance_for_update(tx, database_id)?; - let next = checked_balance_add(balance, operation.credits)?; - update_database_credits_balance(tx, database_id, next, &config, now)?; - insert_database_ledger( - tx, - DatabaseLedgerInsert { - database_id, - kind: "credit_purchase_repair_complete", - amount_credits: operation.credits, - balance_after_credits: next, - payment_amount_e8s: Some(operation.payment_amount_e8s), - caller: &operation.caller, - method: Some("repair_database_credit_purchase_complete"), - cycles_delta: None, - config: None, - ledger_block_index: Some(ledger_block_index), - now, - }, - )?; - delete_pending_credits_operation(tx, operation_id)?; - Ok(next as u64) - }) - } - - pub fn repair_database_credit_purchase_cancel( - &self, - database_id: &str, - operation_id: u64, - caller: &str, - now: i64, - ) -> Result<(), String> { - self.write_index(|tx| { - let operation = load_pending_credits_operation(tx, operation_id)?; - require_pending_database_kind(&operation, database_id, "credit_purchase")?; - require_pending_operation_status( - &operation, - &[CREDIT_OPERATION_STATUS_AMBIGUOUS], - "cancel credit purchase repair", - )?; - let status = load_database_status(tx, database_id)?; - let is_payer = operation.caller == caller; - let is_owner = load_member_role(tx, database_id, caller)? - .map(|role| role == DatabaseRole::Owner) - .unwrap_or(false); - if !is_payer && !is_owner { - return Err(format!( - "caller is not credit purchase payer or database owner: {database_id}" - )); - } - let active_mount_id: Option = tx - .query_row( - "SELECT active_mount_id FROM databases WHERE database_id = ?1", - params![database_id], - |row| crate::sqlite::row_get(row, 0), - ) - .map_err(|error| error.to_string())?; - if status == DatabaseStatus::Pending && active_mount_id.is_some() { - return Err( - "pending database activation already started; complete credit purchase repair" - .to_string(), - ); - } - let balance = database_balance_for_update(tx, database_id)?; - insert_database_ledger( - tx, - DatabaseLedgerInsert { - database_id, - kind: "credit_purchase_repair_cancelled", - amount_credits: 0, - balance_after_credits: balance, - payment_amount_e8s: Some(operation.payment_amount_e8s), - caller, - method: Some("repair_database_credit_purchase_cancel"), - cycles_delta: None, - config: None, - ledger_block_index: None, - now, - }, - )?; - delete_pending_credits_operation(tx, operation_id)?; - Ok(()) + .map(DatabaseCyclesPendingPurchaseRaw::into_public) + .collect::, _>>() }) } - pub fn require_database_write_credits_available( - &self, - database_id: &str, - ) -> Result<(), String> { + pub fn require_database_write_cycles_available(&self, database_id: &str) -> Result<(), String> { self.read_index(|conn| { - let config = load_credits_config(conn)?; - require_database_write_credits_available_for_conn(conn, database_id, &config) + let config = load_cycles_billing_config(conn)?; + require_database_write_cycles_available_for_conn(conn, database_id, &config) }) } @@ -1141,7 +1011,7 @@ impl VfsService { database_id: &str, caller: &str, required_role: RequiredRole, - ) -> Result { + ) -> Result { self.read_index(|conn| { let role = load_database_status(conn, database_id).and_then(|_| { load_member_role(conn, database_id, caller)? @@ -1152,13 +1022,13 @@ impl VfsService { "principal lacks required database role: {database_id}" )); } - let config = load_credits_config(conn)?; - require_database_write_credits_available_for_conn(conn, database_id, &config)?; + let config = load_cycles_billing_config(conn)?; + require_database_write_cycles_available_for_conn(conn, database_id, &config)?; Ok(config) }) } - pub fn check_database_write_credits( + pub fn check_database_write_cycles( &self, database_id: &str, caller: &str, @@ -1167,12 +1037,12 @@ impl VfsService { return Err("anonymous caller not allowed".to_string()); } self.require_role(database_id, caller, RequiredRole::Writer)?; - self.require_database_write_credits_available(database_id) + self.require_database_write_cycles_available(database_id) } pub fn charge_database_update( &self, - config: &CreditsConfig, + config: &CyclesBillingConfig, database_id: &str, caller: &str, method: &str, @@ -1220,7 +1090,7 @@ impl VfsService { } let result = self.database_store(meta)?.run_fs_migrations(); if result.is_ok() { - self.refresh_logical_size(database_id)?; + let _ = self.refresh_logical_size(database_id); } result } @@ -1233,7 +1103,7 @@ impl VfsService { ) -> Result<(), String> { let database_id = request.database_id.as_str(); self.require_role(database_id, caller, RequiredRole::Owner)?; - self.require_no_pending_credits_operations(database_id)?; + self.require_no_pending_cycles_operations(database_id)?; let status = self.read_index(|conn| load_database_status(conn, database_id))?; if !matches!(status, DatabaseStatus::Pending | DatabaseStatus::Active) { return Err(format!( @@ -1257,19 +1127,15 @@ impl VfsService { }) } - fn require_no_pending_credits_operations(&self, database_id: &str) -> Result<(), String> { + fn require_no_pending_cycles_operations(&self, database_id: &str) -> Result<(), String> { self.read_index(|conn| { - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM database_credit_pending_operations - WHERE database_id = ?1", - params![database_id], - |row| crate::sqlite::row_get(row, 0), - ) - .map_err(|error| error.to_string())?; - if count > 0 { + let pending = first_database_cycles_pending_purchase_status(conn, database_id)?; + if let Some(pending) = pending { return Err(format!( - "database has pending credit operation: {database_id}" + "database has pending cycle operation: {database_id}; operation_id={}; status={}; required_action={}", + pending.operation_id, + pending.status, + pending.required_action )); } Ok(()) @@ -1283,6 +1149,7 @@ impl VfsService { now: i64, ) -> Result { self.require_role(database_id, caller, RequiredRole::Owner)?; + self.require_no_pending_cycles_operations(database_id)?; let meta = self.database_meta(database_id)?; let size_bytes = self.database_size(&meta)?; self.write_index(|conn| { @@ -1351,7 +1218,6 @@ impl VfsService { conn.execute( "UPDATE databases SET status = 'archived', - active_mount_id = NULL, snapshot_hash = ?2, restore_size_bytes = NULL, archived_at_ms = ?3, @@ -1414,13 +1280,15 @@ impl VfsService { "database size exceeds limit: {size_bytes} > {MAX_DATABASE_SIZE_BYTES}" )); } + self.require_no_pending_cycles_operations(database_id)?; let rollback = self.database_restore_rollback(database_id)?; if rollback.status != DatabaseStatus::Archived { return Err("database restore can only begin from archived status".to_string()); } + let mount_id = rollback + .active_mount_id + .ok_or_else(|| format!("archived database has no mount: {database_id}"))?; self.write_index(|tx| { - let mount_id = allocate_mount_id(tx)?; - record_mount_history(tx, database_id, mount_id, "restore", now)?; record_database_restore_session(tx, &rollback, now)?; tx.execute( "DELETE FROM database_restore_chunks WHERE database_id = ?1", @@ -1611,11 +1479,16 @@ impl VfsService { now: i64, ) -> Result<(), String> { self.require_role(database_id, caller, RequiredRole::Owner)?; - self.require_database_not_pending(database_id)?; if caller == principal && role != DatabaseRole::Owner { return Err("owner cannot downgrade own access".to_string()); } self.write_index(|conn| { + if !database_member_exists(conn, database_id, principal)? { + let member_count = database_member_count_for_conn(conn, database_id)?; + if member_count >= MAX_DATABASE_MEMBERS_PER_DATABASE { + return Err("too many database members".to_string()); + } + } conn.execute( "INSERT INTO database_members (database_id, principal, role, created_at_ms) VALUES (?1, ?2, ?3, ?4) @@ -1796,7 +1669,7 @@ impl VfsService { .read_node(&request.database_id, &principal, &request.request_path)? .ok_or_else(|| format!("url ingest request not found: {}", request.request_path))?; validate_url_ingest_request_node(&node, &principal)?; - self.require_database_write_credits_available(&request.database_id) + self.require_database_write_cycles_available(&request.database_id) } pub fn authorize_ops_answer_session( @@ -1854,7 +1727,7 @@ impl VfsService { .ok_or_else(|| "ops answer session is missing or expired".to_string()) })?; self.require_role(&request.database_id, &principal, RequiredRole::Reader)?; - self.require_database_write_credits_available(&request.database_id)?; + self.require_database_write_cycles_available(&request.database_id)?; Ok(OpsAnswerSessionCheckResult { principal }) } @@ -1901,7 +1774,7 @@ impl VfsService { if source.etag != request.source_etag { return Err("source run session source etag is stale".to_string()); } - self.require_database_write_credits_available(&request.database_id)?; + self.require_database_write_cycles_available(&request.database_id)?; Ok(()) } @@ -1940,7 +1813,7 @@ impl VfsService { store.write_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -1978,15 +1851,15 @@ impl VfsService { self.with_database_store(&database_id, caller, RequiredRole::Writer, |store| { store.write_node(write_request, now) })?; - self.write_source_run_session( + let _ = self.write_source_run_session( &database_id, &path, &write.node.etag, &session_nonce, caller, now, - )?; - self.refresh_logical_size(&database_id)?; + ); + let _ = self.refresh_logical_size(&database_id); Ok(WriteSourceForGenerationResult { write, session_nonce, @@ -2008,7 +1881,7 @@ impl VfsService { store.write_nodes(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2025,7 +1898,7 @@ impl VfsService { store.delete_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2048,7 +1921,7 @@ impl VfsService { store.append_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2065,7 +1938,7 @@ impl VfsService { store.edit_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2082,7 +1955,7 @@ impl VfsService { store.mkdir_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2102,7 +1975,7 @@ impl VfsService { store.move_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2207,7 +2080,7 @@ impl VfsService { store.multi_edit_node(request, now) }); if result.is_ok() { - self.refresh_logical_size(&database_id)?; + let _ = self.refresh_logical_size(&database_id); } result } @@ -2315,14 +2188,6 @@ impl VfsService { ) } - fn require_database_not_pending(&self, database_id: &str) -> Result<(), String> { - let status = self.read_index(|conn| load_database_status(conn, database_id))?; - if status == DatabaseStatus::Pending { - return Err(format!("database is pending: {database_id}")); - } - Ok(()) - } - fn database_meta_with_statuses( &self, database_id: &str, @@ -2635,7 +2500,7 @@ impl VfsService { } #[cfg(not(target_arch = "wasm32"))] -fn run_index_migrations(conn: &mut Connection, config: &CreditsConfig) -> Result<(), String> { +fn run_index_migrations(conn: &mut Connection, config: &CyclesBillingConfig) -> Result<(), String> { if sqlite_master_entry_exists(conn, "table", "schema_migrations")? { let tx = conn.transaction().map_err(|error| error.to_string())?; ensure_existing_index_schema_is_latest(&tx, Some(config))?; @@ -2649,12 +2514,16 @@ fn run_index_migrations(conn: &mut Connection, config: &CreditsConfig) -> Result )); } } - validate_credits_config(config)?; + if let Some(table) = legacy_credit_index_table_name(conn)? { + return Err(format!( + "unsupported index schema: {table} exists without supported schema_migrations; recreate the index database" + )); + } + validate_cycles_billing_config(config)?; let tx = conn.transaction().map_err(|error| error.to_string())?; create_schema_migrations(&tx)?; create_fresh_index_schema(&tx)?; - insert_credits_config(&tx, config)?; - insert_credits_config_version(&tx)?; + insert_cycles_billing_config(&tx, config)?; for &version in INDEX_SCHEMA_VERSIONS { insert_schema_migration_now(&tx, version)?; } @@ -2664,7 +2533,7 @@ fn run_index_migrations(conn: &mut Connection, config: &CreditsConfig) -> Result #[cfg(not(target_arch = "wasm32"))] fn run_index_migrations_for_upgrade( conn: &mut Connection, - config: Option<&CreditsConfig>, + config: Option<&CyclesBillingConfig>, ) -> Result<(), String> { if sqlite_master_entry_exists(conn, "table", "schema_migrations")? { let tx = conn.transaction().map_err(|error| error.to_string())?; @@ -2673,14 +2542,14 @@ fn run_index_migrations_for_upgrade( return Ok(()); } let config = - config.ok_or_else(|| "credits config required for fresh index upgrade".to_string())?; + config.ok_or_else(|| "cycles config required for fresh index upgrade".to_string())?; run_index_migrations(conn, config) } #[cfg(target_arch = "wasm32")] fn run_index_migrations_in_tx( conn: &Transaction<'_>, - config: &CreditsConfig, + config: &CyclesBillingConfig, ) -> Result<(), String> { if wasm_index_table_exists(conn, "schema_migrations")? { ensure_existing_index_schema_is_latest(conn, Some(config))?; @@ -2693,11 +2562,15 @@ fn run_index_migrations_in_tx( )); } } - validate_credits_config(config)?; + if let Some(table) = legacy_credit_index_table_name_tx(conn)? { + return Err(format!( + "unsupported index schema: {table} exists without schema_migrations" + )); + } + validate_cycles_billing_config(config)?; create_schema_migrations(conn)?; create_fresh_index_schema(conn)?; - insert_credits_config(conn, config)?; - insert_credits_config_version(conn)?; + insert_cycles_billing_config(conn, config)?; for &version in INDEX_SCHEMA_VERSIONS { insert_schema_migration_zero(conn, version)?; } @@ -2707,37 +2580,32 @@ fn run_index_migrations_in_tx( #[cfg(target_arch = "wasm32")] fn run_index_migrations_in_tx_for_upgrade( conn: &Transaction<'_>, - config: Option<&CreditsConfig>, + config: Option<&CyclesBillingConfig>, ) -> Result<(), String> { if wasm_index_table_exists(conn, "schema_migrations")? { ensure_existing_index_schema_is_latest(conn, config)?; return Ok(()); } let config = - config.ok_or_else(|| "credits config required for fresh index upgrade".to_string())?; + config.ok_or_else(|| "cycles config required for fresh index upgrade".to_string())?; run_index_migrations_in_tx(conn, config) } enum IndexSchemaState { Latest, - NeedsPendingOperationStatus, Mainnet011, } fn ensure_existing_index_schema_is_latest( conn: &Transaction<'_>, - config: Option<&CreditsConfig>, + config: Option<&CyclesBillingConfig>, ) -> Result<(), String> { match classify_existing_index_schema_state(conn)? { IndexSchemaState::Latest => validate_index_schema(conn), - IndexSchemaState::NeedsPendingOperationStatus => { - apply_pending_operation_status_index_migration(conn)?; - validate_index_schema(conn) - } IndexSchemaState::Mainnet011 => { let config = config - .ok_or_else(|| "credits config required for first credits upgrade".to_string())?; - validate_credits_config(config)?; + .ok_or_else(|| "cycles config required for first cycles upgrade".to_string())?; + validate_cycles_billing_config(config)?; validate_pre_billing_index_schema(conn)?; apply_mainnet_011_to_latest_index_migration(conn, config)?; validate_index_schema(conn) @@ -2748,13 +2616,30 @@ fn ensure_existing_index_schema_is_latest( fn classify_existing_index_schema_state( conn: &Transaction<'_>, ) -> Result { - if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_CREDITS_CONFIG_VERSION)? { - return if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS)? - { - Ok(IndexSchemaState::Latest) - } else { - Ok(IndexSchemaState::NeedsPendingOperationStatus) - }; + let legacy_billing_marker: Option = conn + .query_row( + "SELECT version + FROM schema_migrations + WHERE version LIKE '%credit%' + ORDER BY version + LIMIT 1", + params![], + |row| row.get(0), + ) + .optional() + .map_err(|error| error.to_string())?; + if let Some(version) = legacy_billing_marker { + return Err(format!( + "unsupported partial billing index schema: migration {version} is already applied" + )); + } + if let Some(table) = legacy_credit_index_table_name_tx(conn)? { + return Err(format!( + "unsupported partial billing index schema: table {table} already exists" + )); + } + if migration_applied_tx(conn, INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX)? { + return Ok(IndexSchemaState::Latest); } if !migration_applied_tx(conn, INDEX_SCHEMA_VERSION_SOURCE_RUN_SESSIONS)? { return Err(format!( @@ -2780,47 +2665,17 @@ fn classify_existing_index_schema_state( fn apply_mainnet_011_to_latest_index_migration( conn: &Transaction<'_>, - config: &CreditsConfig, + config: &CyclesBillingConfig, ) -> Result<(), String> { conn.execute_batch(INDEX_011_TO_LATEST_SQL) .map_err(|error| error.to_string())?; - conn.execute( - "INSERT INTO database_credit_accounts - (database_id, balance_credits, suspended_at_ms, created_at_ms, updated_at_ms) - SELECT database_id, 0, 0, 0, 0 FROM databases", - params![], - ) - .map_err(|error| error.to_string())?; - insert_credits_config(conn, config)?; - insert_credits_config_version(conn)?; - conn.execute( - "UPDATE databases SET status = 'active' WHERE status = 'hot'", - params![], - ) - .map_err(|error| error.to_string())?; - purge_hard_deleted_database_rows(conn)?; + insert_cycles_billing_config(conn, config)?; for &version in POST_011_INDEX_SCHEMA_VERSIONS { insert_schema_migration_now(conn, version)?; } Ok(()) } -fn apply_pending_operation_status_index_migration(conn: &Transaction<'_>) -> Result<(), String> { - if !index_column_exists( - conn, - "database_credit_pending_operations", - "operation_status", - )? { - conn.execute( - "ALTER TABLE database_credit_pending_operations - ADD COLUMN operation_status TEXT NOT NULL DEFAULT 'ambiguous'", - params![], - ) - .map_err(|error| error.to_string())?; - } - insert_schema_migration_now(conn, INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS) -} - fn create_schema_migrations(conn: &Transaction<'_>) -> Result<(), String> { conn.execute( "CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL)", @@ -2854,29 +2709,26 @@ fn create_fresh_index_schema(conn: &Transaction<'_>) -> Result<(), String> { .map_err(|error| error.to_string()) } -fn default_credits_config() -> CreditsConfig { - CreditsConfig { +fn default_cycles_billing_config() -> CyclesBillingConfig { + CyclesBillingConfig { kinic_ledger_canister_id: "aaaaa-aa".to_string(), - sns_governance_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), - credits_per_kinic: DEFAULT_CREDITS_PER_KINIC, - min_update_credits: DEFAULT_MIN_UPDATE_CREDITS, + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), + cycles_per_kinic: DEFAULT_CYCLES_PER_KINIC, + min_update_cycles: DEFAULT_MIN_UPDATE_CYCLES, } } -fn validate_credits_config(config: &CreditsConfig) -> Result<(), String> { +fn validate_cycles_billing_config(config: &CyclesBillingConfig) -> Result<(), String> { validate_principal_text(&config.kinic_ledger_canister_id)?; - validate_principal_text(&config.sns_governance_id)?; - if config.credits_per_kinic == 0 { - return Err("credits_per_kinic must be positive".to_string()); - } - if config.min_update_credits == 0 { - return Err("min_update_credits must be positive".to_string()); + validate_principal_text(&config.billing_authority_id)?; + if config.cycles_per_kinic == 0 { + return Err("cycles_per_kinic must be positive".to_string()); } - if !KINIC_E8S_PER_TOKEN.is_multiple_of(config.credits_per_kinic) { - return Err("credits_per_kinic must divide 100000000".to_string()); + if config.min_update_cycles == 0 { + return Err("min_update_cycles must be positive".to_string()); } - amount_to_i64(config.credits_per_kinic)?; - amount_to_i64(config.min_update_credits)?; + amount_to_i64(config.cycles_per_kinic)?; + amount_to_i64(config.min_update_cycles)?; Ok(()) } @@ -2889,34 +2741,32 @@ fn validate_principal_text(value: &str) -> Result<(), String> { Ok(()) } -fn insert_credits_config(conn: &Transaction<'_>, config: &CreditsConfig) -> Result<(), String> { +fn insert_cycles_billing_config( + conn: &Transaction<'_>, + config: &CyclesBillingConfig, +) -> Result<(), String> { conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", + "INSERT INTO cycles_billing_config (key, value) VALUES (?1, ?2)", params!["kinic_ledger_canister_id", config.kinic_ledger_canister_id], ) .map_err(|error| error.to_string())?; conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params!["sns_governance_id", config.sns_governance_id], + "INSERT INTO cycles_billing_config (key, value) VALUES (?1, ?2)", + params!["billing_authority_id", config.billing_authority_id], ) .map_err(|error| error.to_string())?; - set_credits_config_value(conn, "credits_per_kinic", config.credits_per_kinic)?; - set_credits_config_value(conn, "min_update_credits", config.min_update_credits)?; + set_cycles_billing_config_value(conn, "cycles_per_kinic", config.cycles_per_kinic)?; + set_cycles_billing_config_value(conn, "min_update_cycles", config.min_update_cycles)?; Ok(()) } -fn insert_credits_config_version(conn: &Transaction<'_>) -> Result<(), String> { - conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params!["config_version", DEFAULT_CREDITS_CONFIG_VERSION.to_string()], - ) - .map_err(|error| error.to_string())?; - Ok(()) -} - -fn set_credits_config_value(conn: &Transaction<'_>, key: &str, value: u64) -> Result<(), String> { +fn set_cycles_billing_config_value( + conn: &Transaction<'_>, + key: &str, + value: u64, +) -> Result<(), String> { conn.execute( - "INSERT INTO credits_config (key, value) + "INSERT INTO cycles_billing_config (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value", params![key, value.to_string()], @@ -2943,10 +2793,14 @@ const INDEX_SCHEMA_VERSIONS: &[&str] = &[ INDEX_SCHEMA_VERSION_BILLING_PENDING_LEDGER_DETAILS, INDEX_SCHEMA_VERSION_ACTIVE_STATUS, INDEX_SCHEMA_VERSION_HARD_DELETE_DATABASES, - INDEX_SCHEMA_VERSION_CREDIT_LEDGER_ONLY, - INDEX_SCHEMA_VERSION_FIXED_CYCLES_PER_CREDIT, - INDEX_SCHEMA_VERSION_CREDITS_CONFIG_VERSION, - INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS, + INDEX_SCHEMA_VERSION_CYCLES_LEDGER_ONLY, + INDEX_SCHEMA_VERSION_FIXED_CYCLES_ACCOUNTING, + INDEX_SCHEMA_VERSION_CYCLES_BILLING_CONFIG_VERSION, + INDEX_SCHEMA_VERSION_CYCLES_PENDING_OPERATION_STATUS, + INDEX_SCHEMA_VERSION_CYCLES, + INDEX_SCHEMA_VERSION_STORAGE_BILLING, + INDEX_SCHEMA_VERSION_DIRECT_CYCLES, + INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX, ]; const INDEX_SCHEMA_TABLES_WITHOUT_MIGRATIONS: &[&str] = &[ @@ -2958,10 +2812,10 @@ const INDEX_SCHEMA_TABLES_WITHOUT_MIGRATIONS: &[&str] = &[ "ops_answer_sessions", "source_run_sessions", "database_restore_sessions", - "database_credit_accounts", - "database_credit_ledger", - "database_credit_pending_operations", - "credits_config", + "database_cycle_accounts", + "database_cycle_ledger", + "database_cycle_pending_operations", + "cycles_billing_config", ]; const POST_011_INDEX_SCHEMA_VERSIONS: &[&str] = &[ @@ -2971,17 +2825,21 @@ const POST_011_INDEX_SCHEMA_VERSIONS: &[&str] = &[ INDEX_SCHEMA_VERSION_BILLING_PENDING_LEDGER_DETAILS, INDEX_SCHEMA_VERSION_ACTIVE_STATUS, INDEX_SCHEMA_VERSION_HARD_DELETE_DATABASES, - INDEX_SCHEMA_VERSION_CREDIT_LEDGER_ONLY, - INDEX_SCHEMA_VERSION_FIXED_CYCLES_PER_CREDIT, - INDEX_SCHEMA_VERSION_CREDITS_CONFIG_VERSION, - INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS, + INDEX_SCHEMA_VERSION_CYCLES_LEDGER_ONLY, + INDEX_SCHEMA_VERSION_FIXED_CYCLES_ACCOUNTING, + INDEX_SCHEMA_VERSION_CYCLES_BILLING_CONFIG_VERSION, + INDEX_SCHEMA_VERSION_CYCLES_PENDING_OPERATION_STATUS, + INDEX_SCHEMA_VERSION_CYCLES, + INDEX_SCHEMA_VERSION_STORAGE_BILLING, + INDEX_SCHEMA_VERSION_DIRECT_CYCLES, + INDEX_SCHEMA_VERSION_CYCLES_PENDING_LEDGER_BLOCK_INDEX, ]; const POST_011_INDEX_SCHEMA_TABLES: &[&str] = &[ - "database_credit_accounts", - "database_credit_ledger", - "database_credit_pending_operations", - "credits_config", + "database_cycle_accounts", + "database_cycle_ledger", + "database_cycle_pending_operations", + "cycles_billing_config", ]; fn validate_pre_billing_index_schema(conn: &Transaction<'_>) -> Result<(), String> { @@ -3110,10 +2968,10 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { "databases", "database_restore_chunks", "database_restore_sessions", - "database_credit_accounts", - "database_credit_ledger", - "database_credit_pending_operations", - "credits_config", + "database_cycle_accounts", + "database_cycle_ledger", + "database_cycle_pending_operations", + "cycles_billing_config", ] { if !tx_sqlite_master_entry_exists(conn, "table", table)? { return Err(format!("unsupported index schema: missing table {table}")); @@ -3145,34 +3003,39 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { &["database_id", "offset_bytes", "end_bytes", "bytes"][..], ), ( - "database_credit_accounts", - &["database_id", "balance_credits", "suspended_at_ms"][..], + "database_cycle_accounts", + &[ + "database_id", + "balance_cycles", + "suspended_at_ms", + "storage_charged_at_ms", + ][..], ), ( - "database_credit_ledger", + "database_cycle_ledger", &[ "entry_id", "database_id", "kind", - "amount_credits", - "balance_after_credits", + "amount_cycles", + "balance_after_cycles", "payment_amount_e8s", "caller", "method", "cycles_delta", - "credits_per_kinic", + "cycles_per_kinic", "ledger_block_index", "created_at_ms", ][..], ), ( - "database_credit_pending_operations", + "database_cycle_pending_operations", &[ "operation_id", "database_id", "kind", "caller", - "credits", + "cycles", "payment_amount_e8s", "from_owner", "from_subaccount", @@ -3181,6 +3044,7 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { "ledger_fee_e8s", "ledger_created_at_time_ns", "operation_status", + "ledger_block_index", "created_at_ms", ][..], ), @@ -3196,8 +3060,8 @@ fn validate_index_schema(conn: &Transaction<'_>) -> Result<(), String> { for index in [ "databases_active_mount_id_idx", "database_restore_chunks_database_id_idx", - "database_credit_ledger_database_idx", - "database_credit_pending_operations_database_idx", + "database_cycle_ledger_database_idx", + "database_cycle_pending_operations_database_idx", ] { if !tx_sqlite_master_entry_exists(conn, "index", index)? { return Err(format!("unsupported index schema: missing index {index}")); @@ -3247,6 +3111,39 @@ fn migration_applied_tx(conn: &Transaction<'_>, version: &str) -> Result Result, String> { + conn.query_row( + "SELECT name + FROM sqlite_master + WHERE type = 'table' + AND (name LIKE 'database_' || 'credit_%' + OR name = 'credits_' || 'config') + ORDER BY name + LIMIT 1", + params![], + |row| crate::sqlite::row_get::(row, 0), + ) + .optional() + .map_err(|error| error.to_string()) +} + +fn legacy_credit_index_table_name_tx(conn: &Transaction<'_>) -> Result, String> { + conn.query_row( + "SELECT name + FROM sqlite_master + WHERE type = 'table' + AND (name LIKE 'database_' || 'credit_%' + OR name = 'credits_' || 'config') + ORDER BY name + LIMIT 1", + params![], + |row| crate::sqlite::row_get::(row, 0), + ) + .optional() + .map_err(|error| error.to_string()) +} + #[cfg(not(target_arch = "wasm32"))] fn sqlite_master_entry_exists( conn: &Connection, @@ -3263,30 +3160,29 @@ fn sqlite_master_entry_exists( .map_err(|error| error.to_string()) } -fn load_credits_config(conn: &Connection) -> Result { - Ok(CreditsConfig { - kinic_ledger_canister_id: load_credits_config_text(conn, "kinic_ledger_canister_id")?, - sns_governance_id: load_credits_config_text(conn, "sns_governance_id")?, - credits_per_kinic: load_credits_config_u64(conn, "credits_per_kinic")?, - min_update_credits: load_credits_config_u64(conn, "min_update_credits")?, +fn load_cycles_billing_config(conn: &Connection) -> Result { + Ok(CyclesBillingConfig { + kinic_ledger_canister_id: load_cycles_billing_config_text( + conn, + "kinic_ledger_canister_id", + )?, + billing_authority_id: load_cycles_billing_config_text(conn, "billing_authority_id")?, + cycles_per_kinic: load_cycles_billing_config_u64(conn, "cycles_per_kinic")?, + min_update_cycles: load_cycles_billing_config_u64(conn, "min_update_cycles")?, }) } -fn load_credits_config_version(conn: &Connection) -> Result { - load_credits_config_u64(conn, "config_version") -} - -fn load_credits_config_text(conn: &Connection, key: &str) -> Result { +fn load_cycles_billing_config_text(conn: &Connection, key: &str) -> Result { conn.query_row( - "SELECT value FROM credits_config WHERE key = ?1", + "SELECT value FROM cycles_billing_config WHERE key = ?1", params![key], |row| crate::sqlite::row_get(row, 0), ) .map_err(|error| error.to_string()) } -fn load_credits_config_u64(conn: &Connection, key: &str) -> Result { - let value = load_credits_config_text(conn, key)?; +fn load_cycles_billing_config_u64(conn: &Connection, key: &str) -> Result { + let value = load_cycles_billing_config_text(conn, key)?; value.parse::().map_err(|error| error.to_string()) } @@ -3333,21 +3229,46 @@ fn amount_to_i64(amount: u64) -> Result { i64::try_from(amount).map_err(|_| "amount exceeds i64 limit".to_string()) } -fn credits_to_i64(credits: u64) -> Result { - let credits = i64::try_from(credits).map_err(|_| "credits exceeds i64 limit".to_string())?; - if credits <= 0 { - return Err("credit purchase credits must be positive".to_string()); +fn cycles_to_i64(cycles: u64) -> Result { + let cycles = i64::try_from(cycles).map_err(|_| "cycles exceeds i64 limit".to_string())?; + if cycles <= 0 { + return Err("cycles purchase cycles must be positive".to_string()); } - Ok(credits) + Ok(cycles) +} + +pub fn cycles_for_payment_amount_e8s( + payment_amount_e8s: u64, + config: &CyclesBillingConfig, +) -> Result { + if payment_amount_e8s == 0 { + return Err("cycles purchase payment amount must be positive".to_string()); + } + if config.cycles_per_kinic == 0 { + return Err("cycles_per_kinic must be positive".to_string()); + } + let cycles = u128::from(payment_amount_e8s) + .checked_mul(u128::from(config.cycles_per_kinic)) + .ok_or_else(|| "cycles purchase amount overflow".to_string())? + / u128::from(kinic_base_units_per_token()); + let cycles = + u64::try_from(cycles).map_err(|_| "cycles purchase amount exceeds u64".to_string())?; + if cycles == 0 { + return Err("cycles purchase amount is too small".to_string()); + } + Ok(cycles) } -pub fn payment_amount_e8s_for_credits(credits: u64, config: &CreditsConfig) -> Result { - let e8s_per_credit = KINIC_E8S_PER_TOKEN - .checked_div(config.credits_per_kinic) - .ok_or_else(|| "credits_per_kinic must be positive".to_string())?; - credits - .checked_mul(e8s_per_credit) - .ok_or_else(|| "credit purchase payment amount overflow".to_string()) +fn validate_cycles_purchase_minimum( + amount_cycles: u64, + min_expected_cycles: u64, +) -> Result<(), String> { + if amount_cycles < min_expected_cycles { + return Err(format!( + "cycles purchase quote changed: amount_cycles {amount_cycles} is below min_expected_cycles {min_expected_cycles}" + )); + } + Ok(()) } fn millis_to_nanos(value: i64) -> Result { @@ -3367,50 +3288,56 @@ fn checked_balance_add(balance: i64, amount: i64) -> Result { Ok(next) } -fn validate_database_credit_purchase_for_conn( +fn validate_database_cycles_purchase_for_conn( conn: &Connection, database_id: &str, - credits: i64, + cycles: i64, ) -> Result<(), String> { let status = load_database_status(conn, database_id)?; + if !matches!(status, DatabaseStatus::Pending | DatabaseStatus::Active) { + return Err(format!( + "database is {}: {database_id}", + status_to_db(status) + )); + } if !database_has_owner(conn, database_id)? { return Err(format!("database has no owner: {database_id}")); } let balance: i64 = conn .query_row( - "SELECT balance_credits FROM database_credit_accounts WHERE database_id = ?1", + "SELECT balance_cycles FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| crate::sqlite::row_get(row, 0), ) .optional() .map_err(|error| error.to_string())? - .ok_or_else(|| format!("database credits account not found: {database_id}"))?; - let pending_credit_purchase: i64 = conn + .ok_or_else(|| format!("database cycles account not found: {database_id}"))?; + let pending_cycles_purchase: i64 = conn .query_row( - "SELECT COALESCE(SUM(credits), 0) - FROM database_credit_pending_operations - WHERE database_id = ?1 AND kind = 'credit_purchase'", + "SELECT COALESCE(SUM(cycles), 0) + FROM database_cycle_pending_operations + WHERE database_id = ?1 AND kind = 'cycles_purchase'", params![database_id], |row| crate::sqlite::row_get(row, 0), ) .map_err(|error| error.to_string())?; - if status == DatabaseStatus::Pending && pending_credit_purchase > 0 { + if status == DatabaseStatus::Pending && pending_cycles_purchase > 0 { return Err(format!("database activation is pending: {database_id}")); } - let reserved = checked_balance_add(balance, pending_credit_purchase)?; - checked_balance_add(reserved, credits)?; + let reserved = checked_balance_add(balance, pending_cycles_purchase)?; + checked_balance_add(reserved, cycles)?; Ok(()) } -fn require_database_write_credits_available_for_conn( +fn require_database_write_cycles_available_for_conn( conn: &Connection, database_id: &str, - config: &CreditsConfig, + config: &CyclesBillingConfig, ) -> Result<(), String> { let (balance, suspended_at_ms): (i64, Option) = conn .query_row( - "SELECT balance_credits, suspended_at_ms - FROM database_credit_accounts + "SELECT balance_cycles, suspended_at_ms + FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| { @@ -3422,37 +3349,21 @@ fn require_database_write_credits_available_for_conn( ) .optional() .map_err(|error| error.to_string())? - .ok_or_else(|| format!("database credits account not found: {database_id}"))?; + .ok_or_else(|| format!("database cycles account not found: {database_id}"))?; if suspended_at_ms.is_some() { - return Err(format!("database credits are suspended: {database_id}")); + return Err(format!("database cycles are suspended: {database_id}")); } - if balance < credits_to_i64(config.min_update_credits)? { - return Err(format!( - "database credits balance is too low: {database_id}" - )); - } - Ok(()) -} - -fn purge_hard_deleted_database_rows(conn: &Connection) -> Result<(), String> { - let mut stmt = conn - .prepare("SELECT database_id FROM databases WHERE status = 'deleted'") - .map_err(|error| error.to_string())?; - let database_ids = crate::sqlite::query_map(&mut stmt, params![], |row| { - crate::sqlite::row_get::(row, 0) - }) - .map_err(|error| error.to_string())?; - for database_id in database_ids { - delete_database_index_rows(conn, &database_id)?; + if balance < cycles_to_i64(config.min_update_cycles)? { + return Err(format!("database cycles balance is too low: {database_id}")); } Ok(()) } fn delete_database_index_rows(conn: &Connection, database_id: &str) -> Result<(), String> { for table in [ - "database_credit_pending_operations", - "database_credit_ledger", - "database_credit_accounts", + "database_cycle_pending_operations", + "database_cycle_ledger", + "database_cycle_accounts", "database_members", "database_restore_chunks", "database_restore_sessions", @@ -3468,6 +3379,57 @@ fn delete_database_index_rows(conn: &Connection, database_id: &str) -> Result<() Ok(()) } +fn purge_expired_unstarted_pending_databases( + conn: &Transaction<'_>, + caller: &str, + now: i64, +) -> Result<(), String> { + let expires_before = now.saturating_sub(PENDING_DATABASE_TTL_MS); + let expired_database_ids = { + let mut stmt = conn + .prepare( + "SELECT d.database_id + FROM databases d + JOIN database_members m ON m.database_id = d.database_id + WHERE d.status = 'pending' + AND d.active_mount_id IS NULL + AND d.created_at_ms <= ?2 + AND m.principal = ?1 + AND m.role = 'owner' + AND NOT EXISTS ( + SELECT 1 + FROM database_cycle_pending_operations p + WHERE p.database_id = d.database_id + ) + ORDER BY d.created_at_ms ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map(&mut stmt, params![caller, expires_before], |row| { + crate::sqlite::row_get::(row, 0) + }) + .map_err(|error| error.to_string())? + }; + for database_id in expired_database_ids { + delete_database_index_rows(conn, &database_id)?; + } + Ok(()) +} + +fn pending_database_count_for_caller(conn: &Connection, caller: &str) -> Result { + conn.query_row( + "SELECT COUNT(*) + FROM databases d + JOIN database_members m ON m.database_id = d.database_id + WHERE d.status = 'pending' + AND d.active_mount_id IS NULL + AND m.principal = ?1 + AND m.role = 'owner'", + params![caller], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string()) +} + fn complete_pending_database_activation( conn: &Connection, database_id: &str, @@ -3497,28 +3459,36 @@ fn complete_pending_database_activation( params![database_id, now], ) .map_err(|error| error.to_string())?; + conn.execute( + "UPDATE database_cycle_accounts + SET storage_charged_at_ms = COALESCE(storage_charged_at_ms, ?2), + updated_at_ms = ?2 + WHERE database_id = ?1", + params![database_id, now], + ) + .map_err(|error| error.to_string())?; Ok(()) } fn database_balance_for_update(conn: &Transaction<'_>, database_id: &str) -> Result { conn.query_row( - "SELECT balance_credits FROM database_credit_accounts WHERE database_id = ?1", + "SELECT balance_cycles FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| crate::sqlite::row_get(row, 0), ) .optional() .map_err(|error| error.to_string())? - .ok_or_else(|| format!("database credits account not found: {database_id}")) + .ok_or_else(|| format!("database cycles account not found: {database_id}")) } -fn update_database_credits_balance( +fn update_database_cycles_balance( conn: &Transaction<'_>, database_id: &str, balance: i64, - config: &CreditsConfig, + config: &CyclesBillingConfig, now: i64, ) -> Result<(), String> { - let min = credits_to_i64(config.min_update_credits)?; + let min = cycles_to_i64(config.min_update_cycles)?; let suspended_at_ms = if balance >= min { None } else { Some(now) }; let values = vec![ crate::sqlite::text_value(database_id), @@ -3528,8 +3498,8 @@ fn update_database_credits_balance( ]; crate::sqlite::execute_values( conn, - "UPDATE database_credit_accounts - SET balance_credits = ?2, suspended_at_ms = ?3, updated_at_ms = ?4 + "UPDATE database_cycle_accounts + SET balance_cycles = ?2, suspended_at_ms = ?3, updated_at_ms = ?4 WHERE database_id = ?1", &values, ) @@ -3537,24 +3507,118 @@ fn update_database_credits_balance( Ok(()) } -struct PendingCreditsOperation { - operation_id: u64, +fn load_storage_cycle_account( + conn: &Connection, + database_id: &str, +) -> Result { + conn.query_row( + "SELECT balance_cycles, suspended_at_ms, storage_charged_at_ms + FROM database_cycle_accounts + WHERE database_id = ?1", + params![database_id], + |row| { + Ok(StorageCycleAccount { + balance_cycles: crate::sqlite::row_get(row, 0)?, + suspended_at_ms: crate::sqlite::row_get(row, 1)?, + storage_charged_at_ms: crate::sqlite::row_get(row, 2)?, + }) + }, + ) + .optional() + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("database cycles account not found: {database_id}")) +} + +fn update_database_logical_size( + conn: &Transaction<'_>, + database_id: &str, + size_bytes: u64, +) -> Result<(), String> { + conn.execute( + "UPDATE databases + SET logical_size_bytes = ?2 + WHERE database_id = ?1", + params![database_id, i64::try_from(size_bytes).unwrap_or(i64::MAX)], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn update_database_storage_account( + conn: &Transaction<'_>, + database_id: &str, + balance_cycles: i64, + suspended_at_ms: Option, + storage_charged_at_ms: i64, + now: i64, +) -> Result<(), String> { + let values = vec![ + crate::sqlite::text_value(database_id), + crate::sqlite::integer_value(balance_cycles), + crate::sqlite::nullable_integer_value(suspended_at_ms), + crate::sqlite::integer_value(storage_charged_at_ms), + crate::sqlite::integer_value(now), + ]; + crate::sqlite::execute_values( + conn, + "UPDATE database_cycle_accounts + SET balance_cycles = ?2, + suspended_at_ms = ?3, + storage_charged_at_ms = ?4, + updated_at_ms = ?5 + WHERE database_id = ?1", + &values, + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +struct PendingCyclesOperation { database_id: String, kind: String, caller: String, - credits: i64, + cycles: i64, payment_amount_e8s: i64, - from_owner: Option, - from_subaccount: Option>, - to_owner: Option, - to_subaccount: Option>, - ledger_fee_e8s: Option, - ledger_created_at_time_ns: Option, operation_status: String, + ledger_block_index: Option, +} + +struct DatabaseCyclesPendingPurchaseRaw { + operation_id: i64, + database_id: String, + caller: String, + status: String, + amount_cycles: i64, + payment_amount_e8s: i64, + ledger_block_index: Option, created_at_ms: i64, } -struct PendingCreditsLedgerDetails<'a> { +impl DatabaseCyclesPendingPurchaseRaw { + fn into_public(self) -> Result { + let amount_cycles = u64::try_from(self.amount_cycles).map_err(|error| error.to_string())?; + let payment_amount_e8s = + u64::try_from(self.payment_amount_e8s).map_err(|error| error.to_string())?; + let operation_id = u64::try_from(self.operation_id).map_err(|error| error.to_string())?; + let ledger_block_index = self + .ledger_block_index + .map(u64::try_from) + .transpose() + .map_err(|error| error.to_string())?; + Ok(DatabaseCyclesPendingPurchase { + operation_id, + database_id: self.database_id, + status: self.status.clone(), + amount_cycles, + payment_amount_e8s, + ledger_block_index, + created_at_ms: self.created_at_ms, + required_action: pending_cycles_required_action(&self.status).to_string(), + }) + } +} + +struct PendingCyclesLedgerDetails<'a> { from_owner: &'a str, from_subaccount: Option<&'a [u8]>, to_owner: &'a str, @@ -3563,34 +3627,34 @@ struct PendingCreditsLedgerDetails<'a> { ledger_created_at_time_ns: i64, } -struct PendingCreditsOperationInsert<'a> { +struct PendingCyclesOperationInsert<'a> { database_id: &'a str, kind: &'a str, caller: &'a str, - credits: i64, + cycles: i64, payment_amount_e8s: i64, - ledger: PendingCreditsLedgerDetails<'a>, + ledger: PendingCyclesLedgerDetails<'a>, operation_status: &'a str, now: i64, } -struct PendingCreditsOperationMatch<'a> { +struct PendingCyclesOperationMatch<'a> { operation_id: u64, database_id: &'a str, kind: &'a str, caller: &'a str, - credits: i64, + cycles: i64, } -fn insert_pending_credits_operation( +fn insert_pending_cycles_operation( conn: &Transaction<'_>, - operation: PendingCreditsOperationInsert<'_>, + operation: PendingCyclesOperationInsert<'_>, ) -> Result { let values = vec![ crate::sqlite::text_value(operation.database_id), crate::sqlite::text_value(operation.kind), crate::sqlite::text_value(operation.caller), - crate::sqlite::integer_value(operation.credits), + crate::sqlite::integer_value(operation.cycles), crate::sqlite::integer_value(operation.payment_amount_e8s), crate::sqlite::text_value(operation.ledger.from_owner), crate::sqlite::nullable_blob_value(operation.ledger.from_subaccount.map(Vec::from)), @@ -3603,8 +3667,8 @@ fn insert_pending_credits_operation( ]; crate::sqlite::execute_values( conn, - "INSERT INTO database_credit_pending_operations - (database_id, kind, caller, credits, payment_amount_e8s, from_owner, from_subaccount, + "INSERT INTO database_cycle_pending_operations + (database_id, kind, caller, cycles, payment_amount_e8s, from_owner, from_subaccount, to_owner, to_subaccount, ledger_fee_e8s, ledger_created_at_time_ns, operation_status, created_at_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", @@ -3615,27 +3679,27 @@ fn insert_pending_credits_operation( u64::try_from(operation_id).map_err(|error| error.to_string()) } -fn load_pending_credits_operation( - conn: &Transaction<'_>, +fn load_pending_cycles_operation( + conn: &Connection, operation_id: u64, -) -> Result { +) -> Result { let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; conn.query_row( - "SELECT operation_id, database_id, kind, caller, credits, payment_amount_e8s, - from_owner, from_subaccount, to_owner, to_subaccount, ledger_fee_e8s, - ledger_created_at_time_ns, operation_status, created_at_ms - FROM database_credit_pending_operations + "SELECT database_id, kind, caller, cycles, payment_amount_e8s, + from_owner, from_subaccount, to_owner, to_subaccount, + ledger_fee_e8s, ledger_created_at_time_ns, operation_status, ledger_block_index + FROM database_cycle_pending_operations WHERE operation_id = ?1", params![operation_id], - map_pending_credits_operation, + map_pending_cycles_operation, ) .optional() .map_err(|error| error.to_string())? - .ok_or_else(|| "pending credit operation not found".to_string()) + .ok_or_else(|| "pending cycle operation not found".to_string()) } fn require_pending_operation_status( - operation: &PendingCreditsOperation, + operation: &PendingCyclesOperation, allowed: &[&str], action: &str, ) -> Result<(), String> { @@ -3646,132 +3710,151 @@ fn require_pending_operation_status( return Ok(()); } Err(format!( - "cannot {action}; credit purchase operation is {}", + "cannot {action}; cycle purchase operation is {}", operation.operation_status )) } -fn require_pending_database_kind( - operation: &PendingCreditsOperation, - database_id: &str, - kind: &str, -) -> Result<(), String> { - if operation.database_id != database_id || operation.kind != kind { - return Err("pending credit operation mismatch".to_string()); - } - Ok(()) -} - -fn load_required_pending_credits_operation( +fn load_required_pending_cycles_operation( conn: &Transaction<'_>, - expected: PendingCreditsOperationMatch<'_>, -) -> Result { - let operation = load_pending_credits_operation(conn, expected.operation_id)?; + expected: PendingCyclesOperationMatch<'_>, +) -> Result { + let operation = load_pending_cycles_operation(conn, expected.operation_id)?; if operation.database_id != expected.database_id || operation.kind != expected.kind || operation.caller != expected.caller - || operation.credits != expected.credits + || operation.cycles != expected.cycles { - return Err("pending credit operation mismatch".to_string()); + return Err("pending cycle operation mismatch".to_string()); } Ok(operation) } -fn delete_pending_credits_operation( +fn delete_pending_cycles_operation( conn: &Transaction<'_>, operation_id: u64, ) -> Result<(), String> { let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; conn.execute( - "DELETE FROM database_credit_pending_operations WHERE operation_id = ?1", + "DELETE FROM database_cycle_pending_operations WHERE operation_id = ?1", params![operation_id], ) .map_err(|error| error.to_string())?; Ok(()) } -fn update_pending_credits_operation_status( - conn: &Transaction<'_>, - operation_id: u64, - status: &str, +fn ensure_no_pending_cycles_purchase_for_caller( + conn: &Connection, + database_id: &str, + caller: &str, ) -> Result<(), String> { - let operation_id = i64::try_from(operation_id).map_err(|error| error.to_string())?; - conn.execute( - "UPDATE database_credit_pending_operations - SET operation_status = ?2 - WHERE operation_id = ?1", - params![operation_id, status], - ) - .map_err(|error| error.to_string())?; + let count: i64 = conn + .query_row( + "SELECT COUNT(*) + FROM database_cycle_pending_operations + WHERE database_id = ?1 + AND caller = ?2 + AND kind = 'cycles_purchase'", + params![database_id, caller], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + if count > 0 { + return Err("cycles purchase already pending for caller".to_string()); + } Ok(()) } -fn map_pending_credits_operation( +fn load_database_cycles_pending_purchase_statuses( + conn: &Connection, + database_id: &str, +) -> Result, String> { + let mut stmt = conn + .prepare( + "SELECT operation_id, database_id, caller, operation_status, cycles, + payment_amount_e8s, ledger_block_index, created_at_ms + FROM database_cycle_pending_operations + WHERE database_id = ?1 AND kind = 'cycles_purchase' + ORDER BY operation_id ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map( + &mut stmt, + params![database_id], + map_database_cycles_pending_purchase_raw, + ) + .map_err(|error| error.to_string()) +} + +fn first_database_cycles_pending_purchase_status( + conn: &Connection, + database_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT operation_id, database_id, caller, operation_status, cycles, + payment_amount_e8s, ledger_block_index, created_at_ms + FROM database_cycle_pending_operations + WHERE database_id = ?1 AND kind = 'cycles_purchase' + ORDER BY operation_id ASC + LIMIT 1", + params![database_id], + map_database_cycles_pending_purchase_raw, + ) + .optional() + .map_err(|error| error.to_string())? + .map(DatabaseCyclesPendingPurchaseRaw::into_public) + .transpose() +} + +fn map_database_cycles_pending_purchase_raw( row: &crate::sqlite::Row<'_>, -) -> crate::sqlite::Result { - let operation_id: i64 = crate::sqlite::row_get(row, 0)?; - Ok(PendingCreditsOperation { - operation_id: operation_id.max(0) as u64, +) -> crate::sqlite::Result { + Ok(DatabaseCyclesPendingPurchaseRaw { + operation_id: crate::sqlite::row_get(row, 0)?, database_id: crate::sqlite::row_get(row, 1)?, - kind: crate::sqlite::row_get(row, 2)?, - caller: crate::sqlite::row_get(row, 3)?, - credits: crate::sqlite::row_get(row, 4)?, + caller: crate::sqlite::row_get(row, 2)?, + status: crate::sqlite::row_get(row, 3)?, + amount_cycles: crate::sqlite::row_get(row, 4)?, payment_amount_e8s: crate::sqlite::row_get(row, 5)?, - from_owner: crate::sqlite::row_get(row, 6)?, - from_subaccount: crate::sqlite::row_get(row, 7)?, - to_owner: crate::sqlite::row_get(row, 8)?, - to_subaccount: crate::sqlite::row_get(row, 9)?, - ledger_fee_e8s: crate::sqlite::row_get(row, 10)?, - ledger_created_at_time_ns: crate::sqlite::row_get(row, 11)?, - operation_status: crate::sqlite::row_get(row, 12)?, - created_at_ms: crate::sqlite::row_get(row, 13)?, + ledger_block_index: crate::sqlite::row_get(row, 6)?, + created_at_ms: crate::sqlite::row_get(row, 7)?, }) } -fn pending_credits_operation_to_public( - operation: PendingCreditsOperation, -) -> Result { - Ok(DatabaseCreditPendingOperation { - operation_id: operation.operation_id, - database_id: operation.database_id, - kind: operation.kind, - caller: operation.caller, - operation_status: operation.operation_status, - credits: non_negative_i64_to_u64(operation.credits, "credits")?, - payment_amount_e8s: non_negative_i64_to_u64( - operation.payment_amount_e8s, - "payment_amount_e8s", - )?, - from_owner: operation.from_owner, - from_subaccount: operation.from_subaccount, - to_owner: operation.to_owner, - to_subaccount: operation.to_subaccount, - ledger_fee_e8s: operation - .ledger_fee_e8s - .map(|value| non_negative_i64_to_u64(value, "ledger_fee_e8s")) - .transpose()?, - ledger_created_at_time_ns: operation - .ledger_created_at_time_ns - .map(|value| non_negative_i64_to_u64(value, "ledger_created_at_time_ns")) - .transpose()?, - created_at_ms: operation.created_at_ms, - }) +fn pending_cycles_required_action(status: &str) -> &'static str { + match status { + CYCLES_OPERATION_STATUS_IN_FLIGHT => "wait_for_ledger_result", + CYCLES_OPERATION_STATUS_AMBIGUOUS | CYCLES_OPERATION_STATUS_COMPLETED => { + "billing_authority_review" + } + _ => "billing_authority_review", + } } -fn non_negative_i64_to_u64(value: i64, field: &str) -> Result { - u64::try_from(value).map_err(|_| format!("pending operation {field} is negative")) +fn map_pending_cycles_operation( + row: &crate::sqlite::Row<'_>, +) -> crate::sqlite::Result { + Ok(PendingCyclesOperation { + database_id: crate::sqlite::row_get(row, 0)?, + kind: crate::sqlite::row_get(row, 1)?, + caller: crate::sqlite::row_get(row, 2)?, + cycles: crate::sqlite::row_get(row, 3)?, + payment_amount_e8s: crate::sqlite::row_get(row, 4)?, + operation_status: crate::sqlite::row_get(row, 11)?, + ledger_block_index: crate::sqlite::row_get(row, 12)?, + }) } struct DatabaseLedgerInsert<'a> { database_id: &'a str, kind: &'a str, - amount_credits: i64, - balance_after_credits: i64, + amount_cycles: i64, + balance_after_cycles: i64, payment_amount_e8s: Option, caller: &'a str, method: Option<&'a str>, cycles_delta: Option, - config: Option<&'a CreditsConfig>, + config: Option<&'a CyclesBillingConfig>, ledger_block_index: Option, now: i64, } @@ -3782,10 +3865,24 @@ struct DatabaseCharge<'a> { method: &'a str, cycles_delta: u128, now: i64, - config: &'a CreditsConfig, + config: &'a CyclesBillingConfig, computed_charge: i64, } +struct StorageChargeInput<'a> { + database_id: &'a str, + caller: &'a str, + size_bytes: u64, + now: i64, + config: &'a CyclesBillingConfig, +} + +struct StorageCycleAccount { + balance_cycles: i64, + suspended_at_ms: Option, + storage_charged_at_ms: Option, +} + fn insert_database_ledger( conn: &Transaction<'_>, entry: DatabaseLedgerInsert<'_>, @@ -3793,8 +3890,8 @@ fn insert_database_ledger( let values = vec![ crate::sqlite::text_value(entry.database_id), crate::sqlite::text_value(entry.kind), - crate::sqlite::integer_value(entry.amount_credits), - crate::sqlite::integer_value(entry.balance_after_credits), + crate::sqlite::integer_value(entry.amount_cycles), + crate::sqlite::integer_value(entry.balance_after_cycles), crate::sqlite::nullable_integer_value(entry.payment_amount_e8s), crate::sqlite::text_value(entry.caller), entry @@ -3809,7 +3906,7 @@ fn insert_database_ledger( crate::sqlite::nullable_integer_value( entry .config - .map(|config| i64::try_from(config.credits_per_kinic).unwrap_or(i64::MAX)), + .map(|config| i64::try_from(config.cycles_per_kinic).unwrap_or(i64::MAX)), ), crate::sqlite::nullable_integer_value( entry @@ -3820,9 +3917,9 @@ fn insert_database_ledger( ]; crate::sqlite::execute_values( conn, - "INSERT INTO database_credit_ledger - (database_id, kind, amount_credits, balance_after_credits, payment_amount_e8s, - caller, method, cycles_delta, credits_per_kinic, ledger_block_index, created_at_ms) + "INSERT INTO database_cycle_ledger + (database_id, kind, amount_cycles, balance_after_cycles, payment_amount_e8s, + caller, method, cycles_delta, cycles_per_kinic, ledger_block_index, created_at_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", &values, ) @@ -3830,21 +3927,114 @@ fn insert_database_ledger( Ok(()) } +fn settle_database_storage_charge_in_tx( + tx: &Transaction<'_>, + input: StorageChargeInput<'_>, +) -> Result<(), String> { + let account = load_storage_cycle_account(tx, input.database_id)?; + update_database_logical_size(tx, input.database_id, input.size_bytes)?; + let Some(charged_at_ms) = account.storage_charged_at_ms else { + update_database_storage_account( + tx, + input.database_id, + account.balance_cycles, + account.suspended_at_ms, + input.now, + input.now, + )?; + return Ok(()); + }; + let elapsed_ms = input.now.saturating_sub(charged_at_ms); + if elapsed_ms < STORAGE_BILLING_INTERVAL_MS { + return Ok(()); + } + let storage_cycles = compute_storage_charge_cycles(input.size_bytes, elapsed_ms)?; + if storage_cycles == 0 { + update_database_storage_account( + tx, + input.database_id, + account.balance_cycles, + account.suspended_at_ms, + input.now, + input.now, + )?; + return Ok(()); + } + let charge_cycles = i64::try_from(storage_cycles) + .map_err(|_| "storage charge exceeds i64 limit".to_string())?; + + let paid_cycles = account.balance_cycles.min(charge_cycles).max(0); + let next_balance = account.balance_cycles.saturating_sub(paid_cycles); + let min_balance = cycles_to_i64(input.config.min_update_cycles)?; + let should_suspend = paid_cycles < charge_cycles || next_balance < min_balance; + let suspended_at_ms = if should_suspend { + account.suspended_at_ms.or(Some(input.now)) + } else { + None + }; + let newly_suspended = should_suspend && account.suspended_at_ms.is_none(); + update_database_storage_account( + tx, + input.database_id, + next_balance, + suspended_at_ms, + input.now, + input.now, + )?; + if paid_cycles > 0 { + insert_database_ledger( + tx, + DatabaseLedgerInsert { + database_id: input.database_id, + kind: "storage_charge", + amount_cycles: -paid_cycles, + balance_after_cycles: next_balance, + payment_amount_e8s: None, + caller: input.caller, + method: Some("storage_billing"), + cycles_delta: Some(storage_cycles), + config: Some(input.config), + ledger_block_index: None, + now: input.now, + }, + )?; + } + if newly_suspended { + insert_database_ledger( + tx, + DatabaseLedgerInsert { + database_id: input.database_id, + kind: "suspend", + amount_cycles: 0, + balance_after_cycles: next_balance, + payment_amount_e8s: None, + caller: input.caller, + method: Some("storage_billing"), + cycles_delta: Some(storage_cycles), + config: Some(input.config), + ledger_block_index: None, + now: input.now, + }, + )?; + } + Ok(()) +} + fn charge_database_update_in_tx( tx: &Transaction<'_>, charge: DatabaseCharge<'_>, ) -> Result<(), String> { let balance = database_balance_for_update(tx, charge.database_id)?; - let amount = balance.min(charge.computed_charge); - let next = balance - amount; - update_database_credits_balance(tx, charge.database_id, next, charge.config, charge.now)?; + let paid_cycles = balance.max(0).min(charge.computed_charge); + let next = balance.max(0) - paid_cycles; + update_database_cycles_balance(tx, charge.database_id, next, charge.config, charge.now)?; insert_database_ledger( tx, DatabaseLedgerInsert { database_id: charge.database_id, kind: "charge", - amount_credits: -amount, - balance_after_credits: next, + amount_cycles: -paid_cycles, + balance_after_cycles: next, payment_amount_e8s: None, caller: charge.caller, method: Some(charge.method), @@ -3854,56 +4044,52 @@ fn charge_database_update_in_tx( now: charge.now, }, )?; - if charge.computed_charge > balance { - insert_database_ledger( - tx, - DatabaseLedgerInsert { - database_id: charge.database_id, - kind: "suspend", - amount_credits: 0, - balance_after_credits: next, - payment_amount_e8s: None, - caller: charge.caller, - method: Some(charge.method), - cycles_delta: Some(charge.cycles_delta), - config: Some(charge.config), - ledger_block_index: None, - now: charge.now, - }, - )?; - } Ok(()) } fn compute_update_charge(cycles_delta: u128) -> Result { - let charge = cycles_delta.div_ceil(CYCLES_PER_CREDIT); - i64::try_from(charge).map_err(|_| "cycle charge exceeds i64 limit".to_string()) + i64::try_from(cycles_delta).map_err(|_| "cycle charge exceeds i64 limit".to_string()) +} + +fn compute_storage_charge_cycles(size_bytes: u64, elapsed_ms: i64) -> Result { + if elapsed_ms <= 0 || size_bytes == 0 { + return Ok(0); + } + let elapsed_seconds = u128::try_from(elapsed_ms / 1000) + .map_err(|_| "storage billing elapsed time is negative".to_string())?; + let byte_seconds = u128::from(size_bytes) + .checked_mul(elapsed_seconds) + .ok_or_else(|| "storage byte seconds overflow".to_string())?; + byte_seconds + .checked_mul(STORAGE_CYCLES_PER_GIB_SECOND) + .ok_or_else(|| "storage charge cycles overflow".to_string()) + .map(|cycles| cycles / GIB_BYTES) } fn page_limit(limit: u32) -> u32 { limit.clamp(1, 100) } -fn map_database_credits_entry( +fn map_database_cycles_entry( row: &crate::sqlite::Row<'_>, -) -> crate::sqlite::Result { +) -> crate::sqlite::Result { let entry_id: i64 = crate::sqlite::row_get(row, 0)?; let balance_after: i64 = crate::sqlite::row_get(row, 4)?; let payment_amount_e8s: Option = crate::sqlite::row_get(row, 5)?; let cycles_delta: Option = crate::sqlite::row_get(row, 8)?; - let credits_per_kinic: Option = crate::sqlite::row_get(row, 9)?; + let cycles_per_kinic: Option = crate::sqlite::row_get(row, 9)?; let ledger_block_index: Option = crate::sqlite::row_get(row, 10)?; - Ok(DatabaseCreditEntry { + Ok(DatabaseCycleEntry { entry_id: entry_id.max(0) as u64, database_id: crate::sqlite::row_get(row, 1)?, kind: crate::sqlite::row_get(row, 2)?, - amount_credits: crate::sqlite::row_get(row, 3)?, - balance_after_credits: balance_after.max(0) as u64, + amount_cycles: crate::sqlite::row_get(row, 3)?, + balance_after_cycles: balance_after.max(0) as u64, payment_amount_e8s: payment_amount_e8s.map(|value| value.max(0) as u64), caller: crate::sqlite::row_get(row, 6)?, method: crate::sqlite::row_get(row, 7)?, cycles_delta: cycles_delta.map(|value| value.max(0) as u64), - credits_per_kinic: credits_per_kinic.map(|value| value.max(0) as u64), + cycles_per_kinic: cycles_per_kinic.map(|value| value.max(0) as u64), ledger_block_index: ledger_block_index.map(|value| value.max(0) as u64), created_at_ms: crate::sqlite::row_get(row, 11)?, }) @@ -4029,9 +4215,9 @@ fn parse_frontmatter_fields(content: &str) -> Result Result Option { +fn frontmatter_scalar(value: &str) -> Result, String> { if value == "null" || value == "~" { - return None; + return Ok(None); } if value.len() >= 2 && value.starts_with('"') && value.ends_with('"') { - return Some(value[1..value.len() - 1].to_string()); + return parse_json_string_literal(value).map(Some); } if value.len() >= 2 && value.starts_with('\'') && value.ends_with('\'') { - return Some(value[1..value.len() - 1].to_string()); + return Ok(Some(value[1..value.len() - 1].replace("''", "'"))); } - Some(value.to_string()) + Ok(Some(value.to_string())) +} + +fn frontmatter_end(rest: &str) -> Option { + rest.find("\n---\n").or_else(|| { + rest.ends_with("\n---") + .then_some(rest.len() - "\n---".len()) + }) +} + +fn parse_json_string_literal(value: &str) -> Result { + let body = value + .strip_prefix('"') + .and_then(|inner| inner.strip_suffix('"')) + .ok_or_else(|| "url ingest request frontmatter quoted scalar is invalid".to_string())?; + let mut chars = body.chars(); + let mut decoded = String::new(); + while let Some(ch) = chars.next() { + if ch == '\\' { + let escaped = chars.next().ok_or_else(invalid_quoted_scalar)?; + decode_json_escape(escaped, &mut chars, &mut decoded)?; + continue; + } + if ch.is_control() { + return Err(invalid_quoted_scalar()); + } + decoded.push(ch); + } + Ok(decoded) +} + +fn decode_json_escape( + escaped: char, + chars: &mut std::str::Chars<'_>, + decoded: &mut String, +) -> Result<(), String> { + match escaped { + '"' => decoded.push('"'), + '\\' => decoded.push('\\'), + '/' => decoded.push('/'), + 'b' => decoded.push('\u{0008}'), + 'f' => decoded.push('\u{000c}'), + 'n' => decoded.push('\n'), + 'r' => decoded.push('\r'), + 't' => decoded.push('\t'), + 'u' => { + let code = parse_json_hex4(chars)?; + if (0xD800..=0xDBFF).contains(&code) { + let slash = chars.next().ok_or_else(invalid_quoted_scalar)?; + let marker = chars.next().ok_or_else(invalid_quoted_scalar)?; + if slash != '\\' || marker != 'u' { + return Err(invalid_quoted_scalar()); + } + let low = parse_json_hex4(chars)?; + if !(0xDC00..=0xDFFF).contains(&low) { + return Err(invalid_quoted_scalar()); + } + let scalar = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + decoded.push(char::from_u32(scalar).ok_or_else(invalid_quoted_scalar)?); + } else if (0xDC00..=0xDFFF).contains(&code) { + return Err(invalid_quoted_scalar()); + } else { + decoded.push(char::from_u32(code).ok_or_else(invalid_quoted_scalar)?); + } + } + _ => return Err(invalid_quoted_scalar()), + } + Ok(()) +} + +fn parse_json_hex4(chars: &mut std::str::Chars<'_>) -> Result { + let mut code = 0u32; + for _ in 0..4 { + code *= 16; + code += chars + .next() + .and_then(|ch| ch.to_digit(16)) + .ok_or_else(invalid_quoted_scalar)?; + } + Ok(code) +} + +fn invalid_quoted_scalar() -> String { + "url ingest request frontmatter quoted scalar is invalid".to_string() } fn expect_frontmatter( @@ -4483,7 +4752,21 @@ fn load_databases(conn: &Connection) -> Result, String> { let mut stmt = conn.prepare( "SELECT database_id, name, db_file_name, active_mount_id, schema_version, logical_size_bytes, status FROM databases - WHERE status IN ('pending', 'active', 'archiving', 'restoring') AND active_mount_id IS NOT NULL + WHERE status IN ('pending', 'active', 'archiving', 'archived', 'restoring') AND active_mount_id IS NOT NULL + ORDER BY mount_id ASC", + ) + .map_err(|error| error.to_string())?; + crate::sqlite::query_map(&mut stmt, params![], map_database_meta) + .map_err(|error| error.to_string()) +} + +fn load_active_databases_for_storage_billing( + conn: &Connection, +) -> Result, String> { + let mut stmt = conn.prepare( + "SELECT database_id, name, db_file_name, active_mount_id, schema_version, logical_size_bytes, status + FROM databases + WHERE active_mount_id IS NOT NULL ORDER BY mount_id ASC", ) .map_err(|error| error.to_string())?; @@ -4524,26 +4807,26 @@ fn load_database_summaries_for_caller( let mut stmt = conn .prepare( "SELECT d.database_id, d.name, d.status, m.role, d.logical_size_bytes, - COALESCE(b.balance_credits, 0), b.suspended_at_ms, + COALESCE(b.balance_cycles, 0), b.suspended_at_ms, d.archived_at_ms FROM databases d INNER JOIN database_members m ON m.database_id = d.database_id - LEFT JOIN database_credit_accounts b ON b.database_id = d.database_id + LEFT JOIN database_cycle_accounts b ON b.database_id = d.database_id WHERE m.principal = ?1 ORDER BY d.database_id ASC", ) .map_err(|error| error.to_string())?; crate::sqlite::query_map(&mut stmt, params![caller], |row| { let logical_size_bytes: i64 = crate::sqlite::row_get(row, 4)?; - let credits_balance: i64 = crate::sqlite::row_get(row, 5)?; + let cycles_balance: i64 = crate::sqlite::row_get(row, 5)?; Ok(DatabaseSummary { database_id: crate::sqlite::row_get(row, 0)?, name: crate::sqlite::row_get(row, 1)?, status: status_from_db(&crate::sqlite::row_get::(row, 2)?)?, role: role_from_db(&crate::sqlite::row_get::(row, 3)?)?, logical_size_bytes: logical_size_bytes.max(0) as u64, - credits_balance: Some(credits_balance.max(0) as u64), - credits_suspended_at_ms: crate::sqlite::row_get(row, 6)?, + cycles_balance: Some(cycles_balance.max(0) as u64), + cycles_suspended_at_ms: crate::sqlite::row_get(row, 6)?, archived_at_ms: crate::sqlite::row_get(row, 7)?, }) }) @@ -4594,6 +4877,30 @@ fn load_member_role( .map_err(|error| error.to_string()) } +fn database_member_exists( + conn: &Connection, + database_id: &str, + principal: &str, +) -> Result { + conn.query_row( + "SELECT 1 FROM database_members WHERE database_id = ?1 AND principal = ?2", + params![database_id, principal], + |row| crate::sqlite::row_get::(row, 0), + ) + .optional() + .map_err(|error| error.to_string()) + .map(|value| value.is_some()) +} + +fn database_member_count_for_conn(conn: &Connection, database_id: &str) -> Result { + conn.query_row( + "SELECT COUNT(*) FROM database_members WHERE database_id = ?1", + params![database_id], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string()) +} + fn role_from_db(role: &str) -> crate::sqlite::Result { match role { "owner" => Ok(DatabaseRole::Owner), @@ -4658,16 +4965,60 @@ mod tests { use super::*; - fn test_credits_config() -> CreditsConfig { - CreditsConfig { + #[test] + fn url_ingest_frontmatter_requires_whole_line_terminator() { + let fields = parse_frontmatter_fields( + "---\nkind: \"kinic.url_ingest_request\"\nstatus: queued\nnote: ---not-a-terminator\nrequested_by: alice\n---\n# Body\n", + ) + .expect("frontmatter should parse at the real terminator"); + + assert_eq!( + fields.get("kind").and_then(|value| value.as_deref()), + Some("kinic.url_ingest_request") + ); + assert_eq!( + fields + .get("requested_by") + .and_then(|value| value.as_deref()), + Some("alice") + ); + } + + #[test] + fn url_ingest_frontmatter_unescapes_json_quoted_scalars() { + let fields = parse_frontmatter_fields( + "---\nkind: kinic.url_ingest_request\nrequested_by: \"principal-\\\"1\\\"-\\uD83D\\uDE00\"\n---\n# Body\n", + ) + .expect("frontmatter should parse quoted scalars"); + + assert_eq!( + fields + .get("requested_by") + .and_then(|value| value.as_deref()), + Some("principal-\"1\"-😀") + ); + } + + #[test] + fn url_ingest_frontmatter_rejects_invalid_json_quoted_scalars() { + let error = parse_frontmatter_fields( + "---\nkind: kinic.url_ingest_request\nrequested_by: \"principal-\\q\"\n---\n# Body\n", + ) + .expect_err("invalid JSON escape must not be accepted as a raw quoted value"); + + assert!(error.contains("quoted scalar")); + } + + fn test_cycles_billing_config() -> CyclesBillingConfig { + CyclesBillingConfig { kinic_ledger_canister_id: "aaaaa-aa".to_string(), - sns_governance_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), - credits_per_kinic: DEFAULT_CREDITS_PER_KINIC, - min_update_credits: DEFAULT_MIN_UPDATE_CREDITS, + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai".to_string(), + cycles_per_kinic: DEFAULT_CYCLES_PER_KINIC, + min_update_cycles: DEFAULT_MIN_UPDATE_CYCLES, } } - fn write_pre_credits_schema(index_path: &Path) { + fn write_pre_cycles_schema(index_path: &Path) { let conn = Connection::open(index_path).expect("index DB should open"); conn.execute_batch( "CREATE TABLE schema_migrations ( @@ -4780,48 +5131,48 @@ mod tests { FOREIGN KEY (database_id) REFERENCES databases(database_id) );", ) - .expect("pre-credits schema should write"); + .expect("pre-cycles schema should write"); } #[test] fn old_upgrade_migrations_require_config() { let dir = tempdir().expect("tempdir should create"); let index_path = dir.path().join("index.sqlite3"); - write_pre_credits_schema(&index_path); + write_pre_cycles_schema(&index_path); let service = VfsService::new(index_path, dir.path().join("databases")); let error = service .run_index_migrations_for_upgrade(None) .expect_err("old index should require config"); - assert!(error.contains("credits config required for first credits upgrade")); + assert!(error.contains("cycles config required for first cycles upgrade")); } #[test] fn old_upgrade_migrations_apply_with_config() { let dir = tempdir().expect("tempdir should create"); let index_path = dir.path().join("index.sqlite3"); - write_pre_credits_schema(&index_path); + write_pre_cycles_schema(&index_path); let service = VfsService::new(index_path.clone(), dir.path().join("databases")); - let config = test_credits_config(); + let config = test_cycles_billing_config(); service .run_index_migrations_for_upgrade(Some(config.clone())) .expect("old index should upgrade"); assert_eq!( - service.credits_config().expect("config should load"), + service.cycles_billing_config().expect("config should load"), config ); let conn = Connection::open(&index_path).expect("index DB should reopen"); let marker: String = conn .query_row( "SELECT version FROM schema_migrations - WHERE version = 'database_index:018_credit_ledger_only'", + WHERE version = 'database_index:018_cycles_ledger_only'", params![], |row| row.get(0), ) - .expect("credit ledger only marker should exist"); + .expect("cycle ledger only marker should exist"); let usage_table_count: i64 = conn .query_row( "SELECT COUNT(*) FROM sqlite_master @@ -4830,153 +5181,129 @@ mod tests { |row| row.get(0), ) .expect("usage table count should load"); - assert_eq!(marker, "database_index:018_credit_ledger_only"); + assert_eq!(marker, "database_index:018_cycles_ledger_only"); assert_eq!(usage_table_count, 0); } #[test] - fn upgrade_migrations_accept_no_config_after_credits_initial() { + fn upgrade_migrations_accept_no_config_after_cycles_initial() { let dir = tempdir().expect("tempdir should create"); let service = VfsService::new( dir.path().join("index.sqlite3"), dir.path().join("databases"), ); - let config = test_credits_config(); + let config = test_cycles_billing_config(); service .run_index_migrations_with_config(config.clone()) .expect("initial migrations should run"); service .run_index_migrations_for_upgrade(None) - .expect("post-credits upgrade should not need config"); + .expect("post-cycles upgrade should not need config"); assert_eq!( - service.credits_config().expect("config should load"), + service.cycles_billing_config().expect("config should load"), config ); } #[test] - fn upgrade_migration_marks_existing_pending_credit_operations_ambiguous() { + fn partial_billing_schema_is_rejected_for_upgrade() { let dir = tempdir().expect("tempdir should create"); let index_path = dir.path().join("index.sqlite3"); - let conn = Connection::open(&index_path).expect("index DB should open"); - conn.execute( - "CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL)", - params![], - ) - .expect("schema_migrations should create"); - conn.execute_batch( - &FRESH_INDEX_SCHEMA_SQL.replace(" operation_status TEXT NOT NULL,\n", ""), - ) - .expect("pre-status schema should create"); - for version in INDEX_SCHEMA_VERSIONS - .iter() - .copied() - .filter(|version| *version != INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS) - { - conn.execute( - "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, 0)", - params![version], - ) - .expect("migration marker should insert"); - } - let config = test_credits_config(); - conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params![ - "kinic_ledger_canister_id", - config.kinic_ledger_canister_id.as_str() - ], - ) - .expect("ledger config should insert"); - conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params!["sns_governance_id", config.sns_governance_id.as_str()], - ) - .expect("governance config should insert"); - conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params!["credits_per_kinic", config.credits_per_kinic.to_string()], - ) - .expect("rate config should insert"); - conn.execute( - "INSERT INTO credits_config (key, value) VALUES (?1, ?2)", - params!["min_update_credits", config.min_update_credits.to_string()], - ) - .expect("minimum config should insert"); - conn.execute( - "INSERT INTO credits_config (key, value) VALUES ('config_version', '1')", - params![], - ) - .expect("version config should insert"); - conn.execute( - "INSERT INTO databases - (database_id, name, db_file_name, mount_id, active_mount_id, status, schema_version, - logical_size_bytes, created_at_ms, updated_at_ms) - VALUES ('db_old', 'Old', '', 0, NULL, 'pending', ?1, 0, 1, 1)", - params![DATABASE_SCHEMA_VERSION], - ) - .expect("database should insert"); - conn.execute( - "INSERT INTO database_credit_accounts - (database_id, balance_credits, suspended_at_ms, created_at_ms, updated_at_ms) - VALUES ('db_old', 0, 1, 1, 1)", - params![], - ) - .expect("account should insert"); + write_pre_cycles_schema(&index_path); + let conn = Connection::open(&index_path).expect("index DB should reopen"); + let legacy_marker = format!("database_index:020_{}config_version", "credits_"); conn.execute( - "INSERT INTO database_credit_pending_operations - (database_id, kind, caller, credits, payment_amount_e8s, from_owner, from_subaccount, - to_owner, to_subaccount, ledger_fee_e8s, ledger_created_at_time_ns, created_at_ms) - VALUES ('db_old', 'credit_purchase', '2vxsx-fae', 10, 1000, '2vxsx-fae', NULL, - 'aaaaa-aa', NULL, 10000, 1700000000000000000, 1)", - params![], + "INSERT INTO schema_migrations (version, applied_at) + VALUES (?1, 0)", + params![legacy_marker], ) - .expect("legacy pending operation should insert"); + .expect("legacy billing marker should insert"); drop(conn); - let service = VfsService::new(index_path.clone(), dir.path().join("databases")); + let service = VfsService::new(index_path, dir.path().join("databases")); + + let error = service + .run_index_migrations_for_upgrade(Some(test_cycles_billing_config())) + .expect_err("partial billing schema should be unsupported"); + + assert!(error.contains("unsupported partial billing index schema")); + assert!(error.contains("database_index:020_")); + } + #[test] + fn apply_database_cycles_purchase_rejects_in_flight_operation() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); service - .run_index_migrations_for_upgrade(None) - .expect("status migration should apply"); + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("default", "2vxsx-fae", 1) + .expect("database should create"); + let operation_id = service + .begin_database_cycles_purchase("default", "2vxsx-fae", 1_000_000, 2) + .expect("cycle purchase should begin"); + let cycles = cycles_for_payment_amount_e8s( + 1_000_000, + &service.cycles_billing_config().expect("config should load"), + ) + .expect("cycles should compute"); - let conn = Connection::open(&index_path).expect("index DB should reopen"); - let status: String = conn - .query_row( - "SELECT operation_status FROM database_credit_pending_operations WHERE database_id = 'db_old'", - params![], - |row| row.get(0), - ) - .expect("operation status should load"); - let marker: String = conn - .query_row( - "SELECT version FROM schema_migrations WHERE version = ?1", - params![INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS], - |row| row.get(0), - ) - .expect("status migration marker should exist"); - assert_eq!(status, CREDIT_OPERATION_STATUS_AMBIGUOUS); - assert_eq!(marker, INDEX_SCHEMA_VERSION_CREDIT_PENDING_OPERATION_STATUS); - drop(conn); + let error = service + .apply_database_cycles_purchase(operation_id, "default", "2vxsx-fae", cycles, 1, 2) + .expect_err("in-flight operation must not apply before ledger completion"); + assert!(error.contains("cycle purchase operation is in_flight")); + } + + #[test] + fn ambiguous_database_cycles_purchase_blocks_duplicate_until_repair() { + let dir = tempdir().expect("tempdir should create"); + let index_path = dir.path().join("index.sqlite3"); + let service = VfsService::new(index_path.clone(), dir.path().join("databases")); service - .repair_database_credit_purchase_cancel("db_old", 1, "2vxsx-fae", 2) - .expect("migrated ambiguous operation should remain cancellable"); + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("default", "payer", 1) + .expect("database should create"); + let operation_id = service + .begin_database_cycles_purchase("default", "payer", 1_000_000, 2) + .expect("cycle purchase should begin"); + let cycles = cycles_for_payment_amount_e8s( + 1_000_000, + &service.cycles_billing_config().expect("config should load"), + ) + .expect("cycles should compute"); - let conn = Connection::open(&index_path).expect("index DB should reopen after cancel"); - let pending_count: i64 = conn + service + .mark_database_cycles_purchase_ambiguous(operation_id, "default", "payer", cycles) + .expect("operation should become ambiguous"); + let duplicate = service + .begin_database_cycles_purchase("default", "payer", 1_000_000, 3) + .expect_err("ambiguous operation should block duplicate"); + let conn = Connection::open(index_path).expect("index DB should reopen"); + let status: String = conn .query_row( - "SELECT COUNT(*) FROM database_credit_pending_operations WHERE database_id = 'db_old'", - params![], + "SELECT operation_status FROM database_cycle_pending_operations WHERE operation_id = ?1", + params![i64::try_from(operation_id).expect("operation id should fit")], |row| row.get(0), ) - .expect("pending count should load"); - assert_eq!(pending_count, 0); + .expect("pending status should load"); + + assert_eq!(status, "ambiguous"); + assert!( + duplicate.contains("database activation is pending") + || duplicate.contains("cycles purchase already pending") + ); } #[test] - fn index_sql_json_returns_credits_json_rows() { + fn index_sql_json_returns_cycles_json_rows() { let dir = tempdir().expect("tempdir should create"); let service = VfsService::new( dir.path().join("index.sqlite3"), @@ -4989,30 +5316,36 @@ mod tests { .create_database("default", "2vxsx-fae", 1_700_000_000_000) .expect("database should create"); let operation_id = service - .begin_database_credit_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) - .expect("credit purchase should begin"); + .begin_database_cycles_purchase("default", "2vxsx-fae", 1_000_000, 1_700_000_000_001) + .expect("cycle purchase should begin"); + let cycles = cycles_for_payment_amount_e8s( + 1_000_000, + &service.cycles_billing_config().expect("config should load"), + ) + .expect("cycles should compute"); service - .mark_database_credit_purchase_completed( + .complete_database_cycles_purchase_ledger_transfer( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, + 1, ) - .expect("credit purchase should be marked completed"); + .expect("ledger transfer should complete"); service - .credit_database_purchase( + .apply_database_cycles_purchase( operation_id, "default", "2vxsx-fae", - 1_000_000, + cycles, 1, 1_700_000_000_001, ) - .expect("credit purchase should credit"); + .expect("cycle purchase should cycle"); let result = service .query_index_sql_json( - "SELECT json_object('credit_purchase_credits', COALESCE(SUM(amount_credits), 0)) FROM database_credit_ledger WHERE kind = 'credit_purchase' LIMIT 1", + "SELECT json_object('cycles_purchase_cycles', COALESCE(SUM(amount_cycles), 0)) FROM database_cycle_ledger WHERE kind = 'cycles_purchase' LIMIT 1", 10, ) .expect("index SQL should query"); @@ -5021,7 +5354,7 @@ mod tests { assert_eq!(result.row_count, 1); assert_eq!( result.rows, - vec![r#"{"credit_purchase_credits":1000000}"#.to_string()] + vec![format!(r#"{{"cycles_purchase_cycles":{cycles}}}"#)] ); } @@ -5071,18 +5404,18 @@ mod tests { #[test] fn index_sql_json_rejects_mutating_sql() { for sql in [ - "UPDATE database_credit_accounts SET balance_credits = 0", - "DELETE FROM database_credit_ledger", - "INSERT INTO database_credit_ledger (database_id) VALUES ('x')", + "UPDATE database_cycle_accounts SET balance_cycles = 0", + "DELETE FROM database_cycle_ledger", + "INSERT INTO database_cycle_ledger (database_id) VALUES ('x')", "CREATE TABLE x (id INTEGER)", - "DROP TABLE database_credit_ledger", - "ALTER TABLE database_credit_ledger ADD COLUMN x INTEGER", - "REPLACE INTO credits_config (key, value) VALUES ('x', 'y')", + "DROP TABLE database_cycle_ledger", + "ALTER TABLE database_cycle_ledger ADD COLUMN x INTEGER", + "REPLACE INTO cycles_billing_config (key, value) VALUES ('x', 'y')", "VACUUM", - "PRAGMA table_info(database_credit_ledger)", + "PRAGMA table_info(database_cycle_ledger)", "ATTACH DATABASE 'x' AS x", "DETACH DATABASE x", - "REINDEX database_credit_ledger_database_idx", + "REINDEX database_cycle_ledger_database_idx", "ANALYZE", "SELECT json_object('ok', 1); SELECT json_object('ok', 2)", ] { @@ -5110,4 +5443,429 @@ mod tests { assert!(error.contains("one non-null TEXT JSON column")); } + + #[test] + fn storage_billing_daily_cycles_match_subnet_rate() { + let one_gib_cycles = + compute_storage_charge_cycles(GIB_BYTES as u64, STORAGE_BILLING_INTERVAL_MS) + .expect("1GiB storage cycles should compute"); + assert_eq!(one_gib_cycles, 10_972_800_000); + + let ten_mib = 10 * 1024 * 1024; + let ten_mib_cycles = compute_storage_charge_cycles(ten_mib, STORAGE_BILLING_INTERVAL_MS) + .expect("10MiB storage cycles should compute"); + assert_eq!(ten_mib_cycles, 107_156_250); + } + + #[test] + fn storage_billing_charges_raw_storage_cycles() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_balance(&service, "alpha", 1_000); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: 1, + now: STORAGE_BILLING_INTERVAL_MS, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + let (balance, charged_at, amount) = service + .read_index(|conn| { + let account = load_storage_cycle_account(conn, "alpha")?; + let amount: i64 = conn + .query_row( + "SELECT amount_cycles FROM database_cycle_ledger + WHERE database_id = 'alpha' AND kind = 'storage_charge'", + params![], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + Ok(( + account.balance_cycles, + account.storage_charged_at_ms, + amount, + )) + }) + .expect("account should load"); + assert_eq!(balance, 990); + assert_eq!(charged_at, Some(STORAGE_BILLING_INTERVAL_MS)); + assert_eq!(amount, -10); + } + + #[test] + fn storage_billing_zero_cycles_updates_cursor_without_ledger() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_balance(&service, "alpha", 1_000); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: 0, + now: STORAGE_BILLING_INTERVAL_MS, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + let (balance, charged_at, ledger_count) = service + .read_index(|conn| { + let account = load_storage_cycle_account(conn, "alpha")?; + let ledger_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM database_cycle_ledger WHERE database_id = 'alpha'", + params![], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + Ok(( + account.balance_cycles, + account.storage_charged_at_ms, + ledger_count, + )) + }) + .expect("account should load"); + assert_eq!(balance, 1_000); + assert_eq!(charged_at, Some(STORAGE_BILLING_INTERVAL_MS)); + assert_eq!(ledger_count, 0); + } + + #[test] + fn storage_billing_skips_less_than_interval() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_balance(&service, "alpha", 1_000); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: GIB_BYTES as u64, + now: STORAGE_BILLING_INTERVAL_MS - 1, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + let (balance, charged_at) = service + .read_index(|conn| { + let account = load_storage_cycle_account(conn, "alpha")?; + Ok((account.balance_cycles, account.storage_charged_at_ms)) + }) + .expect("account should load"); + assert_eq!(balance, 1_000); + assert_eq!(charged_at, Some(0)); + } + + #[test] + fn storage_billing_suspends_when_balance_is_insufficient() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_balance(&service, "alpha", 100); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: GIB_BYTES as u64, + now: STORAGE_BILLING_INTERVAL_MS, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: GIB_BYTES as u64, + now: STORAGE_BILLING_INTERVAL_MS * 2, + config: &config, + }, + ) + }) + .expect("second storage charge should settle"); + + let (balance, suspended_at, charged_at, kinds, amount) = service + .read_index(|conn| { + let account = load_storage_cycle_account(conn, "alpha")?; + let mut stmt = conn + .prepare( + "SELECT kind FROM database_cycle_ledger + WHERE database_id = 'alpha' + ORDER BY entry_id ASC", + ) + .map_err(|error| error.to_string())?; + let kinds = crate::sqlite::query_map(&mut stmt, params![], |row| { + crate::sqlite::row_get::(row, 0) + }) + .map_err(|error| error.to_string())?; + let amount: i64 = conn + .query_row( + "SELECT amount_cycles FROM database_cycle_ledger + WHERE database_id = 'alpha' AND kind = 'storage_charge'", + params![], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + Ok(( + account.balance_cycles, + account.suspended_at_ms, + account.storage_charged_at_ms, + kinds, + amount, + )) + }) + .expect("ledger should load"); + assert_eq!(balance, 0); + assert_eq!(suspended_at, Some(STORAGE_BILLING_INTERVAL_MS)); + assert_eq!(charged_at, Some(STORAGE_BILLING_INTERVAL_MS * 2)); + assert_eq!(kinds, vec!["storage_charge", "suspend"]); + assert_eq!(amount, -100); + } + + #[test] + fn storage_billing_exact_charge_consumes_balance_and_suspends() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_balance(&service, "alpha", 10); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: 1, + now: STORAGE_BILLING_INTERVAL_MS, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + let (balance, suspended_at, kinds, amount) = storage_test_account_and_ledger(&service); + assert_eq!(balance, 0); + assert_eq!(suspended_at, Some(STORAGE_BILLING_INTERVAL_MS)); + assert_eq!(kinds, vec!["storage_charge", "suspend"]); + assert_eq!(amount, -10); + } + + #[test] + fn storage_billing_keeps_existing_suspension_timestamp() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + service + .create_database("alpha", "owner", 0) + .expect("database should create"); + set_test_database_account(&service, "alpha", 10, Some(123)); + let config = service.cycles_billing_config().expect("config should load"); + + service + .write_index(|tx| { + settle_database_storage_charge_in_tx( + tx, + StorageChargeInput { + database_id: "alpha", + caller: "canister", + size_bytes: 1, + now: STORAGE_BILLING_INTERVAL_MS, + config: &config, + }, + ) + }) + .expect("storage charge should settle"); + + let (balance, suspended_at, kinds, amount) = storage_test_account_and_ledger(&service); + assert_eq!(balance, 0); + assert_eq!(suspended_at, Some(123)); + assert_eq!(kinds, vec!["storage_charge"]); + assert_eq!(amount, -10); + } + + #[test] + fn storage_billing_loads_mounted_databases() { + let dir = tempdir().expect("tempdir should create"); + let service = VfsService::new( + dir.path().join("index.sqlite3"), + dir.path().join("databases"), + ); + service + .run_index_migrations() + .expect("index migrations should run"); + for (database_id, status, mount_id) in [ + ("active", "active", Some(11_i64)), + ("pending", "pending", Some(12_i64)), + ("archived", "archived", Some(13_i64)), + ("restoring", "restoring", Some(14_i64)), + ("deleted", "deleted", None), + ] { + service + .write_index(|tx| { + tx.execute( + "INSERT INTO databases + (database_id, name, db_file_name, mount_id, active_mount_id, status, + schema_version, logical_size_bytes, created_at_ms, updated_at_ms) + VALUES (?1, ?1, ?1, COALESCE(?3, 0), ?3, ?2, ?4, 0, 0, 0)", + params![database_id, status, mount_id, DATABASE_SCHEMA_VERSION], + ) + .map_err(|error| error.to_string())?; + Ok(()) + }) + .expect("database row should insert"); + } + + let database_ids = service + .read_index(load_active_databases_for_storage_billing) + .expect("storage billing databases should load") + .into_iter() + .map(|meta| meta.database_id) + .collect::>(); + + assert_eq!( + database_ids, + vec!["active", "pending", "archived", "restoring"] + ); + } + + fn storage_test_account_and_ledger( + service: &VfsService, + ) -> (i64, Option, Vec, i64) { + service + .read_index(|conn| { + let account = load_storage_cycle_account(conn, "alpha")?; + let mut stmt = conn + .prepare( + "SELECT kind FROM database_cycle_ledger + WHERE database_id = 'alpha' + ORDER BY entry_id ASC", + ) + .map_err(|error| error.to_string())?; + let kinds = crate::sqlite::query_map(&mut stmt, params![], |row| { + crate::sqlite::row_get::(row, 0) + }) + .map_err(|error| error.to_string())?; + let amount: i64 = conn + .query_row( + "SELECT amount_cycles FROM database_cycle_ledger + WHERE database_id = 'alpha' AND kind = 'storage_charge'", + params![], + |row| crate::sqlite::row_get(row, 0), + ) + .map_err(|error| error.to_string())?; + Ok(( + account.balance_cycles, + account.suspended_at_ms, + kinds, + amount, + )) + }) + .expect("storage account and ledger should load") + } + + fn set_test_database_balance(service: &VfsService, database_id: &str, balance: i64) { + set_test_database_account(service, database_id, balance, None); + } + + fn set_test_database_account( + service: &VfsService, + database_id: &str, + balance: i64, + suspended_at_ms: Option, + ) { + service + .write_index(|tx| { + tx.execute( + "UPDATE database_cycle_accounts + SET balance_cycles = ?2, suspended_at_ms = ?3 + WHERE database_id = ?1", + params![database_id, balance, suspended_at_ms], + ) + .map_err(|error| error.to_string())?; + Ok(()) + }) + .expect("test database account should update"); + } } diff --git a/crates/vfs_runtime/tests/database_service.rs b/crates/vfs_runtime/tests/database_service.rs index d0a42b80..52eaa1cb 100644 --- a/crates/vfs_runtime/tests/database_service.rs +++ b/crates/vfs_runtime/tests/database_service.rs @@ -7,16 +7,17 @@ use rusqlite::{Connection, OptionalExtension, params}; use sha2::{Digest, Sha256}; use tempfile::tempdir; use vfs_runtime::{ - CreditsPendingLedgerDetailsInput, DEFAULT_LLM_WRITER_PRINCIPAL, - DatabaseCreditPurchaseWithLedgerDetails, MAX_ARCHIVE_CHUNK_BYTES, MAX_DATABASE_SIZE_BYTES, - MAX_RESTORE_CHUNK_BYTES, VfsService, + CyclesPendingLedgerDetailsInput, DEFAULT_LLM_WRITER_PRINCIPAL, + DatabaseCyclesPurchaseWithLedgerDetails, MAX_ARCHIVE_CHUNK_BYTES, MAX_DATABASE_SIZE_BYTES, + MAX_RESTORE_CHUNK_BYTES, VfsService, cycles_for_payment_amount_e8s, }; use vfs_types::{ - AppendNodeRequest, CreditsConfigUpdate, DatabaseRole, DatabaseStatus, DeleteDatabaseRequest, - DeleteNodeRequest, KINIC_LEDGER_FEE_E8S, MkdirNodeRequest, NodeKind, - OpsAnswerSessionCheckRequest, OpsAnswerSessionRequest, SearchNodesRequest, SearchPreviewMode, - SourceRunSessionCheckRequest, UrlIngestTriggerSessionCheckRequest, - UrlIngestTriggerSessionRequest, WriteNodeRequest, WriteSourceForGenerationRequest, + AppendNodeRequest, CyclesBillingConfigUpdate, DatabaseRole, DatabaseStatus, + DeleteDatabaseRequest, DeleteNodeRequest, KINIC_LEDGER_FEE_E8S, MkdirNodeRequest, + MoveNodeRequest, NodeKind, OpsAnswerSessionCheckRequest, OpsAnswerSessionRequest, + SearchNodesRequest, SearchPreviewMode, SourceRunSessionCheckRequest, + UrlIngestTriggerSessionCheckRequest, UrlIngestTriggerSessionRequest, WriteNodeRequest, + WriteSourceForGenerationRequest, }; fn service() -> VfsService { @@ -61,31 +62,55 @@ fn mainnet_011_index_upgrades_to_latest() { .expect("database status should load"); let balance: i64 = conn .query_row( - "SELECT balance_credits FROM database_credit_accounts WHERE database_id = 'db_existing'", + "SELECT balance_cycles FROM database_cycle_accounts WHERE database_id = 'db_existing'", params![], |row| row.get(0), ) - .expect("database credits account should exist"); + .expect("database cycles account should exist"); let suspended_at_ms: Option = conn .query_row( - "SELECT suspended_at_ms FROM database_credit_accounts WHERE database_id = 'db_existing'", + "SELECT suspended_at_ms FROM database_cycle_accounts WHERE database_id = 'db_existing'", params![], |row| row.get(0), ) - .expect("database credits suspension should exist"); + .expect("database cycles suspension should exist"); + let storage_columns: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_accounts') + WHERE name = 'storage_charged_at_ms'", + params![], + |row| row.get(0), + ) + .expect("storage charged cursor column should load"); + let removed_storage_columns: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_accounts') + WHERE name = 'storage_unbilled_cycles'", + params![], + |row| row.get(0), + ) + .expect("removed storage column count should load"); let pending_details_columns: i64 = conn .query_row( - "SELECT COUNT(*) FROM pragma_table_info('database_credit_pending_operations') + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_pending_operations') WHERE name IN ('from_owner', 'from_subaccount', 'to_owner', 'to_subaccount', 'ledger_fee_e8s', 'ledger_created_at_time_ns')", params![], |row| row.get(0), ) .expect("pending details columns should load"); + let pending_ledger_block_columns: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_pending_operations') + WHERE name = 'ledger_block_index'", + params![], + |row| row.get(0), + ) + .expect("pending ledger block column count should load"); let ledger_cycles_column_count: i64 = conn .query_row( - "SELECT COUNT(*) FROM pragma_table_info('database_credit_ledger') - WHERE name = 'cycles_per_credit'", + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_ledger') + WHERE name = 'cycles_per_cycle'", params![], |row| row.get(0), ) @@ -102,14 +127,17 @@ fn mainnet_011_index_upgrades_to_latest() { assert_eq!(status, "active"); assert_eq!(balance, 0); assert_eq!(suspended_at_ms, Some(0)); + assert_eq!(storage_columns, 1); + assert_eq!(removed_storage_columns, 0); assert_eq!(pending_details_columns, 6); + assert_eq!(pending_ledger_block_columns, 1); assert_eq!(ledger_cycles_column_count, 0); assert_eq!(usage_table_count, 0); assert_eq!( - schema_migration_count(&root, "database_index:020_credits_config_version"), + schema_migration_count(&root, "database_index:020_cycles_billing_config_version"), 1 ); - assert_eq!(credits_config_version(&root), 1); + assert_eq!(cycles_billing_config_key_count(&root, "config_version"), 0); } fn write_mainnet_011_index_schema(index_path: &std::path::Path, status: &str) { @@ -254,7 +282,7 @@ fn partial_billing_index_schema_is_rejected() { let conn = Connection::open(&index_path).expect("index should reopen"); conn.execute( "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, 0)", - params!["database_index:012_credits_initial"], + params!["database_index:012_cycles_initial"], ) .expect("partial billing marker should insert"); drop(conn); @@ -302,7 +330,7 @@ fn pre_011_index_schema_is_rejected() { } #[test] -fn cycles_per_credit_ledger_schema_is_rejected() { +fn cycles_per_cycle_ledger_schema_is_rejected() { let dir = tempdir().expect("tempdir should create"); let root = dir.keep(); let index_path = root.join("index.sqlite3"); @@ -312,27 +340,27 @@ fn cycles_per_credit_ledger_schema_is_rejected() { version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL ); - CREATE TABLE database_credit_ledger ( + CREATE TABLE database_cycle_ledger ( entry_id INTEGER PRIMARY KEY AUTOINCREMENT, database_id TEXT NOT NULL, kind TEXT NOT NULL, - amount_credits INTEGER NOT NULL, - balance_after_credits INTEGER NOT NULL, + amount_cycles INTEGER NOT NULL, + balance_after_cycles INTEGER NOT NULL, payment_amount_e8s INTEGER, caller TEXT NOT NULL, method TEXT, cycles_delta INTEGER, - credits_per_kinic INTEGER, - cycles_per_credit INTEGER, + cycles_per_kinic INTEGER, + cycles_per_cycle INTEGER, ledger_block_index INTEGER, created_at_ms INTEGER NOT NULL ); - CREATE TABLE credits_config ( + CREATE TABLE cycles_billing_config ( key TEXT PRIMARY KEY, value TEXT NOT NULL );", ) - .expect("legacy credits schema should create"); + .expect("legacy cycles schema should create"); for version in [ "database_index:000_initial", "database_index:001_lifecycle", @@ -345,7 +373,7 @@ fn cycles_per_credit_ledger_schema_is_rejected() { "database_index:009_restore_chunk_bytes", "database_index:010_database_name_breaking", "database_index:011_source_run_sessions", - "database_index:012_credits_initial", + "database_index:012_cycles_initial", ] { conn.execute( "INSERT INTO schema_migrations (version, applied_at) VALUES (?1, 0)", @@ -358,13 +386,13 @@ fn cycles_per_credit_ledger_schema_is_rejected() { let service = VfsService::new(index_path.clone(), root.join("databases")); let error = service .run_index_migrations() - .expect_err("cycles_per_credit schema should reject"); + .expect_err("cycles_per_cycle schema should reject"); let conn = Connection::open(index_path).expect("index should reopen"); let ledger_cycles_column_count: i64 = conn .query_row( - "SELECT COUNT(*) FROM pragma_table_info('database_credit_ledger') - WHERE name = 'cycles_per_credit'", + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_ledger') + WHERE name = 'cycles_per_cycle'", params![], |row| row.get(0), ) @@ -458,79 +486,92 @@ fn database_member_count(root: &std::path::Path, database_id: &str) -> i64 { .expect("member count should load") } -fn database_credits_balance(root: &std::path::Path, database_id: &str) -> i64 { +fn database_cycles_balance(root: &std::path::Path, database_id: &str) -> i64 { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( - "SELECT balance_credits FROM database_credit_accounts WHERE database_id = ?1", + "SELECT balance_cycles FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| row.get(0), ) - .expect("database credits balance should load") + .expect("database cycles balance should load") } -fn database_credits_suspended_at(root: &std::path::Path, database_id: &str) -> Option { +fn database_cycles_suspended_at(root: &std::path::Path, database_id: &str) -> Option { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( - "SELECT suspended_at_ms FROM database_credit_accounts WHERE database_id = ?1", + "SELECT suspended_at_ms FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| row.get(0), ) - .expect("database credits suspension should load") + .expect("database cycles suspension should load") } -fn credits_config_version(root: &std::path::Path) -> i64 { +fn cycles_billing_config_key_count(root: &std::path::Path, key: &str) -> i64 { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( - "SELECT value FROM credits_config WHERE key = 'config_version'", - params![], - |row| row.get::<_, String>(0), + "SELECT COUNT(*) FROM cycles_billing_config WHERE key = ?1", + params![key], + |row| row.get(0), ) - .expect("credits config version should load") - .parse() - .expect("credits config version should be numeric") + .expect("cycles config key count should load") } fn database_pending_operation_count(root: &std::path::Path, database_id: &str) -> i64 { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( - "SELECT COUNT(*) FROM database_credit_pending_operations WHERE database_id = ?1", + "SELECT COUNT(*) FROM database_cycle_pending_operations WHERE database_id = ?1", params![database_id], |row| row.get(0), ) - .expect("pending credit operation count should load") + .expect("pending cycle operation count should load") } -fn credit_database( +fn cycle_database( service: &VfsService, database_id: &str, caller: &str, - amount_credits: u64, + payment_amount_e8s: u64, block_index: u64, now: i64, ) -> u64 { let operation_id = service - .begin_database_credit_purchase(database_id, caller, amount_credits, now) - .expect("database credit purchase should begin"); - service - .mark_database_credit_purchase_completed(operation_id, database_id, caller, amount_credits) - .expect("database credit purchase should be marked completed"); + .begin_database_cycles_purchase(database_id, caller, payment_amount_e8s, now) + .expect("database cycle purchase should begin"); + let cycles = cycles_for_payment(service, database_id, payment_amount_e8s); service - .credit_database_purchase( + .complete_database_cycles_purchase_ledger_transfer( operation_id, database_id, caller, - amount_credits, + cycles, block_index, - now, ) - .expect("database credit purchase should credit") + .expect("database cycle purchase ledger transfer should complete"); + service + .apply_database_cycles_purchase(operation_id, database_id, caller, cycles, block_index, now) + .expect("database cycle purchase should cycle") +} + +fn cycles_for_payment(service: &VfsService, database_id: &str, payment_amount_e8s: u64) -> u64 { + service + .validate_database_cycles_purchase(database_id, payment_amount_e8s) + .expect("database cycle purchase should validate"); + let config = service + .cycles_billing_config() + .expect("cycles config should load"); + cycles_for_payment_amount_e8s(payment_amount_e8s, &config) + .expect("database cycle purchase amount should compute") +} + +fn default_cycles_for_payment(payment_amount_e8s: u64) -> u64 { + payment_amount_e8s * 2_345 } fn database_ledger_kinds(root: &std::path::Path, database_id: &str) -> Vec { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); let mut stmt = conn .prepare( - "SELECT kind FROM database_credit_ledger + "SELECT kind FROM database_cycle_ledger WHERE database_id = ?1 ORDER BY entry_id ASC", ) @@ -781,11 +822,11 @@ fn archive_bytes_for_chunk_size( } #[test] -fn index_migrations_create_credit_ledger_only_schema_once() { +fn index_migrations_create_cycle_ledger_only_schema_once() { let (service, root) = service_with_root(); let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); - for table_name in ["database_mount_history", "database_credit_ledger"] { + for table_name in ["database_mount_history", "database_cycle_ledger"] { let table_exists: i64 = conn .query_row( "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", @@ -805,15 +846,15 @@ fn index_migrations_create_credit_ledger_only_schema_once() { assert_eq!(usage_table_exists, 0); let usage_column_exists: i64 = conn .query_row( - "SELECT COUNT(*) FROM pragma_table_info('database_credit_ledger') + "SELECT COUNT(*) FROM pragma_table_info('database_cycle_ledger') WHERE name = 'usage_event_id'", params![], |row| row.get(0), ) - .expect("credit ledger column lookup should work"); + .expect("cycle ledger column lookup should work"); assert_eq!(usage_column_exists, 0); assert_eq!( - schema_migration_count(&root, "database_index:018_credit_ledger_only"), + schema_migration_count(&root, "database_index:018_cycles_ledger_only"), 1 ); assert_eq!( @@ -845,7 +886,7 @@ fn index_migrations_create_credit_ledger_only_schema_once() { .run_index_migrations() .expect("index migrations should be idempotent"); assert_eq!( - schema_migration_count(&root, "database_index:018_credit_ledger_only"), + schema_migration_count(&root, "database_index:018_cycles_ledger_only"), 1 ); assert_eq!( @@ -880,7 +921,7 @@ fn url_ingest_trigger_session_requires_writer_and_allows_replay() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); let request_path = "/Sources/ingest-requests/1.md"; write_url_ingest_request(&service, "owner", "alpha", request_path, "queued", "owner"); @@ -911,7 +952,7 @@ fn url_ingest_trigger_session_requires_default_llm_writer() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); let request_path = "/Sources/ingest-requests/1.md"; write_url_ingest_request(&service, "owner", "alpha", request_path, "queued", "owner"); service @@ -949,7 +990,7 @@ fn url_ingest_trigger_session_rejects_invalid_request_nodes() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .grant_database_access("alpha", "owner", "other", DatabaseRole::Reader, 2) .expect("reader grant should succeed"); @@ -1059,7 +1100,7 @@ fn url_ingest_trigger_session_rejects_expired_and_unknown_nonce() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); let request_path = "/Sources/ingest-requests/1.md"; write_url_ingest_request(&service, "owner", "alpha", request_path, "queued", "owner"); @@ -1095,7 +1136,7 @@ fn url_ingest_trigger_session_rejects_expired_and_unknown_nonce() { } #[test] -fn url_ingest_trigger_session_check_requires_write_credits_database() { +fn url_ingest_trigger_session_check_requires_write_cycles_database() { let service = service(); service .create_database("alpha", "owner", 1) @@ -1108,7 +1149,7 @@ fn url_ingest_trigger_session_check_requires_write_credits_database() { url_ingest_session_request("alpha", "session-1"), 100, ) - .expect("session should authorize before credits changes"); + .expect("session should authorize before cycles changes"); let error = service .check_url_ingest_trigger_session( @@ -1117,7 +1158,7 @@ fn url_ingest_trigger_session_check_requires_write_credits_database() { ) .expect_err("suspended database should reject session check"); - assert!(error.contains("database credits are suspended")); + assert!(error.contains("database cycles are suspended")); } #[test] @@ -1126,7 +1167,7 @@ fn url_ingest_trigger_session_check_allows_generating_status() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); let request_path = "/Sources/ingest-requests/1.md"; write_url_ingest_request( &service, @@ -1158,7 +1199,7 @@ fn source_for_generation_writes_source_and_authorizes_bound_session() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .grant_database_access("alpha", "owner", "writer", DatabaseRole::Writer, 2) .expect("writer grant should succeed"); @@ -1268,7 +1309,7 @@ fn source_run_session_requires_funded_database() { 101, ) .expect_err("suspended database should reject source run session"); - assert!(error.contains("database credits are suspended")); + assert!(error.contains("database cycles are suspended")); } #[test] @@ -1314,7 +1355,7 @@ fn ops_answer_session_allows_database_members_and_replay() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .grant_database_access("alpha", "owner", "writer", DatabaseRole::Writer, 2) .expect("writer grant should succeed"); @@ -1347,7 +1388,7 @@ fn ops_answer_session_rejects_anonymous_and_non_members() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .grant_database_access("alpha", "owner", "2vxsx-fae", DatabaseRole::Reader, 2) .expect("anonymous public grant should succeed"); @@ -1372,24 +1413,24 @@ fn ops_answer_session_rejects_anonymous_and_non_members() { } #[test] -fn ops_answer_session_check_requires_write_credits_database() { +fn ops_answer_session_check_requires_write_cycles_database() { let service = service(); service .create_database("alpha", "owner", 1) .expect("database should create"); service .authorize_ops_answer_session("owner", ops_answer_session_request("alpha", "session-1"), 0) - .expect("session should authorize before credits changes"); + .expect("session should authorize before cycles changes"); let error = service .check_ops_answer_session(ops_answer_session_check_request("alpha", "session-1"), 1) .expect_err("suspended database should reject ops answer session check"); - assert!(error.contains("database credits are suspended")); + assert!(error.contains("database cycles are suspended")); } #[test] -fn check_database_write_credits_requires_writer_and_funded_database() { +fn check_database_write_cycles_requires_writer_and_funded_database() { let service = service(); service .create_database("alpha", "owner", 1) @@ -1402,29 +1443,29 @@ fn check_database_write_credits_requires_writer_and_funded_database() { .expect("reader grant should succeed"); let suspended = service - .check_database_write_credits("alpha", "writer") + .check_database_write_cycles("alpha", "writer") .expect_err("suspended database should reject writer"); - assert!(suspended.contains("database credits are suspended")); + assert!(suspended.contains("database cycles are suspended")); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 4); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 4); service - .check_database_write_credits("alpha", "owner") - .expect("owner should pass write credits check"); + .check_database_write_cycles("alpha", "owner") + .expect("owner should pass write cycles check"); service - .check_database_write_credits("alpha", "writer") - .expect("writer should pass write credits check"); + .check_database_write_cycles("alpha", "writer") + .expect("writer should pass write cycles check"); let reader = service - .check_database_write_credits("alpha", "reader") - .expect_err("reader should fail write credits check"); + .check_database_write_cycles("alpha", "reader") + .expect_err("reader should fail write cycles check"); assert!(reader.contains("principal lacks required database role")); let anonymous = service - .check_database_write_credits("alpha", "2vxsx-fae") - .expect_err("anonymous should fail write credits check"); + .check_database_write_cycles("alpha", "2vxsx-fae") + .expect_err("anonymous should fail write cycles check"); assert!(anonymous.contains("anonymous caller not allowed")); let missing = service - .check_database_write_credits("alpha", "missing") - .expect_err("non-member should fail write credits check"); + .check_database_write_cycles("alpha", "missing") + .expect_err("non-member should fail write cycles check"); assert!(missing.contains("principal has no access")); } @@ -1434,7 +1475,7 @@ fn ops_answer_session_rechecks_current_role_after_revoke() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .grant_database_access("alpha", "owner", "reader", DatabaseRole::Reader, 2) .expect("reader grant should succeed"); @@ -1470,7 +1511,7 @@ fn ops_answer_session_rejects_invalid_and_expired_nonce() { service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000_000, 1, 2); + cycle_database(&service, "alpha", "owner", 1_000_000, 1, 2); service .authorize_ops_answer_session("owner", ops_answer_session_request("alpha", "session-1"), 0) @@ -1513,9 +1554,9 @@ fn database_create_returns_generated_id_and_name() { assert_eq!(result.database_id.len(), 15); assert_eq!(result.name, "Team skills"); assert_eq!(database_member_count(&root, &result.database_id), 2); - assert_eq!(database_credits_balance(&root, &result.database_id), 0); + assert_eq!(database_cycles_balance(&root, &result.database_id), 0); assert_eq!( - database_credits_suspended_at(&root, &result.database_id), + database_cycles_suspended_at(&root, &result.database_id), Some(1) ); assert!(database_ledger_kinds(&root, &result.database_id).is_empty()); @@ -1527,7 +1568,7 @@ fn database_create_returns_generated_id_and_name() { } #[test] -fn pending_database_creation_defers_mount_slot_until_credit_purchase_activation() { +fn pending_database_creation_defers_mount_slot_until_cycles_purchase_activation() { let (service, root) = service_with_root(); let pending = service @@ -1537,9 +1578,9 @@ fn pending_database_creation_defers_mount_slot_until_credit_purchase_activation( assert!(pending.database_id.starts_with("db_")); assert_eq!(pending.name, "Pending"); assert_eq!(database_member_count(&root, &pending.database_id), 2); - assert_eq!(database_credits_balance(&root, &pending.database_id), 0); + assert_eq!(database_cycles_balance(&root, &pending.database_id), 0); assert_eq!( - database_credits_suspended_at(&root, &pending.database_id), + database_cycles_suspended_at(&root, &pending.database_id), Some(1) ); assert_eq!(mount_history_count(&root), 0); @@ -1553,76 +1594,48 @@ fn pending_database_creation_defers_mount_slot_until_credit_purchase_activation( .expect_err("pending DB should reject VFS reads") .contains("database is pending") ); - assert!( - service - .rename_database(&pending.database_id, "owner", "Renamed", 2) - .expect_err("pending DB should reject rename") - .contains("database is pending") - ); - assert!( - service - .grant_database_access( - &pending.database_id, - "owner", - "reader", - DatabaseRole::Reader, - 2 - ) - .expect_err("pending DB should reject grants") - .contains("database is pending") - ); - assert!( - service - .revoke_database_access(&pending.database_id, "owner", "reader") - .expect_err("pending DB should reject revokes") - .contains("database is pending") - ); - assert!( - service - .list_database_members(&pending.database_id, "owner") - .expect_err("pending DB should reject member listing") - .contains("database is pending") - ); service - .validate_database_credit_purchase(&pending.database_id, 500) - .expect("anonymous preview should accept pending DB credit purchase"); + .validate_database_cycles_purchase(&pending.database_id, 500) + .expect("validation should accept pending DB cycle purchase"); let operation_id = service - .begin_database_credit_purchase(&pending.database_id, "payer", 1_000_000, 2) - .expect("credit purchase should begin"); + .begin_database_cycles_purchase(&pending.database_id, "payer", 1_000_000, 2) + .expect("cycle purchase should begin"); assert_eq!(mount_history_count(&root), 0); assert_eq!( database_index_row(&root, &pending.database_id), ("pending".to_string(), None, 0, None) ); let meta = service - .activate_pending_database_for_credit_purchase(&pending.database_id, 2) + .activate_pending_database_for_cycles_purchase(&pending.database_id, 2) .expect("pending activation should prepare") .expect("pending activation should allocate mount"); assert_eq!(meta.mount_id, 11); service .run_pending_database_migrations(&pending.database_id) .expect("pending migrations should run"); + let purchased_cycles = default_cycles_for_payment(1_000_000); service - .mark_database_credit_purchase_completed( + .complete_database_cycles_purchase_ledger_transfer( operation_id, &pending.database_id, "payer", - 1_000_000, + purchased_cycles, + 42, ) - .expect("credit purchase should be marked completed"); + .expect("cycle purchase ledger transfer should complete"); let balance = service - .credit_database_purchase( + .apply_database_cycles_purchase( operation_id, &pending.database_id, "payer", - 1_000_000, + purchased_cycles, 42, 4, ) - .expect("credit purchase should activate and credit"); + .expect("cycle purchase should activate and cycle"); - assert_eq!(balance, 1_000_000); + assert_eq!(balance, purchased_cycles); let row = database_index_row(&root, &pending.database_id); assert_eq!(row.0, "active"); assert_eq!(row.1, Some(11)); @@ -1634,130 +1647,215 @@ fn pending_database_creation_defers_mount_slot_until_credit_purchase_activation( } #[test] -fn pending_database_can_be_deleted_before_activation() { - let (service, root) = service_with_root(); - let pending = service - .reserve_pending_generated_database("Delete pending", "owner", 1) - .expect("pending database should create"); +fn pending_database_creation_limits_unresolved_per_caller() { + let service = service(); - service - .delete_database(delete_request(&pending.database_id), "owner", 2) - .expect("pending database should delete"); + for offset in 0..3 { + service + .reserve_pending_generated_database( + &format!("Pending {offset}"), + "owner", + 1_700_000_000_000 + offset, + ) + .expect("pending database should create within limit"); + } - assert_eq!(database_member_count(&root, &pending.database_id), 0); - assert!(!database_index_row_exists(&root, &pending.database_id)); + let error = service + .reserve_pending_generated_database("Pending 3", "owner", 1_700_000_000_010) + .expect_err("fourth unresolved pending database should fail"); + assert!(error.contains("too many pending databases for caller")); + + service + .reserve_pending_generated_database("Other caller", "other", 1_700_000_000_010) + .expect("pending limit should be per caller"); } #[test] -fn pending_database_credit_purchase_cancel_does_not_allocate_mount_slot() { +fn pending_database_creation_purges_expired_unstarted_reservations() { let (service, root) = service_with_root(); - let pending = service - .reserve_pending_generated_database("Cancel", "owner", 1) - .expect("pending database should create"); + let mut expired_ids = Vec::new(); + for offset in 0..3 { + let pending = service + .reserve_pending_generated_database(&format!("Expired {offset}"), "owner", offset) + .expect("expired pending database should create"); + expired_ids.push(pending.database_id); + } - let operation_id = service - .begin_database_credit_purchase(&pending.database_id, "payer", 500, 3) - .expect("credit purchase should begin"); - service - .cancel_database_credit_purchase(operation_id, &pending.database_id, "payer", 500) - .expect("ledger reject cancel should delete operation"); + let fresh = service + .reserve_pending_generated_database("Fresh", "owner", 86_400_003) + .expect("expired unstarted pending databases should be purged before limit check"); - assert_eq!(mount_history_count(&root), 0); - assert_eq!( - database_index_row(&root, &pending.database_id), - ("pending".to_string(), None, 0, None) - ); - let active = service - .create_database("active", "owner", 5) - .expect("active database should use first mount"); - assert_eq!(active.mount_id, 11); + assert!(database_index_row_exists(&root, &fresh.database_id)); + for database_id in expired_ids { + assert!(!database_index_row_exists(&root, &database_id)); + } } #[test] -fn direct_credit_purchase_cancel_rejects_non_in_flight_operations() { +fn pending_database_creation_preserves_in_flight_cycle_operations_during_cleanup() { let (service, root) = service_with_root(); - for database_id in ["completed-cancel", "ambiguous-cancel"] { + let protected = service + .reserve_pending_generated_database("In flight", "owner", 0) + .expect("pending database should create"); + service + .begin_database_cycles_purchase(&protected.database_id, "payer", 500, 1) + .expect("cycle purchase should begin"); + for offset in 0..2 { service - .create_database(database_id, "owner", 1) - .expect("database should create"); + .reserve_pending_generated_database(&format!("Expired {offset}"), "owner", offset + 2) + .expect("expired pending database should create"); } - let completed = service - .begin_database_credit_purchase("completed-cancel", "payer", 500, 2) - .expect("completed operation should begin"); - service - .mark_database_credit_purchase_completed(completed, "completed-cancel", "payer", 500) - .expect("completed operation should be marked completed"); - let completed_error = service - .cancel_database_credit_purchase(completed, "completed-cancel", "payer", 500) - .expect_err("completed operation should not be directly cancellable"); - assert!(completed_error.contains("credit purchase operation is completed")); + service + .reserve_pending_generated_database("Fresh", "owner", 86_400_003) + .expect("unprotected expired pending databases should be purged"); + + assert!(database_index_row_exists(&root, &protected.database_id)); assert_eq!( - database_pending_operation_count(&root, "completed-cancel"), + database_pending_operation_count(&root, &protected.database_id), 1 ); +} - let ambiguous = service - .begin_database_credit_purchase("ambiguous-cancel", "payer", 700, 3) - .expect("ambiguous operation should begin"); +#[test] +fn pending_database_creation_preserves_activation_started_reservations_during_cleanup() { + let (service, root) = service_with_root(); + let activated = service + .reserve_pending_generated_database("Activating", "owner", 0) + .expect("pending database should create"); service - .mark_database_credit_purchase_ambiguous(ambiguous, "ambiguous-cancel", "payer", 700, 4) - .expect("ambiguous operation should be marked ambiguous"); - let ambiguous_error = service - .cancel_database_credit_purchase(ambiguous, "ambiguous-cancel", "payer", 700) - .expect_err("ambiguous operation should not be directly cancellable"); - assert!(ambiguous_error.contains("credit purchase operation is ambiguous")); - assert_eq!( - database_pending_operation_count(&root, "ambiguous-cancel"), - 1 - ); + .activate_pending_database_for_cycles_purchase(&activated.database_id, 1) + .expect("pending activation should start") + .expect("pending activation should allocate mount"); + for offset in 0..2 { + service + .reserve_pending_generated_database(&format!("Expired {offset}"), "owner", offset + 2) + .expect("expired pending database should create"); + } + service - .repair_database_credit_purchase_cancel("ambiguous-cancel", ambiguous, "payer", 5) - .expect("ambiguous operation should remain repair cancellable"); + .reserve_pending_generated_database("Fresh", "owner", 86_400_003) + .expect("unactivated expired pending databases should be purged"); + + assert!(database_index_row_exists(&root, &activated.database_id)); assert_eq!( - database_pending_operation_count(&root, "ambiguous-cancel"), - 0 + database_index_row(&root, &activated.database_id), + ("pending".to_string(), Some(11), 0, None) ); } #[test] -fn pending_database_credit_purchase_cancel_rejects_after_activation_started() { +fn pending_database_cycles_purchase_cancel_does_not_allocate_mount_slot() { let (service, root) = service_with_root(); let pending = service - .reserve_pending_generated_database("Started", "owner", 1) + .reserve_pending_generated_database("Cancel", "owner", 1) .expect("pending database should create"); + let operation_id = service - .begin_database_credit_purchase(&pending.database_id, "payer", 500, 2) - .expect("credit purchase should begin"); + .begin_database_cycles_purchase(&pending.database_id, "payer", 500, 3) + .expect("cycle purchase should begin"); + let purchased_cycles = default_cycles_for_payment(500); service - .mark_database_credit_purchase_ambiguous( + .cancel_database_cycles_purchase( operation_id, &pending.database_id, "payer", - 500, - 3, + purchased_cycles, ) - .expect("credit purchase ambiguity should record"); + .expect("ledger reject cancel should delete operation"); + + assert_eq!(mount_history_count(&root), 0); + assert_eq!( + database_index_row(&root, &pending.database_id), + ("pending".to_string(), None, 0, None) + ); + let active = service + .create_database("active", "owner", 5) + .expect("active database should use first mount"); + assert_eq!(active.mount_id, 11); +} + +#[test] +fn cleanup_database_cycles_purchase_discards_started_pending_activation() { + let (service, root) = service_with_root(); + let pending = service + .reserve_pending_generated_database("Started cleanup", "owner", 1) + .expect("pending database should create"); + let operation_id = service + .begin_database_cycles_purchase(&pending.database_id, "payer", 500, 2) + .expect("cycle purchase should begin"); + let purchased_cycles = default_cycles_for_payment(500); let meta = service - .activate_pending_database_for_credit_purchase(&pending.database_id, 4) + .activate_pending_database_for_cycles_purchase(&pending.database_id, 4) .expect("pending activation should prepare") .expect("activation should allocate mount"); assert_eq!(meta.mount_id, 11); - let error = service - .repair_database_credit_purchase_cancel(&pending.database_id, operation_id, "payer", 5) - .expect_err("started activation should require complete repair"); + service + .cleanup_database_cycles_purchase_after_no_credit( + operation_id, + &pending.database_id, + "payer", + purchased_cycles, + ) + .expect("no-credit cleanup should discard reservation"); - assert!(error.contains("complete credit purchase repair")); assert_eq!( database_pending_operation_count(&root, &pending.database_id), - 1 - ); - assert_eq!(mount_history_count(&root), 1); - assert_eq!( - database_index_row(&root, &pending.database_id), - ("pending".to_string(), Some(11), 0, None) + 0 ); + assert_eq!(mount_history_count(&root), 0); + assert!(!database_index_row_exists(&root, &pending.database_id)); +} + +#[test] +fn cycles_purchase_rejects_archive_restore_statuses() { + let service = service(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + + let archive_info = service + .begin_database_archive("alpha", "owner", 2) + .expect("archive should begin"); + let archiving = service + .validate_database_cycles_purchase("alpha", 500) + .expect_err("archiving database should reject purchase"); + assert!(archiving.contains("database is archiving")); + + let archive = read_archive_in_chunks(&service, "alpha", archive_info.size_bytes, 17); + let snapshot_hash = sha256_bytes(&archive); + service + .finalize_database_archive("alpha", "owner", snapshot_hash.clone(), 3) + .expect("archive should finalize"); + let archived = service + .validate_database_cycles_purchase("alpha", 500) + .expect_err("archived database should reject purchase"); + assert!(archived.contains("database is archived")); + + service + .begin_database_restore("alpha", "owner", snapshot_hash, archive_info.size_bytes, 4) + .expect("restore should begin"); + let restoring = service + .validate_database_cycles_purchase("alpha", 500) + .expect_err("restoring database should reject purchase"); + assert!(restoring.contains("database is restoring")); +} + +#[test] +fn lifecycle_operations_reject_pending_cycle_purchase() { + let service = service(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + service + .begin_database_cycles_purchase("alpha", "payer", 500, 2) + .expect("cycle purchase should begin"); + + let archive = service + .begin_database_archive("alpha", "owner", 3) + .expect_err("archive should reject pending cycle operation"); + assert!(archive.contains("pending cycle operation")); } #[test] @@ -1800,105 +1898,114 @@ fn database_create_rejects_duplicate_requested_id_for_internal_setup() { } #[test] -fn requested_database_create_starts_with_zero_credits_balance() { +fn requested_database_create_starts_with_zero_cycles_balance() { let (service, root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); - assert_eq!(database_credits_balance(&root, "alpha"), 0); - assert_eq!(database_credits_suspended_at(&root, "alpha"), Some(1)); + assert_eq!(database_cycles_balance(&root, "alpha"), 0); + assert_eq!(database_cycles_suspended_at(&root, "alpha"), Some(1)); assert!(database_ledger_kinds(&root, "alpha").is_empty()); } #[test] -fn reservation_starts_with_zero_credits_balance() { +fn reservation_starts_with_zero_cycles_balance() { let (service, root) = service_with_root(); service .reserve_database("reserved", "Reserved", "owner", 1) .expect("reservation should create"); - assert_eq!(database_credits_balance(&root, "reserved"), 0); - assert_eq!(database_credits_suspended_at(&root, "reserved"), Some(1)); + assert_eq!(database_cycles_balance(&root, "reserved"), 0); + assert_eq!(database_cycles_suspended_at(&root, "reserved"), Some(1)); assert!(database_ledger_kinds(&root, "reserved").is_empty()); } #[test] -fn database_credits_purchase_allows_authenticated_non_owner() { +fn database_cycles_purchase_allows_authenticated_non_owner() { let (service, _root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 1_000, 1, 3); + cycle_database(&service, "alpha", "owner", 1_000, 1, 3); service - .begin_database_credit_purchase("alpha", "stranger", 100, 4) - .expect("authenticated non-owner should be allowed to purchase credits"); + .begin_database_cycles_purchase("alpha", "stranger", 100, 4) + .expect("authenticated non-owner should be allowed to purchase cycles"); } #[test] -fn credits_config_version_changes_only_for_effective_rate_updates() { +fn cycles_billing_config_update_changes_only_mutable_values() { let (service, root) = service_with_root(); - assert_eq!(credits_config_version(&root), 1); + assert_eq!(cycles_billing_config_key_count(&root, "config_version"), 0); service - .update_credits_config( - CreditsConfigUpdate { - credits_per_kinic: 1_000, - min_update_credits: 1, + .update_cycles_billing_config( + CyclesBillingConfigUpdate { + cycles_per_kinic: 234_500_000_000, + min_update_cycles: 1_000_000, }, "rrkah-fqaaa-aaaaa-aaaaq-cai", ) - .expect("same config should update without version bump"); - assert_eq!(credits_config_version(&root), 1); + .expect("same config should update"); + assert_eq!(cycles_billing_config_key_count(&root, "config_version"), 0); service - .update_credits_config( - CreditsConfigUpdate { - credits_per_kinic: 2_000, - min_update_credits: 1, + .update_cycles_billing_config( + CyclesBillingConfigUpdate { + cycles_per_kinic: 469_000_000_000, + min_update_cycles: 1_000_000, }, "rrkah-fqaaa-aaaaa-aaaaq-cai", ) .expect("changed config should update"); - assert_eq!(credits_config_version(&root), 2); + assert_eq!( + service + .cycles_billing_config() + .expect("cycles config should load") + .cycles_per_kinic, + 469_000_000_000 + ); + assert_eq!(cycles_billing_config_key_count(&root, "config_version"), 0); } #[test] -fn credit_purchase_preview_returns_fixed_payment_inputs() { +fn cycles_purchase_validation_accepts_current_payment_inputs() { let (service, _root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); - let preview = service - .preview_database_credit_purchase("alpha", 500) - .expect("preview should succeed"); + service + .validate_database_cycles_purchase("alpha", 50_000) + .expect("purchase should validate"); + let config = service + .cycles_billing_config() + .expect("cycles config should load"); - assert_eq!(preview.payment_amount_e8s, 50_000_000); - assert_eq!(preview.ledger_fee_e8s, KINIC_LEDGER_FEE_E8S); - assert_eq!(preview.credits_per_kinic, 1_000); - assert_eq!(preview.config_version, 1); + assert_eq!( + cycles_for_payment_amount_e8s(50_000, &config).expect("cycles should compute"), + 117_250_000 + ); } #[test] -fn credit_purchase_begin_rejects_stale_expected_values_before_pending_create() { +fn cycles_purchase_begin_returns_current_config_amount() { let (service, root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); - let stale_amount = service - .begin_database_credit_purchase_with_ledger_details( - DatabaseCreditPurchaseWithLedgerDetails { + let start = service + .begin_database_cycles_purchase_with_ledger_details( + DatabaseCyclesPurchaseWithLedgerDetails { database_id: "alpha", caller: "payer", - credits: 500, - expected_payment_amount_e8s: 50_000_001, - expected_config_version: 1, - ledger: CreditsPendingLedgerDetailsInput { + payment_amount_e8s: 50_000, + min_expected_cycles: 1, + ledger: CyclesPendingLedgerDetailsInput { from_owner: "payer", from_subaccount: None, to_owner: "canister", @@ -1909,128 +2016,111 @@ fn credit_purchase_begin_rejects_stale_expected_values_before_pending_create() { now: 2, }, ) - .expect_err("stale amount should reject"); - assert!(stale_amount.contains("payment amount changed")); - assert_eq!(database_pending_operation_count(&root, "alpha"), 0); - - let stale_version = service - .begin_database_credit_purchase_with_ledger_details( - DatabaseCreditPurchaseWithLedgerDetails { - database_id: "alpha", - caller: "payer", - credits: 500, - expected_payment_amount_e8s: 50_000_000, - expected_config_version: 2, - ledger: CreditsPendingLedgerDetailsInput { - from_owner: "payer", - from_subaccount: None, - to_owner: "canister", - to_subaccount: None, - ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, - ledger_created_at_time_ns: 3_000_000, - }, - now: 3, - }, - ) - .expect_err("stale version should reject"); - assert!(stale_version.contains("credits config changed")); - assert_eq!(database_pending_operation_count(&root, "alpha"), 0); + .expect("purchase should begin"); + assert_eq!(start.amount_cycles, 117_250_000); + assert_eq!(database_pending_operation_count(&root, "alpha"), 1); } #[test] -fn database_credit_purchase_settlement_survives_owner_role_change() { +fn database_cycles_purchase_settlement_survives_owner_role_change() { let (service, root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); let operation_id = service - .begin_database_credit_purchase("alpha", "owner", 500, 2) - .expect("owner should start credit purchase"); + .begin_database_cycles_purchase("alpha", "owner", 500, 2) + .expect("owner should start cycle purchase"); + let purchased_cycles = cycles_for_payment(&service, "alpha", 500); service .grant_database_access("alpha", "owner", "replacement", DatabaseRole::Owner, 2) .expect("replacement owner should grant"); service .revoke_database_access("alpha", "replacement", "owner") .expect("replacement should revoke original owner"); - service - .mark_database_credit_purchase_completed(operation_id, "alpha", "owner", 500) - .expect("credit purchase should be marked completed"); + service + .complete_database_cycles_purchase_ledger_transfer( + operation_id, + "alpha", + "owner", + purchased_cycles, + 7, + ) + .expect("cycle purchase ledger transfer should complete"); let balance = service - .credit_database_purchase(operation_id, "alpha", "owner", 500, 7, 3) - .expect("started credit purchase should settle"); + .apply_database_cycles_purchase(operation_id, "alpha", "owner", purchased_cycles, 7, 3) + .expect("started cycle purchase should settle"); - assert_eq!(balance, 500); - assert_eq!(database_credits_balance(&root, "alpha"), 500); + assert_eq!(balance, purchased_cycles); + assert_eq!( + database_cycles_balance(&root, "alpha"), + purchased_cycles as i64 + ); assert_eq!( database_ledger_kinds(&root, "alpha"), - vec!["credit_purchase"] + vec!["cycles_purchase"] ); } #[test] -fn pending_database_credit_purchase_blocks_delete_until_resolved() { +fn pending_database_cycles_purchase_blocks_delete_until_resolved() { let (service, root) = service_with_root(); - for database_id in ["complete", "cancel", "ambiguous"] { + for database_id in ["complete", "cancel"] { service .create_database(database_id, "owner", 1) .expect("database should create"); } let complete = service - .begin_database_credit_purchase("complete", "owner", 500, 2) - .expect("credit purchase should begin"); + .begin_database_cycles_purchase("complete", "owner", 500, 2) + .expect("cycle purchase should begin"); let cancel = service - .begin_database_credit_purchase("cancel", "owner", 500, 2) - .expect("credit purchase should begin"); - let ambiguous = service - .begin_database_credit_purchase("ambiguous", "owner", 500, 2) - .expect("credit purchase should begin"); + .begin_database_cycles_purchase("cancel", "owner", 500, 2) + .expect("cycle purchase should begin"); + let purchased_cycles = cycles_for_payment(&service, "complete", 500); - for database_id in ["complete", "cancel", "ambiguous"] { + for database_id in ["complete", "cancel"] { let error = service .delete_database(delete_request(database_id), "owner", 3) - .expect_err("pending credit purchase should block delete"); - assert!(error.contains("pending credit operation")); + .expect_err("pending cycle purchase should block delete"); + assert!(error.contains("pending cycle operation")); assert_eq!(database_pending_operation_count(&root, database_id), 1); } service - .mark_database_credit_purchase_completed(complete, "complete", "owner", 500) - .expect("credit purchase should be marked completed"); - service - .credit_database_purchase(complete, "complete", "owner", 500, 10, 4) - .expect("credit purchase should complete"); + .complete_database_cycles_purchase_ledger_transfer( + complete, + "complete", + "owner", + purchased_cycles, + 10, + ) + .expect("cycle purchase ledger transfer should complete"); service - .cancel_database_credit_purchase(cancel, "cancel", "owner", 500) - .expect("credit purchase should cancel"); + .apply_database_cycles_purchase(complete, "complete", "owner", purchased_cycles, 10, 4) + .expect("cycle purchase should complete"); service - .mark_database_credit_purchase_ambiguous(ambiguous, "ambiguous", "owner", 500, 4) - .expect("ambiguous credit purchase should record"); + .cancel_database_cycles_purchase(cancel, "cancel", "owner", purchased_cycles) + .expect("cycle purchase should cancel"); for database_id in ["complete", "cancel"] { assert_eq!(database_pending_operation_count(&root, database_id), 0); service .delete_database(delete_request(database_id), "owner", 5) - .expect("resolved credit purchase should allow delete"); + .expect("resolved cycle purchase should allow delete"); } - assert_eq!(database_pending_operation_count(&root, "ambiguous"), 1); - let error = service - .delete_database(delete_request("ambiguous"), "owner", 5) - .expect_err("ambiguous credit purchase should keep delete blocked"); - assert!(error.contains("pending credit operation")); } #[test] -fn delete_database_removes_index_rows_and_discards_remaining_credits() { +fn delete_database_removes_index_rows_and_discards_remaining_cycles() { let (service, root) = service_with_root(); service .create_database("funded", "owner", 1) .expect("database should create"); - credit_database(&service, "funded", "owner", 100_000_000, 1, 2); + cycle_database(&service, "funded", "owner", 100_000_000, 1, 2); service .delete_database(delete_request("funded"), "owner", 3) - .expect("remaining credits should be discarded on delete"); + .expect("remaining cycles should be discarded on delete"); assert!(!database_index_row_exists(&root, "funded")); assert_eq!(database_member_count(&root, "funded"), 0); assert_eq!(database_pending_operation_count(&root, "funded"), 0); @@ -2038,7 +2128,7 @@ fn delete_database_removes_index_rows_and_discards_remaining_credits() { } #[test] -fn pending_credits_operations_are_visible_to_owner_and_governance_only() { +fn cycles_history_redacts_principals_for_non_owner_readers() { let (service, _root) = service_with_root(); service .create_database("alpha", "owner", 1) @@ -2049,111 +2139,10 @@ fn pending_credits_operations_are_visible_to_owner_and_governance_only() { service .grant_database_access("alpha", "owner", "reader", DatabaseRole::Reader, 2) .expect("reader should be granted"); - service - .begin_database_credit_purchase("alpha", "payer", 500, 3) - .expect("credit purchase should begin"); - - let owner_page = service - .list_database_credit_pending_operations("alpha", "owner", None, 10) - .expect("owner should list pending operations"); - assert_eq!(owner_page.entries.len(), 1); - assert_eq!(owner_page.entries[0].kind, "credit_purchase"); - assert_eq!(owner_page.entries[0].caller, "payer"); - assert_eq!(owner_page.entries[0].operation_status, "in_flight"); - assert_eq!(owner_page.entries[0].credits, 500); - assert_eq!(owner_page.entries[0].payment_amount_e8s, 50_000_000); - assert_eq!(owner_page.entries[0].ledger_fee_e8s, Some(0)); - - let governance_page = service - .list_database_credit_pending_operations("alpha", "rrkah-fqaaa-aaaaa-aaaaq-cai", None, 10) - .expect("governance should list pending operations"); - assert_eq!(governance_page.entries.len(), 1); - - for caller in ["writer", "reader", "2vxsx-fae"] { - let error = service - .list_database_credit_pending_operations("alpha", caller, None, 10) - .expect_err("non-owner should not list pending operations"); - assert!( - error.contains("principal lacks required database role") - || error.contains("principal has no access") - ); - } -} - -#[test] -fn repair_credit_purchase_cancel_allows_payer_or_owner_only() { - let (service, root) = service_with_root(); - for database_id in ["payer-cancel", "owner-cancel", "reject-cancel"] { - service - .create_database(database_id, "owner", 1) - .expect("database should create"); - service - .grant_database_access(database_id, "owner", "writer", DatabaseRole::Writer, 2) - .expect("writer should be granted"); - service - .grant_database_access(database_id, "owner", "reader", DatabaseRole::Reader, 3) - .expect("reader should be granted"); - } - - let payer_cancel = service - .begin_database_credit_purchase("payer-cancel", "payer", 500, 4) - .expect("payer cancel operation should begin"); - service - .mark_database_credit_purchase_ambiguous(payer_cancel, "payer-cancel", "payer", 500, 5) - .expect("payer cancel should mark ambiguous"); - service - .repair_database_credit_purchase_cancel("payer-cancel", payer_cancel, "payer", 6) - .expect("payer should cancel own pending purchase"); - assert_eq!(database_pending_operation_count(&root, "payer-cancel"), 0); - - let owner_cancel = service - .begin_database_credit_purchase("owner-cancel", "payer", 700, 7) - .expect("owner cancel operation should begin"); - service - .mark_database_credit_purchase_ambiguous(owner_cancel, "owner-cancel", "payer", 700, 8) - .expect("owner cancel should mark ambiguous"); - service - .repair_database_credit_purchase_cancel("owner-cancel", owner_cancel, "owner", 9) - .expect("owner should cancel pending purchase"); - assert_eq!( - database_ledger_kinds(&root, "owner-cancel"), - vec![ - "credit_purchase_ambiguous", - "credit_purchase_repair_cancelled" - ] - ); - - let reject_cancel = service - .begin_database_credit_purchase("reject-cancel", "payer", 900, 10) - .expect("reject cancel operation should begin"); - service - .mark_database_credit_purchase_ambiguous(reject_cancel, "reject-cancel", "payer", 900, 11) - .expect("reject cancel should mark ambiguous"); - for caller in ["writer", "reader", "outsider"] { - let error = service - .repair_database_credit_purchase_cancel("reject-cancel", reject_cancel, caller, 12) - .expect_err("non-payer non-owner should reject"); - assert!(error.contains("not credit purchase payer or database owner")); - assert_eq!(database_pending_operation_count(&root, "reject-cancel"), 1); - } -} - -#[test] -fn credits_history_redacts_principals_for_non_owner_readers() { - let (service, _root) = service_with_root(); - service - .create_database("alpha", "owner", 1) - .expect("database should create"); - service - .grant_database_access("alpha", "owner", "writer", DatabaseRole::Writer, 2) - .expect("writer should be granted"); - service - .grant_database_access("alpha", "owner", "reader", DatabaseRole::Reader, 2) - .expect("reader should be granted"); - credit_database(&service, "alpha", "payer-principal", 500, 42, 3); + cycle_database(&service, "alpha", "payer-principal", 500, 42, 3); let reader_entry = service - .list_database_credit_entries("alpha", "reader", None, 10) + .list_database_cycle_entries("alpha", "reader", None, 10) .expect("reader should list history") .entries .remove(0); @@ -2161,167 +2150,64 @@ fn credits_history_redacts_principals_for_non_owner_readers() { assert_eq!(reader_entry.ledger_block_index, Some(42)); let writer_entry = service - .list_database_credit_entries("alpha", "writer", None, 10) + .list_database_cycle_entries("alpha", "writer", None, 10) .expect("writer should list history") .entries .remove(0); assert_eq!(writer_entry.caller, "redacted"); let owner_entry = service - .list_database_credit_entries("alpha", "owner", None, 10) + .list_database_cycle_entries("alpha", "owner", None, 10) .expect("owner should list history") .entries .remove(0); assert_eq!(owner_entry.caller, "payer-principal"); - let governance_entry = service - .list_database_credit_entries("alpha", "rrkah-fqaaa-aaaaa-aaaaq-cai", None, 10) - .expect("governance should list history without membership") + let billing_authority_entry = service + .list_database_cycle_entries("alpha", "rrkah-fqaaa-aaaaa-aaaaq-cai", None, 10) + .expect("billing authority should list history without membership") .entries .remove(0); - assert_eq!(governance_entry.caller, "payer-principal"); + assert_eq!(billing_authority_entry.caller, "payer-principal"); let error = service - .list_database_credit_entries("alpha", "outsider", None, 10) + .list_database_cycle_entries("alpha", "outsider", None, 10) .expect_err("outsider should not list history"); assert!(error.contains("principal has no access")); } #[test] -fn verified_complete_allows_authenticated_caller_and_owner_cancel() { - let (service, root) = service_with_root(); - service - .create_database("complete", "owner", 1) - .expect("database should create"); - service - .create_database("cancel", "owner", 1) - .expect("database should create"); - let complete = service - .begin_database_credit_purchase("complete", "payer", 500, 2) - .expect("credit purchase should begin"); - let cancel = service - .begin_database_credit_purchase("cancel", "payer", 700, 2) - .expect("credit purchase should begin"); - service - .mark_database_credit_purchase_ambiguous(complete, "complete", "payer", 500, 3) - .expect("credit purchase ambiguity should record"); - service - .mark_database_credit_purchase_ambiguous(cancel, "cancel", "payer", 700, 3) - .expect("credit purchase ambiguity should record"); - let pending = service - .list_database_credit_pending_operations("complete", "owner", None, 10) - .expect("owner should list ambiguous pending operations") - .entries; - assert_eq!(pending[0].operation_status, "ambiguous"); - - let balance = service - .repair_database_credit_purchase_complete("complete", complete, 77, 4) - .expect("authenticated caller should complete verified credit purchase"); - assert_eq!(balance, 500); - service - .repair_database_credit_purchase_cancel("cancel", cancel, "owner", 4) - .expect("owner should cancel ambiguous credit purchase after verification"); - - assert_eq!(database_credits_balance(&root, "complete"), 500); - assert_eq!(database_credits_balance(&root, "cancel"), 0); - assert_eq!(database_pending_operation_count(&root, "complete"), 0); - assert_eq!(database_pending_operation_count(&root, "cancel"), 0); - assert_eq!( - database_ledger_kinds(&root, "complete"), - vec![ - "credit_purchase_ambiguous", - "credit_purchase_repair_complete" - ] - ); - assert_eq!( - database_ledger_kinds(&root, "cancel"), - vec![ - "credit_purchase_ambiguous", - "credit_purchase_repair_cancelled" - ] - ); - let entries = service - .list_database_credit_entries("complete", "owner", None, 10) - .expect("credits entries should load") - .entries; - assert_eq!(entries[0].amount_credits, 0); - assert_eq!(entries[0].balance_after_credits, 0); - assert_eq!(entries[1].amount_credits, 500); - assert_eq!(entries[1].balance_after_credits, 500); - assert_eq!(entries[0].caller, "payer"); - assert_eq!(entries[1].caller, "payer"); - assert_eq!(entries[1].ledger_block_index, Some(77)); - - let cancel_entries = service - .list_database_credit_entries("cancel", "owner", None, 10) - .expect("cancel entries should load") - .entries; - assert_eq!(cancel_entries[1].caller, "owner"); - assert_eq!(cancel_entries[0].amount_credits, 0); - assert_eq!(cancel_entries[0].balance_after_credits, 0); - assert_eq!(cancel_entries[1].amount_credits, 0); - assert_eq!(cancel_entries[1].payment_amount_e8s, Some(70_000_000)); - assert_eq!(cancel_entries[1].balance_after_credits, 0); - assert_eq!(cancel_entries[1].ledger_block_index, None); -} - -#[test] -fn verified_repair_mark_blocks_cancel_before_credit_apply() { - let (service, root) = service_with_root(); +fn cycles_history_paginates_with_clamped_limits() { + let (service, _root) = service_with_root(); service - .create_database("complete", "owner", 1) + .create_database("alpha", "owner", 1) .expect("database should create"); - let operation_id = service - .begin_database_credit_purchase("complete", "payer", 500, 2) - .expect("credit purchase should begin"); - service - .mark_database_credit_purchase_ambiguous(operation_id, "complete", "payer", 500, 3) - .expect("credit purchase ambiguity should record"); + for index in 0..105 { + cycle_database(&service, "alpha", "owner", 500, index + 1, index as i64 + 2); + } - service - .mark_database_credit_purchase_repair_completed("complete", operation_id) - .expect("verified repair should mark operation completed"); - let cancel_error = service - .repair_database_credit_purchase_cancel("complete", operation_id, "owner", 4) - .expect_err("completed repair should not be cancellable"); - assert!(cancel_error.contains("credit purchase operation is completed")); + let minimum_page = service + .list_database_cycle_entries("alpha", "owner", None, 0) + .expect("minimum page should load"); + assert_eq!(minimum_page.entries.len(), 1); + assert_eq!(minimum_page.entries[0].entry_id, 1); + assert_eq!(minimum_page.next_cursor, Some(1)); - let balance = service - .repair_database_credit_purchase_complete("complete", operation_id, 77, 5) - .expect("completed repair should still credit database"); - assert_eq!(balance, 500); - assert_eq!(database_credits_balance(&root, "complete"), 500); - assert_eq!(database_pending_operation_count(&root, "complete"), 0); -} + let first_page = service + .list_database_cycle_entries("alpha", "owner", None, 200) + .expect("first clamped page should load"); + assert_eq!(first_page.entries.len(), 100); + assert_eq!(first_page.entries[0].entry_id, 1); + assert_eq!(first_page.entries[99].entry_id, 100); + assert_eq!(first_page.next_cursor, Some(100)); -#[test] -fn verified_repair_mark_accepts_in_flight_credit_purchase() { - let (service, root) = service_with_root(); - service - .create_database("complete", "owner", 1) - .expect("database should create"); - let operation_id = service - .begin_database_credit_purchase("complete", "payer", 500, 2) - .expect("credit purchase should begin"); - let operation = service - .get_database_credit_pending_operation_for_complete("complete", operation_id) - .expect("verified repair should inspect in-flight operation"); - assert_eq!(operation.operation_id, operation_id); - - service - .mark_database_credit_purchase_repair_completed("complete", operation_id) - .expect("verified repair should mark in-flight operation completed"); - let cancel_error = service - .repair_database_credit_purchase_cancel("complete", operation_id, "owner", 4) - .expect_err("completed repair should not be cancellable"); - assert!(cancel_error.contains("credit purchase operation is completed")); - - let balance = service - .repair_database_credit_purchase_complete("complete", operation_id, 77, 5) - .expect("completed repair should credit database"); - assert_eq!(balance, 500); - assert_eq!(database_credits_balance(&root, "complete"), 500); - assert_eq!(database_pending_operation_count(&root, "complete"), 0); + let second_page = service + .list_database_cycle_entries("alpha", "owner", first_page.next_cursor, 200) + .expect("second clamped page should load"); + assert_eq!(second_page.entries.len(), 5); + assert_eq!(second_page.entries[0].entry_id, 101); + assert_eq!(second_page.entries[4].entry_id, 105); + assert_eq!(second_page.next_cursor, None); } #[test] @@ -2351,45 +2237,97 @@ fn database_rename_requires_owner() { } #[test] -fn zero_cycle_charge_skips_credit_ledger() { +fn zero_cycle_charge_skips_cycle_ledger() { let (service, root) = service_with_root(); service .create_database("alpha", "owner", 1) .expect("database should create"); - credit_database(&service, "alpha", "owner", 500, 7, 2); + let purchased_cycles = cycle_database(&service, "alpha", "owner", 5_000, 7, 2); let config = service - .credits_config() - .expect("credits config should load"); + .cycles_billing_config() + .expect("cycles config should load"); service .charge_database_update(&config, "alpha", "owner", "write_node", 0, 3) .expect("zero-cycle update should skip charge"); - assert_eq!(database_credits_balance(&root, "alpha"), 500); + assert_eq!( + database_cycles_balance(&root, "alpha"), + purchased_cycles as i64 + ); assert_eq!( database_ledger_kinds(&root, "alpha"), - vec!["credit_purchase"] + vec!["cycles_purchase"] ); service - .charge_database_update(&config, "alpha", "owner", "write_node", 1_000_000_000, 4) - .expect("charged update should record credit ledger"); + .charge_database_update(&config, "alpha", "owner", "write_node", 1_000_000, 4) + .expect("charged update should record cycle ledger"); + + let after_first_charge = purchased_cycles as i64 - 1_000_000; + assert_eq!(database_cycles_balance(&root, "alpha"), after_first_charge); + service + .charge_database_update(&config, "alpha", "owner", "write_node", 1_000_001, 5) + .expect("raw update cycle charge should record cycle ledger"); - assert_eq!(database_credits_balance(&root, "alpha"), 499); + let after_second_charge = after_first_charge - 1_000_001; + assert_eq!(database_cycles_balance(&root, "alpha"), after_second_charge); service - .charge_database_update(&config, "alpha", "owner", "write_node", 1_000_000_001, 5) - .expect("rounded-up update should record credit ledger"); + .charge_database_update( + &config, + "alpha", + "owner", + "write_node", + u128::try_from(after_second_charge).expect("remaining balance should fit") + 1, + 6, + ) + .expect("overdrawn update cycle charge should consume remaining balance"); + assert_eq!(database_cycles_balance(&root, "alpha"), 0); + assert_eq!(database_cycles_suspended_at(&root, "alpha"), Some(6)); + assert_eq!( + database_ledger_kinds(&root, "alpha"), + vec!["cycles_purchase", "charge", "charge", "charge"] + ); + + let overdrawn_entries = service + .list_database_cycle_entries("alpha", "owner", None, 10) + .expect("cycle entries should load") + .entries; + assert_eq!(overdrawn_entries[3].kind, "charge"); + assert_eq!(overdrawn_entries[3].amount_cycles, -after_second_charge); + assert_eq!( + overdrawn_entries[3].cycles_delta, + Some(after_second_charge as u64 + 1) + ); - assert_eq!(database_credits_balance(&root, "alpha"), 497); + let (service, root) = service_with_root(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + let purchased_cycles = cycle_database(&service, "alpha", "owner", 5_000, 7, 2); + let config = service + .cycles_billing_config() + .expect("cycles config should load"); + + service + .charge_database_update( + &config, + "alpha", + "owner", + "write_node", + u128::from(purchased_cycles), + 7, + ) + .expect("exact balance cycle charge should succeed"); + + assert_eq!(database_cycles_balance(&root, "alpha"), 0); let entries = service - .list_database_credit_entries("alpha", "owner", None, 10) - .expect("credit entries should load") + .list_database_cycle_entries("alpha", "owner", None, 10) + .expect("cycle entries should load") .entries; - assert_eq!(entries.len(), 3); + assert_eq!(entries.len(), 2); assert_eq!(entries[1].kind, "charge"); - assert_eq!(entries[1].amount_credits, -1); - assert_eq!(entries[2].kind, "charge"); - assert_eq!(entries[2].amount_credits, -2); + assert_eq!(entries[1].amount_cycles, -(purchased_cycles as i64)); } #[test] @@ -2452,6 +2390,44 @@ fn lists_database_summaries_for_caller_memberships_only() { assert!(outsider_summaries.is_empty()); } +#[test] +fn grant_database_access_enforces_member_limit_for_new_members_only() { + let (service, root) = service_with_root(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + + for index in 0..30 { + service + .grant_database_access( + "alpha", + "owner", + &format!("member-{index}"), + DatabaseRole::Reader, + 2 + index, + ) + .expect("member grant should fit limit"); + } + assert_eq!(database_member_count(&root, "alpha"), 32); + + service + .grant_database_access("alpha", "owner", "member-0", DatabaseRole::Writer, 40) + .expect("existing member role update should ignore member cap"); + + let error = service + .grant_database_access("alpha", "owner", "member-30", DatabaseRole::Reader, 41) + .expect_err("new member beyond cap should fail"); + assert!(error.contains("too many database members")); + + service + .revoke_database_access("alpha", "owner", "member-1") + .expect("member revoke should succeed"); + service + .grant_database_access("alpha", "owner", "member-30", DatabaseRole::Reader, 42) + .expect("new member should fit after revoke"); + assert_eq!(database_member_count(&root, "alpha"), 32); +} + #[test] fn discards_failed_database_reservation_for_retry() { let (service, root) = service_with_root(); @@ -2472,6 +2448,131 @@ fn discards_failed_database_reservation_for_retry() { assert_eq!(database_member_count(&root, "retryable"), 2); } +#[test] +fn database_cycles_purchase_rejects_duplicate_pending_operation_for_caller() { + let (service, root) = service_with_root(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + + let first = service + .begin_database_cycles_purchase("alpha", "payer", 500, 2) + .expect("first purchase should begin"); + let duplicate = service + .begin_database_cycles_purchase("alpha", "payer", 600, 3) + .expect_err("same caller should not create duplicate pending purchase"); + assert!(duplicate.contains("cycles purchase already pending for caller")); + + service + .begin_database_cycles_purchase("alpha", "other-payer", 600, 4) + .expect("different caller can begin separate purchase"); + + let cycles = default_cycles_for_payment(500); + service + .cancel_database_cycles_purchase(first, "alpha", "payer", cycles) + .expect("first purchase should cancel"); + service + .begin_database_cycles_purchase("alpha", "payer", 700, 5) + .expect("caller can begin after pending operation resolves"); + + assert_eq!(database_pending_operation_count(&root, "alpha"), 2); +} + +#[test] +fn database_cycles_purchase_rejects_when_cycles_below_minimum_quote() { + let (service, root) = service_with_root(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + let cycles = default_cycles_for_payment(50_000); + + let error = service + .begin_database_cycles_purchase_with_ledger_details( + DatabaseCyclesPurchaseWithLedgerDetails { + database_id: "alpha", + caller: "payer", + payment_amount_e8s: 50_000, + min_expected_cycles: cycles + 1, + ledger: CyclesPendingLedgerDetailsInput { + from_owner: "payer", + from_subaccount: None, + to_owner: "canister", + to_subaccount: None, + ledger_fee_e8s: KINIC_LEDGER_FEE_E8S, + ledger_created_at_time_ns: 2_000_000, + }, + now: 2, + }, + ) + .expect_err("stale quote should reject before pending operation"); + + assert!(error.contains("below min_expected_cycles")); + assert_eq!(database_pending_operation_count(&root, "alpha"), 0); +} + +#[test] +fn lists_pending_cycles_purchases_for_owner_authority_and_payer() { + let service = service(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + let operation_id = service + .begin_database_cycles_purchase("alpha", "payer", 500, 2) + .expect("purchase should begin"); + + let owner = service + .list_database_cycles_pending_purchases("alpha", "owner") + .expect("owner should view pending purchase"); + assert_eq!(owner.len(), 1); + assert_eq!(owner[0].operation_id, operation_id); + assert_eq!(owner[0].status, "in_flight"); + assert_eq!(owner[0].required_action, "wait_for_ledger_result"); + + let payer = service + .list_database_cycles_pending_purchases("alpha", "payer") + .expect("payer should view own pending purchase"); + assert_eq!(payer, owner); + + let authority = service + .cycles_billing_config() + .expect("config should load") + .billing_authority_id; + let authority_view = service + .list_database_cycles_pending_purchases("alpha", &authority) + .expect("billing authority should view pending purchase"); + assert_eq!(authority_view, owner); + + let error = service + .list_database_cycles_pending_purchases("alpha", "stranger") + .expect_err("unrelated caller should reject"); + assert!(error.contains("cannot view pending cycle purchases")); +} + +#[test] +fn delete_database_reports_pending_cycles_purchase_action() { + let service = service(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + let operation_id = service + .begin_database_cycles_purchase("alpha", "payer", 500, 2) + .expect("purchase should begin"); + + let error = service + .delete_database( + DeleteDatabaseRequest { + database_id: "alpha".to_string(), + }, + "owner", + 3, + ) + .expect_err("delete should block while pending purchase exists"); + + assert!(error.contains(&format!("operation_id={operation_id}"))); + assert!(error.contains("status=in_flight")); + assert!(error.contains("required_action=wait_for_ledger_result")); +} + #[test] fn rejects_invalid_database_ids() { let service = service(); @@ -2866,7 +2967,12 @@ fn archives_and_restores_database_bytes() { ); assert_eq!( database_index_row(&root, "alpha"), - ("archived".to_string(), None, archive.size_bytes, None) + ( + "archived".to_string(), + archiving_mount_id, + archive.size_bytes, + None, + ) ); assert!( service @@ -2948,7 +3054,7 @@ fn archives_and_restores_database_bytes() { } #[test] -fn restored_mount_id_is_not_reused_after_rearchive() { +fn restore_reuses_archived_mount_id_after_rearchive() { let (service, root) = service_with_root(); service .create_database("alpha", "owner", 1) @@ -2979,6 +3085,7 @@ fn restored_mount_id_is_not_reused_after_rearchive() { let restored = service .begin_database_restore("alpha", "owner", snapshot_hash, archive.size_bytes, 4) .expect("restore should begin"); + assert_eq!(restored.mount_id, 11); service .write_database_restore_chunk("alpha", "owner", 0, &bytes) .expect("restore chunk should write"); @@ -3001,7 +3108,7 @@ fn restored_mount_id_is_not_reused_after_rearchive() { assert_ne!(beta.mount_id, restored.mount_id); assert_eq!( mount_history_row(&root, restored.mount_id), - ("alpha".to_string(), "restore".to_string()) + ("alpha".to_string(), "create".to_string()) ); } @@ -3504,14 +3611,19 @@ fn cancel_database_restore_returns_archived_database_and_removes_partial_state() assert_eq!( database_index_row(&root, "alpha"), - ("archived".to_string(), None, archive.size_bytes, None) + ( + "archived".to_string(), + Some(restore.meta.mount_id), + archive.size_bytes, + None, + ) ); assert_eq!(database_restore_chunk_count(&root, "alpha"), 0); assert_eq!(database_restore_session_count(&root, "alpha"), 0); assert!(!restoring_file.exists()); assert_eq!( mount_history_row(&root, restore.meta.mount_id), - ("alpha".to_string(), "restore".to_string()) + ("alpha".to_string(), "create".to_string()) ); } @@ -3649,19 +3761,24 @@ fn rollback_database_restore_begin_restores_archived_state() { .expect("restore begin should rollback"); assert_eq!( database_index_row(&root, "alpha"), - ("archived".to_string(), None, archive.size_bytes, None) + ( + "archived".to_string(), + Some(failed_mount_id), + archive.size_bytes, + None, + ) ); assert_eq!(database_restore_chunk_count(&root, "alpha"), 0); assert_eq!(database_restore_session_count(&root, "alpha"), 0); assert_eq!( mount_history_row(&root, failed_mount_id), - ("alpha".to_string(), "restore".to_string()) + ("alpha".to_string(), "create".to_string()) ); let retry = service .begin_database_restore_session("alpha", "owner", snapshot_hash, archive.size_bytes, 7) .expect("restore should retry"); - assert_ne!(retry.meta.mount_id, failed_mount_id); + assert_eq!(retry.meta.mount_id, failed_mount_id); } #[test] @@ -3805,6 +3922,43 @@ fn append_node_validates_effective_kind_paths() { .expect_err("kind=None under sources should be treated as file"); assert!(error.contains("source path must use source kind")); + ensure_parent_folders( + &service, + "owner", + "alpha", + "/Sources/skill-runs/review/1700000000000.md", + 3, + ); + let error = service + .write_node( + "owner", + WriteNodeRequest { + database_id: "alpha".to_string(), + path: "/Sources/skill-runs/review/1700000000000.md".to_string(), + kind: NodeKind::File, + content: "bad".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 4, + ) + .expect_err("skill run source path should reject file kind"); + assert!(error.contains("source path must use source kind")); + service + .write_node( + "owner", + WriteNodeRequest { + database_id: "alpha".to_string(), + path: "/Sources/skill-runs/review/1700000000000.md".to_string(), + kind: NodeKind::Source, + content: "source".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 5, + ) + .expect("skill run source path should accept source kind"); + ensure_parent_folders(&service, "owner", "alpha", "/Sources/raw/good/good.md", 3); let source = service .write_node( @@ -3854,3 +4008,57 @@ fn append_node_validates_effective_kind_paths() { .expect("kind=None should create wiki file"); assert_eq!(wiki.node.kind, NodeKind::File); } + +#[test] +fn move_node_validates_source_target_path() { + let service = service(); + service + .create_database("alpha", "owner", 1) + .expect("database should create"); + ensure_parent_folders(&service, "owner", "alpha", "/Sources/raw/web/abc.md", 2); + ensure_parent_folders(&service, "owner", "alpha", "/Sources/raw/web/wrong.md", 2); + ensure_parent_folders(&service, "owner", "alpha", "/Sources/raw/chatgpt/def.md", 2); + let source = service + .write_node( + "owner", + WriteNodeRequest { + database_id: "alpha".to_string(), + path: "/Sources/raw/web/abc.md".to_string(), + kind: NodeKind::Source, + content: "source".to_string(), + metadata_json: "{}".to_string(), + expected_etag: None, + }, + 3, + ) + .expect("source should write"); + + let error = service + .move_node( + "owner", + MoveNodeRequest { + database_id: "alpha".to_string(), + from_path: "/Sources/raw/web/abc.md".to_string(), + to_path: "/Sources/raw/web/wrong.txt".to_string(), + expected_etag: Some(source.node.etag.clone()), + overwrite: false, + }, + 4, + ) + .expect_err("non-canonical source target should fail"); + assert!(error.contains("canonical form")); + + service + .move_node( + "owner", + MoveNodeRequest { + database_id: "alpha".to_string(), + from_path: "/Sources/raw/web/abc.md".to_string(), + to_path: "/Sources/raw/chatgpt/def.md".to_string(), + expected_etag: Some(source.node.etag), + overwrite: false, + }, + 5, + ) + .expect("canonical source target should pass"); +} diff --git a/crates/vfs_runtime/tests/database_service_pbt.rs b/crates/vfs_runtime/tests/database_service_pbt.rs index f1890c42..e38503bb 100644 --- a/crates/vfs_runtime/tests/database_service_pbt.rs +++ b/crates/vfs_runtime/tests/database_service_pbt.rs @@ -1,5 +1,5 @@ // Where: crates/vfs_runtime/tests/database_service_pbt.rs -// What: Property tests for database credits and lifecycle operation sequences. +// What: Property tests for database cycles and lifecycle operation sequences. // Why: Randomized state-machine checks catch partial updates across balances, status, and mounts. use std::path::{Path, PathBuf}; @@ -7,15 +7,15 @@ use proptest::prelude::*; use proptest::test_runner::{Config as ProptestConfig, FileFailurePersistence}; use sha2::{Digest, Sha256}; use tempfile::{TempDir, tempdir}; -use vfs_runtime::{CYCLES_PER_CREDIT, VfsService}; +use vfs_runtime::{VfsService, cycles_for_payment_amount_e8s}; use vfs_types::DatabaseStatus; const OWNER: &str = "owner"; -const INITIAL_DATABASE_CREDITS: u64 = 1_000; +const INITIAL_DATABASE_PAYMENT_E8S: u64 = 1_000; #[derive(Clone, Debug)] enum RuntimeOp { - PurchaseDatabaseCredits { amount: u64 }, + PurchaseDatabaseCycles { amount: u64 }, Charge { cycles_delta: u128 }, ArchiveFinalize, RestoreArchived, @@ -23,7 +23,7 @@ enum RuntimeOp { #[derive(Debug)] struct Model { - database_credits: u64, + database_cycles: u64, status: DatabaseStatus, archive_bytes: Option>, archive_hash: Option>, @@ -49,7 +49,7 @@ fn property_config() -> ProptestConfig { fn operation_strategy() -> impl Strategy { prop_oneof![ - 4 => (1_u64..=250_000).prop_map(|amount| RuntimeOp::PurchaseDatabaseCredits { amount }), + 4 => (1_u64..=250_000).prop_map(|amount| RuntimeOp::PurchaseDatabaseCycles { amount }), 4 => (0_u128..=20_000_u128).prop_map(|cycles_delta| RuntimeOp::Charge { cycles_delta }), 2 => Just(RuntimeOp::ArchiveFinalize), 2 => Just(RuntimeOp::RestoreArchived), @@ -70,33 +70,50 @@ fn service_with_root() -> TestService { } } -fn create_seeded_database(service: &VfsService) -> String { +fn create_seeded_database(service: &VfsService) -> (String, u64) { let meta = service .create_generated_database("PBT database", OWNER, 2) .expect("database should create"); - credit_database( + let initial_cycles = purchase_database_cycles( service, &meta.database_id, OWNER, - INITIAL_DATABASE_CREDITS, + INITIAL_DATABASE_PAYMENT_E8S, 1, 3, ) - .expect("database seed should credit"); - meta.database_id + .expect("database seed should cycle"); + (meta.database_id, initial_cycles) } -fn credit_database( +fn purchase_database_cycles( service: &VfsService, database_id: &str, caller: &str, - credits: u64, + payment_amount_e8s: u64, block_index: u64, now: i64, ) -> Result { - let operation_id = service.begin_database_credit_purchase(database_id, caller, credits, now)?; - service.mark_database_credit_purchase_completed(operation_id, database_id, caller, credits)?; - service.credit_database_purchase(operation_id, database_id, caller, credits, block_index, now) + let operation_id = + service.begin_database_cycles_purchase(database_id, caller, payment_amount_e8s, now)?; + let config = service.cycles_billing_config()?; + let cycles = cycles_for_payment_amount_e8s(payment_amount_e8s, &config)?; + service.complete_database_cycles_purchase_ledger_transfer( + operation_id, + database_id, + caller, + cycles, + block_index, + )?; + service.apply_database_cycles_purchase( + operation_id, + database_id, + caller, + cycles, + block_index, + now, + )?; + Ok(cycles) } fn status_and_mount(service: &VfsService, database_id: &str) -> (DatabaseStatus, Option) { @@ -144,30 +161,23 @@ fn database_bytes(root: &Path, service: &VfsService, database_id: &str) -> (Vec< } fn charge_amount(cycles_delta: u128) -> u64 { - cycles_delta - .div_ceil(CYCLES_PER_CREDIT) - .try_into() - .expect("generated charge fits u64") + u64::try_from(cycles_delta).expect("generated charge fits u64") } fn assert_invariants(service: &VfsService, database_id: &str, model: &Model) { let database_entries = service - .list_database_credit_entries(database_id, OWNER, None, 100) + .list_database_cycle_entries(database_id, OWNER, None, 100) .expect("database ledger should load") .entries; let database_after = database_entries .last() .expect("database ledger should not be empty") - .balance_after_credits; - assert_eq!(database_after, model.database_credits); + .balance_after_cycles; + assert_eq!(database_after, model.database_cycles); let (status, mount_id) = status_and_mount(service, database_id); assert_eq!(status, model.status); - if matches!(status, DatabaseStatus::Archived) { - assert_eq!(mount_id, None); - } else { - assert!(mount_id.is_some()); - } + assert!(mount_id.is_some()); let infos = service.list_database_infos().expect("infos should load"); let mut mount_ids = infos @@ -191,16 +201,25 @@ fn apply_operation( step: i64, ) { match operation { - RuntimeOp::PurchaseDatabaseCredits { amount } => { - let result = - credit_database(service, database_id, OWNER, amount, step as u64 + 10, step); - result.expect("database credit purchase should succeed"); - model.database_credits += amount; + RuntimeOp::PurchaseDatabaseCycles { amount } => { + let result = purchase_database_cycles( + service, + database_id, + OWNER, + amount, + step as u64 + 10, + step, + ); + if model.status == DatabaseStatus::Active { + model.database_cycles += result.expect("database cycle purchase should succeed"); + } else { + assert!(result.is_err()); + } } RuntimeOp::Charge { cycles_delta } => { let config = service - .credits_config() - .expect("credits config should load"); + .cycles_billing_config() + .expect("cycles config should load"); let result = service.charge_database_update( &config, database_id, @@ -209,9 +228,9 @@ fn apply_operation( cycles_delta, step, ); - result.expect("database charge should record against credit account"); - let charge = model.database_credits.min(charge_amount(cycles_delta)); - model.database_credits -= charge; + result.expect("database charge should record against cycle account"); + let charge = model.database_cycles.min(charge_amount(cycles_delta)); + model.database_cycles -= charge; } RuntimeOp::ArchiveFinalize => { if model.status == DatabaseStatus::Active { @@ -286,9 +305,9 @@ proptest! { fn database_service_pbt(operations in prop::collection::vec(operation_strategy(), 1..40)) { let env = service_with_root(); let service = &env.service; - let database_id = create_seeded_database(service); + let (database_id, initial_cycles) = create_seeded_database(service); let mut model = Model { - database_credits: INITIAL_DATABASE_CREDITS, + database_cycles: initial_cycles, status: DatabaseStatus::Active, archive_bytes: None, archive_hash: None, diff --git a/crates/vfs_runtime/tests/database_service_pbt_ext.rs b/crates/vfs_runtime/tests/database_service_pbt_ext.rs index da0ae905..3aa4d971 100644 --- a/crates/vfs_runtime/tests/database_service_pbt_ext.rs +++ b/crates/vfs_runtime/tests/database_service_pbt_ext.rs @@ -1,5 +1,5 @@ // Where: crates/vfs_runtime/tests/database_service_pbt_ext.rs -// What: Supplemental property tests for credits suspension, mount history, and restore chunks. +// What: Supplemental property tests for cycles suspension, mount history, and restore chunks. // Why: The main PBT covers common flows; these tests target branch-risky edge state. use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -9,15 +9,15 @@ use proptest::test_runner::{Config as ProptestConfig, FileFailurePersistence}; use rusqlite::{Connection, params}; use sha2::{Digest, Sha256}; use tempfile::{TempDir, tempdir}; -use vfs_runtime::{CYCLES_PER_CREDIT, DEFAULT_MIN_UPDATE_CREDITS, VfsService}; +use vfs_runtime::{DEFAULT_MIN_UPDATE_CYCLES, VfsService, cycles_for_payment_amount_e8s}; use vfs_types::{DatabaseStatus, DeleteDatabaseRequest, NodeKind, WriteNodeRequest}; const OWNER: &str = "owner"; -const DEPOSIT_CREDITS: u64 = 1_000; +const DEPOSIT_PAYMENT_E8S: u64 = 1_000; #[derive(Clone, Debug)] -enum CreditsOp { - PurchaseDatabaseCredits { amount: u64 }, +enum CyclesOp { + PurchaseDatabaseCycles { amount: u64 }, Charge { cycles_delta: u128 }, } @@ -59,21 +59,21 @@ fn service_with_root() -> TestService { } } -fn create_billed_database(service: &VfsService, name: &str, now: i64) -> String { +fn create_billed_database(service: &VfsService, name: &str, now: i64) -> (String, u64) { let database_id = service .create_generated_database(name, OWNER, now + 1) .expect("database should create") .database_id; - credit_database( + let deposit_cycles = purchase_database_cycles( service, &database_id, OWNER, - DEPOSIT_CREDITS, + DEPOSIT_PAYMENT_E8S, now as u64, now + 2, ) - .expect("database seed should credit"); - database_id + .expect("database seed should cycle"); + (database_id, deposit_cycles) } fn delete_request(database_id: &str) -> DeleteDatabaseRequest { @@ -82,24 +82,41 @@ fn delete_request(database_id: &str) -> DeleteDatabaseRequest { } } -fn credit_database( +fn purchase_database_cycles( service: &VfsService, database_id: &str, caller: &str, - credits: u64, + payment_amount_e8s: u64, block_index: u64, now: i64, ) -> Result { - let operation_id = service.begin_database_credit_purchase(database_id, caller, credits, now)?; - service.mark_database_credit_purchase_completed(operation_id, database_id, caller, credits)?; - service.credit_database_purchase(operation_id, database_id, caller, credits, block_index, now) + let operation_id = + service.begin_database_cycles_purchase(database_id, caller, payment_amount_e8s, now)?; + let config = service.cycles_billing_config()?; + let cycles = cycles_for_payment_amount_e8s(payment_amount_e8s, &config)?; + service.complete_database_cycles_purchase_ledger_transfer( + operation_id, + database_id, + caller, + cycles, + block_index, + )?; + service.apply_database_cycles_purchase( + operation_id, + database_id, + caller, + cycles, + block_index, + now, + )?; + Ok(cycles) } fn db_account(root: &Path, database_id: &str) -> (u64, Option) { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); conn.query_row( - "SELECT balance_credits, suspended_at_ms - FROM database_credit_accounts + "SELECT balance_cycles, suspended_at_ms + FROM database_cycle_accounts WHERE database_id = ?1", params![database_id], |row| { @@ -108,7 +125,7 @@ fn db_account(root: &Path, database_id: &str) -> (u64, Option) { Ok((balance.max(0) as u64, suspended_at_ms)) }, ) - .expect("credit account should exist") + .expect("cycle account should exist") } fn status_and_mount(service: &VfsService, database_id: &str) -> (DatabaseStatus, Option) { @@ -122,16 +139,15 @@ fn status_and_mount(service: &VfsService, database_id: &str) -> (DatabaseStatus, } fn charge_amount(cycles_delta: u128) -> u64 { - let variable = cycles_delta.div_ceil(CYCLES_PER_CREDIT); - u64::try_from(variable).expect("generated charge fits u64") + u64::try_from(cycles_delta).expect("generated charge fits u64") } fn assert_database_ledger_chain(root: &Path, database_id: &str, expected_balance: u64) { let conn = Connection::open(root.join("index.sqlite3")).expect("index should open"); let mut stmt = conn .prepare( - "SELECT kind, amount_credits, balance_after_credits, method, cycles_delta - FROM database_credit_ledger + "SELECT kind, amount_cycles, balance_after_cycles, method, cycles_delta + FROM database_cycle_ledger WHERE database_id = ?1 ORDER BY entry_id ASC", ) @@ -154,9 +170,9 @@ fn assert_database_ledger_chain(root: &Path, database_id: &str, expected_balance balance += amount; assert_eq!(balance_after, balance, "database ledger chain broke"); match kind.as_str() { - "credit_purchase" => assert!(amount > 0), + "cycles_purchase" => assert!(amount > 0), "charge" => assert!(amount <= 0), - "delete_credit_discard" => assert!(amount <= 0), + "delete_cycle_discard" => assert!(amount <= 0), "suspend" => { assert_eq!(amount, 0); assert!(method.is_some()); @@ -257,10 +273,10 @@ fn finalize_restore_from_bytes( .expect("complete restore should finalize"); } -fn credits_operation_strategy() -> impl Strategy { +fn cycles_operation_strategy() -> impl Strategy { prop_oneof![ - 4 => (1_u64..=2_000_000).prop_map(|amount| CreditsOp::PurchaseDatabaseCredits { amount }), - 5 => (0_u128..=8_000_000_000_u128).prop_map(|cycles_delta| CreditsOp::Charge { cycles_delta }), + 4 => (1_u64..=2_000_000).prop_map(|amount| CyclesOp::PurchaseDatabaseCycles { amount }), + 5 => (0_u128..=8_000_000_000_u128).prop_map(|cycles_delta| CyclesOp::Charge { cycles_delta }), ] } @@ -276,27 +292,29 @@ proptest! { #![proptest_config(property_config())] #[test] - fn database_service_pbt_credits_suspension_and_ledger_chain( - operations in prop::collection::vec(credits_operation_strategy(), 1..50), + fn database_service_pbt_cycles_suspension_and_ledger_chain( + operations in prop::collection::vec(cycles_operation_strategy(), 1..50), ) { let env = service_with_root(); let service = &env.service; - let database_id = create_billed_database(service, "credits-pbt", 1); - let mut database_balance = DEPOSIT_CREDITS; + let (database_id, deposit_cycles) = create_billed_database(service, "cycles-pbt", 1); + let mut database_balance = deposit_cycles; for (index, operation) in operations.into_iter().enumerate() { let now = index as i64 + 100; match operation { - CreditsOp::PurchaseDatabaseCredits { amount } => { - credit_database(service, &database_id, OWNER, amount, now as u64, now) - .expect("credit purchase should succeed"); - database_balance += amount; + CyclesOp::PurchaseDatabaseCycles { amount } => { + let purchased_cycles = + purchase_database_cycles(service, &database_id, OWNER, amount, now as u64, now) + .expect("cycle purchase should succeed"); + database_balance += purchased_cycles; } - CreditsOp::Charge { cycles_delta } => { + CyclesOp::Charge { cycles_delta } => { let before = database_balance; let config = service - .credits_config() - .expect("credits config should load"); + .cycles_billing_config() + .expect("cycles config should load"); + let computed = charge_amount(cycles_delta); service .charge_database_update( &config, @@ -307,25 +325,16 @@ proptest! { now, ) .expect("charge should record"); - let computed = charge_amount(cycles_delta); - database_balance = database_balance.saturating_sub(computed); - let entries = service - .list_database_credit_entries(&database_id, OWNER, None, 100) - .expect("database credits entries should load") - .entries; - if computed > before { - assert_eq!(entries[entries.len() - 2].kind, "charge"); - assert_eq!(entries[entries.len() - 1].kind, "suspend"); - } + database_balance -= computed.min(before); } } let (stored_balance, suspended_at_ms) = db_account(&env.root, &database_id); assert_eq!(stored_balance, database_balance); - assert_eq!(suspended_at_ms.is_some(), database_balance < DEFAULT_MIN_UPDATE_CREDITS); + assert_eq!(suspended_at_ms.is_some(), database_balance < DEFAULT_MIN_UPDATE_CYCLES); assert_eq!( - service.require_database_write_credits_available(&database_id).is_ok(), - database_balance >= DEFAULT_MIN_UPDATE_CREDITS + service.require_database_write_cycles_available(&database_id).is_ok(), + database_balance >= DEFAULT_MIN_UPDATE_CYCLES ); assert_database_ledger_chain(&env.root, &database_id, database_balance); } @@ -339,7 +348,7 @@ proptest! { ) { let env = service_with_root(); let service = &env.service; - let database_id = create_billed_database(service, "restore-pbt", 1); + let (database_id, _deposit_cycles) = create_billed_database(service, "restore-pbt", 1); let content = format!("restore body split={split_bias} deleted={restore_deleted} cancel={cancel_first}"); service .write_node( @@ -438,7 +447,7 @@ proptest! { } #[test] - fn database_service_pbt_mount_history_never_reuses_ids( + fn database_service_pbt_restore_does_not_allocate_mount_ids( operations in prop::collection::vec(mount_operation_strategy(), 1..30), ) { let env = service_with_root(); @@ -475,7 +484,6 @@ proptest! { .finalize_database_archive(database_id, OWNER, hash.clone(), now + 1) .expect("archive should finalize"); finalize_restore_from_bytes(service, database_id, &bytes, hash, size, bytes.len() / 2, now + 2); - expected_mount_events += 1; } } _ => {} diff --git a/crates/vfs_store/src/fs_store.rs b/crates/vfs_store/src/fs_store.rs index cdbcbb33..490aeba6 100644 --- a/crates/vfs_store/src/fs_store.rs +++ b/crates/vfs_store/src/fs_store.rs @@ -52,11 +52,13 @@ const CONTEXT_LINK_LIMIT: u32 = 20; const CONTEXT_SEARCH_LIMIT: u32 = 10; const WRITE_NODES_BATCH_LIMIT_MAX: usize = 100; const TOKEN_CHAR_APPROX: usize = 4; +const SYNC_RESPONSE_BYTE_BUDGET: usize = 1_500_000; const SNAPSHOT_REVISION_NO_LONGER_CURRENT: &str = "snapshot_revision is no longer current"; const SNAPSHOT_SESSION_INVALID: &str = "snapshot_session_id is invalid"; const SNAPSHOT_REVISION_CURSOR_REQUIRED: &str = "snapshot_revision is required when cursor is set"; const TARGET_SNAPSHOT_CURSOR_REQUIRED: &str = "target_snapshot_revision is required when cursor is set"; +const SYNC_RESPONSE_ITEM_TOO_LARGE: &str = "sync response item exceeds byte budget"; const LIST_ROOT_CHILD_ROWS_SQL: &str = "\ SELECT child.path, child.kind, @@ -870,7 +872,7 @@ impl FsStore { snapshot.revision, limit + 1, )?; - let next_cursor = page_next_cursor(&mut nodes, limit); + let next_cursor = page_nodes_by_limit_and_budget(&mut nodes, limit)?; Ok(ExportSnapshotResponse { snapshot_revision: scoped_snapshot_revision(&prefix, snapshot.revision), snapshot_session_id: None, @@ -950,7 +952,13 @@ impl FsStore { cursor.as_deref(), limit + 1, )?; - let next_cursor = page_next_cursor(&mut paths, limit); + let limit_had_more = paths.len() > limit as usize; + if limit_had_more { + paths.truncate(limit as usize); + } + let mut next_cursor = None; + let mut used_bytes = sync_response_base_bytes(&target_snapshot_revision); + let mut last_returned_path = None; for path in paths { if load_path_last_change_revision(conn, &path)? > target_snapshot.revision { return Err( @@ -959,11 +967,27 @@ impl FsStore { ); } let current_node = load_node(conn, &path)?; + let item_bytes = current_node + .as_ref() + .map(estimated_node_response_bytes) + .unwrap_or_else(|| estimated_removed_path_response_bytes(&path)); + if !sync_item_fits_budget(used_bytes, item_bytes) { + if changed_nodes.is_empty() && removed_paths.is_empty() { + return Err(SYNC_RESPONSE_ITEM_TOO_LARGE.to_string()); + } + next_cursor = last_returned_path.clone(); + break; + } + used_bytes = used_bytes.saturating_add(item_bytes); + last_returned_path = Some(path.clone()); match current_node { Some(node) => changed_nodes.push(node), None => removed_paths.push(path), } } + if next_cursor.is_none() && limit_had_more { + next_cursor = last_returned_path; + } Ok(FetchUpdatesResponse { snapshot_revision: target_snapshot_revision, changed_nodes, @@ -1186,15 +1210,57 @@ fn path_in_prefix(path: &str, prefix: &str) -> bool { prefix == "/" || path == prefix || path.starts_with(&format!("{prefix}/")) } -fn page_next_cursor(items: &mut Vec, limit: i64) -> Option -where - T: PageCursorPath, -{ - if items.len() <= limit as usize { - return None; +fn page_nodes_by_limit_and_budget( + nodes: &mut Vec, + limit: i64, +) -> Result, String> { + let limit_had_more = nodes.len() > limit as usize; + if limit_had_more { + nodes.truncate(limit as usize); + } + let mut used_bytes = sync_response_base_bytes(""); + let mut keep_len = 0_usize; + for node in nodes.iter() { + let item_bytes = estimated_node_response_bytes(node); + if !sync_item_fits_budget(used_bytes, item_bytes) { + if keep_len == 0 { + return Err(SYNC_RESPONSE_ITEM_TOO_LARGE.to_string()); + } + break; + } + used_bytes = used_bytes.saturating_add(item_bytes); + keep_len += 1; } - items.truncate(limit as usize); - items.last().map(PageCursorPath::cursor_path) + let budget_had_more = keep_len < nodes.len(); + if budget_had_more { + nodes.truncate(keep_len); + } + if limit_had_more || budget_had_more { + return Ok(nodes.last().map(PageCursorPath::cursor_path)); + } + Ok(None) +} + +fn sync_item_fits_budget(used_bytes: usize, item_bytes: usize) -> bool { + used_bytes.saturating_add(item_bytes) <= SYNC_RESPONSE_BYTE_BUDGET +} + +fn sync_response_base_bytes(revision: &str) -> usize { + 256_usize.saturating_add(revision.len()) +} + +fn estimated_removed_path_response_bytes(path: &str) -> usize { + 32_usize.saturating_add(path.len()) +} + +fn estimated_node_response_bytes(node: &Node) -> usize { + 128_usize + .saturating_add(node.path.len()) + .saturating_add(node.content.len()) + .saturating_add(node.etag.len()) + .saturating_add(node.metadata_json.len()) + .saturating_add(std::mem::size_of_val(&node.created_at)) + .saturating_add(std::mem::size_of_val(&node.updated_at)) } trait PageCursorPath { diff --git a/crates/vfs_store/tests/fs_store_sync.rs b/crates/vfs_store/tests/fs_store_sync.rs index 742a1dfc..3c1564bb 100644 --- a/crates/vfs_store/tests/fs_store_sync.rs +++ b/crates/vfs_store/tests/fs_store_sync.rs @@ -836,6 +836,64 @@ fn export_snapshot_pages_nodes_by_path() { assert_eq!(second.next_cursor, None); } +#[test] +fn export_snapshot_pages_by_byte_budget() { + let (_dir, store) = new_store(); + let content = "x".repeat(800_000); + write_node(&store, "/Wiki/a.md", &content, None, 10); + write_node(&store, "/Wiki/b.md", &content, None, 11); + + let first = store + .export_snapshot(ExportSnapshotRequest { + database_id: "default".to_string(), + prefix: Some("/Wiki".to_string()), + limit: 100, + cursor: None, + snapshot_revision: None, + snapshot_session_id: None, + }) + .expect("byte-budgeted snapshot should succeed"); + + assert_eq!(first.nodes.len(), 2); + assert_eq!(first.nodes[0].path, "/Wiki"); + assert_eq!(first.nodes[1].path, "/Wiki/a.md"); + assert_eq!(first.next_cursor, Some("/Wiki/a.md".to_string())); + + let second = store + .export_snapshot(ExportSnapshotRequest { + database_id: "default".to_string(), + prefix: Some("/Wiki".to_string()), + limit: 100, + cursor: first.next_cursor, + snapshot_revision: Some(first.snapshot_revision.clone()), + snapshot_session_id: None, + }) + .expect("second snapshot page should succeed"); + assert_eq!(second.nodes.len(), 1); + assert_eq!(second.nodes[0].path, "/Wiki/b.md"); + assert_eq!(second.next_cursor, None); +} + +#[test] +fn export_snapshot_rejects_single_node_over_byte_budget() { + let (_dir, store) = new_store(); + let content = "x".repeat(1_600_000); + write_node(&store, "/Wiki/huge.md", &content, None, 10); + + let error = store + .export_snapshot(ExportSnapshotRequest { + database_id: "default".to_string(), + prefix: Some("/Wiki/huge.md".to_string()), + limit: 100, + cursor: None, + snapshot_revision: None, + snapshot_session_id: None, + }) + .expect_err("single huge snapshot node should reject"); + + assert!(error.contains("byte budget")); +} + #[test] fn export_snapshot_allows_prefix_external_change_between_pages() { let (_dir, store) = new_store(); @@ -1179,6 +1237,83 @@ fn fetch_updates_pages_changed_and_removed_paths_to_fixed_target() { assert_eq!(second.next_cursor, None); } +#[test] +fn fetch_updates_pages_by_byte_budget() { + let (_dir, store) = new_store(); + let base = store + .export_snapshot(ExportSnapshotRequest { + database_id: "default".to_string(), + prefix: Some("/Wiki".to_string()), + limit: 100, + cursor: None, + snapshot_revision: None, + snapshot_session_id: None, + }) + .expect("base snapshot should succeed"); + let content = "x".repeat(800_000); + write_node(&store, "/Wiki/a.md", &content, None, 10); + write_node(&store, "/Wiki/b.md", &content, None, 11); + + let first = store + .fetch_updates(FetchUpdatesRequest { + database_id: "default".to_string(), + known_snapshot_revision: base.snapshot_revision.clone(), + prefix: Some("/Wiki".to_string()), + limit: 100, + cursor: None, + target_snapshot_revision: None, + }) + .expect("byte-budgeted updates should succeed"); + + assert_eq!(first.changed_nodes.len(), 1); + assert_eq!(first.changed_nodes[0].path, "/Wiki/a.md"); + assert_eq!(first.next_cursor, Some("/Wiki/a.md".to_string())); + + let second = store + .fetch_updates(FetchUpdatesRequest { + database_id: "default".to_string(), + known_snapshot_revision: base.snapshot_revision, + prefix: Some("/Wiki".to_string()), + limit: 100, + cursor: first.next_cursor, + target_snapshot_revision: Some(first.snapshot_revision), + }) + .expect("second byte-budgeted updates page should succeed"); + assert_eq!(second.changed_nodes.len(), 1); + assert_eq!(second.changed_nodes[0].path, "/Wiki/b.md"); + assert_eq!(second.next_cursor, None); +} + +#[test] +fn fetch_updates_rejects_single_node_over_byte_budget() { + let (_dir, store) = new_store(); + let base = store + .export_snapshot(ExportSnapshotRequest { + database_id: "default".to_string(), + prefix: Some("/Wiki/huge.md".to_string()), + limit: 100, + cursor: None, + snapshot_revision: None, + snapshot_session_id: None, + }) + .expect("base snapshot should succeed"); + let content = "x".repeat(1_600_000); + write_node(&store, "/Wiki/huge.md", &content, None, 10); + + let error = store + .fetch_updates(FetchUpdatesRequest { + database_id: "default".to_string(), + known_snapshot_revision: base.snapshot_revision, + prefix: Some("/Wiki/huge.md".to_string()), + limit: 100, + cursor: None, + target_snapshot_revision: None, + }) + .expect_err("single huge update node should reject"); + + assert!(error.contains("byte budget")); +} + #[test] fn fetch_updates_rejects_when_paged_target_path_changes_after_target() { let (_dir, store) = new_store(); diff --git a/crates/vfs_types/src/fs.rs b/crates/vfs_types/src/fs.rs index 2f0ad67e..393b6418 100644 --- a/crates/vfs_types/src/fs.rs +++ b/crates/vfs_types/src/fs.rs @@ -57,91 +57,71 @@ pub struct DatabaseSummary { pub status: DatabaseStatus, pub role: DatabaseRole, pub logical_size_bytes: u64, - pub credits_balance: Option, - pub credits_suspended_at_ms: Option, + pub cycles_balance: Option, + pub cycles_suspended_at_ms: Option, pub archived_at_ms: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct CreditsConfig { +pub struct CyclesBillingConfig { pub kinic_ledger_canister_id: String, - pub sns_governance_id: String, - pub credits_per_kinic: u64, - pub min_update_credits: u64, + pub billing_authority_id: String, + pub cycles_per_kinic: u64, + pub min_update_cycles: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct CreditsConfigUpdate { - pub credits_per_kinic: u64, - pub min_update_credits: u64, +pub struct CyclesBillingConfigUpdate { + pub cycles_per_kinic: u64, + pub min_update_cycles: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditPurchasePreview { - pub payment_amount_e8s: u64, - pub ledger_fee_e8s: u64, - pub credits_per_kinic: u64, - pub config_version: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditPurchaseRequest { +pub struct DatabaseCyclesPurchaseRequest { pub database_id: String, - pub credits: u64, - pub expected_payment_amount_e8s: u64, - pub expected_config_version: u64, + pub payment_amount_e8s: u64, + pub min_expected_cycles: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct CreditsPurchaseResult { +pub struct CyclesPurchaseResult { pub block_index: u64, - pub balance_credits: u64, + pub amount_cycles: u64, + pub balance_cycles: u64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditEntry { +pub struct DatabaseCycleEntry { pub entry_id: u64, pub database_id: String, pub kind: String, - pub amount_credits: i64, - pub balance_after_credits: u64, + pub amount_cycles: i64, + pub balance_after_cycles: u64, pub payment_amount_e8s: Option, pub caller: String, pub method: Option, pub cycles_delta: Option, - pub credits_per_kinic: Option, + pub cycles_per_kinic: Option, pub ledger_block_index: Option, pub created_at_ms: i64, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditEntryPage { - pub entries: Vec, +pub struct DatabaseCycleEntryPage { + pub entries: Vec, pub next_cursor: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditPendingOperation { +pub struct DatabaseCyclesPendingPurchase { pub operation_id: u64, pub database_id: String, - pub kind: String, - pub caller: String, - pub operation_status: String, - pub credits: u64, + pub status: String, + pub amount_cycles: u64, pub payment_amount_e8s: u64, - pub from_owner: Option, - pub from_subaccount: Option>, - pub to_owner: Option, - pub to_subaccount: Option>, - pub ledger_fee_e8s: Option, - pub ledger_created_at_time_ns: Option, + pub ledger_block_index: Option, pub created_at_ms: i64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] -pub struct DatabaseCreditPendingOperationPage { - pub entries: Vec, - pub next_cursor: Option, + pub required_action: String, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] diff --git a/crates/vfs_types/src/lib.rs b/crates/vfs_types/src/lib.rs index 70dd6a7e..0319c7a1 100644 --- a/crates/vfs_types/src/lib.rs +++ b/crates/vfs_types/src/lib.rs @@ -8,7 +8,12 @@ use serde::{Deserialize, Serialize}; pub use fs::*; -pub const KINIC_LEDGER_FEE_E8S: u64 = 10_000; +pub const KINIC_DECIMALS: u8 = 8; +pub const KINIC_LEDGER_FEE_E8S: u64 = 100_000; + +pub fn kinic_base_units_per_token() -> u64 { + 10_u64.pow(u32::from(KINIC_DECIMALS)) +} #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, CandidType)] pub struct Status { diff --git a/crates/wiki_domain/src/lib.rs b/crates/wiki_domain/src/lib.rs index 05106832..2973dafd 100644 --- a/crates/wiki_domain/src/lib.rs +++ b/crates/wiki_domain/src/lib.rs @@ -13,17 +13,20 @@ pub const PUBLIC_SKILL_REGISTRY_ROOT: &str = SKILL_REGISTRY_ROOT; pub const RAW_SOURCES_PREFIX: &str = "/Sources/raw"; pub const SESSION_SOURCES_PREFIX: &str = "/Sources/sessions"; pub const SKILL_RUNS_PREFIX: &str = "/Sources/skill-runs"; +const MAX_SOURCE_PROVIDER_LEN: usize = 32; +const MAX_SOURCE_ID_LEN: usize = 128; pub fn validate_source_path_for_kind(path: &str, kind: &NodeKind) -> Result<(), String> { let is_source_path = path_matches_prefix_boundary(path, RAW_SOURCES_PREFIX) - || path_matches_prefix_boundary(path, SESSION_SOURCES_PREFIX); + || path_matches_prefix_boundary(path, SESSION_SOURCES_PREFIX) + || path_matches_prefix_boundary(path, SKILL_RUNS_PREFIX); if *kind == NodeKind::Folder { return Ok(()); } if *kind != NodeKind::Source { if is_source_path { return Err(format!( - "source path must use source kind under {RAW_SOURCES_PREFIX} or {SESSION_SOURCES_PREFIX}: {path}" + "source path must use source kind under {RAW_SOURCES_PREFIX}, {SESSION_SOURCES_PREFIX}, or {SKILL_RUNS_PREFIX}: {path}" )); } return Ok(()); @@ -80,12 +83,9 @@ fn path_matches_prefix_boundary(path: &str, prefix: &str) -> bool { fn validate_source_path_under_prefix(path: &str, prefix: &str) -> Result<(), String> { let relative = path - .strip_prefix(prefix) + .strip_prefix(&format!("{prefix}/")) .ok_or_else(|| format!("source path must stay under {prefix}: {path}"))?; - let segments = relative - .split('/') - .filter(|segment| !segment.is_empty()) - .collect::>(); + let segments = relative.split('/').collect::>(); if segments.len() != 2 { return Err(format!( "source path must use canonical form {prefix}//.md: {path}" @@ -94,7 +94,12 @@ fn validate_source_path_under_prefix(path: &str, prefix: &str) -> Result<(), Str let [directory_name, file_name] = segments.as_slice() else { unreachable!(); }; - if directory_name.is_empty() || *file_name != format!("{directory_name}.md") { + let Some(file_stem) = file_name.strip_suffix(".md") else { + return Err(format!( + "source path must use canonical form {prefix}//.md: {path}" + )); + }; + if !is_safe_source_segment(directory_name) || file_stem != *directory_name { return Err(format!( "source path must use canonical form {prefix}//.md: {path}" )); @@ -104,12 +109,9 @@ fn validate_source_path_under_prefix(path: &str, prefix: &str) -> Result<(), Str fn validate_raw_source_path(path: &str) -> Result<(), String> { let relative = path - .strip_prefix(RAW_SOURCES_PREFIX) + .strip_prefix(&format!("{RAW_SOURCES_PREFIX}/")) .ok_or_else(|| format!("source path must stay under {RAW_SOURCES_PREFIX}: {path}"))?; - let segments = relative - .split('/') - .filter(|segment| !segment.is_empty()) - .collect::>(); + let segments = relative.split('/').collect::>(); if segments.len() != 2 { return Err(format!( "source path must use canonical form {RAW_SOURCES_PREFIX}//.md: {path}" @@ -130,6 +132,9 @@ fn validate_raw_source_path(path: &str) -> Result<(), String> { } fn is_safe_source_segment(value: &str) -> bool { + if value.len() > MAX_SOURCE_ID_LEN || value.contains("..") { + return false; + } let mut chars = value.chars(); let Some(first) = chars.next() else { return false; @@ -140,6 +145,9 @@ fn is_safe_source_segment(value: &str) -> bool { } fn is_safe_provider_segment(value: &str) -> bool { + if value.len() > MAX_SOURCE_PROVIDER_LEN { + return false; + } let mut chars = value.chars(); let Some(first) = chars.next() else { return false; @@ -154,12 +162,9 @@ fn is_source_segment_char(value: char) -> bool { fn validate_skill_run_source_path(path: &str) -> Result<(), String> { let relative = path - .strip_prefix(SKILL_RUNS_PREFIX) + .strip_prefix(&format!("{SKILL_RUNS_PREFIX}/")) .ok_or_else(|| format!("source path must stay under {SKILL_RUNS_PREFIX}: {path}"))?; - let segments = relative - .split('/') - .filter(|segment| !segment.is_empty()) - .collect::>(); + let segments = relative.split('/').collect::>(); if segments.len() != 2 { return Err(format!( "skill run source path must use canonical form {SKILL_RUNS_PREFIX}//.md: {path}" @@ -168,10 +173,12 @@ fn validate_skill_run_source_path(path: &str) -> Result<(), String> { let [name, file_name] = segments.as_slice() else { unreachable!(); }; - if name.is_empty() - || !file_name.ends_with(".md") - || file_name.trim_end_matches(".md").is_empty() - { + let Some(file_stem) = file_name.strip_suffix(".md") else { + return Err(format!( + "skill run source path must use canonical form {SKILL_RUNS_PREFIX}//.md: {path}" + )); + }; + if !is_safe_source_segment(name) || !is_safe_source_segment(file_stem) { return Err(format!( "skill run source path must use canonical form {SKILL_RUNS_PREFIX}//.md: {path}" )); @@ -181,9 +188,11 @@ fn validate_skill_run_source_path(path: &str) -> Result<(), String> { #[cfg(test)] mod tests { + use vfs_types::NodeKind; + use super::{ RAW_SOURCES_PREFIX, SKILL_RUNS_PREFIX, WIKI_ROOT_PATH, normalize_wiki_remote_path, - validate_canonical_source_path, wiki_relative_path, + validate_canonical_source_path, validate_source_path_for_kind, wiki_relative_path, }; #[test] @@ -213,24 +222,75 @@ mod tests { assert!(error.contains("source path must stay under")); } + #[test] + fn canonical_source_path_rejects_provider_and_id_over_limits() { + let long_provider = "a".repeat(33); + let long_id = "a".repeat(129); + + for path in [ + format!("{RAW_SOURCES_PREFIX}/{long_provider}/ok.md"), + format!("{RAW_SOURCES_PREFIX}/chatgpt/{long_id}.md"), + ] { + let error = validate_canonical_source_path(&path) + .expect_err("overlong provider or id should fail"); + assert!(error.contains("canonical form")); + } + } + + #[test] + fn canonical_source_path_rejects_dotdot_inside_source_id() { + let error = validate_canonical_source_path("/Sources/raw/chatgpt/a..b.md") + .expect_err("dotdot inside raw source id should fail"); + assert!(error.contains("canonical form")); + } + #[test] fn canonical_source_path_accepts_skill_runs() { let path = format!("{SKILL_RUNS_PREFIX}/legal-review/1700000000000.md"); assert!(validate_canonical_source_path(&path).is_ok()); } + #[test] + fn canonical_source_path_accepts_sessions() { + assert!(validate_canonical_source_path("/Sources/sessions/session-1/session-1.md").is_ok()); + } + + #[test] + fn skill_runs_prefix_requires_source_kind() { + let path = format!("{SKILL_RUNS_PREFIX}/legal-review/1700000000000.md"); + let error = validate_source_path_for_kind(&path, &NodeKind::File) + .expect_err("skill run source path should reject file kind"); + assert!(error.contains("source kind")); + assert!(validate_source_path_for_kind(&path, &NodeKind::Source).is_ok()); + } + #[test] fn canonical_source_path_rejects_malformed_skill_runs() { for path in [ "/Sources/skill-runs/legal-review", "/Sources/skill-runs/legal-review/", "/Sources/skill-runs/legal-review/run.txt", + "/Sources/skill-runs/../...md", + "/Sources/skill-runs/legal-review/run..1.md", "/Sources/skill-runsfoo/legal-review/run.md", ] { assert!(validate_canonical_source_path(path).is_err()); } } + #[test] + fn canonical_source_path_rejects_empty_and_dotdot_segments() { + for path in [ + "/Sources/raw//chatgpt/alpha.md", + "/Sources/raw/chatgpt//alpha.md", + "/Sources/sessions//session.md", + "/Sources/sessions/../...md", + "/Sources/sessions/session-1/session..1.md", + ] { + assert!(validate_canonical_source_path(path).is_err(), "{path}"); + } + } + #[test] fn wiki_relative_path_strips_wiki_root() { assert_eq!( diff --git a/docs/CLI.md b/docs/CLI.md index de924451..9929994a 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -30,8 +30,6 @@ Internet Identity-backed identities are the default authenticated path. Non-II ` Mainnet commands default to the Kinic VFS canister. Use `--canister-id` only to select a different canister explicitly. DB-backed VFS commands require an explicit database selection from `--database-id`, `VFS_DATABASE_ID`, `.kinic/config.toml`, or user config. No production `default` database is created implicitly. This is a breaking change for older single-DB clients that omitted `database_id`. -`recent-nodes` and the canister `recent_nodes` query were removed. Use `list-nodes --prefix --recursive --json` for scoped inventory, or `search-remote` / `search-path-remote` for recall. - ```bash cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id --database-id status ``` @@ -40,12 +38,12 @@ Use `--local` for the default local replica host, or `--replica-host` for a proj ```bash cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --local --database-id status -cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --replica-host http://127.0.0.1:8001 --database-id status +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --replica-host http://127.0.0.1:8011 --database-id status ``` `--replica-host` takes precedence over configured hosts. `--database-id` takes precedence over `VFS_DATABASE_ID`. -List, search, and graph commands default to the VFS root `/`. +List, search, glob, and graph commands default to the VFS root `/`. Pass `--prefix /Wiki` or `--path /Wiki` when the human-facing wiki tree is the intended scope. Without `--canister-id`, the CLI reads configuration from: @@ -80,18 +78,28 @@ cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --identity-mode anonymous --da Create a database before reading or writing: ```bash +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id cycles config +# Approve the VFS canister on the listed KINIC ICRC-2 ledger before CLI cycle purchase. The allowance must cover the KINIC amount plus ledger transfer fee. DB_ID="$(cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database create "")" cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database list -# Complete the first credit purchase in the browser wallet flow before writes: -# https://wiki.kinic.xyz/credits?canisterId=&databaseId=$DB_ID +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database purchase-cycles "$DB_ID" 1.25 +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database cycles "$DB_ID" 1.25 +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database cycles-history "$DB_ID" +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database cycles-pending "$DB_ID" cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database grant "$DB_ID" reader cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database link "$DB_ID" cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- write-node --path /Wiki/file.md --input file.md cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- search-remote "budget" --prefix /Wiki --top-k 10 --json ``` -`database create ` creates a generated pending database ID and prints it on success. The DB becomes writable after the first successful credit purchase in the browser wallet flow. -`database list` prints databases attached to the caller principal. +`cycles config` prints the KINIC ledger canister, billing authority principal, `cycles_per_kinic`, `min_update_cycles`, and fixed ledger transfer fee `100_000 e8s`. +`database create ` creates a generated pending database ID with zero DB cycles balance and prints it on success. It does not allocate a DB mount until the first successful cycle purchase. +`database purchase-cycles ` pulls the KINIC payment from the caller through the ledger allowance already approved outside the CLI and adds raw cycles to the DB cycles balance. Any authenticated payer can purchase cycles for an existing DB. The allowance must include the fixed ledger transfer fee. +`database cycles ` opens `https://wiki.kinic.xyz/cycles?...` for wallet-based OISY or Plug funding. This command does not use the CLI identity. The browser flow is limited to the configured canonical wiki canister, approves `payment_amount_e8s + ledger_fee_e8s` with a 30 minute expiry, and purchases cycles using the current canister config. The wallet also pays the approve transaction fee from its balance. The first successful purchase activates a pending DB. +`database cycles-history [--json]` lists DB cycles ledger entries. Reader and writer principals see payer/caller principals as `redacted`; DB owner and billing authority see full details. +`database cycles-pending [--json]` lists pending purchase operations visible to the DB owner, billing authority, or payer. Output includes `operation_id`, `status`, and `required_action`. +`database list` prints databases attached to the caller principal, including DB cycles balance and suspension time. +Successful DB updates consume DB cycles balance. Browser write surfaces disable writes when the DB is suspended, below `min_update_cycles`, or cycles config cannot be loaded. URL ingest and query-answer sessions are checked again before external Worker or DeepSeek execution, so a session issued before suspension can still fail after DB cycles balance changes. Database names are a breaking index-schema change. Existing local or canister index databases from older builds must be recreated; no automatic backfill is provided. @@ -146,7 +154,7 @@ Writes, database grants, archive operations, private Skill Registry writes, and ## Archive and Restore Archive exports one database as SQLite snapshot bytes and then finalizes the database into `archived` status. -Restore imports that snapshot into an `archived` or `deleted` database and returns it to `hot`. +Restore imports that snapshot into an `archived` database and returns it to `active`. The canister verifies the SHA-256 digest during both flows. ```bash @@ -171,7 +179,7 @@ Manual cancel is available when a database is left in `archiving`: cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database archive-cancel ``` -If restore fails after it begins, the CLI attempts to cancel the restore automatically so the database returns to its previous `archived` or `deleted` state. Manual cancel is available for an interrupted restore: +If restore fails after it begins, the CLI attempts to cancel the restore automatically so the database returns to its previous `archived` state. Manual cancel is available for an interrupted restore: ```bash cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id database restore-cancel diff --git a/docs/DB_LIFECYCLE.md b/docs/DB_LIFECYCLE.md index 2e774317..ec212ac7 100644 --- a/docs/DB_LIFECYCLE.md +++ b/docs/DB_LIFECYCLE.md @@ -24,17 +24,17 @@ Stable-memory mount IDs are partitioned by purpose: - `11..=32767`: user DB slots - `32768..=65534`: reserved -The index DB tracks database metadata, membership, and credits history. User DBs hold VFS node data, search data, and link data. +The index DB tracks database metadata, membership, and cycles history. User DBs hold VFS node data, search data, and link data. The index DB startup path ensures the latest schema. Fresh index DBs are created directly at the latest schema, and already-latest DBs are validated only. The only supported automatic migration is the production mainnet `database_index:011_source_run_sessions` to latest upgrade. Partial billing schemas, index DBs without `schema_migrations`, and pre-011 schemas are rejected instead of repaired. -Pending DBs have index metadata and credit accounts but no stable-memory mount ID. Active, archiving, or restoring DBs consume one active user DB slot. Archived DBs release their active mount, but v1 does not recycle stable-memory mount IDs for another database. A pending DB consumes a mount ID only after the first successful credit purchase activates it. +Pending DBs have index metadata and cycle accounts but no stable-memory mount ID. Active, archiving, or restoring DBs consume one active user DB slot. Archived DBs release their active mount, but v1 does not recycle stable-memory mount IDs for another database. A pending DB consumes a mount ID only after the first successful cycle purchase activates it. ## Status Databases move through five statuses: -- `pending`: metadata reserved, no mounted SQLite DB yet, only listing, credit purchase/repair, and delete are available +- `pending`: metadata reserved, no mounted SQLite DB yet, only cycle purchase and owner management are available - `active`: mounted and usable for VFS read/write/search/list - `archiving`: mounted for chunk export, VFS operations rejected until finalize succeeds - `archived`: not mounted, active mount released, snapshot metadata retained @@ -44,69 +44,74 @@ Only `active` DBs are available to normal VFS APIs. ## Size Tracking -`logical_size_bytes` tracks the SQLite file size for a database. +`logical_size_bytes` is the billable SQLite bytes for an active database. -It is updated after VFS mutations and restore finalization. It is useful for visibility and planning, but it is not a stable-memory credits or shrink metric. +It is updated after VFS mutations, restore finalization, and storage billing settle. SQLite free pages are included. Index DB bytes, canister heap, and shared management state are excluded. Deleting or archiving a DB releases the active mount. It does not imply that canister stable memory shrinks or that the stable-memory mount ID is reused. -## Credits +## Cycles -KINIC credits uses one internal DB-scoped balance: +KINIC cycles uses one internal DB-scoped balance: -- DB credits balance: KINIC pulled from the external ledger directly into a reserved DB +- DB cycles balance: KINIC pulled from the external ledger directly into a reserved DB -DB creation uses `create_database(display_name)`. It creates a generated `database_id`, owner membership, and a zero DB credits balance without allocating a stable-memory mount ID. The DB remains `pending` and credits-suspended until its first successful credit purchase activates the mounted SQLite DB. +DB creation uses `create_database(display_name)`. It creates a generated `database_id`, owner membership, and a zero DB cycles balance without allocating a stable-memory mount ID. The DB remains `pending` and cycles-suspended until its first successful cycle purchase activates the mounted SQLite DB. -While a DB is pending, callers can use `list_databases`, `preview_database_credit_purchase`, `purchase_database_credits`, credit purchase repair/list/cancel endpoints, and `delete_database`. `rename_database`, `grant_database_access`, `revoke_database_access`, `list_database_members`, and VFS read/write/search/list APIs require activation first. +External ledger calls are limited to DB cycles purchase: -External ledger calls are limited to DB credit purchase: +- `purchase_database_cycles(DatabaseCyclesPurchaseRequest)` pulls the KINIC payment from the caller through ICRC-2 `approve` + `icrc2_transfer_from` and mints cycles into that DB cycles balance. The request includes `payment_amount_e8s` and `min_expected_cycles`; credited cycles are computed from the current `cycles_per_kinic` before the ledger call and must be at least `min_expected_cycles`. The approved allowance must cover `payment_amount_e8s + ledger_fee_e8s`. -- `preview_database_credit_purchase(database_id, credits)` returns `payment_amount_e8s`, `ledger_fee_e8s`, `credits_per_kinic`, and `config_version`. -- `purchase_database_credits(DatabaseCreditPurchaseRequest)` pulls the KINIC payment from the caller through ICRC-2 `approve` + `icrc2_transfer_from` and mints credits into that DB credits balance. The request must include the previewed `expected_payment_amount_e8s` and `expected_config_version`; mismatch rejects before pending operation creation and before ledger transfer. The approved allowance must cover `payment_amount_e8s + ledger_fee_e8s`. +Any authenticated caller can cycle purchase an existing DB that still has an owner, including callers with no DB role. The payer is recorded in the DB ledger entry. Reader and writer cycles history redacts payer/caller principals, while DB owner and billing authority can read full payer/caller details. Once the ledger call starts, normal completion or explicit ledger-error cancellation resolves the started operation even if membership changes during the await. Ambiguous ledger results keep the pending operation as `ambiguous` for billing-authority review. If ledger transfer succeeds but local activation or cycles apply fails, the completed pending operation remains for billing-authority review. Owner, billing authority, and payer can inspect pending purchase status through `list_database_cycles_pending_purchases(database_id)` or CLI `database cycles-pending `. -Any authenticated caller can credit purchase an existing DB that still has an owner, including callers with no DB role. `preview_database_credit_purchase` is intentionally callable by anonymous callers so wallet UIs can validate a database target before requesting approval. The payer is recorded in the DB ledger entry. Reader and writer credits history redacts payer/caller principals, while DB owner and SNS governance can read full payer/caller details. Once the ledger call starts, completion, cancellation, or ambiguous recording resolves the started operation even if membership changes during the await. +Successful DB update calls are charged after execution. The charge is raw cycle usage: -Successful DB update calls are charged after execution. The charge is: +```text +cycles_delta +``` + +Cycles are stored as raw integer cycles. The default purchase rate is `1 KINIC = 234_500_000_000 cycles` (`0.2345 Tcycle`), controlled by `cycles_per_kinic`. Before a metered update, the caller role is checked first, then the DB cycles balance must be at least `min_update_cycles` and the DB must not be suspended. Non-members receive access errors without learning cycles state. If the post-update charge exceeds the DB cycles balance, the remaining DB cycles are fully charged, the balance becomes `0`, and the DB is suspended. + +Storage billing settles every 24h from a canister timer, with controller-only `settle_database_storage_charges()` as recovery path. Only active DBs are charged. The 13-node subnet rate is fixed at `127_000 cycles / GiB / sec`: ```text -ceil(cycles_delta / 1_000_000_000) +storage_cycles = logical_size_bytes * elapsed_seconds * 127_000 / 2^30 ``` -The default purchase rate is `1 KINIC = 1000 credits`, controlled by `credits_per_kinic`. Runtime consumption is fixed at `1 credit = 1_000_000_000 cycles`. Before a metered update, the caller role is checked first, then the DB credits balance must be at least `min_update_credits` and the DB must not be suspended. Non-members receive access errors without learning credits state. If the post-update charge exceeds the DB credits balance, the remaining balance is fully consumed, the DB is suspended, and the update result remains successful. +Storage charges write `kind = "storage_charge"` ledger entries for actually collected cycles. Insufficient-balance unpaid cycles are not carried forward or tracked as debt in v1. The residual cost above the remaining balance is forgiven as subsidy/suspension policy, the remaining balance is consumed, and the DB is suspended. The next settle uses the updated cursor. Debt tracking is a separate follow-up. -`database_credit_ledger` is the credits source of truth. `amount_credits` is the effective credits delta applied to the DB balance, and `balance_after_credits` is the resulting balance. Successful charged update calls are recorded there directly. Ledger-backed credit purchase and repair entries store ledger block indexes in `ledger_block_index`. +`database_cycle_ledger` is the cycles source of truth. Successful charged update calls are recorded there directly. Ledger-backed cycle purchase entries store ledger block indexes in `ledger_block_index`. -Credits history redacts payer/caller principals for reader and writer callers. DB owner and SNS governance can read full credits history. Pending credit operations remain visible only to DB owner and SNS governance and include `operation_status` for repair decisions. New credits history fields must not carry payer/caller principals unless the same redaction policy is applied. +Cycles history redacts payer/caller principals for reader and writer callers. DB owner and billing authority can read full cycles history. Pending cycle purchase status is visible only to owner, billing authority, and the payer of that operation. New cycles history fields must not carry payer/caller principals unless the same redaction policy is applied. -`kinic_ledger_canister_id` and `sns_governance_id` are fixed at init. SNS governance may update only rate and minimum-balance fields by calling `update_credits_config` with a Candid-encoded `CreditsConfigUpdate` blob. `config_version` starts at `1` and increments only when `credits_per_kinic` or `min_update_credits` actually changes. +`kinic_ledger_canister_id` and `billing_authority_id` are fixed at init. The billing authority may update only rate and minimum-balance fields by calling `update_cycles_billing_config` with a `CyclesBillingConfigUpdate` record. -`scripts/deploy/wiki_credits_args.sh` is the shared deploy-arg generator. `scripts/local/deploy_wiki.sh` deploys to `local-wiki`; if `SNS_GOVERNANCE_ID` is unset, local deploy uses `icp identity principal`. `KINIC_LEDGER_CANISTER_ID` is always explicit. The deploy scripts do not create a ledger canister. +`scripts/local/deploy_wiki.sh` carries local development init args. If `BILLING_AUTHORITY_ID` is unset, local deploy uses `icp identity principal`. The deploy script does not create a ledger canister by itself. Local cycle purchase smoke should use `scripts/local/setup_kinic_ledger.sh` or `scripts/smoke/local_canister_archive_restore.sh`, which creates or validates a project-local ICRC ledger and deploys the wiki with that ledger ID. -Fresh install and reinstall do not support no-arg deploy. A pre-credits schema upgrade also requires `CreditsConfig`. After credits config has been initialized, no-arg post-upgrade can reuse stored config, but official deploy and upgrade steps still pass `CreditsConfig` every time. Production deploy must use `scripts/mainnet/deploy_wiki.sh` with `KINIC_LEDGER_CANISTER_ID` and `SNS_GOVERNANCE_ID`; the script rejects unset, empty, or anonymous values before install. Production and staging principal values are managed as deploy environment variables, not committed repo data. These principal values cannot be changed after init. +Unit tests do not deploy a ledger. They mock ledger transfer outcomes inside the canister test harness. Production deploy must use `scripts/mainnet/deploy_wiki.sh` with explicit `KINIC_LEDGER_CANISTER_ID` and `BILLING_AUTHORITY_ID`. The script rejects unset, empty, or anonymous values before install. These principal values cannot be changed after init. -Unit tests do not deploy a ledger. They mock ledger transfer outcomes inside the canister test harness. `scripts/smoke/local_canister_post_upgrade.sh` verifies deploy with `CreditsConfig`, pending DB persistence, and upgrade with the same config. Ledger-backed credit purchase smoke is separate. +Upgrade compatibility: + +- `post_upgrade` accepts no arg, a bare `CyclesBillingConfig`, or `opt CyclesBillingConfig`. +- The first upgrade from the pre-billing mainnet index schema requires a valid `CyclesBillingConfig`; missing or invalid principals trap before migration. +- After `cycles_billing_config` exists in the index schema, no-arg upgrade is supported and the stored config remains authoritative. +- The only supported automatic billing upgrade is the production pre-billing mainnet `database_index:011_source_run_sessions` schema to latest. Partial billing schemas and legacy credit schemas are unsupported; recreate or reinstall those DBs instead of auto-converting them. Normal operator flow: 1. Owner creates a pending DB with `create_database(display_name)`. -2. Payer previews the DB credit purchase, then approves the VFS canister on the KINIC ICRC-2 ledger for the DB credit amount plus ledger transfer fee. Browser approve uses the current allowance as `expected_allowance` and expires after 30 minutes. The approve transaction fee is paid separately by the wallet. -3. Payer calls `purchase_database_credits` with the previewed expected amount and config version. If the DB is pending, the canister starts the ledger transfer first, then allocates and migrates the DB mount only after the ledger transfer succeeds. The DB becomes active when mount migration and balance credit both complete. -4. Successful DB updates consume DB credits balance. -5. DB delete discards any remaining credits. - -URL ingest and query-answer sessions can expire after issuance if the DB becomes suspended or drops below the minimum update balance. Browser write UI also treats suspended, low-balance, or credits-config-unavailable DBs as not writable. Browser and worker paths re-check credits before forwarding to external Worker or DeepSeek calls. URL ingest source generation carries the original `sessionNonce` through the queue and re-checks the session immediately before DeepSeek. - -Treasury sweep, DB-specific ledger subaccounts, and repair browser UI are not implemented. +2. Payer approves the VFS canister on the KINIC ICRC-2 ledger for the payment amount plus ledger transfer fee. Browser approve uses the current allowance as `expected_allowance` and expires after 30 minutes. The approve transaction fee is paid separately by the wallet. +3. Payer calls `purchase_database_cycles` with the payment amount. If the DB is pending, the canister starts the ledger transfer first, then allocates and migrates the DB mount only after the ledger transfer succeeds. The DB becomes active when mount migration and balance cycle both complete. +4. Successful DB updates consume DB cycles balance. +5. DB delete discards any remaining cycles. -If DB credit purchase receives an explicit ledger error, the credit purchase is cancelled. If the inter-canister call or response decoding is ambiguous, the operation remains pending and the DB ledger records `credit_purchase_ambiguous` with `amount_credits = 0`. If the ledger transfer succeeds but local DB activation or credit application fails, the operation remains pending and the error returns the pending `operation_id` and `ledger_block_index` for verified completion. Pending operations store the expected ledger from/to accounts, fee, memo inputs, and `created_at_time` so completion can validate the exact transfer. +URL ingest and query-answer sessions can expire after issuance if the DB becomes suspended or drops below the minimum update balance. Browser write UI also treats suspended, low-balance, or cycles-config-unavailable DBs as not writable. Browser and worker paths re-check cycles before forwarding to external Worker or DeepSeek calls. URL ingest source generation carries the original `sessionNonce` through the queue and re-checks the session immediately before DeepSeek. -Pending operations block DB delete until they are resolved: +Treasury sweep, DB-specific ledger subaccounts, repair browser UI, purchase retry API, and ambiguous purchase repair/cancel API are not implemented. -- `repair_database_credit_purchase_complete(database_id, operation_id, ledger_block_index)` -- `repair_database_credit_purchase_cancel(database_id, operation_id)` +DB cycle purchase credits internal cycles only after `icrc2_transfer_from` returns `Ok(block_index)` and local activation/apply both finish. Explicit ledger errors cancel the `in_flight` operation without credit. Ambiguous inter-canister call or response decoding stores `operation_status = "ambiguous"` without credit and returns an error containing the `operation_id`. If ledger transfer succeeds but local DB activation or cycle application fails, the canister stores `operation_status = "completed"` with the ledger block index, does not credit cycles, and returns a local apply error containing the `operation_id` and ledger block index. -Complete checks the ledger transaction at `ledger_block_index` against the pending operation before changing DB credits balance. The canister entrypoint accepts any non-anonymous caller when the ledger block proves the payment; the official CLI defaults to Internet Identity and requires explicit `--allow-non-ii-identity` opt-in for non-II operator identities. The completed ledger entry records the original payer from the pending operation as `caller`, not the repair executor. If local activation or credit application fails during complete, the pending operation remains and the returned error includes the operation and block identifiers. Cancel repair is allowed only for the original payer or DB owner, and is rejected once pending DB activation has started. Cancel writes `credit_purchase_repair_cancelled` with the cancel caller, pending payment amount, current balance, `amount_credits = 0`, and no ledger block index. DB owner and SNS governance can inspect pending operations. +Pending cycle operations are temporary state for transfer-in-flight, ambiguous ledger result review, ledger-success-before-local-apply review, memo correlation, and duplicate purchase guard. Owner, billing authority, and the payer of each operation can inspect them through `list_database_cycles_pending_purchases(database_id)` or CLI `database cycles-pending `. The public status exposes `operation_id`, `database_id`, `status`, `amount_cycles`, `payment_amount_e8s`, `ledger_block_index`, `created_at_ms`, and `required_action`; unrelated callers are rejected. ## Delete @@ -115,11 +120,11 @@ Complete checks the ledger transaction at `ledger_block_index` against the pendi Delete is a hard delete: - the SQLite DB file is removed where file deletion is available -- DB membership, credits, pending operations, restore chunks, restore sessions, and transient sessions are removed from the index +- DB membership, cycles, pending operations, restore chunks, restore sessions, and transient sessions are removed from the index - `database_mount_history` is retained so the stable-memory mount ID is not reused by another DB in v1 - the stable-memory mount ID is not reused by another DB in v1 -Delete requires no pending credit purchase operations. The request carries only `database_id`. Remaining DB credits are discarded with the deleted index rows. +Delete requires no pending cycle purchase operations. The request carries only `database_id`. Remaining DB cycles are discarded with the deleted index rows. Delete is treated as irreversible. If recovery is required, archive first and store the exported bytes outside the canister. Deleted DBs are absent from `list_databases` and subsequent DB operations return `database not found`. diff --git a/docs/PUBLIC_SMOKE.md b/docs/PUBLIC_SMOKE.md index be51420f..2373ca8c 100644 --- a/docs/PUBLIC_SMOKE.md +++ b/docs/PUBLIC_SMOKE.md @@ -4,26 +4,30 @@ Use this flow before publishing a Browser build with CLI setup instructions. The ## Local Canister -The local credits smoke prepares a project-local ICRC ledger when `KINIC_LEDGER_CANISTER_ID` is unset. Set `KINIC_LEDGER_WASM` if the ICRC ledger wasm is not in a known local cache path. `scripts/local/deploy_wiki.sh` defaults `SNS_GOVERNANCE_ID` to `icp identity principal` when it is not set. +The local cycles smoke prepares a project-local ICRC ledger when `KINIC_LEDGER_CANISTER_ID` is unset. Set `KINIC_LEDGER_WASM` if the ICRC ledger wasm is not in a known local cache path. `scripts/local/deploy_wiki.sh` defaults `BILLING_AUTHORITY_ID` to `icp identity principal` when it is not set. ```bash icp network start -d -e local-wiki ICP_ENVIRONMENT=local-wiki scripts/smoke/local_canister_archive_restore.sh ``` -The smoke stores the generated ledger ID in `.icp/cache/local-kinic-ledger/local-wiki.id`, deploys the wiki with that ledger ID, approves the wiki canister on the ledger, performs smoke-only credit purchases through direct canister calls, and verifies archive/restore plus CLI reads and writes. Resolve the local wiki canister ID from `.icp/cache/mappings/local-wiki.ids.json`, or pass `CANISTER_ID` explicitly. -The wiki canister constructor requires `CreditsConfig`; no-arg fresh install and reinstall are unsupported. +The smoke stores the generated ledger ID in `.icp/cache/local-kinic-ledger/local-wiki.id`, deploys the wiki with that ledger ID, approves the wiki canister on the ledger, and verifies archive/restore plus CLI cycle purchase. Resolve the local wiki canister ID from `.icp/cache/mappings/local-wiki.ids.json`, or pass `CANISTER_ID` explicitly. ## CLI and Browser Read Smoke -Create an active database, write one file, and grant anonymous reader access for Browser reads. Newly created databases are pending until the first browser wallet credit purchase completes. +Create a database, write one file, and grant anonymous reader access for Browser reads: ```bash CANISTER_ID= -REPLICA_HOST=http://127.0.0.1:8001 +REPLICA_HOST=http://127.0.0.1:8011 +KINIC_LEDGER_CANISTER_ID="$(cat .icp/cache/local-kinic-ledger/local-wiki.id)" DB_NAME="${DB_NAME:-Public Smoke}" DB_ID="$(cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --allow-non-ii-identity --replica-host "$REPLICA_HOST" --canister-id "$CANISTER_ID" database create "$DB_NAME")" -# Complete the first credit purchase in the Browser wallet flow before write-node. +icp canister call "${KINIC_LEDGER_CANISTER_ID}" icrc2_approve \ + "(record { spender = record { owner = principal \"${CANISTER_ID}\"; subaccount = null }; amount = 200000000 : nat; expected_allowance = null; expires_at = null; fee = null; memo = null; from_subaccount = null; created_at_time = null })" \ + -e local-wiki -o candid +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --allow-non-ii-identity --replica-host "$REPLICA_HOST" --canister-id "$CANISTER_ID" \ + database purchase-cycles "$DB_ID" 1 printf '# Public Smoke\n\nalpha browser smoke\n' > /tmp/llm-wiki-smoke.md cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --allow-non-ii-identity --replica-host "$REPLICA_HOST" --canister-id "$CANISTER_ID" --database-id "$DB_ID" \ write-node --path /Wiki/smoke.md --input /tmp/llm-wiki-smoke.md @@ -35,7 +39,7 @@ Start the Browser with local env values: ```bash cd wikibrowser -NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8001 \ +NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8011 \ NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID="$CANISTER_ID" \ pnpm dev ``` @@ -57,21 +61,12 @@ ICP_ENVIRONMENT=local-wiki scripts/smoke/local_canister_archive_restore.sh That script runs the dedicated Rust archive/restore smoke and then verifies the public CLI commands: +- `database purchase-cycles` - `database archive-export` - `database archive-restore` - `read-node` -The Rust smoke also verifies the deployed local canister path for archive/restore, upgrade persistence, FTS search, outgoing links, and isolation between two databases. The script targets the project-local replica with `--replica-host http://127.0.0.1:8001`. - -## Post-upgrade Args Smoke - -Run the constructor and upgrade argument smoke: - -```bash -KINIC_LEDGER_CANISTER_ID= scripts/smoke/local_canister_post_upgrade.sh -``` - -This deploys with `CreditsConfig`, creates one pending DB, upgrades with the same config, then verifies `get_credits_config` and DB metadata persistence. It does not perform a ledger credit purchase. +The Rust smoke also verifies the deployed local canister path for archive/restore, upgrade persistence, FTS search, outgoing links, and isolation between two databases. The script targets the project-local replica from `icp network status`. ## Public Deployment Smoke diff --git a/docs/SKILL_REGISTRY.md b/docs/SKILL_REGISTRY.md index 27a3424f..327ef20c 100644 --- a/docs/SKILL_REGISTRY.md +++ b/docs/SKILL_REGISTRY.md @@ -182,7 +182,7 @@ Command responsibilities: - `skill set-status`: move a package through `draft`, `reviewed`, `promoted`, or `deprecated`. - `skill import github`: import package files from a GitHub source. - `skill propose-improvement`: write evidence-backed proposal records. -- `skill approve-proposal`: mark a proposal approved; it does not apply the diff. +- `skill approve-proposal`: mark a proposal reviewed; it does not apply the diff. - `skill install`: write a downstream lockfile only; it does not place files into an agent runtime. Share access with database member commands: @@ -354,11 +354,12 @@ cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- skill propose-improvement lega --runs /Sources/skill-runs/legal-review/123.md \ --summary "Tighten missing-approval checks" \ --diff-file ./proposal.diff -cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- skill approve-proposal legal-review /Wiki/skills/legal-review/improvement-proposals/123.md +cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- skill approve-proposal legal-review /Wiki/skills/legal-review/proposals/123 ``` -`approve-proposal` marks the proposal approved. It does not apply the diff to `SKILL.md`; update the source package and run `skill upsert`. -Approval only accepts proposal nodes under the target skill's `improvement-proposals/` directory with matching proposal frontmatter. +`propose-improvement` writes a v1 proposal directory containing `proposal.md`, `diff.md`, `candidate/SKILL.md`, `metrics.json`, and `status.md`. +`approve-proposal` updates `status.md` to `reviewed`. It does not apply the diff to `SKILL.md`; use `apply-proposal` for gated candidate application. +Approval only accepts proposal roots under the target skill's `proposals/` directory with matching proposal frontmatter and status metadata. ## Example diff --git a/docs/payment.md b/docs/payment.md new file mode 100644 index 00000000..98dd6843 --- /dev/null +++ b/docs/payment.md @@ -0,0 +1,256 @@ +# Payment + +## 用語 + +- `KINIC`: 外部 ICRC ledger 上の支払い token。金額は e8s で扱う。 +- `cycles`: DB ごとの内部残高。KINIC から換算された raw cycles を整数で保持する。 +- `database_cycle_accounts`: DB ごとの cycles 残高、停止時刻、ストレージ課金カーソルを保持する。 +- `database_cycle_ledger`: cycles 履歴の正本。購入、更新課金、ストレージ課金、停止を記録する。 +- `database_cycle_pending_operations`: transfer 中、ledger 結果曖昧、または ledger 成功後 local apply 前の cycles 購入を保持する。 + +## 課金設定 + +`cycles_billing_config` は index DB に保存される。 + +| key | 意味 | +| --- | --- | +| `kinic_ledger_canister_id` | KINIC ledger canister。初期化時に固定される。 | +| `billing_authority_id` | 課金設定更新と cycles 履歴の全件閲覧権限 principal。初期化時に固定される。 | +| `cycles_per_kinic` | `1 KINIC` あたりの付与 cycles。 | +| `min_update_cycles` | metered update 開始に必要な最低 DB cycles 残高。 | + +既定値は以下である。 + +| 項目 | 値 | +| --- | --- | +| `cycles_per_kinic` | `234_500_000_000` (`1 KINIC = 0.2345 Tcycle`) | +| `min_update_cycles` | `1_000_000` | +| KINIC ledger fee | `100_000 e8s` | +| KINIC decimals | `8` | + +`update_cycles_billing_config` は認証済み caller のみ呼べる。caller が `billing_authority_id` と一致しない場合は拒否する。引数は `CyclesBillingConfigUpdate` record で、変更できる値は `cycles_per_kinic` と `min_update_cycles` のみである。 + +mainnet deploy script は `KINIC_LEDGER_CANISTER_ID` と `BILLING_AUTHORITY_ID` を明示必須にする。`kinic_ledger_canister_id` は支払い ledger canister の ID であり、billing authority でも rate 設定でもない。 + +設定値は次を満たす必要がある。 + +- `kinic_ledger_canister_id` と `billing_authority_id` は valid principal text かつ anonymous ではない。 +- `cycles_per_kinic` と `min_update_cycles` は 0 ではなく、`i64` に収まる。 + +## DB 作成と残高初期状態 + +`create_database(display_name)` は generated `database_id`、owner membership、cycles account を作成する。DB は `pending` になり、stable-memory mount はまだ割り当てない。初期 `balance_cycles` は `0`、`suspended_at_ms` は作成時刻、`storage_charged_at_ms` は `NULL` である。 + +pending DB は、最初の cycles 購入が ledger 成功後にローカル反映まで完了した時点で active 化する。active 化では mount ID を割り当て、DB migration を実行し、`storage_charged_at_ms` を active 化時刻で初期化する。 + +古い pending DB は、同じ owner が新規 pending DB を作成する際に、作成から 24h 超過かつ pending cycles operation がない場合だけ purge 対象になる。owner ごとの未開始 pending DB 上限は 3 件である。 + +## 購入量計算 + +cycles は次の式で計算する。`10^KINIC_DECIMALS` は KINIC decimals 8 に由来する base-unit scale であり、現在は `100_000_000` である。小数は整数除算で切り捨てる。 + +```text +cycles = payment_amount_e8s * cycles_per_kinic / 10^KINIC_DECIMALS +``` + +購入実行時の現行 `cycles_per_kinic` で確定する。UI や CLI は同じ計算で見積 cycles を作り、`min_expected_cycles` として request に含める。ledger 転送前の再計算結果が `min_expected_cycles` 未満なら拒否する。 + +購入は ledger 転送前に以下を拒否する。 + +- `payment_amount_e8s` が `0` +- `payment_amount_e8s` が `i64` に収まらない +- `cycles_per_kinic` が `0` +- 乗算結果が overflow する +- 換算後 cycles が `0` +- 換算後 cycles が `i64` に収まらない +- DB が存在しない +- DB status が `pending` / `active` 以外 +- DB に owner が存在しない +- cycles account が存在しない +- pending DB に `cycles_purchase` pending operation が既にある +- 換算後 cycles が `min_expected_cycles` 未満 +- 現在残高 + pending cycles + 購入 cycles が overflow する + +## 購入実行 + +`purchase_database_cycles(DatabaseCyclesPurchaseRequest)` は update で、anonymous caller は拒否される。DB role は不要であり、認証済み caller なら既存 DB に cycles を購入できる。 + +request は以下を含む。 + +- `database_id` +- `payment_amount_e8s` +- `min_expected_cycles` + +検証通過後、canister は現行 config で cycles を計算し、`database_cycle_pending_operations` に `kind = "cycles_purchase"`、`operation_status = "in_flight"` の行を作成する。pending operation には payer、cycles、支払い e8s、ledger fee、ledger created_at_time、from/to account、ledger 成功後の block index を保存する。 + +ledger 転送は KINIC ledger の `icrc2_transfer_from` を使う。 + +- from: caller principal +- to: canister principal +- amount: request の `payment_amount_e8s` +- fee: 固定 `ledger_fee_e8s` +- memo: `kvfs:cp:{operation_id}` +- created_at_time: purchase 開始時刻 +- spender: canister principal + +ledger が `Duplicate` error を返し、`duplicate_of` が `u64` に変換できる場合は成功扱いで進む。ledger が成功した場合、pending operation を `completed` にし、ledger block index を保存する。その後、対象 DB が pending なら mount 割当と DB migration を実行し、cycles 残高へ反映する。反映成功時は `database_cycle_ledger` に以下を記録し、pending operation を削除する。 + +| column | 値 | +| --- | --- | +| `kind` | `cycles_purchase` | +| `amount_cycles` | 購入 cycles | +| `balance_after_cycles` | 反映後 DB cycles 残高 | +| `payment_amount_e8s` | 支払い e8s | +| `caller` | payer | +| `method` | `purchase_database_cycles` | +| `ledger_block_index` | ledger transfer block index、または `Duplicate.duplicate_of` | + +ledger が `BadFee` error を返し、`expected_fee` が `u64` に変換できる場合、canister は pending operation 削除を試行し、`icrc2_transfer_from failed: BadFee expected fee ...; re-approve with the current ledger fee and retry` を返す。その他の明示的 ledger error でも pending operation 削除を試行し、caller へ ledger error を返す。この場合 cycles ledger は増えない。 + +ledger call、response decode、または結果判定が曖昧な場合、canister は `operation_status = "ambiguous"` の pending operation と DB reservation を保持する。cycles ledger は増えない。caller には operation ID を含む billing authority review required error を返す。v1 では repair / cancel API は提供しない。 + +ledger 成功後に pending DB activation、migration、または cycles 残高反映が失敗した場合、`completed` pending operation と DB 予約を残す。cycles ledger は増えず、cycles 残高も増えない。caller には operation ID と ledger block index を含む local apply error を返す。v1 では retry API は提供しない。 + +browser `/cycles` は approve 後の purchase failure でも通常の error 表示だけを行う。復旧パネルやブラウザ保存は行わない。 + +## Wallet UI + +`/cycles` route は canister ID を URL から受け取らない。`NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID` を canister ID として使う。query から読む値は以下だけである。 + +- `database_id` または `databaseId` +- `kinic` + +`database_id` は必須で、`[a-zA-Z0-9_-]+` のみ許可される。`kinic` は初期入力値であり、購入額は UI 上で編集できる。未指定時の初期入力は `1`。 + +KINIC 入力は正の数だけ許可する。小数は最大 8 桁、e8s 換算後は `u64::MAX` 以下でなければならない。 + +OISY と Plug の wallet flow は、購入直前に canister config を取得する。承認 allowance は次である。 + +```text +approved_allowance_e8s = payment_amount_e8s + ledger_fee_e8s +``` + +approve の transaction fee は wallet 残高から別途支払われる。approve は現在 allowance を `expected_allowance` として渡し、30 分後に expire する。approve 後に purchase が失敗した場合、UI は approval が expire まで残る旨を error に含める。 + +UI は `NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID` と request canister ID が一致しない場合に拒否する。Plug は VFS canister と KINIC ledger canister を whitelist して接続する。OISY は ICRC wallet の call-canister 結果 certificate を検証し、method、canister、arg、reply を照合する。 + +## ICRC-21 consent + +`icrc21_canister_call_consent_message` は `purchase_database_cycles` だけ対応する。他 method は unsupported になる。 + +consent 生成時は request arg を `DatabaseCyclesPurchaseRequest` として decode し、現行 config で cycles を計算する。計算結果が `min_expected_cycles` 未満、または DB や支払額が購入条件を満たさない場合は unsupported になる。 + +表示内容は database ID、cycles、payment KINIC、allowance に含める ledger fee、spender canister を含む。 + +## 更新課金 + +metered update は実行前に `prepare_metered_update` で認可と残高確認を行う。順序は次である。 + +1. DB status を読み、caller の DB role を読む。 +2. required role を満たさない場合は拒否する。 +3. `cycles_billing_config` を読む。 +4. cycles account の `suspended_at_ms` が `Some` なら拒否する。 +5. `balance_cycles < min_update_cycles` なら拒否する。 + +`check_database_write_cycles(database_id)` は anonymous caller を明示的に拒否し、writer 以上の role と cycles 利用可能状態を確認する。 + +canister の metered update wrapper は、更新前後の canister cycle balance 差分を `cycles_delta` として計算する。更新関数が `Ok` を返した場合だけ、`charge_database_update` を実行する。 + +更新課金額は `cycles_delta` そのものである。`cycles_delta` が `i64` に収まらない場合は `cycle charge exceeds i64 limit` になる。`cycles_delta == 0` の場合、残高更新も ledger 記録も行わない。 + +`charge_database_update` は現在残高より請求額が大きい場合、残高を全徴収して `balance_after_cycles = 0` にし、DB を suspended にする。ledger の `amount_cycles` は実際に徴収した cycles、`cycles_delta` は実測請求額を記録する。 + +課金成功時は残高から実徴収額を引く。更新後残高が `min_update_cycles` 未満なら `suspended_at_ms` を課金時刻に設定し、それ以上なら `NULL` にする。ledger には以下を記録する。 + +| column | 値 | +| --- | --- | +| `kind` | `charge` | +| `amount_cycles` | 実徴収額 `paid_cycles` の負数 | +| `balance_after_cycles` | 課金後残高 | +| `caller` | update caller | +| `method` | update method name | +| `cycles_delta` | `cycles_delta` | +| `cycles_per_kinic` | 課金時 config の `cycles_per_kinic` | + +残高ぴったりの請求は成功し、残高は `0` になり suspended になる。 + +## ストレージ課金 + +ストレージ課金は active DB だけ対象である。canister timer は 24h ごとに `settle_database_storage_charges` を呼ぶ。controller は同じ entrypoint を手動実行できる。 + +各 active DB について現在の DB size を測定し、`logical_size_bytes` を更新する。`storage_charged_at_ms` が `NULL` の場合は課金せず、課金カーソルを現在時刻に設定する。 + +前回課金時刻から 24h 未満の場合は何もしない。24h 以上経過した場合、次の式で課金 cycles を計算する。 + +```text +storage_cycles = logical_size_bytes * elapsed_seconds * 127_000 / 2^30 +``` + +`elapsed_seconds` は `elapsed_ms / 1000` の整数除算である。`logical_size_bytes == 0` または `elapsed_ms <= 0` の場合、課金 cycles は `0` になる。課金 cycles が `0` の場合、ledger は記録せず、課金カーソルだけ現在時刻に更新する。 + +残高が不足する場合、支払える分だけ `paid_cycles = max(min(balance_cycles, charge_cycles), 0)` として徴収する。未払い分は debt として追跡せず、v1 subsidy/suspension policy として残高超過分を forgive する。課金後残高が `min_update_cycles` 未満、または `paid_cycles < charge_cycles` の場合は suspended になる。既に suspended の場合は元の `suspended_at_ms` を維持する。次回 settle は更新済み cursor から再計算する。storage debt 実装は別 PR の follow-up とする。 + +`paid_cycles > 0` の場合、ledger に `kind = "storage_charge"` を記録する。新たに suspended になった場合は続けて `kind = "suspend"` を記録する。`suspend` の `amount_cycles` は `0` である。 + +1 GiB を 24h 保持した場合の課金は `10_972_800_000 cycles`。10 MiB を 24h 保持した場合の課金は `107_156_250 cycles`。 + +## 履歴と権限 + +`list_database_cycle_entries(database_id, cursor, limit)` は cycles ledger を entry ID 昇順で返す。`limit` は `1..=100` に clamp される。`next_cursor` は取得件数が limit を超えた場合に返る。 + +閲覧権限は以下である。 + +| caller | 結果 | +| --- | --- | +| billing authority | DB member でなくても全履歴を閲覧できる。caller は redacted されない。 | +| owner | 全履歴を閲覧できる。caller は redacted されない。 | +| writer | 履歴を閲覧できる。各 entry の `caller` は `redacted`。 | +| reader | 履歴を閲覧できる。各 entry の `caller` は `redacted`。 | +| member ではない caller | 拒否される。 | + +Pending cycles purchase は owner、billing authority、payer が `list_database_cycles_pending_purchases(database_id)` または CLI `database cycles-pending ` で確認できる。返却値は `operation_id`、`database_id`、`status`、`amount_cycles`、`payment_amount_e8s`、`ledger_block_index`、`created_at_ms`、`required_action` を含む。無関係 caller は拒否される。 + +`required_action` は `status` から決まる。 + +| `status` | `required_action` | +| --- | --- | +| `in_flight` | `wait_for_ledger_result` | +| `ambiguous` | `billing_authority_review` | +| `completed` | `billing_authority_review` | +| その他 | `billing_authority_review` | + +## Ledger 結果と no-credit + +`purchase_database_cycles` は `icrc2_transfer_from` が `Ok(block_index)` を返し、local activation/apply も完了した場合だけ内部 cycles 残高へ credit する。明確な ledger error は pending operation を削除し、残高と ledger entry を変更しない。 + +inter-canister call 失敗や response decode 失敗など ledger 結果が曖昧な場合、canister は `ambiguous` pending operation と DB reservation を保持し、cycles ledger と cycles 残高は更新しない。pending は確認専用であり、v1 では repair / cancel API はない。 + +ledger transfer 成功後に pending DB activation や cycles apply が失敗した場合は、`completed` pending operation と DB reservation を残す。`database_cycle_ledger` と cycles 残高は更新しない。pending は確認専用であり、v1 では retry API はない。 + +## 削除 + +`delete_database(DeleteDatabaseRequest)` は owner のみ実行できる。DB status は `pending` または `active` でなければならない。 + +削除前に pending cycles operation が 1 件でも存在する場合は拒否する。拒否 error は最初の該当 operation の `operation_id`、`status`、`required_action` を含む。ledger 成功後の local apply 失敗では `completed` pending operation が残るため、この状態が解消されるまで削除できない。 + +削除成功時は DB file を削除できる環境では削除し、index DB から以下を削除する。 + +- `database_cycle_pending_operations` +- `database_cycle_ledger` +- `database_cycle_accounts` +- `database_members` +- restore/session 系 rows +- `databases` + +残った cycles balance は返金されず破棄される。 + +## 実装根拠 + +- `crates/vfs_runtime/src/lib.rs`: cycles 設定、pending operation、残高反映、更新課金、ストレージ課金、履歴権限、削除条件。 +- `crates/vfs_canister/src/lib.rs`: canister entrypoint、認証、ICRC ledger call、ICRC-21 consent、metered update wrapper。 +- `crates/vfs_types/src/fs.rs`: 公開 Candid/serde 型。 +- `crates/vfs_types/src/lib.rs`: fixed KINIC ledger fee。 +- `wikibrowser/lib/cycles-wallet.ts`: OISY/Plug approve と purchase flow。 +- `wikibrowser/lib/cycles-url.ts`: `/cycles` の database ID と KINIC 入力 validation。 +- `wikibrowser/app/cycles/page.tsx`: canister ID を環境変数から固定する route 挙動。 +- `crates/vfs_runtime/tests/database_service.rs`: 購入、履歴 redaction、pending operation 表示、更新課金、削除の期待挙動。 diff --git a/extensions/wiki-clipper/pnpm-workspace.yaml b/extensions/wiki-clipper/pnpm-workspace.yaml index 00f6fc47..5ed0b5af 100644 --- a/extensions/wiki-clipper/pnpm-workspace.yaml +++ b/extensions/wiki-clipper/pnpm-workspace.yaml @@ -1,2 +1,2 @@ allowBuilds: - esbuild: set this to true or false + esbuild: true diff --git a/extensions/wiki-clipper/popup/popup.js b/extensions/wiki-clipper/popup/popup.js index deb0dccb..a6184f6e 100644 --- a/extensions/wiki-clipper/popup/popup.js +++ b/extensions/wiki-clipper/popup/popup.js @@ -71,7 +71,7 @@ createDatabaseForm.addEventListener("submit", async (event) => { name ); await refreshAuthAndDatabases(created); - statusText.textContent = "Database created. Purchase credits before capture."; + statusText.textContent = "Database created. Purchase cycles before capture."; } catch (error) { statusText.textContent = error instanceof Error ? error.message : String(error); } finally { @@ -154,12 +154,12 @@ function renderDatabaseOptions(databases, selectedDatabaseId, placeholder = "No const option = document.createElement("option"); option.value = database.databaseId; const label = databaseOptionLabel(database, nameCounts.get(databaseNameKey(database.name)) || 1); - option.disabled = !database.writeCreditsAvailable; - option.textContent = database.writeCreditsAvailable ? label : `${label} - ${database.creditsReason}`; + option.disabled = !database.writeCyclesAvailable; + option.textContent = database.writeCyclesAvailable ? label : `${label} - ${database.cyclesReason}`; option.title = database.databaseId; databaseSelect.append(option); } - const selectable = databases.filter((database) => database.writeCreditsAvailable); + const selectable = databases.filter((database) => database.writeCyclesAvailable); if (selectable.length === 0) { databaseSelect.value = ""; databaseSelect.disabled = true; diff --git a/extensions/wiki-clipper/scripts/check-candid-drift.mjs b/extensions/wiki-clipper/scripts/check-candid-drift.mjs index b69ddd0e..ec8ec7b4 100644 --- a/extensions/wiki-clipper/scripts/check-candid-drift.mjs +++ b/extensions/wiki-clipper/scripts/check-candid-drift.mjs @@ -18,18 +18,18 @@ const expectedTypes = { role: "DatabaseRole", logical_size_bytes: "nat64", database_id: "text", - archived_at_ms: "opt int64", - credits_balance: "opt nat64", - credits_suspended_at_ms: "opt int64" + cycles_balance: "opt nat64", + cycles_suspended_at_ms: "opt int64", + archived_at_ms: "opt int64" } }, - CreditsConfig: { + CyclesBillingConfig: { kind: "record", fields: { kinic_ledger_canister_id: "text", - sns_governance_id: "text", - credits_per_kinic: "nat64", - min_update_credits: "nat64" + billing_authority_id: "text", + cycles_per_kinic: "nat64", + min_update_cycles: "nat64" } }, CreateDatabaseRequest: { kind: "record", fields: { name: "text" } }, @@ -77,12 +77,13 @@ const expectedTypes = { WriteSourceForGenerationResult: { kind: "record", fields: { write: "WriteNodeResult", session_nonce: "text" } } }; const actorExpectedTypes = { - ...expectedTypes + ...expectedTypes, + DatabaseStatus: { kind: "variant", fields: { Hot: "null", Pending: "null", Active: "null", Restoring: "null", Archiving: "null", Archived: "null" } } }; const expectedMethods = { authorize_url_ingest_trigger_session: { input: ["UrlIngestTriggerSessionRequest"], output: "ResultUnit", mode: "update" }, - get_credits_config: { input: [], output: "ResultCreditsConfig", mode: "query" }, + get_cycles_billing_config: { input: [], output: "ResultCyclesBillingConfig", mode: "query" }, create_database: { input: ["CreateDatabaseRequest"], output: "ResultCreateDatabase", mode: "update" }, list_databases: { input: [], output: "ResultDatabases", mode: "query" }, mkdir_node: { input: ["MkdirNodeRequest"], output: "ResultMkdirNode", mode: "update" }, @@ -134,7 +135,7 @@ function parseDidMethods(source) { for (const raw of service.split(";")) { const line = raw.trim().replace(/\s+/g, " "); if (!line) continue; - const match = line.match(/^(\w+)\s*:\s*\(([^)]*)\)\s*->\s*\(([^)]*?)(?:,\s*)?\)(?:\s+(\w+))?$/); + const match = line.match(/^(\w+)\s*:\s*\(([^)]*)\)\s*->\s*\(([^)]*)\)(?:\s+(\w+))?$/); if (!match || !(match[1] in expectedMethods)) continue; methods[match[1]] = { input: splitDidInputs(match[2]), @@ -191,12 +192,12 @@ function normalizeDidShape(value) { function normalizeDidResult(value) { const normalized = normalizeDidShape(value).replace(/,$/, ""); if (normalized === "Result_1") return "ResultUnit"; - if (normalized === "Result_9") return "ResultCreditsConfig"; + if (normalized === "Result_9") return "ResultCyclesBillingConfig"; if (normalized === "Result_4") return "ResultCreateDatabase"; if (normalized === "Result_16") return "ResultDatabases"; if (normalized === "Result_18") return "ResultMkdirNode"; - if (normalized === "Result_25") return "ResultNode"; - if (normalized === "Result_30") return "ResultWriteSourceForGeneration"; + if (normalized === "Result_24") return "ResultNode"; + if (normalized === "Result_29") return "ResultWriteSourceForGeneration"; if (normalized === "Result") return "ResultWriteNode"; return normalized; } @@ -228,7 +229,7 @@ function splitActorInputs(value) { function actorResultName(okShape) { const normalized = normalizeActorShape(okShape); if (normalized === "null") return "ResultUnit"; - if (normalized === "CreditsConfig") return "ResultCreditsConfig"; + if (normalized === "CyclesBillingConfig") return "ResultCyclesBillingConfig"; if (normalized === "CreateDatabaseResult") return "ResultCreateDatabase"; if (normalized === "Vec(DatabaseSummary)") return "ResultDatabases"; if (normalized === "MkdirNodeResult") return "ResultMkdirNode"; diff --git a/extensions/wiki-clipper/src/current-tab-export.js b/extensions/wiki-clipper/src/current-tab-export.js index a5d27cfd..79c75292 100644 --- a/extensions/wiki-clipper/src/current-tab-export.js +++ b/extensions/wiki-clipper/src/current-tab-export.js @@ -141,6 +141,9 @@ async function ensureAuthenticatedForExport(send) { throw new Error("extension auth status is unavailable"); } const response = await send({ type: "auth-status" }); + if (response?.ok === false) { + throw new Error(responseError(response, "extension auth status failed")); + } if (!response?.result?.isAuthenticated) { throw new Error(EXPORT_LOGIN_REQUIRED_MESSAGE); } @@ -258,6 +261,20 @@ async function saveCaptureResult(result, config, send) { } try { const response = await send({ type: "save-source", capture: result.capture, config }); + if (response?.ok === false) { + return { + ok: false, + title: result.capture.conversationTitle || result.target.title, + provider: result.capture.provider, + captureMethod: result.capture.captureMethod, + path: response.result?.path, + created: response.result?.created, + sourceSaved: Boolean(response.result?.path), + generationQueued: false, + generationError: null, + error: responseError(response, "save-source failed") + }; + } const saved = response.result || {}; const generationQueued = saved.generationQueued === true; return { @@ -282,6 +299,10 @@ async function saveCaptureResult(result, config, send) { } } +function responseError(response, fallback) { + return response?.error || response?.result?.error || fallback; +} + export function advanceState(state, event) { const progress = { ...state.progress, diff --git a/extensions/wiki-clipper/src/offscreen.js b/extensions/wiki-clipper/src/offscreen.js index 210c7829..5e4bd3eb 100644 --- a/extensions/wiki-clipper/src/offscreen.js +++ b/extensions/wiki-clipper/src/offscreen.js @@ -5,9 +5,9 @@ import { authSnapshot as defaultAuthSnapshot } from "./auth-client.js"; import { buildUrlIngestRequest } from "./url-ingest-request.js"; import { createVfsActor as defaultCreateVfsActor, - getCreditsConfigOrNull, + getCyclesBillingConfigOrNull, normalizeWritableDatabases, - requireDatabaseWriteCreditsAvailable + requireDatabaseWriteCyclesAvailable } from "./vfs-actor.js"; const URL_INGEST_TRIGGER_URL = "https://wiki.kinic.xyz/api/url-ingest/trigger"; @@ -50,7 +50,7 @@ export async function queueUrlIngest(tab, config) { if (!config?.databaseId) throw new Error("database id is required"); const snapshot = await authenticatedSnapshot(); const actor = await vfsActorFactory({ ...config, identity: snapshot.identity }); - await requireDatabaseWriteCreditsAvailable(actor, config.databaseId); + await requireDatabaseWriteCyclesAvailable(actor, config.databaseId); const session = await ensureTriggerSession(actor, config.databaseId, snapshot.principal); const request = buildUrlIngestRequest({ url: tab.url, @@ -86,7 +86,7 @@ export async function saveRawSource(rawSource, config) { if (!config?.databaseId) throw new Error("database id is required"); const snapshot = await authenticatedSnapshot(); const actor = await vfsActorFactory({ ...config, identity: snapshot.identity }); - await requireDatabaseWriteCreditsAvailable(actor, config.databaseId); + await requireDatabaseWriteCyclesAvailable(actor, config.databaseId); const existing = await actor.read_node(config.databaseId, rawSource.path); if ("Err" in existing) throw new Error(existing.Err); const expected = existing.Ok[0]?.etag ? [existing.Ok[0].etag] : []; @@ -148,12 +148,12 @@ export async function listWritableDatabases(config) { if (!config?.canisterId) throw new Error("canister id is required"); const snapshot = await authenticatedSnapshot(); const actor = await vfsActorFactory({ ...config, identity: snapshot.identity }); - const [result, creditsConfig] = await Promise.all([ + const [result, cyclesConfig] = await Promise.all([ actor.list_databases(), - getCreditsConfigOrNull(actor) + getCyclesBillingConfigOrNull(actor) ]); if ("Err" in result) throw new Error(result.Err); - return normalizeWritableDatabases(result.Ok, creditsConfig); + return normalizeWritableDatabases(result.Ok, cyclesConfig); } export function setOffscreenDepsForTest(deps = {}) { diff --git a/extensions/wiki-clipper/src/raw-source.js b/extensions/wiki-clipper/src/raw-source.js index e80b8f75..13591ab9 100644 --- a/extensions/wiki-clipper/src/raw-source.js +++ b/extensions/wiki-clipper/src/raw-source.js @@ -7,8 +7,8 @@ export function buildRawSource(capture, now = new Date()) { if (!capture.messages || capture.messages.length === 0) { throw new Error("no conversation messages found"); } - const sourceId = sourceIdForCapture(capture, now); - const provider = slug(capture.provider || "conversation"); + const provider = safeProvider(capture.provider || "conversation"); + const sourceId = sourceIdForCapture(capture, now, provider); const path = `/Sources/raw/${provider}/${sourceFileStemForCapture(capture, sourceId)}.md`; const metadata = { provider: capture.provider, @@ -34,25 +34,24 @@ export function buildRawSource(capture, now = new Date()) { }; } -function sourceIdForCapture(capture, now) { - const provider = slug(capture.provider || "conversation"); +function sourceIdForCapture(capture, now, provider = safeProvider(capture.provider || "conversation")) { const conversationId = conversationIdFromUrl(capture.url); if ((capture.provider === "chatgpt" || capture.provider === "claude") && conversationId) { - return `${provider}-${slug(conversationId)}`; + return `${provider}-${safeSourceStem(slug(conversationId))}`; } const title = slug(capture.conversationTitle || "untitled"); const date = now.toISOString().slice(0, 10).replace(/-/g, ""); const fingerprint = hashText(`${capture.url}\n${capture.conversationTitle}`); - return `${provider}-${date}-${title}-${fingerprint}`.slice(0, 96); + return `${provider}-${safeSourceStem(`${date}-${title}-${fingerprint}`)}`; } function sourceFileStemForCapture(capture, sourceId) { const conversationId = conversationIdFromUrl(capture.url); if ((capture.provider === "chatgpt" || capture.provider === "claude") && conversationId) { - return slug(conversationId); + return safeSourceStem(slug(conversationId)); } - const provider = slug(capture.provider || "conversation"); - return sourceId.startsWith(`${provider}-`) ? sourceId.slice(provider.length + 1) : sourceId; + const provider = safeProvider(capture.provider || "conversation"); + return safeSourceStem(sourceId.startsWith(`${provider}-`) ? sourceId.slice(provider.length + 1) : sourceId); } function conversationIdFromUrl(value) { @@ -121,6 +120,20 @@ function slug(value) { return normalized || "untitled"; } +function safeProvider(value) { + const normalized = String(value).toLowerCase().replace(/[^a-z0-9]+/g, ""); + return /^[a-z][a-z0-9]{0,31}$/.test(normalized) ? normalized : "conversation"; +} + +function safeSourceStem(value) { + const normalized = String(value) + .replace(/[^A-Za-z0-9._-]+/g, "-") + .replace(/\.{2,}/g, "-") + .replace(/^-+|-+$/g, "") || "source"; + if (normalized.length <= 128) return normalized; + return `${normalized.slice(0, 119)}-${hashText(normalized)}`; +} + function hashText(value) { let hash = 2166136261; for (let index = 0; index < value.length; index += 1) { diff --git a/extensions/wiki-clipper/src/service-worker.js b/extensions/wiki-clipper/src/service-worker.js index f000e7a3..2347d2bb 100644 --- a/extensions/wiki-clipper/src/service-worker.js +++ b/extensions/wiki-clipper/src/service-worker.js @@ -352,7 +352,7 @@ function errorStatus(message, url = "") { } function shouldOpenSettingsForError(message) { - return message === "UNAUTHENTICATED" || /credits|balance/i.test(String(message || "")); + return message === "UNAUTHENTICATED" || /cycles|balance/i.test(String(message || "")); } function setupRequiredStatus(url = "") { diff --git a/extensions/wiki-clipper/src/url-ingest-request.js b/extensions/wiki-clipper/src/url-ingest-request.js index 9e905aa1..1784e209 100644 --- a/extensions/wiki-clipper/src/url-ingest-request.js +++ b/extensions/wiki-clipper/src/url-ingest-request.js @@ -9,7 +9,7 @@ export const URL_INGEST_STATUS_KEY = "kinic-url-ingest-status-v1"; export function buildUrlIngestRequest({ url, requestedBy, now = new Date(), uuid = crypto.randomUUID() }) { const normalizedUrl = normalizedHttpUrl(url); const requestedAt = now.toISOString(); - const requestId = `${now.getTime()}-${uuid}`; + const requestId = safeIngestRequestId(now, uuid); const requestPath = `/Sources/ingest-requests/${requestId}.md`; return { requestPath, @@ -40,6 +40,22 @@ export function buildUrlIngestRequest({ url, requestedBy, now = new Date(), uuid }; } +export function safeIngestRequestId(now, uuid) { + const suffix = String(uuid || "").trim(); + if (!isSafeRequestSegment(suffix) || suffix.length > 96) { + throw new Error("URL ingest request id is invalid."); + } + const requestId = `${now.getTime()}-${suffix}`; + if (!isSafeRequestSegment(requestId) || requestId.length > 128) { + throw new Error("URL ingest request id is invalid."); + } + return requestId; +} + +function isSafeRequestSegment(value) { + return /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) && value !== "." && value !== ".." && !value.includes(".."); +} + export function normalizedHttpUrl(value) { let url; try { diff --git a/extensions/wiki-clipper/src/vfs-actor.js b/extensions/wiki-clipper/src/vfs-actor.js index 71d017a8..0a64b7f5 100644 --- a/extensions/wiki-clipper/src/vfs-actor.js +++ b/extensions/wiki-clipper/src/vfs-actor.js @@ -17,11 +17,12 @@ export async function createVfsActor({ canisterId, host, identity }) { function idlFactory({ IDL: idl }) { const DatabaseRole = idl.Variant({ Reader: idl.Null, Writer: idl.Null, Owner: idl.Null }); const DatabaseStatus = idl.Variant({ + Hot: idl.Null, + Pending: idl.Null, Active: idl.Null, Restoring: idl.Null, Archiving: idl.Null, - Archived: idl.Null, - Pending: idl.Null + Archived: idl.Null }); const DatabaseSummary = idl.Record({ status: DatabaseStatus, @@ -29,15 +30,15 @@ function idlFactory({ IDL: idl }) { role: DatabaseRole, logical_size_bytes: idl.Nat64, database_id: idl.Text, - archived_at_ms: idl.Opt(idl.Int64), - credits_balance: idl.Opt(idl.Nat64), - credits_suspended_at_ms: idl.Opt(idl.Int64) + cycles_balance: idl.Opt(idl.Nat64), + cycles_suspended_at_ms: idl.Opt(idl.Int64), + archived_at_ms: idl.Opt(idl.Int64) }); - const CreditsConfig = idl.Record({ + const CyclesBillingConfig = idl.Record({ kinic_ledger_canister_id: idl.Text, - sns_governance_id: idl.Text, - credits_per_kinic: idl.Nat64, - min_update_credits: idl.Nat64 + billing_authority_id: idl.Text, + cycles_per_kinic: idl.Nat64, + min_update_cycles: idl.Nat64 }); const CreateDatabaseRequest = idl.Record({ name: idl.Text }); const CreateDatabaseResult = idl.Record({ database_id: idl.Text, name: idl.Text }); @@ -86,7 +87,7 @@ function idlFactory({ IDL: idl }) { }); return idl.Service({ authorize_url_ingest_trigger_session: idl.Func([UrlIngestTriggerSessionRequest], [idl.Variant({ Ok: idl.Null, Err: idl.Text })], []), - get_credits_config: idl.Func([], [idl.Variant({ Ok: CreditsConfig, Err: idl.Text })], ["query"]), + get_cycles_billing_config: idl.Func([], [idl.Variant({ Ok: CyclesBillingConfig, Err: idl.Text })], ["query"]), create_database: idl.Func([CreateDatabaseRequest], [idl.Variant({ Ok: CreateDatabaseResult, Err: idl.Text })], []), list_databases: idl.Func([], [idl.Variant({ Ok: idl.Vec(DatabaseSummary), Err: idl.Text })], ["query"]), mkdir_node: idl.Func([MkdirNodeRequest], [idl.Variant({ Ok: MkdirNodeResult, Err: idl.Text })], []), @@ -111,40 +112,40 @@ export async function createDatabaseWithActor(actor, name) { export async function listWritableDatabases(config) { const actor = await createVfsActor(config); - const [databaseResult, creditsConfig] = await Promise.all([ + const [databaseResult, cyclesConfig] = await Promise.all([ actor.list_databases(), - getCreditsConfigOrNull(actor) + getCyclesBillingConfigOrNull(actor) ]); if ("Err" in databaseResult) { throw new Error(databaseResult.Err); } - return normalizeWritableDatabases(databaseResult.Ok, creditsConfig); + return normalizeWritableDatabases(databaseResult.Ok, cyclesConfig); } -export async function requireDatabaseWriteCreditsAvailable(actor, databaseId) { - const [databaseResult, creditsConfigResult] = await Promise.all([ +export async function requireDatabaseWriteCyclesAvailable(actor, databaseId) { + const [databaseResult, cyclesConfigResult] = await Promise.all([ actor.list_databases(), - actor.get_credits_config() + actor.get_cycles_billing_config() ]); if ("Err" in databaseResult) throw new Error(databaseResult.Err); - if ("Err" in creditsConfigResult) throw new Error(`Credits config unavailable: ${creditsConfigResult.Err}`); - const config = normalizeCreditsConfig(creditsConfigResult.Ok); + if ("Err" in cyclesConfigResult) throw new Error(`Cycles config unavailable: ${cyclesConfigResult.Err}`); + const config = normalizeCyclesBillingConfig(cyclesConfigResult.Ok); const databases = databaseResult.Ok.map(normalizeDatabaseSummary); const database = databases.find((entry) => entry.databaseId === databaseId); - if (!database) throw new Error(`Database credits state unavailable: ${databaseId}`); - const reason = databaseCreditsDisabledReason(database, config); + if (!database) throw new Error(`Database cycles state unavailable: ${databaseId}`); + const reason = databaseCyclesDisabledReason(database, config); if (reason) throw new Error(reason); } -export function normalizeWritableDatabases(rawDatabases, creditsConfig = null) { +export function normalizeWritableDatabases(rawDatabases, cyclesConfig = null) { return rawDatabases.map(normalizeDatabaseSummary).filter((database) => { return database.status === "Active" && (database.role === "Owner" || database.role === "Writer"); }).map((database) => { - const reason = databaseCreditsDisabledReason(database, creditsConfig); + const reason = databaseCyclesDisabledReason(database, cyclesConfig); return { ...database, - writeCreditsAvailable: !reason, - creditsReason: reason + writeCyclesAvailable: !reason, + cyclesReason: reason }; }); } @@ -163,38 +164,39 @@ function normalizeDatabaseSummary(raw) { role: variantKey(raw.role), status: normalizeDatabaseStatus(raw.status), logicalSizeBytes: raw.logical_size_bytes?.toString?.() ?? String(raw.logical_size_bytes ?? "0"), - creditsBalance: raw.credits_balance?.[0]?.toString?.() ?? "0", - creditsSuspendedAtMs: raw.credits_suspended_at_ms?.[0]?.toString?.() ?? null + cyclesBalance: raw.cycles_balance?.[0]?.toString?.() ?? "0", + cyclesSuspendedAtMs: raw.cycles_suspended_at_ms?.[0]?.toString?.() ?? null }; } function normalizeDatabaseStatus(status) { - return variantKey(status); + const key = variantKey(status); + return key === "Hot" ? "Active" : key; } -export async function getCreditsConfigOrNull(actor) { - const result = await actor.get_credits_config(); +export async function getCyclesBillingConfigOrNull(actor) { + const result = await actor.get_cycles_billing_config(); if ("Err" in result) return null; - return normalizeCreditsConfig(result.Ok); + return normalizeCyclesBillingConfig(result.Ok); } -function normalizeCreditsConfig(raw) { +function normalizeCyclesBillingConfig(raw) { return { - minUpdateCredits: raw.min_update_credits?.toString?.() ?? String(raw.min_update_credits ?? "0") + minUpdateCycles: raw.min_update_cycles?.toString?.() ?? String(raw.min_update_cycles ?? "0") }; } -function databaseCreditsDisabledReason(database, config) { - const balance = parseCredits(database.creditsBalance); - const minimum = parseCredits(config?.minUpdateCredits); - if (!config) return "Credits config unavailable."; - if (database.status === "Pending") return "Database activation is pending until its first credit purchase completes."; - if (database.creditsSuspendedAtMs) return "Database credits are suspended."; - if (balance < minimum) return "Database credits balance is below the minimum update balance."; +function databaseCyclesDisabledReason(database, config) { + const balance = parseCycles(database.cyclesBalance); + const minimum = parseCycles(config?.minUpdateCycles); + if (!config) return "Cycles config unavailable."; + if (database.status === "Pending") return "Database activation is pending until its first cycle purchase completes."; + if (database.cyclesSuspendedAtMs) return "Database cycles are suspended."; + if (balance < minimum) return "Database cycles balance is below the minimum update balance."; return null; } -function parseCredits(value) { +function parseCycles(value) { return typeof value === "string" && /^[0-9]+$/.test(value) ? BigInt(value) : 0n; } diff --git a/extensions/wiki-clipper/store-listing/assets/marquee-1400x560.png b/extensions/wiki-clipper/store-listing/assets/marquee-1400x560.png index f9051266..642ba1cd 100644 Binary files a/extensions/wiki-clipper/store-listing/assets/marquee-1400x560.png and b/extensions/wiki-clipper/store-listing/assets/marquee-1400x560.png differ diff --git a/extensions/wiki-clipper/tests/current-tab-export.test.mjs b/extensions/wiki-clipper/tests/current-tab-export.test.mjs index 85c584cd..72af298a 100644 --- a/extensions/wiki-clipper/tests/current-tab-export.test.mjs +++ b/extensions/wiki-clipper/tests/current-tab-export.test.mjs @@ -81,6 +81,35 @@ test("startCurrentTabExport stops before fetching when II is not authenticated", } }); +test("startCurrentTabExport reports auth-status bridge failures before fetching", async () => { + const originalFetch = globalThis.fetch; + let fetchCalled = false; + globalThis.fetch = async () => { + fetchCalled = true; + throw new Error("fetch should not run"); + }; + try { + await assert.rejects( + () => + startCurrentTabExport({ + limit: 1, + config: { canisterId: "abc", databaseId: "team-db", host: "https://icp0.io" }, + originalUrl: "https://chatgpt.com/", + callbacks: { + send: async (message) => { + assert.deepEqual(message, { type: "auth-status" }); + return { ok: false, error: "auth bridge failed" }; + } + } + }), + /auth bridge failed/ + ); + assert.equal(fetchCalled, false); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("fetchRecentConversationTargets paginates, dedupes, and limits", async () => { const calls = []; const fetchImpl = chatGptFetch(async (url, init) => { @@ -265,6 +294,21 @@ test("exportTarget treats saved source with failed generation queue as partial e assert.match(state.logs[0].message, /worker trigger failed: HTTP 502/); }); +test("exportTarget reports save-source bridge failures as save failures", async () => { + const target = { id: "abc", title: "Project", url: "https://chatgpt.com/c/abc" }; + const event = await exportTarget( + target, + { canisterId: "canister", host: "http://127.0.0.1:8001" }, + async () => ({ ok: false, error: "save failed" }), + chatGptFetch(async () => jsonResponse(conversationPayload("abc", "Project"))) + ); + + assert.equal(event.ok, false); + assert.equal(event.sourceSaved, false); + assert.equal(event.generationQueued, false); + assert.equal(event.error, "save failed"); +}); + test("exportTarget does not save API failures or empty conversations", async () => { let saveCount = 0; const failed = await exportTarget( diff --git a/extensions/wiki-clipper/tests/offscreen.test.mjs b/extensions/wiki-clipper/tests/offscreen.test.mjs index bf237768..c7cb67a9 100644 --- a/extensions/wiki-clipper/tests/offscreen.test.mjs +++ b/extensions/wiki-clipper/tests/offscreen.test.mjs @@ -17,7 +17,7 @@ test("queueUrlIngest writes request and triggers via wiki route", async () => { createVfsActor: async (config) => { calls.push(["create", config.identity, config.databaseId]); return { - ...writeCreditsActorMethods(), + ...writeCyclesActorMethods(), async authorize_url_ingest_trigger_session(request) { calls.push(["session", request.database_id, request.session_nonce]); return { Ok: null }; @@ -68,7 +68,7 @@ test("queueUrlIngest keeps request result when trigger fails", async () => { authSnapshot: async () => ({ isAuthenticated: true, identity: { tag: "identity" }, principal: "principal-1" }), fetch: async () => new Response("nope", { status: 502 }), createVfsActor: async () => ({ - ...writeCreditsActorMethods(), + ...writeCyclesActorMethods(), async mkdir_node(request) { return { Ok: { created: true, path: request.path } }; }, @@ -101,7 +101,7 @@ test("queueUrlIngest rejects before writing when session authorize fails", async return Response.json({ accepted: true }); }, createVfsActor: async () => ({ - ...writeCreditsActorMethods(), + ...writeCyclesActorMethods(), async write_node() { calls.push(["write"]); return { Ok: { created: true, node: { etag: "etag-request" } } }; @@ -135,7 +135,7 @@ test("queueUrlIngest reuses session nonce inside ttl", async () => { return Response.json({ accepted: true }); }, createVfsActor: async () => ({ - ...writeCreditsActorMethods(), + ...writeCyclesActorMethods(), async authorize_url_ingest_trigger_session(request) { calls.push(["session", request.session_nonce]); return { Ok: null }; @@ -175,7 +175,7 @@ test("saveRawSource writes with authenticated identity", async () => { createVfsActor: async (config) => { calls.push(["create", config.identity, config.databaseId]); return { - ...writeCreditsActorMethods(), + ...writeCyclesActorMethods(), async read_node(databaseId, path) { calls.push(["read", databaseId, path]); return { Ok: [{ etag: "etag-1" }] }; @@ -230,12 +230,12 @@ test("saveRawSource rejects unauthenticated sessions", async () => { } }); -test("queueUrlIngest rejects low credit balance before writing", async () => { +test("queueUrlIngest rejects low cycle balance before writing", async () => { const calls = []; setOffscreenDepsForTest({ authSnapshot: async () => ({ isAuthenticated: true, identity: { tag: "identity" }, principal: "principal-1" }), createVfsActor: async () => ({ - ...writeCreditsActorMethods({ balanceCredits: 9_999n }), + ...writeCyclesActorMethods({ balanceCycles: 9_999n }), async authorize_url_ingest_trigger_session() { calls.push(["session"]); return { Ok: null }; @@ -262,12 +262,12 @@ test("queueUrlIngest rejects low credit balance before writing", async () => { } }); -test("saveRawSource rejects suspended credits before writing", async () => { +test("saveRawSource rejects suspended cycles before writing", async () => { const calls = []; setOffscreenDepsForTest({ authSnapshot: async () => ({ isAuthenticated: true, identity: { tag: "identity" }, principal: "principal-1" }), createVfsActor: async () => ({ - ...writeCreditsActorMethods({ suspendedAtMs: 1n }), + ...writeCyclesActorMethods({ suspendedAtMs: 1n }), async read_node() { calls.push(["read"]); return { Ok: [] }; @@ -283,7 +283,7 @@ test("saveRawSource rejects suspended credits before writing", async () => { }) }); try { - await assert.rejects(() => saveRawSource(rawSource(), config()), /credits are suspended/); + await assert.rejects(() => saveRawSource(rawSource(), config()), /cycles are suspended/); assert.deepEqual(calls, []); } finally { @@ -345,13 +345,13 @@ test("listWritableDatabases returns active writable database summaries", async ( ] }; }, - async get_credits_config() { + async get_cycles_billing_config() { return { Ok: { kinic_ledger_canister_id: "ryjl3-tyaaa-aaaaa-aaaba-cai", - sns_governance_id: "rrkah-fqaaa-aaaaa-aaaaq-cai", - credits_per_kinic: 1n, - min_update_credits: 10_000n + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai", + cycles_per_kinic: 1n, + min_update_cycles: 10_000n } }; } @@ -365,10 +365,10 @@ test("listWritableDatabases returns active writable database summaries", async ( role: "Writer", status: "Active", logicalSizeBytes: "0", - creditsBalance: "20000", - creditsSuspendedAtMs: null, - writeCreditsAvailable: true, - creditsReason: null + cyclesBalance: "20000", + cyclesSuspendedAtMs: null, + writeCyclesAvailable: true, + cyclesReason: null } ]); } finally { @@ -393,7 +393,7 @@ function config() { }; } -function writeCreditsActorMethods({ databaseId = "team-db", balanceCredits = 20_000n, suspendedAtMs = null } = {}) { +function writeCyclesActorMethods({ databaseId = "team-db", balanceCycles = 20_000n, suspendedAtMs = null } = {}) { return { async list_databases() { return { @@ -404,20 +404,20 @@ function writeCreditsActorMethods({ databaseId = "team-db", balanceCredits = 20_ role: { Writer: null }, status: { Active: null }, logical_size_bytes: 0n, - credits_balance: [balanceCredits], - credits_suspended_at_ms: suspendedAtMs === null ? [] : [suspendedAtMs], + cycles_balance: [balanceCycles], + cycles_suspended_at_ms: suspendedAtMs === null ? [] : [suspendedAtMs], archived_at_ms: [] } ] }; }, - async get_credits_config() { + async get_cycles_billing_config() { return { Ok: { kinic_ledger_canister_id: "ryjl3-tyaaa-aaaaa-aaaba-cai", - sns_governance_id: "rrkah-fqaaa-aaaaa-aaaaq-cai", - credits_per_kinic: 1n, - min_update_credits: 10_000n + billing_authority_id: "rrkah-fqaaa-aaaaa-aaaaq-cai", + cycles_per_kinic: 1n, + min_update_cycles: 10_000n } }; } @@ -431,8 +431,8 @@ function rawDatabase(databaseId, name, role, status) { role: { [role]: null }, status: { [status]: null }, logical_size_bytes: 0n, - credits_balance: [20_000n], - credits_suspended_at_ms: [], + cycles_balance: [20_000n], + cycles_suspended_at_ms: [], archived_at_ms: [] }; } diff --git a/extensions/wiki-clipper/tests/raw-source.test.mjs b/extensions/wiki-clipper/tests/raw-source.test.mjs index be6ab1ec..b2c42a4b 100644 --- a/extensions/wiki-clipper/tests/raw-source.test.mjs +++ b/extensions/wiki-clipper/tests/raw-source.test.mjs @@ -68,6 +68,34 @@ test("buildRawSource keeps a stable path for Claude conversations", () => { assert.equal(JSON.parse(raw.metadataJson).conversation_id, "claude-abc"); }); +test("buildRawSource truncates long conversation ids to a canonical source filename", () => { + const longId = `conversation-${"a".repeat(220)}`; + const raw = buildRawSource({ + provider: "chatgpt", + conversationTitle: "Long ID", + url: `https://chatgpt.com/c/${longId}`, + capturedAt: "2026-05-01T00:00:00.000Z", + messages: [{ role: "user", content: "Hello" }] + }); + const fileName = raw.path.split("/").at(-1); + + assert.match(raw.path, /^\/Sources\/raw\/chatgpt\/[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.md$/); + assert.equal(fileName.length <= 131, true); + assert.equal(JSON.parse(raw.metadataJson).conversation_id, longId); +}); + +test("buildRawSource removes dotdot from conversation source filenames", () => { + const raw = buildRawSource({ + provider: "chatgpt", + conversationTitle: "Dotdot", + url: "https://chatgpt.com/c/a..b", + capturedAt: "2026-05-01T00:00:00.000Z", + messages: [{ role: "user", content: "Hello" }] + }); + + assert.equal(raw.path, "/Sources/raw/chatgpt/a-b.md"); +}); + test("buildRawSource rejects empty captures", () => { assert.throws( () => diff --git a/extensions/wiki-clipper/tests/service-worker.test.mjs b/extensions/wiki-clipper/tests/service-worker.test.mjs index 91795340..c68cb74e 100644 --- a/extensions/wiki-clipper/tests/service-worker.test.mjs +++ b/extensions/wiki-clipper/tests/service-worker.test.mjs @@ -222,12 +222,12 @@ test("unauthenticated save-source opens settings once", async () => { } }); -test("credits-disabled save-source opens settings once", async () => { +test("cycles-disabled save-source opens settings once", async () => { const syncStorage = memoryStorage(); const settingsTabs = []; resetSettingsOpenThrottleForTest(); const restore = installChromeForSettings(syncStorage, settingsTabs); - setOffscreenBridgeForTest(async () => ({ ok: false, error: "Database credits balance is below the minimum update balance." })); + setOffscreenBridgeForTest(async () => ({ ok: false, error: "Database cycles balance is below the minimum update balance." })); try { const message = { type: "save-source", @@ -285,12 +285,12 @@ test("action click opens settings when database config is incomplete", async () ]); }); -test("action click opens settings and stores status when credits is disabled", async () => { +test("action click opens settings and stores status when cycles is disabled", async () => { const calls = []; const response = await handleActionClick( { url: "https://example.com/", title: "Example" }, actionDeps({ - sendOffscreen: async () => ({ ok: false, error: "Database credits are suspended." }), + sendOffscreen: async () => ({ ok: false, error: "Database cycles are suspended." }), openSettings: async () => calls.push(["settings"]), writeStatus: async (status) => calls.push(["status", status.status, status.message]), setBadge: async (text) => calls.push(["badge", text]) @@ -299,7 +299,7 @@ test("action click opens settings and stores status when credits is disabled", a assert.equal(response.ok, false); assert.deepEqual(calls, [ ["badge", "..."], - ["status", "error", "Database credits are suspended."], + ["status", "error", "Database cycles are suspended."], ["badge", "ERR"], ["settings"] ]); diff --git a/extensions/wiki-clipper/tests/settings.test.mjs b/extensions/wiki-clipper/tests/settings.test.mjs index 934b8c1f..4411baca 100644 --- a/extensions/wiki-clipper/tests/settings.test.mjs +++ b/extensions/wiki-clipper/tests/settings.test.mjs @@ -138,9 +138,9 @@ test("database dropdown options include only active owner and writer databases", rawDatabase("writer-db", "Writer", "Active", 20_000n), rawDatabase("reader-db", "Reader", "Active", 20_000n), rawDatabase("archived-db", "Owner", "Archived", 20_000n) - ], { minUpdateCredits: "10000" }); + ], { minUpdateCycles: "10000" }); assert.deepEqual( - databases.map((database) => [database.databaseId, database.name, database.role, database.status, database.writeCreditsAvailable]), + databases.map((database) => [database.databaseId, database.name, database.role, database.status, database.writeCyclesAvailable]), [ ["owner-db", "owner-db name", "Owner", "Active", true], ["writer-db", "writer-db name", "Writer", "Active", true] @@ -148,29 +148,29 @@ test("database dropdown options include only active owner and writer databases", ); }); -test("database dropdown keeps credits-disabled writer databases with reasons", () => { +test("database dropdown keeps cycles-disabled writer databases with reasons", () => { const databases = normalizeWritableDatabases([ rawDatabase("active-db", "Owner", "Active", 20_000n), rawDatabase("low-db", "Writer", "Active", 9_999n), rawDatabase("suspended-db", "Writer", "Active", 20_000n, 1n) - ], { minUpdateCredits: "10000" }); + ], { minUpdateCycles: "10000" }); assert.deepEqual( - databases.map((database) => [database.databaseId, database.writeCreditsAvailable, database.creditsReason]), + databases.map((database) => [database.databaseId, database.writeCyclesAvailable, database.cyclesReason]), [ ["active-db", true, null], - ["low-db", false, "Database credits balance is below the minimum update balance."], - ["suspended-db", false, "Database credits are suspended."] + ["low-db", false, "Database cycles balance is below the minimum update balance."], + ["suspended-db", false, "Database cycles are suspended."] ] ); }); -test("database dropdown disables writer databases when credits config is unavailable", () => { +test("database dropdown disables writer databases when cycles config is unavailable", () => { const databases = normalizeWritableDatabases([ rawDatabase("owner-db", "Owner", "Active", 20_000n) ]); assert.deepEqual( - databases.map((database) => [database.databaseId, database.writeCreditsAvailable, database.creditsReason]), - [["owner-db", false, "Credits config unavailable."]] + databases.map((database) => [database.databaseId, database.writeCyclesAvailable, database.cyclesReason]), + [["owner-db", false, "Cycles config unavailable."]] ); }); @@ -251,17 +251,17 @@ test("settings docs describe automatic database save", () => { assert.doesNotMatch(storeAssets, /Refresh/); }); -function rawDatabase(databaseId, role, status, nameOrBalance = 20_000n, creditsSuspendedAtMs = null) { +function rawDatabase(databaseId, role, status, nameOrBalance = 20_000n, cyclesSuspendedAtMs = null) { const name = typeof nameOrBalance === "string" ? nameOrBalance : `${databaseId} name`; - const creditsBalance = typeof nameOrBalance === "bigint" ? nameOrBalance : 20_000n; + const cyclesBalance = typeof nameOrBalance === "bigint" ? nameOrBalance : 20_000n; return { database_id: databaseId, name, role: { [role]: null }, status: { [status]: null }, logical_size_bytes: 0n, - credits_balance: [creditsBalance], - credits_suspended_at_ms: creditsSuspendedAtMs === null ? [] : [creditsSuspendedAtMs], + cycles_balance: [cyclesBalance], + cycles_suspended_at_ms: cyclesSuspendedAtMs === null ? [] : [cyclesSuspendedAtMs], archived_at_ms: [] }; } diff --git a/extensions/wiki-clipper/tests/url-ingest-request.test.mjs b/extensions/wiki-clipper/tests/url-ingest-request.test.mjs index 719864cb..097067ee 100644 --- a/extensions/wiki-clipper/tests/url-ingest-request.test.mjs +++ b/extensions/wiki-clipper/tests/url-ingest-request.test.mjs @@ -3,7 +3,7 @@ // Why: Extension-created requests must match the worker/browser contract. import assert from "node:assert/strict"; import test from "node:test"; -import { buildUrlIngestRequest, normalizedHttpUrl } from "../src/url-ingest-request.js"; +import { buildUrlIngestRequest, normalizedHttpUrl, safeIngestRequestId } from "../src/url-ingest-request.js"; test("buildUrlIngestRequest creates a file request with frontmatter", () => { const request = buildUrlIngestRequest({ @@ -31,3 +31,12 @@ test("normalizedHttpUrl accepts only http and https", () => { assert.equal(normalizedHttpUrl("http://example.com/#x"), "http://example.com/"); assert.throws(() => normalizedHttpUrl("chrome://extensions"), /http or https/); }); + +test("safeIngestRequestId rejects non-canonical path segments", () => { + const now = new Date("2026-05-13T00:00:00.000Z"); + assert.equal(safeIngestRequestId(now, "uuid-1"), "1778630400000-uuid-1"); + assert.throws(() => safeIngestRequestId(now, "../x"), /request id/); + assert.throws(() => safeIngestRequestId(now, "x/y"), /request id/); + assert.throws(() => safeIngestRequestId(now, ""), /request id/); + assert.throws(() => safeIngestRequestId(now, "x".repeat(97)), /request id/); +}); diff --git a/icp.yaml b/icp.yaml index 204c98e6..653079d7 100644 --- a/icp.yaml +++ b/icp.yaml @@ -12,7 +12,7 @@ networks: ii: true gateway: bind: 127.0.0.1 - port: 8001 + port: 8011 environments: - name: local-wiki diff --git a/plugins/runtime/kinic_agent_runtime/evolve.py b/plugins/runtime/kinic_agent_runtime/evolve.py index f9b609c3..c090615c 100644 --- a/plugins/runtime/kinic_agent_runtime/evolve.py +++ b/plugins/runtime/kinic_agent_runtime/evolve.py @@ -365,7 +365,7 @@ def source_runs_from_job(content: str) -> list[str]: continue if in_source_runs: if line.startswith(" - "): - paths.append(line[4:].strip().strip('"')) + paths.append(clean_yaml_scalar(line[4:])) continue if line and not line.startswith(" "): break @@ -381,17 +381,31 @@ def frontmatter_scalar(content: str, key: str) -> str | None: continue field, value = line.split(":", 1) if field.strip() == key: - return value.strip().strip('"') + return clean_yaml_scalar(value) return None def frontmatter_lines(content: str) -> list[str] | None: if not content.startswith("---\n"): return None - end = content.find("\n---", 4) - if end < 0: - return None - return content[4:end].splitlines() + lines: list[str] = [] + for line in content[4:].splitlines(): + if line == "---": + return lines + lines.append(line) + return None + + +def clean_yaml_scalar(value: str) -> str: + trimmed = value.strip() + if trimmed.startswith('"') and trimmed.endswith('"'): + parsed = json.loads(trimmed) + if isinstance(parsed, str): + return parsed + raise ValueError("quoted YAML scalar must be a string") + if trimmed.startswith("'") and trimmed.endswith("'"): + return trimmed[1:-1].replace("''", "'") + return trimmed def skill_id_from_job_content(content: str) -> str | None: @@ -450,7 +464,11 @@ def validate_candidate(current_skill: str, candidate: str) -> dict[str, Any]: def same_declared_identity(current_skill: str, candidate: str) -> bool: current_title = first_heading(current_skill) candidate_title = first_heading(candidate) - return not current_title or not candidate_title or current_title.lower() == candidate_title.lower() + if not current_title: + return True + if not candidate_title: + return False + return current_title.lower() == candidate_title.lower() def first_heading(content: str) -> str | None: diff --git a/plugins/runtime/tests/test_evolve.py b/plugins/runtime/tests/test_evolve.py new file mode 100644 index 00000000..e90d5768 --- /dev/null +++ b/plugins/runtime/tests/test_evolve.py @@ -0,0 +1,27 @@ +"""Where: plugins/runtime/tests/test_evolve.py +What: Regression tests for skill evolution frontmatter helpers. +Why: Job metadata parsing gates proposal application and source evidence reads. +""" + +import unittest + +from kinic_agent_runtime import evolve + + +class EvolveParsingTests(unittest.TestCase): + def test_frontmatter_uses_whole_line_terminator(self) -> None: + content = "---\nstatus: running\n---not-a-terminator\nskill_id: bad\n---\n# Body\n" + self.assertEqual(evolve.frontmatter_scalar(content, "skill_id"), "bad") + + def test_frontmatter_unescapes_json_quoted_scalars(self) -> None: + content = '---\nskill_id: "Skill\\nID"\nsource_runs:\n - "/Sources/run\\\"1.md"\n---\n# Body\n' + self.assertEqual(evolve.frontmatter_scalar(content, "skill_id"), "Skill\nID") + self.assertEqual(evolve.source_runs_from_job(content), ['/Sources/run"1.md']) + + def test_identity_gate_requires_candidate_h1_when_current_has_h1(self) -> None: + self.assertFalse(evolve.same_declared_identity("# Skill\n", "## Skill\n")) + self.assertTrue(evolve.same_declared_identity("# Skill\n", "# Skill\n")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/check-regression-groups.mjs b/scripts/check-regression-groups.mjs new file mode 100644 index 00000000..807511fb --- /dev/null +++ b/scripts/check-regression-groups.mjs @@ -0,0 +1,80 @@ +// Where: scripts/check-regression-groups.mjs +// What: Verify that the grouped bug-regression coverage required by the 100-bug plan remains wired. +// Why: The concrete regressions live in language-specific tests; this guard keeps the plan-level groups visible. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); + +const groups = { + security_path_validation: [ + ["crates/wiki_domain/src/lib.rs", "canonical_source_path_rejects_prefix_lookalikes"], + ["crates/vfs_canister/src/tests.rs", "fs_entrypoints_reject_noncanonical_source_paths"], + ["workers/wiki-generator/tests/source-path.test.ts", "/Sources/rawfoo/alpha/alpha.md"], + ["extensions/wiki-clipper/tests/url-ingest-request.test.mjs", "safeIngestRequestId rejects non-canonical path segments"] + ], + skill_registry_schema: [ + ["crates/vfs_cli_app/src/skill_registry_tests.rs", "skill_approve_proposal_rejects_wrong_path_and_frontmatter"], + ["crates/vfs_cli_app/src/skill_registry_tests.rs", "skill_upsert_rejects_noncanonical_skill_ids_before_writing"], + ["crates/vfs_cli_app/src/skill_registry_tests.rs", "skill_set_status_removes_stale_status_metadata"], + ["wikibrowser/scripts/check-skill-registry.mjs", "improvement-proposals|kinic\\.skill_improvement_proposal"], + ["wikibrowser/scripts/check-skill-registry.mjs", "skill..v1"], + ["skill-registry-web/scripts/check-skill-registry-web.mjs", "baseEtag"], + ["docs/SKILL_REGISTRY.md", "/Wiki/skills//proposals//"] + ], + frontmatter_markdown: [ + ["workers/wiki-generator/tests/frontmatter.test.ts", "frontmatter parser requires a whole-line terminator"], + ["crates/vfs_cli_app/src/skill_registry_tests.rs", "docs/Project (Alpha).md"], + ["crates/vfs_cli_app/src/skill_registry_tests.rs", "docs/usage.md \\\"Usage\\\""], + ["wikibrowser/scripts/check-skill-registry.mjs", "docs/Project Plan.md"], + ["wikibrowser/scripts/check-skill-registry.mjs", "docs/usage.md \\\"Usage\\\""], + ["skill-registry-web/scripts/check-skill-registry-web.mjs", "docs/Project Plan.md"], + ["skill-registry-web/scripts/check-skill-registry-web.mjs", "docs/usage.md \\\"Usage\\\""] + ], + worker_jobs: [ + ["workers/wiki-generator/tests/processing.test.ts", "missing queued source is recorded as failed"], + ["workers/wiki-generator/tests/processing.test.ts", "kind: \"url_ingest\""], + ["workers/wiki-generator/src/processing.ts", "url_ingest requestPath is non-canonical"], + ["workers/wiki-generator/src/jobs.ts", "attempts = 0"], + ["workers/wiki-generator/src/jobs.ts", "target_path = NULL"], + ["workers/wiki-generator/tests/openai.test.ts", "non-JSON DeepSeek failures before parsing"] + ], + extension_capture: [ + ["extensions/wiki-clipper/tests/raw-source.test.mjs", "truncates long conversation ids to a canonical source filename"], + ["extensions/wiki-clipper/tests/raw-source.test.mjs", "removes dotdot from conversation source filenames"], + ["extensions/wiki-clipper/tests/url-ingest-request.test.mjs", "safeIngestRequestId"], + ["wikibrowser/lib/url-ingest.ts", "safeIngestRequestId(Date.now(), crypto.randomUUID())"] + ], + canister_ci_filter: [ + [".github/workflows/ci.yml", "crates/(vfs_canister|vfs_runtime|vfs_types|vfs_store|wiki_domain)/"], + [".github/workflows/ci.yml", "set_output rust_all"] + ], + canister_cycles_billing: [ + ["scripts/smoke/local_canister_post_upgrade.sh", "scripts/local/deploy_wiki.sh --mode upgrade"], + ["docs/payment.md", "billing_authority_review"], + ["docs/payment.md", "v1 では repair / cancel API は提供しない"], + ["docs/payment.md", "v1 では retry API は提供しない"], + ["docs/payment.md", "残高を全徴収"] + ] +}; + +for (const [group, checks] of Object.entries(groups)) { + for (const [relativePath, marker] of checks) { + const content = readFileSync(join(root, relativePath), "utf8"); + assert.match(content, new RegExp(escapeRegExp(marker)), `${group}: missing marker ${marker} in ${relativePath}`); + } +} + +assert.doesNotMatch( + readFileSync(join(root, "docs/payment.md"), "utf8"), + /自動 repair API と cancel repair API は提供しない/, + "canister_cycles_billing: obsolete repair wording remains in docs/payment.md" +); + +console.log(`Regression groups OK: ${Object.keys(groups).join(", ")}`); + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/scripts/deploy/wiki_credits_args.sh b/scripts/deploy/wiki_credits_args.sh deleted file mode 100755 index d28d31d1..00000000 --- a/scripts/deploy/wiki_credits_args.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Where: scripts/deploy/wiki_credits_args.sh -# What: Emit the Candid CreditsConfig tuple used by wiki canister install/upgrade. -# Why: Fresh install requires explicit ledger/governance config, so deploy scripts share one source. - -ANONYMOUS_PRINCIPAL="2vxsx-fae" - -required_principal() { - local name="$1" - local value="${!name:-}" - if [[ -z "$value" ]]; then - echo "${name} is required" >&2 - exit 1 - fi - if [[ "$value" == "$ANONYMOUS_PRINCIPAL" ]]; then - echo "${name} must not be anonymous" >&2 - exit 1 - fi - printf '%s' "$value" -} - -positive_u64() { - local name="$1" - local default="$2" - local value="${!name:-$default}" - if [[ ! "$value" =~ ^[0-9]+$ || "$value" == "0" ]]; then - echo "${name} must be a positive integer" >&2 - exit 1 - fi - printf '%s' "$value" -} - -ledger="$(required_principal KINIC_LEDGER_CANISTER_ID)" -governance="$(required_principal SNS_GOVERNANCE_ID)" -credits_per_kinic="$(positive_u64 CREDITS_PER_KINIC 1000)" -min_update_credits="$(positive_u64 MIN_UPDATE_CREDITS 1)" - -printf '(record { kinic_ledger_canister_id = "%s"; sns_governance_id = "%s"; credits_per_kinic = %s : nat64; min_update_credits = %s : nat64 })\n' \ - "$ledger" \ - "$governance" \ - "$credits_per_kinic" \ - "$min_update_credits" diff --git a/scripts/local/deploy_wiki.sh b/scripts/local/deploy_wiki.sh index 53b78a5e..706d72e0 100755 --- a/scripts/local/deploy_wiki.sh +++ b/scripts/local/deploy_wiki.sh @@ -2,15 +2,14 @@ set -euo pipefail # Where: scripts/local/deploy_wiki.sh -# What: Deploy the wiki canister locally with explicit credits init args. -# Why: Local credits tests need a stable ledger ID while production keeps explicit env validation. +# What: Deploy the wiki canister locally with explicit cycles billing init args. +# Why: Local cycles tests need a stable ledger ID while production keeps explicit env validation. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" ICP_ENVIRONMENT="${ICP_ENVIRONMENT:-local-wiki}" KINIC_LEDGER_CANISTER_ID="${KINIC_LEDGER_CANISTER_ID:-}" -SNS_GOVERNANCE_ID="${SNS_GOVERNANCE_ID:-}" -MODE="${MODE:-auto}" +BILLING_AUTHORITY_ID="${BILLING_AUTHORITY_ID:-}" case "${ICP_ENVIRONMENT}" in local | local-wiki) ;; @@ -24,8 +23,8 @@ current_identity_principal() { icp identity principal } -if [[ -z "${SNS_GOVERNANCE_ID}" ]]; then - SNS_GOVERNANCE_ID="$(current_identity_principal)" +if [[ -z "${BILLING_AUTHORITY_ID}" ]]; then + BILLING_AUTHORITY_ID="$(current_identity_principal)" fi require_principal_env() { @@ -42,26 +41,26 @@ require_principal_env() { } require_principal_env KINIC_LEDGER_CANISTER_ID -require_principal_env SNS_GOVERNANCE_ID +require_principal_env BILLING_AUTHORITY_ID -ARGS_FILE="$(mktemp "${TMPDIR:-/tmp}/wiki-local-credits-init.XXXXXX.did")" +ARGS_FILE="$(mktemp "${TMPDIR:-/tmp}/wiki-local-cycles-init.XXXXXX.did")" trap 'rm -f "${ARGS_FILE}"' EXIT cat >"${ARGS_FILE}" <&2 + echo "local wiki cycles init args generated for ${ICP_ENVIRONMENT}" >&2 echo "KINIC_LEDGER_CANISTER_ID=${KINIC_LEDGER_CANISTER_ID}" >&2 - echo "SNS_GOVERNANCE_ID=${SNS_GOVERNANCE_ID}" >&2 + echo "BILLING_AUTHORITY_ID=${BILLING_AUTHORITY_ID}" >&2 exit 0 fi cd "${REPO_ROOT}" -icp deploy wiki -e "${ICP_ENVIRONMENT}" --mode "${MODE}" --args-file "${ARGS_FILE}" "$@" +icp deploy wiki -e "${ICP_ENVIRONMENT}" --args-file "${ARGS_FILE}" "$@" diff --git a/scripts/local/setup_kinic_ledger.sh b/scripts/local/setup_kinic_ledger.sh index 4eb25d3f..813b9242 100755 --- a/scripts/local/setup_kinic_ledger.sh +++ b/scripts/local/setup_kinic_ledger.sh @@ -2,7 +2,7 @@ set -euo pipefail # Where: scripts/local/setup_kinic_ledger.sh -# What: Prepare a project-local ICRC ledger for KINIC credit purchase smoke tests. +# What: Prepare a project-local ICRC ledger for KINIC cycle purchase smoke tests. # Why: The wiki canister stores the ledger principal at init, so local smoke needs a real ledger before deploy. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -11,7 +11,7 @@ ICP_ENVIRONMENT="${ICP_ENVIRONMENT:-local-wiki}" LEDGER_ID_DIR="${REPO_ROOT}/.icp/cache/local-kinic-ledger" LEDGER_ID_FILE="${LEDGER_ID_DIR}/${ICP_ENVIRONMENT}.id" DEFAULT_INITIAL_BALANCE_E8S="${KINIC_LEDGER_INITIAL_BALANCE_E8S:-100000000000}" -LEDGER_TRANSFER_FEE_E8S="${KINIC_LEDGER_TRANSFER_FEE_E8S:-10000}" +LEDGER_TRANSFER_FEE_E8S="${KINIC_LEDGER_TRANSFER_FEE_E8S:-100000}" case "${ICP_ENVIRONMENT}" in local | local-wiki) ;; diff --git a/scripts/mainnet/deploy_wiki.sh b/scripts/mainnet/deploy_wiki.sh index 6a176e70..224f522e 100755 --- a/scripts/mainnet/deploy_wiki.sh +++ b/scripts/mainnet/deploy_wiki.sh @@ -2,12 +2,13 @@ set -euo pipefail # Where: scripts/mainnet/deploy_wiki.sh -# What: Deploy the wiki canister to mainnet with explicit credits init args. -# Why: Credits ledger and SNS principals are immutable after init, so placeholders must never reach production. +# What: Deploy the wiki canister to mainnet with cycles billing init args. +# Why: Cycles ledger and billing authority principals are immutable after init, so init values must be concrete deploy-time principals. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" ANONYMOUS_PRINCIPAL="2vxsx-fae" +BILLING_AUTHORITY_ID="${BILLING_AUTHORITY_ID:-}" require_principal_env() { local name="$1" @@ -27,22 +28,23 @@ require_principal_env() { } require_principal_env KINIC_LEDGER_CANISTER_ID -require_principal_env SNS_GOVERNANCE_ID +require_principal_env BILLING_AUTHORITY_ID -ARGS_FILE="$(mktemp "${TMPDIR:-/tmp}/wiki-credits-init.XXXXXX.did")" +ARGS_FILE="$(mktemp "${TMPDIR:-/tmp}/wiki-cycles-init.XXXXXX.did")" trap 'rm -f "${ARGS_FILE}"' EXIT cat >"${ARGS_FILE}" <&2 + echo "mainnet wiki cycles init args validated" >&2 + echo "BILLING_AUTHORITY_ID=${BILLING_AUTHORITY_ID}" >&2 exit 0 fi diff --git a/scripts/setup-wikibrowser-ii-e2e.sh b/scripts/setup-wikibrowser-ii-e2e.sh index c2c468e7..79e3ed36 100755 --- a/scripts/setup-wikibrowser-ii-e2e.sh +++ b/scripts/setup-wikibrowser-ii-e2e.sh @@ -57,7 +57,7 @@ ensure_canister_id "$FRONTEND_CANISTER_ID_FILE" II_BACKEND_CANISTER_ID="$(tr -d '[:space:]' < "$BACKEND_CANISTER_ID_FILE")" II_FRONTEND_CANISTER_ID="$(tr -d '[:space:]' < "$FRONTEND_CANISTER_ID_FILE")" WIKI_CANISTER_ID="$(node -e 'const fs=require("fs"); const file=process.argv[1]; const ids=JSON.parse(fs.readFileSync(file,"utf8")); if(!ids.wiki) throw new Error("wiki canister id is missing"); process.stdout.write(ids.wiki);' "$MAPPING_FILE")" -II_FRONTEND_INIT_ARGS="$(printf '(record { backend_canister_id = principal "%s"; backend_origin = "http://%s.raw.localhost:8001"; related_origins = null; fetch_root_key = opt true; analytics_config = null; dummy_auth = opt opt record { prompt_for_index = false }; dev_csp = opt true })' "$II_BACKEND_CANISTER_ID" "$II_BACKEND_CANISTER_ID")" +II_FRONTEND_INIT_ARGS="$(printf '(record { backend_canister_id = principal "%s"; backend_origin = "http://%s.raw.localhost:8011"; related_origins = null; fetch_root_key = opt true; analytics_config = null; dummy_auth = opt opt record { prompt_for_index = false }; dev_csp = opt true })' "$II_BACKEND_CANISTER_ID" "$II_BACKEND_CANISTER_ID")" if ! icp canister install "$II_BACKEND_CANISTER_ID" \ -e local-wiki \ @@ -67,7 +67,7 @@ if ! icp canister install "$II_BACKEND_CANISTER_ID" \ -y; then icp canister create --detached -e local-wiki --quiet > "$BACKEND_CANISTER_ID_FILE" II_BACKEND_CANISTER_ID="$(tr -d '[:space:]' < "$BACKEND_CANISTER_ID_FILE")" - II_FRONTEND_INIT_ARGS="$(printf '(record { backend_canister_id = principal "%s"; backend_origin = "http://%s.raw.localhost:8001"; related_origins = null; fetch_root_key = opt true; analytics_config = null; dummy_auth = opt opt record { prompt_for_index = false }; dev_csp = opt true })' "$II_BACKEND_CANISTER_ID" "$II_BACKEND_CANISTER_ID")" + II_FRONTEND_INIT_ARGS="$(printf '(record { backend_canister_id = principal "%s"; backend_origin = "http://%s.raw.localhost:8011"; related_origins = null; fetch_root_key = opt true; analytics_config = null; dummy_auth = opt opt record { prompt_for_index = false }; dev_csp = opt true })' "$II_BACKEND_CANISTER_ID" "$II_BACKEND_CANISTER_ID")" icp canister install "$II_BACKEND_CANISTER_ID" \ -e local-wiki \ --mode reinstall \ @@ -93,11 +93,11 @@ if ! icp canister install "$II_FRONTEND_CANISTER_ID" \ fi { - printf 'NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8001\n' + printf 'NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8011\n' printf 'NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=%s\n' "$WIKI_CANISTER_ID" - printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8001\n' "$II_FRONTEND_CANISTER_ID" + printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8011\n' "$II_FRONTEND_CANISTER_ID" } > "$ENV_FILE" printf 'Wrote %s\n' "$ENV_FILE" printf 'NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID=%s\n' "$WIKI_CANISTER_ID" -printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8001\n' "$II_FRONTEND_CANISTER_ID" +printf 'NEXT_PUBLIC_II_PROVIDER_URL=http://%s.raw.localhost:8011\n' "$II_FRONTEND_CANISTER_ID" diff --git a/scripts/smoke/local_canister_archive_restore.sh b/scripts/smoke/local_canister_archive_restore.sh index d30b57ac..ab4f8bd7 100755 --- a/scripts/smoke/local_canister_archive_restore.sh +++ b/scripts/smoke/local_canister_archive_restore.sh @@ -9,9 +9,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" ICP_ENVIRONMENT="${ICP_ENVIRONMENT:-local-wiki}" IDS_FILE="${REPO_ROOT}/.icp/cache/mappings/${ICP_ENVIRONMENT}.ids.json" -SMOKE_CREDIT_PURCHASE_CREDITS="${SMOKE_CREDIT_PURCHASE_CREDITS:-1000}" -SMOKE_CREDIT_PURCHASE_COUNT="${SMOKE_CREDIT_PURCHASE_COUNT:-3}" -SMOKE_CREDITS_ALLOWANCE_E8S="${SMOKE_CREDITS_ALLOWANCE_E8S:-400000000}" +SMOKE_CYCLE_PURCHASE_E8S="${SMOKE_CYCLE_PURCHASE_E8S:-100000000}" +SMOKE_CYCLE_PURCHASE_KINIC="${SMOKE_CYCLE_PURCHASE_KINIC:-1}" +SMOKE_CYCLE_PURCHASE_COUNT="${SMOKE_CYCLE_PURCHASE_COUNT:-3}" +SMOKE_CYCLES_ALLOWANCE_E8S="${SMOKE_CYCLES_ALLOWANCE_E8S:-400000000}" case "${ICP_ENVIRONMENT}" in local | local-wiki) ;; @@ -81,86 +82,40 @@ canister_has_module() { } wiki_ledger_canister_id() { - icp canister call wiki get_credits_config '()' -e "${ICP_ENVIRONMENT}" -o candid 2>/dev/null \ + icp canister call wiki get_cycles_billing_config '()' -e "${ICP_ENVIRONMENT}" -o candid 2>/dev/null \ | awk -F'"' '/kinic_ledger_canister_id/ { print $2; exit }' } deploy_wiki() { ICP_ENVIRONMENT="${ICP_ENVIRONMENT}" \ KINIC_LEDGER_CANISTER_ID="${KINIC_LEDGER_CANISTER_ID}" \ - SNS_GOVERNANCE_ID="${SNS_GOVERNANCE_ID}" \ + BILLING_AUTHORITY_ID="${BILLING_AUTHORITY_ID}" \ bash scripts/local/deploy_wiki.sh "$@" } -approve_credits_allowance() { +approve_cycles_allowance() { local canister_id="$1" - echo "approving ${SMOKE_CREDITS_ALLOWANCE_E8S} e8s for wiki canister ${canister_id}" >&2 + echo "approving ${SMOKE_CYCLES_ALLOWANCE_E8S} e8s for wiki canister ${canister_id}" >&2 local approve_result if ! approve_result="$(icp canister call "${KINIC_LEDGER_CANISTER_ID}" icrc2_approve \ - "(record { spender = record { owner = principal \"${canister_id}\"; subaccount = null }; amount = ${SMOKE_CREDITS_ALLOWANCE_E8S} : nat; expected_allowance = null; expires_at = null; fee = null; memo = null; from_subaccount = null; created_at_time = null })" \ + "(record { spender = record { owner = principal \"${canister_id}\"; subaccount = null }; amount = ${SMOKE_CYCLES_ALLOWANCE_E8S} : nat; expected_allowance = null; expires_at = null; fee = null; memo = null; from_subaccount = null; created_at_time = null })" \ -e "${ICP_ENVIRONMENT}" -o candid)"; then - echo "KINIC approve failed. Ensure the current identity has enough local KINIC balance for ${SMOKE_CREDIT_PURCHASE_COUNT} credit purchases plus ledger fees." >&2 + echo "KINIC approve failed. Ensure the current identity has enough local KINIC balance for ${SMOKE_CYCLE_PURCHASE_COUNT} cycle purchases plus ledger fees." >&2 exit 1 fi if [[ "${approve_result}" == *"Err"* ]]; then echo "KINIC approve returned an error: ${approve_result}" >&2 - echo "Ensure the current identity has enough local KINIC balance for ${SMOKE_CREDIT_PURCHASE_COUNT} credit purchases plus ledger fees." >&2 - exit 1 - fi -} - -candid_nat64_field() { - local field="$1" - node -e ' - const fs = require("fs"); - const [field] = process.argv.slice(1); - const text = fs.readFileSync(0, "utf8"); - const match = text.match(new RegExp(`${field}\\s*=\\s*([0-9_]+)\\s*:\\s*nat64`)); - if (!match) process.exit(1); - process.stdout.write(match[1].replaceAll("_", "")); - ' "$field" -} - -purchase_smoke_database_credits() { - local database_id="$1" - local credits="$2" - echo "purchasing ${credits} credits for smoke database ${database_id}" >&2 - local preview_result - if ! preview_result="$(icp canister call "${CANISTER_ID}" preview_database_credit_purchase \ - "(\"${database_id}\", ${credits} : nat64)" \ - -e "${ICP_ENVIRONMENT}" -o candid)"; then - echo "credit purchase preview failed for ${database_id}" >&2 - exit 1 - fi - if [[ "${preview_result}" == *"Err"* ]]; then - echo "credit purchase preview returned an error for ${database_id}: ${preview_result}" >&2 - exit 1 - fi - - local expected_payment_amount_e8s - local expected_config_version - expected_payment_amount_e8s="$(printf '%s' "${preview_result}" | candid_nat64_field payment_amount_e8s)" - expected_config_version="$(printf '%s' "${preview_result}" | candid_nat64_field config_version)" - - local purchase_result - if ! purchase_result="$(icp canister call "${CANISTER_ID}" purchase_database_credits \ - "(record { database_id = \"${database_id}\"; credits = ${credits} : nat64; expected_payment_amount_e8s = ${expected_payment_amount_e8s} : nat64; expected_config_version = ${expected_config_version} : nat64 })" \ - -e "${ICP_ENVIRONMENT}" -o candid)"; then - echo "credit purchase failed for ${database_id}" >&2 - exit 1 - fi - if [[ "${purchase_result}" == *"Err"* ]]; then - echo "credit purchase returned an error for ${database_id}: ${purchase_result}" >&2 + echo "Ensure the current identity has enough local KINIC balance for ${SMOKE_CYCLE_PURCHASE_COUNT} cycle purchases plus ledger fees." >&2 exit 1 fi } cd "${REPO_ROOT}" -validate_unsigned_integer SMOKE_CREDIT_PURCHASE_CREDITS -validate_unsigned_integer SMOKE_CREDIT_PURCHASE_COUNT -validate_unsigned_integer SMOKE_CREDITS_ALLOWANCE_E8S -if [[ -z "${SNS_GOVERNANCE_ID:-}" ]]; then - export SNS_GOVERNANCE_ID="$(current_identity_principal)" +validate_unsigned_integer SMOKE_CYCLE_PURCHASE_E8S +validate_unsigned_integer SMOKE_CYCLE_PURCHASE_COUNT +validate_unsigned_integer SMOKE_CYCLES_ALLOWANCE_E8S +if [[ -z "${BILLING_AUTHORITY_ID:-}" ]]; then + export BILLING_AUTHORITY_ID="$(current_identity_principal)" fi if [[ -z "${REPLICA_HOST:-}" ]]; then @@ -171,7 +126,7 @@ export REPLICA_HOST LEDGER_SETUP_OUTPUT="$(ICP_ENVIRONMENT="${ICP_ENVIRONMENT}" bash scripts/local/setup_kinic_ledger.sh)" KINIC_LEDGER_CANISTER_ID="${LEDGER_SETUP_OUTPUT#KINIC_LEDGER_CANISTER_ID=}" export KINIC_LEDGER_CANISTER_ID -export SMOKE_CREDIT_PURCHASE_CREDITS +export SMOKE_CYCLE_PURCHASE_E8S if ! CANISTER_ID="$(resolve_canister_id)"; then echo "local wiki canister id not found; deploying wiki to ${ICP_ENVIRONMENT} environment" >&2 @@ -194,7 +149,7 @@ fi CANISTER_ID="$(resolve_canister_id)" export CANISTER_ID -approve_credits_allowance "${CANISTER_ID}" +approve_cycles_allowance "${CANISTER_ID}" echo "running local canister archive/restore smoke against ${CANISTER_ID} at ${REPLICA_HOST}" >&2 TMP_DIR="$(mktemp -d)" @@ -217,7 +172,7 @@ CLI_DB_NAME="${CLI_DB_NAME:-Archive smoke CLI}" CLI_DB="$(cd "$CLI_WORKSPACE" && "${VFS[@]}" database create "$CLI_DB_NAME")" ( cd "$CLI_WORKSPACE" - purchase_smoke_database_credits "$CLI_DB" "$SMOKE_CREDIT_PURCHASE_CREDITS" + "${VFS[@]}" database purchase-cycles "$CLI_DB" "$SMOKE_CYCLE_PURCHASE_KINIC" "${VFS[@]}" --database-id "$CLI_DB" write-node --path /Wiki/smoke.md --input "$INPUT_FILE" "${VFS[@]}" database archive-export "$CLI_DB" --output "$ARCHIVE_FILE" --chunk-size 65536 --json "${VFS[@]}" database archive-restore "$CLI_DB" --input "$ARCHIVE_FILE" --chunk-size 65536 --json diff --git a/scripts/smoke/local_canister_post_upgrade.sh b/scripts/smoke/local_canister_post_upgrade.sh index a302db86..c7ac346d 100755 --- a/scripts/smoke/local_canister_post_upgrade.sh +++ b/scripts/smoke/local_canister_post_upgrade.sh @@ -2,23 +2,43 @@ set -euo pipefail # Where: scripts/smoke/local_canister_post_upgrade.sh -# What: Smoke local install/upgrade with explicit CreditsConfig and pending DB persistence. +# What: Smoke local install/upgrade with explicit cycles config and pending DB persistence. # Why: Constructor args are operationally required, and post_upgrade must preserve initialized state. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -IDS_FILE="${REPO_ROOT}/.icp/cache/mappings/local-wiki.ids.json" -REPLICA_HOST="${REPLICA_HOST:-http://127.0.0.1:8001}" +ICP_ENVIRONMENT="${ICP_ENVIRONMENT:-local-wiki}" +IDS_FILE="${REPO_ROOT}/.icp/cache/mappings/${ICP_ENVIRONMENT}.ids.json" +REPLICA_HOST="${REPLICA_HOST:-http://127.0.0.1:8011}" +SMOKE_CYCLE_PURCHASE_E8S="${SMOKE_CYCLE_PURCHASE_E8S:-100000000}" +SMOKE_CYCLES_ALLOWANCE_E8S="${SMOKE_CYCLES_ALLOWANCE_E8S:-200000000}" + +case "${ICP_ENVIRONMENT}" in + local | local-wiki) ;; + *) + echo "ICP_ENVIRONMENT must be local or local-wiki for local smoke" >&2 + exit 1 + ;; +esac + +validate_unsigned_integer() { + local name="$1" + local value="${!name:-}" + if [[ ! "${value}" =~ ^[0-9]+$ ]]; then + echo "${name} must be an unsigned integer" >&2 + exit 1 + fi +} + +current_identity_name() { + icp identity list | awk '$1 == "*" { print $2; found = 1 } END { if (!found) exit 1 }' +} resolve_canister_id() { if [[ -n "${VFS_CANISTER_ID:-}" ]]; then printf '%s\n' "${VFS_CANISTER_ID}" return 0 fi - if [[ -n "${CANISTER_ID:-}" ]]; then - printf '%s\n' "${CANISTER_ID}" - return 0 - fi if [[ -f "${IDS_FILE}" ]]; then node -e ' const fs = require("fs"); @@ -31,27 +51,67 @@ resolve_canister_id() { ' "${IDS_FILE}" return 0 fi + if [[ -n "${CANISTER_ID:-}" ]]; then + printf '%s\n' "${CANISTER_ID}" + return 0 + fi return 1 } +approve_cycles_allowance() { + local canister_id="$1" + echo "approving ${SMOKE_CYCLES_ALLOWANCE_E8S} e8s for wiki canister ${canister_id}" >&2 + local approve_result + if ! approve_result="$(icp canister call "${KINIC_LEDGER_CANISTER_ID}" icrc2_approve \ + "(record { spender = record { owner = principal \"${canister_id}\"; subaccount = null }; amount = ${SMOKE_CYCLES_ALLOWANCE_E8S} : nat; expected_allowance = null; expires_at = null; fee = null; memo = null; from_subaccount = null; created_at_time = null })" \ + -e "${ICP_ENVIRONMENT}" -o candid)"; then + echo "KINIC approve failed. Ensure the current identity has enough local KINIC balance for the smoke cycle purchase plus ledger fees." >&2 + exit 1 + fi + if [[ "${approve_result}" == *"Err"* ]]; then + echo "KINIC approve returned an error: ${approve_result}" >&2 + echo "Ensure the current identity has enough local KINIC balance for the smoke cycle purchase plus ledger fees." >&2 + exit 1 + fi +} + cd "${REPO_ROOT}" +validate_unsigned_integer SMOKE_CYCLE_PURCHASE_E8S +validate_unsigned_integer SMOKE_CYCLES_ALLOWANCE_E8S TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT if [[ -z "${VFS_IDENTITY_PEM_PATH:-}" ]]; then VFS_IDENTITY_PEM_PATH="${TMP_DIR}/identity.pem" - icp identity export > "$VFS_IDENTITY_PEM_PATH" + IDENTITY_NAME="$(current_identity_name)" + if [[ -z "${IDENTITY_NAME}" ]]; then + echo "current icp identity name could not be resolved" >&2 + exit 1 + fi + (umask 077 && icp identity export "${IDENTITY_NAME}" > "$VFS_IDENTITY_PEM_PATH") export VFS_IDENTITY_PEM_PATH fi +if [[ -z "${BILLING_AUTHORITY_ID:-}" ]]; then + BILLING_AUTHORITY_ID="$(icp identity principal)" + export BILLING_AUTHORITY_ID +fi + +LEDGER_SETUP_OUTPUT="$(ICP_ENVIRONMENT="${ICP_ENVIRONMENT}" bash scripts/local/setup_kinic_ledger.sh)" +KINIC_LEDGER_CANISTER_ID="${LEDGER_SETUP_OUTPUT#KINIC_LEDGER_CANISTER_ID=}" +export ICP_ENVIRONMENT +export KINIC_LEDGER_CANISTER_ID +export SMOKE_CYCLE_PURCHASE_E8S + scripts/local/deploy_wiki.sh CANISTER_ID="$(resolve_canister_id)" export CANISTER_ID export REPLICA_HOST +approve_cycles_allowance "${CANISTER_ID}" STATE_FILE="${TMP_DIR}/local_canister_post_upgrade_state.json" cargo run -p kinic-vfs-cli --bin local_canister_post_upgrade_smoke -- --state-output "$STATE_FILE" -MODE=upgrade scripts/local/deploy_wiki.sh +scripts/local/deploy_wiki.sh --mode upgrade cargo run -p kinic-vfs-cli --bin local_canister_post_upgrade_smoke -- --verify-state "$STATE_FILE" diff --git a/skill-registry-web/app/skills/skill-registry-client.tsx b/skill-registry-web/app/skills/skill-registry-client.tsx index cb47ab27..1078d1c3 100644 --- a/skill-registry-web/app/skills/skill-registry-client.tsx +++ b/skill-registry-web/app/skills/skill-registry-client.tsx @@ -42,6 +42,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { const canisterId = process.env.NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID ?? ""; const refreshSeqRef = useRef(0); const [authClient, setAuthClient] = useState(null); + const [activeIdentity, setActiveIdentity] = useState(null); const [principal, setPrincipal] = useState(null); const [skills, setSkills] = useState([]); const [loadState, setLoadState] = useState("idle"); @@ -117,10 +118,12 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { setAuthClient(client); if (await client.isAuthenticated()) { const identity = client.getIdentity(); + setActiveIdentity(identity); setPrincipal(identity.getPrincipal().toText()); await loadRole(identity); await loadCatalog(identity); } else { + setActiveIdentity(null); listDatabaseMembersPublic(canisterId, databaseId).then(setMembers).catch(() => setMembers([])); await loadCatalog(); } @@ -142,6 +145,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { ...authLoginOptions(), onSuccess: () => { const identity = authClient.getIdentity(); + setActiveIdentity(identity); setPrincipal(identity.getPrincipal().toText()); void loadRole(identity); void loadCatalog(identity); @@ -157,6 +161,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { if (!authClient) return; refreshSeqRef.current += 1; await authClient.logout(); + setActiveIdentity(null); setPrincipal(null); setDatabaseRole(null); setSkills([]); @@ -170,9 +175,8 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { const selectedSkill = useMemo(() => skills.find((skill) => skill.manifest.id === selectedSkillId) ?? skills[0] ?? null, [selectedSkillId, skills]); const pendingJobs = jobs.filter((job) => job.status === "queued" || job.status === "running").length; const conflictJobs = jobs.filter((job) => job.status === "conflict").length; - const identity = authClient?.getIdentity(); const writable = databaseRole === "writer" || databaseRole === "owner"; - const packageManager = usePackageManager({ canisterId, databaseId, identity, writable, refresh: loadCatalog, errorMessage }); + const packageManager = usePackageManager({ canisterId, databaseId, identity: activeIdentity ?? undefined, writable, refresh: loadCatalog, errorMessage }); function actionFor(skill: CatalogSkill): ActionDraft { return actions[skill.manifest.id] ?? DEFAULT_ACTION; } @@ -182,16 +186,16 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { } async function runSkillAction(skill: CatalogSkill, operation: (identity: Identity, draft: ActionDraft) => Promise, clearRun = false) { - if (!identity) { + if (!activeIdentity) { patchAction(skill, { error: "Login is required." }); return; } const draft = actionFor(skill); patchAction(skill, { busy: true, error: null }); try { - await operation(identity, draft); + await operation(activeIdentity, draft); patchAction(skill, clearRun ? { busy: false, runTask: "", runNotes: "", message: "Operation completed." } : { busy: false, message: "Operation completed." }); - await loadCatalog(identity); + await loadCatalog(activeIdentity); } catch (cause) { patchAction(skill, { busy: false, error: errorMessage(cause) }); } @@ -242,7 +246,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { principal={principal} onLogin={() => void login()} onLogout={() => void logout()} - onRefresh={() => void loadCatalog(authClient?.getIdentity())} + onRefresh={() => void loadCatalog(activeIdentity ?? undefined)} />
diff --git a/skill-registry-web/app/skills/skill-registry-ui.tsx b/skill-registry-web/app/skills/skill-registry-ui.tsx index 71b3db0b..9f64bf12 100644 --- a/skill-registry-web/app/skills/skill-registry-ui.tsx +++ b/skill-registry-web/app/skills/skill-registry-ui.tsx @@ -193,8 +193,8 @@ export function ProposalList({ 
{proposal.metricsPreview}
- - + +
{preview?.proposalPath === proposal.proposalRoot ?

Preview: {preview.targetPath} +{preview.additions} -{preview.removals}

: null}
@@ -203,6 +203,10 @@ export function ProposalList({ ); } +function proposalCanApply(proposal: SkillProposal): boolean { + return proposal.status === "proposed" || proposal.status === "reviewed"; +} + function EventList({ skill }: { skill: CatalogSkill }) { if (skill.events.length === 0) return

No skill events.

; return ( diff --git a/skill-registry-web/lib/markdown-frontmatter.ts b/skill-registry-web/lib/markdown-frontmatter.ts index 5f8332d8..23226baf 100644 --- a/skill-registry-web/lib/markdown-frontmatter.ts +++ b/skill-registry-web/lib/markdown-frontmatter.ts @@ -11,10 +11,11 @@ export type MarkdownFrontmatter = { export function splitMarkdownFrontmatter(content: string): MarkdownFrontmatter | null { if (!content.startsWith("---\n")) return null; const rest = content.slice(4); - const end = rest.indexOf("\n---"); + const match = rest.match(/\n---(?:\n|$)/); + const end = match?.index ?? -1; if (end < 0) return null; const frontmatter = rest.slice(0, end); - const bodyStart = end + "\n---".length; + const bodyStart = end + match![0].length; const body = rest.slice(bodyStart).replace(/^\n+/, ""); return { fields: flattenFrontmatter(frontmatter), @@ -50,11 +51,14 @@ function flattenFrontmatter(frontmatter: string): FrontmatterField[] { function cleanValue(value: string): string { const trimmed = value.trim(); - if ( - (trimmed.startsWith("\"") && trimmed.endsWith("\"")) || - (trimmed.startsWith("'") && trimmed.endsWith("'")) - ) { - return trimmed.slice(1, -1); + if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) { + try { + const parsed: unknown = JSON.parse(trimmed); + return typeof parsed === "string" ? parsed : ""; + } catch { + return ""; + } } + if (trimmed.startsWith("'") && trimmed.endsWith("'")) return trimmed.slice(1, -1).replace(/''/g, "'"); return trimmed; } diff --git a/skill-registry-web/lib/paths.ts b/skill-registry-web/lib/paths.ts index 204711fd..51b8fdde 100644 --- a/skill-registry-web/lib/paths.ts +++ b/skill-registry-web/lib/paths.ts @@ -104,13 +104,14 @@ export function hrefForMarkdownLink(canisterId: string, databaseId: string, curr return null; } const target = splitMarkdownHref(trimmed); - if (trimmed.startsWith("/Wiki") || trimmed.startsWith("/Sources")) { - return appendMarkdownSuffix(hrefForPath(canisterId, databaseId, target.path, undefined, undefined, undefined, undefined, readMode), target, readMode); + const targetPath = decodeMarkdownHrefPath(target.path); + if (isInternalWikiPath(targetPath)) { + return appendMarkdownSuffix(hrefForPath(canisterId, databaseId, targetPath, undefined, undefined, undefined, undefined, readMode), target, readMode); } - if (trimmed.startsWith("/")) { + if (targetPath.startsWith("/")) { return null; } - return appendMarkdownSuffix(hrefForPath(canisterId, databaseId, resolveRelativeWikiPath(currentPath, target.path), undefined, undefined, undefined, undefined, readMode), target, readMode); + return appendMarkdownSuffix(hrefForPath(canisterId, databaseId, resolveRelativeWikiPath(currentPath, targetPath), undefined, undefined, undefined, undefined, readMode), target, readMode); } export function parentPath(path: string): string | null { @@ -145,6 +146,10 @@ function isExternalHref(href: string): boolean { return /^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith("//"); } +function isInternalWikiPath(path: string): boolean { + return path === "/Wiki" || path.startsWith("/Wiki/") || path === "/Sources" || path.startsWith("/Sources/"); +} + function appendMarkdownSuffix(baseHref: string, target: MarkdownHrefTarget, readMode?: string | null): string { const params = new URLSearchParams(target.query); if (readMode === "anonymous") { @@ -154,6 +159,14 @@ function appendMarkdownSuffix(baseHref: string, target: MarkdownHrefTarget, read return `${baseHref.split("?")[0]}${queryString}${target.hash}`; } +function decodeMarkdownHrefPath(path: string): string { + try { + return decodeURIComponent(path); + } catch { + return path; + } +} + function splitMarkdownHref(href: string): MarkdownHrefTarget { const hashIndex = href.indexOf("#"); const pathAndQuery = hashIndex === -1 ? href : href.slice(0, hashIndex); diff --git a/skill-registry-web/lib/skill-manifest.ts b/skill-registry-web/lib/skill-manifest.ts index 2f8a1f85..ba764b26 100644 --- a/skill-registry-web/lib/skill-manifest.ts +++ b/skill-registry-web/lib/skill-manifest.ts @@ -34,6 +34,8 @@ export function parseSkillManifest(content: string): SkillManifest | null { const version = scalar(values, "version"); const entry = scalar(values, "entry"); if (!id || !version || !entry) return null; + if (!isSafeSkillId(id)) return null; + if (entry !== "SKILL.md") return null; return { kind: "kinic.skill", schemaVersion: "1", @@ -53,6 +55,10 @@ export function parseSkillManifest(content: string): SkillManifest | null { }; } +function isSafeSkillId(value: string): boolean { + return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value) && !value.includes(".."); +} + export function isSkillRegistryPath(path: string): boolean { return path === "/Wiki/skills" || path.startsWith("/Wiki/skills/"); } @@ -99,8 +105,8 @@ export function skillAccessHint(mode: string | null, roles: string[], authentica function extractFrontmatter(content: string): string | null { if (!content.startsWith("---\n")) return null; const rest = content.slice(4); - const end = rest.indexOf("\n---"); - return end >= 0 ? rest.slice(0, end) : null; + const match = rest.match(/\n---(?:\n|$)/); + return match?.index === undefined ? null : rest.slice(0, match.index); } function formatBoolean(value: boolean): string { @@ -150,5 +156,14 @@ function nested(values: Record, parent: string): Record { - const id = proposalRoot.split("/").pop() ?? proposalRoot; + const parsedRoot = parseProposalRoot(proposalRoot); + if (!parsedRoot) return null; + const { skillId, proposalId: id } = parsedRoot; const candidatePath = `${proposalRoot}/candidate/SKILL.md`; const metricsPath = `${proposalRoot}/metrics.json`; + const statusPath = `${proposalRoot}/status.md`; const [candidate, metrics, status] = await Promise.all([ readRegistryNode(canisterId, databaseId, candidatePath, identity), readRegistryNode(canisterId, databaseId, metricsPath, identity), - readRegistryNode(canisterId, databaseId, `${proposalRoot}/status.md`, identity) + readRegistryNode(canisterId, databaseId, statusPath, identity) ]); - if (!candidate || !metrics) return null; + if (!candidate || !metrics || !status) return null; const metricsJson = parseJsonObject(metrics.content); - const statusFields = status ? frontmatterFields(status.content) : {}; + const baseEtag = stringField(metricsJson, "base_etag"); + const statusFields = frontmatterFields(status.content); + const proposalStatus = parseProposalStatus(statusFields.status); + if (!baseEtag) return null; + if (statusFields.kind !== "kinic.skill_evolution_proposal_status" || statusFields.schema_version !== "1") return null; + if (statusFields.skill_id !== skillId || statusFields.proposal_id !== id) return null; + if (!proposalStatus || !statusFields.recorded_at || Number.isNaN(Date.parse(statusFields.recorded_at))) return null; return { proposalRoot, candidatePath, metricsPath, - statusPath: status?.path ?? null, + statusPath, id, title: id, - status: statusFields.status ?? "proposed", - createdAt: stringField(metricsJson, "created_at") ?? String(metricsJson.created_at_ms ?? ""), + status: proposalStatus, + createdAt: stringField(metricsJson, "created_at") ?? statusFields.recorded_at, sourceRuns: arrayStringField(metricsJson, "source_runs"), candidatePreview: candidate.content.slice(0, 1200), - baseEtag: stringField(metricsJson, "base_etag"), - appliedAt: statusFields.recorded_at ?? null, + baseEtag, + appliedAt: statusFields.recorded_at, metricsPreview: metrics.content.slice(0, 2000) }; } +function parseProposalStatus(value: string | undefined): ProposalStatus | null { + if (value === "proposed" || value === "reviewed" || value === "auto_applied" || value === "gate_failed" || value === "conflict") return value; + return null; +} + async function loadVersions(canisterId: string, databaseId: string, basePath: string, identity?: Identity): Promise { const entries = await listRegistryChildren(canisterId, databaseId, `${basePath}/versions`, identity); return entries @@ -157,6 +171,18 @@ function isFileEntry(entry: ChildNode): boolean { return entry.kind !== "directory" && entry.kind !== "folder"; } +function parseProposalRoot(proposalRoot: string): { skillId: string; proposalId: string } | null { + const match = proposalRoot.match(/^\/Wiki\/skills\/([^/]+)\/proposals\/([^/]+)$/); + if (!match) return null; + const [, skillId, proposalId] = match; + if (!isSafePathSegment(skillId) || !isSafePathSegment(proposalId)) return null; + return { skillId, proposalId }; +} + +function isSafePathSegment(value: string): boolean { + return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value) && !value.includes(".."); +} + function summarizeRuns(runs: SkillRunEvidence[]): SkillRunSummary { const summary: SkillRunSummary = { runs: 0, success: 0, partial: 0, fail: 0, lastUsedAt: null, lastOutcome: null }; for (const run of runs) { diff --git a/skill-registry-web/lib/skill-registry-diff.ts b/skill-registry-web/lib/skill-registry-diff.ts index dc118118..c0a0ebc9 100644 --- a/skill-registry-web/lib/skill-registry-diff.ts +++ b/skill-registry-web/lib/skill-registry-diff.ts @@ -1,5 +1,6 @@ import type { Identity } from "@icp-sdk/core/agent"; import type { CatalogSkill, SkillProposal } from "@/lib/skill-registry-catalog"; +import { assertProposalStatus } from "@/lib/skill-registry-operations"; import type { WikiNode } from "@/lib/types"; import { readNode, writeNodeAuthenticated } from "@/lib/vfs-client"; import { ensureParentFoldersAuthenticated } from "@/lib/vfs-folders"; @@ -7,6 +8,7 @@ import { ensureParentFoldersAuthenticated } from "@/lib/vfs-folders"; export type ProposalDiffPreview = { proposalPath: string; targetPath: string; + baseEtag: string; nextContent: string; currentEtag: string; metadataJson: string; @@ -21,13 +23,16 @@ export async function previewApplyProposalDiff(canisterId: string, databaseId: s requireNode(canisterId, databaseId, proposal.metricsPath, identity) ]); assertProposalGates(metrics.content); - if (proposal.baseEtag && proposal.baseEtag !== current.etag) { + assertSafeProposalId(proposal.id); + if (!proposal.baseEtag) throw new Error("Proposal metrics base_etag is required."); + if (proposal.baseEtag !== current.etag) { throw new Error("Current SKILL.md etag no longer matches proposal base_etag."); } const counts = lineDelta(current.content, candidate.content); return { proposalPath: proposal.proposalRoot, targetPath: `${skill.basePath}/SKILL.md`, + baseEtag: proposal.baseEtag, nextContent: candidate.content, currentEtag: current.etag, metadataJson: current.metadataJson, @@ -37,14 +42,25 @@ export async function previewApplyProposalDiff(canisterId: string, databaseId: s } export async function applyProposalDiff(canisterId: string, databaseId: string, identity: Identity, proposal: SkillProposal, preview: ProposalDiffPreview): Promise { + assertSafeProposalId(proposal.id); + if (proposal.proposalRoot !== preview.proposalPath) throw new Error("Proposal changed since preview."); + if (!proposal.baseEtag) throw new Error("Proposal metrics base_etag is required."); + if (proposal.baseEtag !== preview.baseEtag) throw new Error("Proposal base_etag changed since preview."); + if (!preview.targetPath.endsWith("/SKILL.md")) throw new Error("Proposal target must be SKILL.md."); const basePath = preview.targetPath.replace(/\/SKILL\.md$/, ""); const current = await requireNode(canisterId, databaseId, preview.targetPath, identity); if (current.etag !== preview.currentEtag) throw new Error("Current SKILL.md changed since preview."); - const [manifest, metrics] = await Promise.all([ + if (current.etag !== proposal.baseEtag) throw new Error("Current SKILL.md etag no longer matches proposal base_etag."); + const skillId = preview.targetPath.split("/").at(-2); + if (!skillId) throw new Error("Skill id is missing."); + const statusPath = `${proposal.proposalRoot}/status.md`; + const [manifest, metrics, status] = await Promise.all([ readNode(canisterId, databaseId, `${basePath}/manifest.md`, identity), - requireNode(canisterId, databaseId, proposal.metricsPath, identity) + requireNode(canisterId, databaseId, proposal.metricsPath, identity), + requireNode(canisterId, databaseId, statusPath, identity) ]); assertProposalGates(metrics.content); + assertProposalStatus(status.content, skillId, proposal.id, ["proposed", "reviewed"]); const versionId = `${Date.now()}-${(await sha256Hex(current.content)).slice(0, 12)}`; const versionBase = `${basePath}/versions/${versionId}`; const versionSkillPath = `${versionBase}/SKILL.md`; @@ -75,15 +91,14 @@ export async function applyProposalDiff(canisterId: string, databaseId: string, metadataJson: preview.metadataJson, expectedEtag: preview.currentEtag }); - const statusPath = `${proposal.proposalRoot}/status.md`; await ensureParentFoldersAuthenticated(canisterId, databaseId, identity, statusPath); await writeNodeAuthenticated(canisterId, identity, { databaseId, path: statusPath, kind: "file", - content: ["---", "kind: kinic.skill_evolution_proposal_status", "schema_version: 1", `proposal_id: ${JSON.stringify(proposal.id)}`, "status: auto_applied", `recorded_at: ${new Date().toISOString()}`, "---", "# Proposal Status"].join("\n"), - metadataJson: "{}", - expectedEtag: null + content: ["---", "kind: kinic.skill_evolution_proposal_status", "schema_version: 1", `skill_id: ${JSON.stringify(skillId)}`, `proposal_id: ${JSON.stringify(proposal.id)}`, "status: auto_applied", `recorded_at: ${new Date().toISOString()}`, "---", "# Proposal Status"].join("\n"), + metadataJson: status.metadataJson, + expectedEtag: status.etag }); } @@ -134,6 +149,12 @@ function parseJsonObject(content: string): Record { } } +function assertSafeProposalId(value: string): void { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value) || value.includes("..")) { + throw new Error("Proposal id must be a single safe path segment."); + } +} + async function sha256Hex(content: string): Promise { const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(content)); return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); diff --git a/skill-registry-web/lib/skill-registry-operations.ts b/skill-registry-web/lib/skill-registry-operations.ts index 1556846b..45014345 100644 --- a/skill-registry-web/lib/skill-registry-operations.ts +++ b/skill-registry-web/lib/skill-registry-operations.ts @@ -16,12 +16,17 @@ export async function updateSkillStatus( reason: string ): Promise { const node = await requireNode(canisterId, databaseId, skill.manifestPath, identity); - const updates: Record = { status }; + const updates: Record = { status }; if (status === "promoted") updates.promoted_at = new Date().toISOString(); if (status === "deprecated") { updates.deprecated_at = new Date().toISOString(); if (reason.trim()) updates.deprecated_reason = reason.trim(); } + if (status !== "promoted") updates.promoted_at = null; + if (status !== "deprecated") { + updates.deprecated_at = null; + updates.deprecated_reason = null; + } await writeNodeAuthenticated(canisterId, identity, { databaseId, path: skill.manifestPath, @@ -85,16 +90,16 @@ export async function recordSkillRun( } export async function approveSkillProposal(canisterId: string, databaseId: string, identity: Identity, skill: CatalogSkill, proposalPath: string): Promise { - if (!proposalPath.startsWith(`${skill.basePath}/proposals/`)) throw new Error("Proposal path is outside this skill package."); - const statusPath = `${proposalPath}/status.md`; - await ensureParentFoldersAuthenticated(canisterId, databaseId, identity, statusPath); + const { proposalId, statusPath } = proposalStatusPathForSkill(skill, proposalPath); + const current = await requireNode(canisterId, databaseId, statusPath, identity); + assertProposalStatus(current.content, skill.manifest.id, proposalId, ["proposed"]); await writeNodeAuthenticated(canisterId, identity, { databaseId, path: statusPath, kind: "file", - content: ["---", "kind: kinic.skill_evolution_proposal_status", "schema_version: 1", `skill_id: ${JSON.stringify(skill.manifest.id)}`, "status: reviewed", `recorded_at: ${new Date().toISOString()}`, "---", "# Proposal Status"].join("\n"), - metadataJson: "{}", - expectedEtag: null + content: ["---", "kind: kinic.skill_evolution_proposal_status", "schema_version: 1", `skill_id: ${JSON.stringify(skill.manifest.id)}`, `proposal_id: ${JSON.stringify(proposalId)}`, "status: reviewed", `recorded_at: ${new Date().toISOString()}`, "---", "# Proposal Status"].join("\n"), + metadataJson: current.metadataJson, + expectedEtag: current.etag }); await recordSkillEvent(canisterId, databaseId, identity, skill.manifest.id, { action: "proposal.review", @@ -143,10 +148,10 @@ async function requireNode(canisterId: string, databaseId: string, path: string, return node; } -function replaceRootFrontmatter(content: string, updates: Record): string { +function replaceRootFrontmatter(content: string, updates: Record): string { if (!content.startsWith("---\n")) throw new Error("Markdown frontmatter is missing."); const rest = content.slice(4); - const end = rest.indexOf("\n---"); + const end = frontmatterEnd(rest); if (end < 0) throw new Error("Markdown frontmatter terminator is missing."); const lines = rest.slice(0, end).split("\n"); const pending = new Set(Object.keys(updates)); @@ -156,18 +161,73 @@ function replaceRootFrontmatter(content: string, updates: Record const key = match[1].trim(); if (!(key in updates)) return line; pending.delete(key); - return `${key}: ${quoteYaml(updates[key])}`; - }); + const update = updates[key]; + if (update === null) return null; + return `${key}: ${quoteYaml(update)}`; + }).filter((line): line is string => line !== null); for (const key of pending) { - replaced.push(`${key}: ${quoteYaml(updates[key])}`); + const update = updates[key]; + if (update === null) continue; + replaced.push(`${key}: ${quoteYaml(update)}`); } return `---\n${replaced.join("\n")}${rest.slice(end)}`; } +function frontmatterEnd(rest: string): number { + const match = rest.match(/\n---(?:\n|$)/); + return match?.index ?? -1; +} + function quoteYaml(value: string): string { return JSON.stringify(value); } +function proposalStatusPathForSkill(skill: CatalogSkill, proposalPath: string): { proposalId: string; statusPath: string } { + const prefix = `${skill.basePath}/proposals/`; + if (!proposalPath.startsWith(prefix)) throw new Error("Proposal path is outside this skill package."); + const proposalId = proposalPath.slice(prefix.length); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(proposalId) || proposalId.includes("..")) { + throw new Error("Proposal id must be a single safe path segment."); + } + return { proposalId, statusPath: `${proposalPath}/status.md` }; +} + +export function assertProposalStatus(content: string, skillId: string, proposalId: string, allowedStatuses: readonly string[]): void { + const fields = frontmatterFields(content); + if (fields.kind !== "kinic.skill_evolution_proposal_status") throw new Error("Proposal status kind is invalid."); + if (fields.schema_version !== "1") throw new Error("Proposal status schema_version is invalid."); + if (fields.skill_id !== skillId) throw new Error("Proposal status skill_id does not match."); + if (fields.proposal_id !== proposalId) throw new Error("Proposal status proposal_id does not match."); + if (!fields.recorded_at || Number.isNaN(Date.parse(fields.recorded_at))) throw new Error("Proposal status recorded_at is invalid."); + if (!fields.status || !allowedStatuses.includes(fields.status)) throw new Error("Proposal status is not in an updateable state."); +} + +function frontmatterFields(content: string): Record { + if (!content.startsWith("---\n")) return {}; + const rest = content.slice(4); + const end = frontmatterEnd(rest); + if (end < 0) return {}; + const fields: Record = {}; + for (const line of rest.slice(0, end).split("\n")) { + const match = line.match(/^([^:\s][^:]*):(.*)$/); + if (!match) continue; + fields[match[1].trim()] = cleanYamlScalar(match[2].trim()); + } + return fields; +} + +function cleanYamlScalar(value: string): string { + if (value.startsWith('"') && value.endsWith('"')) { + try { + const parsed: unknown = JSON.parse(value); + return typeof parsed === "string" ? parsed : value; + } catch { + return value; + } + } + return value; +} + async function sha256Hex(value: string): Promise { const bytes = new TextEncoder().encode(value); const digest = await crypto.subtle.digest("SHA-256", bytes); diff --git a/skill-registry-web/lib/skill-registry-package.ts b/skill-registry-web/lib/skill-registry-package.ts index 14286555..af0fff10 100644 --- a/skill-registry-web/lib/skill-registry-package.ts +++ b/skill-registry-web/lib/skill-registry-package.ts @@ -72,7 +72,7 @@ function normalizeFiles(files: SkillPackageFile[], skillId: string): SkillPackag } const skill = cleaned.get("SKILL.md"); if (!skill) throw new Error("SKILL.md is required."); - cleaned.set("manifest.md", normalizeManifestForSkill(skillId, cleaned.get("manifest.md") ?? manifestForSkill(skillId, skill))); + cleaned.set("manifest.md", normalizeManifestForSkill(skillId, cleaned.get("manifest.md") ?? manifestForSkill(skillId, skill), skill)); return [...cleaned.entries()].map(([name, content]) => ({ name, content })).sort((left, right) => left.name.localeCompare(right.name)); } @@ -87,6 +87,8 @@ function normalizeGitHubManifest(files: SkillPackageFile[], skillId: string, sou function manifestForSkill(skillId: string, skill: string): string { const title = frontmatterValue(skill, "metadata.title") ?? skillId; const summary = frontmatterValue(skill, "description") ?? ""; + const category = frontmatterValue(skill, "metadata.category"); + const license = frontmatterValue(skill, "license"); return [ "---", "kind: kinic.skill", @@ -96,18 +98,24 @@ function manifestForSkill(skillId: string, skill: string): string { "entry: SKILL.md", `title: ${JSON.stringify(title)}`, `summary: ${JSON.stringify(summary)}`, + ...(category ? ["tags:", ` - ${JSON.stringify(category)}`] : []), "status: draft", + ...(license ? ["provenance:", ` license: ${JSON.stringify(license)}`] : []), "---", `# ${title}` ].join("\n"); } -function normalizeManifestForSkill(skillId: string, content: string): string { +function normalizeManifestForSkill(skillId: string, content: string, skill: string): string { let next = content.startsWith("---\n") ? content : manifestForSkill(skillId, ""); next = setRootFrontmatterField(next, "kind", "kinic.skill"); next = setRootFrontmatterField(next, "schema_version", "1"); next = setRootFrontmatterField(next, "id", skillId); next = setRootFrontmatterField(next, "entry", "SKILL.md"); + next = fillRootFrontmatterField(next, "title", frontmatterValue(skill, "metadata.title")); + next = fillRootFrontmatterField(next, "summary", frontmatterValue(skill, "description")); + next = fillListFrontmatterField(next, "tags", frontmatterValue(skill, "metadata.category")); + next = fillNestedFrontmatterField(next, "provenance", "license", frontmatterValue(skill, "license")); return next; } @@ -118,13 +126,13 @@ function setManifestProvenance(content: string, source: GitHubSource, sha: strin ` revision: ${JSON.stringify(sha)}` ]; if (content.includes("\nprovenance:\n")) return content.replace(/\nprovenance:\n(?: .+\n?)*/m, `\nprovenance:\n${fields.join("\n")}\n`); - return content.replace(/\n---/, `\nprovenance:\n${fields.join("\n")}\n---`); + return insertBeforeFrontmatterTerminator(content, ["provenance:", ...fields]); } function setRootFrontmatterField(content: string, key: string, value: string): string { if (!content.startsWith("---\n")) throw new Error("manifest.md frontmatter is required."); const rest = content.slice(4); - const end = rest.indexOf("\n---"); + const end = frontmatterEnd(rest); if (end < 0) throw new Error("manifest.md frontmatter terminator is missing."); const lines = rest.slice(0, end).split("\n"); let replaced = false; @@ -138,6 +146,21 @@ function setRootFrontmatterField(content: string, key: string, value: string): s return `---\n${next.join("\n")}${rest.slice(end)}`; } +function fillRootFrontmatterField(content: string, key: string, value: string | null): string { + if (!value || frontmatterValue(content, key)) return content; + return setRootFrontmatterField(content, key, value); +} + +function fillListFrontmatterField(content: string, key: string, value: string | null): string { + if (!value || frontmatterHasListItems(content, key)) return content; + return setListFrontmatterField(content, key, [value]); +} + +function fillNestedFrontmatterField(content: string, parent: string, child: string, value: string | null): string { + if (!value || frontmatterValue(content, `${parent}.${child}`)) return content; + return setNestedFrontmatterField(content, parent, child, value); +} + type GitHubSource = { owner: string; repo: string; path: string | null }; function parseGitHubSource(value: string): GitHubSource { @@ -170,8 +193,8 @@ async function fetchOptionalText(url: string): Promise { function markdownPackageLinks(content: string): string[] { const names = new Set(); - for (const match of content.matchAll(/\]\(([^)]+)\)/g)) { - const name = cleanPackageFileName(match[1].split(/[?#\s]/)[0] ?? ""); + for (const target of markdownLinkTargets(content)) { + const name = cleanPackageFileName(cleanMarkdownDestination(target)); if (name) names.add(name); } return [...names]; @@ -179,29 +202,108 @@ function markdownPackageLinks(content: string): string[] { function cleanSkillId(value: string): string { const id = value.trim(); - if (!/^[a-z0-9][a-z0-9_-]*$/.test(id)) throw new Error("Skill id must use lowercase letters, numbers, _ or -."); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(id) || id.includes("..")) { + throw new Error("Skill id must be a single path-safe segment."); + } return id; } function cleanPackageFileName(value: string): string | null { const name = value.trim().replace(/^\.\//, ""); if (!name.endsWith(".md") || name.startsWith("/") || name.includes("..") || name.includes("://")) return null; + const segments = name.split("/"); + if (segments.some((segment) => !segment || segment === ".")) return null; return name; } function frontmatterValue(content: string, key: string): string | null { - const start = content.startsWith("---\n") ? content.indexOf("\n---", 4) : -1; + return frontmatterField(content, key)?.value ?? null; +} + +function frontmatterField(content: string, key: string): { value: string } | null { + if (!content.startsWith("---\n")) return null; + const rest = content.slice(4); + const start = frontmatterEnd(rest); if (start < 0) return null; const parent = key.split(".")[0]; + const child = key.split(".")[1]; let inParent = false; - for (const line of content.slice(4, start).split("\n")) { - if (line.startsWith(`${parent}:`)) inParent = true; - const match = key.includes(".") && inParent ? line.trim().match(new RegExp(`^${key.split(".")[1]}:\\s*(.*)$`)) : line.match(new RegExp(`^${key}:\\s*(.*)$`)); - if (match) return cleanYaml(match[1]); + for (const line of rest.slice(0, start).split("\n")) { + if (!line.startsWith(" ") && !line.startsWith("\t")) { + inParent = line.startsWith(`${parent}:`); + } + const match = child && inParent ? line.trim().match(new RegExp(`^${child}:\\s*(.*)$`)) : line.match(new RegExp(`^${key}:\\s*(.*)$`)); + if (match) return { value: cleanYaml(match[1]) }; } return null; } +function frontmatterHasListItems(content: string, key: string): boolean { + if (!content.startsWith("---\n")) return false; + const rest = content.slice(4); + const end = frontmatterEnd(rest); + if (end < 0) return false; + let inList = false; + for (const line of rest.slice(0, end).split("\n")) { + if (!line.startsWith(" ") && !line.startsWith("\t")) { + const match = line.match(/^([^:\s][^:]*):/); + inList = Boolean(match && match[1].trim() === key); + continue; + } + if (inList && line.trim().startsWith("- ")) return true; + } + return false; +} + +function setListFrontmatterField(content: string, key: string, values: string[]): string { + if (!content.startsWith("---\n")) throw new Error("manifest.md frontmatter is required."); + const rest = content.slice(4); + const end = frontmatterEnd(rest); + if (end < 0) throw new Error("manifest.md frontmatter terminator is missing."); + const lines = rest.slice(0, end).split("\n"); + const replacement = [key + ":", ...values.map((value) => ` - ${JSON.stringify(value)}`)]; + const index = lines.findIndex((line) => line.match(/^([^:\s][^:]*):/)?.[1].trim() === key); + if (index < 0) return `---\n${[...lines, ...replacement].join("\n")}${rest.slice(end)}`; + let after = index + 1; + while (after < lines.length && (lines[after].startsWith(" ") || lines[after].startsWith("\t"))) after += 1; + return `---\n${[...lines.slice(0, index), ...replacement, ...lines.slice(after)].join("\n")}${rest.slice(end)}`; +} + +function setNestedFrontmatterField(content: string, parent: string, child: string, value: string): string { + if (!content.startsWith("---\n")) throw new Error("manifest.md frontmatter is required."); + const rest = content.slice(4); + const end = frontmatterEnd(rest); + if (end < 0) throw new Error("manifest.md frontmatter terminator is missing."); + const lines = rest.slice(0, end).split("\n"); + let parentIndex = -1; + let childIndex = -1; + let inParent = false; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (!line.startsWith(" ") && !line.startsWith("\t")) { + const match = line.match(/^([^:\s][^:]*):/); + inParent = Boolean(match && match[1].trim() === parent); + if (inParent) parentIndex = index; + continue; + } + if (inParent && line.trim().match(new RegExp(`^${child}:`))) { + childIndex = index; + break; + } + } + const replacement = ` ${child}: ${JSON.stringify(value)}`; + if (childIndex >= 0) { + const next = [...lines]; + next[childIndex] = replacement; + return `---\n${next.join("\n")}${rest.slice(end)}`; + } + if (parentIndex >= 0) { + const next = [...lines.slice(0, parentIndex + 1), replacement, ...lines.slice(parentIndex + 1)]; + return `---\n${next.join("\n")}${rest.slice(end)}`; + } + return `---\n${[...lines, parent + ":", replacement].join("\n")}${rest.slice(end)}`; +} + function cleanGitHubPath(value: string): string | null { const path = value.trim().replace(/^\/+|\/+$/g, ""); if (!path) return null; @@ -211,7 +313,111 @@ function cleanGitHubPath(value: string): string | null { function cleanYaml(value: string): string { const trimmed = value.trim(); - return trimmed.startsWith("\"") && trimmed.endsWith("\"") ? trimmed.slice(1, -1) : trimmed; + if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) { + try { + const parsed: unknown = JSON.parse(trimmed); + if (typeof parsed === "string") return parsed; + throw new Error("Invalid quoted YAML scalar."); + } catch { + throw new Error("Invalid quoted YAML scalar."); + } + } + if (trimmed.startsWith("'") && trimmed.endsWith("'")) return trimmed.slice(1, -1).replace(/''/g, "'"); + return trimmed; +} + +function frontmatterEnd(rest: string): number { + const match = rest.match(/\n---(?:\n|$)/); + return match?.index ?? -1; +} + +function insertBeforeFrontmatterTerminator(content: string, lines: string[]): string { + if (!content.startsWith("---\n")) throw new Error("manifest.md frontmatter is required."); + const rest = content.slice(4); + const end = frontmatterEnd(rest); + if (end < 0) throw new Error("manifest.md frontmatter terminator is missing."); + const absoluteEnd = 4 + end; + return `${content.slice(0, absoluteEnd)}\n${lines.join("\n")}${content.slice(absoluteEnd)}`; +} + +function markdownLinkTargets(content: string): string[] { + const targets: string[] = []; + let index = 0; + while (index < content.length) { + const open = content.indexOf("](", index); + if (open < 0) break; + let cursor = open + 2; + if (content[cursor] === "<") { + const close = content.indexOf(">", cursor + 1); + if (close >= 0 && content[close + 1] === ")") { + targets.push(content.slice(cursor, close + 1)); + index = close + 2; + continue; + } + } + let depth = 0; + while (cursor < content.length) { + const char = content[cursor]; + if (char === "(") depth += 1; + if (char === ")") { + if (depth === 0) break; + depth -= 1; + } + cursor += 1; + } + if (cursor < content.length) targets.push(content.slice(open + 2, cursor)); + index = cursor + 1; + } + return targets; +} + +function cleanMarkdownDestination(value: string): string { + const trimmed = value.trim(); + const withoutTitle = markdownDestinationWithoutTitle(trimmed); + const destination = withoutTitle.startsWith("<") && withoutTitle.endsWith(">") ? withoutTitle.slice(1, -1) : withoutTitle; + return destination.split(/[?#]/)[0]?.trim() ?? ""; +} + +function markdownDestinationWithoutTitle(value: string): string { + const trimmed = value.trim(); + if (trimmed.startsWith("<")) { + const close = trimmed.indexOf(">"); + if (close > 0) { + const destination = trimmed.slice(1, close); + const suffix = trimmed.slice(close + 1).trim(); + if (!suffix || isMarkdownTitleSuffix(suffix)) return destination; + } + } + return stripQuotedMarkdownTitle(trimmed, "\"") ?? stripQuotedMarkdownTitle(trimmed, "'") ?? stripParenthesizedMarkdownTitle(trimmed) ?? trimmed; +} + +function stripQuotedMarkdownTitle(value: string, quote: string): string | null { + if (!value.endsWith(quote)) return null; + for (let index = value.length - 2; index > 0; index -= 1) { + if (value[index] === quote && /\s/.test(value[index - 1] ?? "")) { + const destination = value.slice(0, index - 1).trimEnd(); + if (isMarkdownDestinationCandidate(destination)) return destination; + } + } + return null; +} + +function stripParenthesizedMarkdownTitle(value: string): string | null { + if (!value.endsWith(")")) return null; + const titleStart = value.lastIndexOf(" ("); + if (titleStart < 0) return null; + const destination = value.slice(0, titleStart).trimEnd(); + return isMarkdownDestinationCandidate(destination) ? destination : null; +} + +function isMarkdownTitleSuffix(value: string): boolean { + return (value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")) || (value.startsWith("(") && value.endsWith(")")); +} + +function isMarkdownDestinationCandidate(value: string): boolean { + const unwrapped = value.startsWith("<") && value.endsWith(">") ? value.slice(1, -1) : value; + const destination = unwrapped.split(/[?#]/)[0]?.trim() ?? ""; + return Boolean(destination && !destination.startsWith("#") && !destination.startsWith("/") && !destination.includes("://") && destination.endsWith(".md")); } function isCommitPayload(value: unknown): value is { sha: string } { diff --git a/skill-registry-web/lib/types.ts b/skill-registry-web/lib/types.ts index 62c4a7d6..d675d2e2 100644 --- a/skill-registry-web/lib/types.ts +++ b/skill-registry-web/lib/types.ts @@ -105,8 +105,8 @@ export type DatabaseSummary = { role: DatabaseRole; status: DatabaseStatus; logicalSizeBytes: string; - creditsBalance: string | null; - creditsSuspendedAtMs: string | null; + cyclesBalance: string | null; + cyclesSuspendedAtMs: string | null; archivedAtMs: string | null; }; diff --git a/skill-registry-web/lib/vfs-client.ts b/skill-registry-web/lib/vfs-client.ts index e9d18330..f2051577 100644 --- a/skill-registry-web/lib/vfs-client.ts +++ b/skill-registry-web/lib/vfs-client.ts @@ -15,7 +15,7 @@ type Variant = Record; type RawNode = { path: string; kind: Variant; content: string; created_at: bigint; updated_at: bigint; etag: string; metadata_json: string }; type RawNodeMutationAck = { path: string; kind: Variant; updated_at: bigint; etag: string }; type RawChild = { path: string; name: string; kind: Variant; updated_at: [] | [bigint]; etag: [] | [string]; size_bytes: [] | [bigint]; is_virtual: boolean; has_children: boolean }; -type RawDatabaseSummary = { status: Variant; role: Variant; logical_size_bytes: bigint; database_id: string; name: string; archived_at_ms: [] | [bigint]; credits_balance: [] | [bigint]; credits_suspended_at_ms: [] | [bigint] }; +type RawDatabaseSummary = { status: Variant; role: Variant; logical_size_bytes: bigint; database_id: string; name: string; archived_at_ms: [] | [bigint]; cycles_balance: [] | [bigint]; cycles_suspended_at_ms: [] | [bigint] }; type RawDatabaseMember = { database_id: string; principal: string; role: Variant; created_at_ms: bigint }; type RawWriteNodeRequest = { database_id: string; path: string; kind: Variant; content: string; metadata_json: string; expected_etag: [] | [string] }; type RawWriteNodeResult = { created: boolean; node: RawNodeMutationAck }; @@ -179,8 +179,8 @@ function normalizeDatabaseSummary(raw: RawDatabaseSummary): DatabaseSummary { role: normalizeDatabaseRole(raw.role), status: normalizeDatabaseStatus(raw.status), logicalSizeBytes: raw.logical_size_bytes.toString(), - creditsBalance: raw.credits_balance[0]?.toString() ?? null, - creditsSuspendedAtMs: raw.credits_suspended_at_ms[0]?.toString() ?? null, + cyclesBalance: raw.cycles_balance[0]?.toString() ?? null, + cyclesSuspendedAtMs: raw.cycles_suspended_at_ms[0]?.toString() ?? null, archivedAtMs: raw.archived_at_ms[0]?.toString() ?? null, }; } diff --git a/skill-registry-web/lib/vfs-idl.ts b/skill-registry-web/lib/vfs-idl.ts index 9f709781..e49f08a4 100644 --- a/skill-registry-web/lib/vfs-idl.ts +++ b/skill-registry-web/lib/vfs-idl.ts @@ -20,8 +20,8 @@ export const idlFactory: ActorInterfaceFactory = ({ IDL: idl }) => { database_id: idl.Text, name: idl.Text, archived_at_ms: idl.Opt(idl.Int64), - credits_balance: idl.Opt(idl.Nat64), - credits_suspended_at_ms: idl.Opt(idl.Int64) + cycles_balance: idl.Opt(idl.Nat64), + cycles_suspended_at_ms: idl.Opt(idl.Int64) }); const CreateDatabaseRequest = idl.Record({ name: idl.Text }); const CreateDatabaseResult = idl.Record({ name: idl.Text, database_id: idl.Text }); diff --git a/skill-registry-web/lib/wiki-helpers.ts b/skill-registry-web/lib/wiki-helpers.ts index 5f2d495f..6f9610b5 100644 --- a/skill-registry-web/lib/wiki-helpers.ts +++ b/skill-registry-web/lib/wiki-helpers.ts @@ -78,7 +78,7 @@ export function inferNoteRole(path: string): string { if (name === "plans.md") return "plans"; if (name === "summary.md") return "summary"; if (name === "open_questions.md") return "open_questions"; - if (path.startsWith("/Sources/raw")) return "raw_source"; + if (path === "/Sources/raw" || path.startsWith("/Sources/raw/")) return "raw_source"; if (path.endsWith(".md")) return "markdown_note"; return "directory"; } diff --git a/skill-registry-web/package.json b/skill-registry-web/package.json index 48035322..ddb0717e 100644 --- a/skill-registry-web/package.json +++ b/skill-registry-web/package.json @@ -8,6 +8,7 @@ "lint": "eslint .", "test": "node scripts/check-skill-registry-web.mjs", "typecheck": "next typegen && node scripts/normalize-typegen-tsconfig.mjs && tsc --noEmit", + "audit:moderate": "pnpm audit --audit-level moderate", "normalize:typegen": "node scripts/normalize-typegen-tsconfig.mjs" }, "dependencies": { @@ -25,13 +26,14 @@ "autoprefixer": "10.4.22", "eslint": "9.39.1", "eslint-config-next": "16.2.4", - "postcss": "8.5.11", + "postcss": "8.5.12", "tailwindcss": "3.4.18", "typescript": "5.9.3" }, "pnpm": { "overrides": { - "next>postcss": "8.5.10" + "brace-expansion": "5.0.6", + "postcss": "8.5.12" } }, "packageManager": "pnpm@11.0.8" diff --git a/skill-registry-web/pnpm-lock.yaml b/skill-registry-web/pnpm-lock.yaml index 7d57a1dc..f26756b5 100644 --- a/skill-registry-web/pnpm-lock.yaml +++ b/skill-registry-web/pnpm-lock.yaml @@ -38,7 +38,7 @@ importers: version: 19.2.3(@types/react@19.2.7) autoprefixer: specifier: 10.4.22 - version: 10.4.22(postcss@8.5.11) + version: 10.4.22(postcss@8.5.12) eslint: specifier: 9.39.1 version: 9.39.1(jiti@1.21.7) @@ -46,8 +46,8 @@ importers: specifier: 16.2.4 version: 16.2.4(@typescript-eslint/parser@8.59.0(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) postcss: - specifier: 8.5.11 - version: 8.5.11 + specifier: 8.5.12 + version: 8.5.12 tailwindcss: specifier: 3.4.18 version: 3.4.18 @@ -780,8 +780,8 @@ packages: brace-expansion@1.1.14: resolution: {integrity: sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==} - brace-expansion@5.0.5: - resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} + brace-expansion@5.0.6: + resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==} engines: {node: 18 || 20 || >=22} braces@3.0.3: @@ -1668,8 +1668,8 @@ packages: resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==} engines: {node: ^10 || ^12 || >=14} - postcss@8.5.11: - resolution: {integrity: sha512-5dDj8+lmvA8XB78SmzGI8NlQoksv7IfutGWeVZxiixHbO+p4LDPT3wuG/D9sM/wrjZZ9I+Siy/e117vbFPxSZg==} + postcss@8.5.12: + resolution: {integrity: sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==} engines: {node: ^10 || ^12 || >=14} prelude-ls@1.2.1: @@ -2676,14 +2676,14 @@ snapshots: async-function@1.0.0: {} - autoprefixer@10.4.22(postcss@8.5.11): + autoprefixer@10.4.22(postcss@8.5.12): dependencies: browserslist: 4.28.2 caniuse-lite: 1.0.30001791 fraction.js: 5.3.4 normalize-range: 0.1.2 picocolors: 1.1.1 - postcss: 8.5.11 + postcss: 8.5.12 postcss-value-parser: 4.2.0 available-typed-arrays@1.0.7: @@ -2709,7 +2709,7 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 - brace-expansion@5.0.5: + brace-expansion@5.0.6: dependencies: balanced-match: 4.0.4 @@ -3540,7 +3540,7 @@ snapshots: minimatch@10.2.5: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 5.0.6 minimatch@3.1.5: dependencies: @@ -3568,7 +3568,7 @@ snapshots: '@swc/helpers': 0.5.15 baseline-browser-mapping: 2.10.29 caniuse-lite: 1.0.30001792 - postcss: 8.4.31 + postcss: 8.5.12 react: 19.2.5 react-dom: 19.2.5(react@19.2.5) styled-jsx: 5.1.6(@babel/core@7.29.0)(react@19.2.5) @@ -3688,28 +3688,28 @@ snapshots: possible-typed-array-names@1.1.0: {} - postcss-import@15.1.0(postcss@8.5.11): + postcss-import@15.1.0(postcss@8.5.12): dependencies: - postcss: 8.5.11 + postcss: 8.5.12 postcss-value-parser: 4.2.0 read-cache: 1.0.0 resolve: 1.22.12 - postcss-js@4.1.0(postcss@8.5.11): + postcss-js@4.1.0(postcss@8.5.12): dependencies: camelcase-css: 2.0.1 - postcss: 8.5.11 + postcss: 8.5.12 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.11): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.12): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 - postcss: 8.5.11 + postcss: 8.5.12 - postcss-nested@6.2.0(postcss@8.5.11): + postcss-nested@6.2.0(postcss@8.5.12): dependencies: - postcss: 8.5.11 + postcss: 8.5.12 postcss-selector-parser: 6.1.2 postcss-selector-parser@6.1.2: @@ -3725,7 +3725,7 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postcss@8.5.11: + postcss@8.5.12: dependencies: nanoid: 3.3.12 picocolors: 1.1.1 @@ -4027,11 +4027,11 @@ snapshots: normalize-path: 3.0.0 object-hash: 3.0.0 picocolors: 1.1.1 - postcss: 8.5.11 - postcss-import: 15.1.0(postcss@8.5.11) - postcss-js: 4.1.0(postcss@8.5.11) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.11) - postcss-nested: 6.2.0(postcss@8.5.11) + postcss: 8.5.12 + postcss-import: 15.1.0(postcss@8.5.12) + postcss-js: 4.1.0(postcss@8.5.12) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.12) + postcss-nested: 6.2.0(postcss@8.5.12) postcss-selector-parser: 6.1.2 resolve: 1.22.12 sucrase: 3.35.1 diff --git a/skill-registry-web/scripts/check-skill-registry-web.mjs b/skill-registry-web/scripts/check-skill-registry-web.mjs index e328f9cc..f1dbee86 100644 --- a/skill-registry-web/scripts/check-skill-registry-web.mjs +++ b/skill-registry-web/scripts/check-skill-registry-web.mjs @@ -1,12 +1,16 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; +import ts from "typescript"; const route = readFileSync(new URL("../app/skills/[databaseId]/page.tsx", import.meta.url), "utf8"); const client = readFileSync(new URL("../app/skills/skill-registry-client.tsx", import.meta.url), "utf8"); const panels = readFileSync(new URL("../app/skills/skill-registry-panels.tsx", import.meta.url), "utf8"); +const catalog = readFileSync(new URL("../lib/skill-registry-catalog.ts", import.meta.url), "utf8"); const details = readFileSync(new URL("../lib/skill-registry-details.ts", import.meta.url), "utf8"); const diff = readFileSync(new URL("../lib/skill-registry-diff.ts", import.meta.url), "utf8"); const operations = readFileSync(new URL("../lib/skill-registry-operations.ts", import.meta.url), "utf8"); +const packages = readFileSync(new URL("../lib/skill-registry-package.ts", import.meta.url), "utf8"); +const wikiHelpers = readFileSync(new URL("../lib/wiki-helpers.ts", import.meta.url), "utf8"); const types = readFileSync(new URL("../lib/types.ts", import.meta.url), "utf8"); const vfsClient = readFileSync(new URL("../lib/vfs-client.ts", import.meta.url), "utf8"); const vfsIdl = readFileSync(new URL("../lib/vfs-idl.ts", import.meta.url), "utf8"); @@ -20,19 +24,43 @@ assert.match(panels, /Skill Registry Dashboard/); assert.match(panels, /EvolutionJobsPanel/); assert.match(panels, /Current SKILL\.md/); assert.doesNotMatch(panels, /authenticated=\{false\}/); +assert.match(readFileSync(new URL("../app/skills/skill-registry-ui.tsx", import.meta.url), "utf8"), /proposal\.status === "proposed" \|\| proposal\.status === "reviewed"/); assert.match(client, /handlersFor\(selectedSkill\)/); assert.match(details, /\/Wiki\/skill-evolution-jobs/); assert.match(details, /candidate\/SKILL\.md/); assert.match(details, /metrics\.json/); +assert.match(details, /parseProposalRoot/); +assert.match(details, /statusFields\.skill_id !== skillId/); +assert.match(details, /statusFields\.proposal_id !== id/); +assert.match(catalog, /statusPath: string;/); +assert.match(catalog, /baseEtag: string;/); +assert.match(catalog, /export type ProposalStatus = "proposed" \| "reviewed" \| "auto_applied" \| "gate_failed" \| "conflict"/); +assert.doesNotMatch(catalog, /statusPath: string \| null/); +assert.doesNotMatch(catalog, /baseEtag: string \| null/); +assert.match(details, /parseProposalStatus\(statusFields\.status\)/); +assert.match(details, /statusFields\.kind !== "kinic\.skill_evolution_proposal_status"/); +assert.match(details, /statusFields\.schema_version !== "1"/); assert.match(diff, /candidatePath/); assert.match(diff, /base_etag/); +assert.match(diff, /baseEtag: string/); +assert.match(diff, /Proposal metrics base_etag is required/); +assert.match(diff, /Proposal changed since preview/); assert.match(diff, /assertProposalGates/); +assert.match(diff, /assertProposalStatus\(status\.content, skillId, proposal\.id, \["proposed", "reviewed"\]\)/); assert.match(diff, /candidate_score_gate/); assert.match(diff, /heading_consistency_gate/); assert.match(diff, /permission_gate/); assert.match(diff, /proposal\.metricsPath/); assert.equal((diff.match(/proposal\.metricsPath/g) ?? []).length, 2); assert.match(operations, /kinic.skill_evolution_proposal_status/); +assert.match(operations, /assertProposalStatus\(current\.content, skill\.manifest\.id, proposalId, \["proposed"\]\)/); +assert.match(operations, /proposalStatusPathForSkill/); +assert.match(operations, /Proposal id must be a single safe path segment/); +assert.doesNotMatch(operations, /proposalPath\.split\("\/"\)\.pop/); +assert.match(operations, /frontmatterEnd\(rest\)/); +assert.doesNotMatch(operations, /indexOf\("\\n---"\)/); +assert.match(wikiHelpers, /path === "\/Sources\/raw" \|\| path\.startsWith\("\/Sources\/raw\/"\)/); +assert.doesNotMatch(wikiHelpers, /path\.startsWith\("\/Sources\/raw"\)/); assert.match(types, /DatabaseStatus = "pending" \| "active" \| "restoring" \| "archiving" \| "archived"/); assert.doesNotMatch(vfsIdl, /Hot: idl\.Null/); assert.match(vfsIdl, /Pending: idl\.Null/); @@ -43,5 +71,84 @@ assert.match(vfsClient, /"Pending" in status/); assert.doesNotMatch(vfsClient, /: "hot"/); assert.doesNotMatch(client, /from ["']..\/..\/..\/wikibrowser/); assert.doesNotMatch(panels, /from ["']..\/..\/..\/wikibrowser/); +assert.match(packages, /markdownLinkTargets/); +const packageParser = await importSkillRegistryPackageForTest("../lib/skill-registry-package.ts"); +const manifestParser = await importSkillManifestForTest("../lib/skill-manifest.ts"); +const normalizedManifest = packageParser.normalizeManifestForSkill( + "Skill.v1", + "---\nkind: kinic.skill\nschema_version: 1\nid: Skill.v1\nversion: 0.1.0\nentry: SKILL.md\ntitle: Existing\ntags:\n - Existing\nprovenance:\n license: Existing-License\n---\n# Manifest\n", + "---\nmetadata:\n title: Generated\n category: Generated\ndescription: Generated summary\nlicense: Generated-License\n---\n# Skill\n" +); +assert.equal((normalizedManifest.match(/\ntags:/g) ?? []).length, 1); +assert.match(normalizedManifest, / - Existing/); +assert.match(normalizedManifest, /summary: "Generated summary"/); +assert.doesNotMatch(normalizedManifest, /title: "Generated"/); +assert.doesNotMatch(normalizedManifest, / - Generated/); +assert.doesNotMatch(normalizedManifest, /Generated-License/); +const normalizedEmptyManifest = packageParser.normalizeManifestForSkill( + "Skill.v1", + "---\nkind: kinic.skill\nschema_version: 1\nid: Skill.v1\nversion: 0.1.0\nentry: SKILL.md\ntitle: \"\"\nsummary: \"\"\ntags:\nprovenance:\n license: \"\"\n---\n# Manifest\n", + "---\nmetadata:\n title: Generated\n category: Generated\ndescription: Generated summary\nlicense: Generated-License\n---\n# Skill\n" +); +assert.equal((normalizedEmptyManifest.match(/\ntags:/g) ?? []).length, 1); +assert.equal((normalizedEmptyManifest.match(/\n license:/g) ?? []).length, 1); +assert.match(normalizedEmptyManifest, /title: "Generated"/); +assert.match(normalizedEmptyManifest, /summary: "Generated summary"/); +assert.match(normalizedEmptyManifest, / - "Generated"/); +assert.match(normalizedEmptyManifest, / license: "Generated-License"/); +assert.deepEqual( + packageParser.markdownPackageLinks([ + "[Plan](docs/Project Plan.md)", + "[Angle]()", + "[Alpha](docs/Project (Alpha).md)", + "[Titled](docs/usage.md \"Usage\")", + "[Angle titled]( 'Project plan')", + "[Parenthesized title](docs/reference.md (Reference))", + "[Ignored](https://example.com/docs/External.md)", + "[Escape](../escape.md)", + "[Empty](docs//Broken.md)", + "[Dot](docs/./Hidden.md)" + ].join("\n")), + ["docs/Project Plan.md", "docs/Project (Alpha).md", "docs/usage.md", "docs/reference.md"] +); +assert.equal(packageParser.cleanSkillId("Skill.v1"), "Skill.v1"); +assert.throws(() => packageParser.cleanSkillId("skill..v1"), /single path-safe segment/); +assert.throws(() => packageParser.cleanSkillId("a".repeat(129)), /single path-safe segment/); +assert.equal( + manifestParser.parseSkillManifest("---\nkind: kinic.skill\nschema_version: 1\nid: Skill.v1\nversion: 0.1.0\nentry: SKILL.md\n---\n# Skill")?.id, + "Skill.v1" +); +assert.equal( + manifestParser.parseSkillManifest("---\nkind: kinic.skill\nschema_version: 1\nid: skill..v1\nversion: 0.1.0\nentry: SKILL.md\n---\n# Skill"), + null +); console.log("Skill Registry web checks OK"); + +async function importSkillRegistryPackageForTest(relativePath) { + const sourcePath = new URL(relativePath, import.meta.url); + const source = readFileSync(sourcePath, "utf8") + .replace(/^import .+;\n/gm, "") + .concat("\nexport { markdownPackageLinks, cleanSkillId, normalizeManifestForSkill };\n"); + const compiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.ES2022, + target: ts.ScriptTarget.ES2022 + } + }).outputText; + const moduleUrl = `data:text/javascript;base64,${Buffer.from(compiled).toString("base64")}`; + return import(moduleUrl); +} + +async function importSkillManifestForTest(relativePath) { + const sourcePath = new URL(relativePath, import.meta.url); + const source = readFileSync(sourcePath, "utf8"); + const compiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.ES2022, + target: ts.ScriptTarget.ES2022 + } + }).outputText; + const moduleUrl = `data:text/javascript;base64,${Buffer.from(compiled).toString("base64")}`; + return import(moduleUrl); +} diff --git a/wikibrowser/README.md b/wikibrowser/README.md index aba23761..ee3b4f3c 100644 --- a/wikibrowser/README.md +++ b/wikibrowser/README.md @@ -28,8 +28,8 @@ cargo run -p kinic-vfs-cli --bin kinic-vfs-cli -- --canister-id da ```bash # local icp network -NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8001 -NEXT_PUBLIC_II_PROVIDER_URL=http://id.ai.localhost:8001 +NEXT_PUBLIC_WIKI_IC_HOST=http://127.0.0.1:8011 +NEXT_PUBLIC_II_PROVIDER_URL=http://id.ai.localhost:8011 NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID= # mainnet / Cloudflare Workers @@ -83,7 +83,7 @@ Submitting a URL writes one request node to the same database: Ingest request nodes are regular `file` nodes. Only fetched raw web evidence under `/Sources/raw//.md` is stored as `source`. -When `KINIC_WIKI_GENERATOR_URL` and the `KINIC_WIKI_WORKER_TOKEN` secret are set, the browser asks the VFS canister to authorize a 30 minute session trigger ticket for the II caller, writes the request, then calls `/api/url-ingest/trigger`. That server route checks the canister session ticket and configured canister id before forwarding `canisterId`, `databaseId`, `requestPath`, and `sessionNonce` to the generator Worker with bearer auth. The ticket is replayable within its TTL; duplicate jobs are handled by Worker/job idempotency and rate limits. Writer access and DB credits state are re-checked after issuance, so role revocation, suspension, or low balance can invalidate an existing ticket before its TTL. `Origin` is only a CORS allowlist, not the authorization boundary. +When `KINIC_WIKI_GENERATOR_URL` and the `KINIC_WIKI_WORKER_TOKEN` secret are set, the browser asks the VFS canister to authorize a 30 minute session trigger ticket for the II caller, writes the request, then calls `/api/url-ingest/trigger`. That server route checks the canister session ticket and configured canister id before forwarding `canisterId`, `databaseId`, `requestPath`, and `sessionNonce` to the generator Worker with bearer auth. The ticket is replayable within its TTL; duplicate jobs are handled by Worker/job idempotency and rate limits. Writer access and DB cycles state are re-checked after issuance, so role revocation, suspension, or low balance can invalidate an existing ticket before its TTL. `Origin` is only a CORS allowlist, not the authorization boundary. The worker fetches supported `http` / `https` HTML or text URLs, writes the normalized source to `/Sources/raw//.md`, then generates one review-ready page under `/Wiki/conversations`. Source run tickets are replayable within their TTL so `/api/source/run` can be retried after temporary Worker failures; duplicate source runs are handled by Worker/job idempotency. The generator Worker principal must have writer access to the target database. New databases include the default LLM writer service principal as a `writer` member so URL ingest and page generation can run immediately. Owners can revoke that member, but URL ingest sessions will fail while the service principal lacks writer access. @@ -109,7 +109,7 @@ pnpm e2e:ii:setup pnpm e2e:ii ``` -The wiki canister constructor requires `CreditsConfig`; use the deploy wrapper instead of no-arg `icp deploy`. +The wiki canister constructor requires cycles billing config; use the deploy wrapper instead of no-arg `icp deploy`. `next-env.d.ts` is generated by Next and is intentionally ignored. `pnpm typecheck` runs `next typegen` before `tsc` so clean checkouts do not need to commit that file. diff --git a/wikibrowser/app/api/source/run/route.ts b/wikibrowser/app/api/source/run/route.ts index fe910cdc..9f195ddf 100644 --- a/wikibrowser/app/api/source/run/route.ts +++ b/wikibrowser/app/api/source/run/route.ts @@ -113,7 +113,8 @@ function parseSourceRunRequest(value: unknown): SourceRunRequest | string { } function isCanonicalSourcePath(path: string): boolean { - return /^\/Sources\/raw\/[a-z][a-z0-9]{0,31}\/[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.md$/.test(path); + const match = path.match(/^\/Sources\/raw\/[a-z][a-z0-9]{0,31}\/([A-Za-z0-9][A-Za-z0-9._-]{0,127})\.md$/); + return !!match && !match[1].includes(".."); } function allowedOrigin(request: Request): string | null { diff --git a/wikibrowser/app/api/url-ingest/trigger/route.ts b/wikibrowser/app/api/url-ingest/trigger/route.ts index b7677278..03e8a8ef 100644 --- a/wikibrowser/app/api/url-ingest/trigger/route.ts +++ b/wikibrowser/app/api/url-ingest/trigger/route.ts @@ -104,12 +104,18 @@ function parseTriggerRequest(value: unknown): TriggerRequest | string { if (typeof requestPath !== "string" || !requestPath) return "requestPath is required"; if (typeof sessionNonce !== "string" || !sessionNonce) return "sessionNonce is required"; if (sessionNonce.length > 128) return "sessionNonce is too long"; - if (!requestPath.startsWith("/Sources/ingest-requests/") || !requestPath.endsWith(".md")) { + if (!isIngestRequestPath(requestPath)) { return "requestPath must be a URL ingest request path"; } return { canisterId, databaseId, requestPath, sessionNonce }; } +function isIngestRequestPath(path: string): boolean { + if (!path.startsWith("/Sources/ingest-requests/")) return false; + const name = path.slice("/Sources/ingest-requests/".length); + return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}\.md$/.test(name) && !name.includes(".."); +} + function allowedOrigin(request: Request): string | null { const origin = request.headers.get("origin"); if (!origin || !ALLOWED_ORIGINS.has(origin)) return null; diff --git a/wikibrowser/app/credits/credits-client.tsx b/wikibrowser/app/cycles/cycles-client.tsx similarity index 80% rename from wikibrowser/app/credits/credits-client.tsx rename to wikibrowser/app/cycles/cycles-client.tsx index 2574aa1d..bfa5bbed 100644 --- a/wikibrowser/app/credits/credits-client.tsx +++ b/wikibrowser/app/cycles/cycles-client.tsx @@ -1,38 +1,38 @@ -// Where: /credits client UI. -// What: collects a KINIC amount locally, then submits wallet approval and credits purchase. -// Why: CLI/query can seed credits, and the final purchase amount remains user-editable. +// Where: /cycles client UI. +// What: collects a KINIC amount locally, then submits wallet approval and cycles purchase. +// Why: CLI/query can seed KINIC, and the final purchase amount remains user-editable. "use client"; import Link from "next/link"; import { CheckCircle2, CircleAlert, Info, PlugZap, Wallet } from "lucide-react"; import { useEffect, useMemo, useState, type ReactNode } from "react"; -import { parseCreditsAmountInput, parseCreditsTarget } from "@/lib/credits-url"; -import { connectOisyWallet, connectPlugWallet, purchaseCreditsWithOisy, purchaseCreditsWithPlug, type ConnectedOisyWallet, type ConnectedPlugWallet } from "@/lib/credits-wallet"; -import { formatTokenAmountFromE8s } from "@/lib/credit-amount"; +import { parseKinicAmountE8sInput, parseCyclesTarget } from "@/lib/cycles-url"; +import { connectOisyWallet, connectPlugWallet, purchaseCyclesWithOisy, purchaseCyclesWithPlug, type ConnectedOisyWallet, type ConnectedPlugWallet } from "@/lib/cycles-wallet"; +import { formatTokenAmountFromE8s } from "@/lib/kinic-amount"; -type CreditsStatus = "idle" | "connecting" | "running" | "success" | "error"; -type CreditsProvider = "oisy" | "plug"; +type CyclesStatus = "idle" | "connecting" | "running" | "success" | "error"; +type CyclesProvider = "oisy" | "plug"; -type CreditsClientProps = { +type CyclesClientProps = { canisterId: string; databaseId: string; - initialCredits?: string; + initialKinic?: string; }; -export function CreditsClient({ canisterId, databaseId, initialCredits }: CreditsClientProps) { - const [status, setStatus] = useState("idle"); +export function CyclesClient({ canisterId, databaseId, initialKinic }: CyclesClientProps) { + const [status, setStatus] = useState("idle"); const [message, setMessage] = useState(null); - const [provider, setProvider] = useState(null); - const [amount, setAmount] = useState(() => (initialCredits?.trim() ? initialCredits : "1")); + const [provider, setProvider] = useState(null); + const [amount, setAmount] = useState(() => (initialKinic?.trim() ? initialKinic : "1")); const [oisyWallet, setOisyWallet] = useState(null); const [plugWallet, setPlugWallet] = useState(null); const configuredCanisterId = process.env.NEXT_PUBLIC_KINIC_WIKI_CANISTER_ID ?? ""; const parsedTarget = useMemo(() => { const params = new URLSearchParams(); params.set("database_id", databaseId); - return parseCreditsTarget(params); + return parseCyclesTarget(params); }, [databaseId]); - const parsedAmount = useMemo(() => parseCreditsAmountInput(amount), [amount]); + const parsedAmount = useMemo(() => parseKinicAmountE8sInput(amount), [amount]); const error = typeof parsedTarget === "string" ? parsedTarget @@ -59,7 +59,7 @@ export function CreditsClient({ canisterId, databaseId, initialCredits }: Credit }; }, [oisyWallet]); - async function connect(nextProvider: CreditsProvider) { + async function connect(nextProvider: CyclesProvider) { setStatus("connecting"); setProvider(nextProvider); setMessage(null); @@ -91,17 +91,17 @@ export function CreditsClient({ canisterId, databaseId, initialCredits }: Credit setProvider(selectedProvider); setMessage(null); try { - const request = { canisterId, databaseId: parsedTarget.databaseId, credits: parsedAmount }; + const request = { canisterId, databaseId: parsedTarget.databaseId, paymentAmountE8s: parsedAmount }; const result = selectedProvider === "oisy" && activeOisyWallet - ? await purchaseCreditsWithOisy(request, activeOisyWallet) + ? await purchaseCyclesWithOisy(request, activeOisyWallet) : activePlugWallet - ? await purchaseCreditsWithPlug(request, activePlugWallet) + ? await purchaseCyclesWithPlug(request, activePlugWallet) : null; if (!result) return; - const balance = result.balanceCredits ? `credits balance ${result.balanceCredits}` : "credits purchase accepted"; + const balance = result.balanceCycles ? `cycles balance ${result.balanceCycles}` : "cycles purchase accepted"; setMessage( - `${result.provider} approve block ${result.approveBlockIndex}; purchased credits ${result.creditedCredits}; paid ${formatTokenAmountFromE8s(result.paymentAmountE8s)} KINIC; approved allowance ${formatTokenAmountFromE8s(result.approvedAllowanceE8s)}; ledger transfer fee in allowance ${formatTokenAmountFromE8s(result.transferFeeE8s)}; ${balance}` + `${result.provider} purchased cycles ${result.purchasedCycles}; paid ${formatTokenAmountFromE8s(result.paymentAmountE8s)} KINIC; approved allowance ${formatTokenAmountFromE8s(result.approvedAllowanceE8s)}; ledger transfer fee in allowance ${formatTokenAmountFromE8s(result.transferFeeE8s)}; ${balance}` ); if (selectedProvider === "oisy") setOisyWallet(null); setStatus("success"); @@ -118,14 +118,14 @@ export function CreditsClient({ canisterId, databaseId, initialCredits }: Credit Database dashboard -

Database purchase

+

Database cycles purchase

- +
@@ -225,14 +225,14 @@ function WalletConnect({ ); } -function purchaseButtonLabel(selectedProvider: CreditsProvider | null, status: CreditsStatus, activeProvider: CreditsProvider | null): string { +function purchaseButtonLabel(selectedProvider: CyclesProvider | null, status: CyclesStatus, activeProvider: CyclesProvider | null): string { if (status === "running" && activeProvider === selectedProvider) { if (selectedProvider === "oisy") return "Processing OISY"; if (selectedProvider === "plug") return "Processing Plug"; } - if (selectedProvider === "oisy") return "Purchase credits with OISY"; - if (selectedProvider === "plug") return "Purchase credits with Plug"; - return "Purchase credits"; + if (selectedProvider === "oisy") return "Purchase cycles with OISY"; + if (selectedProvider === "plug") return "Purchase cycles with Plug"; + return "Purchase cycles"; } function Field({ label, value }: { label: string; value: string }) { diff --git a/wikibrowser/app/credits/page.tsx b/wikibrowser/app/cycles/page.tsx similarity index 57% rename from wikibrowser/app/credits/page.tsx rename to wikibrowser/app/cycles/page.tsx index 276b42c3..ef703b13 100644 --- a/wikibrowser/app/credits/page.tsx +++ b/wikibrowser/app/cycles/page.tsx @@ -1,23 +1,23 @@ -// Where: /credits route. +// Where: /cycles route. // What: passes the configured canister and target database into the client. -// Why: CLI/query can seed credits, but canister selection must not come from URL input. +// Why: CLI/query can seed cycles, but canister selection must not come from URL input. import type { Metadata } from "next"; -import { CreditsClient } from "./credits-client"; +import { CyclesClient } from "./cycles-client"; export const metadata: Metadata = { - title: "Kinic Wiki Credits", - description: "Fund a Kinic Wiki database credits balance with a wallet." + title: "Kinic Wiki Cycles", + description: "Fund a Kinic Wiki database cycles balance with a wallet." }; type PageSearchParams = Promise>; -export default async function CreditsPage({ searchParams }: { searchParams: PageSearchParams }) { +export default async function CyclesPage({ searchParams }: { searchParams: PageSearchParams }) { const params = await searchParams; return ( - ); } diff --git a/wikibrowser/app/dashboard/dashboard-client.tsx b/wikibrowser/app/dashboard/dashboard-client.tsx index 47028ccc..bc4fd8c8 100644 --- a/wikibrowser/app/dashboard/dashboard-client.tsx +++ b/wikibrowser/app/dashboard/dashboard-client.tsx @@ -8,12 +8,11 @@ import type { BusyAction } from "./access-control"; import { AuthControls, OwnerPanel, PendingDatabasePanel, ReadonlyMembersPanel, StatusPanel, SummaryPanel } from "./dashboard-ui"; import { CycleBattery } from "@/components/cycle-battery"; import { AUTH_CLIENT_CREATE_OPTIONS, authLoginOptions } from "@/lib/auth"; -import type { CreditsConfig, DatabaseMember, DatabaseRole, DatabaseSummary } from "@/lib/types"; +import type { CyclesBillingConfig, DatabaseMember, DatabaseRole, DatabaseSummary } from "@/lib/types"; import { deleteDatabaseAuthenticated, - getCreditsConfig, + getCyclesBillingConfig, grantDatabaseAccessAuthenticated, - listDatabaseCreditPendingOperationsAuthenticated, listDatabaseMembersAuthenticated, listDatabaseMembersPublic, listDatabasesAuthenticated, @@ -32,9 +31,8 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) const [authClient, setAuthClient] = useState(null); const [principal, setPrincipal] = useState(null); const [databases, setDatabases] = useState([]); - const [creditsConfig, setCreditsConfig] = useState(null); + const [cyclesConfig, setCyclesBillingConfig] = useState(null); const [members, setMembers] = useState([]); - const [pendingOperationCount, setPendingOperationCount] = useState(0); const [loadState, setLoadState] = useState("idle"); const [error, setError] = useState(null); const [warning, setWarning] = useState(null); @@ -61,9 +59,8 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) if (!nextDatabaseId) { setPrincipal(client?.getIdentity().getPrincipal().toText() ?? null); setDatabases([]); - setCreditsConfig(null); + setCyclesBillingConfig(null); setMembers([]); - setPendingOperationCount(0); setError(null); setWarning(null); setMemberError(null); @@ -76,8 +73,8 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) setMemberError(null); try { const identity = client?.getIdentity() ?? null; - const [creditsResult, publicResult, memberResult] = await Promise.allSettled([ - getCreditsConfig(canisterId), + const [cyclesResult, publicResult, memberResult] = await Promise.allSettled([ + getCyclesBillingConfig(canisterId), listDatabasesPublic(canisterId), identity ? listDatabasesAuthenticated(canisterId, identity) : Promise.resolve([]) ]); @@ -94,31 +91,24 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) const nextDatabase = nextDatabases.find((item) => item.databaseId === nextDatabaseId) ?? null; setPrincipal(identity?.getPrincipal().toText() ?? null); setDatabases(nextDatabases); - setCreditsConfig(creditsResult.status === "fulfilled" ? creditsResult.value : null); + setCyclesBillingConfig(cyclesResult.status === "fulfilled" ? cyclesResult.value : null); setMembers([]); - setPendingOperationCount(0); if (publicResult.status === "rejected") { setWarning(`Public database list unavailable: ${errorMessage(publicResult.reason)}`); } if (memberResult.status === "rejected") { setMemberError(`Member database list unavailable: ${errorMessage(memberResult.reason)}`); } - if (identity && nextDatabase?.role === "owner" && nextDatabase.status === "active") { - try { - const nextMembers = await listDatabaseMembersAuthenticated(canisterId, identity, nextDatabaseId); - if (!isCurrentRefresh()) return; - setMembers(nextMembers); - } catch (cause) { - if (!isCurrentRefresh()) return; - setMemberError(errorMessage(cause)); - } - try { - const pendingOperations = await listDatabaseCreditPendingOperationsAuthenticated(canisterId, identity, nextDatabaseId); - if (!isCurrentRefresh()) return; - setPendingOperationCount(pendingOperations.length); - } catch { - if (!isCurrentRefresh()) return; - setPendingOperationCount(0); + if (identity && nextDatabase?.role === "owner") { + if (nextDatabase.status === "active") { + try { + const nextMembers = await listDatabaseMembersAuthenticated(canisterId, identity, nextDatabaseId); + if (!isCurrentRefresh()) return; + setMembers(nextMembers); + } catch (cause) { + if (!isCurrentRefresh()) return; + setMemberError(errorMessage(cause)); + } } } else if (nextDatabase?.publicReadable && nextDatabase.status === "active") { try { @@ -184,9 +174,8 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) await authClient.logout(); setPrincipal(null); setDatabases([]); - setCreditsConfig(null); + setCyclesBillingConfig(null); setMembers([]); - setPendingOperationCount(0); setError(null); setWarning(null); setMemberError(null); @@ -296,20 +285,19 @@ export function DashboardDatabaseClient({ databaseId }: { databaseId: string }) {warning ? : null} {actionMessage ? : null} - {database ? : null} + {database ? : null} {database ? ( canDeletePendingDatabase ? ( ) : canManage ? ( @@ -62,9 +62,9 @@ export function SummaryPanel({ - +
- {purchaseHref ? } label="Credits" /> : null} + {purchaseHref ? } label="Cycles" /> : null} {openHref ? } label="Open" /> : null} {active && publicReadable && routable ? (

Reserved database

-

This database is reserved until the first credit purchase completes. VFS, skills, and member management are available after activation.

+

This database is reserved until the first cycle purchase completes. VFS, skills, and member management are available after activation.

- + ); } export function OwnerPanel(props: { - creditsBalance: string; + cyclesBalance: string; busy: boolean; busyAction: BusyAction | null; databaseId: string; databaseName: string; members: DatabaseMember[]; - pendingOperationCount: number; principal: string; onDelete: () => Promise; onGrant: (principalText: string, role: DatabaseRole) => void; @@ -136,7 +135,7 @@ export function OwnerPanel(props: { if (principalText === LLM_WRITER_PRINCIPAL) { setPendingAction({ title: llmWriterButtonLabel, - message: `Grant writer access to ${LLM_WRITER_LABEL}. Worker writes can create and update wiki drafts, and stop when role or credits state changes.`, + message: `Grant writer access to ${LLM_WRITER_LABEL}. Worker writes can create and update wiki drafts, and stop when role or cycles state changes.`, confirmLabel: llmWriterButtonLabel, principalText, role: "writer", @@ -261,12 +260,11 @@ export function OwnerPanel(props: { {pendingAction ? setPendingAction(null)} onConfirm={confirmPendingAction} /> : null} diff --git a/wikibrowser/app/dashboard/database-danger-zone.tsx b/wikibrowser/app/dashboard/database-danger-zone.tsx index 73c17e90..93923114 100644 --- a/wikibrowser/app/dashboard/database-danger-zone.tsx +++ b/wikibrowser/app/dashboard/database-danger-zone.tsx @@ -9,18 +9,16 @@ import type { BusyAction } from "./access-control"; import { ActionButton } from "./action-button"; export function DatabaseDangerZone(props: { - creditsBalance: string; + cyclesBalance: string; busy: boolean; busyAction: BusyAction | null; databaseId: string; databaseName: string; - pendingOperationCount: number; onDelete: () => Promise; }) { const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); const [deleteError, setDeleteError] = useState(null); - const hasPendingOperations = props.pendingOperationCount > 0; - const deleteDisabled = props.busy || hasPendingOperations; + const deleteDisabled = props.busy; function openDeleteDialog() { setDeleteError(null); setDeleteDialogOpen(true); @@ -45,7 +43,7 @@ export function DatabaseDangerZone(props: {

{props.databaseName} / {props.databaseId}

- +
@@ -113,13 +111,6 @@ function ConfirmDeleteDatabaseDialog(props: { ); } -function DeleteCreditsNotice({ pendingOperationCount }: { pendingOperationCount: number }) { - if (pendingOperationCount > 0) { - return ( -

- Resolve pending credit operations before deleting. Pending operations: {pendingOperationCount} -

- ); - } - return

Remaining credits will be discarded.

; +function DeleteCyclesNotice() { + return

Remaining cycles will be discarded.

; } diff --git a/wikibrowser/app/home-ui.tsx b/wikibrowser/app/home-ui.tsx index a61fdab5..4ed7d25c 100644 --- a/wikibrowser/app/home-ui.tsx +++ b/wikibrowser/app/home-ui.tsx @@ -3,8 +3,8 @@ import Link from "next/link"; import { BookOpen, Settings, Share2, Wallet } from "lucide-react"; import type { ReactNode } from "react"; -import { databaseCreditsView, databaseCreditsHref } from "@/lib/credits-state"; -import type { CreditsConfig, DatabaseSummary } from "@/lib/types"; +import { databaseCyclesView, databaseCyclesHref } from "@/lib/cycles-state"; +import type { CyclesBillingConfig, DatabaseSummary } from "@/lib/types"; import { isRoutableDatabaseId, publicDatabasePath, xShareDatabaseHref } from "@/lib/share-links"; const OFFICIAL_KINIC_WIKI_DATABASE_ID = "db_kva4v2twg6jv"; @@ -57,14 +57,14 @@ export function AuthControls({ } export function DatabaseBody({ - creditsConfig, + cyclesConfig, loading, myDatabases, principal, publicError, publicDatabases }: { - creditsConfig: CreditsConfig | null; + cyclesConfig: CyclesBillingConfig | null; loading: boolean; myDatabases: DatabaseRow[]; principal: string | null; @@ -73,12 +73,12 @@ export function DatabaseBody({ }) { if (loading) return
Loading databases...
; if (!principal) { - return ; + return ; } return (
- - + +
); } @@ -110,7 +110,7 @@ export function OfficialKinicWikiPanel() { } function DatabaseSection({ - creditsConfig, + cyclesConfig, description, emptyMessage, mode, @@ -119,7 +119,7 @@ function DatabaseSection({ showTitle = true, title }: { - creditsConfig: CreditsConfig | null; + cyclesConfig: CyclesBillingConfig | null; description: string; emptyMessage: string; mode: "member" | "public"; @@ -150,7 +150,7 @@ function DatabaseSection({ {!showTitle && publicError && mode === "public" ?

{publicError}

: null}
{rows.map((database) => ( - + ))}
@@ -161,7 +161,7 @@ function DatabaseSection({ Role Status Logical size - Credits + Cycles Open Share {mode === "member" ? Skills : null} @@ -170,7 +170,7 @@ function DatabaseSection({ {rows.map((database) => ( - + ))} @@ -179,7 +179,7 @@ function DatabaseSection({ ); } -function DatabaseTableRow({ creditsConfig, database, mode }: { creditsConfig: CreditsConfig | null; database: DatabaseRow; mode: "member" | "public" }) { +function DatabaseTableRow({ cyclesConfig, database, mode }: { cyclesConfig: CyclesBillingConfig | null; database: DatabaseRow; mode: "member" | "public" }) { const active = isActiveRoutableDatabase(database); return ( @@ -193,15 +193,11 @@ function DatabaseTableRow({ creditsConfig, database, mode }: { creditsConfig: Cr {database.role} {database.status} {formatBytes(database.logicalSizeBytes)} - {databaseCreditsView(database, creditsConfig).summary} + {databaseCyclesView(database, cyclesConfig).summary}
- {active ? ( - } label="Open" /> - ) : -} - {active && mode === "member" && database.publicReadable ? ( - } label="Open public" /> - ) : null} + {active ? } label="Open" /> : -} + {active && mode === "member" && database.publicReadable ? } label="Open public" /> : null}
{active && database.publicReadable ? : -} @@ -213,7 +209,7 @@ function DatabaseTableRow({ creditsConfig, database, mode }: { creditsConfig: Cr Registry ) : null} - } label="Credits" /> + } label="Cycles" />
) : null} @@ -224,7 +220,7 @@ function DatabaseTableRow({ creditsConfig, database, mode }: { creditsConfig: Cr ); } -function DatabaseMobileCard({ creditsConfig, database, mode }: { creditsConfig: CreditsConfig | null; database: DatabaseRow; mode: "member" | "public" }) { +function DatabaseMobileCard({ cyclesConfig, database, mode }: { cyclesConfig: CyclesBillingConfig | null; database: DatabaseRow; mode: "member" | "public" }) { const active = isActiveRoutableDatabase(database); return (
@@ -237,7 +233,7 @@ function DatabaseMobileCard({ creditsConfig, database, mode }: { creditsConfig: - +
{active ? ( @@ -254,7 +250,7 @@ function DatabaseMobileCard({ creditsConfig, database, mode }: { creditsConfig: ) : null ) : null} {mode === "member" ? ( - } label="Credits" /> + } label="Cycles" /> ) : null} {active && database.publicReadable ? : null} } label="Access" /> diff --git a/wikibrowser/app/page.tsx b/wikibrowser/app/page.tsx index 08a2f9bb..c68255dd 100644 --- a/wikibrowser/app/page.tsx +++ b/wikibrowser/app/page.tsx @@ -8,8 +8,8 @@ import { Plus, TerminalSquare } from "lucide-react"; import { CreateDatabaseDialog } from "./create-database-dialog"; import { AuthControls, CreatedDatabasePanel, DatabaseBody, OfficialKinicWikiPanel, StatusPanel } from "./home-ui"; import { AUTH_CLIENT_CREATE_OPTIONS, authLoginOptions } from "@/lib/auth"; -import type { CreditsConfig, DatabaseSummary } from "@/lib/types"; -import { createDatabaseAuthenticated, getCreditsConfig, listDatabasesAuthenticated, listDatabasesPublic } from "@/lib/vfs-client"; +import type { CyclesBillingConfig, DatabaseSummary } from "@/lib/types"; +import { createDatabaseAuthenticated, getCyclesBillingConfig, listDatabasesAuthenticated, listDatabasesPublic } from "@/lib/vfs-client"; import type { DatabaseRow } from "./home-ui"; type LoadState = "idle" | "loading" | "ready" | "error"; @@ -20,7 +20,7 @@ export default function HomePage() { const [authClient, setAuthClient] = useState(null); const [principal, setPrincipal] = useState(null); const [databases, setDatabases] = useState([]); - const [creditsConfig, setCreditsConfig] = useState(null); + const [cyclesConfig, setCyclesBillingConfig] = useState(null); const [loadState, setLoadState] = useState("idle"); const [error, setError] = useState(null); const [publicError, setPublicError] = useState(null); @@ -45,8 +45,8 @@ export default function HomePage() { setWarning(null); try { const identity = client?.getIdentity() ?? null; - const [creditsResult, publicResult, memberResult] = await Promise.allSettled([ - getCreditsConfig(canisterId), + const [cyclesResult, publicResult, memberResult] = await Promise.allSettled([ + getCyclesBillingConfig(canisterId), listDatabasesPublic(canisterId), identity ? listDatabasesAuthenticated(canisterId, identity) : Promise.resolve([]) ]); @@ -58,7 +58,7 @@ export default function HomePage() { const nextDatabases = mergeDatabaseRows(memberDatabases, publicDatabases); if (!isCurrentRefresh()) return; setDatabases(nextDatabases); - setCreditsConfig(creditsResult.status === "fulfilled" ? creditsResult.value : null); + setCyclesBillingConfig(cyclesResult.status === "fulfilled" ? cyclesResult.value : null); setPrincipal(identity?.getPrincipal().toText() ?? null); setPublicError(publicResult.status === "rejected" ? `Public database list unavailable: ${errorMessage(publicResult.reason)}` : null); setWarning(listWarning(memberResult)); @@ -115,7 +115,7 @@ export default function HomePage() { if (!authClient) return; await authClient.logout(); setPrincipal(null); - setCreditsConfig(null); + setCyclesBillingConfig(null); setCreatedDatabase(null); setCreateDialogOpen(false); setNewDatabaseName(""); @@ -223,7 +223,7 @@ export default function HomePage() {
- + ) : (
@@ -233,7 +233,7 @@ export default function HomePage() {

Public databases open without login. Login with Internet Identity to show My databases linked to your principal.

- + )} diff --git a/wikibrowser/app/skills/skill-registry-client.tsx b/wikibrowser/app/skills/skill-registry-client.tsx index 73d6dc35..e46bcebe 100644 --- a/wikibrowser/app/skills/skill-registry-client.tsx +++ b/wikibrowser/app/skills/skill-registry-client.tsx @@ -9,13 +9,13 @@ import { PackageManager, RoleBanner } from "@/app/skills/skill-registry-manageme import { usePackageManager } from "@/app/skills/skill-registry-package-state"; import { EmptyState, SkillCard, StatusPanel, SummaryStrip } from "@/app/skills/skill-registry-ui"; import { AUTH_CLIENT_CREATE_OPTIONS, authLoginOptions } from "@/lib/auth"; -import { databaseCreditsDisabledReason, databaseCanWrite } from "@/lib/credits-state"; +import { databaseCyclesDisabledReason, databaseCanWrite } from "@/lib/cycles-state"; import { filterSkills, loadSkillCatalog, summarizeSkills, type CatalogSkill, type StatusFilter } from "@/lib/skill-registry-catalog"; import { loadSkillCatalogDetails } from "@/lib/skill-registry-details"; import { applyProposalDiff, previewApplyProposalDiff, type ProposalDiffPreview } from "@/lib/skill-registry-diff"; import { approveSkillProposal, recordSkillEvent, recordSkillRun, updateSkillStatus, type RunOutcome, type SkillStatus } from "@/lib/skill-registry-operations"; -import type { CreditsConfig, DatabaseRole, DatabaseSummary } from "@/lib/types"; -import { getCreditsConfig, listDatabasesAuthenticated } from "@/lib/vfs-client"; +import type { CyclesBillingConfig, DatabaseRole, DatabaseSummary } from "@/lib/types"; +import { getCyclesBillingConfig, listDatabasesAuthenticated } from "@/lib/vfs-client"; type LoadState = "idle" | "loading" | "ready" | "error"; type ActionDraft = { @@ -55,7 +55,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { const [actions, setActions] = useState>({}); const [databaseRole, setDatabaseRole] = useState(null); const [databaseSummary, setDatabaseSummary] = useState(null); - const [creditsConfig, setCreditsConfig] = useState(null); + const [cyclesConfig, setCyclesConfig] = useState(null); const loadCatalog = useCallback( async (identity?: Identity) => { @@ -102,13 +102,13 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { } catch { setDatabaseSummary(null); setDatabaseRole(null); - setCreditsConfig(null); + setCyclesConfig(null); return; } try { - setCreditsConfig(await getCreditsConfig(canisterId)); + setCyclesConfig(await getCyclesBillingConfig(canisterId)); } catch { - setCreditsConfig(null); + setCyclesConfig(null); } }, [canisterId, databaseId]); @@ -170,7 +170,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { setPrincipal(null); setDatabaseRole(null); setDatabaseSummary(null); - setCreditsConfig(null); + setCyclesConfig(null); setSkills([]); setError(null); setLoadState("idle"); @@ -180,8 +180,8 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { const filteredSkills = useMemo(() => filterSkills(skills, query, statusFilter), [skills, query, statusFilter]); const summary = useMemo(() => summarizeSkills(skills), [skills]); const identity = authClient?.getIdentity(); - const creditsReason = databaseCreditsDisabledReason(databaseSummary, creditsConfig); - const writable = databaseCanWrite(databaseSummary, creditsConfig); + const cyclesReason = databaseCyclesDisabledReason(databaseSummary, cyclesConfig); + const writable = databaseCanWrite(databaseSummary, cyclesConfig); const packageManager = usePackageManager({ canisterId, databaseId, identity, writable, refresh: refreshSkillRegistry, errorMessage }); function actionFor(skill: CatalogSkill): ActionDraft { @@ -303,7 +303,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { }), true ), - approveProposal: (proposal) => void runSkillAction(skill, (activeIdentity) => approveSkillProposal(canisterId, databaseId, activeIdentity, skill, proposal.path)), + approveProposal: (proposal) => void runSkillAction(skill, (activeIdentity) => approveSkillProposal(canisterId, databaseId, activeIdentity, skill, proposal.proposalRoot)), previewProposal: (proposal) => void runSkillAction( skill, @@ -316,7 +316,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { ), applyProposal: (proposal) => void runSkillAction(skill, async (activeIdentity, draft) => { - if (!draft.preview || draft.preview.proposalPath !== proposal.path) throw new Error("Preview this proposal before applying."); + if (!draft.preview || draft.preview.proposalPath !== proposal.proposalRoot) throw new Error("Preview this proposal before applying."); await applyProposalDiff(canisterId, databaseId, activeIdentity, proposal, draft.preview); await recordSkillEvent(canisterId, databaseId, activeIdentity, skill.manifest.id, { action: "proposal.apply", targetPath: draft.preview.targetPath, result: "applied" }); }) @@ -329,7 +329,7 @@ export function SkillRegistryClient({ databaseId }: { databaseId: string }) { ) : null}
diff --git a/wikibrowser/app/skills/skill-registry-management-ui.tsx b/wikibrowser/app/skills/skill-registry-management-ui.tsx index 0ad2f478..583542ed 100644 --- a/wikibrowser/app/skills/skill-registry-management-ui.tsx +++ b/wikibrowser/app/skills/skill-registry-management-ui.tsx @@ -3,13 +3,11 @@ import type { ReactNode } from "react"; import { ChevronDown, Github, ShieldCheck, Upload } from "lucide-react"; import type { DatabaseRole } from "@/lib/types"; -import type { SkillCatalog } from "@/lib/skill-registry-package"; export type PackageDraft = { source: string; reference: string; id: string; - catalog: SkillCatalog; skill: string; manifest: string; provenance: string; @@ -24,9 +22,9 @@ export type PackageHandlers = { pasteUpsert: () => void; }; -export function RoleBanner({ creditsReason, role, principal }: { creditsReason: string | null; role: DatabaseRole | null; principal: string | null }) { +export function RoleBanner({ cyclesReason, role, principal }: { cyclesReason: string | null; role: DatabaseRole | null; principal: string | null }) { const roleWritable = role === "writer" || role === "owner"; - const writable = roleWritable && !creditsReason; + const writable = roleWritable && !cyclesReason; return (
@@ -35,7 +33,7 @@ export function RoleBanner({ creditsReason, role, principal }: { creditsReason:

Database Role: {role ?? (principal ? "unknown" : "anonymous")}

-

{writable ? "Write operations enabled." : roleWritable && creditsReason ? creditsReason : "Writer or owner access required."}

+

{writable ? "Write operations enabled." : roleWritable && cyclesReason ? cyclesReason : "Writer or owner access required."}

@@ -77,7 +75,7 @@ export function PackageManager({ onChange={(event) => handlers.setDraft({ source: event.target.value })} /> -
+
handlers.setDraft({ id: event.target.value })} /> - - handlers.setDraft({ catalog })} /> -

Public GitHub repositories only. Browser API cannot prune deleted files.