From 77a75b1b34215f5f0e53fa287dd73c59c1fc8b4a Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:07:02 -0700 Subject: [PATCH 01/19] Add FOCUS 1.3 ingestion (#2124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds IngestionSetup_v1_3.kql with Costs/Prices/CommitmentDiscountUsage/ Recommendations/Transactions transforms and final tables renamed for FOCUS 1.3. Cost and Usage gains 8 new FOCUS 1.3 columns: AllocatedMethodId/Details/ResourceId/ResourceName/Tags (data-generator split cost allocation), ContractApplied (per-row contract commitment application), ServiceProviderName + HostProviderName (replacing the deprecated ProviderName/PublisherName, with empty-fallback for back compat). Costs_raw now carries the new columns so downstream v1_2 transforms keep working and v1_3 transforms can read them directly. The v1_3 file is wired into the Bicep deployment alongside v1_0 and v1_2. The unversioned Costs() function still aliases to v1_2 — phase 2 adds Costs_v1_3() and retargets Latest. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 3 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../scripts/IngestionSetup_RawTables.kql | 8 + .../Analytics/scripts/IngestionSetup_v1_3.kql | 1966 +++++++++++++++++ 4 files changed, 1977 insertions(+), 1 deletion(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 2dd495e60..0963fb494 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -33,7 +33,8 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index b880e7108..42cd2738b 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -378,6 +378,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) scripts: { v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') + v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 94fead987..83a3be40e 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -359,6 +359,11 @@ // Costs_raw table -- Redefine all columns .alter table Costs_raw ( + AllocatedMethodDetails: string, // FOCUS 1.3+ (JSON) + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: string, // FOCUS 1.3+ (JSON) AvailabilityZone: string, // FOCUS 0.5+ BilledCost: real, // FOCUS 0.5+ BillingAccountId: string, // FOCUS 0.5+ @@ -385,9 +390,11 @@ CommitmentDiscountUnit: string, // FOCUS 1.1+ ConsumedQuantity: real, // FOCUS 1.0+ ConsumedUnit: string, // FOCUS 1.0+ + ContractApplied: string, // FOCUS 1.3+ (JSON) ContractedCost: real, // FOCUS 1.0+ ContractedUnitPrice: real, // FOCUS 1.0+ EffectiveCost: real, // FOCUS 1.0-preview+ + HostProviderName: string, // FOCUS 1.3+ InvoiceId: string, // FOCUS 1.2+ InvoiceIssuerName: string, // FOCUS 0.5+ ListCost: real, // FOCUS 1.0-preview+ @@ -406,6 +413,7 @@ ResourceType: string, // FOCUS 1.0-preview+ ServiceCategory: string, // FOCUS 0.5+ ServiceName: string, // FOCUS 0.5+ + ServiceProviderName: string, // FOCUS 1.3+ ServiceSubcategory: string, // FOCUS 1.1+ SkuId: string, // FOCUS 1.0-preview+ SkuMeter: string, // FOCUS 1.1+ diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql new file mode 100644 index 000000000..70d29a2ed --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql @@ -0,0 +1,1966 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database +// See known issues @ https://github.com/microsoft/finops-toolkit/issues/1111 +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +//===| Prices |========================================================================================================= +// Supported versions: +// - MS EA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-ea +// - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca +//====================================================================================================================== + +// Prices_transform_v1_3 function +.create-or-alter function +with (docstring='Transforms Prices_raw into FOCUS 1.3.', folder='Prices') +Prices_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + let prices = materialize( + Prices_raw + | extend PricingCurrency = coalesce(Currency, CurrencyCode) // CurrencyCode last as a fallback only + | extend x_SkuId = coalesce(SkuId, SkuID) + | extend x_SkuMeterId = coalesce(MeterId, MeterID) + | extend x_SkuProductId = coalesce(ProductId, ProductID) + | extend x_SkuTerm = isoMonths(Term) + | project-rename + SkuMeter = MeterName, + x_BaseUnitPrice = BasePrice, + x_EffectivePeriodEnd = EffectiveEndDate, + x_EffectivePeriodStart = EffectiveStartDate, + x_PricingUnitDescription = UnitOfMeasure, + x_SkuIncludedQuantity = IncludedQuantity, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuMeterType = MeterType, + x_SkuOfferId = OfferID, + x_SkuPartNumber = PartNumber, + x_SkuPriceType = PriceType, + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTier = TierMinimumUnits + | extend ContractedUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', UnitPrice, real(null)) // UnitPrice for savings plan is not the on-demand unit price + | extend ListUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', MarketPrice, real(null)) // MarketPrice for savings plan is not the list price + | extend ChargeCategory = case( + x_SkuPriceType == 'Consumption', 'Usage', + x_SkuPriceType == 'ReservedInstance', 'Purchase', + x_SkuPriceType == 'SavingsPlan', 'Usage', // Savings plan prices are for committed usage, not the purchase + '' + ) + | extend SkuPriceIdv2 = strcat(case(x_SkuPriceType == 'Consumption', 'OD', x_SkuPriceType == 'ReservedInstance', 'RI', x_SkuPriceType == 'SavingsPlan', 'SP', 'XX'), substring(ChargeCategory, 0, 1), x_SkuTerm, '_', x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType, '_', x_SkuTier, x_SkuOfferId) + | extend x_BillingAccountId = iff(BillingAccountId startswith '/', split(BillingAccountId, '/')[-1], coalesce(BillingAccountId, EnrollmentNumber)) + | extend x_BillingProfileId = iff(BillingProfileId startswith '/', split(BillingProfileId, '/')[-1], coalesce(BillingProfileId, EnrollmentNumber)) + | extend tmp_SavingsPlanKey = strcat(x_SkuMeterId, x_SkuProductId, x_SkuId, x_SkuTier, x_SkuOfferId) + // + // Get latest ingested row based on the unique ID + | extend x_IngestionTime = ingestion_time() + ); + // + // Meters for reservations and savings plans to identify commitment eligibility + let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; + let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // + // Copy list/base/contracted prices from on-demand SKUs + prices + | where x_SkuPriceType == 'SavingsPlan' + // If we use join, specify the shuffle key + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) + | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) + | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) + | project-away ListUnitPrice1, ContractedUnitPrice1, x_BaseUnitPrice1, tmp_SavingsPlanKey + | union ((prices | where x_SkuPriceType != 'SavingsPlan')) + // + // Set CommitmentDiscountCategory for reuse + | extend CommitmentDiscountCategory = case( + x_SkuPriceType == 'ReservedInstance', 'Usage', + x_SkuPriceType == 'SavingsPlan', 'Spend', + '' + ) + // + // Calculate commitment discount eligibility + // TODO: Would a join be faster? + // TODO: Check this to ensure it's correct + | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + // + // TODO: Implement x_CommitmentDiscountNormalizedRatio + | extend x_CommitmentDiscountNormalizedRatio = real(null) + // + // Add PricingUnit and x_PricingBlockSize + // TODO: Compare join vs. lookup perf -- | join kind=leftouter (PricingUnits) on x_PricingUnitDescription | project-away x_PricingUnitDescription1 + | lookup kind=leftouter (PricingUnits) on x_PricingUnitDescription + // + | extend x_EffectiveUnitPrice = iff(x_SkuPriceType == 'SavingsPlan', UnitPrice, real(null)) // Savings plan prices are for the effective price, not the contracted price + | extend x_EffectiveUnitPriceDiscount = ContractedUnitPrice - x_EffectiveUnitPrice + | extend x_ContractedUnitPriceDiscount = ListUnitPrice - ContractedUnitPrice + | extend x_TotalUnitPriceDiscount = ListUnitPrice - x_EffectiveUnitPrice + | project + BillingAccountId = tolower(case( + BillingProfileId startswith '/', BillingProfileId, + BillingAccountId startswith '/', BillingAccountId, + strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId, iff(x_BillingProfileId == x_BillingAccountId, '', strcat('/billingprofiles/', x_BillingProfileId))) + )), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName, x_BillingProfileId), + BillingCurrency = coalesce(BillingCurrency, CurrencyCode, Currency), // Currency last as a fallback only + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType = case( + x_SkuPriceType == 'ReservedInstance', 'Reservation', + x_SkuPriceType == 'SavingsPlan', 'Savings plan', + '' + ), + CommitmentDiscountUnit = case( + isempty(CommitmentDiscountCategory), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), PricingUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', PricingUnit), + '' + ), + ContractedUnitPrice, + ListUnitPrice, + PricingCategory = case( + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed', + '' + ), + PricingCurrency, + PricingUnit, + SkuId = coalesce(ProductId, ProductID), + SkuMeter, + SkuPriceId = strcat(x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType), + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement = case( + strlen(x_BillingAccountId) > 32, 'MCA', + strlen(x_BillingAccountId) < 32, 'EA', + 'Unknown' + ), + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent = 1.0 * x_ContractedUnitPriceDiscount / ListUnitPrice * 100, + x_EffectivePeriodEnd = startofmonth(x_EffectivePeriodEnd + 1h), + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent = 1.0 * x_EffectiveUnitPriceDiscount / ContractedUnitPrice * 100, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory = case( + x_SkuPriceType == 'Consumption' and (x_SkuIncludedQuantity > 0 or x_SkuTier > 0), 'Tiered', + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed Spend', + '' + ), + x_PricingUnitDescription, + x_SkuDescription = Product, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName = coalesce(x_SourceName, 'Cost Management'), + x_SourceProvider = coalesce(x_SourceProvider, 'Microsoft'), + x_SourceType = coalesce(x_SourceType, 'PriceSheet'), + x_SourceVersion = coalesce(x_SourceVersion, '2023-05-01'), + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 +} + +// Prices_final_v1_3 table +.create-merge table Prices_final_v1_3 ( + BillingAccountId: string, + BillingAccountName: string, + BillingCurrency: string, + ChargeCategory: string, + CommitmentDiscountCategory: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ContractedUnitPrice: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, // Azure + PricingUnit: string, + SkuId: string, + SkuMeter: string, // Azure + SkuPriceId: string, + SkuPriceIdv2: string, // Hubs add-on + x_BaseUnitPrice: real, // Azure + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure MCA + x_BillingProfileId: string, // Azure MCA + x_CommitmentDiscountNormalizedRatio: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_ContractedUnitPriceDiscount: real, // Hubs add-on + x_ContractedUnitPriceDiscountPercent: real, // Hubs add-on + x_EffectivePeriodEnd: datetime, // Azure + x_EffectivePeriodStart: datetime, // Azure + x_EffectiveUnitPrice: real, // Azure + x_EffectiveUnitPriceDiscount: real, // Hubs add-on + x_EffectiveUnitPriceDiscountPercent: real, // Hubs add-on + x_IngestionTime: datetime, // Hubs add-on + x_PricingBlockSize: real, // Hubs add-on + x_PricingSubcategory: string, // Hubs add-on + x_PricingUnitDescription: string, // Azure + x_SkuDescription: string, // Azure + x_SkuId: string, // Azure + x_SkuIncludedQuantity: real, // Azure EA + x_SkuMeterCategory: string, // Azure + x_SkuMeterId: string, // Azure + x_SkuMeterSubcategory: string, // Azure + x_SkuMeterType: string, // Azure + x_SkuPriceType: string, // Azure + x_SkuProductId: string, // Azure + x_SkuRegion: string, // Azure + x_SkuServiceFamily: string, // Azure + x_SkuOfferId: string, // Azure EA + x_SkuPartNumber: string, // Azure EA + x_SkuTerm: int, // Azure + x_SkuTier: real, // Azure MCA + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_TotalUnitPriceDiscount: real, // Hubs add-on + x_TotalUnitPriceDiscountPercent: real // Hubs add-on +) + +// Update policy for Prices_raw -> Prices_final_v1_3 +.alter table Prices_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Prices_raw", + "Query": "Prices_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Cost and usage |================================================================================================= +// Supported versions: +// - MS: 1.2-preview, 1.0, 1.0-preview(v1) +// https://aka.ms/costmgmt/exports/focus +// - AWS: 1.0 +// https://docs.aws.amazon.com/cur/latest/userguide/table-dictionary-focus-1-0-aws-columns.html +// - GCP: Jan-Jun 2024 +// https://cloud.google.com/resources/google-cloud-focus?e=48754805&hl=en +// Links to (Aug 2024): https://services.google.com/fh/files/misc/focus_guide_v1.pdf +// See also: +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/standard-usage +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage +// - OCI: 1.0 +// https://docs.oracle.com/iaas/Content/Billing/Concepts/costusagereportsoverview.htm#costreports__focus-cost-report-schema +// +// Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. +//====================================================================================================================== + +// Costs_transform_v1_3 function +.create-or-alter function +with (docstring='All costs transformed to FOCUS 1.3.', folder='Costs') +Costs_transform_v1_3() +{ + let checkString = (column: string, oldValue: string, newValue: string) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkInt = (column: string, oldValue: int, newValue: int) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkReal = (column: string, oldValue: real, newValue: real) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + Costs_raw + // + // Dedupe rows + | extend x_IngestionTime = ingestion_time() + | extend x_ChargeId = '' + // TODO: Consider adding a unique charge ID per row + // hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // // 1. Resource hierarchy (including resource name), highest to lowest + // BillingAccountId, + // x_InvoiceSectionId, + // x_AccountOwnerId, + // SubAccountId, + // x_ResourceGroupName, + // ResourceName, + // // 2. Resource details + // ResourceId, + // RegionId, + // Tags, + // CommitmentDiscountId, + // x_CostCenter, + // // 4. Meter details + // SkuPriceId, + // x_SkuMeterId, + // x_SkuPartNumber, + // x_SkuOfferId, + // x_SkuDetails, + // // 5. Date + // ChargePeriodStart + // )) + // + // Identify data quality issues + // TODO: Remove x_SourceChanges in v1_3 (or later) + | extend x_SourceChanges = trim_end(',', strcat( + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based', 'InvalidChargeFrequency,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and EffectiveCost > 0, 'InvalidEffectiveCost,', ''), + iff((isempty(ContractedCost) or ContractedCost == 0) and EffectiveCost != 0, 'MissingContractedCost,', ''), + iff((isempty(ContractedUnitPrice) or ContractedUnitPrice == 0) and x_EffectiveUnitPrice != 0, 'MissingContractedUnitPrice,', ''), + iff(ListCost < ContractedCost, 'ListCostLessThanContractedCost,', ''), + iff(ContractedCost < EffectiveCost, 'ContractedCostLessThanEffectiveCost,', ''), + iff((isempty(ListCost) or ListCost == 0) and (ContractedCost != 0 or EffectiveCost != 0), 'MissingListCost,', ''), + iff((isempty(ListUnitPrice) or ListUnitPrice == 0) and (ContractedUnitPrice != 0 or x_EffectiveUnitPrice != 0), 'MissingListUnitPrice,', ''), + iff((isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001) + or (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001), + 'XEffectiveUnitPriceRoundingError,', ''), + iff(ConsumedQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingConsumedQuantity,', ''), + iff(PricingQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingPricingQuantity,', ''), + iff(isempty(ProviderName), 'MissingProviderName,', ''), + iff(isempty(PublisherName), 'MissingPublisherName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceId), 'MissingResourceId,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceName), 'MissingResourceName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceType), 'MissingResourceType,', ''), + iff(BilledCost > 0 and x_BilledUnitPrice == 0, 'MissingXBilledUnitPrice,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(x_ResourceType), 'MissingXResourceType,', ''), + iff(PricingCategory == 'Standard' and isnotempty(CommitmentDiscountId) and ChargeCategory == 'Usage', 'PricingCategoryShouldBeCommitted,', ''), + iff(x_SkuTerm == '1Year' or x_SkuTerm == '3Years' or x_SkuTerm == '5Years', 'SkuTermShouldBeAnInteger,', '') + )) + // + // Handle provider columns that moved to FOCUS + | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) + // + // Backup original prices/costs before the merge + | extend old_ContractedCost = ContractedCost + | extend old_ContractedUnitPrice = ContractedUnitPrice + | extend old_ListCost = ListCost + | extend old_ListUnitPrice = ListUnitPrice + | extend old_x_EffectiveUnitPrice = x_EffectiveUnitPrice + // + // Fix columns needed in other changes + | extend old_ProviderName = ProviderName, ProviderName = case( + isnotempty(ProviderName), ProviderName, + isnotempty(coalesce(x_CostCategories, x_Discount, x_Operation, x_ServiceCode, x_UsageType)), 'AWS', + isnotempty(coalesce(tostring(UsageAmount), tostring(x_Cost), x_Credits, x_CostType, tostring(x_CurrencyConversionRate), tostring(x_ExportTime), x_Project, x_ServiceId)), 'GCP', + isnotempty(coalesce(x_BillingProfileId, x_InvoiceSectionId)), 'Microsoft', + '' + ) + // + // Identify source + | extend x_SourceName = coalesce(x_SourceName, iff(isnotempty(x_BillingProfileId), 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, case( + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', + isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', + isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), + '' + )) + // Append version check error code + | extend x_SourceChanges = iff(x_SourceVersion == '1.0', x_SourceChanges, + strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) + ) + // + // Fix quantities + | extend old_PricingQuantity = PricingQuantity, PricingQuantity = case( + PricingQuantity != 0 or (EffectiveCost == 0 and BilledCost == 0), PricingQuantity, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(EffectiveCost) and EffectiveCost != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0, EffectiveCost / x_EffectiveUnitPrice, + PricingQuantity + ) + | extend old_ConsumedQuantity = ConsumedQuantity, ConsumedQuantity = case( + isnotempty(ConsumedQuantity) and ConsumedQuantity != 0, ConsumedQuantity, + ChargeCategory == 'Usage', PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + ConsumedQuantity + ) + // + // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID + | extend tmp_MissingPrices = ProviderName == 'Microsoft' + and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0) + and x_EffectiveUnitPrice != 0 + and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused') + and isnotempty(strcat(x_SkuMeterId, x_SkuOfferId)) + | as allCosts + | where tmp_MissingPrices + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | as costsWithMissingPrices + | join kind=leftouter ( + Prices_final_v1_3 + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) + | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit + ) on tmp_ReservationPriceLookupKey + // + // Select the best price to use for each row + | extend x_EffectiveUnitPrice = case( + // If price is a rounding error away from the billed price, use the billed price + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001, x_BilledUnitPrice, + // If price is a rounding error away from the contracted price, use the contracted price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001, ContractedUnitPrice, + x_EffectiveUnitPrice + ) + | extend ContractedUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ContractedUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ContractedUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // If billed price is available, assume the billed price is the same as contracted price to support aggregations + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, x_EffectiveUnitPrice, + // Otherwise, assume the effective price is the same as contracted price to support aggregations + x_EffectiveUnitPrice + ) + | extend ListUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ListUnitPrice) and ListUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ListUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ListUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ListUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // Otherwise, assume the contracted price is the same as list price to support aggregations + ContractedUnitPrice + ) + // Calculate missing costs based on new prices -- If cost is already correct, keep that; if not and price is available, recalculate the cost; otherwise, keep the existing cost + | extend ContractedCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ContractedCost) and ContractedCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedCost, + // ContractedCost is 0 in all other scenarios... + // If 0 and there's a billed cost and prices are the same, use BilledCost + isnotempty(BilledCost) and BilledCost != 0 and ContractedUnitPrice == x_BilledUnitPrice, BilledCost, + // If 0 and there's a billed cost and prices are the same, use EffectiveCost + isnotempty(EffectiveCost) and EffectiveCost != 0 and ContractedUnitPrice == x_EffectiveUnitPrice, EffectiveCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0, ContractedUnitPrice * PricingQuantity, + // If 0 and there's no price, assume EffectiveCost + isempty(ContractedUnitPrice) or ContractedUnitPrice == 0, EffectiveCost, + // Fall back to the original value for any unhandled scenarios + ContractedCost + ) + | extend ListCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ListCost) and ListCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ListCost, + // ListCost is 0 in all other scenarios... + // If 0 and there's a contracted cost and prices are the same, use ContractedCost + isnotempty(ContractedCost) and ContractedCost != 0 and ListUnitPrice == ContractedUnitPrice, ContractedCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ListUnitPrice) and ListUnitPrice != 0, ListUnitPrice * PricingQuantity, + // If 0 and there's no price, assume ContractedCost + isempty(ListUnitPrice) or ListUnitPrice == 0, ContractedCost, + // Fall back to the original value for any unhandled scenarios + ListCost + ) + // Merge the rest of the unmodified cost records and remove excess columns + | union (allCosts | where not(tmp_MissingPrices)) + | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1 + // + | extend SkuPriceDetails = parse_json(SkuPriceDetails) + | extend Tags = parse_json(Tags) + | extend x_SkuDetails = parse_json(x_SkuDetails) + // + // Handle FOCUS 1.0-preview + | extend old_ChargeSubcategory = ChargeSubcategory + | extend old_ChargeCategory = ChargeCategory, ChargeCategory = case( + // Handle FOCUS 1.0-preview ChargeSubcategory + ChargeSubcategory == 'Credit', 'Credit', + ChargeSubcategory == 'Refund', 'Purchase', // We are assuming purchase refunds since we don't have data to indicate usage refunds + ChargeCategory + ) + | extend old_ChargeClass = ChargeClass, ChargeClass = case(ChargeSubcategory == 'Refund', 'Correction', ChargeClass) + // + // Populate CapacityReservationId when not specified + | extend CapacityReservationId = coalesce(CapacityReservationId, tostring(coalesce(x_SkuDetails.VMCapacityReservationId, SkuPriceDetails.VMCapacityReservationId, SkuPriceDetails.x_VMCapacityReservationId))) + | extend old_CapacityReservationStatus = CapacityReservationStatus, CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + isnotempty(CapacityReservationStatus), CapacityReservationStatus, + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + // + // BUG: ChargeFrequency shows "Usage-Based" for monthly recurring savings plan purchases + | extend old_ChargeFrequency = ChargeFrequency, ChargeFrequency = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based' and ProviderName == 'Microsoft' and x_SourceVersion startswith '1.0', 'Recurring', ChargeFrequency) + // + // Commitment discounts + | extend x_CommitmentDiscountNormalizedRatio = case( + // Calculate from CommitmentDiscountQuantity, if specified + isnotempty(CommitmentDiscountQuantity) and CommitmentDiscountQuantity != 0, CommitmentDiscountQuantity / PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, SkuPriceDetails.RINormalizationRatio, SkuPriceDetails.x_RINormalizationRatio, dynamic(1))) + ) + | extend old_CommitmentDiscountQuantity = CommitmentDiscountQuantity, CommitmentDiscountQuantity = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountQuantity), CommitmentDiscountQuantity, + // FOCUS 1.0-preview, 1.0 + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend old_CommitmentDiscountUnit = CommitmentDiscountUnit, CommitmentDiscountUnit = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountUnit), CommitmentDiscountUnit, + // FOCUS 1.0 + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend old_CommitmentDiscountStatus = CommitmentDiscountStatus, CommitmentDiscountStatus = case( + // FOCUS 1.0+ + isnotempty(CommitmentDiscountStatus), CommitmentDiscountStatus, + // FOCUS 1.0-preview + ChargeSubcategory == 'Used Commitment', 'Used', + ChargeSubcategory == 'Unused Commitment', 'Unused', + '' + ) + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // Pricing + | extend old_x_AmortizationClass = x_AmortizationClass, x_AmortizationClass = case( + // FOCUS 1.3 + isnotempty(x_AmortizationClass), x_AmortizationClass, + // FOCUS 1.0-preview+ + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + | extend old_PricingCategory = PricingCategory, PricingCategory = case( + // FOCUS 1.0+ + isnotempty(PricingCategory), PricingCategory, + // FOCUS 1.0-preview + PricingCategory == 'On-Demand', 'Standard', + PricingCategory == 'Commitment-Based', 'Committed', + '' + ) + // + // Commitment discount utilization + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // BUG: Fix ContractedCost that has bad values + | extend ContractedCost = iff(ProviderName == 'Microsoft' and isnotempty(PricingQuantity) and isnotempty(x_PricingBlockSize) and ContractedCost != ContractedUnitPrice * PricingQuantity, ContractedUnitPrice * PricingQuantity, ContractedCost) + // + // Handle FOCUS 1.0-preview UsageQuantity/Unit + | extend ConsumedQuantity = iff(ChargeCategory == 'Usage', coalesce(ConsumedQuantity, UsageQuantity, UsageAmount), real(null)) + | extend old_ConsumedUnit = ConsumedUnit, ConsumedUnit = iff(ChargeCategory == 'Usage' and isnotempty(ConsumedQuantity), coalesce(ConsumedUnit, UsageUnit, 'Units'), '') + // + // Convert IDs to lowercase for consistency + | extend BillingAccountId = tolower(BillingAccountId) + | extend CommitmentDiscountId = tolower(CommitmentDiscountId) + // + // BUG: Remove EffectiveCost for commitment discount purchases + | extend old_EffectiveCost = EffectiveCost, EffectiveCost = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), EffectiveCost) + | extend old_x_EffectiveCostInUsd = x_EffectiveCostInUsd, x_EffectiveCostInUsd = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), x_EffectiveCostInUsd) + // + // Clean up resource columns + | extend old_ResourceId = ResourceId, ResourceId = case( + isnotempty(ResourceId), ResourceId, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), CommitmentDiscountId, + ResourceId + ) + | extend old_ResourceName = ResourceName, ResourceName = tolower(case( + isnotempty(ResourceName), ResourceName, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountName), CommitmentDiscountName, + isnotempty(ResourceId), parse_resourceid(ResourceId).ResourceName, + ResourceName + )) + | extend old_x_ResourceType = x_ResourceType, x_ResourceType = case( + isnotempty(x_ResourceType), x_ResourceType, + isnotempty(ResourceId), parse_resourceid(ResourceId).x_ResourceType, + x_ResourceType + ) + | extend old_ResourceType = ResourceType, ResourceType = case( + // Use existing resource type display name unless it's an internal resource type ID + isnotempty(ResourceType) and tolower(ResourceType) != tolower(x_ResourceType) and ResourceType !contains '/', ResourceType, + // Use CommitmentDiscountType for commitment discount purchases + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountType), CommitmentDiscountType, + // Look up display name from internal type + isnotempty(x_ResourceType), coalesce(tostring(resource_type(x_ResourceType).SingularDisplayName), ResourceType, x_ResourceType), + ResourceType + ) + // + // Handle missing values + | extend old_PublisherName = PublisherName, PublisherName = case(PublisherName == 'Microsoft Corporation', 'Microsoft', isnotempty(PublisherName), PublisherName, x_PublisherCategory == 'Cloud Provider', ProviderName, '') + // + // Handle FOCUS 1.0-preview Region column + | extend old_Region = Region + | extend old_RegionId = RegionId, RegionId = coalesce(RegionId, iff(ProviderName == 'Microsoft', replace_string(tolower(Region), ' ', ''), Region)) + | extend RegionName = coalesce(RegionName, Region) + // + // SKU properties + | extend x_SkuCoreCount = toint(coalesce(SkuPriceDetails.CoreCount, SkuPriceDetails.x_VCPUs, x_SkuDetails.VCPUs, SkuPriceDetails.x_VCores, x_SkuDetails.VCores, SkuPriceDetails.x_vCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(SkuPriceDetails.InstanceType, SkuPriceDetails.x_ServiceType, x_SkuDetails.ServiceType, SkuPriceDetails.x_ServerSku, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + isnotempty(SkuPriceDetails.OperatingSystem), SkuPriceDetails.OperatingSystem, + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Canonical', 'Linux', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend SkuPriceDetails = case( + // FOCUS 1.3 + isnotempty(SkuPriceDetails), SkuPriceDetails, + // FOCUS 1.0-preview, 1.0 + parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // + // Azure Hybrid Benefit + | extend tmp_SqlAhb = tolower(coalesce(x_SkuDetails.AHB, SkuPriceDetails.x_AHB)) + | extend x_SkuLicenseType = case( + ChargeCategory != 'Usage', '', + x_SkuMeterCategory in ('Virtual Machines', 'Virtual Machine Licenses') and (x_SkuMeterSubcategory contains 'Windows' or coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL'), 'Windows Server', + isnotempty(tmp_SqlAhb) or x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isempty(x_SkuLicenseType), '', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL' or tmp_SqlAhb == 'true' or x_SkuMeterSubcategory contains 'Azure Hybrid Benefit', 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not Enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount) or isempty(x_SkuLicenseType), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + // + // Savings + | extend x_CommitmentDiscountSavings = iff(isempty(ContractedCost) or ContractedCost == 0 or ContractedCost - EffectiveCost < 0.0001, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - ContractedCost < 0.0001, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - EffectiveCost < 0.0001, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(isempty(ContractedUnitPrice) or ContractedUnitPrice == 0 or ContractedUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - ContractedUnitPrice < 0.0001, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // + // Minor fixes + | extend old_BillingPeriodEnd = BillingPeriodEnd, BillingPeriodEnd = startofmonth(BillingPeriodEnd) + | extend old_BillingPeriodStart = BillingPeriodStart, BillingPeriodStart = startofmonth(BillingPeriodStart) + // + // Sort columns and apply final transforms + | project + AllocatedMethodDetails = parse_json(AllocatedMethodDetails), // FOCUS 1.3+ + AllocatedMethodId, // FOCUS 1.3+ + AllocatedResourceId, // FOCUS 1.3+ + AllocatedResourceName, // FOCUS 1.3+ + AllocatedTags = parse_json(AllocatedTags), // FOCUS 1.3+ + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), + ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), + EffectiveCost, + HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceId = coalesce(InvoiceId, x_InvoiceId), + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName = iff(isempty(ServiceProviderName), ProviderName, ServiceProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat + ServiceSubcategory, // TODO: Populate ServiceSubcategory from ServiceName when missing + SkuId, + SkuMeter = coalesce(SkuMeter, x_SkuMeterName), + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName = iff(isempty(SubAccountId), '', SubAccountName), + SubAccountType, + Tags, + x_AccountId = iff(x_AccountId == '-2', '', x_AccountId), + x_AccountName = iff(x_AccountId == '-2', '', x_AccountName), + x_AccountOwnerId = iff(x_AccountId == '-2', '', x_AccountOwnerId), + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement = case( + ProviderName == 'Microsoft' and x_BillingAccountId == x_BillingProfileId, 'EA', + ProviderName == 'Microsoft' and x_BillingAccountId != x_BillingProfileId, 'MCA', + ProviderName + ), + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility = '', // TODO: Add x_CommitmentDiscountSpendEligibility for Costs + x_CommitmentDiscountUsageEligibility = '', // TODO: Add x_CommitmentDiscountUsageEligibility for Costs + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd = coalesce(x_ContractedCostInUsd, x_OnDemandCostInUsd), + x_CostAllocationRuleName, + x_CostCategories = parse_json(x_CostCategories), + x_CostCenter, + x_CostType, + x_Credits = parse_json(x_Credits), + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount = parse_json(x_Discount), + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId = case( + x_InvoiceSectionId == '-2', '', + x_InvoiceSectionId + ), + x_InvoiceSectionName = case( + x_InvoiceSectionName == 'Unassigned', '', + x_InvoiceSectionName + ), + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription = iff(x_PricingUnitDescription == 'Unassigned', '', x_PricingUnitDescription), + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName = tolower(x_ResourceGroupName), + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, // TODO: Populate from ServiceName when missing + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues = bag_merge( + checkString('BillingPeriodEnd', old_BillingPeriodEnd, BillingPeriodEnd), + checkString('BillingPeriodStart', old_BillingPeriodStart, BillingPeriodStart), + checkString('CapacityReservationStatus', old_CapacityReservationStatus, CapacityReservationStatus), + checkString('ChargeCategory', old_ChargeCategory, ChargeCategory), + checkString('ChargeClass', old_ChargeClass, ChargeClass), + checkString('ChargeSubcategory', old_ChargeSubcategory, ''), // Not included in final schema; use empty string + checkString('ChargeFrequency', old_ChargeFrequency, ChargeFrequency), + checkReal('CommitmentDiscountQuantity', old_CommitmentDiscountQuantity, CommitmentDiscountQuantity), + checkString('CommitmentDiscountUnit', old_CommitmentDiscountUnit, CommitmentDiscountUnit), + checkString('CommitmentDiscountStatus', old_CommitmentDiscountStatus, CommitmentDiscountStatus), + checkReal('ConsumedQuantity', old_ConsumedQuantity, ConsumedQuantity), + checkString('ConsumedUnit', old_ConsumedUnit, ConsumedUnit), + checkReal('ContractedCost', old_ContractedCost, ContractedCost), + checkReal('ContractedUnitPrice', old_ContractedUnitPrice, ContractedUnitPrice), + checkReal('EffectiveCost', old_EffectiveCost, EffectiveCost), + checkReal('ListCost', old_ListCost, ListCost), + checkReal('ListUnitPrice', old_ListUnitPrice, ListUnitPrice), + checkString('PricingCategory', old_PricingCategory, PricingCategory), + checkReal('PricingQuantity', old_PricingQuantity, PricingQuantity), + checkString('ProviderName', old_ProviderName, ProviderName), + checkString('PublisherName', old_PublisherName, PublisherName), + checkString('Region', old_Region, ''), // Not included in final schema; use empty string + checkString('RegionId', old_RegionId, RegionId), + checkString('ResourceId', old_ResourceId, ResourceId), + checkString('ResourceName', old_ResourceName, ResourceName), + checkString('ResourceType', old_ResourceType, ResourceType), + checkString('x_AmortizationClass', old_x_AmortizationClass, x_AmortizationClass), + checkReal('x_EffectiveCostInUsd', old_x_EffectiveCostInUsd, x_EffectiveCostInUsd), + checkReal('x_EffectiveUnitPrice', old_x_EffectiveUnitPrice, x_EffectiveUnitPrice), + checkString('x_ResourceType', old_x_ResourceType, x_ResourceType) + ), + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + +// Costs_final_v1_3 table +.create-merge table Costs_final_v1_3 ( + AllocatedMethodDetails: dynamic, // FOCUS 1.3+ + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: dynamic, // FOCUS 1.3+ + AvailabilityZone: string, + BilledCost: real, + BillingAccountId: string, + BillingAccountName: string, + BillingAccountType: string, + BillingCurrency: string, + BillingPeriodEnd: datetime, + BillingPeriodStart: datetime, + CapacityReservationId: string, + CapacityReservationStatus: string, + ChargeCategory: string, + ChargeClass: string, + ChargeDescription: string, + ChargeFrequency: string, + ChargePeriodEnd: datetime, + ChargePeriodStart: datetime, + CommitmentDiscountCategory: string, + CommitmentDiscountId: string, + CommitmentDiscountName: string, + CommitmentDiscountQuantity: real, + CommitmentDiscountStatus: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceId: string, + InvoiceIssuerName: string, + ListCost: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, + PricingQuantity: real, + PricingUnit: string, + ProviderName: string, + PublisherName: string, + RegionId: string, + RegionName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + ServiceCategory: string, + ServiceName: string, + ServiceProviderName: string, // FOCUS 1.3+ + ServiceSubcategory: string, + SkuId: string, + SkuMeter: string, + SkuPriceDetails: dynamic, + SkuPriceId: string, + SubAccountId: string, + SubAccountName: string, + SubAccountType: string, + Tags: dynamic, + x_AccountId: string, // Azure 1.0-preview(v1)+ + x_AccountName: string, // Azure 1.0-preview(v1)+ + x_AccountOwnerId: string, // Azure 1.0-preview(v1)+ + x_AmortizationClass: string, // Azure 1.2-preview+ + x_BilledCostInUsd: real, // Azure 1.0-preview(v1)+ + x_BilledUnitPrice: real, // Azure 1.0-preview(v1)+ + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure 1.0-preview(v1)+ + x_BillingAccountName: string, // Azure 1.0-preview(v1)+ + x_BillingExchangeRate: real, // Azure 1.0-preview(v1)+ + x_BillingExchangeRateDate: datetime, // Azure 1.0-preview(v1)+ + x_BillingItemCode: string, // Alibaba 1.0 + x_BillingItemName: string, // Alibaba 1.0 + x_BillingProfileId: string, // Azure 1.0-preview(v1)+ + x_BillingProfileName: string, // Azure 1.0-preview(v1)+ + x_ChargeId: string, // Azure 1.0-preview(v1) only + x_CommitmentDiscountNormalizedRatio: real, // Azure 1.2-preview+ + x_CommitmentDiscountPercent: real, // Hubs add-on + x_CommitmentDiscountSavings: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_CommitmentDiscountUtilizationAmount: real, // Hubs add-on + x_CommitmentDiscountUtilizationPotential: real, // Hubs add-on + x_CommodityCode: string, // Alibaba 1.0 + x_CommodityName: string, // Alibaba 1.0 + x_ComponentName: string, // Tencent 1.0 + x_ComponentType: string, // Tencent 1.0 + x_ConsumedCoreHours: real, // Hubs add-on + x_ContractedCostInUsd: real, // Azure 1.0+ + x_CostAllocationRuleName: string, // Azure 1.0-preview(v1)+ + x_CostCategories: dynamic, // AWS 1.0 (JSON) + x_CostCenter: string, // Azure 1.0-preview(v1)+ + x_CostType: string, // GCP Jan 2024 + x_Credits: dynamic, // GCP Jan 2024 + x_CurrencyConversionRate: real, // GCP Jun 2024 + x_CustomerId: string, // Azure 1.0-preview(v1)+ + x_CustomerName: string, // Azure 1.0-preview(v1)+ + x_Discount: dynamic, // AWS 1.0 (JSON) + x_EffectiveCostInUsd: real, // Azure 1.0-preview(v1)+ + x_EffectiveUnitPrice: real, // Azure 1.0-preview(v1)+ + x_ExportTime: datetime, // GCP Jan 2024 / Tencent 1.0 + x_IngestionTime: datetime, // Hubs add-on + x_InstanceID: string, // Alibaba 1.0 + x_InvoiceIssuerId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionName: string, // Azure 1.0-preview(v1)+ + x_ListCostInUsd: real, // Azure 1.0-preview(v1)+ + x_Location: string, // GCP Jan 2024 + x_NegotiatedDiscountPercent:real, // Hubs add-on + x_NegotiatedDiscountSavings:real, // Hubs add-on + x_Operation: string, // AWS 1.0 + x_OwnerAccountID: string, // Tencent 1.0 + x_PartnerCreditApplied: string, // Azure 1.0-preview(v1)+ + x_PartnerCreditRate: string, // Azure 1.0-preview(v1)+ + x_PricingBlockSize: real, // Azure 1.0-preview(v1)+ + x_PricingSubcategory: string, // Azure 1.0-preview(v1)+ + x_PricingUnitDescription: string, // Azure 1.0-preview(v1)+ + x_Project: string, // GCP Jan 2024 + x_PublisherCategory: string, // Azure 1.0-preview(v1)+ + x_PublisherId: string, // Azure 1.0-preview(v1)+ + x_ResellerId: string, // Azure 1.0-preview(v1)+ + x_ResellerName: string, // Azure 1.0-preview(v1)+ + x_ResourceGroupName: string, // Azure 1.0-preview(v1)+ + x_ResourceType: string, // Azure 1.0-preview(v1)+ + x_ServiceCode: string, // AWS 1.0 + x_ServiceId: string, // GCP Jan 2024 + x_ServiceModel: string, // Azure 1.2-preview+ + x_ServicePeriodEnd: datetime, // Azure 1.0-preview(v1)+ + x_ServicePeriodStart: datetime, // Azure 1.0-preview(v1)+ + x_SkuCoreCount: int, // Hubs add-on + x_SkuDescription: string, // Azure 1.0-preview(v1)+ + x_SkuDetails: dynamic, // Azure 1.0-preview(v1)+ + x_SkuInstanceType: string, // Hubs add-on + x_SkuIsCreditEligible: bool, // Azure 1.0-preview(v1)+ + x_SkuLicenseQuantity: int, // Hubs add-on + x_SkuLicenseStatus: string, // Hubs add-on + x_SkuLicenseType: string, // Hubs add-on + x_SkuLicenseUnit: string, // Hubs add-on + x_SkuMeterCategory: string, // Azure 1.0-preview(v1)+ + x_SkuMeterId: string, // Azure 1.0-preview(v1)+ + x_SkuMeterSubcategory: string, // Azure 1.0-preview(v1)+ + x_SkuOfferId: string, // Azure 1.0-preview(v1)+ + x_SkuOperatingSystem: string, // Hubs add-on + x_SkuOrderId: string, // Azure 1.0-preview(v1)+ + x_SkuOrderName: string, // Azure 1.0-preview(v1)+ + x_SkuPartNumber: string, // Azure 1.0-preview(v1)+ + x_SkuPlanName: string, // Azure 1.2-preview+ + x_SkuRegion: string, // Azure 1.0-preview(v1)+ + x_SkuServiceFamily: string, // Azure 1.0-preview(v1)+ + x_SkuTerm: int, // Azure 1.0-preview(v1)+ + x_SkuTier: string, // Azure 1.0-preview(v1)+ + x_SourceChanges: string, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceValues: dynamic, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubproductName: string, // Tencent 1.0 + x_TotalDiscountPercent: real, // Hubs add-on + x_TotalSavings: real, // Hubs add-on + x_UsageType: string // AWS 1.0 +) + +// Update policy for Costs_raw -> Costs_final_v1_3 table +.alter table Costs_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Costs_raw", + "Query": "Costs_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Actual costs |=================================================================================================== +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// ActualCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +ActualCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + ActualCosts_raw + | where ChargeType in ('Purchase', 'Refund') and isnotempty(ReservationId) + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for ActualCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "ActualCosts_raw", + "Query": "ActualCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Amortized costs |================================================================================================ +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// AmortizedCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +AmortizedCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + AmortizedCosts_raw + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for AmortizedCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "AmortizedCosts_raw", + "Query": "AmortizedCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| CommitmentDiscountUsage |======================================================================================== +// Supported versions: +// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea +// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca +//====================================================================================================================== + +// CommitmentDiscountUsage_transform_v1_3 function +.create-or-alter function +with (docstring='All commitment discount usage transformed to FOCUS 1.3. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + CommitmentDiscountUsage_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Handle resource columns + | extend ResourceId = tolower(InstanceId) + | extend tmp_ResourceDetails = parse_resourceid(ResourceId) + | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) + | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) + | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) + | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) + | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + // + // Sort columns and apply final transforms + | project + ChargePeriodEnd = UsageDate + 1d, + ChargePeriodStart = UsageDate, + CommitmentDiscountCategory = 'Usage', + CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), + CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, + CommitmentDiscountType = 'Reservation', + CommitmentDiscountUnit = case( + InstanceFlexibilityRatio == 1, 'Hours', + InstanceFlexibilityRatio != 1, 'Normalized Hours', + '' + ), + ConsumedQuantity = UsedHours, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount = TotalReservedQuantity, + x_CommitmentDiscountCommittedAmount = ReservedHours, + // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, + x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), + x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, + x_IngestionTime = ingestion_time(), + x_ResourceGroupName, + x_ResourceType, + // x_RowId = hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // CommitmentDiscountId, + // ResourceId, + // ChargePeriodStart + // )), + x_ServiceModel, + x_SkuOrderId = ReservationOrderId, + x_SkuSize = iff(SkuName == 'NA', '', SkuName), + x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), + x_SourceProvider = coalesce(x_SourceProvider, ProviderName), + x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), + x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) +} + +// CommitmentDiscountUsage_final_v1_3 table +.create-merge table CommitmentDiscountUsage_final_v1_3 ( + ChargePeriodEnd: datetime, // Hubs add-on + ChargePeriodStart: datetime, // MS 2023-03-01 + CommitmentDiscountCategory: string, // Hubs add-on + CommitmentDiscountId: string, // MS 2023-03-01 + CommitmentDiscountQuantity: real, // MS 2023-03-01 + CommitmentDiscountType: string, // Hubs add-on + CommitmentDiscountUnit: string, // Hubs add-on + ConsumedQuantity: real, // MS 2023-03-01 + ProviderName: string, // Hubs add-on + ResourceId: string, // MS 2023-03-01 + ResourceName: string, // Hubs add-on + ResourceType: string, // Hubs add-on + ServiceCategory: string, // Hubs add-on + ServiceName: string, // Hubs add-on + ServiceSubcategory: string, // Hubs add-on + SubAccountId: string, // Hubs add-on + x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 + x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 + x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 + x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 + x_IngestionTime: datetime, // Hubs add-on + x_ResourceGroupName: string, // Hubs add-on + x_ResourceType: string, // Hubs add-on + x_ServiceModel: string, // Hubs add-on + x_SkuOrderId: string, // MS 2023-03-01 + x_SkuSize: string, // MS 2023-03-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_3 table +.alter table CommitmentDiscountUsage_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "CommitmentDiscountUsage_raw", + "Query": "CommitmentDiscountUsage_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Recommendations |================================================================================================ +// Supported datasets/versions: +// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea +// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca +//====================================================================================================================== + +// Recommendations_transform_v1_3 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.3.', folder='Recommendations') +Recommendations_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + | extend x_IngestionTime = ingestion_time() + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Convert JSON cost columns to real + | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) + | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) + | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) + // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // + // Build recommendation details + | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location + | extend x_RecommendationDetails = case( + // Use incoming x_RecommendationDetails first + isnotempty(x_RecommendationDetails), x_RecommendationDetails, + // Create one for reservation recommendations if needed + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', Scope, + 'LookbackPeriodDuration', case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), + '' + ), + 'LookbackPeriodStart', FirstUsageDate, + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'RegionName', RegionName, + 'SkuMeterId', MeterId, + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future + | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) + | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) + // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // + | project + ProviderName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations + ResourceType, + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, + x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), + x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), + x_IngestionTime, + x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationDate, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), + x_RecommendationDetails, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_final_v1_3 table +.create-merge table Recommendations_final_v1_3 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: real, + x_EffectiveCostBefore: real, + x_EffectiveCostSavings: real, + x_IngestionTime: datetime, + x_RecommendationCategory: string, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_RecommendationId: string, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Update policy for Recommendations_raw -> Recommendations_final_v1_3 table +.alter table Recommendations_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Transactions |=================================================================================================== +// Supported versions: +// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea +// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca +//====================================================================================================================== + +// Transactions_transform_v1_3 function +.create-or-alter function +with (docstring='All transactions transformed to FOCUS 1.3.', folder='Transactions') +Transactions_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Transactions_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Handle BillingPeriodStart/End + | extend BillingMonth = tostring(BillingMonth) + | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) + | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) + // + // Sort columns and apply final transforms + | project + BilledCost = Amount, + BillingAccountId = case( + BillingProfileId startswith '/', BillingProfileId, + isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), + isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), + '' + ), + BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), + BillingCurrency = Currency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory = case( + EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', + 'Adjustment' + ), + ChargeClass = case( + EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario + EventType == 'Refund', 'Correction', + '' + ), + ChargeDescription = Description, + ChargeFrequency = case( + BillingFrequency == 'OneTime', 'One-Time', + BillingFrequency == 'Recurring', 'Recurring', + BillingFrequency + ), + ChargePeriodStart = EventDate, + InvoiceId, + PricingQuantity = Quantity, + PricingUnit = 'Reservations', + ProviderName, + RegionId = Region, + RegionName = Region, + SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), + SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerEmail, + x_CostCenter = CostCenter, + x_InvoiceNumber = Invoice, + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), + x_IngestionTime = ingestion_time(), + x_MonetaryCommitment = MonetaryCommitment, + x_Overage = Overage, + x_PurchasingBillingAccountId = PurchasingEnrollment, + x_SkuOrderId = ReservationOrderId, + x_SkuOrderName = ReservationOrderName, + x_SkuSize = ArmSkuName, + x_SkuTerm = isoMonths(Term), + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId = PurchasingSubscriptionGuid, + x_TransactionType = EventType +} + +// Transactions_final_v1_3 table +.create-merge table Transactions_final_v1_3 ( + BilledCost: real, // MS CM EA+MCA 2023-05-01 + BillingAccountId: string, // MS CM EA+MCA 2023-05-01 + BillingAccountName: string, // MS CM EA+MCA 2023-05-01 + BillingCurrency: string, // MS CM EA+MCA 2023-05-01 + BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 + BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + ChargeCategory: string, // Hubs add-on + ChargeClass: string, // Hubs add-on + ChargeDescription: string, // MS CM EA+MCA 2023-05-01 + ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 + ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + InvoiceId: string, // MS CM MCA 2023-05-01 + PricingQuantity: real, // MS CM EA+MCA 2023-05-01 + PricingUnit: string, // Hubs add-on + ProviderName: string, // Hubs add-on + RegionId: string, // MS CM EA+MCA 2023-05-01 + RegionName: string, // MS CM EA+MCA 2023-05-01 + SubAccountId: string, // MS CM EA+MCA 2023-05-01 + SubAccountName: string, // MS CM EA+MCA 2023-05-01 + x_AccountName: string, // MS CM EA 2023-05-01 + x_AccountOwnerId: string, // MS CM EA 2023-05-01 + x_CostCenter: string, // MS CM EA 2023-05-01 + x_InvoiceNumber: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 + x_IngestionTime: datetime, // Hubs add-on + x_MonetaryCommitment: real, // MS CM EA 2023-05-01 + x_Overage: real, // MS CM EA 2023-05-01 + x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 + x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 + x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 + x_SkuSize: string, // MS CM EA+MCA 2023-05-01 + x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 + x_TransactionType: string // MS CM EA+MCA 2023-05-01 +) + +// Update policy for Transactions_raw -> Transactions_final_v1_3 table +.alter table Transactions_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Transactions_raw", + "Query": "Transactions_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` From f70c43d88579087110b353769e8b63c18d98b837 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:21:44 -0700 Subject: [PATCH 02/19] Add FOCUS 1.3 hub functions (#2127) Adds HubSetup_v1_3.kql with Costs_v1_3, Prices_v1_3, CommitmentDiscountUsage_v1_3, Recommendations_v1_3, and Transactions_v1_3 that union the new Costs_final_v1_3 with the existing Costs_final_v1_2 and Costs_final_v1_0 tables. For v1_2 data unioned into the v1_3 view, the 8 new FOCUS 1.3 columns default to null/empty and ServiceProviderName/HostProviderName are populated from the deprecated ProviderName/PublisherName for back compat. The same defaults apply to the v1_0 union arm on top of the existing v1_0 -> v1_2 conversion. HubSetup_Latest.kql aliases now point to *_v1_3 functions so the unversioned Costs(), Prices(), etc. return the latest schema. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 1 + .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../Analytics/scripts/HubSetup_Latest.kql | 10 +- .../Analytics/scripts/HubSetup_v1_3.kql | 618 ++++++++++++++++++ 4 files changed, 625 insertions(+), 5 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 0963fb494..4eff22919 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -44,6 +44,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 42cd2738b..ebf106beb 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -414,6 +414,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { scripts: { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') + v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 8de86a1aa..1895b39db 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -13,7 +13,7 @@ with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_2() + CommitmentDiscountUsage_v1_3() } @@ -21,7 +21,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_2() + Costs_v1_3() } @@ -29,7 +29,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_2() + Prices_v1_3() } @@ -37,7 +37,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_2() + Recommendations_v1_3() } @@ -45,5 +45,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_2() + Transactions_v1_3() } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql new file mode 100644 index 000000000..62995ffa5 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql @@ -0,0 +1,618 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Hub database / FOCUS 1.3 functions +// Used for reporting with backward compatibility. +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + + +// CommitmentDiscountUsage_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.3.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_3() +{ + database('Ingestion').CommitmentDiscountUsage_final_v1_3 + | union ( + database('Ingestion').CommitmentDiscountUsage_final_v1_0 + // Convert decimal to real + | extend + ConsumedQuantity = toreal(ConsumedQuantity), + x_CommitmentDiscountCommittedCount = toreal(x_CommitmentDiscountCommittedCount), + x_CommitmentDiscountCommittedAmount = toreal(x_CommitmentDiscountCommittedAmount), + x_CommitmentDiscountNormalizedRatio = toreal(x_CommitmentDiscountNormalizedRatio) + // Add new columns + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceSubcategory) on x_ResourceType + | extend CommitmentDiscountQuantity = ConsumedQuantity * x_CommitmentDiscountNormalizedRatio + | extend CommitmentDiscountUnit = case( + x_CommitmentDiscountNormalizedRatio == 1, 'Hours', + x_CommitmentDiscountNormalizedRatio > 1, 'Normalized Hours', + '' + ) + ) + | project + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountQuantity, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount, + x_CommitmentDiscountCommittedAmount, + x_CommitmentDiscountNormalizedGroup, + x_CommitmentDiscountNormalizedRatio, + x_IngestionTime, + x_ResourceGroupName, + x_ResourceType, + x_ServiceModel, + x_SkuOrderId, + x_SkuSize, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Costs_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') +Costs_v1_3() +{ + database('Ingestion').Costs_final_v1_3 + | union ( + database('Ingestion').Costs_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + ConsumedQuantity = toreal(ConsumedQuantity), + ContractedCost = toreal(ContractedCost), + ContractedUnitPrice = toreal(ContractedUnitPrice), + EffectiveCost = toreal(EffectiveCost), + ListCost = toreal(ListCost), + ListUnitPrice = toreal(ListUnitPrice), + PricingQuantity = toreal(PricingQuantity), + x_BilledCostInUsd = toreal(x_BilledCostInUsd), + x_BilledUnitPrice = toreal(x_BilledUnitPrice), + x_BillingExchangeRate = toreal(x_BillingExchangeRate), + x_ContractedCostInUsd = toreal(x_ContractedCostInUsd), + x_CurrencyConversionRate = toreal(x_CurrencyConversionRate), + x_EffectiveCostInUsd = toreal(x_EffectiveCostInUsd), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_ListCostInUsd = toreal(x_ListCostInUsd), + x_PricingBlockSize = toreal(x_PricingBlockSize) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId, + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + // Add new columns + | lookup kind=leftouter (Services | where isnotempty(x_ResourceType) | summarize take_any(ServiceSubcategory), take_any(x_ServiceModel) by x_ResourceType) on x_ResourceType + | extend CapacityReservationId = tostring(x_SkuDetails.VMCapacityReservationId) + | extend CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + | extend x_CommitmentDiscountNormalizedRatio = case( + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, dynamic(1))) + ) + | extend CommitmentDiscountQuantity = case( + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend CommitmentDiscountUnit = case( + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend x_AmortizationClass = case( + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + // Hubs add-ons + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + | extend x_SkuCoreCount = toint(coalesce(x_SkuDetails.VCPUs, x_SkuDetails.VCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(x_SkuDetails.ServiceType, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + x_SkuDetails.ImageType == 'Canonical', 'Linux', + x_SkuDetails.ImageType == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + x_SkuDetails.ImageType + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend tmp_SqlAhb = tolower(x_SkuDetails.AHB) + | extend x_SkuLicenseType = case( + x_SkuDetails.ImageType contains 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isnotempty(x_SkuLicenseType) or tmp_SqlAhb == 'true' or (x_SkuMeterSubcategory contains 'Azure Hybrid Benefit'), 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + | extend x_CommitmentDiscountSavings = iff(ContractedCost < EffectiveCost, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(ListCost < ContractedCost, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(ListCost < EffectiveCost, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(ContractedUnitPrice == 0, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // SkuPriceDetails conversion -- Must be after hubs add-ons + | extend SkuPriceDetails = parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | union ( + database('Ingestion').Costs_final_v1_2 + // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) + | project + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied, + ContractedCost, + ContractedUnitPrice, + EffectiveCost, + HostProviderName, + InvoiceId, + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName, + ServiceSubcategory, + SkuId, + SkuMeter, + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName, + SubAccountType, + Tags, + x_AccountId, + x_AccountName, + x_AccountOwnerId, + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd, + x_CostAllocationRuleName, + x_CostCategories, + x_CostCenter, + x_CostType, + x_Credits, + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount, + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName, + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues, + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + + +// Prices_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all prices aligned to FOCUS 1.3.', folder = 'Prices') +Prices_v1_3() +{ + database('Ingestion').Prices_final_v1_3 + | union ( + database('Ingestion').Prices_final_v1_0 + // Convert decimal to real + | extend + ContractedUnitPrice = toreal(ContractedUnitPrice), + ListUnitPrice = toreal(ListUnitPrice), + x_BaseUnitPrice = toreal(x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = toreal(x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = toreal(x_ContractedUnitPriceDiscountPercent), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_EffectiveUnitPriceDiscount = toreal(x_EffectiveUnitPriceDiscount), + x_EffectiveUnitPriceDiscountPercent = toreal(x_EffectiveUnitPriceDiscountPercent), + x_PricingBlockSize = toreal(x_PricingBlockSize), + x_SkuIncludedQuantity = toreal(x_SkuIncludedQuantity), + x_SkuTier = toreal(x_SkuTier), + x_TotalUnitPriceDiscount = toreal(x_TotalUnitPriceDiscount), + x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) + // Rename columns + | project-rename + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + ) + | project + BillingAccountId, + BillingAccountName, + BillingCurrency, + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType, + CommitmentDiscountUnit, + ContractedUnitPrice, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingUnit, + SkuId, + SkuMeter, + SkuPriceId, + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent, + x_EffectivePeriodEnd, + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_SkuDescription, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent +} + + +// Recommendations_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all recommendations aligned to FOCUS 1.3.', folder = 'Recommendations') +Recommendations_v1_3() +{ + database('Ingestion').Recommendations_final_v1_3 + | union ( + database('Ingestion').Recommendations_final_v1_0 + // Convert decimal to real + | extend + x_EffectiveCostAfter = toreal(x_EffectiveCostAfter), + x_EffectiveCostBefore = toreal(x_EffectiveCostBefore), + x_EffectiveCostSavings = toreal(x_EffectiveCostSavings) + ) + | project + ProviderName, + ResourceId, + ResourceName, + ResourceType, + SubAccountId, + SubAccountName, + x_EffectiveCostAfter, + x_EffectiveCostBefore, + x_EffectiveCostSavings, + x_IngestionTime, + x_RecommendationCategory, + x_RecommendationDate, + x_RecommendationDescription, + x_RecommendationDetails, + x_RecommendationId, + x_ResourceGroupName, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Transactions_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all transactions aligned to FOCUS 1.3.', folder = 'Transactions') +Transactions_v1_3() +{ + database('Ingestion').Transactions_final_v1_3 + | union ( + database('Ingestion').Transactions_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + PricingQuantity = toreal(PricingQuantity), + x_MonetaryCommitment = toreal(x_MonetaryCommitment), + x_Overage = toreal(x_Overage) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId + ) + | project + BilledCost, + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodStart, + InvoiceId, + PricingQuantity, + PricingUnit, + ProviderName, + RegionId, + RegionName, + SubAccountId, + SubAccountName, + x_AccountName, + x_AccountOwnerId, + x_CostCenter, + x_InvoiceNumber, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_IngestionTime, + x_MonetaryCommitment, + x_Overage, + x_PurchasingBillingAccountId, + x_SkuOrderId, + x_SkuOrderName, + x_SkuSize, + x_SkuTerm, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId, + x_TransactionType +} + + +//====================================================================================================================== +// Latest FOCUS version +//====================================================================================================================== + +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage() +{ + CommitmentDiscountUsage_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') +Costs() +{ + Costs_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') +Prices() +{ + Prices_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') +Recommendations() +{ + Recommendations_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') +Transactions() +{ + Transactions_v1_3() +} From 2bfa61179c5f33bcb39c282c183eb32aca39f4ba Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:35:24 -0700 Subject: [PATCH 03/19] Add ContractCommitment dataset (#2129) Adds the FOCUS 1.3 Contract Commitment supplemental dataset: - ContractCommitment_raw table in IngestionSetup_RawTables.kql with the 14 mandatory FOCUS 1.3 columns plus standard hubs source metadata. - ContractCommitment_transform_v1_3 + ContractCommitment_final_v1_3 in IngestionSetup_v1_3.kql, with update policy mapping raw to final. - ContractCommitment_v1_3 hub function in HubSetup_v1_3.kql that reads the final table directly (no older versions to union). - Unversioned ContractCommitment alias in HubSetup_Latest.kql. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Analytics/scripts/HubSetup_Latest.kql | 8 +++ .../Analytics/scripts/HubSetup_v1_3.kql | 9 +++ .../scripts/IngestionSetup_RawTables.kql | 68 ++++++++++++++++++ .../Analytics/scripts/IngestionSetup_v1_3.kql | 69 +++++++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 1895b39db..26012ec5c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -17,6 +17,14 @@ CommitmentDiscountUsage() } +.create-or-alter function +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') +ContractCommitment() +{ + ContractCommitment_v1_3() +} + + .create-or-alter function with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql index 62995ffa5..c1fd9d363 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql @@ -66,6 +66,15 @@ CommitmentDiscountUsage_v1_3() } +// ContractCommitment_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.3.', folder = 'ContractCommitment') +ContractCommitment_v1_3() +{ + database('Ingestion').ContractCommitment_final_v1_3 +} + + // Costs_final_v1_3 .create-or-alter function with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 83a3be40e..ca500e58d 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -335,6 +335,74 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// ContractCommitment_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Redefine all columns +.alter table ContractCommitment_raw ( + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + x_SourceName: string, // Hubs v1_3+ + x_SourceProvider: string, // Hubs v1_3+ + x_SourceType: string, // Hubs v1_3+ + x_SourceVersion: string // Hubs v1_3+ +) + +// ContractCommitment_raw ingestion mapping +.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +``` +[ + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// ContractCommitment_raw retention policy (clear historical data) +.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled + +// ContractCommitment_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable ContractCommitment_raw streaming ingestion (required for Fabric) +.alter table ContractCommitment_raw policy streamingingestion disable + + //===| Costs |========================================================================================================== // Supported versions: // - MS: 1.0, 1.0-preview(v1) -- See https://aka.ms/costmgmt/exports/focus diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql index 70d29a2ed..b7b051e18 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql @@ -1663,6 +1663,75 @@ CommitmentDiscountUsage_transform_v1_3() ``` +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitment_transform_v1_3 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.3.', folder='Contract commitments') +ContractCommitment_transform_v1_3() +{ + ContractCommitment_raw + | project + BillingCurrency, + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + InvoiceIssuerName, + PricingCurrency, + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitment_final_v1_3 table +.create-merge table ContractCommitment_final_v1_3 ( + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_3 table +.alter table ContractCommitment_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitment_raw", + "Query": "ContractCommitment_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + //===| Recommendations |================================================================================================ // Supported datasets/versions: // - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea From e489389400e35c841cb4b108afbce568cd8d1c71 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:39:32 -0700 Subject: [PATCH 04/19] Add FOCUS 1.4-preview ingestion (#2131) Adds IngestionSetup_v1_4.kql as a preview schema for FOCUS 1.4 working_draft. Cost and Usage drops the deprecated ProviderName and PublisherName columns (removed in 1.4 per spec). Contract Commitment expands by 14 new columns: BenefitCategory, ContractCommitmentApplicability (JSON), Created, DiscountPercentage, DurationType, FulfillmentInterval, LastUpdated, LifecycleStatus, Model, OfferCategory, PaymentInterval, PaymentModel, PaymentUpfrontPercentage, PricingCurrencyContractCommitmentCost. The internal ProviderName/PublisherName fix-up logic in the Costs transform stays so HostProviderName / ServiceProviderName can fall back to those values when source data does not yet supply them. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 3 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../scripts/IngestionSetup_RawTables.kql | 100 +- .../Analytics/scripts/IngestionSetup_v1_4.kql | 2061 +++++++++++++++++ 4 files changed, 2128 insertions(+), 37 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 4eff22919..431f540b8 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -34,7 +34,8 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index ebf106beb..543b7c455 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -379,6 +379,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') + v1_4: loadTextContent('scripts/IngestionSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index ca500e58d..cdc281659 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -348,48 +348,76 @@ // ContractCommitment_raw table -- Redefine all columns .alter table ContractCommitment_raw ( - BillingCurrency: string, // FOCUS 1.3+ - ContractCommitmentCategory: string, // FOCUS 1.3+ - ContractCommitmentCost: real, // FOCUS 1.3+ - ContractCommitmentId: string, // FOCUS 1.3+ - ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ - ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ - ContractCommitmentQuantity: real, // FOCUS 1.3+ - ContractCommitmentType: string, // FOCUS 1.3+ - ContractCommitmentUnit: string, // FOCUS 1.3+ - ContractId: string, // FOCUS 1.3+ - ContractPeriodEnd: datetime, // FOCUS 1.3+ - ContractPeriodStart: datetime, // FOCUS 1.3+ - InvoiceIssuerName: string, // FOCUS 1.3+ - PricingCurrency: string, // FOCUS 1.3+ - x_SourceName: string, // Hubs v1_3+ - x_SourceProvider: string, // Hubs v1_3+ - x_SourceType: string, // Hubs v1_3+ - x_SourceVersion: string // Hubs v1_3+ + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_3+ + x_SourceProvider: string, // Hubs v1_3+ + x_SourceType: string, // Hubs v1_3+ + x_SourceVersion: string // Hubs v1_3+ ) // ContractCommitment_raw ingestion mapping .create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" ``` [ - { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, - { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, - { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, - { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, - { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, - { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, - { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, - { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, - { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, - { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, - { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, - { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, - { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, - { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, - { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, - { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, - { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, - { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } + { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentApplicability", "Properties": { "Field": "ContractCommitmentApplicability" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "Created", "Properties": { "Field": "Created" } }, + { "Column": "DiscountPercentage", "Properties": { "Field": "DiscountPercentage" } }, + { "Column": "DurationType", "Properties": { "Field": "DurationType" } }, + { "Column": "FulfillmentInterval", "Properties": { "Field": "FulfillmentInterval" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "LastUpdated", "Properties": { "Field": "LastUpdated" } }, + { "Column": "LifecycleStatus", "Properties": { "Field": "LifecycleStatus" } }, + { "Column": "Model", "Properties": { "Field": "Model" } }, + { "Column": "OfferCategory", "Properties": { "Field": "OfferCategory" } }, + { "Column": "PaymentInterval", "Properties": { "Field": "PaymentInterval" } }, + { "Column": "PaymentModel", "Properties": { "Field": "PaymentModel" } }, + { "Column": "PaymentUpfrontPercentage", "Properties": { "Field": "PaymentUpfrontPercentage" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "PricingCurrencyContractCommitmentCost", "Properties": { "Field": "PricingCurrencyContractCommitmentCost" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } ] ``` diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql new file mode 100644 index 000000000..9ff8edcd8 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -0,0 +1,2061 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database +// See known issues @ https://github.com/microsoft/finops-toolkit/issues/1111 +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +//===| Prices |========================================================================================================= +// Supported versions: +// - MS EA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-ea +// - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca +//====================================================================================================================== + +// Prices_transform_v1_4 function +.create-or-alter function +with (docstring='Transforms Prices_raw into FOCUS 1.4-preview.', folder='Prices') +Prices_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + let prices = materialize( + Prices_raw + | extend PricingCurrency = coalesce(Currency, CurrencyCode) // CurrencyCode last as a fallback only + | extend x_SkuId = coalesce(SkuId, SkuID) + | extend x_SkuMeterId = coalesce(MeterId, MeterID) + | extend x_SkuProductId = coalesce(ProductId, ProductID) + | extend x_SkuTerm = isoMonths(Term) + | project-rename + SkuMeter = MeterName, + x_BaseUnitPrice = BasePrice, + x_EffectivePeriodEnd = EffectiveEndDate, + x_EffectivePeriodStart = EffectiveStartDate, + x_PricingUnitDescription = UnitOfMeasure, + x_SkuIncludedQuantity = IncludedQuantity, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuMeterType = MeterType, + x_SkuOfferId = OfferID, + x_SkuPartNumber = PartNumber, + x_SkuPriceType = PriceType, + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTier = TierMinimumUnits + | extend ContractedUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', UnitPrice, real(null)) // UnitPrice for savings plan is not the on-demand unit price + | extend ListUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', MarketPrice, real(null)) // MarketPrice for savings plan is not the list price + | extend ChargeCategory = case( + x_SkuPriceType == 'Consumption', 'Usage', + x_SkuPriceType == 'ReservedInstance', 'Purchase', + x_SkuPriceType == 'SavingsPlan', 'Usage', // Savings plan prices are for committed usage, not the purchase + '' + ) + | extend SkuPriceIdv2 = strcat(case(x_SkuPriceType == 'Consumption', 'OD', x_SkuPriceType == 'ReservedInstance', 'RI', x_SkuPriceType == 'SavingsPlan', 'SP', 'XX'), substring(ChargeCategory, 0, 1), x_SkuTerm, '_', x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType, '_', x_SkuTier, x_SkuOfferId) + | extend x_BillingAccountId = iff(BillingAccountId startswith '/', split(BillingAccountId, '/')[-1], coalesce(BillingAccountId, EnrollmentNumber)) + | extend x_BillingProfileId = iff(BillingProfileId startswith '/', split(BillingProfileId, '/')[-1], coalesce(BillingProfileId, EnrollmentNumber)) + | extend tmp_SavingsPlanKey = strcat(x_SkuMeterId, x_SkuProductId, x_SkuId, x_SkuTier, x_SkuOfferId) + // + // Get latest ingested row based on the unique ID + | extend x_IngestionTime = ingestion_time() + ); + // + // Meters for reservations and savings plans to identify commitment eligibility + let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; + let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // + // Copy list/base/contracted prices from on-demand SKUs + prices + | where x_SkuPriceType == 'SavingsPlan' + // If we use join, specify the shuffle key + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) + | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) + | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) + | project-away ListUnitPrice1, ContractedUnitPrice1, x_BaseUnitPrice1, tmp_SavingsPlanKey + | union ((prices | where x_SkuPriceType != 'SavingsPlan')) + // + // Set CommitmentDiscountCategory for reuse + | extend CommitmentDiscountCategory = case( + x_SkuPriceType == 'ReservedInstance', 'Usage', + x_SkuPriceType == 'SavingsPlan', 'Spend', + '' + ) + // + // Calculate commitment discount eligibility + // TODO: Would a join be faster? + // TODO: Check this to ensure it's correct + | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + // + // TODO: Implement x_CommitmentDiscountNormalizedRatio + | extend x_CommitmentDiscountNormalizedRatio = real(null) + // + // Add PricingUnit and x_PricingBlockSize + // TODO: Compare join vs. lookup perf -- | join kind=leftouter (PricingUnits) on x_PricingUnitDescription | project-away x_PricingUnitDescription1 + | lookup kind=leftouter (PricingUnits) on x_PricingUnitDescription + // + | extend x_EffectiveUnitPrice = iff(x_SkuPriceType == 'SavingsPlan', UnitPrice, real(null)) // Savings plan prices are for the effective price, not the contracted price + | extend x_EffectiveUnitPriceDiscount = ContractedUnitPrice - x_EffectiveUnitPrice + | extend x_ContractedUnitPriceDiscount = ListUnitPrice - ContractedUnitPrice + | extend x_TotalUnitPriceDiscount = ListUnitPrice - x_EffectiveUnitPrice + | project + BillingAccountId = tolower(case( + BillingProfileId startswith '/', BillingProfileId, + BillingAccountId startswith '/', BillingAccountId, + strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId, iff(x_BillingProfileId == x_BillingAccountId, '', strcat('/billingprofiles/', x_BillingProfileId))) + )), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName, x_BillingProfileId), + BillingCurrency = coalesce(BillingCurrency, CurrencyCode, Currency), // Currency last as a fallback only + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType = case( + x_SkuPriceType == 'ReservedInstance', 'Reservation', + x_SkuPriceType == 'SavingsPlan', 'Savings plan', + '' + ), + CommitmentDiscountUnit = case( + isempty(CommitmentDiscountCategory), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), PricingUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', PricingUnit), + '' + ), + ContractedUnitPrice, + ListUnitPrice, + PricingCategory = case( + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed', + '' + ), + PricingCurrency, + PricingUnit, + SkuId = coalesce(ProductId, ProductID), + SkuMeter, + SkuPriceId = strcat(x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType), + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement = case( + strlen(x_BillingAccountId) > 32, 'MCA', + strlen(x_BillingAccountId) < 32, 'EA', + 'Unknown' + ), + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent = 1.0 * x_ContractedUnitPriceDiscount / ListUnitPrice * 100, + x_EffectivePeriodEnd = startofmonth(x_EffectivePeriodEnd + 1h), + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent = 1.0 * x_EffectiveUnitPriceDiscount / ContractedUnitPrice * 100, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory = case( + x_SkuPriceType == 'Consumption' and (x_SkuIncludedQuantity > 0 or x_SkuTier > 0), 'Tiered', + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed Spend', + '' + ), + x_PricingUnitDescription, + x_SkuDescription = Product, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName = coalesce(x_SourceName, 'Cost Management'), + x_SourceProvider = coalesce(x_SourceProvider, 'Microsoft'), + x_SourceType = coalesce(x_SourceType, 'PriceSheet'), + x_SourceVersion = coalesce(x_SourceVersion, '2023-05-01'), + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 +} + +// Prices_final_v1_4 table +.create-merge table Prices_final_v1_4 ( + BillingAccountId: string, + BillingAccountName: string, + BillingCurrency: string, + ChargeCategory: string, + CommitmentDiscountCategory: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ContractedUnitPrice: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, // Azure + PricingUnit: string, + SkuId: string, + SkuMeter: string, // Azure + SkuPriceId: string, + SkuPriceIdv2: string, // Hubs add-on + x_BaseUnitPrice: real, // Azure + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure MCA + x_BillingProfileId: string, // Azure MCA + x_CommitmentDiscountNormalizedRatio: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_ContractedUnitPriceDiscount: real, // Hubs add-on + x_ContractedUnitPriceDiscountPercent: real, // Hubs add-on + x_EffectivePeriodEnd: datetime, // Azure + x_EffectivePeriodStart: datetime, // Azure + x_EffectiveUnitPrice: real, // Azure + x_EffectiveUnitPriceDiscount: real, // Hubs add-on + x_EffectiveUnitPriceDiscountPercent: real, // Hubs add-on + x_IngestionTime: datetime, // Hubs add-on + x_PricingBlockSize: real, // Hubs add-on + x_PricingSubcategory: string, // Hubs add-on + x_PricingUnitDescription: string, // Azure + x_SkuDescription: string, // Azure + x_SkuId: string, // Azure + x_SkuIncludedQuantity: real, // Azure EA + x_SkuMeterCategory: string, // Azure + x_SkuMeterId: string, // Azure + x_SkuMeterSubcategory: string, // Azure + x_SkuMeterType: string, // Azure + x_SkuPriceType: string, // Azure + x_SkuProductId: string, // Azure + x_SkuRegion: string, // Azure + x_SkuServiceFamily: string, // Azure + x_SkuOfferId: string, // Azure EA + x_SkuPartNumber: string, // Azure EA + x_SkuTerm: int, // Azure + x_SkuTier: real, // Azure MCA + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_TotalUnitPriceDiscount: real, // Hubs add-on + x_TotalUnitPriceDiscountPercent: real // Hubs add-on +) + +// Update policy for Prices_raw -> Prices_final_v1_4 +.alter table Prices_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Prices_raw", + "Query": "Prices_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Cost and usage |================================================================================================= +// Supported versions: +// - MS: 1.2-preview, 1.0, 1.0-preview(v1) +// https://aka.ms/costmgmt/exports/focus +// - AWS: 1.0 +// https://docs.aws.amazon.com/cur/latest/userguide/table-dictionary-focus-1-0-aws-columns.html +// - GCP: Jan-Jun 2024 +// https://cloud.google.com/resources/google-cloud-focus?e=48754805&hl=en +// Links to (Aug 2024): https://services.google.com/fh/files/misc/focus_guide_v1.pdf +// See also: +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/standard-usage +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage +// - OCI: 1.0 +// https://docs.oracle.com/iaas/Content/Billing/Concepts/costusagereportsoverview.htm#costreports__focus-cost-report-schema +// +// Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. +//====================================================================================================================== + +// Costs_transform_v1_4 function +.create-or-alter function +with (docstring='All costs transformed to FOCUS 1.4-preview.', folder='Costs') +Costs_transform_v1_4() +{ + let checkString = (column: string, oldValue: string, newValue: string) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkInt = (column: string, oldValue: int, newValue: int) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkReal = (column: string, oldValue: real, newValue: real) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + Costs_raw + // + // Dedupe rows + | extend x_IngestionTime = ingestion_time() + | extend x_ChargeId = '' + // TODO: Consider adding a unique charge ID per row + // hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // // 1. Resource hierarchy (including resource name), highest to lowest + // BillingAccountId, + // x_InvoiceSectionId, + // x_AccountOwnerId, + // SubAccountId, + // x_ResourceGroupName, + // ResourceName, + // // 2. Resource details + // ResourceId, + // RegionId, + // Tags, + // CommitmentDiscountId, + // x_CostCenter, + // // 4. Meter details + // SkuPriceId, + // x_SkuMeterId, + // x_SkuPartNumber, + // x_SkuOfferId, + // x_SkuDetails, + // // 5. Date + // ChargePeriodStart + // )) + // + // Identify data quality issues + // TODO: Remove x_SourceChanges in v1_3 (or later) + | extend x_SourceChanges = trim_end(',', strcat( + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based', 'InvalidChargeFrequency,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and EffectiveCost > 0, 'InvalidEffectiveCost,', ''), + iff((isempty(ContractedCost) or ContractedCost == 0) and EffectiveCost != 0, 'MissingContractedCost,', ''), + iff((isempty(ContractedUnitPrice) or ContractedUnitPrice == 0) and x_EffectiveUnitPrice != 0, 'MissingContractedUnitPrice,', ''), + iff(ListCost < ContractedCost, 'ListCostLessThanContractedCost,', ''), + iff(ContractedCost < EffectiveCost, 'ContractedCostLessThanEffectiveCost,', ''), + iff((isempty(ListCost) or ListCost == 0) and (ContractedCost != 0 or EffectiveCost != 0), 'MissingListCost,', ''), + iff((isempty(ListUnitPrice) or ListUnitPrice == 0) and (ContractedUnitPrice != 0 or x_EffectiveUnitPrice != 0), 'MissingListUnitPrice,', ''), + iff((isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001) + or (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001), + 'XEffectiveUnitPriceRoundingError,', ''), + iff(ConsumedQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingConsumedQuantity,', ''), + iff(PricingQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingPricingQuantity,', ''), + iff(isempty(ProviderName), 'MissingProviderName,', ''), + iff(isempty(PublisherName), 'MissingPublisherName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceId), 'MissingResourceId,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceName), 'MissingResourceName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceType), 'MissingResourceType,', ''), + iff(BilledCost > 0 and x_BilledUnitPrice == 0, 'MissingXBilledUnitPrice,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(x_ResourceType), 'MissingXResourceType,', ''), + iff(PricingCategory == 'Standard' and isnotempty(CommitmentDiscountId) and ChargeCategory == 'Usage', 'PricingCategoryShouldBeCommitted,', ''), + iff(x_SkuTerm == '1Year' or x_SkuTerm == '3Years' or x_SkuTerm == '5Years', 'SkuTermShouldBeAnInteger,', '') + )) + // + // Handle provider columns that moved to FOCUS + | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) + // + // Backup original prices/costs before the merge + | extend old_ContractedCost = ContractedCost + | extend old_ContractedUnitPrice = ContractedUnitPrice + | extend old_ListCost = ListCost + | extend old_ListUnitPrice = ListUnitPrice + | extend old_x_EffectiveUnitPrice = x_EffectiveUnitPrice + // + // Fix columns needed in other changes + | extend old_ProviderName = ProviderName, ProviderName = case( + isnotempty(ProviderName), ProviderName, + isnotempty(coalesce(x_CostCategories, x_Discount, x_Operation, x_ServiceCode, x_UsageType)), 'AWS', + isnotempty(coalesce(tostring(UsageAmount), tostring(x_Cost), x_Credits, x_CostType, tostring(x_CurrencyConversionRate), tostring(x_ExportTime), x_Project, x_ServiceId)), 'GCP', + isnotempty(coalesce(x_BillingProfileId, x_InvoiceSectionId)), 'Microsoft', + '' + ) + // + // Identify source + | extend x_SourceName = coalesce(x_SourceName, iff(isnotempty(x_BillingProfileId), 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, case( + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', + isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', + isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), + '' + )) + // Append version check error code + | extend x_SourceChanges = iff(x_SourceVersion == '1.0', x_SourceChanges, + strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) + ) + // + // Fix quantities + | extend old_PricingQuantity = PricingQuantity, PricingQuantity = case( + PricingQuantity != 0 or (EffectiveCost == 0 and BilledCost == 0), PricingQuantity, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(EffectiveCost) and EffectiveCost != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0, EffectiveCost / x_EffectiveUnitPrice, + PricingQuantity + ) + | extend old_ConsumedQuantity = ConsumedQuantity, ConsumedQuantity = case( + isnotempty(ConsumedQuantity) and ConsumedQuantity != 0, ConsumedQuantity, + ChargeCategory == 'Usage', PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + ConsumedQuantity + ) + // + // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID + | extend tmp_MissingPrices = ProviderName == 'Microsoft' + and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0) + and x_EffectiveUnitPrice != 0 + and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused') + and isnotempty(strcat(x_SkuMeterId, x_SkuOfferId)) + | as allCosts + | where tmp_MissingPrices + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | as costsWithMissingPrices + | join kind=leftouter ( + Prices_final_v1_4 + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) + | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit + ) on tmp_ReservationPriceLookupKey + // + // Select the best price to use for each row + | extend x_EffectiveUnitPrice = case( + // If price is a rounding error away from the billed price, use the billed price + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001, x_BilledUnitPrice, + // If price is a rounding error away from the contracted price, use the contracted price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001, ContractedUnitPrice, + x_EffectiveUnitPrice + ) + | extend ContractedUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ContractedUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ContractedUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // If billed price is available, assume the billed price is the same as contracted price to support aggregations + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, x_EffectiveUnitPrice, + // Otherwise, assume the effective price is the same as contracted price to support aggregations + x_EffectiveUnitPrice + ) + | extend ListUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ListUnitPrice) and ListUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ListUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ListUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ListUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // Otherwise, assume the contracted price is the same as list price to support aggregations + ContractedUnitPrice + ) + // Calculate missing costs based on new prices -- If cost is already correct, keep that; if not and price is available, recalculate the cost; otherwise, keep the existing cost + | extend ContractedCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ContractedCost) and ContractedCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedCost, + // ContractedCost is 0 in all other scenarios... + // If 0 and there's a billed cost and prices are the same, use BilledCost + isnotempty(BilledCost) and BilledCost != 0 and ContractedUnitPrice == x_BilledUnitPrice, BilledCost, + // If 0 and there's a billed cost and prices are the same, use EffectiveCost + isnotempty(EffectiveCost) and EffectiveCost != 0 and ContractedUnitPrice == x_EffectiveUnitPrice, EffectiveCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0, ContractedUnitPrice * PricingQuantity, + // If 0 and there's no price, assume EffectiveCost + isempty(ContractedUnitPrice) or ContractedUnitPrice == 0, EffectiveCost, + // Fall back to the original value for any unhandled scenarios + ContractedCost + ) + | extend ListCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ListCost) and ListCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ListCost, + // ListCost is 0 in all other scenarios... + // If 0 and there's a contracted cost and prices are the same, use ContractedCost + isnotempty(ContractedCost) and ContractedCost != 0 and ListUnitPrice == ContractedUnitPrice, ContractedCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ListUnitPrice) and ListUnitPrice != 0, ListUnitPrice * PricingQuantity, + // If 0 and there's no price, assume ContractedCost + isempty(ListUnitPrice) or ListUnitPrice == 0, ContractedCost, + // Fall back to the original value for any unhandled scenarios + ListCost + ) + // Merge the rest of the unmodified cost records and remove excess columns + | union (allCosts | where not(tmp_MissingPrices)) + | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1 + // + | extend SkuPriceDetails = parse_json(SkuPriceDetails) + | extend Tags = parse_json(Tags) + | extend x_SkuDetails = parse_json(x_SkuDetails) + // + // Handle FOCUS 1.0-preview + | extend old_ChargeSubcategory = ChargeSubcategory + | extend old_ChargeCategory = ChargeCategory, ChargeCategory = case( + // Handle FOCUS 1.0-preview ChargeSubcategory + ChargeSubcategory == 'Credit', 'Credit', + ChargeSubcategory == 'Refund', 'Purchase', // We are assuming purchase refunds since we don't have data to indicate usage refunds + ChargeCategory + ) + | extend old_ChargeClass = ChargeClass, ChargeClass = case(ChargeSubcategory == 'Refund', 'Correction', ChargeClass) + // + // Populate CapacityReservationId when not specified + | extend CapacityReservationId = coalesce(CapacityReservationId, tostring(coalesce(x_SkuDetails.VMCapacityReservationId, SkuPriceDetails.VMCapacityReservationId, SkuPriceDetails.x_VMCapacityReservationId))) + | extend old_CapacityReservationStatus = CapacityReservationStatus, CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + isnotempty(CapacityReservationStatus), CapacityReservationStatus, + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + // + // BUG: ChargeFrequency shows "Usage-Based" for monthly recurring savings plan purchases + | extend old_ChargeFrequency = ChargeFrequency, ChargeFrequency = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based' and ProviderName == 'Microsoft' and x_SourceVersion startswith '1.0', 'Recurring', ChargeFrequency) + // + // Commitment discounts + | extend x_CommitmentDiscountNormalizedRatio = case( + // Calculate from CommitmentDiscountQuantity, if specified + isnotempty(CommitmentDiscountQuantity) and CommitmentDiscountQuantity != 0, CommitmentDiscountQuantity / PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, SkuPriceDetails.RINormalizationRatio, SkuPriceDetails.x_RINormalizationRatio, dynamic(1))) + ) + | extend old_CommitmentDiscountQuantity = CommitmentDiscountQuantity, CommitmentDiscountQuantity = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountQuantity), CommitmentDiscountQuantity, + // FOCUS 1.0-preview, 1.0 + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend old_CommitmentDiscountUnit = CommitmentDiscountUnit, CommitmentDiscountUnit = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountUnit), CommitmentDiscountUnit, + // FOCUS 1.0 + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend old_CommitmentDiscountStatus = CommitmentDiscountStatus, CommitmentDiscountStatus = case( + // FOCUS 1.0+ + isnotempty(CommitmentDiscountStatus), CommitmentDiscountStatus, + // FOCUS 1.0-preview + ChargeSubcategory == 'Used Commitment', 'Used', + ChargeSubcategory == 'Unused Commitment', 'Unused', + '' + ) + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // Pricing + | extend old_x_AmortizationClass = x_AmortizationClass, x_AmortizationClass = case( + // FOCUS 1.3 + isnotempty(x_AmortizationClass), x_AmortizationClass, + // FOCUS 1.0-preview+ + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + | extend old_PricingCategory = PricingCategory, PricingCategory = case( + // FOCUS 1.0+ + isnotempty(PricingCategory), PricingCategory, + // FOCUS 1.0-preview + PricingCategory == 'On-Demand', 'Standard', + PricingCategory == 'Commitment-Based', 'Committed', + '' + ) + // + // Commitment discount utilization + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // BUG: Fix ContractedCost that has bad values + | extend ContractedCost = iff(ProviderName == 'Microsoft' and isnotempty(PricingQuantity) and isnotempty(x_PricingBlockSize) and ContractedCost != ContractedUnitPrice * PricingQuantity, ContractedUnitPrice * PricingQuantity, ContractedCost) + // + // Handle FOCUS 1.0-preview UsageQuantity/Unit + | extend ConsumedQuantity = iff(ChargeCategory == 'Usage', coalesce(ConsumedQuantity, UsageQuantity, UsageAmount), real(null)) + | extend old_ConsumedUnit = ConsumedUnit, ConsumedUnit = iff(ChargeCategory == 'Usage' and isnotempty(ConsumedQuantity), coalesce(ConsumedUnit, UsageUnit, 'Units'), '') + // + // Convert IDs to lowercase for consistency + | extend BillingAccountId = tolower(BillingAccountId) + | extend CommitmentDiscountId = tolower(CommitmentDiscountId) + // + // BUG: Remove EffectiveCost for commitment discount purchases + | extend old_EffectiveCost = EffectiveCost, EffectiveCost = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), EffectiveCost) + | extend old_x_EffectiveCostInUsd = x_EffectiveCostInUsd, x_EffectiveCostInUsd = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), x_EffectiveCostInUsd) + // + // Clean up resource columns + | extend old_ResourceId = ResourceId, ResourceId = case( + isnotempty(ResourceId), ResourceId, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), CommitmentDiscountId, + ResourceId + ) + | extend old_ResourceName = ResourceName, ResourceName = tolower(case( + isnotempty(ResourceName), ResourceName, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountName), CommitmentDiscountName, + isnotempty(ResourceId), parse_resourceid(ResourceId).ResourceName, + ResourceName + )) + | extend old_x_ResourceType = x_ResourceType, x_ResourceType = case( + isnotempty(x_ResourceType), x_ResourceType, + isnotempty(ResourceId), parse_resourceid(ResourceId).x_ResourceType, + x_ResourceType + ) + | extend old_ResourceType = ResourceType, ResourceType = case( + // Use existing resource type display name unless it's an internal resource type ID + isnotempty(ResourceType) and tolower(ResourceType) != tolower(x_ResourceType) and ResourceType !contains '/', ResourceType, + // Use CommitmentDiscountType for commitment discount purchases + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountType), CommitmentDiscountType, + // Look up display name from internal type + isnotempty(x_ResourceType), coalesce(tostring(resource_type(x_ResourceType).SingularDisplayName), ResourceType, x_ResourceType), + ResourceType + ) + // + // Handle missing values + | extend old_PublisherName = PublisherName, PublisherName = case(PublisherName == 'Microsoft Corporation', 'Microsoft', isnotempty(PublisherName), PublisherName, x_PublisherCategory == 'Cloud Provider', ProviderName, '') + // + // Handle FOCUS 1.0-preview Region column + | extend old_Region = Region + | extend old_RegionId = RegionId, RegionId = coalesce(RegionId, iff(ProviderName == 'Microsoft', replace_string(tolower(Region), ' ', ''), Region)) + | extend RegionName = coalesce(RegionName, Region) + // + // SKU properties + | extend x_SkuCoreCount = toint(coalesce(SkuPriceDetails.CoreCount, SkuPriceDetails.x_VCPUs, x_SkuDetails.VCPUs, SkuPriceDetails.x_VCores, x_SkuDetails.VCores, SkuPriceDetails.x_vCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(SkuPriceDetails.InstanceType, SkuPriceDetails.x_ServiceType, x_SkuDetails.ServiceType, SkuPriceDetails.x_ServerSku, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + isnotempty(SkuPriceDetails.OperatingSystem), SkuPriceDetails.OperatingSystem, + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Canonical', 'Linux', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend SkuPriceDetails = case( + // FOCUS 1.3 + isnotempty(SkuPriceDetails), SkuPriceDetails, + // FOCUS 1.0-preview, 1.0 + parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // + // Azure Hybrid Benefit + | extend tmp_SqlAhb = tolower(coalesce(x_SkuDetails.AHB, SkuPriceDetails.x_AHB)) + | extend x_SkuLicenseType = case( + ChargeCategory != 'Usage', '', + x_SkuMeterCategory in ('Virtual Machines', 'Virtual Machine Licenses') and (x_SkuMeterSubcategory contains 'Windows' or coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL'), 'Windows Server', + isnotempty(tmp_SqlAhb) or x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isempty(x_SkuLicenseType), '', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL' or tmp_SqlAhb == 'true' or x_SkuMeterSubcategory contains 'Azure Hybrid Benefit', 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not Enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount) or isempty(x_SkuLicenseType), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + // + // Savings + | extend x_CommitmentDiscountSavings = iff(isempty(ContractedCost) or ContractedCost == 0 or ContractedCost - EffectiveCost < 0.0001, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - ContractedCost < 0.0001, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - EffectiveCost < 0.0001, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(isempty(ContractedUnitPrice) or ContractedUnitPrice == 0 or ContractedUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - ContractedUnitPrice < 0.0001, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // + // Minor fixes + | extend old_BillingPeriodEnd = BillingPeriodEnd, BillingPeriodEnd = startofmonth(BillingPeriodEnd) + | extend old_BillingPeriodStart = BillingPeriodStart, BillingPeriodStart = startofmonth(BillingPeriodStart) + // + // Sort columns and apply final transforms + | project + AllocatedMethodDetails = parse_json(AllocatedMethodDetails), // FOCUS 1.3+ + AllocatedMethodId, // FOCUS 1.3+ + AllocatedResourceId, // FOCUS 1.3+ + AllocatedResourceName, // FOCUS 1.3+ + AllocatedTags = parse_json(AllocatedTags), // FOCUS 1.3+ + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), + ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), + EffectiveCost, + HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceId = coalesce(InvoiceId, x_InvoiceId), + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), + PricingQuantity, + PricingUnit, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName = iff(isempty(ServiceProviderName), ProviderName, ServiceProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat + ServiceSubcategory, // TODO: Populate ServiceSubcategory from ServiceName when missing + SkuId, + SkuMeter = coalesce(SkuMeter, x_SkuMeterName), + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName = iff(isempty(SubAccountId), '', SubAccountName), + SubAccountType, + Tags, + x_AccountId = iff(x_AccountId == '-2', '', x_AccountId), + x_AccountName = iff(x_AccountId == '-2', '', x_AccountName), + x_AccountOwnerId = iff(x_AccountId == '-2', '', x_AccountOwnerId), + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement = case( + ProviderName == 'Microsoft' and x_BillingAccountId == x_BillingProfileId, 'EA', + ProviderName == 'Microsoft' and x_BillingAccountId != x_BillingProfileId, 'MCA', + ProviderName + ), + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility = '', // TODO: Add x_CommitmentDiscountSpendEligibility for Costs + x_CommitmentDiscountUsageEligibility = '', // TODO: Add x_CommitmentDiscountUsageEligibility for Costs + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd = coalesce(x_ContractedCostInUsd, x_OnDemandCostInUsd), + x_CostAllocationRuleName, + x_CostCategories = parse_json(x_CostCategories), + x_CostCenter, + x_CostType, + x_Credits = parse_json(x_Credits), + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount = parse_json(x_Discount), + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId = case( + x_InvoiceSectionId == '-2', '', + x_InvoiceSectionId + ), + x_InvoiceSectionName = case( + x_InvoiceSectionName == 'Unassigned', '', + x_InvoiceSectionName + ), + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription = iff(x_PricingUnitDescription == 'Unassigned', '', x_PricingUnitDescription), + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName = tolower(x_ResourceGroupName), + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, // TODO: Populate from ServiceName when missing + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues = bag_merge( + checkString('BillingPeriodEnd', old_BillingPeriodEnd, BillingPeriodEnd), + checkString('BillingPeriodStart', old_BillingPeriodStart, BillingPeriodStart), + checkString('CapacityReservationStatus', old_CapacityReservationStatus, CapacityReservationStatus), + checkString('ChargeCategory', old_ChargeCategory, ChargeCategory), + checkString('ChargeClass', old_ChargeClass, ChargeClass), + checkString('ChargeSubcategory', old_ChargeSubcategory, ''), // Not included in final schema; use empty string + checkString('ChargeFrequency', old_ChargeFrequency, ChargeFrequency), + checkReal('CommitmentDiscountQuantity', old_CommitmentDiscountQuantity, CommitmentDiscountQuantity), + checkString('CommitmentDiscountUnit', old_CommitmentDiscountUnit, CommitmentDiscountUnit), + checkString('CommitmentDiscountStatus', old_CommitmentDiscountStatus, CommitmentDiscountStatus), + checkReal('ConsumedQuantity', old_ConsumedQuantity, ConsumedQuantity), + checkString('ConsumedUnit', old_ConsumedUnit, ConsumedUnit), + checkReal('ContractedCost', old_ContractedCost, ContractedCost), + checkReal('ContractedUnitPrice', old_ContractedUnitPrice, ContractedUnitPrice), + checkReal('EffectiveCost', old_EffectiveCost, EffectiveCost), + checkReal('ListCost', old_ListCost, ListCost), + checkReal('ListUnitPrice', old_ListUnitPrice, ListUnitPrice), + checkString('PricingCategory', old_PricingCategory, PricingCategory), + checkReal('PricingQuantity', old_PricingQuantity, PricingQuantity), + checkString('ProviderName', old_ProviderName, ProviderName), + checkString('PublisherName', old_PublisherName, PublisherName), + checkString('Region', old_Region, ''), // Not included in final schema; use empty string + checkString('RegionId', old_RegionId, RegionId), + checkString('ResourceId', old_ResourceId, ResourceId), + checkString('ResourceName', old_ResourceName, ResourceName), + checkString('ResourceType', old_ResourceType, ResourceType), + checkString('x_AmortizationClass', old_x_AmortizationClass, x_AmortizationClass), + checkReal('x_EffectiveCostInUsd', old_x_EffectiveCostInUsd, x_EffectiveCostInUsd), + checkReal('x_EffectiveUnitPrice', old_x_EffectiveUnitPrice, x_EffectiveUnitPrice), + checkString('x_ResourceType', old_x_ResourceType, x_ResourceType) + ), + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + +// Costs_final_v1_4 table +.create-merge table Costs_final_v1_4 ( + AllocatedMethodDetails: dynamic, // FOCUS 1.3+ + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: dynamic, // FOCUS 1.3+ + AvailabilityZone: string, + BilledCost: real, + BillingAccountId: string, + BillingAccountName: string, + BillingAccountType: string, + BillingCurrency: string, + BillingPeriodEnd: datetime, + BillingPeriodStart: datetime, + CapacityReservationId: string, + CapacityReservationStatus: string, + ChargeCategory: string, + ChargeClass: string, + ChargeDescription: string, + ChargeFrequency: string, + ChargePeriodEnd: datetime, + ChargePeriodStart: datetime, + CommitmentDiscountCategory: string, + CommitmentDiscountId: string, + CommitmentDiscountName: string, + CommitmentDiscountQuantity: real, + CommitmentDiscountStatus: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceId: string, + InvoiceIssuerName: string, + ListCost: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, + PricingQuantity: real, + PricingUnit: string, + // ProviderName and PublisherName removed in FOCUS 1.4 + RegionId: string, + RegionName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + ServiceCategory: string, + ServiceName: string, + ServiceProviderName: string, // FOCUS 1.3+ + ServiceSubcategory: string, + SkuId: string, + SkuMeter: string, + SkuPriceDetails: dynamic, + SkuPriceId: string, + SubAccountId: string, + SubAccountName: string, + SubAccountType: string, + Tags: dynamic, + x_AccountId: string, // Azure 1.0-preview(v1)+ + x_AccountName: string, // Azure 1.0-preview(v1)+ + x_AccountOwnerId: string, // Azure 1.0-preview(v1)+ + x_AmortizationClass: string, // Azure 1.2-preview+ + x_BilledCostInUsd: real, // Azure 1.0-preview(v1)+ + x_BilledUnitPrice: real, // Azure 1.0-preview(v1)+ + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure 1.0-preview(v1)+ + x_BillingAccountName: string, // Azure 1.0-preview(v1)+ + x_BillingExchangeRate: real, // Azure 1.0-preview(v1)+ + x_BillingExchangeRateDate: datetime, // Azure 1.0-preview(v1)+ + x_BillingItemCode: string, // Alibaba 1.0 + x_BillingItemName: string, // Alibaba 1.0 + x_BillingProfileId: string, // Azure 1.0-preview(v1)+ + x_BillingProfileName: string, // Azure 1.0-preview(v1)+ + x_ChargeId: string, // Azure 1.0-preview(v1) only + x_CommitmentDiscountNormalizedRatio: real, // Azure 1.2-preview+ + x_CommitmentDiscountPercent: real, // Hubs add-on + x_CommitmentDiscountSavings: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_CommitmentDiscountUtilizationAmount: real, // Hubs add-on + x_CommitmentDiscountUtilizationPotential: real, // Hubs add-on + x_CommodityCode: string, // Alibaba 1.0 + x_CommodityName: string, // Alibaba 1.0 + x_ComponentName: string, // Tencent 1.0 + x_ComponentType: string, // Tencent 1.0 + x_ConsumedCoreHours: real, // Hubs add-on + x_ContractedCostInUsd: real, // Azure 1.0+ + x_CostAllocationRuleName: string, // Azure 1.0-preview(v1)+ + x_CostCategories: dynamic, // AWS 1.0 (JSON) + x_CostCenter: string, // Azure 1.0-preview(v1)+ + x_CostType: string, // GCP Jan 2024 + x_Credits: dynamic, // GCP Jan 2024 + x_CurrencyConversionRate: real, // GCP Jun 2024 + x_CustomerId: string, // Azure 1.0-preview(v1)+ + x_CustomerName: string, // Azure 1.0-preview(v1)+ + x_Discount: dynamic, // AWS 1.0 (JSON) + x_EffectiveCostInUsd: real, // Azure 1.0-preview(v1)+ + x_EffectiveUnitPrice: real, // Azure 1.0-preview(v1)+ + x_ExportTime: datetime, // GCP Jan 2024 / Tencent 1.0 + x_IngestionTime: datetime, // Hubs add-on + x_InstanceID: string, // Alibaba 1.0 + x_InvoiceIssuerId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionName: string, // Azure 1.0-preview(v1)+ + x_ListCostInUsd: real, // Azure 1.0-preview(v1)+ + x_Location: string, // GCP Jan 2024 + x_NegotiatedDiscountPercent:real, // Hubs add-on + x_NegotiatedDiscountSavings:real, // Hubs add-on + x_Operation: string, // AWS 1.0 + x_OwnerAccountID: string, // Tencent 1.0 + x_PartnerCreditApplied: string, // Azure 1.0-preview(v1)+ + x_PartnerCreditRate: string, // Azure 1.0-preview(v1)+ + x_PricingBlockSize: real, // Azure 1.0-preview(v1)+ + x_PricingSubcategory: string, // Azure 1.0-preview(v1)+ + x_PricingUnitDescription: string, // Azure 1.0-preview(v1)+ + x_Project: string, // GCP Jan 2024 + x_PublisherCategory: string, // Azure 1.0-preview(v1)+ + x_PublisherId: string, // Azure 1.0-preview(v1)+ + x_ResellerId: string, // Azure 1.0-preview(v1)+ + x_ResellerName: string, // Azure 1.0-preview(v1)+ + x_ResourceGroupName: string, // Azure 1.0-preview(v1)+ + x_ResourceType: string, // Azure 1.0-preview(v1)+ + x_ServiceCode: string, // AWS 1.0 + x_ServiceId: string, // GCP Jan 2024 + x_ServiceModel: string, // Azure 1.2-preview+ + x_ServicePeriodEnd: datetime, // Azure 1.0-preview(v1)+ + x_ServicePeriodStart: datetime, // Azure 1.0-preview(v1)+ + x_SkuCoreCount: int, // Hubs add-on + x_SkuDescription: string, // Azure 1.0-preview(v1)+ + x_SkuDetails: dynamic, // Azure 1.0-preview(v1)+ + x_SkuInstanceType: string, // Hubs add-on + x_SkuIsCreditEligible: bool, // Azure 1.0-preview(v1)+ + x_SkuLicenseQuantity: int, // Hubs add-on + x_SkuLicenseStatus: string, // Hubs add-on + x_SkuLicenseType: string, // Hubs add-on + x_SkuLicenseUnit: string, // Hubs add-on + x_SkuMeterCategory: string, // Azure 1.0-preview(v1)+ + x_SkuMeterId: string, // Azure 1.0-preview(v1)+ + x_SkuMeterSubcategory: string, // Azure 1.0-preview(v1)+ + x_SkuOfferId: string, // Azure 1.0-preview(v1)+ + x_SkuOperatingSystem: string, // Hubs add-on + x_SkuOrderId: string, // Azure 1.0-preview(v1)+ + x_SkuOrderName: string, // Azure 1.0-preview(v1)+ + x_SkuPartNumber: string, // Azure 1.0-preview(v1)+ + x_SkuPlanName: string, // Azure 1.2-preview+ + x_SkuRegion: string, // Azure 1.0-preview(v1)+ + x_SkuServiceFamily: string, // Azure 1.0-preview(v1)+ + x_SkuTerm: int, // Azure 1.0-preview(v1)+ + x_SkuTier: string, // Azure 1.0-preview(v1)+ + x_SourceChanges: string, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceValues: dynamic, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubproductName: string, // Tencent 1.0 + x_TotalDiscountPercent: real, // Hubs add-on + x_TotalSavings: real, // Hubs add-on + x_UsageType: string // AWS 1.0 +) + +// Update policy for Costs_raw -> Costs_final_v1_4 table +.alter table Costs_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Costs_raw", + "Query": "Costs_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Actual costs |=================================================================================================== +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// ActualCosts_transform_v1_4 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.4-preview.', folder='Costs') +ActualCosts_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + ActualCosts_raw + | where ChargeType in ('Purchase', 'Refund') and isnotempty(ReservationId) + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for ActualCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "ActualCosts_raw", + "Query": "ActualCosts_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Amortized costs |================================================================================================ +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// AmortizedCosts_transform_v1_4 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.4-preview.', folder='Costs') +AmortizedCosts_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + AmortizedCosts_raw + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for AmortizedCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "AmortizedCosts_raw", + "Query": "AmortizedCosts_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| CommitmentDiscountUsage |======================================================================================== +// Supported versions: +// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea +// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca +//====================================================================================================================== + +// CommitmentDiscountUsage_transform_v1_4 function +.create-or-alter function +with (docstring='All commitment discount usage transformed to FOCUS 1.4-preview. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + CommitmentDiscountUsage_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Handle resource columns + | extend ResourceId = tolower(InstanceId) + | extend tmp_ResourceDetails = parse_resourceid(ResourceId) + | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) + | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) + | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) + | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) + | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + // + // Sort columns and apply final transforms + | project + ChargePeriodEnd = UsageDate + 1d, + ChargePeriodStart = UsageDate, + CommitmentDiscountCategory = 'Usage', + CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), + CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, + CommitmentDiscountType = 'Reservation', + CommitmentDiscountUnit = case( + InstanceFlexibilityRatio == 1, 'Hours', + InstanceFlexibilityRatio != 1, 'Normalized Hours', + '' + ), + ConsumedQuantity = UsedHours, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount = TotalReservedQuantity, + x_CommitmentDiscountCommittedAmount = ReservedHours, + // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, + x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), + x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, + x_IngestionTime = ingestion_time(), + x_ResourceGroupName, + x_ResourceType, + // x_RowId = hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // CommitmentDiscountId, + // ResourceId, + // ChargePeriodStart + // )), + x_ServiceModel, + x_SkuOrderId = ReservationOrderId, + x_SkuSize = iff(SkuName == 'NA', '', SkuName), + x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), + x_SourceProvider = coalesce(x_SourceProvider, ProviderName), + x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), + x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) +} + +// CommitmentDiscountUsage_final_v1_4 table +.create-merge table CommitmentDiscountUsage_final_v1_4 ( + ChargePeriodEnd: datetime, // Hubs add-on + ChargePeriodStart: datetime, // MS 2023-03-01 + CommitmentDiscountCategory: string, // Hubs add-on + CommitmentDiscountId: string, // MS 2023-03-01 + CommitmentDiscountQuantity: real, // MS 2023-03-01 + CommitmentDiscountType: string, // Hubs add-on + CommitmentDiscountUnit: string, // Hubs add-on + ConsumedQuantity: real, // MS 2023-03-01 + ProviderName: string, // Hubs add-on + ResourceId: string, // MS 2023-03-01 + ResourceName: string, // Hubs add-on + ResourceType: string, // Hubs add-on + ServiceCategory: string, // Hubs add-on + ServiceName: string, // Hubs add-on + ServiceSubcategory: string, // Hubs add-on + SubAccountId: string, // Hubs add-on + x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 + x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 + x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 + x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 + x_IngestionTime: datetime, // Hubs add-on + x_ResourceGroupName: string, // Hubs add-on + x_ResourceType: string, // Hubs add-on + x_ServiceModel: string, // Hubs add-on + x_SkuOrderId: string, // MS 2023-03-01 + x_SkuSize: string, // MS 2023-03-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_4 table +.alter table CommitmentDiscountUsage_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "CommitmentDiscountUsage_raw", + "Query": "CommitmentDiscountUsage_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitment_transform_v1_4 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.4-preview.', folder='Contract commitments') +ContractCommitment_transform_v1_4() +{ + ContractCommitment_raw + | project + BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ + BillingCurrency, + ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ + DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ + DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ + FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ + InvoiceIssuerName, + LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ + LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ + Model = column_ifexists('Model', ''), // FOCUS 1.4+ + OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ + PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ + PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ + PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ + PricingCurrency, + PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitment_final_v1_4 table +.create-merge table ContractCommitment_final_v1_4 ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_4 table +.alter table ContractCommitment_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitment_raw", + "Query": "ContractCommitment_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Recommendations |================================================================================================ +// Supported datasets/versions: +// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea +// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca +//====================================================================================================================== + +// Recommendations_transform_v1_4 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.4-preview.', folder='Recommendations') +Recommendations_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + | extend x_IngestionTime = ingestion_time() + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Convert JSON cost columns to real + | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) + | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) + | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) + // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // + // Build recommendation details + | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location + | extend x_RecommendationDetails = case( + // Use incoming x_RecommendationDetails first + isnotempty(x_RecommendationDetails), x_RecommendationDetails, + // Create one for reservation recommendations if needed + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', Scope, + 'LookbackPeriodDuration', case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), + '' + ), + 'LookbackPeriodStart', FirstUsageDate, + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'RegionName', RegionName, + 'SkuMeterId', MeterId, + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future + | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) + | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) + // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // + | project + ProviderName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations + ResourceType, + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, + x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), + x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), + x_IngestionTime, + x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationDate, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), + x_RecommendationDetails, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_final_v1_4 table +.create-merge table Recommendations_final_v1_4 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: real, + x_EffectiveCostBefore: real, + x_EffectiveCostSavings: real, + x_IngestionTime: datetime, + x_RecommendationCategory: string, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_RecommendationId: string, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Update policy for Recommendations_raw -> Recommendations_final_v1_4 table +.alter table Recommendations_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Transactions |=================================================================================================== +// Supported versions: +// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea +// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca +//====================================================================================================================== + +// Transactions_transform_v1_4 function +.create-or-alter function +with (docstring='All transactions transformed to FOCUS 1.4-preview.', folder='Transactions') +Transactions_transform_v1_4() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Transactions_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Handle BillingPeriodStart/End + | extend BillingMonth = tostring(BillingMonth) + | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) + | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) + // + // Sort columns and apply final transforms + | project + BilledCost = Amount, + BillingAccountId = case( + BillingProfileId startswith '/', BillingProfileId, + isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), + isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), + '' + ), + BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), + BillingCurrency = Currency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory = case( + EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', + 'Adjustment' + ), + ChargeClass = case( + EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario + EventType == 'Refund', 'Correction', + '' + ), + ChargeDescription = Description, + ChargeFrequency = case( + BillingFrequency == 'OneTime', 'One-Time', + BillingFrequency == 'Recurring', 'Recurring', + BillingFrequency + ), + ChargePeriodStart = EventDate, + InvoiceId, + PricingQuantity = Quantity, + PricingUnit = 'Reservations', + ProviderName, + RegionId = Region, + RegionName = Region, + SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), + SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerEmail, + x_CostCenter = CostCenter, + x_InvoiceNumber = Invoice, + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), + x_IngestionTime = ingestion_time(), + x_MonetaryCommitment = MonetaryCommitment, + x_Overage = Overage, + x_PurchasingBillingAccountId = PurchasingEnrollment, + x_SkuOrderId = ReservationOrderId, + x_SkuOrderName = ReservationOrderName, + x_SkuSize = ArmSkuName, + x_SkuTerm = isoMonths(Term), + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId = PurchasingSubscriptionGuid, + x_TransactionType = EventType +} + +// Transactions_final_v1_4 table +.create-merge table Transactions_final_v1_4 ( + BilledCost: real, // MS CM EA+MCA 2023-05-01 + BillingAccountId: string, // MS CM EA+MCA 2023-05-01 + BillingAccountName: string, // MS CM EA+MCA 2023-05-01 + BillingCurrency: string, // MS CM EA+MCA 2023-05-01 + BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 + BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + ChargeCategory: string, // Hubs add-on + ChargeClass: string, // Hubs add-on + ChargeDescription: string, // MS CM EA+MCA 2023-05-01 + ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 + ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + InvoiceId: string, // MS CM MCA 2023-05-01 + PricingQuantity: real, // MS CM EA+MCA 2023-05-01 + PricingUnit: string, // Hubs add-on + ProviderName: string, // Hubs add-on + RegionId: string, // MS CM EA+MCA 2023-05-01 + RegionName: string, // MS CM EA+MCA 2023-05-01 + SubAccountId: string, // MS CM EA+MCA 2023-05-01 + SubAccountName: string, // MS CM EA+MCA 2023-05-01 + x_AccountName: string, // MS CM EA 2023-05-01 + x_AccountOwnerId: string, // MS CM EA 2023-05-01 + x_CostCenter: string, // MS CM EA 2023-05-01 + x_InvoiceNumber: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 + x_IngestionTime: datetime, // Hubs add-on + x_MonetaryCommitment: real, // MS CM EA 2023-05-01 + x_Overage: real, // MS CM EA 2023-05-01 + x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 + x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 + x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 + x_SkuSize: string, // MS CM EA+MCA 2023-05-01 + x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 + x_TransactionType: string // MS CM EA+MCA 2023-05-01 +) + +// Update policy for Transactions_raw -> Transactions_final_v1_4 table +.alter table Transactions_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "Transactions_raw", + "Query": "Transactions_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` From 3a68afd443fedfe032f7b9a1bda3ac38dfa0608d Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 11:04:36 -0700 Subject: [PATCH 05/19] Add FOCUS 1.4-preview hub functions (#2133) Adds HubSetup_v1_4.kql with the v1_4-preview-shaped hub functions. Costs_v1_4 unions Costs_final_v1_4, Costs_final_v1_3, Costs_final_v1_2, and Costs_final_v1_0; ProviderName and PublisherName are dropped from the v1_4 project list (removed in 1.4 per spec). ContractCommitment_v1_4 unions ContractCommitment_final_v1_4 with ContractCommitment_final_v1_3 and extends v1_3 data with empty/null defaults for the 14 new FOCUS 1.4 columns. HubSetup_Latest.kql is intentionally NOT retargeted to v1_4. Per the preview convention in data-model.md, unversioned aliases stay pinned to the latest GA schema (v1_3) until v1_4 promotes from preview to GA. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 1 + .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../Analytics/scripts/HubSetup_v1_4.kql | 650 ++++++++++++++++++ 3 files changed, 652 insertions(+) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 431f540b8..ae9e5d128 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -46,6 +46,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 543b7c455..5569beac7 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -416,6 +416,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') + v1_4: loadTextContent('scripts/HubSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql new file mode 100644 index 000000000..fea5b464e --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -0,0 +1,650 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Hub database / FOCUS 1.4-preview functions +// Used for reporting with backward compatibility. +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + + +// CommitmentDiscountUsage_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.4-preview.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_4() +{ + database('Ingestion').CommitmentDiscountUsage_final_v1_4 + | union ( + database('Ingestion').CommitmentDiscountUsage_final_v1_0 + // Convert decimal to real + | extend + ConsumedQuantity = toreal(ConsumedQuantity), + x_CommitmentDiscountCommittedCount = toreal(x_CommitmentDiscountCommittedCount), + x_CommitmentDiscountCommittedAmount = toreal(x_CommitmentDiscountCommittedAmount), + x_CommitmentDiscountNormalizedRatio = toreal(x_CommitmentDiscountNormalizedRatio) + // Add new columns + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceSubcategory) on x_ResourceType + | extend CommitmentDiscountQuantity = ConsumedQuantity * x_CommitmentDiscountNormalizedRatio + | extend CommitmentDiscountUnit = case( + x_CommitmentDiscountNormalizedRatio == 1, 'Hours', + x_CommitmentDiscountNormalizedRatio > 1, 'Normalized Hours', + '' + ) + ) + | project + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountQuantity, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount, + x_CommitmentDiscountCommittedAmount, + x_CommitmentDiscountNormalizedGroup, + x_CommitmentDiscountNormalizedRatio, + x_IngestionTime, + x_ResourceGroupName, + x_ResourceType, + x_ServiceModel, + x_SkuOrderId, + x_SkuSize, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// ContractCommitment_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4-preview.', folder = 'ContractCommitment') +ContractCommitment_v1_4() +{ + database('Ingestion').ContractCommitment_final_v1_4 + | union ( + database('Ingestion').ContractCommitment_final_v1_3 + // FOCUS 1.4 column defaults (data was ingested under v1_3 schema) + | extend BenefitCategory = '' + | extend ContractCommitmentApplicability = dynamic(null) + | extend Created = datetime(null) + | extend DiscountPercentage = real(null) + | extend DurationType = '' + | extend FulfillmentInterval = '' + | extend LastUpdated = datetime(null) + | extend LifecycleStatus = '' + | extend Model = '' + | extend OfferCategory = '' + | extend PaymentInterval = '' + | extend PaymentModel = '' + | extend PaymentUpfrontPercentage = real(null) + | extend PricingCurrencyContractCommitmentCost = real(null) + ) +} + + +// Costs_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.4-preview.', folder = 'Costs') +Costs_v1_4() +{ + database('Ingestion').Costs_final_v1_4 + | union ( + database('Ingestion').Costs_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + ConsumedQuantity = toreal(ConsumedQuantity), + ContractedCost = toreal(ContractedCost), + ContractedUnitPrice = toreal(ContractedUnitPrice), + EffectiveCost = toreal(EffectiveCost), + ListCost = toreal(ListCost), + ListUnitPrice = toreal(ListUnitPrice), + PricingQuantity = toreal(PricingQuantity), + x_BilledCostInUsd = toreal(x_BilledCostInUsd), + x_BilledUnitPrice = toreal(x_BilledUnitPrice), + x_BillingExchangeRate = toreal(x_BillingExchangeRate), + x_ContractedCostInUsd = toreal(x_ContractedCostInUsd), + x_CurrencyConversionRate = toreal(x_CurrencyConversionRate), + x_EffectiveCostInUsd = toreal(x_EffectiveCostInUsd), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_ListCostInUsd = toreal(x_ListCostInUsd), + x_PricingBlockSize = toreal(x_PricingBlockSize) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId, + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + // Add new columns + | lookup kind=leftouter (Services | where isnotempty(x_ResourceType) | summarize take_any(ServiceSubcategory), take_any(x_ServiceModel) by x_ResourceType) on x_ResourceType + | extend CapacityReservationId = tostring(x_SkuDetails.VMCapacityReservationId) + | extend CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + | extend x_CommitmentDiscountNormalizedRatio = case( + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, dynamic(1))) + ) + | extend CommitmentDiscountQuantity = case( + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend CommitmentDiscountUnit = case( + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend x_AmortizationClass = case( + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + // Hubs add-ons + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + | extend x_SkuCoreCount = toint(coalesce(x_SkuDetails.VCPUs, x_SkuDetails.VCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(x_SkuDetails.ServiceType, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + x_SkuDetails.ImageType == 'Canonical', 'Linux', + x_SkuDetails.ImageType == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + x_SkuDetails.ImageType + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend tmp_SqlAhb = tolower(x_SkuDetails.AHB) + | extend x_SkuLicenseType = case( + x_SkuDetails.ImageType contains 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isnotempty(x_SkuLicenseType) or tmp_SqlAhb == 'true' or (x_SkuMeterSubcategory contains 'Azure Hybrid Benefit'), 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + | extend x_CommitmentDiscountSavings = iff(ContractedCost < EffectiveCost, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(ListCost < ContractedCost, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(ListCost < EffectiveCost, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(ContractedUnitPrice == 0, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // SkuPriceDetails conversion -- Must be after hubs add-ons + | extend SkuPriceDetails = parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | union ( + database('Ingestion').Costs_final_v1_2 + // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | union ( + database('Ingestion').Costs_final_v1_3 + // No column adds needed; v1_3 already has the FOCUS 1.3 columns. The + // deprecated ProviderName / PublisherName fall off via the | project + // list below since FOCUS 1.4 removes them. + ) + | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) + | project + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied, + ContractedCost, + ContractedUnitPrice, + EffectiveCost, + HostProviderName, + InvoiceId, + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingQuantity, + PricingUnit, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName, + ServiceSubcategory, + SkuId, + SkuMeter, + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName, + SubAccountType, + Tags, + x_AccountId, + x_AccountName, + x_AccountOwnerId, + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd, + x_CostAllocationRuleName, + x_CostCategories, + x_CostCenter, + x_CostType, + x_Credits, + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount, + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName, + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues, + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + + +// Prices_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all prices aligned to FOCUS 1.4-preview.', folder = 'Prices') +Prices_v1_4() +{ + database('Ingestion').Prices_final_v1_4 + | union ( + database('Ingestion').Prices_final_v1_0 + // Convert decimal to real + | extend + ContractedUnitPrice = toreal(ContractedUnitPrice), + ListUnitPrice = toreal(ListUnitPrice), + x_BaseUnitPrice = toreal(x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = toreal(x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = toreal(x_ContractedUnitPriceDiscountPercent), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_EffectiveUnitPriceDiscount = toreal(x_EffectiveUnitPriceDiscount), + x_EffectiveUnitPriceDiscountPercent = toreal(x_EffectiveUnitPriceDiscountPercent), + x_PricingBlockSize = toreal(x_PricingBlockSize), + x_SkuIncludedQuantity = toreal(x_SkuIncludedQuantity), + x_SkuTier = toreal(x_SkuTier), + x_TotalUnitPriceDiscount = toreal(x_TotalUnitPriceDiscount), + x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) + // Rename columns + | project-rename + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + ) + | project + BillingAccountId, + BillingAccountName, + BillingCurrency, + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType, + CommitmentDiscountUnit, + ContractedUnitPrice, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingUnit, + SkuId, + SkuMeter, + SkuPriceId, + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent, + x_EffectivePeriodEnd, + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_SkuDescription, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent +} + + +// Recommendations_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all recommendations aligned to FOCUS 1.4-preview.', folder = 'Recommendations') +Recommendations_v1_4() +{ + database('Ingestion').Recommendations_final_v1_4 + | union ( + database('Ingestion').Recommendations_final_v1_0 + // Convert decimal to real + | extend + x_EffectiveCostAfter = toreal(x_EffectiveCostAfter), + x_EffectiveCostBefore = toreal(x_EffectiveCostBefore), + x_EffectiveCostSavings = toreal(x_EffectiveCostSavings) + ) + | project + ProviderName, + ResourceId, + ResourceName, + ResourceType, + SubAccountId, + SubAccountName, + x_EffectiveCostAfter, + x_EffectiveCostBefore, + x_EffectiveCostSavings, + x_IngestionTime, + x_RecommendationCategory, + x_RecommendationDate, + x_RecommendationDescription, + x_RecommendationDetails, + x_RecommendationId, + x_ResourceGroupName, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Transactions_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all transactions aligned to FOCUS 1.4-preview.', folder = 'Transactions') +Transactions_v1_4() +{ + database('Ingestion').Transactions_final_v1_4 + | union ( + database('Ingestion').Transactions_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + PricingQuantity = toreal(PricingQuantity), + x_MonetaryCommitment = toreal(x_MonetaryCommitment), + x_Overage = toreal(x_Overage) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId + ) + | project + BilledCost, + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodStart, + InvoiceId, + PricingQuantity, + PricingUnit, + ProviderName, + RegionId, + RegionName, + SubAccountId, + SubAccountName, + x_AccountName, + x_AccountOwnerId, + x_CostCenter, + x_InvoiceNumber, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_IngestionTime, + x_MonetaryCommitment, + x_Overage, + x_PurchasingBillingAccountId, + x_SkuOrderId, + x_SkuOrderName, + x_SkuSize, + x_SkuTerm, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId, + x_TransactionType +} + + +//====================================================================================================================== +// Latest FOCUS version +//====================================================================================================================== + +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage() +{ + CommitmentDiscountUsage_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') +Costs() +{ + Costs_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') +Prices() +{ + Prices_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') +Recommendations() +{ + Recommendations_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') +Transactions() +{ + Transactions_v1_4() +} From 657aa7d2aa8fc991d7b19b5b571557c9f9ec4786 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 11:08:53 -0700 Subject: [PATCH 06/19] Refresh Claude Code plugin skill files for FOCUS 1.3 / 1.4 (#2119) Updates plugin skill files so plugin guidance reflects the new hub schemas and FOCUS columns added in this milestone: - finops-hubs.md: lists the full set of unversioned functions (CommitmentDiscountUsage, ContractCommitment included), notes the v1_0 / v1_2 / v1_3 / v1_4-preview versioned variants, and enumerates the FOCUS 1.3 column additions plus the 1.4 ProviderName / PublisherName removal. - finops-hubs-deployment.md: extends the supported FOCUS export versions to 1.0r2 / 1.2 / 1.2-preview (current Cost Management) and notes that hubs can also ingest 1.3 / 1.4-preview when Cost Management ships those exports. - ftk-database-query.md: expands from four to six analytic functions, adds CommitmentDiscountUsage and ContractCommitment sections, and surfaces the FOCUS 1.3 columns on the Costs() reference table. - ftk-output-style.md: adds FOCUS 1.3+ terminology preferences for ServiceProviderName, HostProviderName, and ContractApplied. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../references/finops-hubs-deployment.md | 3 ++- .../finops-toolkit/references/finops-hubs.md | 5 ++++- .../claude-plugin/agents/ftk-database-query.md | 13 ++++++++++++- .../claude-plugin/output-styles/ftk-output-style.md | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md b/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md index 6ec35855b..1a4c898ec 100644 --- a/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md +++ b/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md @@ -127,7 +127,8 @@ Create exports for each scope you want to monitor: | Resource Group | `subscriptions/{subscription-id}/resourceGroups/{rg-name}` | **Supported Datasets:** -- Cost and usage details (FOCUS) - `1.0`, `1.0r2` + +- Cost and usage details (FOCUS) - `1.0`, `1.0r2`, `1.2`, `1.2-preview` (Cost Management export versions). Hubs can also ingest `1.3` and `1.4-preview` if Cost Management ships those exports. - Price sheet - `2023-05-01` (required for missing prices) - Reservation details - `2023-03-01` - Reservation recommendations - `2023-05-01` (required for Rate optimization report) diff --git a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md index 54f886102..a66bf2b83 100644 --- a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md +++ b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md @@ -50,7 +50,10 @@ All KQL queries are located in `references/queries/`: **Database Rules:** - Always use "Hub" database, NEVER "Ingestion" -- Function-based access: `Costs()`, `Prices()`, `Recommendations()`, `Transactions()` +- Function-based access: `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `ContractCommitment()`, `Recommendations()`, `Transactions()` +- The unversioned functions (`Costs()`, etc.) return the latest GA FOCUS schema. Use versioned functions (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_3()`) to pin to a specific schema. `Costs_v1_4()` is **preview** while FOCUS 1.4 is in working draft and may change. +- FOCUS 1.3 added: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags` (data-generator split cost allocation), `ContractApplied` (per-row contract commitment application), `ServiceProviderName` and `HostProviderName` (replacing deprecated `ProviderName` / `PublisherName`). FOCUS 1.4 removes `ProviderName` and `PublisherName` entirely. +- The `ContractCommitment()` function (FOCUS 1.3+) returns provider-confirmed contract commitment metadata — the dataset feeding `ContractApplied` JSON arrays on each cost row. --- diff --git a/src/templates/claude-plugin/agents/ftk-database-query.md b/src/templates/claude-plugin/agents/ftk-database-query.md index 42e20002b..8492bf1f2 100644 --- a/src/templates/claude-plugin/agents/ftk-database-query.md +++ b/src/templates/claude-plugin/agents/ftk-database-query.md @@ -10,7 +10,7 @@ You are a FinOps Toolkit database specialist with deep expertise in the FinOps h ## Database Architecture -The FinOps hubs database exposes four main analytic functions: +The FinOps hubs database exposes six main analytic functions. The unversioned forms below return the latest GA FOCUS schema; pin to a specific schema with the versioned variants (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_3()`). `Costs_v1_4()` is **preview** while FOCUS 1.4 is in working draft and may change. ### Costs() @@ -28,6 +28,9 @@ The primary table for cost and usage analytics. Aligned with the FOCUS specifica | ChargeCategory | string | Category of the charge (Usage, Purchase) | | PricingCategory | string | Category of pricing (Standard, Spot, Committed) | | CommitmentDiscountStatus | string | Status of commitment discount (Used, Unused) | +| ContractApplied | dynamic | (FOCUS 1.3+) JSON array of contract commitments applied to this row | +| ServiceProviderName | string | (FOCUS 1.3+) Provider that made the resource available; replaces deprecated `ProviderName` (removed in 1.4) | +| HostProviderName | string | (FOCUS 1.3+) Underlying infrastructure provider; replaces deprecated `PublisherName` (removed in 1.4) | | ResourceId | string | Unique identifier for the resource | | ResourceName | string | Name of the resource | | ResourceType | string | Type of resource | @@ -41,6 +44,14 @@ The primary table for cost and usage analytics. Aligned with the FOCUS specifica Price sheets with list, contracted, and effective pricing. Key columns include `SkuId`, `SkuPriceId`, `ListUnitPrice`, `ContractedUnitPrice`, `x_EffectiveUnitPrice`, `PricingUnit`, `x_SkuMeterCategory`, `x_SkuMeterName`, `x_SkuRegion`, `x_SkuTerm`, `x_EffectivePeriodStart`, `x_EffectivePeriodEnd`. +### CommitmentDiscountUsage() + +Reservation and savings plan utilization, joining commitment discounts to the resources that consumed them. Key columns include `ChargePeriodStart`, `ChargePeriodEnd`, `CommitmentDiscountId`, `CommitmentDiscountQuantity`, `CommitmentDiscountUnit`, `ConsumedQuantity`, `ResourceId`, `ServiceName`, `x_CommitmentDiscountCommittedCount`, `x_CommitmentDiscountNormalizedRatio`. + +### ContractCommitment() + +(FOCUS 1.3+) Provider-confirmed contract commitment metadata — the dataset that feeds `ContractApplied` JSON arrays on each row in `Costs()`. Key columns include `ContractCommitmentId`, `ContractCommitmentCategory` (Spend / Usage), `ContractCommitmentCost`, `ContractCommitmentQuantity`, `ContractCommitmentPeriodStart`, `ContractCommitmentPeriodEnd`, `ContractId`, `BillingCurrency`, `InvoiceIssuerName`. FOCUS 1.4 preview adds payment-term and lifecycle columns (`PaymentModel`, `PaymentInterval`, `LifecycleStatus`, etc.). + ### Recommendations() Reservation and savings plan recommendations from Microsoft. Key columns include `x_EffectiveCostBefore`, `x_EffectiveCostAfter`, `x_EffectiveCostSavings`, `x_RecommendationDate`, `x_RecommendationDetails` (dynamic), `SubAccountId`. diff --git a/src/templates/claude-plugin/output-styles/ftk-output-style.md b/src/templates/claude-plugin/output-styles/ftk-output-style.md index 22ad6ec2e..3de91b8ce 100644 --- a/src/templates/claude-plugin/output-styles/ftk-output-style.md +++ b/src/templates/claude-plugin/output-styles/ftk-output-style.md @@ -133,7 +133,7 @@ Every recommendation must include: ## FinOps domain conventions - Reference FinOps Framework capabilities by their official names (e.g., "Managing commitment-based discounts", not "reservation management") -- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost) +- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost). Prefer FOCUS 1.3+ names where applicable: `ServiceProviderName` (replaces deprecated `ProviderName`, removed in 1.4) and `HostProviderName` (replaces deprecated `PublisherName`, removed in 1.4); `ContractApplied` and the `ContractCommitment()` dataset for contract commitment tracking. - Reference maturity levels as Crawl/Walk/Run when discussing FinOps practice maturity - Cite the six FinOps principles when they are relevant to a recommendation - For Azure-specific guidance, reference the official Microsoft documentation URL From f0904613b1d2fee80f35ec0a0d34c8e98b9a10e7 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 11:12:49 -0700 Subject: [PATCH 07/19] Add FOCUS schema tests and changelog entries (#2120) - HubsFocusSchemas.Tests.ps1: 108 Pester tests asserting the FOCUS 1.3 and 1.4-preview schema additions land where expected: Costs_raw additions, ContractCommitment_raw definition with both 1.3 and 1.4 columns, presence of versioned transforms and final tables in IngestionSetup_v1_3 / v1_4, presence of versioned hub functions in HubSetup_v1_3 / v1_4, the union arms in Costs_v1_3 and Costs_v1_4, the Latest aliases pinned to v1_3 (not preview v1_4), and the Bicep / build-config wiring. - changelog.md: adds v15.0.0 hub entries describing the FOCUS 1.3 GA and 1.4-preview schema additions, the new ContractCommitment dataset, the unversioned alias retarget, and the upgrade-procedure refresh. Adds a plugin entry for #2119. Updates ms.date to today. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs-mslearn/toolkit/changelog.md | 14 +- .../Tests/Unit/HubsFocusSchemas.Tests.ps1 | 228 ++++++++++++++++++ 2 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index dc22e673f..651aeb6b5 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 04/29/2026 +ms.date: 05/06/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -31,6 +31,18 @@ The following section lists features and enhancements that are currently in deve - Added Claude Code plugin with skills for FinOps hubs and Azure Cost Management ([#2043](https://github.com/microsoft/finops-toolkit/pull/2043)). - Added 4 agents (CFO, FinOps practitioner, database query, hubs agent), 5 commands (`/ftk-hubs-connect`, `/ftk-hubs-healthCheck`, `/ftk-mom-report`, `/ftk-ytd-report`, `/ftk-cost-optimization`), and an output style. - Linked to the existing KQL query catalog in `src/queries/` from the plugin. +- **Changed** + - Updated plugin skill files to reflect FOCUS 1.3 / 1.4-preview hub schemas, the new `ContractCommitment()` function, and the deprecation of `ProviderName` / `PublisherName` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + +### FinOps hubs v15.0.0 + +- **Added** + - Added FOCUS 1.3 hub schema (`v1_3`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`. New supplemental dataset: `ContractCommitment` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + - Added FOCUS 1.4-preview hub schema (`v1_4`). Drops `ProviderName` and `PublisherName` from cost and usage. Expands `ContractCommitment` by 14 columns (`BenefitCategory`, `ContractCommitmentApplicability`, `Created`, `DiscountPercentage`, `DurationType`, `FulfillmentInterval`, `LastUpdated`, `LifecycleStatus`, `Model`, `OfferCategory`, `PaymentInterval`, `PaymentModel`, `PaymentUpfrontPercentage`, `PricingCurrencyContractCommitmentCost`). Marked **preview** because FOCUS 1.4 is still in working draft and may change before ratification. + - Added unversioned `ContractCommitment()` function aliasing to `ContractCommitment_v1_3()`. +- **Changed** + - Retargeted unversioned `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `Recommendations()`, and `Transactions()` aliases to their `_v1_3` counterparts. Unversioned aliases stay pinned to the latest GA schema (v1_3) — `_v1_4` is opt-in only while in preview. + - Refreshed the canonical "add a new FOCUS version" procedure in [src/templates/finops-hub/docs/README.md](https://github.com/microsoft/finops-toolkit/tree/dev/src/templates/finops-hub/docs/README.md) with multi-version-cycle, plugin, and changelog steps. ### Bicep Registry module pending updates diff --git a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 new file mode 100644 index 000000000..1e9e18432 --- /dev/null +++ b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 @@ -0,0 +1,228 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe 'HubsFocusSchemas' { + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $scriptsPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts' + $rawTablesContent = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_RawTables.kql') -Raw + + $ingestionFiles = @{ + v1_0 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_0.kql') -Raw + v1_2 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_2.kql') -Raw + v1_3 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_3.kql') -Raw + v1_4 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_4.kql') -Raw + } + + $hubFiles = @{ + v1_0 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_0.kql') -Raw + v1_2 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_2.kql') -Raw + v1_3 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_3.kql') -Raw + v1_4 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_4.kql') -Raw + Latest = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_Latest.kql') -Raw + } + + $appBicep = Get-Content -Path (Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep') -Raw + $buildConfig = Get-Content -Path (Join-Path $repoRoot 'src/templates/finops-hub/.build.config') -Raw + } + + Context 'FOCUS 1.3 columns in Costs_raw' { + + BeforeAll { + # Extract just the Costs_raw alter block (not Costs_final or any other table). + $script:costsRawBlock = if ($rawTablesContent -match '(?ms)\.alter table Costs_raw \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + } + + It 'Costs_raw block was extracted' { + $costsRawBlock | Should -Not -BeNullOrEmpty + } + + It 'Adds <_> to Costs_raw' -ForEach @( + 'AllocatedMethodId', 'AllocatedMethodDetails', 'AllocatedResourceId', + 'AllocatedResourceName', 'AllocatedTags', 'ContractApplied', + 'ServiceProviderName', 'HostProviderName' + ) { + $costsRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'ContractCommitment_raw exists with FOCUS 1.3 + 1.4 columns' { + + BeforeAll { + # The Redefine-all-columns alter-table block is the second occurrence; match all and pick it. + $allBlocks = [regex]::Matches($rawTablesContent, '(?ms)\.alter table ContractCommitment_raw \(\r?\n(.*?)\r?\n\)') + $script:contractCommitmentRawBlock = if ($allBlocks.Count -ge 1) { $allBlocks[$allBlocks.Count - 1].Groups[1].Value } else { '' } + } + + It 'Defines ContractCommitment_raw' { + $rawTablesContent | Should -Match '\.alter table ContractCommitment_raw \(' + } + + It 'ContractCommitment_raw column block was extracted' { + $contractCommitmentRawBlock | Should -Not -BeNullOrEmpty + } + + It 'Includes FOCUS 1.3 column <_>' -ForEach @( + 'BillingCurrency', 'ContractCommitmentCategory', 'ContractCommitmentCost', + 'ContractCommitmentId', 'ContractCommitmentPeriodEnd', 'ContractCommitmentPeriodStart', + 'ContractCommitmentQuantity', 'ContractCommitmentType', 'ContractCommitmentUnit', + 'ContractId', 'ContractPeriodEnd', 'ContractPeriodStart', 'InvoiceIssuerName', + 'PricingCurrency' + ) { + $contractCommitmentRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + + It 'Includes FOCUS 1.4 column <_>' -ForEach @( + 'BenefitCategory', 'ContractCommitmentApplicability', 'Created', + 'DiscountPercentage', 'DurationType', 'FulfillmentInterval', 'LastUpdated', + 'LifecycleStatus', 'Model', 'OfferCategory', 'PaymentInterval', 'PaymentModel', + 'PaymentUpfrontPercentage', 'PricingCurrencyContractCommitmentCost' + ) { + $contractCommitmentRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'IngestionSetup_v1_3.kql' { + + BeforeAll { + $script:costsFinalV13Block = if ($ingestionFiles.v1_3 -match '(?ms)\.create-merge table Costs_final_v1_3 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + } + + It 'Defines Costs_transform_v1_3' { + $ingestionFiles.v1_3 | Should -Match 'Costs_transform_v1_3\(\)' + } + + It 'Defines Costs_final_v1_3 table' { + $ingestionFiles.v1_3 | Should -Match '\.create-merge table Costs_final_v1_3' + } + + It 'Defines ContractCommitment_transform_v1_3' { + $ingestionFiles.v1_3 | Should -Match 'ContractCommitment_transform_v1_3\(\)' + } + + It 'Defines ContractCommitment_final_v1_3 table' { + $ingestionFiles.v1_3 | Should -Match '\.create-merge table ContractCommitment_final_v1_3' + } + + It 'Costs_final_v1_3 block was extracted' { + $costsFinalV13Block | Should -Not -BeNullOrEmpty + } + + It 'Costs_final_v1_3 includes FOCUS 1.3 column <_>' -ForEach @( + 'AllocatedMethodId', 'AllocatedMethodDetails', 'AllocatedResourceId', + 'AllocatedResourceName', 'AllocatedTags', 'ContractApplied', + 'ServiceProviderName', 'HostProviderName' + ) { + $costsFinalV13Block | Should -Match "(?m)^\s+$_\s*:" + } + + It 'Costs_final_v1_3 still includes deprecated <_> for back compat' -ForEach @( + 'ProviderName', 'PublisherName' + ) { + $costsFinalV13Block | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'IngestionSetup_v1_4.kql' { + + BeforeAll { + $script:costsFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table Costs_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + $script:contractCommitmentFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table ContractCommitment_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + } + + It 'Defines Costs_transform_v1_4' { + $ingestionFiles.v1_4 | Should -Match 'Costs_transform_v1_4\(\)' + } + + It 'Defines Costs_final_v1_4 table' { + $ingestionFiles.v1_4 | Should -Match '\.create-merge table Costs_final_v1_4' + } + + It 'Costs_final_v1_4 block was extracted' { + $costsFinalV14Block | Should -Not -BeNullOrEmpty + } + + It 'Costs_final_v1_4 does NOT include deprecated <_> (removed in 1.4)' -ForEach @( + 'ProviderName', 'PublisherName' + ) { + $costsFinalV14Block | Should -Not -Match "(?m)^\s+$_\s*:" + } + + It 'ContractCommitment_final_v1_4 includes FOCUS 1.4 column <_>' -ForEach @( + 'BenefitCategory', 'ContractCommitmentApplicability', 'Created', + 'DiscountPercentage', 'DurationType', 'FulfillmentInterval', 'LastUpdated', + 'LifecycleStatus', 'Model', 'OfferCategory', 'PaymentInterval', 'PaymentModel', + 'PaymentUpfrontPercentage', 'PricingCurrencyContractCommitmentCost' + ) { + $contractCommitmentFinalV14Block | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'HubSetup_v1_3.kql' { + + It 'Defines <_>' -ForEach @( + 'CommitmentDiscountUsage_v1_3', 'ContractCommitment_v1_3', + 'Costs_v1_3', 'Prices_v1_3', 'Recommendations_v1_3', 'Transactions_v1_3' + ) { + $hubFiles.v1_3 | Should -Match "$_\(\)" + } + + It 'Costs_v1_3 unions all prior versions' -ForEach @( + 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_3' + ) { + $hubFiles.v1_3 | Should -Match "database\('Ingestion'\)\.$_" + } + } + + Context 'HubSetup_v1_4.kql (preview)' { + + It 'Defines <_>' -ForEach @( + 'CommitmentDiscountUsage_v1_4', 'ContractCommitment_v1_4', + 'Costs_v1_4', 'Prices_v1_4', 'Recommendations_v1_4', 'Transactions_v1_4' + ) { + $hubFiles.v1_4 | Should -Match "$_\(\)" + } + + It 'Costs_v1_4 unions all prior versions' -ForEach @( + 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_3', 'Costs_final_v1_4' + ) { + $hubFiles.v1_4 | Should -Match "database\('Ingestion'\)\.$_" + } + + It 'Marks itself as preview in docstring' { + $hubFiles.v1_4 | Should -Match 'FOCUS 1\.4-preview' + } + } + + Context 'HubSetup_Latest.kql aliases' { + + It 'Aliases <_> to v1_3 (latest GA, not v1_4 preview)' -ForEach @( + 'CommitmentDiscountUsage', 'Costs', 'Prices', 'Recommendations', 'Transactions', + 'ContractCommitment' + ) { + $hubFiles.Latest | Should -Match "(?ms)$_\(\)\s*\{\s*${_}_v1_3\(\)\s*\}" + } + + It 'Does NOT alias to v1_4 (preview must be opt-in)' { + $hubFiles.Latest | Should -Not -Match '_v1_4\(\)' + } + } + + Context 'Bicep wiring' { + + It 'app.bicep loads <_>' -ForEach @( + 'IngestionSetup_v1_3.kql', 'IngestionSetup_v1_4.kql', + 'HubSetup_v1_3.kql', 'HubSetup_v1_4.kql' + ) { + $appBicep | Should -Match ([regex]::Escape($_)) + } + + It '.build.config lists <_>' -ForEach @( + 'IngestionSetup_v1_3.kql', 'IngestionSetup_v1_4.kql', + 'HubSetup_v1_3.kql', 'HubSetup_v1_4.kql' + ) { + $buildConfig | Should -Match ([regex]::Escape($_)) + } + } +} From c56b616547c3f00d912b2c7bcfe2c9b6e60ee555 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:24:08 -0700 Subject: [PATCH 08/19] Add FOCUS 1.4 ingestion (IngestionSetup_v1_4.kql) Consolidates FOCUS 1.3 and 1.4 column additions into a single v1_4 ingestion setup file. Removes the intermediate v1_3 file, adds ContractCommitment_raw table definition, and updates app.bicep and .build.config to reference v1_4. Co-Authored-By: Claude --- src/templates/finops-hub/.build.config | 2 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 2 +- .../scripts/IngestionSetup_RawTables.kql | 96 +++++++++ ...Setup_v1_3.kql => IngestionSetup_v1_4.kql} | 201 +++++++++++++----- 4 files changed, 246 insertions(+), 55 deletions(-) rename src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/{IngestionSetup_v1_3.kql => IngestionSetup_v1_4.kql} (92%) diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 0963fb494..59ccbdc8c 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -34,7 +34,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 42cd2738b..e20e982bd 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -378,7 +378,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) scripts: { v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') - v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') + v1_4: loadTextContent('scripts/IngestionSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 83a3be40e..cdc281659 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -335,6 +335,102 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// ContractCommitment_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Redefine all columns +.alter table ContractCommitment_raw ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_3+ + x_SourceProvider: string, // Hubs v1_3+ + x_SourceType: string, // Hubs v1_3+ + x_SourceVersion: string // Hubs v1_3+ +) + +// ContractCommitment_raw ingestion mapping +.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +``` +[ + { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentApplicability", "Properties": { "Field": "ContractCommitmentApplicability" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "Created", "Properties": { "Field": "Created" } }, + { "Column": "DiscountPercentage", "Properties": { "Field": "DiscountPercentage" } }, + { "Column": "DurationType", "Properties": { "Field": "DurationType" } }, + { "Column": "FulfillmentInterval", "Properties": { "Field": "FulfillmentInterval" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "LastUpdated", "Properties": { "Field": "LastUpdated" } }, + { "Column": "LifecycleStatus", "Properties": { "Field": "LifecycleStatus" } }, + { "Column": "Model", "Properties": { "Field": "Model" } }, + { "Column": "OfferCategory", "Properties": { "Field": "OfferCategory" } }, + { "Column": "PaymentInterval", "Properties": { "Field": "PaymentInterval" } }, + { "Column": "PaymentModel", "Properties": { "Field": "PaymentModel" } }, + { "Column": "PaymentUpfrontPercentage", "Properties": { "Field": "PaymentUpfrontPercentage" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "PricingCurrencyContractCommitmentCost", "Properties": { "Field": "PricingCurrencyContractCommitmentCost" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// ContractCommitment_raw retention policy (clear historical data) +.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled + +// ContractCommitment_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable ContractCommitment_raw streaming ingestion (required for Fabric) +.alter table ContractCommitment_raw policy streamingingestion disable + + //===| Costs |========================================================================================================== // Supported versions: // - MS: 1.0, 1.0-preview(v1) -- See https://aka.ms/costmgmt/exports/focus diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql similarity index 92% rename from src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index 70d29a2ed..84e8d6fcd 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -14,10 +14,10 @@ // - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca //====================================================================================================================== -// Prices_transform_v1_3 function +// Prices_transform_v1_4 function .create-or-alter function -with (docstring='Transforms Prices_raw into FOCUS 1.3.', folder='Prices') -Prices_transform_v1_3() +with (docstring='Transforms Prices_raw into FOCUS 1.4.', folder='Prices') +Prices_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -196,8 +196,8 @@ Prices_transform_v1_3() x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 } -// Prices_final_v1_3 table -.create-merge table Prices_final_v1_3 ( +// Prices_final_v1_4 table +.create-merge table Prices_final_v1_4 ( BillingAccountId: string, BillingAccountName: string, BillingCurrency: string, @@ -255,13 +255,13 @@ Prices_transform_v1_3() x_TotalUnitPriceDiscountPercent: real // Hubs add-on ) -// Update policy for Prices_raw -> Prices_final_v1_3 -.alter table Prices_final_v1_3 policy update +// Update policy for Prices_raw -> Prices_final_v1_4 +.alter table Prices_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Prices_raw", - "Query": "Prices_transform_v1_3()", + "Query": "Prices_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -286,10 +286,10 @@ Prices_transform_v1_3() // Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. //====================================================================================================================== -// Costs_transform_v1_3 function +// Costs_transform_v1_4 function .create-or-alter function -with (docstring='All costs transformed to FOCUS 1.3.', folder='Costs') -Costs_transform_v1_3() +with (docstring='All costs transformed to FOCUS 1.4.', folder='Costs') +Costs_transform_v1_4() { let checkString = (column: string, oldValue: string, newValue: string) { iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) @@ -419,7 +419,7 @@ Costs_transform_v1_3() | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) | as costsWithMissingPrices | join kind=leftouter ( - Prices_final_v1_3 + Prices_final_v1_4 | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit @@ -758,8 +758,7 @@ Costs_transform_v1_3() PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), PricingQuantity, PricingUnit, - ProviderName, - PublisherName, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -921,8 +920,8 @@ Costs_transform_v1_3() x_UsageType } -// Costs_final_v1_3 table -.create-merge table Costs_final_v1_3 ( +// Costs_final_v1_4 table +.create-merge table Costs_final_v1_4 ( AllocatedMethodDetails: dynamic, // FOCUS 1.3+ AllocatedMethodId: string, // FOCUS 1.3+ AllocatedResourceId: string, // FOCUS 1.3+ @@ -966,8 +965,7 @@ Costs_transform_v1_3() PricingCurrency: string, PricingQuantity: real, PricingUnit: string, - ProviderName: string, - PublisherName: string, + // ProviderName and PublisherName removed in FOCUS 1.4 RegionId: string, RegionName: string, ResourceId: string, @@ -1088,13 +1086,13 @@ Costs_transform_v1_3() x_UsageType: string // AWS 1.0 ) -// Update policy for Costs_raw -> Costs_final_v1_3 table -.alter table Costs_final_v1_3 policy update +// Update policy for Costs_raw -> Costs_final_v1_4 table +.alter table Costs_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Costs_raw", - "Query": "Costs_transform_v1_3()", + "Query": "Costs_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1106,10 +1104,10 @@ Costs_transform_v1_3() // - C360-2025-04 //====================================================================================================================== -// ActualCosts_transform_v1_3 function +// ActualCosts_transform_v1_4 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') -ActualCosts_transform_v1_3() +with (docstring='ActualCost exports transformed to FOCUS 1.4.', folder='Costs') +ActualCosts_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 ActualCosts_raw @@ -1316,7 +1314,7 @@ ActualCosts_transform_v1_3() [{ "IsEnabled": true, "Source": "ActualCosts_raw", - "Query": "ActualCosts_transform_v1_3()", + "Query": "ActualCosts_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1328,10 +1326,10 @@ ActualCosts_transform_v1_3() // - C360-2025-04 //====================================================================================================================== -// AmortizedCosts_transform_v1_3 function +// AmortizedCosts_transform_v1_4 function .create-or-alter function -with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') -AmortizedCosts_transform_v1_3() +with (docstring='ActualCost exports transformed to FOCUS 1.4.', folder='Costs') +AmortizedCosts_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 AmortizedCosts_raw @@ -1537,7 +1535,7 @@ AmortizedCosts_transform_v1_3() [{ "IsEnabled": true, "Source": "AmortizedCosts_raw", - "Query": "AmortizedCosts_transform_v1_3()", + "Query": "AmortizedCosts_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1550,10 +1548,10 @@ AmortizedCosts_transform_v1_3() // - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca //====================================================================================================================== -// CommitmentDiscountUsage_transform_v1_3 function +// CommitmentDiscountUsage_transform_v1_4 function .create-or-alter function -with (docstring='All commitment discount usage transformed to FOCUS 1.3. This includes reservationdeatils_raw.', folder='Commitment discounts') -CommitmentDiscountUsage_transform_v1_3() +with (docstring='All commitment discount usage transformed to FOCUS 1.4. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 CommitmentDiscountUsage_raw @@ -1616,8 +1614,8 @@ CommitmentDiscountUsage_transform_v1_3() x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) } -// CommitmentDiscountUsage_final_v1_3 table -.create-merge table CommitmentDiscountUsage_final_v1_3 ( +// CommitmentDiscountUsage_final_v1_4 table +.create-merge table CommitmentDiscountUsage_final_v1_4 ( ChargePeriodEnd: datetime, // Hubs add-on ChargePeriodStart: datetime, // MS 2023-03-01 CommitmentDiscountCategory: string, // Hubs add-on @@ -1650,13 +1648,110 @@ CommitmentDiscountUsage_transform_v1_3() x_SourceVersion: string // Hubs add-on ) -// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_3 table -.alter table CommitmentDiscountUsage_final_v1_3 policy update +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_4 table +.alter table CommitmentDiscountUsage_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "CommitmentDiscountUsage_raw", - "Query": "CommitmentDiscountUsage_transform_v1_3()", + "Query": "CommitmentDiscountUsage_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitment_transform_v1_4 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') +ContractCommitment_transform_v1_4() +{ + ContractCommitment_raw + | project + BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ + BillingCurrency, + ContractCommitmentApplicability = parse_json(column_ifexists('ContractCommitmentApplicability', '')),// FOCUS 1.4+ + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + Created = column_ifexists('Created', datetime(null)), // FOCUS 1.4+ + DiscountPercentage = column_ifexists('DiscountPercentage', real(null)), // FOCUS 1.4+ + DurationType = column_ifexists('DurationType', ''), // FOCUS 1.4+ + FulfillmentInterval = column_ifexists('FulfillmentInterval', ''), // FOCUS 1.4+ + InvoiceIssuerName, + LastUpdated = column_ifexists('LastUpdated', datetime(null)), // FOCUS 1.4+ + LifecycleStatus = column_ifexists('LifecycleStatus', ''), // FOCUS 1.4+ + Model = column_ifexists('Model', ''), // FOCUS 1.4+ + OfferCategory = column_ifexists('OfferCategory', ''), // FOCUS 1.4+ + PaymentInterval = column_ifexists('PaymentInterval', ''), // FOCUS 1.4+ + PaymentModel = column_ifexists('PaymentModel', ''), // FOCUS 1.4+ + PaymentUpfrontPercentage = column_ifexists('PaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ + PricingCurrency, + PricingCurrencyContractCommitmentCost = column_ifexists('PricingCurrencyContractCommitmentCost', real(null)), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitment_final_v1_4 table +.create-merge table ContractCommitment_final_v1_4 ( + BenefitCategory: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + Created: datetime, // FOCUS 1.4+ + DiscountPercentage: real, // FOCUS 1.4+ + DurationType: string, // FOCUS 1.4+ + FulfillmentInterval: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.3+ + LastUpdated: datetime, // FOCUS 1.4+ + LifecycleStatus: string, // FOCUS 1.4+ + Model: string, // FOCUS 1.4+ + OfferCategory: string, // FOCUS 1.4+ + PaymentInterval: string, // FOCUS 1.4+ + PaymentModel: string, // FOCUS 1.4+ + PaymentUpfrontPercentage: real, // FOCUS 1.4+ + PricingCurrency: string, // FOCUS 1.3+ + PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_4 table +.alter table ContractCommitment_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitment_raw", + "Query": "ContractCommitment_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1669,10 +1764,10 @@ CommitmentDiscountUsage_transform_v1_3() // - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca //====================================================================================================================== -// Recommendations_transform_v1_3 function +// Recommendations_transform_v1_4 function .create-or-alter function -with (docstring='All recommendations transformed to FOCUS 1.3.', folder='Recommendations') -Recommendations_transform_v1_3() +with (docstring='All recommendations transformed to FOCUS 1.4.', folder='Recommendations') +Recommendations_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -1777,8 +1872,8 @@ Recommendations_transform_v1_3() x_SourceVersion } -// Recommendations_final_v1_3 table -.create-merge table Recommendations_final_v1_3 ( +// Recommendations_final_v1_4 table +.create-merge table Recommendations_final_v1_4 ( ProviderName: string, ResourceId: string, ResourceName: string, @@ -1801,13 +1896,13 @@ Recommendations_transform_v1_3() x_SourceVersion: string ) -// Update policy for Recommendations_raw -> Recommendations_final_v1_3 table -.alter table Recommendations_final_v1_3 policy update +// Update policy for Recommendations_raw -> Recommendations_final_v1_4 table +.alter table Recommendations_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Recommendations_raw", - "Query": "Recommendations_transform_v1_3()", + "Query": "Recommendations_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] @@ -1820,10 +1915,10 @@ Recommendations_transform_v1_3() // - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca //====================================================================================================================== -// Transactions_transform_v1_3 function +// Transactions_transform_v1_4 function .create-or-alter function -with (docstring='All transactions transformed to FOCUS 1.3.', folder='Transactions') -Transactions_transform_v1_3() +with (docstring='All transactions transformed to FOCUS 1.4.', folder='Transactions') +Transactions_transform_v1_4() { // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 let isoMonths = (duration: string) { @@ -1910,8 +2005,8 @@ Transactions_transform_v1_3() x_TransactionType = EventType } -// Transactions_final_v1_3 table -.create-merge table Transactions_final_v1_3 ( +// Transactions_final_v1_4 table +.create-merge table Transactions_final_v1_4 ( BilledCost: real, // MS CM EA+MCA 2023-05-01 BillingAccountId: string, // MS CM EA+MCA 2023-05-01 BillingAccountName: string, // MS CM EA+MCA 2023-05-01 @@ -1953,13 +2048,13 @@ Transactions_transform_v1_3() x_TransactionType: string // MS CM EA+MCA 2023-05-01 ) -// Update policy for Transactions_raw -> Transactions_final_v1_3 table -.alter table Transactions_final_v1_3 policy update +// Update policy for Transactions_raw -> Transactions_final_v1_4 table +.alter table Transactions_final_v1_4 policy update ``` [{ "IsEnabled": true, "Source": "Transactions_raw", - "Query": "Transactions_transform_v1_3()", + "Query": "Transactions_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] From 8a01133a132a6d130057a04c95515c3c4cc352f3 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:25:49 -0700 Subject: [PATCH 09/19] Add FOCUS 1.4 hub functions (HubSetup_v1_4.kql) Renames HubSetup_v1_3.kql to HubSetup_v1_4.kql, consolidating all hub functions (including ContractCommitment) for FOCUS 1.4 GA. Updates HubSetup_Latest.kql to alias all unversioned functions to v1_4. Updates app.bicep and .build.config to reference v1_4. Co-Authored-By: Claude --- src/templates/finops-hub/.build.config | 2 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 2 +- .../Analytics/scripts/HubSetup_Latest.kql | 18 ++-- .../{HubSetup_v1_3.kql => HubSetup_v1_4.kql} | 88 +++++++++++++------ 4 files changed, 75 insertions(+), 35 deletions(-) rename src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/{HubSetup_v1_3.kql => HubSetup_v1_4.kql} (89%) diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index c8091cb04..e4807f5f0 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -44,7 +44,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 4cede47fe..f374c677c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -414,7 +414,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { scripts: { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') - v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') + v1_4: loadTextContent('scripts/HubSetup_v1_4.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 1895b39db..c3a4985f2 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -13,7 +13,15 @@ with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_3() + CommitmentDiscountUsage_v1_4() +} + + +.create-or-alter function +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') +ContractCommitment() +{ + ContractCommitment_v1_4() } @@ -21,7 +29,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_3() + Costs_v1_4() } @@ -29,7 +37,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_3() + Prices_v1_4() } @@ -37,7 +45,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_3() + Recommendations_v1_4() } @@ -45,5 +53,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_3() + Transactions_v1_4() } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql similarity index 89% rename from src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql rename to src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index 62995ffa5..cabdbdfc5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -2,19 +2,19 @@ // Licensed under the MIT License. //====================================================================================================================== -// Hub database / FOCUS 1.3 functions +// Hub database / FOCUS 1.4 functions // Used for reporting with backward compatibility. //====================================================================================================================== // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script -// CommitmentDiscountUsage_final_v1_3 +// CommitmentDiscountUsage_final_v1_4 .create-or-alter function -with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.3.', folder = 'CommitmentDiscountUsage') -CommitmentDiscountUsage_v1_3() +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.4.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_4() { - database('Ingestion').CommitmentDiscountUsage_final_v1_3 + database('Ingestion').CommitmentDiscountUsage_final_v1_4 | union ( database('Ingestion').CommitmentDiscountUsage_final_v1_0 // Convert decimal to real @@ -66,12 +66,39 @@ CommitmentDiscountUsage_v1_3() } -// Costs_final_v1_3 +// ContractCommitment_final_v1_4 .create-or-alter function -with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') -Costs_v1_3() +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitment') +ContractCommitment_v1_4() { - database('Ingestion').Costs_final_v1_3 + database('Ingestion').ContractCommitment_final_v1_4 + | union ( + database('Ingestion').ContractCommitment_final_v1_3 + // FOCUS 1.4 column defaults (data was ingested under v1_3 schema) + | extend BenefitCategory = '' + | extend ContractCommitmentApplicability = dynamic(null) + | extend Created = datetime(null) + | extend DiscountPercentage = real(null) + | extend DurationType = '' + | extend FulfillmentInterval = '' + | extend LastUpdated = datetime(null) + | extend LifecycleStatus = '' + | extend Model = '' + | extend OfferCategory = '' + | extend PaymentInterval = '' + | extend PaymentModel = '' + | extend PaymentUpfrontPercentage = real(null) + | extend PricingCurrencyContractCommitmentCost = real(null) + ) +} + + +// Costs_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.4.', folder = 'Costs') +Costs_v1_4() +{ + database('Ingestion').Costs_final_v1_4 | union ( database('Ingestion').Costs_final_v1_0 // Convert decimal to real @@ -220,6 +247,12 @@ Costs_v1_3() | extend ServiceProviderName = ProviderName | extend HostProviderName = PublisherName ) + | union ( + database('Ingestion').Costs_final_v1_3 + // No column adds needed; v1_3 already has the FOCUS 1.3 columns. The + // deprecated ProviderName / PublisherName fall off via the | project + // list below since FOCUS 1.4 removes them. + ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project AllocatedMethodDetails, @@ -265,8 +298,7 @@ Costs_v1_3() PricingCurrency, PricingQuantity, PricingUnit, - ProviderName, - PublisherName, + // ProviderName and PublisherName removed in FOCUS 1.4. Use ServiceProviderName / HostProviderName. RegionId, RegionName, ResourceId, @@ -388,12 +420,12 @@ Costs_v1_3() } -// Prices_final_v1_3 +// Prices_final_v1_4 .create-or-alter function -with (docstring = 'Gets all prices aligned to FOCUS 1.3.', folder = 'Prices') -Prices_v1_3() +with (docstring = 'Gets all prices aligned to FOCUS 1.4.', folder = 'Prices') +Prices_v1_4() { - database('Ingestion').Prices_final_v1_3 + database('Ingestion').Prices_final_v1_4 | union ( database('Ingestion').Prices_final_v1_0 // Convert decimal to real @@ -475,12 +507,12 @@ Prices_v1_3() } -// Recommendations_final_v1_3 +// Recommendations_final_v1_4 .create-or-alter function -with (docstring = 'Gets all recommendations aligned to FOCUS 1.3.', folder = 'Recommendations') -Recommendations_v1_3() +with (docstring = 'Gets all recommendations aligned to FOCUS 1.4.', folder = 'Recommendations') +Recommendations_v1_4() { - database('Ingestion').Recommendations_final_v1_3 + database('Ingestion').Recommendations_final_v1_4 | union ( database('Ingestion').Recommendations_final_v1_0 // Convert decimal to real @@ -513,12 +545,12 @@ Recommendations_v1_3() } -// Transactions_final_v1_3 +// Transactions_final_v1_4 .create-or-alter function -with (docstring = 'Gets all transactions aligned to FOCUS 1.3.', folder = 'Transactions') -Transactions_v1_3() +with (docstring = 'Gets all transactions aligned to FOCUS 1.4.', folder = 'Transactions') +Transactions_v1_4() { - database('Ingestion').Transactions_final_v1_3 + database('Ingestion').Transactions_final_v1_4 | union ( database('Ingestion').Transactions_final_v1_0 // Convert decimal to real @@ -582,7 +614,7 @@ Transactions_v1_3() with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_3() + CommitmentDiscountUsage_v1_4() } @@ -590,7 +622,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_3() + Costs_v1_4() } @@ -598,7 +630,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_3() + Prices_v1_4() } @@ -606,7 +638,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_3() + Recommendations_v1_4() } @@ -614,5 +646,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_3() + Transactions_v1_4() } From 58f50cdbaee96dfbb5100156685b28b1b854fae4 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:27:49 -0700 Subject: [PATCH 10/19] Refresh Claude Code plugin skill files for FOCUS 1.4 Updates all FOCUS version references from 1.3/1.4-preview to v1_4 GA. Removes intermediate v1_3 function references. Marks ContractCommitment as FOCUS 1.4+ (not 1.3+). Removes preview designations for v1_4. Co-Authored-By: Claude --- .../references/finops-hubs-deployment.md | 2 +- .../finops-toolkit/references/finops-hubs.md | 6 +++--- .../claude-plugin/agents/ftk-database-query.md | 10 +++++----- .../claude-plugin/output-styles/ftk-output-style.md | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md b/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md index 1a4c898ec..3fc3becb1 100644 --- a/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md +++ b/src/templates/agent-skills/finops-toolkit/references/finops-hubs-deployment.md @@ -128,7 +128,7 @@ Create exports for each scope you want to monitor: **Supported Datasets:** -- Cost and usage details (FOCUS) - `1.0`, `1.0r2`, `1.2`, `1.2-preview` (Cost Management export versions). Hubs can also ingest `1.3` and `1.4-preview` if Cost Management ships those exports. +- Cost and usage details (FOCUS) - `1.0`, `1.0r2`, `1.2`, `1.2-preview` (Cost Management export versions). - Price sheet - `2023-05-01` (required for missing prices) - Reservation details - `2023-03-01` - Reservation recommendations - `2023-05-01` (required for Rate optimization report) diff --git a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md index a66bf2b83..908dee713 100644 --- a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md +++ b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md @@ -51,9 +51,9 @@ All KQL queries are located in `references/queries/`: **Database Rules:** - Always use "Hub" database, NEVER "Ingestion" - Function-based access: `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `ContractCommitment()`, `Recommendations()`, `Transactions()` -- The unversioned functions (`Costs()`, etc.) return the latest GA FOCUS schema. Use versioned functions (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_3()`) to pin to a specific schema. `Costs_v1_4()` is **preview** while FOCUS 1.4 is in working draft and may change. -- FOCUS 1.3 added: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags` (data-generator split cost allocation), `ContractApplied` (per-row contract commitment application), `ServiceProviderName` and `HostProviderName` (replacing deprecated `ProviderName` / `PublisherName`). FOCUS 1.4 removes `ProviderName` and `PublisherName` entirely. -- The `ContractCommitment()` function (FOCUS 1.3+) returns provider-confirmed contract commitment metadata — the dataset feeding `ContractApplied` JSON arrays on each cost row. +- The unversioned functions (`Costs()`, etc.) return the latest GA FOCUS schema (v1_4). Use versioned functions (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_4()`) to pin to a specific schema. +- FOCUS 1.4 added: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags` (data-generator split cost allocation), `ContractApplied` (per-row contract commitment application), `ServiceProviderName` and `HostProviderName` (replacing deprecated `ProviderName` / `PublisherName`, which are removed in 1.4). Also added the `ContractCommitment` supplemental dataset. +- The `ContractCommitment()` function (FOCUS 1.4+) returns provider-confirmed contract commitment metadata — the dataset feeding `ContractApplied` JSON arrays on each cost row. --- diff --git a/src/templates/claude-plugin/agents/ftk-database-query.md b/src/templates/claude-plugin/agents/ftk-database-query.md index 8492bf1f2..e94ee96dc 100644 --- a/src/templates/claude-plugin/agents/ftk-database-query.md +++ b/src/templates/claude-plugin/agents/ftk-database-query.md @@ -10,7 +10,7 @@ You are a FinOps Toolkit database specialist with deep expertise in the FinOps h ## Database Architecture -The FinOps hubs database exposes six main analytic functions. The unversioned forms below return the latest GA FOCUS schema; pin to a specific schema with the versioned variants (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_3()`). `Costs_v1_4()` is **preview** while FOCUS 1.4 is in working draft and may change. +The FinOps hubs database exposes six main analytic functions. The unversioned forms below return the latest GA FOCUS schema (v1_4); pin to a specific schema with the versioned variants (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_4()`). ### Costs() @@ -28,9 +28,9 @@ The primary table for cost and usage analytics. Aligned with the FOCUS specifica | ChargeCategory | string | Category of the charge (Usage, Purchase) | | PricingCategory | string | Category of pricing (Standard, Spot, Committed) | | CommitmentDiscountStatus | string | Status of commitment discount (Used, Unused) | -| ContractApplied | dynamic | (FOCUS 1.3+) JSON array of contract commitments applied to this row | -| ServiceProviderName | string | (FOCUS 1.3+) Provider that made the resource available; replaces deprecated `ProviderName` (removed in 1.4) | -| HostProviderName | string | (FOCUS 1.3+) Underlying infrastructure provider; replaces deprecated `PublisherName` (removed in 1.4) | +| ContractApplied | dynamic | (FOCUS 1.4+) JSON array of contract commitments applied to this row | +| ServiceProviderName | string | (FOCUS 1.4+) Provider that made the resource available; replaces removed `ProviderName` | +| HostProviderName | string | (FOCUS 1.4+) Underlying infrastructure provider; replaces removed `PublisherName` | | ResourceId | string | Unique identifier for the resource | | ResourceName | string | Name of the resource | | ResourceType | string | Type of resource | @@ -50,7 +50,7 @@ Reservation and savings plan utilization, joining commitment discounts to the re ### ContractCommitment() -(FOCUS 1.3+) Provider-confirmed contract commitment metadata — the dataset that feeds `ContractApplied` JSON arrays on each row in `Costs()`. Key columns include `ContractCommitmentId`, `ContractCommitmentCategory` (Spend / Usage), `ContractCommitmentCost`, `ContractCommitmentQuantity`, `ContractCommitmentPeriodStart`, `ContractCommitmentPeriodEnd`, `ContractId`, `BillingCurrency`, `InvoiceIssuerName`. FOCUS 1.4 preview adds payment-term and lifecycle columns (`PaymentModel`, `PaymentInterval`, `LifecycleStatus`, etc.). +(FOCUS 1.4+) Provider-confirmed contract commitment metadata — the dataset that feeds `ContractApplied` JSON arrays on each row in `Costs()`. Key columns include `ContractCommitmentId`, `ContractCommitmentCategory` (Spend / Usage), `ContractCommitmentCost`, `ContractCommitmentQuantity`, `ContractCommitmentPeriodStart`, `ContractCommitmentPeriodEnd`, `ContractId`, `BillingCurrency`, `InvoiceIssuerName`, `PaymentModel`, `PaymentInterval`, `LifecycleStatus`, `DurationType`. ### Recommendations() diff --git a/src/templates/claude-plugin/output-styles/ftk-output-style.md b/src/templates/claude-plugin/output-styles/ftk-output-style.md index 3de91b8ce..c2d3f9457 100644 --- a/src/templates/claude-plugin/output-styles/ftk-output-style.md +++ b/src/templates/claude-plugin/output-styles/ftk-output-style.md @@ -133,7 +133,7 @@ Every recommendation must include: ## FinOps domain conventions - Reference FinOps Framework capabilities by their official names (e.g., "Managing commitment-based discounts", not "reservation management") -- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost). Prefer FOCUS 1.3+ names where applicable: `ServiceProviderName` (replaces deprecated `ProviderName`, removed in 1.4) and `HostProviderName` (replaces deprecated `PublisherName`, removed in 1.4); `ContractApplied` and the `ContractCommitment()` dataset for contract commitment tracking. +- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost). Prefer FOCUS 1.4 names: `ServiceProviderName` (replaces removed `ProviderName`) and `HostProviderName` (replaces removed `PublisherName`); `ContractApplied` and the `ContractCommitment()` dataset for contract commitment tracking. - Reference maturity levels as Crawl/Walk/Run when discussing FinOps practice maturity - Cite the six FinOps principles when they are relevant to a recommendation - For Azure-specific guidance, reference the official Microsoft documentation URL From eba731fdecb9a7db4b8441dc1fbce7b18a122f20 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 17 May 2026 13:29:39 -0700 Subject: [PATCH 11/19] Add FOCUS 1.4 schema tests and changelog entries Updates Pester tests to validate v1_4 (not v1_3) schema files, removes tests for the eliminated intermediate v1_3 schema, and corrects HubSetup_Latest assertions to expect _v1_4() aliases. Updates changelog to reflect FOCUS 1.4 GA release without preview designation. Co-Authored-By: Claude --- docs-mslearn/toolkit/changelog.md | 9 +- .../Tests/Unit/HubsFocusSchemas.Tests.ps1 | 100 ++++-------------- 2 files changed, 25 insertions(+), 84 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 651aeb6b5..ceb1b0eba 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -32,16 +32,15 @@ The following section lists features and enhancements that are currently in deve - Added 4 agents (CFO, FinOps practitioner, database query, hubs agent), 5 commands (`/ftk-hubs-connect`, `/ftk-hubs-healthCheck`, `/ftk-mom-report`, `/ftk-ytd-report`, `/ftk-cost-optimization`), and an output style. - Linked to the existing KQL query catalog in `src/queries/` from the plugin. - **Changed** - - Updated plugin skill files to reflect FOCUS 1.3 / 1.4-preview hub schemas, the new `ContractCommitment()` function, and the deprecation of `ProviderName` / `PublisherName` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + - Updated plugin skill files to reflect FOCUS 1.4 hub schema, the new `ContractCommitment()` function, and the removal of `ProviderName` / `PublisherName` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). ### FinOps hubs v15.0.0 - **Added** - - Added FOCUS 1.3 hub schema (`v1_3`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`. New supplemental dataset: `ContractCommitment` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). - - Added FOCUS 1.4-preview hub schema (`v1_4`). Drops `ProviderName` and `PublisherName` from cost and usage. Expands `ContractCommitment` by 14 columns (`BenefitCategory`, `ContractCommitmentApplicability`, `Created`, `DiscountPercentage`, `DurationType`, `FulfillmentInterval`, `LastUpdated`, `LifecycleStatus`, `Model`, `OfferCategory`, `PaymentInterval`, `PaymentModel`, `PaymentUpfrontPercentage`, `PricingCurrencyContractCommitmentCost`). Marked **preview** because FOCUS 1.4 is still in working draft and may change before ratification. - - Added unversioned `ContractCommitment()` function aliasing to `ContractCommitment_v1_3()`. + - Added FOCUS 1.4 hub schema (`v1_4`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`. Removes deprecated `ProviderName` and `PublisherName`. New supplemental dataset: `ContractCommitment` with 28 columns ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + - Added unversioned `ContractCommitment()` function aliasing to `ContractCommitment_v1_4()`. - **Changed** - - Retargeted unversioned `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `Recommendations()`, and `Transactions()` aliases to their `_v1_3` counterparts. Unversioned aliases stay pinned to the latest GA schema (v1_3) — `_v1_4` is opt-in only while in preview. + - Retargeted unversioned `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `Recommendations()`, and `Transactions()` aliases to their `_v1_4` counterparts. - Refreshed the canonical "add a new FOCUS version" procedure in [src/templates/finops-hub/docs/README.md](https://github.com/microsoft/finops-toolkit/tree/dev/src/templates/finops-hub/docs/README.md) with multi-version-cycle, plugin, and changelog steps. ### Bicep Registry module pending updates diff --git a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 index 1e9e18432..f49ff5483 100644 --- a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 +++ b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 @@ -11,14 +11,12 @@ Describe 'HubsFocusSchemas' { $ingestionFiles = @{ v1_0 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_0.kql') -Raw v1_2 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_2.kql') -Raw - v1_3 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_3.kql') -Raw v1_4 = Get-Content -Path (Join-Path $scriptsPath 'IngestionSetup_v1_4.kql') -Raw } $hubFiles = @{ v1_0 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_0.kql') -Raw v1_2 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_2.kql') -Raw - v1_3 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_3.kql') -Raw v1_4 = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_v1_4.kql') -Raw Latest = Get-Content -Path (Join-Path $scriptsPath 'HubSetup_Latest.kql') -Raw } @@ -27,7 +25,7 @@ Describe 'HubsFocusSchemas' { $buildConfig = Get-Content -Path (Join-Path $repoRoot 'src/templates/finops-hub/.build.config') -Raw } - Context 'FOCUS 1.3 columns in Costs_raw' { + Context 'FOCUS 1.4 columns in Costs_raw' { BeforeAll { # Extract just the Costs_raw alter block (not Costs_final or any other table). @@ -47,7 +45,7 @@ Describe 'HubsFocusSchemas' { } } - Context 'ContractCommitment_raw exists with FOCUS 1.3 + 1.4 columns' { + Context 'ContractCommitment_raw exists with all FOCUS 1.4 columns' { BeforeAll { # The Redefine-all-columns alter-table block is the second occurrence; match all and pick it. @@ -63,7 +61,7 @@ Describe 'HubsFocusSchemas' { $contractCommitmentRawBlock | Should -Not -BeNullOrEmpty } - It 'Includes FOCUS 1.3 column <_>' -ForEach @( + It 'Includes base column <_>' -ForEach @( 'BillingCurrency', 'ContractCommitmentCategory', 'ContractCommitmentCost', 'ContractCommitmentId', 'ContractCommitmentPeriodEnd', 'ContractCommitmentPeriodStart', 'ContractCommitmentQuantity', 'ContractCommitmentType', 'ContractCommitmentUnit', @@ -83,47 +81,6 @@ Describe 'HubsFocusSchemas' { } } - Context 'IngestionSetup_v1_3.kql' { - - BeforeAll { - $script:costsFinalV13Block = if ($ingestionFiles.v1_3 -match '(?ms)\.create-merge table Costs_final_v1_3 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } - } - - It 'Defines Costs_transform_v1_3' { - $ingestionFiles.v1_3 | Should -Match 'Costs_transform_v1_3\(\)' - } - - It 'Defines Costs_final_v1_3 table' { - $ingestionFiles.v1_3 | Should -Match '\.create-merge table Costs_final_v1_3' - } - - It 'Defines ContractCommitment_transform_v1_3' { - $ingestionFiles.v1_3 | Should -Match 'ContractCommitment_transform_v1_3\(\)' - } - - It 'Defines ContractCommitment_final_v1_3 table' { - $ingestionFiles.v1_3 | Should -Match '\.create-merge table ContractCommitment_final_v1_3' - } - - It 'Costs_final_v1_3 block was extracted' { - $costsFinalV13Block | Should -Not -BeNullOrEmpty - } - - It 'Costs_final_v1_3 includes FOCUS 1.3 column <_>' -ForEach @( - 'AllocatedMethodId', 'AllocatedMethodDetails', 'AllocatedResourceId', - 'AllocatedResourceName', 'AllocatedTags', 'ContractApplied', - 'ServiceProviderName', 'HostProviderName' - ) { - $costsFinalV13Block | Should -Match "(?m)^\s+$_\s*:" - } - - It 'Costs_final_v1_3 still includes deprecated <_> for back compat' -ForEach @( - 'ProviderName', 'PublisherName' - ) { - $costsFinalV13Block | Should -Match "(?m)^\s+$_\s*:" - } - } - Context 'IngestionSetup_v1_4.kql' { BeforeAll { @@ -139,11 +96,19 @@ Describe 'HubsFocusSchemas' { $ingestionFiles.v1_4 | Should -Match '\.create-merge table Costs_final_v1_4' } + It 'Defines ContractCommitment_transform_v1_4' { + $ingestionFiles.v1_4 | Should -Match 'ContractCommitment_transform_v1_4\(\)' + } + + It 'Defines ContractCommitment_final_v1_4 table' { + $ingestionFiles.v1_4 | Should -Match '\.create-merge table ContractCommitment_final_v1_4' + } + It 'Costs_final_v1_4 block was extracted' { $costsFinalV14Block | Should -Not -BeNullOrEmpty } - It 'Costs_final_v1_4 does NOT include deprecated <_> (removed in 1.4)' -ForEach @( + It 'Costs_final_v1_4 does NOT include deprecated <_> (removed in FOCUS 1.4)' -ForEach @( 'ProviderName', 'PublisherName' ) { $costsFinalV14Block | Should -Not -Match "(?m)^\s+$_\s*:" @@ -159,23 +124,7 @@ Describe 'HubsFocusSchemas' { } } - Context 'HubSetup_v1_3.kql' { - - It 'Defines <_>' -ForEach @( - 'CommitmentDiscountUsage_v1_3', 'ContractCommitment_v1_3', - 'Costs_v1_3', 'Prices_v1_3', 'Recommendations_v1_3', 'Transactions_v1_3' - ) { - $hubFiles.v1_3 | Should -Match "$_\(\)" - } - - It 'Costs_v1_3 unions all prior versions' -ForEach @( - 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_3' - ) { - $hubFiles.v1_3 | Should -Match "database\('Ingestion'\)\.$_" - } - } - - Context 'HubSetup_v1_4.kql (preview)' { + Context 'HubSetup_v1_4.kql' { It 'Defines <_>' -ForEach @( 'CommitmentDiscountUsage_v1_4', 'ContractCommitment_v1_4', @@ -185,42 +134,35 @@ Describe 'HubsFocusSchemas' { } It 'Costs_v1_4 unions all prior versions' -ForEach @( - 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_3', 'Costs_final_v1_4' + 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_4' ) { $hubFiles.v1_4 | Should -Match "database\('Ingestion'\)\.$_" } - - It 'Marks itself as preview in docstring' { - $hubFiles.v1_4 | Should -Match 'FOCUS 1\.4-preview' - } } Context 'HubSetup_Latest.kql aliases' { - It 'Aliases <_> to v1_3 (latest GA, not v1_4 preview)' -ForEach @( - 'CommitmentDiscountUsage', 'Costs', 'Prices', 'Recommendations', 'Transactions', - 'ContractCommitment' + It 'Aliases <_> to v1_4 (latest GA)' -ForEach @( + 'CommitmentDiscountUsage', 'ContractCommitment', 'Costs', 'Prices', 'Recommendations', 'Transactions' ) { - $hubFiles.Latest | Should -Match "(?ms)$_\(\)\s*\{\s*${_}_v1_3\(\)\s*\}" + $hubFiles.Latest | Should -Match "(?ms)$_\(\)\s*\{\s*${_}_v1_4\(\)\s*\}" } - It 'Does NOT alias to v1_4 (preview must be opt-in)' { - $hubFiles.Latest | Should -Not -Match '_v1_4\(\)' + It 'Does NOT alias to v1_2 or older' { + $hubFiles.Latest | Should -Not -Match '_v1_2\(\)' } } Context 'Bicep wiring' { It 'app.bicep loads <_>' -ForEach @( - 'IngestionSetup_v1_3.kql', 'IngestionSetup_v1_4.kql', - 'HubSetup_v1_3.kql', 'HubSetup_v1_4.kql' + 'IngestionSetup_v1_4.kql', 'HubSetup_v1_4.kql' ) { $appBicep | Should -Match ([regex]::Escape($_)) } It '.build.config lists <_>' -ForEach @( - 'IngestionSetup_v1_3.kql', 'IngestionSetup_v1_4.kql', - 'HubSetup_v1_3.kql', 'HubSetup_v1_4.kql' + 'IngestionSetup_v1_4.kql', 'HubSetup_v1_4.kql' ) { $buildConfig | Should -Match ([regex]::Escape($_)) } From a681e73bb920fb783984173903811519c28c823b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 May 2026 20:30:36 +0000 Subject: [PATCH 12/19] chore: Update ms.date in docs-mslearn files --- docs-mslearn/toolkit/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index ceb1b0eba..2b3b5de43 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 05/06/2026 +ms.date: 05/17/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From f8495a52c3ecbd32701ff39f9ee3d7ced47f61e1 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 00:22:33 -0700 Subject: [PATCH 13/19] Address PR #2126 feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pluralize ContractCommitments_raw / _transform_v1_4 / _final_v1_4 / mapping / update policy - Add BillingPeriods_raw + IngestionSetup_v1_4 transforms/finals/policies (FOCUS 1.4 supplemental dataset) - Add InvoiceDetails_raw + IngestionSetup_v1_4 transforms/finals/policies (FOCUS 1.4 supplemental dataset) - Add 12 ContractCommitment* per-row columns + CommitmentProgramEligibilityDetails + InvoiceDetailId to Costs_raw, Costs_final_v1_4, and Costs_transform_v1_4 - Fix x_Source* annotations on ContractCommitments_raw to Hubs v1_4+ - Update Costs_raw ProviderName/PublisherName annotations to FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) - Update Costs_raw Region annotation: deprecated -> removed - File #2159 for the FOCUS 1.4 native-columns architecture research 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../scripts/IngestionSetup_RawTables.kql | 204 ++++++++++++++--- .../Analytics/scripts/IngestionSetup_v1_4.kql | 206 ++++++++++++++++-- 2 files changed, 363 insertions(+), 47 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index cdc281659..dfb70fda9 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -272,6 +272,58 @@ .alter table AmortizedCosts_raw policy streamingingestion disable +//===| BillingPeriods |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// BillingPeriods_raw table -- Create the table if it doesn't exist +.create-merge table BillingPeriods_raw ( ignore: string ) + +// BillingPeriods_raw table -- Remove all columns to allow changing column types +.alter table BillingPeriods_raw ( ignore: string ) + +// BillingPeriods_raw table -- Redefine all columns +.alter table BillingPeriods_raw ( + BillingPeriodCreated: datetime, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + BillingPeriodStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ +) + +// BillingPeriods_raw ingestion mapping +.create-or-alter table BillingPeriods_raw ingestion parquet mapping "BillingPeriods_raw_mapping" +``` +[ + { "Column": "BillingPeriodCreated", "Properties": { "Field": "BillingPeriodCreated" } }, + { "Column": "BillingPeriodEnd", "Properties": { "Field": "BillingPeriodEnd" } }, + { "Column": "BillingPeriodLastUpdated", "Properties": { "Field": "BillingPeriodLastUpdated" } }, + { "Column": "BillingPeriodStart", "Properties": { "Field": "BillingPeriodStart" } }, + { "Column": "BillingPeriodStatus", "Properties": { "Field": "BillingPeriodStatus" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// BillingPeriods_raw retention policy (clear historical data) +.alter-merge table BillingPeriods_raw policy retention softdelete = 0d recoverability = disabled + +// BillingPeriods_raw retention policy (set the user-defined retention period) +.alter-merge table BillingPeriods_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable BillingPeriods_raw streaming ingestion (required for Fabric) +.alter table BillingPeriods_raw policy streamingingestion disable + + //===| CommitmentDiscountUsage |======================================================================================== // Supported versions: // - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea @@ -335,19 +387,19 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable -//===| Contract commitment |============================================================================================ +//===| Contract commitments |=========================================================================================== // Supported versions: -// - FOCUS 1.3+ (no Cost Management export shipped yet) +// - FOCUS 1.4+ (no Cost Management export shipped yet) //====================================================================================================================== -// ContractCommitment_raw table -- Create the table if it doesn't exist -.create-merge table ContractCommitment_raw ( ignore: string ) +// ContractCommitments_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitments_raw ( ignore: string ) -// ContractCommitment_raw table -- Remove all columns to allow changing column types -.alter table ContractCommitment_raw ( ignore: string ) +// ContractCommitments_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitments_raw ( ignore: string ) -// ContractCommitment_raw table -- Redefine all columns -.alter table ContractCommitment_raw ( +// ContractCommitments_raw table -- Redefine all columns +.alter table ContractCommitments_raw ( BenefitCategory: string, // FOCUS 1.4+ BillingCurrency: string, // FOCUS 1.3+ ContractCommitmentApplicability: string, // FOCUS 1.4+ (JSON) @@ -376,14 +428,14 @@ PaymentUpfrontPercentage: real, // FOCUS 1.4+ PricingCurrency: string, // FOCUS 1.3+ PricingCurrencyContractCommitmentCost: real, // FOCUS 1.4+ - x_SourceName: string, // Hubs v1_3+ - x_SourceProvider: string, // Hubs v1_3+ - x_SourceType: string, // Hubs v1_3+ - x_SourceVersion: string // Hubs v1_3+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ ) -// ContractCommitment_raw ingestion mapping -.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +// ContractCommitments_raw ingestion mapping +.create-or-alter table ContractCommitments_raw ingestion parquet mapping "ContractCommitments_raw_mapping" ``` [ { "Column": "BenefitCategory", "Properties": { "Field": "BenefitCategory" } }, @@ -421,14 +473,14 @@ ] ``` -// ContractCommitment_raw retention policy (clear historical data) -.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled +// ContractCommitments_raw retention policy (clear historical data) +.alter-merge table ContractCommitments_raw policy retention softdelete = 0d recoverability = disabled -// ContractCommitment_raw retention policy (set the user-defined retention period) -.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled +// ContractCommitments_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitments_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled -// Disable ContractCommitment_raw streaming ingestion (required for Fabric) -.alter table ContractCommitment_raw policy streamingingestion disable +// Disable ContractCommitments_raw streaming ingestion (required for Fabric) +.alter table ContractCommitments_raw policy streamingingestion disable //===| Costs |========================================================================================================== @@ -483,14 +535,28 @@ CommitmentDiscountQuantity: real, // FOCUS 1.1+ CommitmentDiscountStatus: string, // FOCUS 1.0+ CommitmentDiscountType: string, // FOCUS 1.0-preview+ - CommitmentDiscountUnit: string, // FOCUS 1.1+ - ConsumedQuantity: real, // FOCUS 1.0+ - ConsumedUnit: string, // FOCUS 1.0+ - ContractApplied: string, // FOCUS 1.3+ (JSON) + CommitmentDiscountUnit: string, // FOCUS 1.1+ + CommitmentProgramEligibilityDetails: string, // FOCUS 1.4+ (JSON) + ConsumedQuantity: real, // FOCUS 1.0+ + ConsumedUnit: string, // FOCUS 1.0+ + ContractApplied: string, // FOCUS 1.3+ (JSON) + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ ContractedCost: real, // FOCUS 1.0+ ContractedUnitPrice: real, // FOCUS 1.0+ EffectiveCost: real, // FOCUS 1.0-preview+ HostProviderName: string, // FOCUS 1.3+ + InvoiceDetailId: string, // FOCUS 1.4+ InvoiceId: string, // FOCUS 1.2+ InvoiceIssuerName: string, // FOCUS 0.5+ ListCost: real, // FOCUS 1.0-preview+ @@ -499,9 +565,9 @@ PricingCurrency: string, // FOCUS 1.2+ PricingQuantity: real, // FOCUS 1.0-preview+ PricingUnit: string, // FOCUS 1.0-preview+ - ProviderName: string, // FOCUS 0.5+ - PublisherName: string, // FOCUS 0.5+ - Region: string, // FOCUS 0.5-1.0-preview (deprecated) + ProviderName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) + PublisherName: string, // FOCUS 0.5-1.3 (removed in 1.4; kept for back compat) + Region: string, // FOCUS 0.5-1.0-preview (removed; kept for back compat) RegionId: string, // FOCUS 1.0+ RegionName: string, // FOCUS 1.0+ ResourceId: string, // FOCUS 0.5+ @@ -770,6 +836,90 @@ .alter table Costs_raw policy streamingingestion disable +//===| InvoiceDetails |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// InvoiceDetails_raw table -- Create the table if it doesn't exist +.create-merge table InvoiceDetails_raw ( ignore: string ) + +// InvoiceDetails_raw table -- Remove all columns to allow changing column types +.alter table InvoiceDetails_raw ( ignore: string ) + +// InvoiceDetails_raw table -- Redefine all columns +.alter table InvoiceDetails_raw ( + BilledCost: real, // FOCUS 1.4+ + BillingAccountId: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + ChargeCategory: string, // FOCUS 1.4+ + InvoiceDetailCreated: datetime, // FOCUS 1.4+ + InvoiceDetailDescription: string, // FOCUS 1.4+ + InvoiceDetailGrain: string, // FOCUS 1.4+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ + InvoiceId: string, // FOCUS 1.4+ + InvoiceIssueDate: datetime, // FOCUS 1.4+ + InvoiceIssueStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + PaymentCurrency: string, // FOCUS 1.4+ + PaymentCurrencyBilledCost: real, // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ + PaymentDueDate: datetime, // FOCUS 1.4+ + PaymentTerms: string, // FOCUS 1.4+ + PurchaseOrderNumber: string, // FOCUS 1.4+ + ReferenceInvoiceId: string, // FOCUS 1.4+ + x_SourceName: string, // Hubs v1_4+ + x_SourceProvider: string, // Hubs v1_4+ + x_SourceType: string, // Hubs v1_4+ + x_SourceVersion: string // Hubs v1_4+ +) + +// InvoiceDetails_raw ingestion mapping +.create-or-alter table InvoiceDetails_raw ingestion parquet mapping "InvoiceDetails_raw_mapping" +``` +[ + { "Column": "BilledCost", "Properties": { "Field": "BilledCost" } }, + { "Column": "BillingAccountId", "Properties": { "Field": "BillingAccountId" } }, + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "BillingPeriodEnd", "Properties": { "Field": "BillingPeriodEnd" } }, + { "Column": "BillingPeriodStart", "Properties": { "Field": "BillingPeriodStart" } }, + { "Column": "ChargeCategory", "Properties": { "Field": "ChargeCategory" } }, + { "Column": "InvoiceDetailCreated", "Properties": { "Field": "InvoiceDetailCreated" } }, + { "Column": "InvoiceDetailDescription", "Properties": { "Field": "InvoiceDetailDescription" } }, + { "Column": "InvoiceDetailGrain", "Properties": { "Field": "InvoiceDetailGrain" } }, + { "Column": "InvoiceDetailId", "Properties": { "Field": "InvoiceDetailId" } }, + { "Column": "InvoiceDetailLastUpdated", "Properties": { "Field": "InvoiceDetailLastUpdated" } }, + { "Column": "InvoiceId", "Properties": { "Field": "InvoiceId" } }, + { "Column": "InvoiceIssueDate", "Properties": { "Field": "InvoiceIssueDate" } }, + { "Column": "InvoiceIssueStatus", "Properties": { "Field": "InvoiceIssueStatus" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "PaymentCurrency", "Properties": { "Field": "PaymentCurrency" } }, + { "Column": "PaymentCurrencyBilledCost", "Properties": { "Field": "PaymentCurrencyBilledCost" } }, + { "Column": "PaymentCurrencyInvoiceDetailId", "Properties": { "Field": "PaymentCurrencyInvoiceDetailId" } }, + { "Column": "PaymentDueDate", "Properties": { "Field": "PaymentDueDate" } }, + { "Column": "PaymentTerms", "Properties": { "Field": "PaymentTerms" } }, + { "Column": "PurchaseOrderNumber", "Properties": { "Field": "PurchaseOrderNumber" } }, + { "Column": "ReferenceInvoiceId", "Properties": { "Field": "ReferenceInvoiceId" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// InvoiceDetails_raw retention policy (clear historical data) +.alter-merge table InvoiceDetails_raw policy retention softdelete = 0d recoverability = disabled + +// InvoiceDetails_raw retention policy (set the user-defined retention period) +.alter-merge table InvoiceDetails_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable InvoiceDetails_raw streaming ingestion (required for Fabric) +.alter table InvoiceDetails_raw policy streamingingestion disable + + //===| Prices |========================================================================================================= // NOTE: Must be before cost details. // diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index 84e8d6fcd..f525f2cf1 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -743,13 +743,27 @@ Costs_transform_v1_4() CommitmentDiscountStatus, CommitmentDiscountType, CommitmentDiscountUnit, + CommitmentProgramEligibilityDetails = parse_json(column_ifexists('CommitmentProgramEligibilityDetails', '')), // FOCUS 1.4+ ConsumedQuantity, ConsumedUnit, ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractCommitmentBenefitCategory = column_ifexists('ContractCommitmentBenefitCategory', ''), // FOCUS 1.4+ + ContractCommitmentCreated = column_ifexists('ContractCommitmentCreated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentDiscountPercentage = column_ifexists('ContractCommitmentDiscountPercentage', real(null)), // FOCUS 1.4+ + ContractCommitmentDurationType = column_ifexists('ContractCommitmentDurationType', ''), // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval = column_ifexists('ContractCommitmentFulfillmentInterval', ''), // FOCUS 1.4+ + ContractCommitmentLastUpdated = column_ifexists('ContractCommitmentLastUpdated', datetime(null)), // FOCUS 1.4+ + ContractCommitmentLifecycleStatus = column_ifexists('ContractCommitmentLifecycleStatus', ''), // FOCUS 1.4+ + ContractCommitmentModel = column_ifexists('ContractCommitmentModel', ''), // FOCUS 1.4+ + ContractCommitmentOfferCategory = column_ifexists('ContractCommitmentOfferCategory', ''), // FOCUS 1.4+ + ContractCommitmentPaymentInterval = column_ifexists('ContractCommitmentPaymentInterval', ''), // FOCUS 1.4+ + ContractCommitmentPaymentModel = column_ifexists('ContractCommitmentPaymentModel', ''), // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage = column_ifexists('ContractCommitmentPaymentUpfrontPercentage', real(null)), // FOCUS 1.4+ ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), EffectiveCost, HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceDetailId = column_ifexists('InvoiceDetailId', ''), // FOCUS 1.4+ InvoiceId = coalesce(InvoiceId, x_InvoiceId), InvoiceIssuerName, ListCost, @@ -949,15 +963,29 @@ Costs_transform_v1_4() CommitmentDiscountQuantity: real, CommitmentDiscountStatus: string, CommitmentDiscountType: string, - CommitmentDiscountUnit: string, - ConsumedQuantity: real, - ConsumedUnit: string, - ContractApplied: dynamic, // FOCUS 1.3+ - ContractedCost: real, - ContractedUnitPrice: real, - EffectiveCost: real, - HostProviderName: string, // FOCUS 1.3+ - InvoiceId: string, + CommitmentDiscountUnit: string, + CommitmentProgramEligibilityDetails: dynamic, // FOCUS 1.4+ (JSON) + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractCommitmentBenefitCategory: string, // FOCUS 1.4+ + ContractCommitmentCreated: datetime, // FOCUS 1.4+ + ContractCommitmentDiscountPercentage: real, // FOCUS 1.4+ + ContractCommitmentDurationType: string, // FOCUS 1.4+ + ContractCommitmentFulfillmentInterval: string, // FOCUS 1.4+ + ContractCommitmentLastUpdated: datetime, // FOCUS 1.4+ + ContractCommitmentLifecycleStatus: string, // FOCUS 1.4+ + ContractCommitmentModel: string, // FOCUS 1.4+ + ContractCommitmentOfferCategory: string, // FOCUS 1.4+ + ContractCommitmentPaymentInterval: string, // FOCUS 1.4+ + ContractCommitmentPaymentModel: string, // FOCUS 1.4+ + ContractCommitmentPaymentUpfrontPercentage: real, // FOCUS 1.4+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceId: string, InvoiceIssuerName: string, ListCost: real, ListUnitPrice: real, @@ -1661,17 +1689,17 @@ CommitmentDiscountUsage_transform_v1_4() ``` -//===| Contract commitment |============================================================================================ +//===| Contract commitments |=========================================================================================== // Supported versions: -// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) //====================================================================================================================== -// ContractCommitment_transform_v1_4 function +// ContractCommitments_transform_v1_4 function .create-or-alter function with (docstring='All contract commitments aligned to FOCUS 1.4.', folder='Contract commitments') -ContractCommitment_transform_v1_4() +ContractCommitments_transform_v1_4() { - ContractCommitment_raw + ContractCommitments_raw | project BenefitCategory = column_ifexists('BenefitCategory', ''), // FOCUS 1.4+ BillingCurrency, @@ -1708,8 +1736,8 @@ ContractCommitment_transform_v1_4() x_SourceVersion = coalesce(x_SourceVersion, '') } -// ContractCommitment_final_v1_4 table -.create-merge table ContractCommitment_final_v1_4 ( +// ContractCommitments_final_v1_4 table +.create-merge table ContractCommitments_final_v1_4 ( BenefitCategory: string, // FOCUS 1.4+ BillingCurrency: string, // FOCUS 1.3+ ContractCommitmentApplicability: dynamic, // FOCUS 1.4+ (JSON) @@ -1745,13 +1773,151 @@ ContractCommitment_transform_v1_4() x_SourceVersion: string // Hubs add-on ) -// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_4 table -.alter table ContractCommitment_final_v1_4 policy update +// Update policy for ContractCommitments_raw -> ContractCommitments_final_v1_4 table +.alter table ContractCommitments_final_v1_4 policy update ``` [{ "IsEnabled": true, - "Source": "ContractCommitment_raw", - "Query": "ContractCommitment_transform_v1_4()", + "Source": "ContractCommitments_raw", + "Query": "ContractCommitments_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| BillingPeriods |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// BillingPeriods_transform_v1_4 function +.create-or-alter function +with (docstring='All billing periods aligned to FOCUS 1.4.', folder='BillingPeriods') +BillingPeriods_transform_v1_4() +{ + BillingPeriods_raw + | project + BillingPeriodCreated = column_ifexists('BillingPeriodCreated', datetime(null)), // FOCUS 1.4+ + BillingPeriodEnd, + BillingPeriodLastUpdated = column_ifexists('BillingPeriodLastUpdated', datetime(null)), // FOCUS 1.4+ + BillingPeriodStart, + BillingPeriodStatus = column_ifexists('BillingPeriodStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// BillingPeriods_final_v1_4 table +.create-merge table BillingPeriods_final_v1_4 ( + BillingPeriodCreated: datetime, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodLastUpdated: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + BillingPeriodStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for BillingPeriods_raw -> BillingPeriods_final_v1_4 table +.alter table BillingPeriods_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "BillingPeriods_raw", + "Query": "BillingPeriods_transform_v1_4()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| InvoiceDetails |================================================================================================= +// Supported versions: +// - FOCUS 1.4+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// InvoiceDetails_transform_v1_4 function +.create-or-alter function +with (docstring='All invoice details aligned to FOCUS 1.4.', folder='InvoiceDetails') +InvoiceDetails_transform_v1_4() +{ + InvoiceDetails_raw + | project + BilledCost, + BillingAccountId, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + InvoiceDetailCreated = column_ifexists('InvoiceDetailCreated', datetime(null)), // FOCUS 1.4+ + InvoiceDetailDescription = column_ifexists('InvoiceDetailDescription', ''), // FOCUS 1.4+ + InvoiceDetailGrain = column_ifexists('InvoiceDetailGrain', ''), // FOCUS 1.4+ + InvoiceDetailId, + InvoiceDetailLastUpdated = column_ifexists('InvoiceDetailLastUpdated', datetime(null)), // FOCUS 1.4+ + InvoiceId, + InvoiceIssueDate = column_ifexists('InvoiceIssueDate', datetime(null)), // FOCUS 1.4+ + InvoiceIssueStatus = column_ifexists('InvoiceIssueStatus', ''), // FOCUS 1.4+ + InvoiceIssuerName, + PaymentCurrency = column_ifexists('PaymentCurrency', ''), // FOCUS 1.4+ + PaymentCurrencyBilledCost = column_ifexists('PaymentCurrencyBilledCost', real(null)), // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId = column_ifexists('PaymentCurrencyInvoiceDetailId', ''), // FOCUS 1.4+ + PaymentDueDate = column_ifexists('PaymentDueDate', datetime(null)), // FOCUS 1.4+ + PaymentTerms = column_ifexists('PaymentTerms', ''), // FOCUS 1.4+ + PurchaseOrderNumber = column_ifexists('PurchaseOrderNumber', ''), // FOCUS 1.4+ + ReferenceInvoiceId = column_ifexists('ReferenceInvoiceId', ''), // FOCUS 1.4+ + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// InvoiceDetails_final_v1_4 table +.create-merge table InvoiceDetails_final_v1_4 ( + BilledCost: real, // FOCUS 1.4+ + BillingAccountId: string, // FOCUS 1.4+ + BillingCurrency: string, // FOCUS 1.4+ + BillingPeriodEnd: datetime, // FOCUS 1.4+ + BillingPeriodStart: datetime, // FOCUS 1.4+ + ChargeCategory: string, // FOCUS 1.4+ + InvoiceDetailCreated: datetime, // FOCUS 1.4+ + InvoiceDetailDescription: string, // FOCUS 1.4+ + InvoiceDetailGrain: string, // FOCUS 1.4+ + InvoiceDetailId: string, // FOCUS 1.4+ + InvoiceDetailLastUpdated: datetime, // FOCUS 1.4+ + InvoiceId: string, // FOCUS 1.4+ + InvoiceIssueDate: datetime, // FOCUS 1.4+ + InvoiceIssueStatus: string, // FOCUS 1.4+ + InvoiceIssuerName: string, // FOCUS 1.4+ + PaymentCurrency: string, // FOCUS 1.4+ + PaymentCurrencyBilledCost: real, // FOCUS 1.4+ + PaymentCurrencyInvoiceDetailId: string, // FOCUS 1.4+ + PaymentDueDate: datetime, // FOCUS 1.4+ + PaymentTerms: string, // FOCUS 1.4+ + PurchaseOrderNumber: string, // FOCUS 1.4+ + ReferenceInvoiceId: string, // FOCUS 1.4+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for InvoiceDetails_raw -> InvoiceDetails_final_v1_4 table +.alter table InvoiceDetails_final_v1_4 policy update +``` +[{ + "IsEnabled": true, + "Source": "InvoiceDetails_raw", + "Query": "InvoiceDetails_transform_v1_4()", "IsTransactional": true, "PropagateIngestionProperties": true }] From 56f9b3015a1a7df981d12a773dd2ad888a26d960 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 00:25:01 -0700 Subject: [PATCH 14/19] Cascade PR #2126 feedback to hub functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename ContractCommitment_v1_4() -> ContractCommitments_v1_4() (and update folder label) - Add BillingPeriods_v1_4() and InvoiceDetails_v1_4() hub functions - Add BillingPeriods(), ContractCommitments() (renamed), and InvoiceDetails() unversioned aliases in HubSetup_Latest.kql - Add CommitmentProgramEligibilityDetails + 12 ContractCommitment* + InvoiceDetailId column defaults to Costs_final_v1_0 and Costs_final_v1_2 union arms (back-compat shims) and add them to the project list 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../Analytics/scripts/HubSetup_Latest.kql | 22 ++- .../Analytics/scripts/HubSetup_v1_4.kql | 128 +++++++++++------- 2 files changed, 101 insertions(+), 49 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index c3a4985f2..bb0bc69fc 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -9,6 +9,14 @@ // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script +.create-or-alter function +with (docstring = 'Gets all billing periods with the latest supported version of the FOCUS schema.', folder = 'BillingPeriods') +BillingPeriods() +{ + BillingPeriods_v1_4() +} + + .create-or-alter function with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() @@ -18,10 +26,10 @@ CommitmentDiscountUsage() .create-or-alter function -with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') -ContractCommitment() +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitments') +ContractCommitments() { - ContractCommitment_v1_4() + ContractCommitments_v1_4() } @@ -33,6 +41,14 @@ Costs() } +.create-or-alter function +with (docstring = 'Gets all invoice details with the latest supported version of the FOCUS schema.', folder = 'InvoiceDetails') +InvoiceDetails() +{ + InvoiceDetails_v1_4() +} + + .create-or-alter function with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index cabdbdfc5..8b813e44a 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -66,30 +66,30 @@ CommitmentDiscountUsage_v1_4() } -// ContractCommitment_final_v1_4 +// ContractCommitments_final_v1_4 .create-or-alter function -with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitment') -ContractCommitment_v1_4() +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.4.', folder = 'ContractCommitments') +ContractCommitments_v1_4() { - database('Ingestion').ContractCommitment_final_v1_4 - | union ( - database('Ingestion').ContractCommitment_final_v1_3 - // FOCUS 1.4 column defaults (data was ingested under v1_3 schema) - | extend BenefitCategory = '' - | extend ContractCommitmentApplicability = dynamic(null) - | extend Created = datetime(null) - | extend DiscountPercentage = real(null) - | extend DurationType = '' - | extend FulfillmentInterval = '' - | extend LastUpdated = datetime(null) - | extend LifecycleStatus = '' - | extend Model = '' - | extend OfferCategory = '' - | extend PaymentInterval = '' - | extend PaymentModel = '' - | extend PaymentUpfrontPercentage = real(null) - | extend PricingCurrencyContractCommitmentCost = real(null) - ) + database('Ingestion').ContractCommitments_final_v1_4 +} + + +// BillingPeriods_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all billing periods aligned to FOCUS 1.4.', folder = 'BillingPeriods') +BillingPeriods_v1_4() +{ + database('Ingestion').BillingPeriods_final_v1_4 +} + + +// InvoiceDetails_final_v1_4 +.create-or-alter function +with (docstring = 'Gets all invoice details aligned to FOCUS 1.4.', folder = 'InvoiceDetails') +InvoiceDetails_v1_4() +{ + database('Ingestion').InvoiceDetails_final_v1_4 } @@ -225,33 +225,55 @@ Costs_v1_4() | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), SkuPriceDetails) - // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) - | extend AllocatedMethodId = '' - | extend AllocatedMethodDetails = dynamic(null) - | extend AllocatedResourceId = '' - | extend AllocatedResourceName = '' - | extend AllocatedTags = dynamic(null) - | extend ContractApplied = dynamic(null) - | extend ServiceProviderName = ProviderName - | extend HostProviderName = PublisherName + // FOCUS 1.4 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend CommitmentProgramEligibilityDetails = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ContractCommitmentBenefitCategory = '' + | extend ContractCommitmentCreated = datetime(null) + | extend ContractCommitmentDiscountPercentage = real(null) + | extend ContractCommitmentDurationType = '' + | extend ContractCommitmentFulfillmentInterval = '' + | extend ContractCommitmentLastUpdated = datetime(null) + | extend ContractCommitmentLifecycleStatus = '' + | extend ContractCommitmentModel = '' + | extend ContractCommitmentOfferCategory = '' + | extend ContractCommitmentPaymentInterval = '' + | extend ContractCommitmentPaymentModel = '' + | extend ContractCommitmentPaymentUpfrontPercentage = real(null) + | extend HostProviderName = PublisherName + | extend InvoiceDetailId = '' + | extend ServiceProviderName = ProviderName ) | union ( database('Ingestion').Costs_final_v1_2 - // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) - | extend AllocatedMethodId = '' - | extend AllocatedMethodDetails = dynamic(null) - | extend AllocatedResourceId = '' - | extend AllocatedResourceName = '' - | extend AllocatedTags = dynamic(null) - | extend ContractApplied = dynamic(null) - | extend ServiceProviderName = ProviderName - | extend HostProviderName = PublisherName - ) - | union ( - database('Ingestion').Costs_final_v1_3 - // No column adds needed; v1_3 already has the FOCUS 1.3 columns. The - // deprecated ProviderName / PublisherName fall off via the | project - // list below since FOCUS 1.4 removes them. + // FOCUS 1.4 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend CommitmentProgramEligibilityDetails = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ContractCommitmentBenefitCategory = '' + | extend ContractCommitmentCreated = datetime(null) + | extend ContractCommitmentDiscountPercentage = real(null) + | extend ContractCommitmentDurationType = '' + | extend ContractCommitmentFulfillmentInterval = '' + | extend ContractCommitmentLastUpdated = datetime(null) + | extend ContractCommitmentLifecycleStatus = '' + | extend ContractCommitmentModel = '' + | extend ContractCommitmentOfferCategory = '' + | extend ContractCommitmentPaymentInterval = '' + | extend ContractCommitmentPaymentModel = '' + | extend ContractCommitmentPaymentUpfrontPercentage = real(null) + | extend HostProviderName = PublisherName + | extend InvoiceDetailId = '' + | extend ServiceProviderName = ProviderName ) | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) | project @@ -283,13 +305,27 @@ Costs_v1_4() CommitmentDiscountStatus, CommitmentDiscountType, CommitmentDiscountUnit, + CommitmentProgramEligibilityDetails, ConsumedQuantity, ConsumedUnit, ContractApplied, + ContractCommitmentBenefitCategory, + ContractCommitmentCreated, + ContractCommitmentDiscountPercentage, + ContractCommitmentDurationType, + ContractCommitmentFulfillmentInterval, + ContractCommitmentLastUpdated, + ContractCommitmentLifecycleStatus, + ContractCommitmentModel, + ContractCommitmentOfferCategory, + ContractCommitmentPaymentInterval, + ContractCommitmentPaymentModel, + ContractCommitmentPaymentUpfrontPercentage, ContractedCost, ContractedUnitPrice, EffectiveCost, HostProviderName, + InvoiceDetailId, InvoiceId, InvoiceIssuerName, ListCost, From cb93641bbe45f76eb08141e1cbc80c449627a4c2 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 00:28:23 -0700 Subject: [PATCH 15/19] Cascade PR #2126 feedback to plugin docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename ContractCommitment() -> ContractCommitments() in skill references and agent docs - Add BillingPeriods() and InvoiceDetails() to the function list and dedicated agent doc sections - Mention the new FOCUS 1.4 per-row columns (CommitmentProgramEligibilityDetails, 12 ContractCommitment* columns, InvoiceDetailId) - Update output-style guidance to reference all three supplemental datasets 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../finops-toolkit/references/finops-hubs.md | 8 +++++--- .../claude-plugin/agents/ftk-database-query.md | 12 ++++++++++-- .../claude-plugin/output-styles/ftk-output-style.md | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md index 908dee713..307c9b1bb 100644 --- a/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md +++ b/src/templates/agent-skills/finops-toolkit/references/finops-hubs.md @@ -50,10 +50,12 @@ All KQL queries are located in `references/queries/`: **Database Rules:** - Always use "Hub" database, NEVER "Ingestion" -- Function-based access: `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `ContractCommitment()`, `Recommendations()`, `Transactions()` +- Function-based access: `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `ContractCommitments()`, `BillingPeriods()`, `InvoiceDetails()`, `Recommendations()`, `Transactions()` - The unversioned functions (`Costs()`, etc.) return the latest GA FOCUS schema (v1_4). Use versioned functions (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_4()`) to pin to a specific schema. -- FOCUS 1.4 added: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags` (data-generator split cost allocation), `ContractApplied` (per-row contract commitment application), `ServiceProviderName` and `HostProviderName` (replacing deprecated `ProviderName` / `PublisherName`, which are removed in 1.4). Also added the `ContractCommitment` supplemental dataset. -- The `ContractCommitment()` function (FOCUS 1.4+) returns provider-confirmed contract commitment metadata — the dataset feeding `ContractApplied` JSON arrays on each cost row. +- FOCUS 1.4 added: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags` (data-generator split cost allocation), `ContractApplied` (per-row contract commitment application), `ServiceProviderName` and `HostProviderName` (replacing removed `ProviderName` / `PublisherName`), `CommitmentProgramEligibilityDetails`, `InvoiceDetailId`, and 12 `ContractCommitment*` per-row columns. Also added three supplemental datasets: `ContractCommitments`, `BillingPeriods`, and `InvoiceDetails`. +- The `ContractCommitments()` function (FOCUS 1.4+) returns provider-confirmed contract commitment metadata — the dataset feeding `ContractApplied` JSON arrays on each cost row. +- The `BillingPeriods()` function (FOCUS 1.4+) returns billing period metadata (start/end/status) for use with `InvoiceDetailId` joins. +- The `InvoiceDetails()` function (FOCUS 1.4+) returns invoice line-item metadata associated with `InvoiceId` on the cost rows. --- diff --git a/src/templates/claude-plugin/agents/ftk-database-query.md b/src/templates/claude-plugin/agents/ftk-database-query.md index e94ee96dc..416607cfd 100644 --- a/src/templates/claude-plugin/agents/ftk-database-query.md +++ b/src/templates/claude-plugin/agents/ftk-database-query.md @@ -10,7 +10,7 @@ You are a FinOps Toolkit database specialist with deep expertise in the FinOps h ## Database Architecture -The FinOps hubs database exposes six main analytic functions. The unversioned forms below return the latest GA FOCUS schema (v1_4); pin to a specific schema with the versioned variants (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_4()`). +The FinOps hubs database exposes eight main analytic functions. The unversioned forms below return the latest GA FOCUS schema (v1_4); pin to a specific schema with the versioned variants (`Costs_v1_0()`, `Costs_v1_2()`, `Costs_v1_4()`). ### Costs() @@ -48,10 +48,18 @@ Price sheets with list, contracted, and effective pricing. Key columns include ` Reservation and savings plan utilization, joining commitment discounts to the resources that consumed them. Key columns include `ChargePeriodStart`, `ChargePeriodEnd`, `CommitmentDiscountId`, `CommitmentDiscountQuantity`, `CommitmentDiscountUnit`, `ConsumedQuantity`, `ResourceId`, `ServiceName`, `x_CommitmentDiscountCommittedCount`, `x_CommitmentDiscountNormalizedRatio`. -### ContractCommitment() +### ContractCommitments() (FOCUS 1.4+) Provider-confirmed contract commitment metadata — the dataset that feeds `ContractApplied` JSON arrays on each row in `Costs()`. Key columns include `ContractCommitmentId`, `ContractCommitmentCategory` (Spend / Usage), `ContractCommitmentCost`, `ContractCommitmentQuantity`, `ContractCommitmentPeriodStart`, `ContractCommitmentPeriodEnd`, `ContractId`, `BillingCurrency`, `InvoiceIssuerName`, `PaymentModel`, `PaymentInterval`, `LifecycleStatus`, `DurationType`. +### BillingPeriods() + +(FOCUS 1.4+) Billing period metadata. Key columns: `BillingPeriodStart`, `BillingPeriodEnd`, `BillingPeriodCreated`, `BillingPeriodLastUpdated`, `BillingPeriodStatus`, `InvoiceIssuerName`. Join on billing period to align cost data with invoice timing. + +### InvoiceDetails() + +(FOCUS 1.4+) Invoice line-item metadata. Key columns: `InvoiceDetailId`, `InvoiceId`, `BilledCost`, `BillingAccountId`, `BillingCurrency`, `ChargeCategory`, `InvoiceIssueDate`, `InvoiceIssueStatus`, `PaymentCurrency`, `PaymentCurrencyBilledCost`, `PaymentDueDate`, `PaymentTerms`, `PurchaseOrderNumber`, `ReferenceInvoiceId`. Join to `Costs()` on `InvoiceDetailId` (FOCUS 1.4+) for invoice reconciliation. + ### Recommendations() Reservation and savings plan recommendations from Microsoft. Key columns include `x_EffectiveCostBefore`, `x_EffectiveCostAfter`, `x_EffectiveCostSavings`, `x_RecommendationDate`, `x_RecommendationDetails` (dynamic), `SubAccountId`. diff --git a/src/templates/claude-plugin/output-styles/ftk-output-style.md b/src/templates/claude-plugin/output-styles/ftk-output-style.md index c2d3f9457..7623a33ad 100644 --- a/src/templates/claude-plugin/output-styles/ftk-output-style.md +++ b/src/templates/claude-plugin/output-styles/ftk-output-style.md @@ -133,7 +133,7 @@ Every recommendation must include: ## FinOps domain conventions - Reference FinOps Framework capabilities by their official names (e.g., "Managing commitment-based discounts", not "reservation management") -- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost). Prefer FOCUS 1.4 names: `ServiceProviderName` (replaces removed `ProviderName`) and `HostProviderName` (replaces removed `PublisherName`); `ContractApplied` and the `ContractCommitment()` dataset for contract commitment tracking. +- Use FOCUS specification terminology when discussing cost data fields (e.g., BilledCost, EffectiveCost, ListCost, ContractedCost). Prefer FOCUS 1.4 names: `ServiceProviderName` (replaces removed `ProviderName`) and `HostProviderName` (replaces removed `PublisherName`); `ContractApplied` plus the `ContractCommitments()`, `BillingPeriods()`, and `InvoiceDetails()` datasets for contract commitment, billing-period, and invoice tracking. - Reference maturity levels as Crawl/Walk/Run when discussing FinOps practice maturity - Cite the six FinOps principles when they are relevant to a recommendation - For Azure-specific guidance, reference the official Microsoft documentation URL From 3ebb967d2bf73c0271b3f9ab3f24e25395247cdb Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Tue, 26 May 2026 20:26:51 -0700 Subject: [PATCH 16/19] Remove leftover singular ContractCommitment() alias A merge artifact from rebasing on the v1_3 ancestry left a singular ContractCommitment() function referencing ContractCommitment_v1_3() in HubSetup_Latest.kql. v1_3 was eliminated and the dataset was pluralized to ContractCommitments_v1_4(); the orphan would fail to load on hub deployment. Remove it. Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- .../Analytics/scripts/HubSetup_Latest.kql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index b546e49c8..bb0bc69fc 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -33,14 +33,6 @@ ContractCommitments() } -.create-or-alter function -with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') -ContractCommitment() -{ - ContractCommitment_v1_3() -} - - .create-or-alter function with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() From 362de4432cc96ed8b9705a607be5757633249631 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 27 May 2026 01:26:45 -0700 Subject: [PATCH 17/19] Cascade PR #2126 feedback to tests + changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: - Pluralize ContractCommitments_raw / _final_v1_4 / _v1_4 assertions - Assert new BillingPeriods_raw + InvoiceDetails_raw datasets exist with their FOCUS 1.4 columns - Assert new BillingPeriods_v1_4(), InvoiceDetails_v1_4(), ContractCommitments_v1_4() hub functions - Assert HubSetup_Latest aliases for BillingPeriods(), ContractCommitments(), InvoiceDetails() - Assert Costs_raw and Costs_final_v1_4 include the 14 new FOCUS 1.4 columns - Assert NO singular ContractCommitment_raw / ContractCommitment_v1_4 / ContractCommitment() exists - Assert Costs_raw keeps removed ProviderName/PublisherName/Region for back compat Changelog: - Expand v15.0.0 entry with the 14 new FOCUS 1.4 columns and the three new supplemental datasets (ContractCommitments, BillingPeriods, InvoiceDetails) - Update plugin entry to list ContractCommitments() / BillingPeriods() / InvoiceDetails() 🤖 Generated with [Claude Code](https://claude.ai/claude-code) Co-Authored-By: Michael Flanakin Co-Authored-By: Claude --- docs-mslearn/toolkit/changelog.md | 6 +- .../Tests/Unit/HubsFocusSchemas.Tests.ps1 | 138 +++++++++++++++--- 2 files changed, 120 insertions(+), 24 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 2b3b5de43..0336977cb 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -32,13 +32,13 @@ The following section lists features and enhancements that are currently in deve - Added 4 agents (CFO, FinOps practitioner, database query, hubs agent), 5 commands (`/ftk-hubs-connect`, `/ftk-hubs-healthCheck`, `/ftk-mom-report`, `/ftk-ytd-report`, `/ftk-cost-optimization`), and an output style. - Linked to the existing KQL query catalog in `src/queries/` from the plugin. - **Changed** - - Updated plugin skill files to reflect FOCUS 1.4 hub schema, the new `ContractCommitment()` function, and the removal of `ProviderName` / `PublisherName` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + - Updated plugin skill files to reflect FOCUS 1.4 hub schema, the new `ContractCommitments()`, `BillingPeriods()`, and `InvoiceDetails()` functions, and the removal of `ProviderName` / `PublisherName` ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). ### FinOps hubs v15.0.0 - **Added** - - Added FOCUS 1.4 hub schema (`v1_4`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`. Removes deprecated `ProviderName` and `PublisherName`. New supplemental dataset: `ContractCommitment` with 28 columns ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). - - Added unversioned `ContractCommitment()` function aliasing to `ContractCommitment_v1_4()`. + - Added FOCUS 1.4 hub schema (`v1_4`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`, `CommitmentProgramEligibilityDetails`, `InvoiceDetailId`, and 12 `ContractCommitment*` per-row columns (`ContractCommitmentBenefitCategory`, `ContractCommitmentCreated`, `ContractCommitmentDiscountPercentage`, `ContractCommitmentDurationType`, `ContractCommitmentFulfillmentInterval`, `ContractCommitmentLastUpdated`, `ContractCommitmentLifecycleStatus`, `ContractCommitmentModel`, `ContractCommitmentOfferCategory`, `ContractCommitmentPaymentInterval`, `ContractCommitmentPaymentModel`, `ContractCommitmentPaymentUpfrontPercentage`). Removes deprecated `ProviderName` and `PublisherName` from the schema (raw tables keep them for back compat). Three new supplemental datasets: `ContractCommitments` (28 columns), `BillingPeriods` (6 columns), and `InvoiceDetails` (22 columns) ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). + - Added unversioned `ContractCommitments()`, `BillingPeriods()`, and `InvoiceDetails()` functions aliasing to their `_v1_4` counterparts. - **Changed** - Retargeted unversioned `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `Recommendations()`, and `Transactions()` aliases to their `_v1_4` counterparts. - Refreshed the canonical "add a new FOCUS version" procedure in [src/templates/finops-hub/docs/README.md](https://github.com/microsoft/finops-toolkit/tree/dev/src/templates/finops-hub/docs/README.md) with multi-version-cycle, plugin, and changelog steps. diff --git a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 index f49ff5483..02f4d3584 100644 --- a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 +++ b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 @@ -39,26 +39,43 @@ Describe 'HubsFocusSchemas' { It 'Adds <_> to Costs_raw' -ForEach @( 'AllocatedMethodId', 'AllocatedMethodDetails', 'AllocatedResourceId', 'AllocatedResourceName', 'AllocatedTags', 'ContractApplied', - 'ServiceProviderName', 'HostProviderName' + 'ServiceProviderName', 'HostProviderName', + 'CommitmentProgramEligibilityDetails', 'InvoiceDetailId', + 'ContractCommitmentBenefitCategory', 'ContractCommitmentCreated', + 'ContractCommitmentDiscountPercentage', 'ContractCommitmentDurationType', + 'ContractCommitmentFulfillmentInterval', 'ContractCommitmentLastUpdated', + 'ContractCommitmentLifecycleStatus', 'ContractCommitmentModel', + 'ContractCommitmentOfferCategory', 'ContractCommitmentPaymentInterval', + 'ContractCommitmentPaymentModel', 'ContractCommitmentPaymentUpfrontPercentage' + ) { + $costsRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + + It 'Keeps deprecated <_> for back compat' -ForEach @( + 'ProviderName', 'PublisherName', 'Region' ) { $costsRawBlock | Should -Match "(?m)^\s+$_\s*:" } } - Context 'ContractCommitment_raw exists with all FOCUS 1.4 columns' { + Context 'ContractCommitments_raw exists with all FOCUS 1.4 columns' { BeforeAll { # The Redefine-all-columns alter-table block is the second occurrence; match all and pick it. - $allBlocks = [regex]::Matches($rawTablesContent, '(?ms)\.alter table ContractCommitment_raw \(\r?\n(.*?)\r?\n\)') - $script:contractCommitmentRawBlock = if ($allBlocks.Count -ge 1) { $allBlocks[$allBlocks.Count - 1].Groups[1].Value } else { '' } + $allBlocks = [regex]::Matches($rawTablesContent, '(?ms)\.alter table ContractCommitments_raw \(\r?\n(.*?)\r?\n\)') + $script:contractCommitmentsRawBlock = if ($allBlocks.Count -ge 1) { $allBlocks[$allBlocks.Count - 1].Groups[1].Value } else { '' } + } + + It 'Defines ContractCommitments_raw (plural)' { + $rawTablesContent | Should -Match '\.alter table ContractCommitments_raw \(' } - It 'Defines ContractCommitment_raw' { - $rawTablesContent | Should -Match '\.alter table ContractCommitment_raw \(' + It 'Does NOT define singular ContractCommitment_raw' { + $rawTablesContent | Should -Not -Match '\.alter table ContractCommitment_raw \(' } - It 'ContractCommitment_raw column block was extracted' { - $contractCommitmentRawBlock | Should -Not -BeNullOrEmpty + It 'ContractCommitments_raw column block was extracted' { + $contractCommitmentsRawBlock | Should -Not -BeNullOrEmpty } It 'Includes base column <_>' -ForEach @( @@ -68,7 +85,7 @@ Describe 'HubsFocusSchemas' { 'ContractId', 'ContractPeriodEnd', 'ContractPeriodStart', 'InvoiceIssuerName', 'PricingCurrency' ) { - $contractCommitmentRawBlock | Should -Match "(?m)^\s+$_\s*:" + $contractCommitmentsRawBlock | Should -Match "(?m)^\s+$_\s*:" } It 'Includes FOCUS 1.4 column <_>' -ForEach @( @@ -77,7 +94,49 @@ Describe 'HubsFocusSchemas' { 'LifecycleStatus', 'Model', 'OfferCategory', 'PaymentInterval', 'PaymentModel', 'PaymentUpfrontPercentage', 'PricingCurrencyContractCommitmentCost' ) { - $contractCommitmentRawBlock | Should -Match "(?m)^\s+$_\s*:" + $contractCommitmentsRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'BillingPeriods_raw exists with FOCUS 1.4 columns' { + + BeforeAll { + $allBlocks = [regex]::Matches($rawTablesContent, '(?ms)\.alter table BillingPeriods_raw \(\r?\n(.*?)\r?\n\)') + $script:billingPeriodsRawBlock = if ($allBlocks.Count -ge 1) { $allBlocks[$allBlocks.Count - 1].Groups[1].Value } else { '' } + } + + It 'Defines BillingPeriods_raw' { + $rawTablesContent | Should -Match '\.alter table BillingPeriods_raw \(' + } + + It 'Includes column <_>' -ForEach @( + 'BillingPeriodCreated', 'BillingPeriodEnd', 'BillingPeriodLastUpdated', + 'BillingPeriodStart', 'BillingPeriodStatus', 'InvoiceIssuerName' + ) { + $billingPeriodsRawBlock | Should -Match "(?m)^\s+$_\s*:" + } + } + + Context 'InvoiceDetails_raw exists with FOCUS 1.4 columns' { + + BeforeAll { + $allBlocks = [regex]::Matches($rawTablesContent, '(?ms)\.alter table InvoiceDetails_raw \(\r?\n(.*?)\r?\n\)') + $script:invoiceDetailsRawBlock = if ($allBlocks.Count -ge 1) { $allBlocks[$allBlocks.Count - 1].Groups[1].Value } else { '' } + } + + It 'Defines InvoiceDetails_raw' { + $rawTablesContent | Should -Match '\.alter table InvoiceDetails_raw \(' + } + + It 'Includes column <_>' -ForEach @( + 'BilledCost', 'BillingAccountId', 'BillingCurrency', 'BillingPeriodEnd', + 'BillingPeriodStart', 'ChargeCategory', 'InvoiceDetailCreated', 'InvoiceDetailDescription', + 'InvoiceDetailGrain', 'InvoiceDetailId', 'InvoiceDetailLastUpdated', 'InvoiceId', + 'InvoiceIssueDate', 'InvoiceIssueStatus', 'InvoiceIssuerName', 'PaymentCurrency', + 'PaymentCurrencyBilledCost', 'PaymentCurrencyInvoiceDetailId', 'PaymentDueDate', + 'PaymentTerms', 'PurchaseOrderNumber', 'ReferenceInvoiceId' + ) { + $invoiceDetailsRawBlock | Should -Match "(?m)^\s+$_\s*:" } } @@ -85,7 +144,7 @@ Describe 'HubsFocusSchemas' { BeforeAll { $script:costsFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table Costs_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } - $script:contractCommitmentFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table ContractCommitment_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + $script:contractCommitmentsFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table ContractCommitments_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } } It 'Defines Costs_transform_v1_4' { @@ -96,43 +155,75 @@ Describe 'HubsFocusSchemas' { $ingestionFiles.v1_4 | Should -Match '\.create-merge table Costs_final_v1_4' } - It 'Defines ContractCommitment_transform_v1_4' { - $ingestionFiles.v1_4 | Should -Match 'ContractCommitment_transform_v1_4\(\)' + It 'Defines ContractCommitments_transform_v1_4 (plural)' { + $ingestionFiles.v1_4 | Should -Match 'ContractCommitments_transform_v1_4\(\)' + } + + It 'Defines ContractCommitments_final_v1_4 table (plural)' { + $ingestionFiles.v1_4 | Should -Match '\.create-merge table ContractCommitments_final_v1_4' + } + + It 'Defines BillingPeriods_transform_v1_4' { + $ingestionFiles.v1_4 | Should -Match 'BillingPeriods_transform_v1_4\(\)' + } + + It 'Defines BillingPeriods_final_v1_4 table' { + $ingestionFiles.v1_4 | Should -Match '\.create-merge table BillingPeriods_final_v1_4' + } + + It 'Defines InvoiceDetails_transform_v1_4' { + $ingestionFiles.v1_4 | Should -Match 'InvoiceDetails_transform_v1_4\(\)' } - It 'Defines ContractCommitment_final_v1_4 table' { - $ingestionFiles.v1_4 | Should -Match '\.create-merge table ContractCommitment_final_v1_4' + It 'Defines InvoiceDetails_final_v1_4 table' { + $ingestionFiles.v1_4 | Should -Match '\.create-merge table InvoiceDetails_final_v1_4' } It 'Costs_final_v1_4 block was extracted' { $costsFinalV14Block | Should -Not -BeNullOrEmpty } - It 'Costs_final_v1_4 does NOT include deprecated <_> (removed in FOCUS 1.4)' -ForEach @( + It 'Costs_final_v1_4 does NOT include removed <_>' -ForEach @( 'ProviderName', 'PublisherName' ) { $costsFinalV14Block | Should -Not -Match "(?m)^\s+$_\s*:" } - It 'ContractCommitment_final_v1_4 includes FOCUS 1.4 column <_>' -ForEach @( + It 'Costs_final_v1_4 includes new FOCUS 1.4 column <_>' -ForEach @( + 'CommitmentProgramEligibilityDetails', 'InvoiceDetailId', + 'ContractCommitmentBenefitCategory', 'ContractCommitmentCreated', + 'ContractCommitmentDiscountPercentage', 'ContractCommitmentDurationType', + 'ContractCommitmentFulfillmentInterval', 'ContractCommitmentLastUpdated', + 'ContractCommitmentLifecycleStatus', 'ContractCommitmentModel', + 'ContractCommitmentOfferCategory', 'ContractCommitmentPaymentInterval', + 'ContractCommitmentPaymentModel', 'ContractCommitmentPaymentUpfrontPercentage' + ) { + $costsFinalV14Block | Should -Match "(?m)^\s+$_\s*:" + } + + It 'ContractCommitments_final_v1_4 includes FOCUS 1.4 column <_>' -ForEach @( 'BenefitCategory', 'ContractCommitmentApplicability', 'Created', 'DiscountPercentage', 'DurationType', 'FulfillmentInterval', 'LastUpdated', 'LifecycleStatus', 'Model', 'OfferCategory', 'PaymentInterval', 'PaymentModel', 'PaymentUpfrontPercentage', 'PricingCurrencyContractCommitmentCost' ) { - $contractCommitmentFinalV14Block | Should -Match "(?m)^\s+$_\s*:" + $contractCommitmentsFinalV14Block | Should -Match "(?m)^\s+$_\s*:" } } Context 'HubSetup_v1_4.kql' { It 'Defines <_>' -ForEach @( - 'CommitmentDiscountUsage_v1_4', 'ContractCommitment_v1_4', - 'Costs_v1_4', 'Prices_v1_4', 'Recommendations_v1_4', 'Transactions_v1_4' + 'BillingPeriods_v1_4', 'CommitmentDiscountUsage_v1_4', 'ContractCommitments_v1_4', + 'Costs_v1_4', 'InvoiceDetails_v1_4', 'Prices_v1_4', 'Recommendations_v1_4', 'Transactions_v1_4' ) { $hubFiles.v1_4 | Should -Match "$_\(\)" } + It 'Does NOT define singular ContractCommitment_v1_4' { + $hubFiles.v1_4 | Should -Not -Match 'ContractCommitment_v1_4\(\)' + } + It 'Costs_v1_4 unions all prior versions' -ForEach @( 'Costs_final_v1_0', 'Costs_final_v1_2', 'Costs_final_v1_4' ) { @@ -143,11 +234,16 @@ Describe 'HubsFocusSchemas' { Context 'HubSetup_Latest.kql aliases' { It 'Aliases <_> to v1_4 (latest GA)' -ForEach @( - 'CommitmentDiscountUsage', 'ContractCommitment', 'Costs', 'Prices', 'Recommendations', 'Transactions' + 'BillingPeriods', 'CommitmentDiscountUsage', 'ContractCommitments', + 'Costs', 'InvoiceDetails', 'Prices', 'Recommendations', 'Transactions' ) { $hubFiles.Latest | Should -Match "(?ms)$_\(\)\s*\{\s*${_}_v1_4\(\)\s*\}" } + It 'Does NOT alias to singular ContractCommitment' { + $hubFiles.Latest | Should -Not -Match '(?m)^ContractCommitment\(\)' + } + It 'Does NOT alias to v1_2 or older' { $hubFiles.Latest | Should -Not -Match '_v1_2\(\)' } From bbb71802cb3849363c16e7b8883a212397c255ee Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 27 May 2026 08:28:04 +0000 Subject: [PATCH 18/19] chore: Update ms.date in docs-mslearn files --- docs-mslearn/toolkit/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 0336977cb..4c50c8c1d 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 05/17/2026 +ms.date: 05/27/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From 2a88b02d542d277fe575ffc1d66b45f7d1c9411e Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 24 Jun 2026 16:48:19 -0700 Subject: [PATCH 19/19] Add x_RecommendationCategory and x_RecommendationSubcategory enrichments Adds two hub-toolkit enrichment columns to the Recommendations dataset in the FOCUS 1.4 hub schema, normalized to a fixed hub taxonomy so reports can group recommendations consistently across providers. - IngestionSetup_v1_4.kql: normalize x_RecommendationCategory into the 5 top-level values (Cost, Operational Excellence, Performance, Reliability, Security); map Azure Advisor's HighAvailability -> Reliability; derive x_RecommendationSubcategory from the documented taxonomy with 'Other' as the fallback. Add both columns to Recommendations_final_v1_4. - HubSetup_v1_4.kql: include both columns in the Recommendations_v1_4() project list. - data-model.md: document the allowed values for both columns and the normalization behavior. - HubsFocusSchemas.Tests.ps1: assert columns exist in the final table, the transform projects them, and the case() expression covers all 5 top-level categories. - changelog.md: note the new enrichment columns under FinOps hubs v15.0.0. Co-Authored-By: Michael Flanakin Co-Authored-By: Claude Opus 4.7 --- docs-mslearn/toolkit/changelog.md | 3 +- docs-mslearn/toolkit/hubs/data-model.md | 55 ++++++++++++++++++- .../Tests/Unit/HubsFocusSchemas.Tests.ps1 | 50 +++++++++++++++++ .../Analytics/scripts/HubSetup_v1_4.kql | 1 + .../Analytics/scripts/IngestionSetup_v1_4.kql | 55 ++++++++++++++++++- 5 files changed, 160 insertions(+), 4 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 4c50c8c1d..97d8f7012 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 05/27/2026 +ms.date: 06/24/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -39,6 +39,7 @@ The following section lists features and enhancements that are currently in deve - **Added** - Added FOCUS 1.4 hub schema (`v1_4`). New cost and usage columns: `AllocatedMethodId`, `AllocatedMethodDetails`, `AllocatedResourceId`, `AllocatedResourceName`, `AllocatedTags`, `ContractApplied`, `ServiceProviderName`, `HostProviderName`, `CommitmentProgramEligibilityDetails`, `InvoiceDetailId`, and 12 `ContractCommitment*` per-row columns (`ContractCommitmentBenefitCategory`, `ContractCommitmentCreated`, `ContractCommitmentDiscountPercentage`, `ContractCommitmentDurationType`, `ContractCommitmentFulfillmentInterval`, `ContractCommitmentLastUpdated`, `ContractCommitmentLifecycleStatus`, `ContractCommitmentModel`, `ContractCommitmentOfferCategory`, `ContractCommitmentPaymentInterval`, `ContractCommitmentPaymentModel`, `ContractCommitmentPaymentUpfrontPercentage`). Removes deprecated `ProviderName` and `PublisherName` from the schema (raw tables keep them for back compat). Three new supplemental datasets: `ContractCommitments` (28 columns), `BillingPeriods` (6 columns), and `InvoiceDetails` (22 columns) ([#2120](https://github.com/microsoft/finops-toolkit/issues/2120)). - Added unversioned `ContractCommitments()`, `BillingPeriods()`, and `InvoiceDetails()` functions aliasing to their `_v1_4` counterparts. + - Added `x_RecommendationCategory` and `x_RecommendationSubcategory` enrichment columns to the `Recommendations` dataset, normalized to a fixed hub taxonomy (Cost, Operational Excellence, Performance, Reliability, Security). See [data model](hubs/data-model.md#recommendations-managed-dataset) for the allowed values. - **Changed** - Retargeted unversioned `Costs()`, `Prices()`, `CommitmentDiscountUsage()`, `Recommendations()`, and `Transactions()` aliases to their `_v1_4` counterparts. - Refreshed the canonical "add a new FOCUS version" procedure in [src/templates/finops-hub/docs/README.md](https://github.com/microsoft/finops-toolkit/tree/dev/src/templates/finops-hub/docs/README.md) with multi-version-cycle, plugin, and changelog steps. diff --git a/docs-mslearn/toolkit/hubs/data-model.md b/docs-mslearn/toolkit/hubs/data-model.md index e06a19519..f8a927f66 100644 --- a/docs-mslearn/toolkit/hubs/data-model.md +++ b/docs-mslearn/toolkit/hubs/data-model.md @@ -3,7 +3,7 @@ title: FinOps hubs data model description: Learn about the tables and functions available in FinOps hubs to build your own queries, reports, and dashboards. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 06/24/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -736,7 +736,8 @@ Columns in the **Recommendations** managed dataset include: | x_EffectiveCostBefore | Real | Source: Microsoft EA+MCA 2023-05-01. | | x_EffectiveCostSavings | Real | Source: Microsoft EA+MCA 2023-05-01. | | x_IngestionTime | Datetime | Source: Hubs add-on. | -| x_RecommendationCategory | String | Source: Hubs recommendations. | +| x_RecommendationCategory | String | Source: Hubs recommendations. Normalized to the hub taxonomy. | +| x_RecommendationSubcategory | String | Source: Hubs add-on. Normalized to the hub taxonomy. | | x_RecommendationDate | Datetime | Source: Microsoft EA+MCA 2023-05-01. | | x_RecommendationDescription | String | Source: Hubs add-on. | | x_RecommendationDetails | Dynamic | Contains source-specific metadata. See notes below the table. | @@ -749,6 +750,56 @@ Columns in the **Recommendations** managed dataset include: The **x_RecommendationDetails** column is a dynamic object that contains source-specific metadata. For Cost Management reservation recommendations, it includes commitment discount details like scope, term, SKU, and region. For hubs recommendations, it includes the recommendation provider, solution, type ID, resource type, and any additional properties from the query. +The **x_RecommendationCategory** and **x_RecommendationSubcategory** columns classify each recommendation against a fixed hub taxonomy so reports can group across providers consistently. Source data is normalized on ingestion: source category values (for example, Azure Advisor's `HighAvailability` → `Reliability`) are mapped to the allowed values below. Unmapped category values become an empty string and unmapped subcategory values become `Other`. + +Allowed **x_RecommendationCategory** values: `Cost`, `Operational Excellence`, `Performance`, `Reliability`, `Security`. + +Allowed **x_RecommendationSubcategory** values, grouped by parent category: + +- Cost + - Autoscaling + - Commitment Discount Coverage + - Commitment Discount Utilization + - Idle Resources (includes unused capacity and orphaned resources) + - Low Utilization + - Negotiated Discount Enablement + - Region Placement + - Right-Sizing + - Scheduling + - Service Selection and Architecture + - SKU Modernization + - Spot Eligibility +- Operational Excellence + - Automation and Process + - Best Practices + - Governance and Policy + - Observability + - Operational Hygiene + - Quotas and Limits + - Resource Consistency +- Performance + - Performance Tuning + - Resource Configuration + - Resource Sizing + - Scaling and Capacity + - Throughput and Latency + - Workload Placement +- Reliability + - Backup Configuration + - Capacity + - Disaster Recovery + - High Availability + - Service Retirement + - Zone Resiliency +- Security + - Compliance Alignment + - Data Protection + - Identity & Access + - Network Security + - Threat Detection + - Vulnerability Exposure +- Other (fallback when no specific subcategory applies) +
## Regions table diff --git a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 index 02f4d3584..c8bd5fcb6 100644 --- a/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 +++ b/src/powershell/Tests/Unit/HubsFocusSchemas.Tests.ps1 @@ -229,6 +229,56 @@ Describe 'HubsFocusSchemas' { ) { $hubFiles.v1_4 | Should -Match "database\('Ingestion'\)\.$_" } + + It 'Recommendations_v1_4 projects <_>' -ForEach @( + 'x_RecommendationCategory', 'x_RecommendationSubcategory' + ) { + $script:recommendationsV14Block = if ($hubFiles.v1_4 -match '(?ms)Recommendations_v1_4\(\)\s*\{(.*?)\n\}') { $Matches[1] } else { '' } + $recommendationsV14Block | Should -Match "(?m)^\s+$_\s*,?\s*$" + } + } + + Context 'Recommendations enrichment in IngestionSetup_v1_4.kql' { + + BeforeAll { + $script:recommendationsTransformV14 = if ($ingestionFiles.v1_4 -match '(?ms)Recommendations_transform_v1_4\(\)\s*\{(.*?)\n\}') { $Matches[1] } else { '' } + $script:recommendationsFinalV14Block = if ($ingestionFiles.v1_4 -match '(?ms)\.create-merge table Recommendations_final_v1_4 \(\r?\n(.*?)\r?\n\)') { $Matches[1] } else { '' } + } + + It 'Recommendations_transform_v1_4 block was extracted' { + $recommendationsTransformV14 | Should -Not -BeNullOrEmpty + } + + It 'Recommendations_final_v1_4 block was extracted' { + $recommendationsFinalV14Block | Should -Not -BeNullOrEmpty + } + + It 'Recommendations_final_v1_4 includes column <_>' -ForEach @( + 'x_RecommendationCategory', 'x_RecommendationSubcategory' + ) { + $recommendationsFinalV14Block | Should -Match "(?m)^\s+$_\s*:" + } + + It 'Recommendations_transform_v1_4 projects <_>' -ForEach @( + 'x_RecommendationCategory', 'x_RecommendationSubcategory' + ) { + $recommendationsTransformV14 | Should -Match "(?m)^\s+$_\s*,?\s*(//.*)?$" + } + + It 'Recommendations_transform_v1_4 normalizes category to <_>' -ForEach @( + "'Cost'", "'Operational Excellence'", "'Performance'", "'Reliability'", "'Security'" + ) { + $recommendationsTransformV14 | Should -Match ([regex]::Escape($_)) + } + + It 'Recommendations_transform_v1_4 maps Advisor HighAvailability to Reliability' { + $recommendationsTransformV14 | Should -Match "HighAvailability" + $recommendationsTransformV14 | Should -Match "'Reliability'" + } + + It 'Recommendations_transform_v1_4 falls back subcategory to Other' { + $recommendationsTransformV14 | Should -Match "'Other'" + } } Context 'HubSetup_Latest.kql aliases' { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql index fc67110b2..b7b232d05 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_4.kql @@ -582,6 +582,7 @@ Recommendations_v1_4() x_EffectiveCostSavings, x_IngestionTime, x_RecommendationCategory, + x_RecommendationSubcategory, x_RecommendationDate, x_RecommendationDescription, x_RecommendationDetails, diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql index f0386ff32..823cf44ad 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_4.kql @@ -2084,6 +2084,57 @@ Recommendations_transform_v1_4() // Set ResourceType display name from x_ResourceType code | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) // + // Normalize x_RecommendationCategory into the hub taxonomy (Cost, Operational Excellence, Performance, Reliability, Security) + // Source: Azure Advisor Category values are PascalCase (Cost, HighAvailability, OperationalExcellence, Performance, Security). + // Reservation recommendations from Cost Management have no category set; default to Cost. + | extend x_RecommendationCategory = tostring(case( + // Empty / null: default reservation recommendations to Cost; leave others empty for unknown + isempty(x_RecommendationCategory), iff(x_SourceType == 'ReservationRecommendations', 'Cost', ''), + // Already normalized values pass through + x_RecommendationCategory in ('Cost', 'Operational Excellence', 'Performance', 'Reliability', 'Security'), x_RecommendationCategory, + // Azure Advisor PascalCase values + x_RecommendationCategory =~ 'Cost', 'Cost', + x_RecommendationCategory =~ 'HighAvailability', 'Reliability', + x_RecommendationCategory =~ 'OperationalExcellence', 'Operational Excellence', + x_RecommendationCategory =~ 'Performance', 'Performance', + x_RecommendationCategory =~ 'Security', 'Security', + // Unknown source value: leave empty + '' + )) + // + // Derive x_RecommendationSubcategory from the hub taxonomy. Default to 'Other' so every row has a value. + // Reservation recommendations always map to 'Commitment Discount Coverage'. For other sources without a + // deterministic mapping we fall back to 'Other'. Source value (if present) comes from x_RecommendationDetails. + | extend tmp_Subcategory = tostring(x_RecommendationDetails.x_RecommendationSubcategory) + | extend x_RecommendationSubcategory = tostring(case( + // Already a documented subcategory value passes through + tmp_Subcategory in ( + // Cost + 'Commitment Discount Coverage', 'Negotiated Discount Enablement', 'Spot Eligibility', + 'Region Placement', 'Service Selection and Architecture', 'Right-Sizing', + 'SKU Modernization', 'Autoscaling', 'Commitment Discount Utilization', 'Scheduling', + 'Low Utilization', 'Idle Resources', + // Operational Excellence + 'Automation and Process', 'Best Practices', 'Governance and Policy', 'Observability', + 'Operational Hygiene', 'Quotas and Limits', 'Resource Consistency', + // Performance + 'Performance Tuning', 'Resource Configuration', 'Resource Sizing', + 'Scaling and Capacity', 'Throughput and Latency', 'Workload Placement', + // Reliability + 'Backup Configuration', 'Capacity', 'Disaster Recovery', 'High Availability', + 'Service Retirement', 'Zone Resiliency', + // Security + 'Compliance Alignment', 'Data Protection', 'Identity & Access', 'Network Security', + 'Threat Detection', 'Vulnerability Exposure', + // Fallback + 'Other' + ), tmp_Subcategory, + // Reservation recommendations are always commitment discount coverage + x_SourceType == 'ReservationRecommendations', 'Commitment Discount Coverage', + // No deterministic mapping yet for Advisor / custom recommendations + 'Other' + )) + // | project ProviderName, ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering @@ -2095,7 +2146,8 @@ Recommendations_transform_v1_4() x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), x_IngestionTime, - x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationCategory, + x_RecommendationSubcategory, x_RecommendationDate, x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), x_RecommendationDetails, @@ -2120,6 +2172,7 @@ Recommendations_transform_v1_4() x_EffectiveCostSavings: real, x_IngestionTime: datetime, x_RecommendationCategory: string, + x_RecommendationSubcategory: string, x_RecommendationDate: datetime, x_RecommendationDescription: string, x_RecommendationDetails: dynamic,