From 77a75b1b34215f5f0e53fa287dd73c59c1fc8b4a Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:07:02 -0700 Subject: [PATCH 1/3] Add FOCUS 1.3 ingestion (#2124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds IngestionSetup_v1_3.kql with Costs/Prices/CommitmentDiscountUsage/ Recommendations/Transactions transforms and final tables renamed for FOCUS 1.3. Cost and Usage gains 8 new FOCUS 1.3 columns: AllocatedMethodId/Details/ResourceId/ResourceName/Tags (data-generator split cost allocation), ContractApplied (per-row contract commitment application), ServiceProviderName + HostProviderName (replacing the deprecated ProviderName/PublisherName, with empty-fallback for back compat). Costs_raw now carries the new columns so downstream v1_2 transforms keep working and v1_3 transforms can read them directly. The v1_3 file is wired into the Bicep deployment alongside v1_0 and v1_2. The unversioned Costs() function still aliases to v1_2 — phase 2 adds Costs_v1_3() and retargets Latest. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 3 +- .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../scripts/IngestionSetup_RawTables.kql | 8 + .../Analytics/scripts/IngestionSetup_v1_3.kql | 1966 +++++++++++++++++ 4 files changed, 1977 insertions(+), 1 deletion(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 2dd495e60..0963fb494 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -33,7 +33,8 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql", - "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql" + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql" ] }, { diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index b880e7108..42cd2738b 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -378,6 +378,7 @@ module ingestion_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) scripts: { v1_0: loadTextContent('scripts/IngestionSetup_v1_0.kql') v1_2: loadTextContent('scripts/IngestionSetup_v1_2.kql') + v1_3: loadTextContent('scripts/IngestionSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 94fead987..83a3be40e 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -359,6 +359,11 @@ // Costs_raw table -- Redefine all columns .alter table Costs_raw ( + AllocatedMethodDetails: string, // FOCUS 1.3+ (JSON) + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: string, // FOCUS 1.3+ (JSON) AvailabilityZone: string, // FOCUS 0.5+ BilledCost: real, // FOCUS 0.5+ BillingAccountId: string, // FOCUS 0.5+ @@ -385,9 +390,11 @@ CommitmentDiscountUnit: string, // FOCUS 1.1+ ConsumedQuantity: real, // FOCUS 1.0+ ConsumedUnit: string, // FOCUS 1.0+ + ContractApplied: string, // FOCUS 1.3+ (JSON) ContractedCost: real, // FOCUS 1.0+ ContractedUnitPrice: real, // FOCUS 1.0+ EffectiveCost: real, // FOCUS 1.0-preview+ + HostProviderName: string, // FOCUS 1.3+ InvoiceId: string, // FOCUS 1.2+ InvoiceIssuerName: string, // FOCUS 0.5+ ListCost: real, // FOCUS 1.0-preview+ @@ -406,6 +413,7 @@ ResourceType: string, // FOCUS 1.0-preview+ ServiceCategory: string, // FOCUS 0.5+ ServiceName: string, // FOCUS 0.5+ + ServiceProviderName: string, // FOCUS 1.3+ ServiceSubcategory: string, // FOCUS 1.1+ SkuId: string, // FOCUS 1.0-preview+ SkuMeter: string, // FOCUS 1.1+ diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql new file mode 100644 index 000000000..70d29a2ed --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql @@ -0,0 +1,1966 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database +// See known issues @ https://github.com/microsoft/finops-toolkit/issues/1111 +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +//===| Prices |========================================================================================================= +// Supported versions: +// - MS EA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-ea +// - MS MCA 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/price-sheet-mca +//====================================================================================================================== + +// Prices_transform_v1_3 function +.create-or-alter function +with (docstring='Transforms Prices_raw into FOCUS 1.3.', folder='Prices') +Prices_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + let prices = materialize( + Prices_raw + | extend PricingCurrency = coalesce(Currency, CurrencyCode) // CurrencyCode last as a fallback only + | extend x_SkuId = coalesce(SkuId, SkuID) + | extend x_SkuMeterId = coalesce(MeterId, MeterID) + | extend x_SkuProductId = coalesce(ProductId, ProductID) + | extend x_SkuTerm = isoMonths(Term) + | project-rename + SkuMeter = MeterName, + x_BaseUnitPrice = BasePrice, + x_EffectivePeriodEnd = EffectiveEndDate, + x_EffectivePeriodStart = EffectiveStartDate, + x_PricingUnitDescription = UnitOfMeasure, + x_SkuIncludedQuantity = IncludedQuantity, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuMeterType = MeterType, + x_SkuOfferId = OfferID, + x_SkuPartNumber = PartNumber, + x_SkuPriceType = PriceType, + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTier = TierMinimumUnits + | extend ContractedUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', UnitPrice, real(null)) // UnitPrice for savings plan is not the on-demand unit price + | extend ListUnitPrice = iff(x_SkuPriceType != 'SavingsPlan', MarketPrice, real(null)) // MarketPrice for savings plan is not the list price + | extend ChargeCategory = case( + x_SkuPriceType == 'Consumption', 'Usage', + x_SkuPriceType == 'ReservedInstance', 'Purchase', + x_SkuPriceType == 'SavingsPlan', 'Usage', // Savings plan prices are for committed usage, not the purchase + '' + ) + | extend SkuPriceIdv2 = strcat(case(x_SkuPriceType == 'Consumption', 'OD', x_SkuPriceType == 'ReservedInstance', 'RI', x_SkuPriceType == 'SavingsPlan', 'SP', 'XX'), substring(ChargeCategory, 0, 1), x_SkuTerm, '_', x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType, '_', x_SkuTier, x_SkuOfferId) + | extend x_BillingAccountId = iff(BillingAccountId startswith '/', split(BillingAccountId, '/')[-1], coalesce(BillingAccountId, EnrollmentNumber)) + | extend x_BillingProfileId = iff(BillingProfileId startswith '/', split(BillingProfileId, '/')[-1], coalesce(BillingProfileId, EnrollmentNumber)) + | extend tmp_SavingsPlanKey = strcat(x_SkuMeterId, x_SkuProductId, x_SkuId, x_SkuTier, x_SkuOfferId) + // + // Get latest ingested row based on the unique ID + | extend x_IngestionTime = ingestion_time() + ); + // + // Meters for reservations and savings plans to identify commitment eligibility + let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; + let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // + // Copy list/base/contracted prices from on-demand SKUs + prices + | where x_SkuPriceType == 'SavingsPlan' + // If we use join, specify the shuffle key + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) + | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) + | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) + | project-away ListUnitPrice1, ContractedUnitPrice1, x_BaseUnitPrice1, tmp_SavingsPlanKey + | union ((prices | where x_SkuPriceType != 'SavingsPlan')) + // + // Set CommitmentDiscountCategory for reuse + | extend CommitmentDiscountCategory = case( + x_SkuPriceType == 'ReservedInstance', 'Usage', + x_SkuPriceType == 'SavingsPlan', 'Spend', + '' + ) + // + // Calculate commitment discount eligibility + // TODO: Would a join be faster? + // TODO: Check this to ensure it's correct + | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + // + // TODO: Implement x_CommitmentDiscountNormalizedRatio + | extend x_CommitmentDiscountNormalizedRatio = real(null) + // + // Add PricingUnit and x_PricingBlockSize + // TODO: Compare join vs. lookup perf -- | join kind=leftouter (PricingUnits) on x_PricingUnitDescription | project-away x_PricingUnitDescription1 + | lookup kind=leftouter (PricingUnits) on x_PricingUnitDescription + // + | extend x_EffectiveUnitPrice = iff(x_SkuPriceType == 'SavingsPlan', UnitPrice, real(null)) // Savings plan prices are for the effective price, not the contracted price + | extend x_EffectiveUnitPriceDiscount = ContractedUnitPrice - x_EffectiveUnitPrice + | extend x_ContractedUnitPriceDiscount = ListUnitPrice - ContractedUnitPrice + | extend x_TotalUnitPriceDiscount = ListUnitPrice - x_EffectiveUnitPrice + | project + BillingAccountId = tolower(case( + BillingProfileId startswith '/', BillingProfileId, + BillingAccountId startswith '/', BillingAccountId, + strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId, iff(x_BillingProfileId == x_BillingAccountId, '', strcat('/billingprofiles/', x_BillingProfileId))) + )), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName, x_BillingProfileId), + BillingCurrency = coalesce(BillingCurrency, CurrencyCode, Currency), // Currency last as a fallback only + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType = case( + x_SkuPriceType == 'ReservedInstance', 'Reservation', + x_SkuPriceType == 'SavingsPlan', 'Savings plan', + '' + ), + CommitmentDiscountUnit = case( + isempty(CommitmentDiscountCategory), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), PricingUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', PricingUnit), + '' + ), + ContractedUnitPrice, + ListUnitPrice, + PricingCategory = case( + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed', + '' + ), + PricingCurrency, + PricingUnit, + SkuId = coalesce(ProductId, ProductID), + SkuMeter, + SkuPriceId = strcat(x_SkuProductId, '_', x_SkuId, '_', x_SkuMeterType), + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement = case( + strlen(x_BillingAccountId) > 32, 'MCA', + strlen(x_BillingAccountId) < 32, 'EA', + 'Unknown' + ), + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent = 1.0 * x_ContractedUnitPriceDiscount / ListUnitPrice * 100, + x_EffectivePeriodEnd = startofmonth(x_EffectivePeriodEnd + 1h), + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent = 1.0 * x_EffectiveUnitPriceDiscount / ContractedUnitPrice * 100, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory = case( + x_SkuPriceType == 'Consumption' and (x_SkuIncludedQuantity > 0 or x_SkuTier > 0), 'Tiered', + x_SkuPriceType == 'Consumption', 'Standard', + x_SkuPriceType == 'ReservedInstance', 'Standard', // Reservation purchases are tracked as "Standard" + x_SkuPriceType == 'SavingsPlan', 'Committed Spend', + '' + ), + x_PricingUnitDescription, + x_SkuDescription = Product, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName = coalesce(x_SourceName, 'Cost Management'), + x_SourceProvider = coalesce(x_SourceProvider, 'Microsoft'), + x_SourceType = coalesce(x_SourceType, 'PriceSheet'), + x_SourceVersion = coalesce(x_SourceVersion, '2023-05-01'), + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent = 1.0 * x_TotalUnitPriceDiscount / ListUnitPrice * 100 +} + +// Prices_final_v1_3 table +.create-merge table Prices_final_v1_3 ( + BillingAccountId: string, + BillingAccountName: string, + BillingCurrency: string, + ChargeCategory: string, + CommitmentDiscountCategory: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ContractedUnitPrice: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, // Azure + PricingUnit: string, + SkuId: string, + SkuMeter: string, // Azure + SkuPriceId: string, + SkuPriceIdv2: string, // Hubs add-on + x_BaseUnitPrice: real, // Azure + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure MCA + x_BillingProfileId: string, // Azure MCA + x_CommitmentDiscountNormalizedRatio: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_ContractedUnitPriceDiscount: real, // Hubs add-on + x_ContractedUnitPriceDiscountPercent: real, // Hubs add-on + x_EffectivePeriodEnd: datetime, // Azure + x_EffectivePeriodStart: datetime, // Azure + x_EffectiveUnitPrice: real, // Azure + x_EffectiveUnitPriceDiscount: real, // Hubs add-on + x_EffectiveUnitPriceDiscountPercent: real, // Hubs add-on + x_IngestionTime: datetime, // Hubs add-on + x_PricingBlockSize: real, // Hubs add-on + x_PricingSubcategory: string, // Hubs add-on + x_PricingUnitDescription: string, // Azure + x_SkuDescription: string, // Azure + x_SkuId: string, // Azure + x_SkuIncludedQuantity: real, // Azure EA + x_SkuMeterCategory: string, // Azure + x_SkuMeterId: string, // Azure + x_SkuMeterSubcategory: string, // Azure + x_SkuMeterType: string, // Azure + x_SkuPriceType: string, // Azure + x_SkuProductId: string, // Azure + x_SkuRegion: string, // Azure + x_SkuServiceFamily: string, // Azure + x_SkuOfferId: string, // Azure EA + x_SkuPartNumber: string, // Azure EA + x_SkuTerm: int, // Azure + x_SkuTier: real, // Azure MCA + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_TotalUnitPriceDiscount: real, // Hubs add-on + x_TotalUnitPriceDiscountPercent: real // Hubs add-on +) + +// Update policy for Prices_raw -> Prices_final_v1_3 +.alter table Prices_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Prices_raw", + "Query": "Prices_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Cost and usage |================================================================================================= +// Supported versions: +// - MS: 1.2-preview, 1.0, 1.0-preview(v1) +// https://aka.ms/costmgmt/exports/focus +// - AWS: 1.0 +// https://docs.aws.amazon.com/cur/latest/userguide/table-dictionary-focus-1-0-aws-columns.html +// - GCP: Jan-Jun 2024 +// https://cloud.google.com/resources/google-cloud-focus?e=48754805&hl=en +// Links to (Aug 2024): https://services.google.com/fh/files/misc/focus_guide_v1.pdf +// See also: +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/standard-usage +// - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage +// - OCI: 1.0 +// https://docs.oracle.com/iaas/Content/Billing/Concepts/costusagereportsoverview.htm#costreports__focus-cost-report-schema +// +// Support for non-Azure data is limited to ingestion only. Data is not transformed across versions. +//====================================================================================================================== + +// Costs_transform_v1_3 function +.create-or-alter function +with (docstring='All costs transformed to FOCUS 1.3.', folder='Costs') +Costs_transform_v1_3() +{ + let checkString = (column: string, oldValue: string, newValue: string) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkInt = (column: string, oldValue: int, newValue: int) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + let checkReal = (column: string, oldValue: real, newValue: real) { + iff(oldValue == newValue, dynamic({}), bag_pack(column, oldValue)) + }; + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + Costs_raw + // + // Dedupe rows + | extend x_IngestionTime = ingestion_time() + | extend x_ChargeId = '' + // TODO: Consider adding a unique charge ID per row + // hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // // 1. Resource hierarchy (including resource name), highest to lowest + // BillingAccountId, + // x_InvoiceSectionId, + // x_AccountOwnerId, + // SubAccountId, + // x_ResourceGroupName, + // ResourceName, + // // 2. Resource details + // ResourceId, + // RegionId, + // Tags, + // CommitmentDiscountId, + // x_CostCenter, + // // 4. Meter details + // SkuPriceId, + // x_SkuMeterId, + // x_SkuPartNumber, + // x_SkuOfferId, + // x_SkuDetails, + // // 5. Date + // ChargePeriodStart + // )) + // + // Identify data quality issues + // TODO: Remove x_SourceChanges in v1_3 (or later) + | extend x_SourceChanges = trim_end(',', strcat( + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based', 'InvalidChargeFrequency,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and EffectiveCost > 0, 'InvalidEffectiveCost,', ''), + iff((isempty(ContractedCost) or ContractedCost == 0) and EffectiveCost != 0, 'MissingContractedCost,', ''), + iff((isempty(ContractedUnitPrice) or ContractedUnitPrice == 0) and x_EffectiveUnitPrice != 0, 'MissingContractedUnitPrice,', ''), + iff(ListCost < ContractedCost, 'ListCostLessThanContractedCost,', ''), + iff(ContractedCost < EffectiveCost, 'ContractedCostLessThanEffectiveCost,', ''), + iff((isempty(ListCost) or ListCost == 0) and (ContractedCost != 0 or EffectiveCost != 0), 'MissingListCost,', ''), + iff((isempty(ListUnitPrice) or ListUnitPrice == 0) and (ContractedUnitPrice != 0 or x_EffectiveUnitPrice != 0), 'MissingListUnitPrice,', ''), + iff((isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001) + or (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001), + 'XEffectiveUnitPriceRoundingError,', ''), + iff(ConsumedQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingConsumedQuantity,', ''), + iff(PricingQuantity == 0 and (EffectiveCost != 0 or BilledCost != 0), 'MissingPricingQuantity,', ''), + iff(isempty(ProviderName), 'MissingProviderName,', ''), + iff(isempty(PublisherName), 'MissingPublisherName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceId), 'MissingResourceId,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceName), 'MissingResourceName,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(ResourceType), 'MissingResourceType,', ''), + iff(BilledCost > 0 and x_BilledUnitPrice == 0, 'MissingXBilledUnitPrice,', ''), + iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and isempty(x_ResourceType), 'MissingXResourceType,', ''), + iff(PricingCategory == 'Standard' and isnotempty(CommitmentDiscountId) and ChargeCategory == 'Usage', 'PricingCategoryShouldBeCommitted,', ''), + iff(x_SkuTerm == '1Year' or x_SkuTerm == '3Years' or x_SkuTerm == '5Years', 'SkuTermShouldBeAnInteger,', '') + )) + // + // Handle provider columns that moved to FOCUS + | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) + // + // Backup original prices/costs before the merge + | extend old_ContractedCost = ContractedCost + | extend old_ContractedUnitPrice = ContractedUnitPrice + | extend old_ListCost = ListCost + | extend old_ListUnitPrice = ListUnitPrice + | extend old_x_EffectiveUnitPrice = x_EffectiveUnitPrice + // + // Fix columns needed in other changes + | extend old_ProviderName = ProviderName, ProviderName = case( + isnotempty(ProviderName), ProviderName, + isnotempty(coalesce(x_CostCategories, x_Discount, x_Operation, x_ServiceCode, x_UsageType)), 'AWS', + isnotempty(coalesce(tostring(UsageAmount), tostring(x_Cost), x_Credits, x_CostType, tostring(x_CurrencyConversionRate), tostring(x_ExportTime), x_Project, x_ServiceId)), 'GCP', + isnotempty(coalesce(x_BillingProfileId, x_InvoiceSectionId)), 'Microsoft', + '' + ) + // + // Identify source + | extend x_SourceName = coalesce(x_SourceName, iff(isnotempty(x_BillingProfileId), 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, case( + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', + isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', + isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), + '' + )) + // Append version check error code + | extend x_SourceChanges = iff(x_SourceVersion == '1.0', x_SourceChanges, + strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) + ) + // + // Fix quantities + | extend old_PricingQuantity = PricingQuantity, PricingQuantity = case( + PricingQuantity != 0 or (EffectiveCost == 0 and BilledCost == 0), PricingQuantity, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(BilledCost) and BilledCost != 0 and isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, BilledCost / x_BilledUnitPrice, + PricingQuantity == 0 and isnotempty(EffectiveCost) and EffectiveCost != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0, EffectiveCost / x_EffectiveUnitPrice, + PricingQuantity + ) + | extend old_ConsumedQuantity = ConsumedQuantity, ConsumedQuantity = case( + isnotempty(ConsumedQuantity) and ConsumedQuantity != 0, ConsumedQuantity, + ChargeCategory == 'Usage', PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + ConsumedQuantity + ) + // + // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID + | extend tmp_MissingPrices = ProviderName == 'Microsoft' + and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0) + and x_EffectiveUnitPrice != 0 + and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused') + and isnotempty(strcat(x_SkuMeterId, x_SkuOfferId)) + | as allCosts + | where tmp_MissingPrices + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | as costsWithMissingPrices + | join kind=leftouter ( + Prices_final_v1_3 + | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId)) + | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey)) + | summarize ListUnitPrice = min(ListUnitPrice), ContractedUnitPrice = min(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit + ) on tmp_ReservationPriceLookupKey + // + // Select the best price to use for each row + | extend x_EffectiveUnitPrice = case( + // If price is a rounding error away from the billed price, use the billed price + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(x_BilledUnitPrice - x_EffectiveUnitPrice) < 0.0001, x_BilledUnitPrice, + // If price is a rounding error away from the contracted price, use the contracted price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0 and isnotempty(x_EffectiveUnitPrice) and x_EffectiveUnitPrice != 0 and abs(ContractedUnitPrice - x_EffectiveUnitPrice) < 0.0001, ContractedUnitPrice, + x_EffectiveUnitPrice + ) + | extend ContractedUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ContractedUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ContractedUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // If billed price is available, assume the billed price is the same as contracted price to support aggregations + isnotempty(x_BilledUnitPrice) and x_BilledUnitPrice != 0, x_EffectiveUnitPrice, + // Otherwise, assume the effective price is the same as contracted price to support aggregations + x_EffectiveUnitPrice + ) + | extend ListUnitPrice = case( + // If price is already correct, keep that + (isnotempty(ListUnitPrice) and ListUnitPrice != 0) or (EffectiveCost == 0 and BilledCost == 0), ListUnitPrice, + // If both prices use the same scale, use the new one + PricingUnit == PricingUnit1 and x_PricingBlockSize == x_PricingBlockSize1, ListUnitPrice1 * x_BillingExchangeRate, + // If prices are the same unit but not the same scale, use the new one but correct the scale + PricingUnit == PricingUnit1 and x_PricingBlockSize != x_PricingBlockSize1 and isnotempty(x_PricingBlockSize) and isnotempty(x_PricingBlockSize1), ListUnitPrice1 * x_BillingExchangeRate / x_PricingBlockSize1 * x_PricingBlockSize, + // Otherwise, assume the contracted price is the same as list price to support aggregations + ContractedUnitPrice + ) + // Calculate missing costs based on new prices -- If cost is already correct, keep that; if not and price is available, recalculate the cost; otherwise, keep the existing cost + | extend ContractedCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ContractedCost) and ContractedCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ContractedCost, + // ContractedCost is 0 in all other scenarios... + // If 0 and there's a billed cost and prices are the same, use BilledCost + isnotempty(BilledCost) and BilledCost != 0 and ContractedUnitPrice == x_BilledUnitPrice, BilledCost, + // If 0 and there's a billed cost and prices are the same, use EffectiveCost + isnotempty(EffectiveCost) and EffectiveCost != 0 and ContractedUnitPrice == x_EffectiveUnitPrice, EffectiveCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ContractedUnitPrice) and ContractedUnitPrice != 0, ContractedUnitPrice * PricingQuantity, + // If 0 and there's no price, assume EffectiveCost + isempty(ContractedUnitPrice) or ContractedUnitPrice == 0, EffectiveCost, + // Fall back to the original value for any unhandled scenarios + ContractedCost + ) + | extend ListCost = case( + // If not set or there's no cost, keep the original value + (isnotempty(ListCost) and ListCost != 0) or (EffectiveCost == 0 and BilledCost == 0), ListCost, + // ListCost is 0 in all other scenarios... + // If 0 and there's a contracted cost and prices are the same, use ContractedCost + isnotempty(ContractedCost) and ContractedCost != 0 and ListUnitPrice == ContractedUnitPrice, ContractedCost, + // If 0 and there's a price, calculate the cost based on the price + isnotempty(ListUnitPrice) and ListUnitPrice != 0, ListUnitPrice * PricingQuantity, + // If 0 and there's no price, assume ContractedCost + isempty(ListUnitPrice) or ListUnitPrice == 0, ContractedCost, + // Fall back to the original value for any unhandled scenarios + ListCost + ) + // Merge the rest of the unmodified cost records and remove excess columns + | union (allCosts | where not(tmp_MissingPrices)) + | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1 + // + | extend SkuPriceDetails = parse_json(SkuPriceDetails) + | extend Tags = parse_json(Tags) + | extend x_SkuDetails = parse_json(x_SkuDetails) + // + // Handle FOCUS 1.0-preview + | extend old_ChargeSubcategory = ChargeSubcategory + | extend old_ChargeCategory = ChargeCategory, ChargeCategory = case( + // Handle FOCUS 1.0-preview ChargeSubcategory + ChargeSubcategory == 'Credit', 'Credit', + ChargeSubcategory == 'Refund', 'Purchase', // We are assuming purchase refunds since we don't have data to indicate usage refunds + ChargeCategory + ) + | extend old_ChargeClass = ChargeClass, ChargeClass = case(ChargeSubcategory == 'Refund', 'Correction', ChargeClass) + // + // Populate CapacityReservationId when not specified + | extend CapacityReservationId = coalesce(CapacityReservationId, tostring(coalesce(x_SkuDetails.VMCapacityReservationId, SkuPriceDetails.VMCapacityReservationId, SkuPriceDetails.x_VMCapacityReservationId))) + | extend old_CapacityReservationStatus = CapacityReservationStatus, CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + isnotempty(CapacityReservationStatus), CapacityReservationStatus, + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + // + // BUG: ChargeFrequency shows "Usage-Based" for monthly recurring savings plan purchases + | extend old_ChargeFrequency = ChargeFrequency, ChargeFrequency = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId) and ChargeFrequency == 'Usage-Based' and ProviderName == 'Microsoft' and x_SourceVersion startswith '1.0', 'Recurring', ChargeFrequency) + // + // Commitment discounts + | extend x_CommitmentDiscountNormalizedRatio = case( + // Calculate from CommitmentDiscountQuantity, if specified + isnotempty(CommitmentDiscountQuantity) and CommitmentDiscountQuantity != 0, CommitmentDiscountQuantity / PricingQuantity / coalesce(x_PricingBlockSize, real(1)), + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, SkuPriceDetails.RINormalizationRatio, SkuPriceDetails.x_RINormalizationRatio, dynamic(1))) + ) + | extend old_CommitmentDiscountQuantity = CommitmentDiscountQuantity, CommitmentDiscountQuantity = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountQuantity), CommitmentDiscountQuantity, + // FOCUS 1.0-preview, 1.0 + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend old_CommitmentDiscountUnit = CommitmentDiscountUnit, CommitmentDiscountUnit = case( + // FOCUS 1.3 + isnotempty(CommitmentDiscountUnit), CommitmentDiscountUnit, + // FOCUS 1.0 + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend old_CommitmentDiscountStatus = CommitmentDiscountStatus, CommitmentDiscountStatus = case( + // FOCUS 1.0+ + isnotempty(CommitmentDiscountStatus), CommitmentDiscountStatus, + // FOCUS 1.0-preview + ChargeSubcategory == 'Used Commitment', 'Used', + ChargeSubcategory == 'Unused Commitment', 'Unused', + '' + ) + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // Pricing + | extend old_x_AmortizationClass = x_AmortizationClass, x_AmortizationClass = case( + // FOCUS 1.3 + isnotempty(x_AmortizationClass), x_AmortizationClass, + // FOCUS 1.0-preview+ + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + | extend old_PricingCategory = PricingCategory, PricingCategory = case( + // FOCUS 1.0+ + isnotempty(PricingCategory), PricingCategory, + // FOCUS 1.0-preview + PricingCategory == 'On-Demand', 'Standard', + PricingCategory == 'Commitment-Based', 'Committed', + '' + ) + // + // Commitment discount utilization + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + // + // BUG: Fix ContractedCost that has bad values + | extend ContractedCost = iff(ProviderName == 'Microsoft' and isnotempty(PricingQuantity) and isnotempty(x_PricingBlockSize) and ContractedCost != ContractedUnitPrice * PricingQuantity, ContractedUnitPrice * PricingQuantity, ContractedCost) + // + // Handle FOCUS 1.0-preview UsageQuantity/Unit + | extend ConsumedQuantity = iff(ChargeCategory == 'Usage', coalesce(ConsumedQuantity, UsageQuantity, UsageAmount), real(null)) + | extend old_ConsumedUnit = ConsumedUnit, ConsumedUnit = iff(ChargeCategory == 'Usage' and isnotempty(ConsumedQuantity), coalesce(ConsumedUnit, UsageUnit, 'Units'), '') + // + // Convert IDs to lowercase for consistency + | extend BillingAccountId = tolower(BillingAccountId) + | extend CommitmentDiscountId = tolower(CommitmentDiscountId) + // + // BUG: Remove EffectiveCost for commitment discount purchases + | extend old_EffectiveCost = EffectiveCost, EffectiveCost = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), EffectiveCost) + | extend old_x_EffectiveCostInUsd = x_EffectiveCostInUsd, x_EffectiveCostInUsd = iff(ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), real(0), x_EffectiveCostInUsd) + // + // Clean up resource columns + | extend old_ResourceId = ResourceId, ResourceId = case( + isnotempty(ResourceId), ResourceId, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountId), CommitmentDiscountId, + ResourceId + ) + | extend old_ResourceName = ResourceName, ResourceName = tolower(case( + isnotempty(ResourceName), ResourceName, + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountName), CommitmentDiscountName, + isnotempty(ResourceId), parse_resourceid(ResourceId).ResourceName, + ResourceName + )) + | extend old_x_ResourceType = x_ResourceType, x_ResourceType = case( + isnotempty(x_ResourceType), x_ResourceType, + isnotempty(ResourceId), parse_resourceid(ResourceId).x_ResourceType, + x_ResourceType + ) + | extend old_ResourceType = ResourceType, ResourceType = case( + // Use existing resource type display name unless it's an internal resource type ID + isnotempty(ResourceType) and tolower(ResourceType) != tolower(x_ResourceType) and ResourceType !contains '/', ResourceType, + // Use CommitmentDiscountType for commitment discount purchases + ChargeCategory == 'Purchase' and isnotempty(CommitmentDiscountType), CommitmentDiscountType, + // Look up display name from internal type + isnotempty(x_ResourceType), coalesce(tostring(resource_type(x_ResourceType).SingularDisplayName), ResourceType, x_ResourceType), + ResourceType + ) + // + // Handle missing values + | extend old_PublisherName = PublisherName, PublisherName = case(PublisherName == 'Microsoft Corporation', 'Microsoft', isnotempty(PublisherName), PublisherName, x_PublisherCategory == 'Cloud Provider', ProviderName, '') + // + // Handle FOCUS 1.0-preview Region column + | extend old_Region = Region + | extend old_RegionId = RegionId, RegionId = coalesce(RegionId, iff(ProviderName == 'Microsoft', replace_string(tolower(Region), ' ', ''), Region)) + | extend RegionName = coalesce(RegionName, Region) + // + // SKU properties + | extend x_SkuCoreCount = toint(coalesce(SkuPriceDetails.CoreCount, SkuPriceDetails.x_VCPUs, x_SkuDetails.VCPUs, SkuPriceDetails.x_VCores, x_SkuDetails.VCores, SkuPriceDetails.x_vCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(SkuPriceDetails.InstanceType, SkuPriceDetails.x_ServiceType, x_SkuDetails.ServiceType, SkuPriceDetails.x_ServerSku, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + isnotempty(SkuPriceDetails.OperatingSystem), SkuPriceDetails.OperatingSystem, + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Canonical', 'Linux', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend SkuPriceDetails = case( + // FOCUS 1.3 + isnotempty(SkuPriceDetails), SkuPriceDetails, + // FOCUS 1.0-preview, 1.0 + parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // + // Azure Hybrid Benefit + | extend tmp_SqlAhb = tolower(coalesce(x_SkuDetails.AHB, SkuPriceDetails.x_AHB)) + | extend x_SkuLicenseType = case( + ChargeCategory != 'Usage', '', + x_SkuMeterCategory in ('Virtual Machines', 'Virtual Machine Licenses') and (x_SkuMeterSubcategory contains 'Windows' or coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL'), 'Windows Server', + isnotempty(tmp_SqlAhb) or x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isempty(x_SkuLicenseType), '', + coalesce(SkuPriceDetails.x_ImageType, x_SkuDetails.ImageType) == 'Windows Server BYOL' or tmp_SqlAhb == 'true' or x_SkuMeterSubcategory contains 'Azure Hybrid Benefit', 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not Enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount) or isempty(x_SkuLicenseType), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + // + // Savings + | extend x_CommitmentDiscountSavings = iff(isempty(ContractedCost) or ContractedCost == 0 or ContractedCost - EffectiveCost < 0.0001, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - ContractedCost < 0.0001, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(isempty(ListCost) or ListCost == 0 or ListCost - EffectiveCost < 0.0001, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(isempty(ContractedUnitPrice) or ContractedUnitPrice == 0 or ContractedUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - ContractedUnitPrice < 0.0001, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(isempty(ListUnitPrice) or ListUnitPrice == 0 or ListUnitPrice - x_EffectiveUnitPrice < 0.0001, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // + // Minor fixes + | extend old_BillingPeriodEnd = BillingPeriodEnd, BillingPeriodEnd = startofmonth(BillingPeriodEnd) + | extend old_BillingPeriodStart = BillingPeriodStart, BillingPeriodStart = startofmonth(BillingPeriodStart) + // + // Sort columns and apply final transforms + | project + AllocatedMethodDetails = parse_json(AllocatedMethodDetails), // FOCUS 1.3+ + AllocatedMethodId, // FOCUS 1.3+ + AllocatedResourceId, // FOCUS 1.3+ + AllocatedResourceName, // FOCUS 1.3+ + AllocatedTags = parse_json(AllocatedTags), // FOCUS 1.3+ + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied = parse_json(ContractApplied), // FOCUS 1.3+ + ContractedCost = coalesce(ContractedCost, x_OnDemandCost, x_Cost), + ContractedUnitPrice = coalesce(ContractedUnitPrice, x_OnDemandUnitPrice), + EffectiveCost, + HostProviderName = iff(isempty(HostProviderName), PublisherName, HostProviderName), // FOCUS 1.3+; falls back to deprecated PublisherName for back compat + InvoiceId = coalesce(InvoiceId, x_InvoiceId), + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency), + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName = iff(isempty(ServiceProviderName), ProviderName, ServiceProviderName), // FOCUS 1.3+; falls back to deprecated ProviderName for back compat + ServiceSubcategory, // TODO: Populate ServiceSubcategory from ServiceName when missing + SkuId, + SkuMeter = coalesce(SkuMeter, x_SkuMeterName), + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName = iff(isempty(SubAccountId), '', SubAccountName), + SubAccountType, + Tags, + x_AccountId = iff(x_AccountId == '-2', '', x_AccountId), + x_AccountName = iff(x_AccountId == '-2', '', x_AccountName), + x_AccountOwnerId = iff(x_AccountId == '-2', '', x_AccountOwnerId), + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement = case( + ProviderName == 'Microsoft' and x_BillingAccountId == x_BillingProfileId, 'EA', + ProviderName == 'Microsoft' and x_BillingAccountId != x_BillingProfileId, 'MCA', + ProviderName + ), + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility = '', // TODO: Add x_CommitmentDiscountSpendEligibility for Costs + x_CommitmentDiscountUsageEligibility = '', // TODO: Add x_CommitmentDiscountUsageEligibility for Costs + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd = coalesce(x_ContractedCostInUsd, x_OnDemandCostInUsd), + x_CostAllocationRuleName, + x_CostCategories = parse_json(x_CostCategories), + x_CostCenter, + x_CostType, + x_Credits = parse_json(x_Credits), + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount = parse_json(x_Discount), + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId = case( + x_InvoiceSectionId == '-2', '', + x_InvoiceSectionId + ), + x_InvoiceSectionName = case( + x_InvoiceSectionName == 'Unassigned', '', + x_InvoiceSectionName + ), + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription = iff(x_PricingUnitDescription == 'Unassigned', '', x_PricingUnitDescription), + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName = tolower(x_ResourceGroupName), + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, // TODO: Populate from ServiceName when missing + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues = bag_merge( + checkString('BillingPeriodEnd', old_BillingPeriodEnd, BillingPeriodEnd), + checkString('BillingPeriodStart', old_BillingPeriodStart, BillingPeriodStart), + checkString('CapacityReservationStatus', old_CapacityReservationStatus, CapacityReservationStatus), + checkString('ChargeCategory', old_ChargeCategory, ChargeCategory), + checkString('ChargeClass', old_ChargeClass, ChargeClass), + checkString('ChargeSubcategory', old_ChargeSubcategory, ''), // Not included in final schema; use empty string + checkString('ChargeFrequency', old_ChargeFrequency, ChargeFrequency), + checkReal('CommitmentDiscountQuantity', old_CommitmentDiscountQuantity, CommitmentDiscountQuantity), + checkString('CommitmentDiscountUnit', old_CommitmentDiscountUnit, CommitmentDiscountUnit), + checkString('CommitmentDiscountStatus', old_CommitmentDiscountStatus, CommitmentDiscountStatus), + checkReal('ConsumedQuantity', old_ConsumedQuantity, ConsumedQuantity), + checkString('ConsumedUnit', old_ConsumedUnit, ConsumedUnit), + checkReal('ContractedCost', old_ContractedCost, ContractedCost), + checkReal('ContractedUnitPrice', old_ContractedUnitPrice, ContractedUnitPrice), + checkReal('EffectiveCost', old_EffectiveCost, EffectiveCost), + checkReal('ListCost', old_ListCost, ListCost), + checkReal('ListUnitPrice', old_ListUnitPrice, ListUnitPrice), + checkString('PricingCategory', old_PricingCategory, PricingCategory), + checkReal('PricingQuantity', old_PricingQuantity, PricingQuantity), + checkString('ProviderName', old_ProviderName, ProviderName), + checkString('PublisherName', old_PublisherName, PublisherName), + checkString('Region', old_Region, ''), // Not included in final schema; use empty string + checkString('RegionId', old_RegionId, RegionId), + checkString('ResourceId', old_ResourceId, ResourceId), + checkString('ResourceName', old_ResourceName, ResourceName), + checkString('ResourceType', old_ResourceType, ResourceType), + checkString('x_AmortizationClass', old_x_AmortizationClass, x_AmortizationClass), + checkReal('x_EffectiveCostInUsd', old_x_EffectiveCostInUsd, x_EffectiveCostInUsd), + checkReal('x_EffectiveUnitPrice', old_x_EffectiveUnitPrice, x_EffectiveUnitPrice), + checkString('x_ResourceType', old_x_ResourceType, x_ResourceType) + ), + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + +// Costs_final_v1_3 table +.create-merge table Costs_final_v1_3 ( + AllocatedMethodDetails: dynamic, // FOCUS 1.3+ + AllocatedMethodId: string, // FOCUS 1.3+ + AllocatedResourceId: string, // FOCUS 1.3+ + AllocatedResourceName: string, // FOCUS 1.3+ + AllocatedTags: dynamic, // FOCUS 1.3+ + AvailabilityZone: string, + BilledCost: real, + BillingAccountId: string, + BillingAccountName: string, + BillingAccountType: string, + BillingCurrency: string, + BillingPeriodEnd: datetime, + BillingPeriodStart: datetime, + CapacityReservationId: string, + CapacityReservationStatus: string, + ChargeCategory: string, + ChargeClass: string, + ChargeDescription: string, + ChargeFrequency: string, + ChargePeriodEnd: datetime, + ChargePeriodStart: datetime, + CommitmentDiscountCategory: string, + CommitmentDiscountId: string, + CommitmentDiscountName: string, + CommitmentDiscountQuantity: real, + CommitmentDiscountStatus: string, + CommitmentDiscountType: string, + CommitmentDiscountUnit: string, + ConsumedQuantity: real, + ConsumedUnit: string, + ContractApplied: dynamic, // FOCUS 1.3+ + ContractedCost: real, + ContractedUnitPrice: real, + EffectiveCost: real, + HostProviderName: string, // FOCUS 1.3+ + InvoiceId: string, + InvoiceIssuerName: string, + ListCost: real, + ListUnitPrice: real, + PricingCategory: string, + PricingCurrency: string, + PricingQuantity: real, + PricingUnit: string, + ProviderName: string, + PublisherName: string, + RegionId: string, + RegionName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + ServiceCategory: string, + ServiceName: string, + ServiceProviderName: string, // FOCUS 1.3+ + ServiceSubcategory: string, + SkuId: string, + SkuMeter: string, + SkuPriceDetails: dynamic, + SkuPriceId: string, + SubAccountId: string, + SubAccountName: string, + SubAccountType: string, + Tags: dynamic, + x_AccountId: string, // Azure 1.0-preview(v1)+ + x_AccountName: string, // Azure 1.0-preview(v1)+ + x_AccountOwnerId: string, // Azure 1.0-preview(v1)+ + x_AmortizationClass: string, // Azure 1.2-preview+ + x_BilledCostInUsd: real, // Azure 1.0-preview(v1)+ + x_BilledUnitPrice: real, // Azure 1.0-preview(v1)+ + x_BillingAccountAgreement: string, // Hubs add-on + x_BillingAccountId: string, // Azure 1.0-preview(v1)+ + x_BillingAccountName: string, // Azure 1.0-preview(v1)+ + x_BillingExchangeRate: real, // Azure 1.0-preview(v1)+ + x_BillingExchangeRateDate: datetime, // Azure 1.0-preview(v1)+ + x_BillingItemCode: string, // Alibaba 1.0 + x_BillingItemName: string, // Alibaba 1.0 + x_BillingProfileId: string, // Azure 1.0-preview(v1)+ + x_BillingProfileName: string, // Azure 1.0-preview(v1)+ + x_ChargeId: string, // Azure 1.0-preview(v1) only + x_CommitmentDiscountNormalizedRatio: real, // Azure 1.2-preview+ + x_CommitmentDiscountPercent: real, // Hubs add-on + x_CommitmentDiscountSavings: real, // Hubs add-on + x_CommitmentDiscountSpendEligibility: string, // Hubs add-on + x_CommitmentDiscountUsageEligibility: string, // Hubs add-on + x_CommitmentDiscountUtilizationAmount: real, // Hubs add-on + x_CommitmentDiscountUtilizationPotential: real, // Hubs add-on + x_CommodityCode: string, // Alibaba 1.0 + x_CommodityName: string, // Alibaba 1.0 + x_ComponentName: string, // Tencent 1.0 + x_ComponentType: string, // Tencent 1.0 + x_ConsumedCoreHours: real, // Hubs add-on + x_ContractedCostInUsd: real, // Azure 1.0+ + x_CostAllocationRuleName: string, // Azure 1.0-preview(v1)+ + x_CostCategories: dynamic, // AWS 1.0 (JSON) + x_CostCenter: string, // Azure 1.0-preview(v1)+ + x_CostType: string, // GCP Jan 2024 + x_Credits: dynamic, // GCP Jan 2024 + x_CurrencyConversionRate: real, // GCP Jun 2024 + x_CustomerId: string, // Azure 1.0-preview(v1)+ + x_CustomerName: string, // Azure 1.0-preview(v1)+ + x_Discount: dynamic, // AWS 1.0 (JSON) + x_EffectiveCostInUsd: real, // Azure 1.0-preview(v1)+ + x_EffectiveUnitPrice: real, // Azure 1.0-preview(v1)+ + x_ExportTime: datetime, // GCP Jan 2024 / Tencent 1.0 + x_IngestionTime: datetime, // Hubs add-on + x_InstanceID: string, // Alibaba 1.0 + x_InvoiceIssuerId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionId: string, // Azure 1.0-preview(v1)+ + x_InvoiceSectionName: string, // Azure 1.0-preview(v1)+ + x_ListCostInUsd: real, // Azure 1.0-preview(v1)+ + x_Location: string, // GCP Jan 2024 + x_NegotiatedDiscountPercent:real, // Hubs add-on + x_NegotiatedDiscountSavings:real, // Hubs add-on + x_Operation: string, // AWS 1.0 + x_OwnerAccountID: string, // Tencent 1.0 + x_PartnerCreditApplied: string, // Azure 1.0-preview(v1)+ + x_PartnerCreditRate: string, // Azure 1.0-preview(v1)+ + x_PricingBlockSize: real, // Azure 1.0-preview(v1)+ + x_PricingSubcategory: string, // Azure 1.0-preview(v1)+ + x_PricingUnitDescription: string, // Azure 1.0-preview(v1)+ + x_Project: string, // GCP Jan 2024 + x_PublisherCategory: string, // Azure 1.0-preview(v1)+ + x_PublisherId: string, // Azure 1.0-preview(v1)+ + x_ResellerId: string, // Azure 1.0-preview(v1)+ + x_ResellerName: string, // Azure 1.0-preview(v1)+ + x_ResourceGroupName: string, // Azure 1.0-preview(v1)+ + x_ResourceType: string, // Azure 1.0-preview(v1)+ + x_ServiceCode: string, // AWS 1.0 + x_ServiceId: string, // GCP Jan 2024 + x_ServiceModel: string, // Azure 1.2-preview+ + x_ServicePeriodEnd: datetime, // Azure 1.0-preview(v1)+ + x_ServicePeriodStart: datetime, // Azure 1.0-preview(v1)+ + x_SkuCoreCount: int, // Hubs add-on + x_SkuDescription: string, // Azure 1.0-preview(v1)+ + x_SkuDetails: dynamic, // Azure 1.0-preview(v1)+ + x_SkuInstanceType: string, // Hubs add-on + x_SkuIsCreditEligible: bool, // Azure 1.0-preview(v1)+ + x_SkuLicenseQuantity: int, // Hubs add-on + x_SkuLicenseStatus: string, // Hubs add-on + x_SkuLicenseType: string, // Hubs add-on + x_SkuLicenseUnit: string, // Hubs add-on + x_SkuMeterCategory: string, // Azure 1.0-preview(v1)+ + x_SkuMeterId: string, // Azure 1.0-preview(v1)+ + x_SkuMeterSubcategory: string, // Azure 1.0-preview(v1)+ + x_SkuOfferId: string, // Azure 1.0-preview(v1)+ + x_SkuOperatingSystem: string, // Hubs add-on + x_SkuOrderId: string, // Azure 1.0-preview(v1)+ + x_SkuOrderName: string, // Azure 1.0-preview(v1)+ + x_SkuPartNumber: string, // Azure 1.0-preview(v1)+ + x_SkuPlanName: string, // Azure 1.2-preview+ + x_SkuRegion: string, // Azure 1.0-preview(v1)+ + x_SkuServiceFamily: string, // Azure 1.0-preview(v1)+ + x_SkuTerm: int, // Azure 1.0-preview(v1)+ + x_SkuTier: string, // Azure 1.0-preview(v1)+ + x_SourceChanges: string, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceValues: dynamic, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubproductName: string, // Tencent 1.0 + x_TotalDiscountPercent: real, // Hubs add-on + x_TotalSavings: real, // Hubs add-on + x_UsageType: string // AWS 1.0 +) + +// Update policy for Costs_raw -> Costs_final_v1_3 table +.alter table Costs_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Costs_raw", + "Query": "Costs_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Actual costs |=================================================================================================== +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// ActualCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +ActualCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + ActualCosts_raw + | where ChargeType in ('Purchase', 'Refund') and isnotempty(ReservationId) + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for ActualCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "ActualCosts_raw", + "Query": "ActualCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Amortized costs |================================================================================================ +// Supported versions: +// - C360-2025-04 +//====================================================================================================================== + +// AmortizedCosts_transform_v1_3 function +.create-or-alter function +with (docstring='ActualCost exports transformed to FOCUS 1.3.', folder='Costs') +AmortizedCosts_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + AmortizedCosts_raw + // + // + // !!! The rest of the query is reused for both the ActualCosts and AmortizedCosts queries -- Copy all changes in both transform functions !!! + // + // + | extend x_AmortizationClass = case( + ChargeType in ('Purchase', 'Refund') and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', 'Amortized Charge', + '' + ) + | extend tmp_ResourceInfo = parse_resourceid(ResourceId) + // TODO: PricingCategory needs to include savings plan usage and spot usage + | extend PricingCategory = case( + x_AmortizationClass == 'Amortized Charge', 'Committed', + ChargeType in ('Usage', 'Purchase'), 'Standard', + '' + ) + | extend x_ResourceType = tostring(tmp_ResourceInfo.x_ResourceType) + | project-rename + PricingQuantity = Quantity, + x_PricingUnitDescription = UnitOfMeasure + | join kind=leftouter (PricingUnits) on x_PricingUnitDescription + | join kind=leftouter (Regions) on ResourceLocation + | join kind=leftouter (ResourceTypes | project x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + // TODO: Add the following in 1.2: PublisherName, x_PublisherCategory, x_Environment + | join kind=leftouter (Services | where isnotempty(x_ResourceType) | project x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + | join kind=leftouter (Services | where isnotempty(x_ConsumedService) | summarize take_any(ServiceName), take_any(ServiceCategory), take_any(ServiceSubcategory), take_any(x_ServiceModel) by ConsumedService = x_ConsumedService) on ConsumedService + | extend CommitmentDiscountCategory = iff(isnotempty(ReservationId), 'Usage', '') // TODO: CommitmentDiscountCategory needs to handle savings plans + | project + AvailabilityZone = AvailabilityZone, + BilledCost = iff(x_AmortizationClass == 'Amortized Charge', real(0), Cost), + BillingAccountId = strcat('/providers/microsoft.billing/billingaccounts/', BillingAccountId, iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), '', strcat('/billingprofiles/', BillingProfileId))), + BillingAccountName = coalesce(BillingProfileName, BillingAccountName), + BillingAccountType = iff(BillingAccountId == BillingProfileId or isempty(BillingProfileId), 'Billing Profile', 'Billing Account'), + BillingCurrency, + BillingPeriodEnd = startofmonth(BillingPeriodEndDate, 1), + BillingPeriodStart = startofmonth(BillingPeriodStartDate), + CapacityReservationId = '', + CapacityReservationStatus = '', + ChargeCategory = case( + ChargeType in ('Usage', 'Purchase', 'Credit', 'Tax'), ChargeType, + ChargeType in ('UnusedReservation', 'UnusedSavingsPlan'), 'Usage', + ChargeType == 'Refund', 'Purchase', + 'Adjustment' + ), + ChargeClass = iff(ChargeType == 'Refund', 'Correction', ''), + ChargeDescription = Product, + ChargeFrequency = case( + Frequency == 'UsageBased', 'Usage-Based', + Frequency == 'OneTime', 'One-Time', + Frequency // "Recurring" and any fallback + ), + ChargePeriodEnd = Date + 1d, + ChargePeriodStart = Date, + ChargeSubcategory = '', + // TODO: CommitmentDiscount* columns need to handle savings plans + CommitmentDiscountCategory, + CommitmentDiscountId = iff(isnotempty(ReservationId), strcat('/providers/microsoft.capacity/reservationorders/', ProductOrderId, '/reservations/', ReservationId), ''), + CommitmentDiscountName = ReservationName, + CommitmentDiscountQuantity = real(null), + CommitmentDiscountStatus = case( + isempty(ReservationId), '', + isnotempty(ReservationId) and ChargeType == 'Usage', 'Used', + ChargeType startswith 'Unused', 'Unused', + 'Unused' + ), + CommitmentDiscountType = iff(isnotempty(ReservationId), 'Reservation', ''), + CommitmentDiscountUnit = '', + ConsumedQuantity = PricingQuantity * x_PricingBlockSize, + ConsumedUnit = PricingUnit, + ContractedCost = UnitPrice * PricingQuantity, + ContractedUnitPrice = UnitPrice, + EffectiveCost = iff(x_AmortizationClass == 'Principal', real(0), Cost), + InvoiceId = '', + InvoiceIssuerName = 'Microsoft', + ListCost = real(null), + ListUnitPrice = real(null), + PricingCategory, + PricingCurrency = iff(BillingAccountId == BillingProfileId, BillingCurrency, ''), + PricingQuantity, + PricingUnit, + ProviderName = 'Microsoft', + PublisherName = iff(PublisherType == 'Marketplace', PublisherName, 'Microsoft'), + Region = '', + RegionId, + RegionName, + ResourceId = tostring(tmp_ResourceInfo.ResourceId), + ResourceName = tostring(tmp_ResourceInfo.ResourceName), + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SkuId = '', + SkuMeter = MeterName, + SkuPriceDetails = '', + SkuPriceId = '', + SubAccountId = strcat('/subscriptions/', SubscriptionId), + SubAccountName = iff(isempty(SubscriptionId), '', SubscriptionName), + SubAccountType = iff(isempty(SubscriptionId), '', 'Subscription'), + Tags = iff(Tags startswith '{', Tags, strcat('{', Tags, '}')), + UsageAmount = real(null), + UsageQuantity = real(null), + UsageUnit = '', + x_AccountId = '', + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerId, + x_AmortizationClass = '', + x_BilledCostInUsd = real(null), + x_BilledUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), UnitPrice), + x_BillingAccountId = BillingAccountId, + x_BillingAccountName = BillingAccountName, + x_BillingExchangeRate = real(null), + x_BillingExchangeRateDate = datetime(null), + x_BillingItemCode = '', + x_BillingItemName = '', + x_BillingProfileId = BillingProfileId, + x_BillingProfileName = BillingProfileName, + x_ChargeId = '', + x_CommodityCode = '', + x_CommodityName = '', + x_ComponentType = '', + x_ComponentName = '', + x_ContractedCostInUsd = real(null), + x_Cost = real(null), + x_CostAllocationRuleName = '', + x_CostCategories = '', + x_CostCenter = CostCenter, + x_CostType = '', + x_Credits = '', + x_CurrencyConversionRate = real(null), + x_CustomerId = '', + x_CustomerName = '', + x_Discount = '', + x_EffectiveCostInUsd = real(null), + x_EffectiveUnitPrice = iff(ChargeType == 'Usage' and isnotempty(ReservationId) and ChargeType != 'UnusedSavingsPlan', real(0), EffectivePrice), + x_ExportTime = datetime(null), + x_InstanceID = '', + x_InvoiceId = '', + x_InvoiceIssuerId = '', + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = InvoiceSection, + x_ListCostInUsd = real(null), + x_Location = '', + x_OnDemandCost = real(null), + x_OnDemandCostInUsd = real(null), + x_OnDemandUnitPrice = real(null), + x_Operation = '', + x_OwnerAccountID = '', + x_PartnerCreditApplied = '', + x_PartnerCreditRate = '', + x_PricingBlockSize, + x_PricingCurrency = '', + x_PricingSubcategory = case( + // TODO: Add x_SkuTier when supported by C360 -- PricingCategory == 'Standard' and isnotempty(x_SkuTier), 'Tiered', + PricingCategory == 'Standard', 'Standard', + PricingCategory == 'Committed', strcat('Committed ', CommitmentDiscountCategory), + PricingCategory == 'Dynamic', 'Spot', + '' + ), + x_PricingUnitDescription, + x_Project = '', + x_PublisherCategory = iff(PublisherType == 'Marketplace', 'Vendor', 'Cloud Provider'), + x_PublisherId = '', + x_ResellerId = '', + x_ResellerName = '', + x_ResourceGroupName = ResourceGroup, + x_ResourceType, + x_ServiceCode = '', + x_ServiceId = '', + x_ServiceModel, + x_ServicePeriodEnd = datetime(null), + x_ServicePeriodStart = datetime(null), + x_SkuDescription = Product, + x_SkuDetails = iff(AdditionalInfo startswith '{', AdditionalInfo, strcat('{', AdditionalInfo, '}')), + x_SkuIsCreditEligible = IsAzureCreditEligible, + x_SkuMeterCategory = MeterCategory, + x_SkuMeterId = MeterId, + x_SkuMeterName = MeterName, + x_SkuMeterSubcategory = MeterSubCategory, + x_SkuOfferId = OfferId, + x_SkuOrderId = ProductOrderId, + x_SkuOrderName = ProductOrderName, + x_SkuPartNumber = PartNumber, + x_SkuPlanName = '', + x_SkuRegion = MeterRegion, + x_SkuServiceFamily = ServiceFamily, + x_SkuTerm = toint(Term), + x_SkuTier = '', + x_SourceName = 'C360', + x_SourceProvider = 'Microsoft', + x_SourceType = 'ActualCost', + x_SourceVersion = 'C360-2025-04', + x_SubproductName = '', + x_UsageType = '' +} + +// Update policy for AmortizedCosts_raw -> Costs_raw table +.alter table Costs_raw policy update +``` +[{ + "IsEnabled": true, + "Source": "AmortizedCosts_raw", + "Query": "AmortizedCosts_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| CommitmentDiscountUsage |======================================================================================== +// Supported versions: +// - MS EA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-ea +// - MS MCA reservation details: 2023-03-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-details-mca +//====================================================================================================================== + +// CommitmentDiscountUsage_transform_v1_3 function +.create-or-alter function +with (docstring='All commitment discount usage transformed to FOCUS 1.3. This includes reservationdeatils_raw.', folder='Commitment discounts') +CommitmentDiscountUsage_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + CommitmentDiscountUsage_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Handle resource columns + | extend ResourceId = tolower(InstanceId) + | extend tmp_ResourceDetails = parse_resourceid(ResourceId) + | extend ResourceName = tostring(tmp_ResourceDetails.ResourceName) + | extend SubAccountId = tostring(tmp_ResourceDetails.SubAccountId) + | extend x_ResourceGroupName = tostring(tmp_ResourceDetails.x_ResourceGroupName) + | extend x_ResourceType = tostring(tmp_ResourceDetails.x_ResourceType) + | lookup kind=leftouter (ResourceTypes | distinct x_ResourceType, ResourceType = SingularDisplayName) on x_ResourceType + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceName, ServiceCategory, ServiceSubcategory, x_ServiceModel) on x_ResourceType + // + // Sort columns and apply final transforms + | project + ChargePeriodEnd = UsageDate + 1d, + ChargePeriodStart = UsageDate, + CommitmentDiscountCategory = 'Usage', + CommitmentDiscountId = tolower(strcat('/providers/microsoft.capacity/reservationorders/', ReservationOrderId, '/reservations/', ReservationId)), + CommitmentDiscountQuantity = UsedHours * InstanceFlexibilityRatio, + CommitmentDiscountType = 'Reservation', + CommitmentDiscountUnit = case( + InstanceFlexibilityRatio == 1, 'Hours', + InstanceFlexibilityRatio != 1, 'Normalized Hours', + '' + ), + ConsumedQuantity = UsedHours, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount = TotalReservedQuantity, + x_CommitmentDiscountCommittedAmount = ReservedHours, + // TODO: Is this needed? -- x_CommitmentDiscountKind = Kind, + x_CommitmentDiscountNormalizedGroup = iff(InstanceFlexibilityGroup == 'NA', '', InstanceFlexibilityGroup), + x_CommitmentDiscountNormalizedRatio = InstanceFlexibilityRatio, + x_IngestionTime = ingestion_time(), + x_ResourceGroupName, + x_ResourceType, + // x_RowId = hash_sha256(strcat( + // // DO NOT CHANGE COLUMNS OR COLUMN ORDER + // CommitmentDiscountId, + // ResourceId, + // ChargePeriodStart + // )), + x_ServiceModel, + x_SkuOrderId = ReservationOrderId, + x_SkuSize = iff(SkuName == 'NA', '', SkuName), + x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)), + x_SourceProvider = coalesce(x_SourceProvider, ProviderName), + x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationDetails', '')), + x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2024-03-01', '')) +} + +// CommitmentDiscountUsage_final_v1_3 table +.create-merge table CommitmentDiscountUsage_final_v1_3 ( + ChargePeriodEnd: datetime, // Hubs add-on + ChargePeriodStart: datetime, // MS 2023-03-01 + CommitmentDiscountCategory: string, // Hubs add-on + CommitmentDiscountId: string, // MS 2023-03-01 + CommitmentDiscountQuantity: real, // MS 2023-03-01 + CommitmentDiscountType: string, // Hubs add-on + CommitmentDiscountUnit: string, // Hubs add-on + ConsumedQuantity: real, // MS 2023-03-01 + ProviderName: string, // Hubs add-on + ResourceId: string, // MS 2023-03-01 + ResourceName: string, // Hubs add-on + ResourceType: string, // Hubs add-on + ServiceCategory: string, // Hubs add-on + ServiceName: string, // Hubs add-on + ServiceSubcategory: string, // Hubs add-on + SubAccountId: string, // Hubs add-on + x_CommitmentDiscountCommittedCount: real, // MS 2023-03-01 + x_CommitmentDiscountCommittedAmount: real, // MS 2023-03-01 + x_CommitmentDiscountNormalizedGroup: string, // MS 2023-03-01 + x_CommitmentDiscountNormalizedRatio: real, // MS 2023-03-01 + x_IngestionTime: datetime, // Hubs add-on + x_ResourceGroupName: string, // Hubs add-on + x_ResourceType: string, // Hubs add-on + x_ServiceModel: string, // Hubs add-on + x_SkuOrderId: string, // MS 2023-03-01 + x_SkuSize: string, // MS 2023-03-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for CommitmentDiscountUsage_raw -> CommitmentDiscountUsage_final_v1_3 table +.alter table CommitmentDiscountUsage_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "CommitmentDiscountUsage_raw", + "Query": "CommitmentDiscountUsage_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Recommendations |================================================================================================ +// Supported datasets/versions: +// - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea +// - MS CM MCA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-mca +//====================================================================================================================== + +// Recommendations_transform_v1_3 function +.create-or-alter function +with (docstring='All recommendations transformed to FOCUS 1.3.', folder='Recommendations') +Recommendations_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Recommendations_raw + | extend x_IngestionTime = ingestion_time() + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationRecommendations', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Convert JSON cost columns to real + | extend CostWithNoReservedInstances = case(isnotempty(CostWithNoReservedInstances), CostWithNoReservedInstances, isnotempty(CostWithNoReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, CostWithNoReservedInstancesJson)), CostWithNoReservedInstances) + | extend NetSavings = case(isnotempty(NetSavings), NetSavings, isnotempty(NetSavingsJson), toreal(extract(@'"value":([0-9\.]+)', 1, NetSavingsJson)), NetSavings) + | extend TotalCostWithReservedInstances = case(isnotempty(TotalCostWithReservedInstances), TotalCostWithReservedInstances, isnotempty(TotalCostWithReservedInstancesJson), toreal(extract(@'"value":([0-9\.]+)', 1, TotalCostWithReservedInstancesJson)), TotalCostWithReservedInstances) + // + // Parse x_RecommendationDetails from JSON string (ARG queries serialize to string for Parquet compatibility) + | extend x_RecommendationDetails = iff(gettype(x_RecommendationDetails) == 'string', parse_json(tostring(x_RecommendationDetails)), x_RecommendationDetails) + // + // Normalize x_RecommendationDetails keys to x_PascalCase (Advisor extendedProperties use camelCase) + // Guard: inject a placeholder key so mv-apply doesn't drop rows with null/empty bags + | extend x_RecommendationDetails = bag_merge(coalesce(x_RecommendationDetails, dynamic({})), bag_pack('__placeholder', '')) + | mv-apply k = bag_keys(x_RecommendationDetails) on ( + where isnotempty(tostring(k)) and tostring(k) != '__placeholder' + | extend newKey = iff(tostring(k) startswith 'x_', tostring(k), strcat('x_', toupper(substring(tostring(k), 0, 1)), substring(tostring(k), 1))) + | summarize x_RecommendationDetails = make_bag(bag_pack(newKey, x_RecommendationDetails[tostring(k)])) + ) + // + // Build recommendation details + | lookup kind=leftouter (Regions | summarize RegionName = make_set(RegionName)[0] by Location = RegionId) on Location + | extend x_RecommendationDetails = case( + // Use incoming x_RecommendationDetails first + isnotempty(x_RecommendationDetails), x_RecommendationDetails, + // Create one for reservation recommendations if needed + x_SourceType == 'ReservationRecommendations', bag_pack( + 'CommitmentDiscountNormalizedGroup', InstanceFlexibilityGroup, + 'CommitmentDiscountNormalizedRatio', InstanceFlexibilityRatio, + 'CommitmentDiscountNormalizedSize', NormalizedSize, + 'CommitmentDiscountResourceType', ResourceType, + 'CommitmentDiscountScope', Scope, + 'LookbackPeriodDuration', case( + LookBackPeriod matches regex @'^Last([0-9]+)Days$', replace_regex(LookBackPeriod, @'^Last([0-9]+)Days$', @'P\1D'), + LookBackPeriod matches regex @'^[0-9]+$', strcat('P', LookBackPeriod, 'D'), + '' + ), + 'LookbackPeriodStart', FirstUsageDate, + 'RecommendedQuantity', RecommendedQuantity, + 'RecommendedQuantityNormalized', RecommendedQuantityNormalized, + 'RegionId', Location, + 'RegionName', RegionName, + 'SkuMeterId', MeterId, + 'SkuPriceDetails', SkuProperties, + 'SkuSize', coalesce(SKU, SkuName), + 'SkuTerm', isoMonths(Term) + ), + dynamic({}) + ) + // + // Prefer specified date, then fall back to generating a date based on reservation recommendation lookback period, then validate to ensure it's not in the future + | extend x_RecommendationDate = coalesce(x_RecommendationDate, FirstUsageDate + (toint(extract(@'^P([0-9]+)D$', 1, tostring(x_RecommendationDetails.LookbackPeriodDuration))) * 1d)) + | extend x_RecommendationDate = iff(x_RecommendationDate > now(), startofday(now()), x_RecommendationDate) + // + // Derive x_ResourceType from ResourceId + | extend tmp_ResourceType = tostring(parse_resourceid(ResourceId).x_ResourceType) + | extend x_RecommendationDetails = iff(isnotempty(tmp_ResourceType), bag_merge(bag_pack('x_ResourceType', tmp_ResourceType), x_RecommendationDetails), x_RecommendationDetails) + // + // Set ResourceType display name from x_ResourceType code + | extend ResourceType = coalesce(ResourceType, tostring(resource_type(tmp_ResourceType).SingularDisplayName), tmp_ResourceType) + // + | project + ProviderName, + ResourceId = tolower(ResourceId), // Force lowercase for consistent grouping/filtering + ResourceName = tolower(iff(tmp_ResourceType =~ 'microsoft.resources/subscriptions', coalesce(SubAccountName, ResourceName), ResourceName)), // Use subscription name for subscription-level recommendations + ResourceType, + SubAccountId = coalesce(SubAccountId, iff(isnotempty(SubscriptionId), strcat('/subscriptions/', SubscriptionId), '')), + SubAccountName, + x_EffectiveCostAfter = coalesce(x_EffectiveCostAfter, TotalCostWithReservedInstances), + x_EffectiveCostBefore = coalesce(x_EffectiveCostBefore, CostWithNoReservedInstances), + x_EffectiveCostSavings = coalesce(x_EffectiveCostSavings, NetSavings, toreal(x_RecommendationDetails.x_SavingsAmount), toreal(x_RecommendationDetails.x_AnnualSavingsAmount) / 12), + x_IngestionTime, + x_RecommendationCategory, // TODO: Set for reservation recommendations + x_RecommendationDate, + x_RecommendationDescription = coalesce(x_RecommendationDescription, tostring(x_RecommendationDetails.x_RecommendationSolution), tostring(x_RecommendationDetails.x_RecommendationSubCategory)), + x_RecommendationDetails, + x_RecommendationId = tolower(x_RecommendationId), // TODO: Set for reservation recommendations; force lowercase for consistent grouping/filtering + x_ResourceGroupName = tolower(x_ResourceGroupName), // Force lowercase for consistent grouping/filtering + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + +// Recommendations_final_v1_3 table +.create-merge table Recommendations_final_v1_3 ( + ProviderName: string, + ResourceId: string, + ResourceName: string, + ResourceType: string, + SubAccountId: string, + SubAccountName: string, + x_EffectiveCostAfter: real, + x_EffectiveCostBefore: real, + x_EffectiveCostSavings: real, + x_IngestionTime: datetime, + x_RecommendationCategory: string, + x_RecommendationDate: datetime, + x_RecommendationDescription: string, + x_RecommendationDetails: dynamic, + x_RecommendationId: string, + x_ResourceGroupName: string, + x_SourceName: string, + x_SourceProvider: string, + x_SourceType: string, + x_SourceVersion: string +) + +// Update policy for Recommendations_raw -> Recommendations_final_v1_3 table +.alter table Recommendations_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Recommendations_raw", + "Query": "Recommendations_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + +//===| Transactions |=================================================================================================== +// Supported versions: +// - MS CM EA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-ea +// - MS CM MCA reservation transactions: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-transactions-mca +//====================================================================================================================== + +// Transactions_transform_v1_3 function +.create-or-alter function +with (docstring='All transactions transformed to FOCUS 1.3.', folder='Transactions') +Transactions_transform_v1_3() +{ + // NOTE: All open issues and questions are tracked @ https://github.com/microsoft/finops-toolkit/issues/1111 + let isoMonths = (duration: string) { + let number = toint(replace_regex(duration, @'[PMY]', '')); + toint(case( + duration == '', int(null), + duration endswith "Y", number * 12, + duration endswith "M", number, + -1 + )) + }; + Transactions_raw + // + // Set ProviderName + | extend ProviderName = 'Microsoft' + // + // Set source columns + | extend x_SourceName = coalesce(x_SourceName, iff(ProviderName == 'Microsoft', 'Cost Management', ProviderName)) + | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) + | extend x_SourceType = coalesce(x_SourceType, iff(ProviderName == 'Microsoft', 'ReservationTransactions', '')) + | extend x_SourceVersion = coalesce(x_SourceVersion, iff(ProviderName == 'Microsoft', '2023-05-01', '')) + // + // Handle BillingPeriodStart/End + | extend BillingMonth = tostring(BillingMonth) + | extend BillingPeriodStart = iff(isempty(BillingMonth), datetime(null), todatetime(strcat(substring(BillingMonth, 0, 4), "-", substring(BillingMonth, 4, 2), "-", substring(BillingMonth, 6, 2)))) + | extend BillingPeriodEnd = iff(isempty(BillingMonth), datetime(null), startofmonth(endofmonth(BillingPeriodStart) + 1d)) + // + // Sort columns and apply final transforms + | project + BilledCost = Amount, + BillingAccountId = case( + BillingProfileId startswith '/', BillingProfileId, + isnotempty(CurrentEnrollmentId), strcat('/providers/Microsoft.Billing/billingAccounts/', CurrentEnrollmentId), + isnotempty(BillingProfileId), strcat('/providers/Microsoft.Billing/billingProfiles/', BillingProfileId), + '' + ), + BillingAccountName = coalesce(BillingProfileName, CurrentEnrollmentId), + BillingCurrency = Currency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory = case( + EventType in ('Cancel', 'Purchase', 'Refund'), 'Purchase', + 'Adjustment' + ), + ChargeClass = case( + EventType == 'Cancel', 'Cancel', // FOCUS does not handle this scenario + EventType == 'Refund', 'Correction', + '' + ), + ChargeDescription = Description, + ChargeFrequency = case( + BillingFrequency == 'OneTime', 'One-Time', + BillingFrequency == 'Recurring', 'Recurring', + BillingFrequency + ), + ChargePeriodStart = EventDate, + InvoiceId, + PricingQuantity = Quantity, + PricingUnit = 'Reservations', + ProviderName, + RegionId = Region, + RegionName = Region, + SubAccountId = iff(isempty(PurchasingSubscriptionGuid), '', strcat('/subscriptions/', PurchasingSubscriptionGuid)), + SubAccountName = iff(isempty(PurchasingSubscriptionGuid), '', PurchasingSubscriptionName), + x_AccountName = AccountName, + x_AccountOwnerId = AccountOwnerEmail, + x_CostCenter = CostCenter, + x_InvoiceNumber = Invoice, + x_InvoiceSectionId = InvoiceSectionId, + x_InvoiceSectionName = coalesce(InvoiceSectionName, DepartmentName), + x_IngestionTime = ingestion_time(), + x_MonetaryCommitment = MonetaryCommitment, + x_Overage = Overage, + x_PurchasingBillingAccountId = PurchasingEnrollment, + x_SkuOrderId = ReservationOrderId, + x_SkuOrderName = ReservationOrderName, + x_SkuSize = ArmSkuName, + x_SkuTerm = isoMonths(Term), + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId = PurchasingSubscriptionGuid, + x_TransactionType = EventType +} + +// Transactions_final_v1_3 table +.create-merge table Transactions_final_v1_3 ( + BilledCost: real, // MS CM EA+MCA 2023-05-01 + BillingAccountId: string, // MS CM EA+MCA 2023-05-01 + BillingAccountName: string, // MS CM EA+MCA 2023-05-01 + BillingCurrency: string, // MS CM EA+MCA 2023-05-01 + BillingPeriodEnd: datetime, // MS CM EA+MCA 2023-05-01 + BillingPeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + ChargeCategory: string, // Hubs add-on + ChargeClass: string, // Hubs add-on + ChargeDescription: string, // MS CM EA+MCA 2023-05-01 + ChargeFrequency: string, // MS CM EA+MCA 2023-05-01 + ChargePeriodStart: datetime, // MS CM EA+MCA 2023-05-01 + InvoiceId: string, // MS CM MCA 2023-05-01 + PricingQuantity: real, // MS CM EA+MCA 2023-05-01 + PricingUnit: string, // Hubs add-on + ProviderName: string, // Hubs add-on + RegionId: string, // MS CM EA+MCA 2023-05-01 + RegionName: string, // MS CM EA+MCA 2023-05-01 + SubAccountId: string, // MS CM EA+MCA 2023-05-01 + SubAccountName: string, // MS CM EA+MCA 2023-05-01 + x_AccountName: string, // MS CM EA 2023-05-01 + x_AccountOwnerId: string, // MS CM EA 2023-05-01 + x_CostCenter: string, // MS CM EA 2023-05-01 + x_InvoiceNumber: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionId: string, // MS CM MCA 2023-05-01 + x_InvoiceSectionName: string, // MS CM MCA 2023-05-01 + x_IngestionTime: datetime, // Hubs add-on + x_MonetaryCommitment: real, // MS CM EA 2023-05-01 + x_Overage: real, // MS CM EA 2023-05-01 + x_PurchasingBillingAccountId: string, // MS CM EA 2023-05-01 + x_SkuOrderId: string, // MS CM EA+MCA 2023-05-01 + x_SkuOrderName: string, // MS CM EA+MCA 2023-05-01 + x_SkuSize: string, // MS CM EA+MCA 2023-05-01 + x_SkuTerm: int, // MS CM EA+MCA 2023-05-01 + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string, // Hubs add-on + x_SubscriptionId: string, // MS CM EA+MCA 2023-05-01 + x_TransactionType: string // MS CM EA+MCA 2023-05-01 +) + +// Update policy for Transactions_raw -> Transactions_final_v1_3 table +.alter table Transactions_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "Transactions_raw", + "Query": "Transactions_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` From f70c43d88579087110b353769e8b63c18d98b837 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:21:44 -0700 Subject: [PATCH 2/3] Add FOCUS 1.3 hub functions (#2127) Adds HubSetup_v1_3.kql with Costs_v1_3, Prices_v1_3, CommitmentDiscountUsage_v1_3, Recommendations_v1_3, and Transactions_v1_3 that union the new Costs_final_v1_3 with the existing Costs_final_v1_2 and Costs_final_v1_0 tables. For v1_2 data unioned into the v1_3 view, the 8 new FOCUS 1.3 columns default to null/empty and ServiceProviderName/HostProviderName are populated from the deprecated ProviderName/PublisherName for back compat. The same defaults apply to the v1_0 union arm on top of the existing v1_0 -> v1_2 conversion. HubSetup_Latest.kql aliases now point to *_v1_3 functions so the unversioned Costs(), Prices(), etc. return the latest schema. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/templates/finops-hub/.build.config | 1 + .../Microsoft.FinOpsHubs/Analytics/app.bicep | 1 + .../Analytics/scripts/HubSetup_Latest.kql | 10 +- .../Analytics/scripts/HubSetup_v1_3.kql | 618 ++++++++++++++++++ 4 files changed, 625 insertions(+), 5 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 0963fb494..4eff22919 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -44,6 +44,7 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_OpenData.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_0.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", + "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 42cd2738b..ebf106beb 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -414,6 +414,7 @@ module hub_VersionedScripts '../../fx/hub-database.bicep' = if (useAzure) { scripts: { v1_0: loadTextContent('scripts/HubSetup_v1_0.kql') v1_2: loadTextContent('scripts/HubSetup_v1_2.kql') + v1_3: loadTextContent('scripts/HubSetup_v1_3.kql') } continueOnErrors: continueOnErrors forceUpdateTag: forceUpdateTag diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 8de86a1aa..1895b39db 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -13,7 +13,7 @@ with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') CommitmentDiscountUsage() { - CommitmentDiscountUsage_v1_2() + CommitmentDiscountUsage_v1_3() } @@ -21,7 +21,7 @@ CommitmentDiscountUsage() with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() { - Costs_v1_2() + Costs_v1_3() } @@ -29,7 +29,7 @@ Costs() with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') Prices() { - Prices_v1_2() + Prices_v1_3() } @@ -37,7 +37,7 @@ Prices() with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') Recommendations() { - Recommendations_v1_2() + Recommendations_v1_3() } @@ -45,5 +45,5 @@ Recommendations() with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') Transactions() { - Transactions_v1_2() + Transactions_v1_3() } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql new file mode 100644 index 000000000..62995ffa5 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql @@ -0,0 +1,618 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Hub database / FOCUS 1.3 functions +// Used for reporting with backward compatibility. +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + + +// CommitmentDiscountUsage_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records aligned to FOCUS 1.3.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage_v1_3() +{ + database('Ingestion').CommitmentDiscountUsage_final_v1_3 + | union ( + database('Ingestion').CommitmentDiscountUsage_final_v1_0 + // Convert decimal to real + | extend + ConsumedQuantity = toreal(ConsumedQuantity), + x_CommitmentDiscountCommittedCount = toreal(x_CommitmentDiscountCommittedCount), + x_CommitmentDiscountCommittedAmount = toreal(x_CommitmentDiscountCommittedAmount), + x_CommitmentDiscountNormalizedRatio = toreal(x_CommitmentDiscountNormalizedRatio) + // Add new columns + | lookup kind=leftouter (Services | distinct x_ResourceType, ServiceSubcategory) on x_ResourceType + | extend CommitmentDiscountQuantity = ConsumedQuantity * x_CommitmentDiscountNormalizedRatio + | extend CommitmentDiscountUnit = case( + x_CommitmentDiscountNormalizedRatio == 1, 'Hours', + x_CommitmentDiscountNormalizedRatio > 1, 'Normalized Hours', + '' + ) + ) + | project + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountQuantity, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ProviderName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceSubcategory, + SubAccountId, + x_CommitmentDiscountCommittedCount, + x_CommitmentDiscountCommittedAmount, + x_CommitmentDiscountNormalizedGroup, + x_CommitmentDiscountNormalizedRatio, + x_IngestionTime, + x_ResourceGroupName, + x_ResourceType, + x_ServiceModel, + x_SkuOrderId, + x_SkuSize, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Costs_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') +Costs_v1_3() +{ + database('Ingestion').Costs_final_v1_3 + | union ( + database('Ingestion').Costs_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + ConsumedQuantity = toreal(ConsumedQuantity), + ContractedCost = toreal(ContractedCost), + ContractedUnitPrice = toreal(ContractedUnitPrice), + EffectiveCost = toreal(EffectiveCost), + ListCost = toreal(ListCost), + ListUnitPrice = toreal(ListUnitPrice), + PricingQuantity = toreal(PricingQuantity), + x_BilledCostInUsd = toreal(x_BilledCostInUsd), + x_BilledUnitPrice = toreal(x_BilledUnitPrice), + x_BillingExchangeRate = toreal(x_BillingExchangeRate), + x_ContractedCostInUsd = toreal(x_ContractedCostInUsd), + x_CurrencyConversionRate = toreal(x_CurrencyConversionRate), + x_EffectiveCostInUsd = toreal(x_EffectiveCostInUsd), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_ListCostInUsd = toreal(x_ListCostInUsd), + x_PricingBlockSize = toreal(x_PricingBlockSize) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId, + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + // Add new columns + | lookup kind=leftouter (Services | where isnotempty(x_ResourceType) | summarize take_any(ServiceSubcategory), take_any(x_ServiceModel) by x_ResourceType) on x_ResourceType + | extend CapacityReservationId = tostring(x_SkuDetails.VMCapacityReservationId) + | extend CapacityReservationStatus = case( + isempty(CapacityReservationId), '', + tolower(x_ResourceType) == 'microsoft.compute/capacityreservationgroups/capacityreservations', 'Unused', + 'Used' + ) + | extend x_CommitmentDiscountNormalizedRatio = case( + // Not applicable + isempty(CommitmentDiscountStatus), real(null), + // Parse from SKU details if not specified explicitly + toreal(coalesce(x_SkuDetails.RINormalizationRatio, dynamic(1))) + ) + | extend CommitmentDiscountQuantity = case( + isempty(CommitmentDiscountStatus), real(null), + CommitmentDiscountCategory == 'Spend', EffectiveCost / coalesce(x_BillingExchangeRate, real(1)), + CommitmentDiscountCategory == 'Usage' and isnotempty(x_CommitmentDiscountNormalizedRatio), PricingQuantity / coalesce(x_PricingBlockSize, real(1)) * x_CommitmentDiscountNormalizedRatio, + real(null) + ) + | extend CommitmentDiscountUnit = case( + isempty(CommitmentDiscountQuantity), '', + CommitmentDiscountCategory == 'Spend', PricingCurrency, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio == real(1), ConsumedUnit, + CommitmentDiscountCategory == 'Usage' and x_CommitmentDiscountNormalizedRatio > real(1), strcat('Normalized ', ConsumedUnit), + '' + ) + | extend x_AmortizationClass = case( + ChargeCategory == 'Purchase' and (tolower(ResourceId) contains '/microsoft.capacity/reservationorders/' or tolower(ResourceId) contains '/microsoft.billingbenefits/savingsplanorders/'), 'Principal', + ChargeCategory == 'Usage' and isnotempty(CommitmentDiscountId) and isnotempty(CommitmentDiscountStatus), 'Amortized Charge', + '' + ) + // Hubs add-ons + | extend x_CommitmentDiscountUtilizationPotential = case( + ChargeCategory == 'Purchase', real(0), + ProviderName == 'Microsoft' and isnotempty(CommitmentDiscountCategory), EffectiveCost, + CommitmentDiscountCategory == 'Usage', ConsumedQuantity, + CommitmentDiscountCategory == 'Spend', EffectiveCost, + real(0) + ) + | extend x_CommitmentDiscountUtilizationAmount = iff(CommitmentDiscountStatus == 'Used', x_CommitmentDiscountUtilizationPotential, real(0)) + | extend x_SkuCoreCount = toint(coalesce(x_SkuDetails.VCPUs, x_SkuDetails.VCores, x_SkuDetails.vCores)) + | extend x_SkuInstanceType = tostring(coalesce(x_SkuDetails.ServiceType, x_SkuDetails.ServerSku)) + | extend x_SkuOperatingSystem = case( + x_SkuDetails.ImageType == 'Canonical', 'Linux', + x_SkuDetails.ImageType == 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory endswith ' Series Windows', 'Windows Server', + x_SkuDetails.ImageType + ) + | extend x_ConsumedCoreHours = iff(ConsumedUnit == 'Hours' and isnotempty(x_SkuCoreCount), x_SkuCoreCount * ConsumedQuantity, real(null)) + | extend tmp_SqlAhb = tolower(x_SkuDetails.AHB) + | extend x_SkuLicenseType = case( + x_SkuDetails.ImageType contains 'Windows Server BYOL', 'Windows Server', + x_SkuMeterSubcategory == 'SQL Server Azure Hybrid Benefit', 'SQL Server', + '' + ) + | extend x_SkuLicenseStatus = case( + isnotempty(x_SkuLicenseType) or tmp_SqlAhb == 'true' or (x_SkuMeterSubcategory contains 'Azure Hybrid Benefit'), 'Enabled', + (x_SkuMeterSubcategory contains 'Windows') or tmp_SqlAhb == 'false', 'Not enabled', + '' + ) + | extend x_SkuLicenseQuantity = case( + isempty(x_SkuCoreCount), int(null), + x_SkuCoreCount <= 8, int(8), + x_SkuCoreCount > 8, x_SkuCoreCount, + int(null) + ) + | extend x_SkuLicenseUnit = iff(isnotempty(x_SkuLicenseQuantity), 'Cores', '') + | extend x_CommitmentDiscountSavings = iff(ContractedCost < EffectiveCost, real(0), ContractedCost - EffectiveCost) + | extend x_NegotiatedDiscountSavings = iff(ListCost < ContractedCost, real(0), ListCost - ContractedCost) + | extend x_TotalSavings = iff(ListCost < EffectiveCost, real(0), ListCost - EffectiveCost) + | extend x_CommitmentDiscountPercent = iff(ContractedUnitPrice == 0, real(0), (ContractedUnitPrice - x_EffectiveUnitPrice) / ContractedUnitPrice) + | extend x_NegotiatedDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice) + | extend x_TotalDiscountPercent = iff(ListUnitPrice == 0, real(0), (ListUnitPrice - x_EffectiveUnitPrice) / ListUnitPrice) + // SkuPriceDetails conversion -- Must be after hubs add-ons + | extend SkuPriceDetails = parse_json(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(replace_regex(tostring(x_SkuDetails) + // Prefix all keys with x_ first to avoid double-prefixing + , @'([\{,])"', @'\1"x_') + // CoreCount for number of CPUs/vCPUs/cores/vCores + , @'"x_(VCPUs|VCores|vCores)":', @'"CoreCount":') + // TODO: DiskMaxIops for disk I/O operations per second (IOPS) + // TODO: DiskSpace for disk size in GiB + // TODO: DiskType for the kind of disk (e.g., SSD, HDD, NVMe) + // TODO: GpuCount for the number of GPUs + // InstanceType for the resource size/SKU (e.g., ArmSkuName) + , @'"x_(ServerSku|ServiceType)":', @'"InstanceType":') + // TODO: InstanceSeries for the size family/series + // TODO: MemorySize for the RAM in GiB + // TODO: NetworkMaxIops for network I/O operations per second (IOPS) + // TODO: NetworkMaxThroughput for network max throughput for data transfer in Mbps + // OperatingSystem for the OS name + , @'("x_ImageType":"Canonical")', @'\1,"OperatingSystem":"Linux"') + , @'("x_ImageType":"Windows Server( BYOL)?")', @'\1,"OperatingSystem":"Windows Server"') + , @'("x_ImageType":("[^"]+"))', @'\1,"OperatingSystem":\2') + // TODO: Redundancy for the level of redundancy (e.g., Local, Zonal, Global) + // TODO: StorageClass for the tier of storage (e.g., Hot, Archive, Nearline) + ) + | extend SkuPriceDetails = iff(isempty(SkuPriceDetails.OperatingSystem) and isnotempty(x_SkuOperatingSystem), + parse_json(replace_string(tostring(SkuPriceDetails), '}', strcat(@',"OperatingSystem":"', x_SkuOperatingSystem, '"}'))), + SkuPriceDetails) + // FOCUS 1.3 column defaults (data was ingested under v1_0 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | union ( + database('Ingestion').Costs_final_v1_2 + // FOCUS 1.3 column defaults (data was ingested under v1_2 schema) + | extend AllocatedMethodId = '' + | extend AllocatedMethodDetails = dynamic(null) + | extend AllocatedResourceId = '' + | extend AllocatedResourceName = '' + | extend AllocatedTags = dynamic(null) + | extend ContractApplied = dynamic(null) + | extend ServiceProviderName = ProviderName + | extend HostProviderName = PublisherName + ) + | extend SkuPriceDetails = iff(isnotempty(SkuPriceDetails), SkuPriceDetails, parse_json(replace_regex(tostring(x_SkuDetails), @'([\{,])"', @'\1"x_'))) + | project + AllocatedMethodDetails, + AllocatedMethodId, + AllocatedResourceId, + AllocatedResourceName, + AllocatedTags, + AvailabilityZone, + BilledCost, + BillingAccountId, + BillingAccountName, + BillingAccountType, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + CapacityReservationId, + CapacityReservationStatus, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodEnd, + ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountQuantity, + CommitmentDiscountStatus, + CommitmentDiscountType, + CommitmentDiscountUnit, + ConsumedQuantity, + ConsumedUnit, + ContractApplied, + ContractedCost, + ContractedUnitPrice, + EffectiveCost, + HostProviderName, + InvoiceId, + InvoiceIssuerName, + ListCost, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingQuantity, + PricingUnit, + ProviderName, + PublisherName, + RegionId, + RegionName, + ResourceId, + ResourceName, + ResourceType, + ServiceCategory, + ServiceName, + ServiceProviderName, + ServiceSubcategory, + SkuId, + SkuMeter, + SkuPriceDetails, + SkuPriceId, + SubAccountId, + SubAccountName, + SubAccountType, + Tags, + x_AccountId, + x_AccountName, + x_AccountOwnerId, + x_AmortizationClass, + x_BilledCostInUsd, + x_BilledUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingAccountName, + x_BillingExchangeRate, + x_BillingExchangeRateDate, + x_BillingItemCode, + x_BillingItemName, + x_BillingProfileId, + x_BillingProfileName, + x_ChargeId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountPercent, + x_CommitmentDiscountSavings, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_CommitmentDiscountUtilizationAmount, + x_CommitmentDiscountUtilizationPotential, + x_CommodityCode, + x_CommodityName, + x_ComponentName, + x_ComponentType, + x_ConsumedCoreHours, + x_ContractedCostInUsd, + x_CostAllocationRuleName, + x_CostCategories, + x_CostCenter, + x_CostType, + x_Credits, + x_CurrencyConversionRate, + x_CustomerId, + x_CustomerName, + x_Discount, + x_EffectiveCostInUsd, + x_EffectiveUnitPrice, + x_ExportTime, + x_IngestionTime, + x_InstanceID, + x_InvoiceIssuerId, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_ListCostInUsd, + x_Location, + x_NegotiatedDiscountPercent, + x_NegotiatedDiscountSavings, + x_Operation, + x_OwnerAccountID, + x_PartnerCreditApplied, + x_PartnerCreditRate, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_Project, + x_PublisherCategory, + x_PublisherId, + x_ResellerId, + x_ResellerName, + x_ResourceGroupName, + x_ResourceType, + x_ServiceCode, + x_ServiceId, + x_ServiceModel, + x_ServicePeriodEnd, + x_ServicePeriodStart, + x_SkuCoreCount, + x_SkuDescription, + x_SkuDetails, + x_SkuInstanceType, + x_SkuIsCreditEligible, + x_SkuLicenseQuantity, + x_SkuLicenseStatus, + x_SkuLicenseType, + x_SkuLicenseUnit, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuOfferId, + x_SkuOperatingSystem, + x_SkuOrderId, + x_SkuOrderName, + x_SkuPartNumber, + x_SkuPlanName, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuTerm, + x_SkuTier, + x_SourceChanges, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceValues, + x_SourceVersion, + x_SubproductName, + x_TotalDiscountPercent, + x_TotalSavings, + x_UsageType +} + + +// Prices_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all prices aligned to FOCUS 1.3.', folder = 'Prices') +Prices_v1_3() +{ + database('Ingestion').Prices_final_v1_3 + | union ( + database('Ingestion').Prices_final_v1_0 + // Convert decimal to real + | extend + ContractedUnitPrice = toreal(ContractedUnitPrice), + ListUnitPrice = toreal(ListUnitPrice), + x_BaseUnitPrice = toreal(x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = toreal(x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = toreal(x_ContractedUnitPriceDiscountPercent), + x_EffectiveUnitPrice = toreal(x_EffectiveUnitPrice), + x_EffectiveUnitPriceDiscount = toreal(x_EffectiveUnitPriceDiscount), + x_EffectiveUnitPriceDiscountPercent = toreal(x_EffectiveUnitPriceDiscountPercent), + x_PricingBlockSize = toreal(x_PricingBlockSize), + x_SkuIncludedQuantity = toreal(x_SkuIncludedQuantity), + x_SkuTier = toreal(x_SkuTier), + x_TotalUnitPriceDiscount = toreal(x_TotalUnitPriceDiscount), + x_TotalUnitPriceDiscountPercent = toreal(x_TotalUnitPriceDiscountPercent) + // Rename columns + | project-rename + PricingCurrency = x_PricingCurrency, + SkuMeter = x_SkuMeterName + ) + | project + BillingAccountId, + BillingAccountName, + BillingCurrency, + ChargeCategory, + CommitmentDiscountCategory, + CommitmentDiscountType, + CommitmentDiscountUnit, + ContractedUnitPrice, + ListUnitPrice, + PricingCategory, + PricingCurrency, + PricingUnit, + SkuId, + SkuMeter, + SkuPriceId, + SkuPriceIdv2, + x_BaseUnitPrice, + x_BillingAccountAgreement, + x_BillingAccountId, + x_BillingProfileId, + x_CommitmentDiscountNormalizedRatio, + x_CommitmentDiscountSpendEligibility, + x_CommitmentDiscountUsageEligibility, + x_ContractedUnitPriceDiscount, + x_ContractedUnitPriceDiscountPercent, + x_EffectivePeriodEnd, + x_EffectivePeriodStart, + x_EffectiveUnitPrice, + x_EffectiveUnitPriceDiscount, + x_EffectiveUnitPriceDiscountPercent, + x_IngestionTime, + x_PricingBlockSize, + x_PricingSubcategory, + x_PricingUnitDescription, + x_SkuDescription, + x_SkuId, + x_SkuIncludedQuantity, + x_SkuMeterCategory, + x_SkuMeterId, + x_SkuMeterSubcategory, + x_SkuMeterType, + x_SkuPriceType, + x_SkuProductId, + x_SkuRegion, + x_SkuServiceFamily, + x_SkuOfferId, + x_SkuPartNumber, + x_SkuTerm, + x_SkuTier, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_TotalUnitPriceDiscount, + x_TotalUnitPriceDiscountPercent +} + + +// Recommendations_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all recommendations aligned to FOCUS 1.3.', folder = 'Recommendations') +Recommendations_v1_3() +{ + database('Ingestion').Recommendations_final_v1_3 + | union ( + database('Ingestion').Recommendations_final_v1_0 + // Convert decimal to real + | extend + x_EffectiveCostAfter = toreal(x_EffectiveCostAfter), + x_EffectiveCostBefore = toreal(x_EffectiveCostBefore), + x_EffectiveCostSavings = toreal(x_EffectiveCostSavings) + ) + | project + ProviderName, + ResourceId, + ResourceName, + ResourceType, + SubAccountId, + SubAccountName, + x_EffectiveCostAfter, + x_EffectiveCostBefore, + x_EffectiveCostSavings, + x_IngestionTime, + x_RecommendationCategory, + x_RecommendationDate, + x_RecommendationDescription, + x_RecommendationDetails, + x_RecommendationId, + x_ResourceGroupName, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion +} + + +// Transactions_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all transactions aligned to FOCUS 1.3.', folder = 'Transactions') +Transactions_v1_3() +{ + database('Ingestion').Transactions_final_v1_3 + | union ( + database('Ingestion').Transactions_final_v1_0 + // Convert decimal to real + | extend + BilledCost = toreal(BilledCost), + PricingQuantity = toreal(PricingQuantity), + x_MonetaryCommitment = toreal(x_MonetaryCommitment), + x_Overage = toreal(x_Overage) + // Rename columns + | project-rename + InvoiceId = x_InvoiceId + ) + | project + BilledCost, + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeCategory, + ChargeClass, + ChargeDescription, + ChargeFrequency, + ChargePeriodStart, + InvoiceId, + PricingQuantity, + PricingUnit, + ProviderName, + RegionId, + RegionName, + SubAccountId, + SubAccountName, + x_AccountName, + x_AccountOwnerId, + x_CostCenter, + x_InvoiceNumber, + x_InvoiceSectionId, + x_InvoiceSectionName, + x_IngestionTime, + x_MonetaryCommitment, + x_Overage, + x_PurchasingBillingAccountId, + x_SkuOrderId, + x_SkuOrderName, + x_SkuSize, + x_SkuTerm, + x_SourceName, + x_SourceProvider, + x_SourceType, + x_SourceVersion, + x_SubscriptionId, + x_TransactionType +} + + +//====================================================================================================================== +// Latest FOCUS version +//====================================================================================================================== + +.create-or-alter function +with (docstring = 'Gets all commitment discount usage records with the latest supported version of the FOCUS schema.', folder = 'CommitmentDiscountUsage') +CommitmentDiscountUsage() +{ + CommitmentDiscountUsage_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') +Costs() +{ + Costs_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all prices with the latest supported version of the FOCUS schema.', folder = 'Prices') +Prices() +{ + Prices_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all recommendations with the latest supported version of the FOCUS schema.', folder = 'Recommendations') +Recommendations() +{ + Recommendations_v1_3() +} + + +.create-or-alter function +with (docstring = 'Gets all transactions with the latest supported version of the FOCUS schema.', folder = 'Transactions') +Transactions() +{ + Transactions_v1_3() +} From 2bfa61179c5f33bcb39c282c183eb32aca39f4ba Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 6 May 2026 08:35:24 -0700 Subject: [PATCH 3/3] Add ContractCommitment dataset (#2129) Adds the FOCUS 1.3 Contract Commitment supplemental dataset: - ContractCommitment_raw table in IngestionSetup_RawTables.kql with the 14 mandatory FOCUS 1.3 columns plus standard hubs source metadata. - ContractCommitment_transform_v1_3 + ContractCommitment_final_v1_3 in IngestionSetup_v1_3.kql, with update policy mapping raw to final. - ContractCommitment_v1_3 hub function in HubSetup_v1_3.kql that reads the final table directly (no older versions to union). - Unversioned ContractCommitment alias in HubSetup_Latest.kql. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Analytics/scripts/HubSetup_Latest.kql | 8 +++ .../Analytics/scripts/HubSetup_v1_3.kql | 9 +++ .../scripts/IngestionSetup_RawTables.kql | 68 ++++++++++++++++++ .../Analytics/scripts/IngestionSetup_v1_3.kql | 69 +++++++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql index 1895b39db..26012ec5c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql @@ -17,6 +17,14 @@ CommitmentDiscountUsage() } +.create-or-alter function +with (docstring = 'Gets all contract commitments with the latest supported version of the FOCUS schema.', folder = 'ContractCommitment') +ContractCommitment() +{ + ContractCommitment_v1_3() +} + + .create-or-alter function with (docstring = 'Gets all cost and usage records with the latest supported version of the FOCUS schema.', folder = 'Costs') Costs() diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql index 62995ffa5..c1fd9d363 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_3.kql @@ -66,6 +66,15 @@ CommitmentDiscountUsage_v1_3() } +// ContractCommitment_final_v1_3 +.create-or-alter function +with (docstring = 'Gets all contract commitments aligned to FOCUS 1.3.', folder = 'ContractCommitment') +ContractCommitment_v1_3() +{ + database('Ingestion').ContractCommitment_final_v1_3 +} + + // Costs_final_v1_3 .create-or-alter function with (docstring = 'Gets all cost and usage records aligned to FOCUS 1.3.', folder = 'Costs') diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 83a3be40e..ca500e58d 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -335,6 +335,74 @@ .alter table CommitmentDiscountUsage_raw policy streamingingestion disable +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet) +//====================================================================================================================== + +// ContractCommitment_raw table -- Create the table if it doesn't exist +.create-merge table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Remove all columns to allow changing column types +.alter table ContractCommitment_raw ( ignore: string ) + +// ContractCommitment_raw table -- Redefine all columns +.alter table ContractCommitment_raw ( + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + x_SourceName: string, // Hubs v1_3+ + x_SourceProvider: string, // Hubs v1_3+ + x_SourceType: string, // Hubs v1_3+ + x_SourceVersion: string // Hubs v1_3+ +) + +// ContractCommitment_raw ingestion mapping +.create-or-alter table ContractCommitment_raw ingestion parquet mapping "ContractCommitment_raw_mapping" +``` +[ + { "Column": "BillingCurrency", "Properties": { "Field": "BillingCurrency" } }, + { "Column": "ContractCommitmentCategory", "Properties": { "Field": "ContractCommitmentCategory" } }, + { "Column": "ContractCommitmentCost", "Properties": { "Field": "ContractCommitmentCost" } }, + { "Column": "ContractCommitmentId", "Properties": { "Field": "ContractCommitmentId" } }, + { "Column": "ContractCommitmentPeriodEnd", "Properties": { "Field": "ContractCommitmentPeriodEnd" } }, + { "Column": "ContractCommitmentPeriodStart", "Properties": { "Field": "ContractCommitmentPeriodStart" } }, + { "Column": "ContractCommitmentQuantity", "Properties": { "Field": "ContractCommitmentQuantity" } }, + { "Column": "ContractCommitmentType", "Properties": { "Field": "ContractCommitmentType" } }, + { "Column": "ContractCommitmentUnit", "Properties": { "Field": "ContractCommitmentUnit" } }, + { "Column": "ContractId", "Properties": { "Field": "ContractId" } }, + { "Column": "ContractPeriodEnd", "Properties": { "Field": "ContractPeriodEnd" } }, + { "Column": "ContractPeriodStart", "Properties": { "Field": "ContractPeriodStart" } }, + { "Column": "InvoiceIssuerName", "Properties": { "Field": "InvoiceIssuerName" } }, + { "Column": "PricingCurrency", "Properties": { "Field": "PricingCurrency" } }, + { "Column": "x_SourceName", "Properties": { "Field": "x_SourceName" } }, + { "Column": "x_SourceProvider", "Properties": { "Field": "x_SourceProvider" } }, + { "Column": "x_SourceType", "Properties": { "Field": "x_SourceType" } }, + { "Column": "x_SourceVersion", "Properties": { "Field": "x_SourceVersion" } } +] +``` + +// ContractCommitment_raw retention policy (clear historical data) +.alter-merge table ContractCommitment_raw policy retention softdelete = 0d recoverability = disabled + +// ContractCommitment_raw retention policy (set the user-defined retention period) +.alter-merge table ContractCommitment_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled + +// Disable ContractCommitment_raw streaming ingestion (required for Fabric) +.alter table ContractCommitment_raw policy streamingingestion disable + + //===| Costs |========================================================================================================== // Supported versions: // - MS: 1.0, 1.0-preview(v1) -- See https://aka.ms/costmgmt/exports/focus diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql index 70d29a2ed..b7b051e18 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_3.kql @@ -1663,6 +1663,75 @@ CommitmentDiscountUsage_transform_v1_3() ``` +//===| Contract commitment |============================================================================================ +// Supported versions: +// - FOCUS 1.3+ (no Cost Management export shipped yet; defines the schema in advance for forward compat) +//====================================================================================================================== + +// ContractCommitment_transform_v1_3 function +.create-or-alter function +with (docstring='All contract commitments aligned to FOCUS 1.3.', folder='Contract commitments') +ContractCommitment_transform_v1_3() +{ + ContractCommitment_raw + | project + BillingCurrency, + ContractCommitmentCategory, + ContractCommitmentCost, + ContractCommitmentId, + ContractCommitmentPeriodEnd, + ContractCommitmentPeriodStart, + ContractCommitmentQuantity, + ContractCommitmentType, + ContractCommitmentUnit, + ContractId, + ContractPeriodEnd, + ContractPeriodStart, + InvoiceIssuerName, + PricingCurrency, + x_IngestionTime = ingestion_time(), + x_SourceName = coalesce(x_SourceName, ''), + x_SourceProvider = coalesce(x_SourceProvider, ''), + x_SourceType = coalesce(x_SourceType, ''), + x_SourceVersion = coalesce(x_SourceVersion, '') +} + +// ContractCommitment_final_v1_3 table +.create-merge table ContractCommitment_final_v1_3 ( + BillingCurrency: string, // FOCUS 1.3+ + ContractCommitmentCategory: string, // FOCUS 1.3+ + ContractCommitmentCost: real, // FOCUS 1.3+ + ContractCommitmentId: string, // FOCUS 1.3+ + ContractCommitmentPeriodEnd: datetime, // FOCUS 1.3+ + ContractCommitmentPeriodStart: datetime, // FOCUS 1.3+ + ContractCommitmentQuantity: real, // FOCUS 1.3+ + ContractCommitmentType: string, // FOCUS 1.3+ + ContractCommitmentUnit: string, // FOCUS 1.3+ + ContractId: string, // FOCUS 1.3+ + ContractPeriodEnd: datetime, // FOCUS 1.3+ + ContractPeriodStart: datetime, // FOCUS 1.3+ + InvoiceIssuerName: string, // FOCUS 1.3+ + PricingCurrency: string, // FOCUS 1.3+ + x_IngestionTime: datetime, // Hubs add-on + x_SourceName: string, // Hubs add-on + x_SourceProvider: string, // Hubs add-on + x_SourceType: string, // Hubs add-on + x_SourceVersion: string // Hubs add-on +) + +// Update policy for ContractCommitment_raw -> ContractCommitment_final_v1_3 table +.alter table ContractCommitment_final_v1_3 policy update +``` +[{ + "IsEnabled": true, + "Source": "ContractCommitment_raw", + "Query": "ContractCommitment_transform_v1_3()", + "IsTransactional": true, + "PropagateIngestionProperties": true +}] +``` + + //===| Recommendations |================================================================================================ // Supported datasets/versions: // - MS CM EA reservation recommendations: 2023-05-01 -- See https://learn.microsoft.com/azure/cost-management-billing/dataset-schema/reservation-recommendations-ea