AOE: migrate to DCR-based ingestion

docs-mslearn/toolkit/changelog.md

@@ -64,8 +64,10 @@ _Released June 2026_
 - **Fixed**
   - Corrected stale and incorrect descriptions for `BilledCost`, `EffectiveCost`, `BillingCurrency`, `BillingProfileId`, `BillingProfileName`, `CommitmentDiscountQuantity`, `ListUnitPrice`, `PricingQuantity`, `PricingUnitDescription`, and `TotalSavingsRunningTotal` to align with FOCUS 1.2 ([#2112](https://github.com/microsoft/finops-toolkit/pull/2112)).
 
-### [Optimization engine](optimization-engine/overview.md) updates
+### [Optimization engine](optimization-engine/overview.md) v15
 
+- **Changed**
+  - Migrated Log Analytics ingestion from the Data Collection API (to be deprecated in September 2026) to a Data Collection Rule, Ingestion API-based solution.
 - **Fixed**
   - Removed call to Azure Classic administrators endpoint (deprecated on May 1, 2026) from Azure RBAC assignments exports ([#2142](https://github.com/microsoft/finops-toolkit/issues/2142)).
 

src/optimization-engine/azuredeploy-nested.bicep

@@ -257,7 +257,7 @@ var csvExportsSchedules = [
   {
     exportSchedule: priceExportsScheduleName
     exportDescription: 'Weekly Pricesheet and Reservation Prices exports'
-    exportTimeOffset: 'PT1H35M'
+    exportTimeOffset: 'PT1H05M'
     exportFrequency: 'Week'
   }
   {
@@ -562,7 +562,7 @@ var csvExports = [
     variableDescription: 'The Storage Account container where Pricesheet exports are dumped to'
     ingestSchedule: 'AzureOptimization_IngestPricesheetWeekly'
     ingestDescription: 'Weekly Pricesheet ingests'
-    ingestTimeOffset: 'PT2H'
+    ingestTimeOffset: 'PT1H35M'
     ingestFrequency: 'Week'
     ingestJobId: pricesheetIngestJobId
     exportSchedule: priceExportsScheduleName
@@ -576,7 +576,7 @@ var csvExports = [
     variableDescription: 'The Storage Account container where Reservations Prices exports are dumped to'
     ingestSchedule: 'AzureOptimization_IngestReservationsPriceWeekly'
     ingestDescription: 'Weekly Reservations Prices ingests'
-    ingestTimeOffset: 'PT2H'
+    ingestTimeOffset: 'PT1H35M'
     ingestFrequency: 'Week'
     ingestJobId: reservationPricesIngestJobId
     exportSchedule: priceExportsScheduleName
@@ -949,7 +949,7 @@ var runbooks = [
   }
   {
     name: consumptionExportsRunbookName
-    version: '2.1.1.0'
+    version: '2.1.2.0'
     description: 'Exports Azure Consumption events to Blob Storage using Azure Consumption API'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/data-collection/${consumptionExportsRunbookName}.ps1')
@@ -1075,7 +1075,7 @@ var runbooks = [
   }
   {
     name: csvIngestRunbookName
-    version: '1.6.2.0'
+    version: '2.0.0.0'
     description: 'Ingests CSV blobs as custom logs to Log Analytics'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/data-collection/${csvIngestRunbookName}.ps1')
@@ -1187,21 +1187,21 @@ var runbooks = [
   }
   {
     name: recommendationsIngestRunbookName
-    version: '1.7.1.0'
+    version: '1.7.2.0'
     description: 'Ingests JSON-based recommendations into an Azure SQL Database'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/recommendations/${recommendationsIngestRunbookName}.ps1')
   }
   {
     name: recommendationsLogAnalyticsIngestRunbookName
-    version: '1.1.1.0'
+    version: '2.0.0.0'
     description: 'Ingests JSON-based recommendations into Log Analytics'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/recommendations/${recommendationsLogAnalyticsIngestRunbookName}.ps1')
   }
   {
     name: suppressionsLogAnalyticsIngestRunbookName
-    version: '1.1.0.0'
+    version: '2.0.0.0'
     description: 'Ingests suppressions into Log Analytics'
     type: 'PowerShell'
     scriptUri: uri(templateLocation, 'runbooks/recommendations/${suppressionsLogAnalyticsIngestRunbookName}.ps1')
@@ -1274,7 +1274,7 @@ var automationVariables = [
   {
     name: 'AzureOptimization_LogAnalyticsChunkSize'
     description: 'The size (in rows) for each chunk of Log Analytics ingestion request'
-    value: 6000
+    value: 150
   }
   {
     name: 'AzureOptimization_StorageBlobsPageSize'
@@ -1570,6 +1570,17 @@ resource logAnalyticsWorkspace 'microsoft.operationalinsights/workspaces@2020-08
   }
 }
 
+resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = {
+  name: '${automationAccountName}-dce'
+  location: projectLocation
+  tags: resourceTags
+  properties: {
+    networkAcls: {
+      publicNetworkAccess: 'Enabled'
+    }
+  }
+}
+
 resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = {
   name: storageAccountName
   location: projectLocation
@@ -1893,6 +1904,15 @@ resource automationVariables_LogAnalyticsWorkspaceKey 'Microsoft.Automation/auto
   }
 }
 
+resource automationVariables_DCEIngestionEndpoint 'Microsoft.Automation/automationAccounts/variables@2020-01-13-preview' = {
+  parent: automationAccount
+  name: 'AzureOptimization_DCEIngestionEndpoint'
+  properties: {
+    description: 'The Logs Ingestion endpoint URL of the Data Collection Endpoint used for DCR-based ingestion'
+    value: '"${dataCollectionEndpoint.properties.logsIngestion.endpoint}"'
+  }
+}
+
 resource automationSchedules_csvExports 'Microsoft.Automation/automationAccounts/schedules@2020-01-13-preview' = [for item in csvExportsSchedules: {
   parent: automationAccount
   name: item.exportSchedule
@@ -2156,3 +2176,5 @@ resource contributorRoleAssignmentGuid_resource 'Microsoft.Authorization/roleAss
 }
 
 output automationPrincipalId string = reference(automationAccount.id, '2019-06-01', 'Full').identity.principalId
+output dceLogsIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint
+output dceResourceId string = dataCollectionEndpoint.id

src/optimization-engine/model/loganalyticsingestcontrol-initialize.sql

@@ -1,155 +1,155 @@
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvmexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argvmexports', '1901-01-01T00:00:00Z', -1, 'VMsV1', 'ARGVirtualMachine')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argdiskexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argdiskexports', '1901-01-01T00:00:00Z', -1, 'DisksV1', 'ARGManagedDisk')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvhdexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argvhdexports', '1901-01-01T00:00:00Z', -1, 'VhdDisksV1', 'ARGUnmanagedDisk')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argavailsetexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argavailsetexports', '1901-01-01T00:00:00Z', -1, 'AvailSetsV1', 'ARGAvailabilitySet')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'advisorexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('advisorexports', '1901-01-01T00:00:00Z', -1, 'AdvisorV1', 'AzureAdvisor')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'remediationlogs')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('remediationlogs', '1901-01-01T00:00:00Z', -1, 'RemediationV1', 'RemediationLogs')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'consumptionexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('consumptionexports', '1901-01-01T00:00:00Z', -1, 'ConsumptionV1', 'AzureConsumption')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'aadobjectsexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('aadobjectsexports', '1901-01-01T00:00:00Z', -1, 'AADObjectsV1', 'AADObjects')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'arglbexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('arglbexports', '1901-01-01T00:00:00Z', -1, 'LoadBalancersV1', 'ARGLoadBalancer')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argappgwexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argappgwexports', '1901-01-01T00:00:00Z', -1, 'AppGatewaysV1', 'ARGAppGateway')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argrescontainersexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argrescontainersexports', '1901-01-01T00:00:00Z', -1, 'ResourceContainersV1', 'ARGResourceContainers')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'rbacexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('rbacexports', '1901-01-01T00:00:00Z', -1, 'RBACAssignmentsV1', 'RBACAssignments')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvnetexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argvnetexports', '1901-01-01T00:00:00Z', -1, 'VNetsV1', 'ARGVirtualNetwork')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argnicexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argnicexports', '1901-01-01T00:00:00Z', -1, 'NICsV1', 'ARGNetworkInterface')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argnsgexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argnsgexports', '1901-01-01T00:00:00Z', -1, 'NSGsV1', 'ARGNSGRule')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argpublicipexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argpublicipexports', '1901-01-01T00:00:00Z', -1, 'PublicIPsV1', 'ARGPublicIP')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvmssexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argvmssexports', '1901-01-01T00:00:00Z', -1, 'VMSSV1', 'ARGVMSS')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argsqldbexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argsqldbexports', '1901-01-01T00:00:00Z', -1, 'SqlDbV1', 'ARGSqlDb')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'azmonitorexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('azmonitorexports', '1901-01-01T00:00:00Z', -1, 'MonitorMetricsV1', 'MonitorMetrics')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'policystateexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('policystateexports', '1901-01-01T00:00:00Z', -1, 'PolicyStatesV1', 'PolicyStates')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'recommendationsexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('recommendationsexports', '2022-12-26T00:00:00Z', -1, 'RecommendationsV1', 'Recommendations')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'reservationsexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('reservationsexports', '1901-01-01T00:00:00Z', -1, 'ReservationsUsageV1', 'ReservationsUsage')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argappserviceplanexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('argappserviceplanexports', '1901-01-01T00:00:00Z', -1, 'AppServicePlansV1', 'AppServicePlans')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'pricesheetexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('pricesheetexports', '1901-01-01T00:00:00Z', -1, 'PricesheetV1', 'Pricesheet')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'reservationspriceexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('reservationspriceexports', '1901-01-01T00:00:00Z', -1, 'ReservationsPriceV1', 'ReservationsPrice')
 END
 
 IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'savingsplansexports')
 BEGIN
-    INSERT INTO [dbo].[LogAnalyticsIngestControl] 
+    INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) 
     VALUES ('savingsplansexports', '1901-01-01T00:00:00Z', -1, 'SavingsPlansUsageV1', 'SavingsPlansUsage')
 END

src/optimization-engine/model/loganalyticsingestcontrol-table.sql

@@ -8,7 +8,8 @@ AND OBJECTPROPERTY(id, N'IsUserTable') = 1)
 			[LastProcessedDateTime] [datetime] NULL,
 			[LastProcessedLine] [int] NULL,
 			[LogAnalyticsSuffix] [varchar](50) NOT NULL,
-			[CollectedType] [varchar](50) NULL
+			[CollectedType] [varchar](50) NULL,
+			[DCRImmutableId] [varchar](100) NULL
 		)
 
 		ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD PRIMARY KEY CLUSTERED 
@@ -18,8 +19,12 @@ AND OBJECTPROPERTY(id, N'IsUserTable') = 1)
 	END
 ELSE
 	BEGIN
-		IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'CollectedType'
-)		BEGIN
+		IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'CollectedType')
+		BEGIN
 			ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD [CollectedType] VARCHAR (50) NULL
 		END
+		IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'DCRImmutableId')
+		BEGIN
+			ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD [DCRImmutableId] VARCHAR (100) NULL
+		END
 	END