diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index ef17e2e00..e3b4d43f0 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -64,8 +64,10 @@ _Released June 2026_ - **Fixed** - Corrected stale and incorrect descriptions for `BilledCost`, `EffectiveCost`, `BillingCurrency`, `BillingProfileId`, `BillingProfileName`, `CommitmentDiscountQuantity`, `ListUnitPrice`, `PricingQuantity`, `PricingUnitDescription`, and `TotalSavingsRunningTotal` to align with FOCUS 1.2 ([#2112](https://github.com/microsoft/finops-toolkit/pull/2112)). -### [Optimization engine](optimization-engine/overview.md) updates +### [Optimization engine](optimization-engine/overview.md) v15 +- **Changed** + - Migrated Log Analytics ingestion from the Data Collection API (to be deprecated in September 2026) to a Data Collection Rule, Ingestion API-based solution. - **Fixed** - Removed call to Azure Classic administrators endpoint (deprecated on May 1, 2026) from Azure RBAC assignments exports ([#2142](https://github.com/microsoft/finops-toolkit/issues/2142)). diff --git a/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 b/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 index cdedc639c..a69fd685b 100644 --- a/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 +++ b/src/optimization-engine/Deploy-AzureOptimizationEngine.ps1 @@ -547,7 +547,8 @@ if ("N", "n" -contains $workspaceReuse) $laWorkspaceResourceGroup = $resourceGroupName $la = Get-AzOperationalInsightsWorkspace -ResourceGroupName $resourceGroupName -Name $laWorkspaceName -ErrorAction SilentlyContinue - if ($null -ne $la) { + if ($null -ne $la) + { Write-Host "(The Log Analytics Workspace was already deployed)" -ForegroundColor Green } } @@ -1123,6 +1124,32 @@ if ("Y", "y" -contains $continueInput) #endregion } + #region Ensuring Data Collection Endpoint was created (in case of partial upgrade) + $dceName = "$automationAccountName-dce" + $dce = Get-AzDataCollectionEndpoint -ResourceGroupName $ResourceGroupName -Name $dceName -ErrorAction SilentlyContinue + if ($null -eq $dce) + { + Write-Host "Creating Data Collection Endpoint $dceName..." -ForegroundColor Green + $dce = New-AzDataCollectionEndpoint -ResourceGroupName $ResourceGroupName -Name $dceName -Location $targetLocation ` + -Tag $resourceTags -NetworkAclsPublicNetworkAccess Enabled + } + if ($null -ne $dce.LogIngestionEndpoint) + { + $dceIngestionEndpointVariableName = "AzureOptimization_DCEIngestionEndpoint" + $dceIngestionEndpointVariable = Get-AzAutomationVariable -ResourceGroupName $resourceGroupName -AutomationAccountName $automationAccountName -Name $dceIngestionEndpointVariableName -ErrorAction SilentlyContinue + if ($null -eq $dceIngestionEndpointVariable) + { + Write-Host "Creating Automation Variable with Data Collection Endpoint ingestion endpoint..." -ForegroundColor Green + New-AzAutomationVariable -Name $dceIngestionEndpointVariableName -ResourceGroupName $resourceGroupName -AutomationAccountName $automationAccountName -Value $dce.LogIngestionEndpoint -Encrypted $false | Out-Null + } + } + else + { + throw "Data Collection Endpoint $dceName does not have a log ingestion endpoint yet. Please, check it was created successfully." + } + #endregion + + #region Schedules reset if ($upgradingSchedules) { @@ -1167,7 +1194,7 @@ if ("Y", "y" -contains $continueInput) $deploymentDate = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd") Write-Host "Setting initial deployment date ($deploymentDate)..." -ForegroundColor Green New-AzAutomationVariable -Name $deploymentDateVariableName -Description "The date of the initial engine deployment" ` - -ResourceGroupName $resourceGroupName -AutomationAccountName $automationAccountName -Value $deploymentDate -Encrypted $false + -ResourceGroupName $resourceGroupName -AutomationAccountName $automationAccountName -Value $deploymentDate -Encrypted $false | Out-Null } #endregion @@ -1185,12 +1212,24 @@ if ("Y", "y" -contains $continueInput) $sa = Get-AzStorageAccount -ResourceGroupName $resourceGroupName -Name $storageAccountName $auto = Get-AzAutomationAccount -Name $automationAccountName -ResourceGroupName $resourceGroupName $spnId = $auto.Identity.PrincipalId - $storageBlobContributorRoleId = "ba92f5b4-2d11-453d-a403-e96b0029c9fe" # Storage Blob Data Contributor role - $roleAssignment = Get-AzRoleAssignment -ObjectId $spnId -Scope $sa.Id -ErrorAction SilentlyContinue - if (-not($roleAssignment) -or -not($roleAssignment.RoleDefinitionId -contains $storageBlobContributorRoleId)) + + if ($spnId) { - Write-Host "Granting Storage Blob Data Contributor role to the Automation Account identity..." -ForegroundColor Green - New-AzRoleAssignment -ObjectId $spnId -RoleDefinitionId $storageBlobContributorRoleId -Scope $sa.Id | Out-Null + $storageBlobContributorRoleId = "ba92f5b4-2d11-453d-a403-e96b0029c9fe" # Storage Blob Data Contributor role + $roleAssignment = Get-AzRoleAssignment -ObjectId $spnId -Scope $sa.Id -ErrorAction SilentlyContinue + if (-not($roleAssignment) -or -not($roleAssignment.RoleDefinitionId -contains $storageBlobContributorRoleId)) + { + Write-Host "Granting Storage Blob Data Contributor role to the Automation Account identity..." -ForegroundColor Green + New-AzRoleAssignment -ObjectId $spnId -RoleDefinitionId $storageBlobContributorRoleId -Scope $sa.Id | Out-Null + } + else + { + Write-Host "Automation Account identity already has the Storage Blob Data Contributor role at the Storage Account level." -ForegroundColor Green + } + } + else + { + Write-Host "Automation Account does not have a system-assigned identity. Manually grant the Storage Blob Data Contributor role to the managed identity used by AOE on the $($sa.Id) resource." -ForegroundColor Yellow } #region Open SQL Server firewall rule @@ -1332,17 +1371,25 @@ if ("Y", "y" -contains $continueInput) $Cmd.ExecuteReader() $Conn.Close() - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlServerEndpoint,1433;Database=$databaseName;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken - $Conn.Open() - - $createUserQuery = (Get-Content -Path "./model/automation-user.sql").Replace("", $automationAccountName) - $Cmd = New-Object system.Data.SqlClient.SqlCommand - $Cmd.Connection = $Conn - $Cmd.CommandTimeout = $SqlTimeout - $Cmd.CommandText = $createUserQuery - $Cmd.ExecuteReader() - $Conn.Close() + if ($spnId) + { + $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlServerEndpoint,1433;Database=$databaseName;Encrypt=True;Connection Timeout=$SqlTimeout;") + $Conn.AccessToken = $dbToken + $Conn.Open() + + $createUserQuery = (Get-Content -Path "./model/automation-user.sql").Replace("", $automationAccountName) + $Cmd = New-Object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn + $Cmd.CommandTimeout = $SqlTimeout + $Cmd.CommandText = $createUserQuery + $Cmd.ExecuteReader() + $Conn.Close() + } + else + { + Write-Host "Automation Account does not have a system-assigned identity. Please, create a SQL user for the identity used by AOE with db_datareader and db_datawriter roles on the $databaseName database." -ForegroundColor Yellow + Write-Host "You can use the T-SQL script in model/automation-user.sql as a template, replacing with the name of the AOE Managed Identity." -ForegroundColor Yellow + } $connectionSuccess = $true } @@ -1365,6 +1412,26 @@ if ("Y", "y" -contains $continueInput) } #endregion + #region DCR-based ingestion setup + Write-Host "Setting up Log Analytics custom tables and Data Collection Rules for DCR-based ingestion..." -ForegroundColor Green + try + { + .\Setup-LogAnalyticsTablesAndDCRs.ps1 ` + -ResourceGroupName $resourceGroupName ` + -AutomationAccountName $automationAccountName ` + -WorkspaceName $laWorkspaceName ` + -WorkspaceResourceGroupName $laWorkspaceResourceGroup ` + -SqlServerName $sqlServerName ` + -SqlDatabaseName $sqlDatabaseName ` + -CloudEnvironment $cloudEnvironment + } + catch + { + Write-Host "Could not complete the Log Analytics tables and DCR setup: $($_.Exception.Message)" -ForegroundColor Yellow + Write-Host "You can re-run Setup-LogAnalyticsTablesAndDCRs.ps1 manually after resolving the issue." -ForegroundColor Yellow + } + #endregion + #region Close SQL Server firewall rule if (-not($sqlServerName -like "*.database.*")) { @@ -1409,95 +1476,86 @@ if ("Y", "y" -contains $continueInput) if (!$silentDeploy) { #region Grant Microsoft Entra ID role to AOE principal - if ($null -eq $spnId) + if ($spnId) { - $auto = Get-AzAutomationAccount -Name $automationAccountName -ResourceGroupName $resourceGroupName - $spnId = $auto.Identity.PrincipalId - if ($null -eq $spnId) + try { - $runAsConnection = Get-AzAutomationConnection -ResourceGroupName $resourceGroupName -AutomationAccountName $automationAccountName -Name AzureRunAsConnection -ErrorAction SilentlyContinue - $runAsAppId = $runAsConnection.FieldDefinitionValues.ApplicationId - if ($runAsAppId) - { - $runAsServicePrincipal = Get-AzADServicePrincipal -ApplicationId $runAsAppId - $spnId = $runAsServicePrincipal.Id - } - } - } - - try - { - Import-Module Microsoft.Graph.Authentication - Import-Module Microsoft.Graph.Identity.DirectoryManagement - - Write-Host "Granting Microsoft Entra ID Global Reader role to the Automation Account (requires administrative permissions in Microsoft Entra and MS Graph PowerShell SDK >= 2.4.0)..." -ForegroundColor Green + Import-Module Microsoft.Graph.Authentication + Import-Module Microsoft.Graph.Identity.DirectoryManagement - #workaround for https://github.com/microsoftgraph/msgraph-sdk-powershell/issues/888 - $localPath = [System.Environment]::GetFolderPath([System.Environment+SpecialFolder]::UserProfile) - if (-not(Get-Item "$localPath\.graph\" -ErrorAction SilentlyContinue)) - { - New-Item -Type Directory "$localPath\.graph" - } + Write-Host "Granting Microsoft Entra ID Global Reader role to the Automation Account (requires administrative permissions in Microsoft Entra and MS Graph PowerShell SDK >= 2.4.0)..." -ForegroundColor Green - switch ($cloudEnvironment) - { - "AzureUSGovernment" - { - $graphEnvironment = "USGov" - break - } - "AzureChinaCloud" + #workaround for https://github.com/microsoftgraph/msgraph-sdk-powershell/issues/888 + $localPath = [System.Environment]::GetFolderPath([System.Environment+SpecialFolder]::UserProfile) + if (-not(Get-Item "$localPath\.graph\" -ErrorAction SilentlyContinue)) { - $graphEnvironment = "China" - break + New-Item -Type Directory "$localPath\.graph" } - "AzureGermanCloud" - { - $graphEnvironment = "Germany" - break - } - default + + switch ($cloudEnvironment) { - $graphEnvironment = "Global" + "AzureUSGovernment" + { + $graphEnvironment = "USGov" + break + } + "AzureChinaCloud" + { + $graphEnvironment = "China" + break + } + "AzureGermanCloud" + { + $graphEnvironment = "Germany" + break + } + default + { + $graphEnvironment = "Global" + } } - } - Connect-MgGraph -Scopes "RoleManagement.ReadWrite.Directory", "Directory.Read.All" -UseDeviceAuthentication -Environment $graphEnvironment -NoWelcome + Connect-MgGraph -Scopes "RoleManagement.ReadWrite.Directory", "Directory.Read.All" -UseDeviceAuthentication -Environment $graphEnvironment -NoWelcome - $globalReaderRole = Get-MgDirectoryRole -ExpandProperty Members -Property Id, Members, DisplayName, RoleTemplateId ` - | Where-Object { $_.RoleTemplateId -eq "f2ef992c-3afb-46b9-b7cf-a126ee74c451" } - $globalReaders = $globalReaderRole.Members.Id - if (-not($globalReaders -contains $spnId)) - { - New-MgDirectoryRoleMemberByRef -DirectoryRoleId $globalReaderRole.Id -BodyParameter @{"@odata.id" = "https://graph.microsoft.com/v1.0/directoryObjects/$spnId" } - Start-Sleep -Seconds 5 $globalReaderRole = Get-MgDirectoryRole -ExpandProperty Members -Property Id, Members, DisplayName, RoleTemplateId ` | Where-Object { $_.RoleTemplateId -eq "f2ef992c-3afb-46b9-b7cf-a126ee74c451" } $globalReaders = $globalReaderRole.Members.Id - if ($globalReaders -contains $spnId) + if (-not($globalReaders -contains $spnId)) { - Write-Host "Role granted." -ForegroundColor Green + New-MgDirectoryRoleMemberByRef -DirectoryRoleId $globalReaderRole.Id -BodyParameter @{"@odata.id" = "https://graph.microsoft.com/v1.0/directoryObjects/$spnId" } + Start-Sleep -Seconds 5 + $globalReaderRole = Get-MgDirectoryRole -ExpandProperty Members -Property Id, Members, DisplayName, RoleTemplateId ` + | Where-Object { $_.RoleTemplateId -eq "f2ef992c-3afb-46b9-b7cf-a126ee74c451" } + $globalReaders = $globalReaderRole.Members.Id + if ($globalReaders -contains $spnId) + { + Write-Host "Role granted." -ForegroundColor Green + } + else + { + throw "Error when trying to grant Global Reader role" + } } else { - throw "Error when trying to grant Global Reader role" + Write-Host "Role was already granted before." -ForegroundColor Green } } - else + catch { - Write-Host "Role was already granted before." -ForegroundColor Green + Write-Host $Error[0] -ForegroundColor Yellow + Write-Host "Could not grant role. If you want Microsoft Entra-based recommendations, please grant the Global Reader role manually to the $automationAccountName managed identity or, for previous versions of AOE, to the Run As Account principal." -ForegroundColor Red } } - catch + else { - Write-Host $Error[0] -ForegroundColor Yellow - Write-Host "Could not grant role. If you want Microsoft Entra-based recommendations, please grant the Global Reader role manually to the $automationAccountName managed identity or, for previous versions of AOE, to the Run As Account principal." -ForegroundColor Red + Write-Host "Automation Account does not have a system-assigned identity. Please grant the Global Reader role manually to the managed identity used by AOE." -ForegroundColor Yellow } #endregion } else { - Write-Host "Could not grant role. If you want Microsoft Entra-based recommendations, please grant the Global Reader role manually to the $automationAccountName managed identity or, for previous versions of AOE, to the Run As Account principal." -ForegroundColor Red + Write-Host "Could not grant role. If you want Microsoft Entra-based recommendations, please grant the Global Reader role manually to the $automationAccountName managed identity." -ForegroundColor Red } Write-Host "Azure Optimization Engine deployment completed! We're almost there..." -ForegroundColor Green diff --git a/src/optimization-engine/Setup-LogAnalyticsTablesAndDCRs.ps1 b/src/optimization-engine/Setup-LogAnalyticsTablesAndDCRs.ps1 new file mode 100644 index 000000000..c9d37aafc --- /dev/null +++ b/src/optimization-engine/Setup-LogAnalyticsTablesAndDCRs.ps1 @@ -0,0 +1,1203 @@ +<# +.SYNOPSIS +Creates or updates the custom Log Analytics tables and Data Collection Rules (DCRs) required +for DCR-based ingestion in the Azure Optimization Engine. + +.DESCRIPTION +This script provisions all custom Log Analytics tables with explicit schemas and one DCR per table, +replacing the legacy Log Analytics Data Collector API (SharedKey) ingestion. It must be run once +on both new and existing AOE installations. On existing installations, it also migrates the SQL +control table to store the DCR immutable IDs. + +The script: +- Reads the Data Collection Endpoint (DCE) logs ingestion endpoint from the AzureOptimization_DCEIngestionEndpoint automation variable (the DCE and variable are expected to be deployed via the Bicep template).- Creates or updates each custom Log Analytics table schema. +- Creates or updates one DCR per table. +- Assigns the Monitoring Metrics Publisher role on the resource group containing the DCRs to the Automation account managed identity. +- Writes the DCR immutable ID for each table into the SQL LogAnalyticsIngestControl table. +- Creates/updates the AzureOptimization_DCEIngestionEndpoint automation variable. +- Removes the AzureOptimization_LogAnalyticsWorkspaceKey automation variable (no longer needed). + +.PARAMETER ResourceGroupName +The resource group where the AOE Automation account and (optionally) Log Analytics workspace reside. + +.PARAMETER AutomationAccountName +The name of the AOE Automation account. + +.PARAMETER WorkspaceName +The name of the Log Analytics workspace where custom tables will be created. + +.PARAMETER WorkspaceResourceGroupName +The resource group of the Log Analytics workspace. Defaults to ResourceGroupName if not specified. + +.PARAMETER WorkspaceSubscriptionId +The subscription ID of the Log Analytics workspace. Defaults to the current subscription if not specified. + +.PARAMETER SqlServerName +The SQL Server hostname (FQDN or short name). If short name, '.database.windows.net' is appended. + +.PARAMETER SqlDatabaseName +The name of the AOE SQL database. + +.PARAMETER CloudEnvironment +The Azure cloud environment. Default is AzureCloud. Supported: AzureCloud, AzureChinaCloud, AzureUSGovernment. + +.EXAMPLE +.\Setup-LogAnalyticsTablesAndDCRs.ps1 ` + -ResourceGroupName "rg-aoe" ` + -AutomationAccountName "aa-aoe" ` + -WorkspaceName "la-aoe" ` + -SqlServerName "sql-aoe" ` + -SqlDatabaseName "db-aoe" + +.LINK +https://aka.ms/AzureOptimizationEngine/deployment +#> +param ( + [Parameter(Mandatory = $true)] + [string] $ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string] $AutomationAccountName, + + [Parameter(Mandatory = $true)] + [string] $WorkspaceName, + + [Parameter(Mandatory = $false)] + [string] $WorkspaceResourceGroupName, + + [Parameter(Mandatory = $false)] + [string] $WorkspaceSubscriptionId, + + [Parameter(Mandatory = $true)] + [string] $SqlServerName, + + [Parameter(Mandatory = $true)] + [string] $SqlDatabaseName, + + [Parameter(Mandatory = $false)] + [string] $CloudEnvironment = "AzureCloud" +) + +$ErrorActionPreference = "Stop" + +if ([string]::IsNullOrEmpty($WorkspaceResourceGroupName)) +{ + $WorkspaceResourceGroupName = $ResourceGroupName +} + +#region Resolve Monitor audience URL per cloud +switch ($CloudEnvironment) +{ + "AzureChinaCloud" { $armEndpoint = "https://management.chinacloudapi.cn" } + "AzureUSGovernment" { $armEndpoint = "https://management.usgovcloudapi.net" } + default { $armEndpoint = "https://management.azure.com" } +} +#endregion + +#region Resolve SQL connection string +if (-not ($SqlServerName -like "*.*")) +{ + switch ($CloudEnvironment) + { + "AzureChinaCloud" { $SqlServerName = "$SqlServerName.database.chinacloudapi.cn" } + "AzureUSGovernment" { $SqlServerName = "$SqlServerName.database.usgovcloudapi.net" } + default { $SqlServerName = "$SqlServerName.database.windows.net" } + } +} +#endregion + +Write-Host "Setting up Log Analytics custom tables and DCRs for the Azure Optimization Engine..." -ForegroundColor Green + +#region Resolve workspace and current subscription context +$context = Get-AzContext +if ([string]::IsNullOrEmpty($WorkspaceSubscriptionId)) +{ + $WorkspaceSubscriptionId = $context.Subscription.Id +} +$currentSubscriptionId = $context.Subscription.Id + +$workspaceResourceId = "/subscriptions/$WorkspaceSubscriptionId/resourceGroups/$WorkspaceResourceGroupName/providers/Microsoft.OperationalInsights/workspaces/$WorkspaceName" +$workspaceLocation = (Get-AzOperationalInsightsWorkspace -ResourceGroupName $WorkspaceResourceGroupName -Name $WorkspaceName).Location +#endregion + +#region Get DCE endpoint and resource ID from automation variable +$dceVar = Get-AzAutomationVariable -ResourceGroupName $ResourceGroupName -AutomationAccountName $AutomationAccountName ` + -Name "AzureOptimization_DCEIngestionEndpoint" -ErrorAction SilentlyContinue +if ($null -eq $dceVar -or [string]::IsNullOrEmpty($dceVar.Value)) +{ + throw "AzureOptimization_DCEIngestionEndpoint automation variable not found. Ensure the Bicep template has been deployed first." +} +$dceLogsIngestionEndpoint = $dceVar.Value +Write-Host "DCE Logs Ingestion endpoint: $dceLogsIngestionEndpoint" -ForegroundColor Cyan + +# Discover the DCE resource ID by searching for DCEs in the resource group whose logsIngestion endpoint matches + +$dceListResponse = Get-AzDataCollectionEndpoint -ResourceGroupName $ResourceGroupName -ErrorAction SilentlyContinue +$matchedDce = $dceListResponse | Where-Object { $_.LogIngestionEndpoint -eq $dceLogsIngestionEndpoint } +if ($null -eq $matchedDce) +{ + throw "Could not find a Data Collection Endpoint in resource group $ResourceGroupName with logsIngestion endpoint '$dceLogsIngestionEndpoint'." +} +$dceResourceId = $matchedDce.Id +Write-Host "DCE resource ID: $dceResourceId" -ForegroundColor Cyan +#endregion + +#region Get automation account managed identity principal ID +$automationAccount = Get-AzAutomationAccount -ResourceGroupName $ResourceGroupName -Name $AutomationAccountName +$automationPrincipalId = $automationAccount.Identity.PrincipalId +if ([string]::IsNullOrEmpty($automationPrincipalId)) +{ + Write-Host "Could not retrieve the managed identity principal ID for the Automation account $AutomationAccountName." -ForegroundColor Yellow + Write-Host "Grant the Metrics Publisher role to the AOE's managed identity on the resource group containing the DCRs after this script completes." -ForegroundColor Yellow +} +else +{ + Write-Host "Retrieved managed identity principal ID for Automation account: $automationPrincipalId" -ForegroundColor Cyan +} +#endregion + +#region Monitoring Metrics Publisher role definition ID +# Role definition: Monitoring Metrics Publisher (3913510d-42f4-4e42-8a64-420c390055eb) +$monitoringMetricsPublisherRoleId = "3913510d-42f4-4e42-8a64-420c390055eb" +#endregion + +#region Table schema definitions +# Each entry defines the columns that the ingest runbooks write. +# Column type mapping from the legacy _suffix convention: +# _s -> string +# _g -> string (GUIDs stored as plain strings in the typed column name) +# _t -> datetime +# _d -> real +# _b -> boolean +# TimeGenerated is mandatory for all custom tables. + +$tableSchemas = @{ + + "VMsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "VMName_s"; type = "string" } + @{ name = "DeploymentModel_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "VMSize_s"; type = "string" } + @{ name = "CoresCount_s"; type = "string" } + @{ name = "MemoryMB_s"; type = "string" } + @{ name = "OSType_s"; type = "string" } + @{ name = "LicenseType_s"; type = "string" } + @{ name = "DataDiskCount_s"; type = "string" } + @{ name = "NicCount_s"; type = "string" } + @{ name = "UsesManagedDisks_s"; type = "string" } + @{ name = "AvailabilitySetId_s"; type = "string" } + @{ name = "BootDiagnosticsEnabled_s"; type = "string" } + @{ name = "BootDiagnosticsStorageAccount_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "PowerState_s"; type = "string" } + @{ name = "ImagePublisher_s"; type = "string" } + @{ name = "ImageOffer_s"; type = "string" } + @{ name = "ImageSku_s"; type = "string" } + @{ name = "ImageVersion_s"; type = "string" } + @{ name = "ImageExactVersion_s"; type = "string" } + @{ name = "OSName_s"; type = "string" } + @{ name = "OSVersion_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + ) + + "DisksV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "DiskName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "OwnerVMId_s"; type = "string" } + @{ name = "DeploymentModel_s"; type = "string" } + @{ name = "DiskType_s"; type = "string" } + @{ name = "TimeCreated_s"; type = "string" } + @{ name = "DiskIOPS_s"; type = "string" } + @{ name = "DiskThroughput_s"; type = "string" } + @{ name = "DiskTier_s"; type = "string" } + @{ name = "DiskState_s"; type = "string" } + @{ name = "EncryptionType_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + @{ name = "Caching_s"; type = "string" } + @{ name = "DiskSizeGB_s"; type = "string" } + @{ name = "SKU_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + ) + + "VhdDisksV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "DiskName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "OwnerVMId_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "DeploymentModel_s"; type = "string" } + @{ name = "DiskType_s"; type = "string" } + @{ name = "Caching_s"; type = "string" } + @{ name = "DiskSizeGB_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + ) + + "AvailSetsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "FaultDomains_s"; type = "string" } + @{ name = "UpdateDomains_s"; type = "string" } + @{ name = "VmCount_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + ) + + "AdvisorV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "Category"; type = "string" } + @{ name = "Impact_s"; type = "string" } + @{ name = "ImpactedArea_s"; type = "string" } + @{ name = "Description_s"; type = "string" } + @{ name = "RecommendationText_s"; type = "string" } + @{ name = "RecommendationTypeId_g"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "InstanceName_g"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "AdditionalInfo_s"; type = "string" } + @{ name = "ResourceGroup"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + ) + + "RemediationV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "ResourceId_s"; type = "string" } + @{ name = "RemediationId_g"; type = "string" } + @{ name = "RemediationName_s"; type = "string" } + @{ name = "PolicyAssignmentId_s"; type = "string" } + @{ name = "PolicyDefinitionId_s"; type = "string" } + @{ name = "ProvisioningState_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "ConsumptionV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "AdditionalInfo_s"; type = "string" } + @{ name = "benefitId_s"; type = "string" } + @{ name = "benefitName_s"; type = "string" } + @{ name = "BillingAccountId_s"; type = "string" } + @{ name = "BillingAccountName_s"; type = "string" } + @{ name = "BillingCurrencyCode_s"; type = "string" } + @{ name = "BillingPeriodEndDate_s"; type = "string" } + @{ name = "BillingPeriodStartDate_s"; type = "string" } + @{ name = "BillingProfileId_s"; type = "string" } + @{ name = "BillingProfileName_s"; type = "string" } + @{ name = "ChargeType_s"; type = "string" } + @{ name = "ConsumedService_s"; type = "string" } + @{ name = "CostAllocationRuleName_s"; type = "string" } + @{ name = "CostCenter_s"; type = "string" } + @{ name = "CostInBillingCurrency_s"; type = "string" } + @{ name = "Date_s"; type = "string" } + @{ name = "EffectivePrice_s"; type = "string" } + @{ name = "Frequency_s"; type = "string" } + @{ name = "InvoiceSectionId_s"; type = "string" } + @{ name = "InvoiceSectionName_s"; type = "string" } + @{ name = "IsAzureCreditEligible_s"; type = "string" } + @{ name = "MeterCategory_s"; type = "string" } + @{ name = "MeterId_g"; type = "string" } + @{ name = "MeterName_s"; type = "string" } + @{ name = "MeterRegion_s"; type = "string" } + @{ name = "MeterSubCategory_s"; type = "string" } + @{ name = "OfferId_s"; type = "string" } + @{ name = "PricingModel_s"; type = "string" } + @{ name = "ProductOrderId_s"; type = "string" } + @{ name = "ProductOrderName_s"; type = "string" } + @{ name = "PublisherName_s"; type = "string" } + @{ name = "PublisherType_s"; type = "string" } + @{ name = "Quantity_s"; type = "string" } + @{ name = "ReservationId_s"; type = "string" } + @{ name = "ReservationName_s"; type = "string" } + @{ name = "ResourceGroup"; type = "string" } + @{ name = "ResourceId"; type = "string" } + @{ name = "ResourceLocation_s"; type = "string" } + @{ name = "ServiceFamily_s"; type = "string" } + @{ name = "ServiceInfo1_s"; type = "string" } + @{ name = "ServiceInfo2_s"; type = "string" } + @{ name = "SubscriptionId"; type = "string" } + @{ name = "SubscriptionName_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "Term_s"; type = "string" } + @{ name = "UnitOfMeasure_s"; type = "string" } + @{ name = "UnitPrice_s"; type = "string" } + ) + + "AADObjectsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "ObjectId_g"; type = "string" } + @{ name = "ObjectType_s"; type = "string" } + @{ name = "ObjectSubType_s"; type = "string" } + @{ name = "DisplayName_s"; type = "string" } + @{ name = "SecurityEnabled_s"; type = "string" } + @{ name = "ApplicationId_g"; type = "string" } + @{ name = "Keys_s"; type = "string" } + @{ name = "PrincipalNames_s"; type = "string" } + @{ name = "Owners_s"; type = "string" } + @{ name = "CreatedDate_t"; type = "datetime" } + @{ name = "DeletedDate_s"; type = "string" } + ) + + "LoadBalancersV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "SkuTier_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "LbType_s"; type = "string" } + @{ name = "LbRulesCount_s"; type = "string" } + @{ name = "InboundNatRulesCount_s"; type = "string" } + @{ name = "OutboundRulesCount_s"; type = "string" } + @{ name = "FrontendIPsCount_s"; type = "string" } + @{ name = "BackendIPCount_s"; type = "string" } + @{ name = "BackendAddressesCount_s"; type = "string" } + @{ name = "InboundNatPoolsCount_s"; type = "string" } + @{ name = "BackendPoolsCount_s"; type = "string" } + @{ name = "ProbesCount_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + ) + + "AppGatewaysV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "SkuTier_s"; type = "string" } + @{ name = "SkuCapacity_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + @{ name = "EnableHttp2_s"; type = "string" } + @{ name = "GatewayIPsCount_s"; type = "string" } + @{ name = "FrontendIPsCount_s"; type = "string" } + @{ name = "FrontendPortsCount_s"; type = "string" } + @{ name = "BackendIPCount_s"; type = "string" } + @{ name = "BackendAddressesCount_s"; type = "string" } + @{ name = "HttpSettingsCount_s"; type = "string" } + @{ name = "HttpListenersCount_s"; type = "string" } + @{ name = "BackendPoolsCount_s"; type = "string" } + @{ name = "ProbesCount_s"; type = "string" } + @{ name = "UrlPathMapsCount_s"; type = "string" } + @{ name = "RequestRoutingRulesCount_s"; type = "string" } + @{ name = "RewriteRulesCount_s"; type = "string" } + @{ name = "RedirectConfsCount_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + ) + + "ResourceContainersV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "ContainerType_s"; type = "string" } + @{ name = "ContainerName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "ResourceCount_s"; type = "string" } + @{ name = "ManagedBy_s"; type = "string" } + @{ name = "ContainerProperties_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "RBACAssignmentsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "Model_s"; type = "string" } + @{ name = "PrincipalId_g"; type = "string" } + @{ name = "PrincipalId_s"; type = "string" } + @{ name = "Scope_s"; type = "string" } + @{ name = "RoleDefinition_s"; type = "string" } + ) + + "VNetsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "VNetName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "Model_s"; type = "string" } + @{ name = "VNetPrefixes_s"; type = "string" } + @{ name = "DNSServers_s"; type = "string" } + @{ name = "PeeringsCount_s"; type = "string" } + @{ name = "EnableDdosProtection_s"; type = "string" } + @{ name = "SubnetName_s"; type = "string" } + @{ name = "SubnetPrefix_s"; type = "string" } + @{ name = "SubnetDelegationsCount_s"; type = "string" } + @{ name = "SubnetTotalPrefixIPs_s"; type = "string" } + @{ name = "SubnetUsedIPs_s"; type = "string" } + @{ name = "SubnetNSGId_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "NICsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "Name_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "IsPrimary_s"; type = "string" } + @{ name = "EnableAcceleratedNetworking_s"; type = "string" } + @{ name = "EnableIPForwarding_s"; type = "string" } + @{ name = "TapConfigurationsCount_s"; type = "string" } + @{ name = "HostedWorkloadsCount_s"; type = "string" } + @{ name = "InternalDomainNameSuffix_s"; type = "string" } + @{ name = "AppliedDnsServers_s"; type = "string" } + @{ name = "DnsServers_s"; type = "string" } + @{ name = "OwnerVMId_s"; type = "string" } + @{ name = "OwnerPEId_s"; type = "string" } + @{ name = "MacAddress_s"; type = "string" } + @{ name = "NicType_s"; type = "string" } + @{ name = "NicNSGId_s"; type = "string" } + @{ name = "PrivateIPAddressVersion_s"; type = "string" } + @{ name = "PrivateIPAllocationMethod_s"; type = "string" } + @{ name = "IsIPConfigPrimary_s"; type = "string" } + @{ name = "PrivateIPAddress_s"; type = "string" } + @{ name = "PublicIPId_s"; type = "string" } + @{ name = "IPConfigName_s"; type = "string" } + @{ name = "SubnetId_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "NSGsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "NSGName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "NicCount_s"; type = "string" } + @{ name = "SubnetCount_s"; type = "string" } + @{ name = "RuleName_s"; type = "string" } + @{ name = "RuleProtocol_s"; type = "string" } + @{ name = "RuleDirection_s"; type = "string" } + @{ name = "RulePriority_s"; type = "string" } + @{ name = "RuleAccess_s"; type = "string" } + @{ name = "RuleDestinationAddresses_s"; type = "string" } + @{ name = "RuleSourceAddresses_s"; type = "string" } + @{ name = "RuleDestinationPorts_s"; type = "string" } + @{ name = "RuleSourcePorts_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "PublicIPsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "Name_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "Model_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "SkuTier_s"; type = "string" } + @{ name = "AllocationMethod_s"; type = "string" } + @{ name = "AddressVersion_s"; type = "string" } + @{ name = "AssociatedResourceId_s"; type = "string" } + @{ name = "PublicIpPrefixId_s"; type = "string" } + @{ name = "IPAddress"; type = "string" } + @{ name = "FQDN_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "VMSSV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "Zones_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "VMSSName_s"; type = "string" } + @{ name = "ComputerNamePrefix_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "VMSSSize_s"; type = "string" } + @{ name = "CoresCount_s"; type = "string" } + @{ name = "MemoryMB_s"; type = "string" } + @{ name = "OSType_s"; type = "string" } + @{ name = "DataDiskCount_s"; type = "string" } + @{ name = "NicCount_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "Capacity_s"; type = "string" } + @{ name = "Priority_s"; type = "string" } + @{ name = "OSDiskSize_s"; type = "string" } + @{ name = "OSDiskCaching_s"; type = "string" } + @{ name = "OSDiskSKU_s"; type = "string" } + @{ name = "SinglePlacementGroup_s"; type = "string" } + @{ name = "UpgradePolicy_s"; type = "string" } + @{ name = "OverProvision_s"; type = "string" } + @{ name = "PlatformFaultDomainCount_s"; type = "string" } + @{ name = "ZoneBalance_s"; type = "string" } + @{ name = "UsesManagedDisks_s"; type = "string" } + @{ name = "ImagePublisher_s"; type = "string" } + @{ name = "ImageOffer_s"; type = "string" } + @{ name = "ImageSku_s"; type = "string" } + @{ name = "ImageVersion_s"; type = "string" } + @{ name = "ImageExactVersion_s"; type = "string" } + ) + + "SqlDbV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "ZoneRedundant_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "DBName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "SkuTier_s"; type = "string" } + @{ name = "SkuCapacity_s"; type = "string" } + @{ name = "ServiceObjectiveName_s"; type = "string" } + @{ name = "StorageAccountType_s"; type = "string" } + @{ name = "LicenseType_s"; type = "string" } + @{ name = "MaxSizeBytes_s"; type = "string" } + @{ name = "MaxLogSizeBytes_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "MonitorMetricsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "ResourceName_s"; type = "string" } + @{ name = "ResourceId"; type = "string" } + @{ name = "MetricNames_s"; type = "string" } + @{ name = "AggregationType_s"; type = "string" } + @{ name = "AggregationOfType_s"; type = "string" } + @{ name = "MetricValue_s"; type = "string" } + @{ name = "TimeGrain_s"; type = "string" } + @{ name = "TimeSpan_s"; type = "string" } + ) + + "PolicyStatesV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "ResourceId"; type = "string" } + @{ name = "ResourceType_s"; type = "string" } + @{ name = "ComplianceState_s"; type = "string" } + @{ name = "ComplianceReason_s"; type = "string" } + @{ name = "Effect_s"; type = "string" } + @{ name = "AssignmentId_s"; type = "string" } + @{ name = "AssignmentName_s"; type = "string" } + @{ name = "InitiativeId_s"; type = "string" } + @{ name = "InitiativeName_s"; type = "string" } + @{ name = "DefinitionId_s"; type = "string" } + @{ name = "DefinitionName_s"; type = "string" } + @{ name = "DefinitionReferenceId_s"; type = "string" } + @{ name = "EvaluatedOn_s"; type = "string" } + @{ name = "StatesCount_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "RecommendationsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "Category"; type = "string" } + @{ name = "ImpactedArea_s"; type = "string" } + @{ name = "Impact_s"; type = "string" } + @{ name = "RecommendationType_s"; type = "string" } + @{ name = "RecommendationSubType_s"; type = "string" } + @{ name = "RecommendationSubTypeId_g"; type = "string" } + @{ name = "RecommendationDescription_s"; type = "string" } + @{ name = "RecommendationAction_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "AdditionalInfo_s"; type = "string" } + @{ name = "ResourceGroup"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "SubscriptionName_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "FitScore_d"; type = "real" } + @{ name = "Tags_s"; type = "string" } + @{ name = "DetailsURL_s"; type = "string" } + @{ name = "GeneratedDate_s"; type = "string" } + ) + + "ReservationsUsageV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "ReservationResourceId_s"; type = "string" } + @{ name = "ReservationOrderId_s"; type = "string" } + @{ name = "ReservationId_g"; type = "string" } + @{ name = "DisplayName_s"; type = "string" } + @{ name = "SKUName_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "ResourceType"; type = "string" } + @{ name = "AppliedScopeType_s"; type = "string" } + @{ name = "Term_s"; type = "string" } + @{ name = "ProvisioningState_s"; type = "string" } + @{ name = "RenewState_s"; type = "string" } + @{ name = "PurchaseDate_s"; type = "string" } + @{ name = "ExpiryDate_s"; type = "string" } + @{ name = "Archived_s"; type = "string" } + @{ name = "ReservedHours_s"; type = "string" } + @{ name = "UsedHours_s"; type = "string" } + @{ name = "UsageDate_s"; type = "string" } + @{ name = "MinUtilPercentage_s"; type = "string" } + @{ name = "AvgUtilPercentage_s"; type = "string" } + @{ name = "MaxUtilPercentage_s"; type = "string" } + @{ name = "PurchasedQuantity_s"; type = "string" } + @{ name = "RemainingQuantity_s"; type = "string" } + @{ name = "TotalReservedQuantity_s"; type = "string" } + @{ name = "UsedQuantity_s"; type = "string" } + @{ name = "UtilizedPercentage_s"; type = "string" } + @{ name = "UtilTrend_s"; type = "string" } + @{ name = "Util1Days_s"; type = "string" } + @{ name = "Util7Days_s"; type = "string" } + @{ name = "Util30Days_s"; type = "string" } + @{ name = "Scope_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "CollectedDate_s"; type = "string" } + ) + + "AppServicePlansV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "SubscriptionGuid_g"; type = "string" } + @{ name = "ResourceGroupName_s"; type = "string" } + @{ name = "ZoneRedundant_s"; type = "string" } + @{ name = "Location_s"; type = "string" } + @{ name = "AppServicePlanName_s"; type = "string" } + @{ name = "InstanceId_s"; type = "string" } + @{ name = "Kind_s"; type = "string" } + @{ name = "SkuName_s"; type = "string" } + @{ name = "SkuTier_s"; type = "string" } + @{ name = "SkuCapacity_s"; type = "string" } + @{ name = "SkuFamily_s"; type = "string" } + @{ name = "SkuSize_s"; type = "string" } + @{ name = "ComputeMode_s"; type = "string" } + @{ name = "NumberOfWorkers_s"; type = "string" } + @{ name = "CurrentNumberOfWorkers_s"; type = "string" } + @{ name = "MaximumNumberOfWorkers_s"; type = "string" } + @{ name = "NumberOfSites_s"; type = "string" } + @{ name = "PlanName_s"; type = "string" } + @{ name = "Tags_s"; type = "string" } + @{ name = "StatusDate_s"; type = "string" } + ) + + "PricesheetV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "MeterID_g"; type = "string" } + @{ name = "MeterName_s"; type = "string" } + @{ name = "MeterCategory_s"; type = "string" } + @{ name = "MeterSubCategory_s"; type = "string" } + @{ name = "MeterRegion_s"; type = "string" } + @{ name = "UnitOfMeasure_s"; type = "string" } + @{ name = "PartNumber_s"; type = "string" } + @{ name = "UnitPrice_s"; type = "string" } + @{ name = "CurrencyCode_s"; type = "string" } + @{ name = "IncludedQuantity_s"; type = "string" } + @{ name = "OfferId_s"; type = "string" } + @{ name = "Term_s"; type = "string" } + @{ name = "PriceType_s"; type = "string" } + ) + + "ReservationsPriceV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "productName_s"; type = "string" } + @{ name = "serviceName_s"; type = "string" } + @{ name = "skuName_s"; type = "string" } + @{ name = "skuId_s"; type = "string" } + @{ name = "armRegionName_s"; type = "string" } + @{ name = "location_s"; type = "string" } + @{ name = "effectiveStartDate_s"; type = "string" } + @{ name = "effectiveEndDate_s"; type = "string" } + @{ name = "unitPrice_s"; type = "string" } + @{ name = "currencyCode_s"; type = "string" } + @{ name = "unitOfMeasure_s"; type = "string" } + @{ name = "armSkuName_s"; type = "string" } + @{ name = "productId_s"; type = "string" } + @{ name = "reservationTerm_s"; type = "string" } + @{ name = "meterName_s"; type = "string" } + ) + + "SavingsPlansUsageV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "SavingsPlanResourceId_s"; type = "string" } + @{ name = "SavingsPlanOrderId_s"; type = "string" } + @{ name = "SavingsPlanId_g"; type = "string" } + @{ name = "DisplayName_s"; type = "string" } + @{ name = "SKUName_s"; type = "string" } + @{ name = "Term_s"; type = "string" } + @{ name = "ProvisioningState_s"; type = "string" } + @{ name = "AppliedScopeType_s"; type = "string" } + @{ name = "RenewState_s"; type = "string" } + @{ name = "PurchaseDate_s"; type = "string" } + @{ name = "BenefitStart_s"; type = "string" } + @{ name = "ExpiryDate_s"; type = "string" } + @{ name = "EffectiveDate_s"; type = "string" } + @{ name = "BillingScopeId_s"; type = "string" } + @{ name = "BillingAccountId_s"; type = "string" } + @{ name = "BillingProfileId_s"; type = "string" } + @{ name = "BillingPlan_s"; type = "string" } + @{ name = "CommitmentGrain_s"; type = "string" } + @{ name = "CommitmentCurrencyCode_s"; type = "string" } + @{ name = "CommitmentAmount_s"; type = "string" } + @{ name = "UtilTrend_s"; type = "string" } + @{ name = "Util1Days_s"; type = "string" } + @{ name = "Util7Days_s"; type = "string" } + @{ name = "Util30Days_s"; type = "string" } + @{ name = "Scope_s"; type = "string" } + @{ name = "TenantGuid_g"; type = "string" } + @{ name = "Cloud_s"; type = "string" } + @{ name = "CollectedDate_s"; type = "string" } + ) + + "SuppressionsV1" = @( + @{ name = "TimeGenerated"; type = "datetime" } + @{ name = "FilterId_g"; type = "string" } + @{ name = "RecommendationSubTypeId_g"; type = "string" } + @{ name = "FilterType_s"; type = "string" } + @{ name = "InstanceId_g"; type = "string" } + @{ name = "InstanceName_s"; type = "string" } + @{ name = "FilterStartDate_t"; type = "datetime" } + @{ name = "FilterEndDate_t"; type = "datetime" } + @{ name = "Author_s"; type = "string" } + @{ name = "Notes_s"; type = "string" } + ) +} + +# Storage container -> LogAnalyticsSuffix mapping used to update SQL control table +$containerSuffixMap = @{ + "argvmexports" = "VMsV1" + "argdiskexports" = "DisksV1" + "argvhdexports" = "VhdDisksV1" + "argavailsetexports" = "AvailSetsV1" + "advisorexports" = "AdvisorV1" + "remediationlogs" = "RemediationV1" + "consumptionexports" = "ConsumptionV1" + "aadobjectsexports" = "AADObjectsV1" + "arglbexports" = "LoadBalancersV1" + "argappgwexports" = "AppGatewaysV1" + "argrescontainersexports" = "ResourceContainersV1" + "rbacexports" = "RBACAssignmentsV1" + "argvnetexports" = "VNetsV1" + "argnicexports" = "NICsV1" + "argnsgexports" = "NSGsV1" + "argpublicipexports" = "PublicIPsV1" + "argvmssexports" = "VMSSV1" + "argsqldbexports" = "SqlDbV1" + "azmonitorexports" = "MonitorMetricsV1" + "policystateexports" = "PolicyStatesV1" + "recommendationsexports" = "RecommendationsV1" + "reservationsexports" = "ReservationsUsageV1" + "argappserviceplanexports" = "AppServicePlansV1" + "pricesheetexports" = "PricesheetV1" + "reservationspriceexports" = "ReservationsPriceV1" + "savingsplansexports" = "SavingsPlansUsageV1" +} +#endregion + +$lognamePrefix = Get-AzAutomationVariable -ResourceGroupName $ResourceGroupName -AutomationAccountName $AutomationAccountName ` + -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue +if (($null -eq $lognamePrefix -or [string]::IsNullOrEmpty($lognamePrefix.Value))) +{ + $lognamePrefix = "AzureOptimization" +} +else +{ + $lognamePrefix = $lognamePrefix.Value +} + +$dcrApiVersion = "2022-06-01" +$tableSchemaApiVersion = "2023-01-01-preview" # Tables API (GET/PUT schema) +$tableMigrateApiVersion = "2021-12-01-preview" # Tables migrate endpoint + +#region Get Log Analytics workspace resource ID in the target subscription context +if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) +{ + Write-Host "Switching to workspace subscription $WorkspaceSubscriptionId..." -ForegroundColor Cyan + Set-AzContext -SubscriptionId $WorkspaceSubscriptionId | Out-Null +} +#endregion + +#region Create/update custom Log Analytics tables and DCRs +Write-Host "Processing $($tableSchemas.Keys.Count) custom Log Analytics tables and DCRs..." -ForegroundColor Green + +$existingTables = Get-AzOperationalInsightsTable -ResourceGroupName $WorkspaceResourceGroupName -WorkspaceName $WorkspaceName + +$dcrSuffixToImmutableId = @{} + +#region Assign Monitoring Metrics Publisher role at resource group scope +Write-Host "Granting Monitoring Metrics Publisher on resource group to Automation MI..." -ForegroundColor Green + +if ($automationPrincipalId) +{ + # Role assignment must be created in the DCR subscription/resource group context + if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) + { + Set-AzContext -SubscriptionId $currentSubscriptionId | Out-Null + } + + $dcrResourceGroupScope = "/subscriptions/$currentSubscriptionId/resourceGroups/$ResourceGroupName" + $existingAssignment = Get-AzRoleAssignment -ObjectId $automationPrincipalId ` + -RoleDefinitionId $monitoringMetricsPublisherRoleId ` + -Scope $dcrResourceGroupScope -ErrorAction SilentlyContinue + if ($null -eq $existingAssignment) + { + New-AzRoleAssignment -ObjectId $automationPrincipalId ` + -RoleDefinitionId $monitoringMetricsPublisherRoleId ` + -Scope $dcrResourceGroupScope | Out-Null + Write-Host " Role assigned on resource group scope." -ForegroundColor Gray + } + else + { + Write-Host " Role already assigned on resource group scope." -ForegroundColor Gray + } +} +else +{ + Write-Host " Automation MI not found; skipping role assignment." -ForegroundColor Yellow +} + +# Restore workspace subscription context for table operations +if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) +{ + Set-AzContext -SubscriptionId $WorkspaceSubscriptionId | Out-Null +} +#endregion + +foreach ($suffix in $tableSchemas.Keys) +{ + $tableName = "$lognamePrefix$suffix" + "_CL" + $streamName = "Custom-$lognamePrefix$suffix" + $dcrName = "AOE-DCR-$suffix" + + Write-Host " [$suffix] Setting up table $tableName..." -ForegroundColor Cyan + + #region Build table columns payload + $columns = @() + foreach ($col in $tableSchemas[$suffix]) + { + $columns += @{ name = $col.name; type = $col.type } + } + + # Build source stream columns from table schema columns by removing the legacy type suffix + # (_s, _g, _t, _d, _b). This keeps table column names backward-compatible while accepting + # CSV/source payload fields without suffixes. + $sourceColumnsByName = [ordered]@{} + $destinationToSource = @{} + foreach ($col in $tableSchemas[$suffix]) + { + $sourceName = $col.name + if ($sourceName -match '^(.*)_[sgtdb]$') + { + $sourceName = $Matches[1] + } + + if (-not $sourceColumnsByName.Contains($sourceName)) + { + $sourceColumnsByName[$sourceName] = @{ name = $sourceName; type = $col.type } + } + + $destinationToSource[$col.name] = $sourceName + } + + $sourceColumns = @($sourceColumnsByName.Values) + + $tablePayload = @{ + properties = @{ + schema = @{ + name = $tableName + columns = $columns + } + } + } | ConvertTo-Json -Depth 10 + + $existingTable = $null + try + { + Write-Host " Checking if table $tableName exists and its type..." -ForegroundColor Yellow + $existingTable = $existingTables | Where-Object { $_.Name -eq $tableName } + } + catch + { + # Table does not yet exist; the PUT below will create it as a DCR-based table. + } + + if ($null -ne $existingTable -and $existingTable.Schema.TableSubType -eq "Classic") + { + # Migrate the classic table to DCR-based before updating the schema. + # This call is idempotent; it has no effect if the table has already been converted. + Write-Host " Table $tableName is a classic (Data Collector API) table. Migrating to DCR-based..." -ForegroundColor Yellow + $migrateUri = "$armEndpoint$workspaceResourceId/tables/$tableName/migrate?api-version=$tableMigrateApiVersion" + Invoke-AzRestMethod -Method POST -Uri $migrateUri -Payload "{}" | Out-Null + Write-Host " Migration completed for $tableName." -ForegroundColor Gray + } + + # Build the DCR transform KQL. + # Stream declarations use unsuffixed source fields, while output maps to suffixed table columns. + # For existing tables whose _g columns are already typed as Guid (created by the legacy Data + # Collector API), cast those specific outputs with toguid() so the transformed output matches + # the existing table schema. + $guidColumnNames = @() + if ($null -ne $existingTable) + { + $guidColumns = @($existingTable.Schema.Columns | Where-Object { $_.Type -eq "Guid" -and $_.Name -in $columns.name }) + if ($guidColumns.Count -gt 0) + { + $guidColumnNames = @($guidColumns | ForEach-Object { $_.Name }) + Write-Host " Existing guid columns detected ($($guidColumns.Count)). Using toguid() transform." -ForegroundColor Gray + } + } + + $projectExpressions = @() + foreach ($col in $columns) + { + $destinationName = $col.name + $sourceName = $destinationToSource[$destinationName] + if ($destinationName -in $guidColumnNames) + { + $projectExpressions += "$destinationName = toguid($sourceName)" + } + else + { + $projectExpressions += "$destinationName = $sourceName" + } + } + $transformKql = "source | project " + ($projectExpressions -join ", ") + + # Only create/update the table schema for new (non-existing) tables. + # Existing tables preserve their schema to avoid column type conflicts + # (e.g. Guid columns cannot be changed to string). + $tablePath = "/subscriptions/$WorkspaceSubscriptionId/resourceGroups/$WorkspaceResourceGroupName/providers/Microsoft.OperationalInsights/workspaces/$WorkspaceName/tables/$tableName`?api-version=$tableSchemaApiVersion" + if ($null -eq $existingTable) + { + $tableResponse = Invoke-AzRestMethod -Method PUT -Path $tablePath -Payload $tablePayload + Write-Host " Table $tableName created with status code $($tableResponse.StatusCode)." -ForegroundColor Gray + if ($tableResponse.StatusCode -ne 200 -and $tableResponse.StatusCode -ne 201) + { + $errorMessage = $tableResponse.Content | ConvertFrom-Json | Select-Object -ExpandProperty error | Select-Object -ExpandProperty message + throw " Failed to create table ${tableName}: ${errorMessage}" + } + } + else + { + Write-Host " Table $tableName already exists, skipping schema update." -ForegroundColor Gray + } + #endregion + + #region Create DCR for this table + Write-Host " [$suffix] Creating DCR $dcrName..." -ForegroundColor Cyan + + $dcrPayload = @{ + location = $workspaceLocation + properties = @{ + dataCollectionEndpointId = $dceResourceId + streamDeclarations = @{ + $streamName = @{ + columns = $sourceColumns + } + } + destinations = @{ + logAnalytics = @( + @{ + workspaceResourceId = $workspaceResourceId + name = "laDest" + } + ) + } + dataFlows = @( + @{ + streams = @($streamName) + destinations = @("laDest") + transformKql = $transformKql + outputStream = "Custom-$tableName" + } + ) + } + } | ConvertTo-Json -Depth 10 + + # Switch context back to automation account subscription for DCR creation + if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) + { + Set-AzContext -SubscriptionId $currentSubscriptionId | Out-Null + } + + $dcrUri = "$armEndpoint/subscriptions/$currentSubscriptionId/resourceGroups/$ResourceGroupName/providers/Microsoft.Insights/dataCollectionRules/$dcrName`?api-version=$dcrApiVersion" + $dcrResponse = Invoke-AzRestMethod -Method PUT -Uri $dcrUri -Payload $dcrPayload + Write-Host " DCR $dcrName creation request completed with status code $($dcrResponse.StatusCode)." -ForegroundColor Gray + if ($dcrResponse.StatusCode -ne 200 -and $dcrResponse.StatusCode -ne 201) + { + $errorMessage = $dcrResponse.Content | ConvertFrom-Json | Select-Object -ExpandProperty error | Select-Object -ExpandProperty message + throw " Failed to create DCR ${dcrName}: ${errorMessage}" + } + else + { + $dcrResponseContent = $dcrResponse.Content | ConvertFrom-Json + } + Write-Host " DCR $dcrName created." -ForegroundColor Gray + $dcrImmutableId = $dcrResponseContent.properties.immutableId + $dcrSuffixToImmutableId[$suffix] = $dcrImmutableId + Write-Host " DCR immutable ID: $dcrImmutableId" -ForegroundColor Gray + + # Switch back to workspace subscription if needed for subsequent table operations + if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) + { + Set-AzContext -SubscriptionId $WorkspaceSubscriptionId | Out-Null + } + #endregion + +} +#endregion + +#region Restore context to current subscription +if ($WorkspaceSubscriptionId -ne $currentSubscriptionId) +{ + Set-AzContext -SubscriptionId $currentSubscriptionId | Out-Null +} +#endregion + +#region Update LogAnalyticsIngestControl SQL table with DCR immutable IDs +Write-Host "Updating SQL LogAnalyticsIngestControl with DCR immutable IDs..." -ForegroundColor Green + +switch ($CloudEnvironment) +{ + "AzureChinaCloud" { $azureSqlDomain = "database.chinacloudapi.cn" } + "AzureUSGovernment" { $azureSqlDomain = "database.usgovcloudapi.net" } + default { $azureSqlDomain = "database.windows.net" } +} +$sqlToken = (Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" -AsSecureString).Token | ConvertFrom-SecureString -AsPlainText + +$sqlConnectionString = "Server=tcp:$SqlServerName,1433;Database=$SqlDatabaseName;Encrypt=True;Connection Timeout=120;" + +try +{ + $sqlConnection = New-Object System.Data.SqlClient.SqlConnection + $sqlConnection.ConnectionString = $sqlConnectionString + $sqlConnection.AccessToken = $sqlToken + $sqlConnection.Open() + + foreach ($container in $containerSuffixMap.Keys) + { + $suffix = $containerSuffixMap[$container] + if ($dcrSuffixToImmutableId.ContainsKey($suffix)) + { + $immutableId = $dcrSuffixToImmutableId[$suffix] + $sqlCommand = $sqlConnection.CreateCommand() + $sqlCommand.CommandText = "UPDATE [dbo].[LogAnalyticsIngestControl] SET DCRImmutableId = @id WHERE StorageContainerName = @container" + $sqlCommand.Parameters.AddWithValue("@id", $immutableId) | Out-Null + $sqlCommand.Parameters.AddWithValue("@container", $container) | Out-Null + $sqlCommand.ExecuteNonQuery() | Out-Null + } + } + + # Update the SuppressionsV1 DCR immutable ID (not in container map, keyed directly) + if ($dcrSuffixToImmutableId.ContainsKey("SuppressionsV1")) + { + # Suppressions are not blob-based; store the immutable ID under a placeholder container name + $suppImmutableId = $dcrSuffixToImmutableId["SuppressionsV1"] + $sqlCommand = $sqlConnection.CreateCommand() + $sqlCommand.CommandText = @" +IF NOT EXISTS (select 1 FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'suppressions') +BEGIN +INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType, DCRImmutableId) +VALUES ('suppressions', '1901-01-01T00:00:00Z', -1, 'SuppressionsV1', 'Suppressions', @id) +END +ELSE +BEGIN +UPDATE [dbo].[LogAnalyticsIngestControl] SET DCRImmutableId = @id WHERE StorageContainerName = 'suppressions' +END +"@ + $sqlCommand.Parameters.AddWithValue("@id", $suppImmutableId) | Out-Null + $sqlCommand.ExecuteNonQuery() | Out-Null + } + + $sqlConnection.Close() + Write-Host "SQL update completed." -ForegroundColor Green +} +catch +{ + Write-Host "Could not update SQL control table: $($_.Exception.Message)" -ForegroundColor Yellow + Write-Host "You may need to update the DCRImmutableId column manually or ensure the managed identity has db_datareader/db_datawriter access." -ForegroundColor Yellow +} +#endregion + +#region Remove the legacy workspace shared key automation variable +Write-Host "Removing legacy AzureOptimization_LogAnalyticsWorkspaceKey automation variable..." -ForegroundColor Green +$keyVar = Get-AzAutomationVariable -ResourceGroupName $ResourceGroupName -AutomationAccountName $AutomationAccountName ` + -Name "AzureOptimization_LogAnalyticsWorkspaceKey" -ErrorAction SilentlyContinue +if ($null -ne $keyVar) +{ + Remove-AzAutomationVariable -ResourceGroupName $ResourceGroupName -AutomationAccountName $AutomationAccountName ` + -Name "AzureOptimization_LogAnalyticsWorkspaceKey" + Write-Host "Legacy workspace key variable removed." -ForegroundColor Green +} +else +{ + Write-Host "Variable already absent." -ForegroundColor Gray +} +#endregion + +Write-Host "Setup-LogAnalyticsTablesAndDCRs completed successfully." -ForegroundColor Green +Write-Host " Tables created: $($tableSchemas.Keys.Count)" -ForegroundColor Green +Write-Host " DCRs created/updated: $($dcrSuffixToImmutableId.Keys.Count)" -ForegroundColor Green diff --git a/src/optimization-engine/azuredeploy-nested.bicep b/src/optimization-engine/azuredeploy-nested.bicep index 3f9032fa8..61e881b6e 100644 --- a/src/optimization-engine/azuredeploy-nested.bicep +++ b/src/optimization-engine/azuredeploy-nested.bicep @@ -257,7 +257,7 @@ var csvExportsSchedules = [ { exportSchedule: priceExportsScheduleName exportDescription: 'Weekly Pricesheet and Reservation Prices exports' - exportTimeOffset: 'PT1H35M' + exportTimeOffset: 'PT1H05M' exportFrequency: 'Week' } { @@ -562,7 +562,7 @@ var csvExports = [ variableDescription: 'The Storage Account container where Pricesheet exports are dumped to' ingestSchedule: 'AzureOptimization_IngestPricesheetWeekly' ingestDescription: 'Weekly Pricesheet ingests' - ingestTimeOffset: 'PT2H' + ingestTimeOffset: 'PT1H35M' ingestFrequency: 'Week' ingestJobId: pricesheetIngestJobId exportSchedule: priceExportsScheduleName @@ -576,7 +576,7 @@ var csvExports = [ variableDescription: 'The Storage Account container where Reservations Prices exports are dumped to' ingestSchedule: 'AzureOptimization_IngestReservationsPriceWeekly' ingestDescription: 'Weekly Reservations Prices ingests' - ingestTimeOffset: 'PT2H' + ingestTimeOffset: 'PT1H35M' ingestFrequency: 'Week' ingestJobId: reservationPricesIngestJobId exportSchedule: priceExportsScheduleName @@ -949,7 +949,7 @@ var runbooks = [ } { name: consumptionExportsRunbookName - version: '2.1.1.0' + version: '2.1.2.0' description: 'Exports Azure Consumption events to Blob Storage using Azure Consumption API' type: 'PowerShell' scriptUri: uri(templateLocation, 'runbooks/data-collection/${consumptionExportsRunbookName}.ps1') @@ -1075,7 +1075,7 @@ var runbooks = [ } { name: csvIngestRunbookName - version: '1.6.2.0' + version: '2.0.0.0' description: 'Ingests CSV blobs as custom logs to Log Analytics' type: 'PowerShell' scriptUri: uri(templateLocation, 'runbooks/data-collection/${csvIngestRunbookName}.ps1') @@ -1187,21 +1187,21 @@ var runbooks = [ } { name: recommendationsIngestRunbookName - version: '1.7.1.0' + version: '1.7.2.0' description: 'Ingests JSON-based recommendations into an Azure SQL Database' type: 'PowerShell' scriptUri: uri(templateLocation, 'runbooks/recommendations/${recommendationsIngestRunbookName}.ps1') } { name: recommendationsLogAnalyticsIngestRunbookName - version: '1.1.1.0' + version: '2.0.0.0' description: 'Ingests JSON-based recommendations into Log Analytics' type: 'PowerShell' scriptUri: uri(templateLocation, 'runbooks/recommendations/${recommendationsLogAnalyticsIngestRunbookName}.ps1') } { name: suppressionsLogAnalyticsIngestRunbookName - version: '1.1.0.0' + version: '2.0.0.0' description: 'Ingests suppressions into Log Analytics' type: 'PowerShell' scriptUri: uri(templateLocation, 'runbooks/recommendations/${suppressionsLogAnalyticsIngestRunbookName}.ps1') @@ -1274,7 +1274,7 @@ var automationVariables = [ { name: 'AzureOptimization_LogAnalyticsChunkSize' description: 'The size (in rows) for each chunk of Log Analytics ingestion request' - value: 6000 + value: 150 } { name: 'AzureOptimization_StorageBlobsPageSize' @@ -1570,6 +1570,17 @@ resource logAnalyticsWorkspace 'microsoft.operationalinsights/workspaces@2020-08 } } +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2022-06-01' = { + name: '${automationAccountName}-dce' + location: projectLocation + tags: resourceTags + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } +} + resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: storageAccountName location: projectLocation @@ -1893,6 +1904,15 @@ resource automationVariables_LogAnalyticsWorkspaceKey 'Microsoft.Automation/auto } } +resource automationVariables_DCEIngestionEndpoint 'Microsoft.Automation/automationAccounts/variables@2020-01-13-preview' = { + parent: automationAccount + name: 'AzureOptimization_DCEIngestionEndpoint' + properties: { + description: 'The Logs Ingestion endpoint URL of the Data Collection Endpoint used for DCR-based ingestion' + value: '"${dataCollectionEndpoint.properties.logsIngestion.endpoint}"' + } +} + resource automationSchedules_csvExports 'Microsoft.Automation/automationAccounts/schedules@2020-01-13-preview' = [for item in csvExportsSchedules: { parent: automationAccount name: item.exportSchedule @@ -2156,3 +2176,5 @@ resource contributorRoleAssignmentGuid_resource 'Microsoft.Authorization/roleAss } output automationPrincipalId string = reference(automationAccount.id, '2019-06-01', 'Full').identity.principalId +output dceLogsIngestionEndpoint string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output dceResourceId string = dataCollectionEndpoint.id diff --git a/src/optimization-engine/model/loganalyticsingestcontrol-initialize.sql b/src/optimization-engine/model/loganalyticsingestcontrol-initialize.sql index b8b931c0c..f84933e98 100644 --- a/src/optimization-engine/model/loganalyticsingestcontrol-initialize.sql +++ b/src/optimization-engine/model/loganalyticsingestcontrol-initialize.sql @@ -1,155 +1,155 @@ IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvmexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argvmexports', '1901-01-01T00:00:00Z', -1, 'VMsV1', 'ARGVirtualMachine') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argdiskexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argdiskexports', '1901-01-01T00:00:00Z', -1, 'DisksV1', 'ARGManagedDisk') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvhdexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argvhdexports', '1901-01-01T00:00:00Z', -1, 'VhdDisksV1', 'ARGUnmanagedDisk') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argavailsetexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argavailsetexports', '1901-01-01T00:00:00Z', -1, 'AvailSetsV1', 'ARGAvailabilitySet') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'advisorexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('advisorexports', '1901-01-01T00:00:00Z', -1, 'AdvisorV1', 'AzureAdvisor') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'remediationlogs') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('remediationlogs', '1901-01-01T00:00:00Z', -1, 'RemediationV1', 'RemediationLogs') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'consumptionexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('consumptionexports', '1901-01-01T00:00:00Z', -1, 'ConsumptionV1', 'AzureConsumption') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'aadobjectsexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('aadobjectsexports', '1901-01-01T00:00:00Z', -1, 'AADObjectsV1', 'AADObjects') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'arglbexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('arglbexports', '1901-01-01T00:00:00Z', -1, 'LoadBalancersV1', 'ARGLoadBalancer') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argappgwexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argappgwexports', '1901-01-01T00:00:00Z', -1, 'AppGatewaysV1', 'ARGAppGateway') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argrescontainersexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argrescontainersexports', '1901-01-01T00:00:00Z', -1, 'ResourceContainersV1', 'ARGResourceContainers') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'rbacexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('rbacexports', '1901-01-01T00:00:00Z', -1, 'RBACAssignmentsV1', 'RBACAssignments') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvnetexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argvnetexports', '1901-01-01T00:00:00Z', -1, 'VNetsV1', 'ARGVirtualNetwork') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argnicexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argnicexports', '1901-01-01T00:00:00Z', -1, 'NICsV1', 'ARGNetworkInterface') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argnsgexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argnsgexports', '1901-01-01T00:00:00Z', -1, 'NSGsV1', 'ARGNSGRule') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argpublicipexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argpublicipexports', '1901-01-01T00:00:00Z', -1, 'PublicIPsV1', 'ARGPublicIP') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argvmssexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argvmssexports', '1901-01-01T00:00:00Z', -1, 'VMSSV1', 'ARGVMSS') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argsqldbexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argsqldbexports', '1901-01-01T00:00:00Z', -1, 'SqlDbV1', 'ARGSqlDb') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'azmonitorexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('azmonitorexports', '1901-01-01T00:00:00Z', -1, 'MonitorMetricsV1', 'MonitorMetrics') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'policystateexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('policystateexports', '1901-01-01T00:00:00Z', -1, 'PolicyStatesV1', 'PolicyStates') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'recommendationsexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('recommendationsexports', '2022-12-26T00:00:00Z', -1, 'RecommendationsV1', 'Recommendations') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'reservationsexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('reservationsexports', '1901-01-01T00:00:00Z', -1, 'ReservationsUsageV1', 'ReservationsUsage') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'argappserviceplanexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('argappserviceplanexports', '1901-01-01T00:00:00Z', -1, 'AppServicePlansV1', 'AppServicePlans') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'pricesheetexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('pricesheetexports', '1901-01-01T00:00:00Z', -1, 'PricesheetV1', 'Pricesheet') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'reservationspriceexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('reservationspriceexports', '1901-01-01T00:00:00Z', -1, 'ReservationsPriceV1', 'ReservationsPrice') END IF NOT EXISTS (SELECT * FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'savingsplansexports') BEGIN - INSERT INTO [dbo].[LogAnalyticsIngestControl] + INSERT INTO [dbo].[LogAnalyticsIngestControl] (StorageContainerName, LastProcessedDateTime, LastProcessedLine, LogAnalyticsSuffix, CollectedType) VALUES ('savingsplansexports', '1901-01-01T00:00:00Z', -1, 'SavingsPlansUsageV1', 'SavingsPlansUsage') END diff --git a/src/optimization-engine/model/loganalyticsingestcontrol-table.sql b/src/optimization-engine/model/loganalyticsingestcontrol-table.sql index 23d87d061..81f2541c4 100644 --- a/src/optimization-engine/model/loganalyticsingestcontrol-table.sql +++ b/src/optimization-engine/model/loganalyticsingestcontrol-table.sql @@ -8,7 +8,8 @@ AND OBJECTPROPERTY(id, N'IsUserTable') = 1) [LastProcessedDateTime] [datetime] NULL, [LastProcessedLine] [int] NULL, [LogAnalyticsSuffix] [varchar](50) NOT NULL, - [CollectedType] [varchar](50) NULL + [CollectedType] [varchar](50) NULL, + [DCRImmutableId] [varchar](100) NULL ) ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD PRIMARY KEY CLUSTERED @@ -18,8 +19,12 @@ AND OBJECTPROPERTY(id, N'IsUserTable') = 1) END ELSE BEGIN - IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'CollectedType' -) BEGIN + IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'CollectedType') + BEGIN ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD [CollectedType] VARCHAR (50) NULL END + IF NOT EXISTS (SELECT * FROM sys.columns WHERE object_id = OBJECT_ID(N'[dbo].[LogAnalyticsIngestControl]') AND name = 'DCRImmutableId') + BEGIN + ALTER TABLE [dbo].[LogAnalyticsIngestControl] ADD [DCRImmutableId] VARCHAR (100) NULL + END END \ No newline at end of file diff --git a/src/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 b/src/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 index 11dabcb0d..cda9b13ab 100644 --- a/src/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 +++ b/src/optimization-engine/runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1 @@ -22,26 +22,32 @@ $ErrorActionPreference = "Stop" $global:hadErrors = $false $global:scopesWithErrors = @() -function Authenticate-AzureWithOption { +function Authenticate-AzureWithOption +{ param ( [string] $authOption = "ManagedIdentity", [string] $cloudEnv = "AzureCloud", [string] $clientID ) - switch ($authOption) { - "UserAssignedManagedIdentity" { + switch ($authOption) + { + "UserAssignedManagedIdentity" + { Connect-AzAccount -Identity -EnvironmentName $cloudEnv -AccountId $clientID break } - Default { #ManagedIdentity + Default + { + #ManagedIdentity Connect-AzAccount -Identity -EnvironmentName $cloudEnv break } } } -function Generate-CostDetails { +function Generate-CostDetails +{ param ( [string] $ScopeId, [string] $ScopeName @@ -54,7 +60,7 @@ function Generate-CostDetails { $body = "{ `"metric`": `"$consumptionMetric`", `"timePeriod`": { `"start`": `"$targetStartDate`", `"end`": `"$targetEndDate`" } }" $result = Invoke-AzRestMethod -Path $CostDetailsApiPath -Method POST -Payload $body $requestResultPath = $result.Headers.Location.PathAndQuery - if ($result.StatusCode -in (200,202)) + if ($result.StatusCode -in (200, 202)) { $tries = 0 $requestSuccess = $false @@ -103,53 +109,54 @@ function Generate-CostDetails { # header normalization between MCA and EA $headerConversion = @{ - additionalInfo = "AdditionalInfo"; - billingAccountId = "BillingAccountId"; - billingAccountName = "BillingAccountName"; - billingCurrency = "BillingCurrencyCode"; - billingPeriodEndDate = "BillingPeriodEndDate"; - billingPeriodStartDate = "BillingPeriodStartDate"; - billingProfileId = "BillingProfileId"; - billingProfileName = "BillingProfileName"; - chargeType = "ChargeType"; - consumedService = "ConsumedService"; - costAllocationRuleName = "CostAllocationRuleName"; - costCenter = "CostCenter"; - costInBillingCurrency = "CostInBillingCurrency"; - date = "Date"; - effectivePrice = "EffectivePrice"; - frequency = "Frequency"; - invoiceSectionId = "InvoiceSectionId"; - invoiceSectionName = "InvoiceSectionName"; - isAzureCreditEligible = "IsAzureCreditEligible"; - meterCategory = "MeterCategory"; - meterId = "MeterId"; - meterName = "MeterName"; - meterRegion = "MeterRegion"; - meterSubCategory = "MeterSubCategory"; - offerId = "OfferId"; - pricingModel = "PricingModel"; - productOrderId = "ProductOrderId"; - productOrderName = "ProductOrderName"; - publisherName = "PublisherName"; - publisherType = "PublisherType"; - quantity = "Quantity"; - reservationId = "ReservationId"; - reservationName = "ReservationName"; - resourceGroupName = "ResourceGroup"; - resourceLocation = "ResourceLocation"; - serviceFamily = "ServiceFamily"; - serviceInfo1 = "ServiceInfo1"; - serviceInfo2 = "ServiceInfo2"; - subscriptionName = "SubscriptionName"; - tags = "Tags"; - term = "Term"; - unitOfMeasure = "UnitOfMeasure"; - unitPrice = "UnitPrice" + additionalInfo = "AdditionalInfo" + billingAccountId = "BillingAccountId" + billingAccountName = "BillingAccountName" + billingCurrency = "BillingCurrencyCode" + billingPeriodEndDate = "BillingPeriodEndDate" + billingPeriodStartDate = "BillingPeriodStartDate" + billingProfileId = "BillingProfileId" + billingProfileName = "BillingProfileName" + chargeType = "ChargeType" + consumedService = "ConsumedService" + costAllocationRuleName = "CostAllocationRuleName" + costCenter = "CostCenter" + costInBillingCurrency = "CostInBillingCurrency" + date = "Date" + effectivePrice = "EffectivePrice" + frequency = "Frequency" + invoiceSectionId = "InvoiceSectionId" + invoiceSectionName = "InvoiceSectionName" + isAzureCreditEligible = "IsAzureCreditEligible" + meterCategory = "MeterCategory" + meterId = "MeterId" + meterName = "MeterName" + meterRegion = "MeterRegion" + meterSubCategory = "MeterSubCategory" + offerId = "OfferId" + pricingModel = "PricingModel" + productOrderId = "ProductOrderId" + productOrderName = "ProductOrderName" + publisherName = "PublisherName" + publisherType = "PublisherType" + quantity = "Quantity" + reservationId = "ReservationId" + reservationName = "ReservationName" + resourceGroupName = "ResourceGroup" + resourceLocation = "ResourceLocation" + serviceFamily = "ServiceFamily" + serviceInfo1 = "ServiceInfo1" + serviceInfo2 = "ServiceInfo2" + subscriptionName = "SubscriptionName" + tags = "Tags" + term = "Term" + unitOfMeasure = "UnitOfMeasure" + unitPrice = "UnitPrice" } $lineCounter = 0 - while ($r.Peek() -ge 0) { + while ($r.Peek() -ge 0) + { $line = $r.ReadLine() $lineCounter++ if ($lineCounter -eq 1) @@ -178,7 +185,7 @@ function Generate-CostDetails { $w.Close() $csvBlobName = [System.IO.Path]::GetFileName($finalCsvExportPath) - $csvProperties = @{"ContentType" = "text/csv"}; + $csvProperties = @{"ContentType" = "text/csv" } Set-AzStorageBlobContent -File $finalCsvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") @@ -414,12 +421,12 @@ else # for each subscription, get billing data -$datetime = (get-date).ToUniversalTime() +$datetime = (Get-Date).ToUniversalTime() $timestamp = $datetime.ToString("yyyy-MM-ddTHH:mm:00.000Z") if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($chinaEAEnrollment)) -and -not([string]::IsNullOrEmpty($chinaEAKey))) { - $targetMonth = $targetStartDate.Substring(0,7) + $targetMonth = $targetStartDate.Substring(0, 7) $consumption = $null $billingEntries = @() @@ -427,7 +434,7 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c $PricesheetApiUri = "https://ea.azure.cn/rest/$chinaEAEnrollment/usage-report?month=$targetMonth&type=pricesheet&fmt=Csv" $Headers = @{} - $Headers.Add("Authorization","Bearer $chinaEAKey") + $Headers.Add("Authorization", "Bearer $chinaEAKey") Write-Output "Getting pricesheet for month $targetMonth (EA enrollment $chinaEAEnrollment)..." @@ -443,12 +450,12 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c Write-Output "Removed pricesheet-$targetMonth.csv from local disk..." - $csvFile2 = $csvFile[2..($csvFile.Count-1)] + $csvFile2 = $csvFile[2..($csvFile.Count - 1)] $headerLine = $csvFile2[0] $columnHeaders = $headerLine.Split(",") for ($i = 0; $i -lt $columnHeaders.Count; $i++) { - if($columnHeaders[$i] -match '.+\((?.+)\)') + if ($columnHeaders[$i] -match '.+\((?.+)\)') { $columnHeaders[$i] = $Matches.ColumnName } @@ -465,7 +472,8 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c $requestSuccess = $false do { - try { + try + { $tries++ Invoke-RestMethod -Method Get -Uri $BillingApiUri -Headers $Headers -OutFile "usagedetails-$targetStartDate.csv" @@ -479,12 +487,12 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c Write-Output "Removed usagedetails-$targetStartDate.csv from local disk..." - $csvFile2 = $csvFile[2..($csvFile.Count-1)] + $csvFile2 = $csvFile[2..($csvFile.Count - 1)] $headerLine = $csvFile2[0] $columnHeaders = $headerLine.Split(",") for ($i = 0; $i -lt $columnHeaders.Count; $i++) { - if($columnHeaders[$i] -match '.+\((?.+)\)') + if ($columnHeaders[$i] -match '.+\((?.+)\)') { $columnHeaders[$i] = $Matches.ColumnName } @@ -496,7 +504,8 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c $consumption = $csvFile2 | ConvertFrom-Csv $requestSuccess = $true } - catch { + catch + { $ErrorMessage = $_.Exception.Message Write-Warning "Error getting consumption data: $ErrorMessage. $tries of 3 tries. Waiting 60 seconds..." Start-Sleep -s 60 @@ -523,7 +532,7 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c { $instanceId = $consumptionLine.'Instance ID'.ToLower() $idParts = $consumptionLine.'Instance ID'.Split("/") - $instanceName = $idParts[$idParts.Count-1].ToLower() + $instanceName = $idParts[$idParts.Count - 1].ToLower() } $rgName = $null @@ -569,31 +578,31 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c } $billingEntry = New-Object PSObject -Property @{ - Timestamp = $timestamp - SubscriptionId = $consumptionLine.SubscriptionGuid - ResourceGroup = $rgName - ResourceName = $instanceName - ResourceId = $instanceId - Date = $consumptionLine.Date - Tags = $consumptionLine.Tags - AdditionalInfo = $consumptionLine.AdditionalInfo - BillingCurrencyCode = "CNY" - ChargeType = "Usage" - ConsumedService = $consumptionLine.'Consumed Service' + Timestamp = $timestamp + SubscriptionId = $consumptionLine.SubscriptionGuid + ResourceGroup = $rgName + ResourceName = $instanceName + ResourceId = $instanceId + Date = $consumptionLine.Date + Tags = $consumptionLine.Tags + AdditionalInfo = $consumptionLine.AdditionalInfo + BillingCurrencyCode = "CNY" + ChargeType = "Usage" + ConsumedService = $consumptionLine.'Consumed Service' CostInBillingCurrency = $convertedCost - EffectivePrice = $convertedPrice - Frequency = "UsageBased" - MeterCategory = $consumptionLine.'Meter Category' - MeterId = $consumptionLine.'Meter ID' - MeterName = $consumptionLine.'Meter Name' - MeterSubCategory = $consumptionLine.'Meter Sub-Category' - PartNumber = $partNumber - ProductName = $consumptionLine.Product - Quantity = $consumptionLine.'Consumed Quantity' - UnitOfMeasure = $consumptionLine.'Unit of Measure' - UnitPrice = $unitPrice - ResourceLocation = $consumptionLine.'Resource Location' - AccountOwnerId = $consumptionLine.AccountOwnerId + EffectivePrice = $convertedPrice + Frequency = "UsageBased" + MeterCategory = $consumptionLine.'Meter Category' + MeterId = $consumptionLine.'Meter ID' + MeterName = $consumptionLine.'Meter Name' + MeterSubCategory = $consumptionLine.'Meter Sub-Category' + PartNumber = $partNumber + ProductName = $consumptionLine.Product + Quantity = $consumptionLine.'Consumed Quantity' + UnitOfMeasure = $consumptionLine.'Unit of Measure' + UnitPrice = $unitPrice + ResourceLocation = $consumptionLine.'Resource Location' + AccountOwnerId = $consumptionLine.AccountOwnerId } $billingEntries += $billingEntry @@ -612,7 +621,7 @@ if ($cloudEnvironment -eq "AzureChinaCloud" -and -not([string]::IsNullOrEmpty($c Write-Output "Exported $($billingEntries.Count) entries as CSV to $csvExportPath" $csvBlobName = $csvExportPath - $csvProperties = @{"ContentType" = "text/csv"}; + $csvProperties = @{"ContentType" = "text/csv" } Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force Write-Output "Uploaded to blob storage!" @@ -625,8 +634,8 @@ else { if ($consumptionScope -eq "Subscription") { - $CostDetailsSupportedQuotaIDs = @('EnterpriseAgreement_2014-09-01','Internal_2014-09-01','CSP_2015-05-01') - $ConsumptionSupportedQuotaIDs = @('PayAsYouGo_2014-09-01','MSDN_2014-09-01') + $CostDetailsSupportedQuotaIDs = @('PayAsYouGo_2014-09-01', 'EnterpriseAgreement_2014-09-01', 'Internal_2014-09-01', 'CSP_2015-05-01') + $ConsumptionSupportedQuotaIDs = @('MSDN_2014-09-01') foreach ($subscription in $subscriptions) { @@ -651,12 +660,14 @@ else $requestSuccess = $false do { - try { + try + { $tries++ $consumption = (Invoke-AzRestMethod -Path $ConsumptionApiPath -Method GET).Content | ConvertFrom-Json $requestSuccess = $true } - catch { + catch + { $ErrorMessage = $_.Exception.Message Write-Warning "Error getting consumption data: $ErrorMessage. $tries of 3 tries. Waiting 60 seconds..." Start-Sleep -s 60 @@ -676,126 +687,136 @@ else $tags = $null } - if ([string]::IsNullOrEmpty($consumptionLine.properties.billingProfileId)) + if ($consumptionLine.kind -eq "legacy") { # legacy consumption schema $billingEntry = New-Object PSObject -Property @{ - Timestamp = $timestamp - AccountName = $consumptionLine.properties.accountName - AccountOwnerId = $consumptionLine.properties.accountOwnerId - AdditionalInfo = $consumptionLine.properties.additionalInfo - benefitId = $consumptionLine.properties.benefitId - benefitName = $consumptionLine.properties.benefitName - BillingAccountId = $consumptionLine.properties.billingAccountId - BillingAccountName = $consumptionLine.properties.billingAccountName - BillingCurrencyCode = $consumptionLine.properties.billingCurrency - BillingPeriodEndDate= $consumptionLine.properties.billingPeriodEndDate - BillingPeriodStartDate= $consumptionLine.properties.billingPeriodStartDate - BillingProfileId = $consumptionLine.properties.billingProfileId - BillingProfileName= $consumptionLine.properties.billingProfileName - ChargeType = $consumptionLine.properties.chargeType - ConsumedService = $consumptionLine.properties.consumedService + Timestamp = $timestamp + AccountName = $consumptionLine.properties.accountName + AccountOwnerId = $consumptionLine.properties.accountOwnerId + AdditionalInfo = $consumptionLine.properties.additionalInfo + benefitId = $consumptionLine.properties.benefitId + benefitName = $consumptionLine.properties.benefitName + BillingAccountId = $consumptionLine.properties.billingAccountId + BillingAccountName = $consumptionLine.properties.billingAccountName + BillingCurrencyCode = $consumptionLine.properties.billingCurrency + BillingPeriodEndDate = $consumptionLine.properties.billingPeriodEndDate + BillingPeriodStartDate = $consumptionLine.properties.billingPeriodStartDate + BillingProfileId = $consumptionLine.properties.billingProfileId + BillingProfileName = $consumptionLine.properties.billingProfileName + ChargeType = $consumptionLine.properties.chargeType + ConsumedService = $consumptionLine.properties.consumedService CostAllocationRuleName = $consumptionLine.properties.costAllocationRuleName - CostCenter = $consumptionLine.properties.costCenter - CostInBillingCurrency = $consumptionLine.properties.cost - Date = (Get-Date $consumptionLine.properties.date).ToString("MM/dd/yyyy") - EffectivePrice = $consumptionLine.properties.effectivePrice - Frequency = $consumptionLine.properties.frequency - InvoiceSectionName = $consumptionLine.properties.invoiceSection - IsAzureCreditEligible = $consumptionLine.properties.isAzureCreditEligible - MeterCategory = $consumptionLine.properties.meterDetails.meterCategory - MeterId = $consumptionLine.properties.meterId - MeterName = $consumptionLine.properties.meterDetails.meterName - MeterRegion = $consumptionLine.properties.meterDetails.meterRegion - MeterSubCategory = $consumptionLine.properties.meterDetails.meterSubCategory - OfferId = $consumptionLine.properties.offerId - PartNumber = $consumptionLine.properties.partNumber - PayGPrice = $consumptionLine.properties.PayGPrice - PlanName = $consumptionLine.properties.planName - PricingModel = $consumptionLine.properties.pricingModel - ProductName = $consumptionLine.properties.product - PublisherName = $consumptionLine.properties.publisherName - PublisherType = $consumptionLine.properties.publisherType - Quantity = $consumptionLine.properties.quantity - ReservationId = $consumptionLine.properties.reservationId - ReservationName = $consumptionLine.properties.reservationName - ResourceGroup = $consumptionLine.properties.resourceGroup - ResourceId = $consumptionLine.properties.resourceId - ResourceLocation = $consumptionLine.properties.resourceLocation - ResourceName = $consumptionLine.properties.resourceName - ServiceFamily = $consumptionLine.properties.meterDetails.serviceFamily - SubscriptionId = $consumptionLine.properties.subscriptionId - SubscriptionName = $consumptionLine.properties.subscriptionName - Tags = $tags - Term = $consumptionLine.properties.term - UnitOfMeasure = $consumptionLine.properties.meterDetails.unitOfMeasure - UnitPrice = $consumptionLine.properties.unitPrice + CostCenter = $consumptionLine.properties.costCenter + CostInBillingCurrency = $consumptionLine.properties.cost + Date = (Get-Date $consumptionLine.properties.date).ToString("MM/dd/yyyy") + EffectivePrice = $consumptionLine.properties.effectivePrice + Frequency = $consumptionLine.properties.frequency + InvoiceSectionName = $consumptionLine.properties.invoiceSection + IsAzureCreditEligible = $consumptionLine.properties.isAzureCreditEligible + MeterCategory = $consumptionLine.properties.meterDetails.meterCategory + MeterId = $consumptionLine.properties.meterId + MeterName = $consumptionLine.properties.meterDetails.meterName + MeterRegion = $consumptionLine.properties.meterDetails.meterRegion + MeterSubCategory = $consumptionLine.properties.meterDetails.meterSubCategory + OfferId = $consumptionLine.properties.offerId + PartNumber = $consumptionLine.properties.partNumber + PayGPrice = $consumptionLine.properties.PayGPrice + PlanName = $consumptionLine.properties.planName + PricingModel = $consumptionLine.properties.pricingModel + ProductName = $consumptionLine.properties.product + PublisherName = $consumptionLine.properties.publisherName + PublisherType = $consumptionLine.properties.publisherType + Quantity = $consumptionLine.properties.quantity + ReservationId = $consumptionLine.properties.reservationId + ReservationName = $consumptionLine.properties.reservationName + ResourceGroup = $consumptionLine.properties.resourceGroup + ResourceId = $consumptionLine.properties.resourceId + ResourceLocation = $consumptionLine.properties.resourceLocation + ResourceName = $consumptionLine.properties.resourceName + ServiceFamily = $consumptionLine.properties.meterDetails.serviceFamily + SubscriptionId = $consumptionLine.properties.subscriptionId + SubscriptionName = $consumptionLine.properties.subscriptionName + Tags = $tags + Term = $consumptionLine.properties.term + UnitOfMeasure = $consumptionLine.properties.meterDetails.unitOfMeasure + UnitPrice = $consumptionLine.properties.unitPrice } } - else + elseif ($consumptionLine.kind -eq "modern") { # MCA consumption schema $billingEntry = New-Object PSObject -Property @{ - Timestamp = $timestamp - AdditionalInfo = $consumptionLine.properties.additionalInfo - benefitId = $consumptionLine.properties.benefitId - benefitName = $consumptionLine.properties.benefitName - BillingAccountId = $consumptionLine.properties.billingAccountId - BillingAccountName = $consumptionLine.properties.billingAccountName - BillingCurrencyCode = $consumptionLine.properties.billingCurrencyCode - BillingPeriodEndDate= $consumptionLine.properties.billingPeriodEndDate - BillingPeriodStartDate= $consumptionLine.properties.billingPeriodStartDate - BillingProfileId = $consumptionLine.properties.billingProfileId - BillingProfileName= $consumptionLine.properties.billingProfileName - ChargeType = $consumptionLine.properties.chargeType - ConsumedService = $consumptionLine.properties.consumedService - CostAllocationRuleName = $consumptionLine.properties.costAllocationRuleName - CostCenter = $consumptionLine.properties.costCenter - CostInBillingCurrency = $consumptionLine.properties.costInBillingCurrency - costInPricingCurrency = $consumptionLine.properties.costInPricingCurrency - costInUSD = $consumptionLine.properties.costInUSD - customerName = $consumptionLine.properties.customerName - Date = (Get-Date $consumptionLine.properties.date).ToString("MM/dd/yyyy") - EffectivePrice = $consumptionLine.properties.effectivePrice - exchangeRate = $consumptionLine.properties.exchangeRate - exchangeRateDate = $consumptionLine.properties.exchangeRateDate + Timestamp = $timestamp + AdditionalInfo = $consumptionLine.properties.additionalInfo + benefitId = $consumptionLine.properties.benefitId + benefitName = $consumptionLine.properties.benefitName + BillingAccountId = $consumptionLine.properties.billingAccountId + BillingAccountName = $consumptionLine.properties.billingAccountName + BillingCurrencyCode = $consumptionLine.properties.billingCurrencyCode + BillingPeriodEndDate = $consumptionLine.properties.billingPeriodEndDate + BillingPeriodStartDate = $consumptionLine.properties.billingPeriodStartDate + BillingProfileId = $consumptionLine.properties.billingProfileId + BillingProfileName = $consumptionLine.properties.billingProfileName + ChargeType = $consumptionLine.properties.chargeType + ConsumedService = $consumptionLine.properties.consumedService + CostAllocationRuleName = $consumptionLine.properties.costAllocationRuleName + CostCenter = $consumptionLine.properties.costCenter + CostInBillingCurrency = $consumptionLine.properties.costInBillingCurrency + costInPricingCurrency = $consumptionLine.properties.costInPricingCurrency + costInUSD = $consumptionLine.properties.costInUSD + customerName = $consumptionLine.properties.customerName + Date = (Get-Date $consumptionLine.properties.date).ToString("MM/dd/yyyy") + EffectivePrice = $consumptionLine.properties.effectivePrice + exchangeRate = $consumptionLine.properties.exchangeRate + exchangeRateDate = $consumptionLine.properties.exchangeRateDate exchangeRatePricingToBilling = $consumptionLine.properties.exchangeRatePricingToBilling - Frequency = $consumptionLine.properties.frequency - invoiceSectionId = $consumptionLine.properties.invoiceSectionId - InvoiceSectionName = $consumptionLine.properties.invoiceSectionName - IsAzureCreditEligible = $consumptionLine.properties.isAzureCreditEligible - MeterCategory = $consumptionLine.properties.meterCategory - MeterId = $consumptionLine.properties.meterId - MeterName = $consumptionLine.properties.meterName - MeterRegion = $consumptionLine.properties.meterRegion - MeterSubCategory = $consumptionLine.properties.meterSubCategory - PartNumber = $consumptionLine.properties.partNumber - paygCostInBillingCurrency = $consumptionLine.properties.paygCostInBillingCurrency - paygCostInUSD = $consumptionLine.properties.paygCostInUSD - PayGPrice = $consumptionLine.properties.payGPrice - PlanName = $consumptionLine.properties.planName - pricingCurrencyCode = $consumptionLine.properties.pricingCurrencyCode - PricingModel = $consumptionLine.properties.pricingModel - ProductName = $consumptionLine.properties.product - productIdentifier = $consumptionLine.properties.productIdentifier - PublisherName = $consumptionLine.properties.publisherName - PublisherType = $consumptionLine.properties.publisherType - Quantity = $consumptionLine.properties.quantity - ReservationId = $consumptionLine.properties.reservationId - ReservationName = $consumptionLine.properties.reservationName - ResourceGroup = $consumptionLine.properties.resourceGroup - ResourceId = $consumptionLine.properties.instanceName - ResourceLocation = $consumptionLine.properties.resourceLocation - resourceLocationNormalized = $consumptionLine.properties.resourceLocationNormalized - ServiceFamily = $consumptionLine.properties.serviceFamily - SubscriptionId = $consumptionLine.properties.subscriptionGuid - SubscriptionName = $consumptionLine.properties.subscriptionName - Tags = $tags - Term = $consumptionLine.properties.term - UnitOfMeasure = $consumptionLine.properties.unitOfMeasure - UnitPrice = $consumptionLine.properties.unitPrice + Frequency = $consumptionLine.properties.frequency + invoiceSectionId = $consumptionLine.properties.invoiceSectionId + InvoiceSectionName = $consumptionLine.properties.invoiceSectionName + IsAzureCreditEligible = $consumptionLine.properties.isAzureCreditEligible + MeterCategory = $consumptionLine.properties.meterCategory + MeterId = $consumptionLine.properties.meterId + MeterName = $consumptionLine.properties.meterName + MeterRegion = $consumptionLine.properties.meterRegion + MeterSubCategory = $consumptionLine.properties.meterSubCategory + PartNumber = $consumptionLine.properties.partNumber + paygCostInBillingCurrency = $consumptionLine.properties.paygCostInBillingCurrency + paygCostInUSD = $consumptionLine.properties.paygCostInUSD + PayGPrice = $consumptionLine.properties.payGPrice + PlanName = $consumptionLine.properties.planName + pricingCurrencyCode = $consumptionLine.properties.pricingCurrencyCode + PricingModel = $consumptionLine.properties.pricingModel + ProductName = $consumptionLine.properties.product + productIdentifier = $consumptionLine.properties.productIdentifier + PublisherName = $consumptionLine.properties.publisherName + PublisherType = $consumptionLine.properties.publisherType + Quantity = $consumptionLine.properties.quantity + ReservationId = $consumptionLine.properties.reservationId + ReservationName = $consumptionLine.properties.reservationName + ResourceGroup = $consumptionLine.properties.resourceGroup + ResourceId = $consumptionLine.properties.instanceName + ResourceLocation = $consumptionLine.properties.resourceLocation + resourceLocationNormalized = $consumptionLine.properties.resourceLocationNormalized + ServiceFamily = $consumptionLine.properties.serviceFamily + SubscriptionId = $consumptionLine.properties.subscriptionGuid + SubscriptionName = $consumptionLine.properties.subscriptionName + Tags = $tags + Term = $consumptionLine.properties.term + UnitOfMeasure = $consumptionLine.properties.unitOfMeasure + UnitPrice = $consumptionLine.properties.unitPrice + } + } + else + { + $global:hadErrors = $true + $global:scopesWithErrors += $ScopeName + if (-not($global:scopesWithErrors -contains ($ScopeName))) + { + Write-Warning "Unknown consumption line kind: $($consumptionLine.kind)" } + continue } $billingEntries += $billingEntry } @@ -822,7 +843,7 @@ else $billingEntries | Export-Csv -Path $csvExportPath -NoTypeInformation $csvBlobName = $csvExportPath - $csvProperties = @{"ContentType" = "text/csv"}; + $csvProperties = @{"ContentType" = "text/csv" } Set-AzStorageBlobContent -File $csvExportPath -Container $storageAccountSinkContainer -Properties $csvProperties -Blob $csvBlobName -Context $saCtx -Force $now = (Get-Date).ToUniversalTime().ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") @@ -858,7 +879,7 @@ else if ($consumptionScope -eq "BillingAccount") { "Starting cost details export process from $targetStartDate to $targetEndDate for Billing Account ID $BillingAccountID..." - Generate-CostDetails -ScopeId "/providers/Microsoft.Billing/billingAccounts/$BillingAccountID" -ScopeName $BillingAccountID.Replace(":","_") + Generate-CostDetails -ScopeId "/providers/Microsoft.Billing/billingAccounts/$BillingAccountID" -ScopeName $BillingAccountID.Replace(":", "_") } if ($consumptionScope -eq "BillingProfile") { diff --git a/src/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 b/src/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 index 2fdcb1bdf..91dabfefb 100644 --- a/src/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 +++ b/src/optimization-engine/runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1 @@ -26,8 +26,7 @@ if ([string]::IsNullOrEmpty($sqldatabase)) { $sqldatabase = "azureoptimization" } -$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" -$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$dceEndpoint = Get-AutomationVariable -Name "AzureOptimization_DCEIngestionEndpoint" $LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) if (-not($LogAnalyticsChunkSize -gt 0)) { @@ -70,80 +69,99 @@ switch ($authenticationOption) #region Functions -# Function to create the authorization signature -function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) +# Sends data to a Log Analytics custom table via the DCR-based Logs Ingestion API. +# Uses the Automation account managed identity bearer token for authentication. +function Send-LogIngestionData($accessToken, $dceEndpoint, $dcrImmutableId, $streamName, $body) { - $xHeaders = "x-ms-date:" + $date - $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource - $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) - $keyBytes = [Convert]::FromBase64String($sharedKey) - $sha256 = New-Object System.Security.Cryptography.HMACSHA256 - $sha256.Key = $keyBytes - $calculatedHash = $sha256.ComputeHash($bytesToHash) - $encodedHash = [Convert]::ToBase64String($calculatedHash) - $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash - return $authorization -} - -# Function to create and post the request -function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) -{ - $method = "POST" - $contentType = "application/json" - $resource = "/api/logs" - $rfc1123date = [DateTime]::UtcNow.ToString("r") - $contentLength = $body.Length - $signature = Build-OMSSignature ` - -workspaceId $workspaceId ` - -sharedKey $sharedKey ` - -date $rfc1123date ` - -contentLength $contentLength ` - -method $method ` - -contentType $contentType ` - -resource $resource - - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" - if ($AzureEnvironment -eq "AzureChinaCloud") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" - } - if ($AzureEnvironment -eq "AzureUSGovernment") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" - } - if ($AzureEnvironment -eq "AzureGermanCloud") - { - throw "Azure Germany isn't supported for the Log Analytics Data Collector API" - } - - $OMSheaders = @{ - "Authorization" = $signature - "Log-Type" = $logType - "x-ms-date" = $rfc1123date - "time-generated-field" = $TimeStampField + $uri = "$dceEndpoint/dataCollectionRules/$dcrImmutableId/streams/$streamName`?api-version=2023-01-01" + $headers = @{ + "Authorization" = "Bearer $accessToken" + "Content-Type" = "application/json" } - try { - - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + $response = Invoke-WebRequest -Uri $uri -Method POST -Headers $headers -Body $body -UseBasicParsing -TimeoutSec 1000 -ErrorAction Stop + return $response.StatusCode } catch { - if ($_.Exception.Response.StatusCode.Value__ -eq 401) + if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { - "REAUTHENTICATING" + return [int]$_.Exception.Response.StatusCode + } + throw + } +} - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 +# Converts CSV string values in a PSObject to the correct types expected by DCR typed columns. +# Columns ending in _d are cast to [double]; all others remain as strings. +function ConvertTo-TypedObject($obj) +{ + $typed = [ordered]@{} + foreach ($prop in $obj.PSObject.Properties) + { + $name = $prop.Name + $value = $prop.Value + if ($name -eq 'Timestamp') + { + # Rename to TimeGenerated as required by Log Analytics custom tables + $typed['TimeGenerated'] = $value + } + elseif ($name.EndsWith('_d')) + { + $d = 0.0 + if ([double]::TryParse($value, [System.Globalization.NumberStyles]::Any, [System.Globalization.CultureInfo]::InvariantCulture, [ref]$d)) + { + $typed[$name] = $d + } + else + { + $typed[$name] = $null + } } else { - return $_.Exception.Response.StatusCode.Value__ + $typed[$name] = $value } } + return [PSCustomObject]$typed +} - return $response.StatusCode +function Close-SqlConnection() +{ + if ($null -ne $script:SqlConnection) + { + if ($script:SqlConnection.State -ne [System.Data.ConnectionState]::Closed) + { + $script:SqlConnection.Close() + } + $script:SqlConnection.Dispose() + $script:SqlConnection = $null + $script:SqlTokenExpiresOn = $null + } +} + +function Get-SqlConnection() +{ + $refreshWindow = (Get-Date).ToUniversalTime().AddMinutes(5) + if ($null -ne $script:SqlConnection -and + $script:SqlConnection.State -eq [System.Data.ConnectionState]::Open -and + $script:SqlTokenExpiresOn -gt $refreshWindow) + { + return $script:SqlConnection + } + + Close-SqlConnection + + $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" + $script:SqlTokenExpiresOn = $dbToken.ExpiresOn.UtcDateTime + $script:SqlConnection = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;Pooling=False") + $script:SqlConnection.AccessToken = $dbToken.Token + $script:SqlConnection.Open() + + return $script:SqlConnection } + #endregion Functions $cloudDetails = Get-AzEnvironment -Name $CloudEnvironment @@ -173,10 +191,7 @@ do $tries++ try { - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() + $Conn = Get-SqlConnection $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandTimeout = $SqlTimeout @@ -190,20 +205,31 @@ do } catch { + Close-SqlConnection Write-Output "Failed to contact SQL at try $tries." Write-Output $Error[0] Start-Sleep -Seconds ($tries * 20) } + finally + { + if ($null -ne $sqlAdapter) + { + $sqlAdapter.Dispose() + } + if ($null -ne $Cmd) + { + $Cmd.Dispose() + } + } } while (-not($connectionSuccess) -and $tries -lt 3) +Close-SqlConnection + if (-not($connectionSuccess)) { throw "Could not establish connection to SQL." } -$Conn.Close() -$Conn.Dispose() - if ($controlRows.Count -eq 0 -or -not($controlRows[0].LastProcessedDateTime)) { throw "Could not find a valid ingestion control row for $storageAccountSinkContainer" @@ -213,9 +239,25 @@ $controlRow = $controlRows[0] $lastProcessedLine = $controlRow.LastProcessedLine $lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") $LogAnalyticsSuffix = $controlRow.LogAnalyticsSuffix +$dcrImmutableId = $controlRow.DCRImmutableId $logname = $lognamePrefix + $LogAnalyticsSuffix +$streamName = "Custom-$lognamePrefix$LogAnalyticsSuffix" -Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table..." +if ([string]::IsNullOrEmpty($dcrImmutableId)) +{ + throw "DCRImmutableId is not set for container $storageAccountSinkContainer. Run Setup-LogAnalyticsTablesAndDCRs.ps1 first." +} + +# Obtain a bearer token for the Logs Ingestion API using the Automation managed identity +switch ($cloudEnvironment) +{ + "AzureChinaCloud" { $monitorAudience = "https://monitor.azure.cn/" } + "AzureUSGovernment" { $monitorAudience = "https://monitor.azure.us/" } + default { $monitorAudience = "https://monitor.azure.com/" } +} +$monitorToken = (Get-AzAccessToken -ResourceUrl $monitorAudience).Token + +Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table (stream $streamName)..." $newProcessedTime = $null @@ -268,11 +310,16 @@ foreach ($blob in $unprocessedBlobs) if (($lineCounter -eq $LogAnalyticsChunkSize -or $r.Peek() -lt 0) -and $linesProcessed -gt 0) { $csvObject = $chunkLines | ConvertFrom-Csv - $jsonObject = ConvertTo-Json -InputObject $csvObject + $typedObjects = $csvObject | ForEach-Object { ConvertTo-TypedObject $_ } + $jsonObject = ConvertTo-Json -InputObject @($typedObjects) -Depth 3 if ($null -ne $jsonObject) { - $res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment + # Refresh token for long-running ingestion jobs + $monitorToken = (Get-AzAccessToken -ResourceUrl $monitorAudience).Token + $res = Send-LogIngestionData -accessToken $monitorToken -dceEndpoint $dceEndpoint ` + -dcrImmutableId $dcrImmutableId -streamName $streamName ` + -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) if ($res -ge 200 -and $res -lt 300) { @@ -309,17 +356,13 @@ foreach ($blob in $unprocessedBlobs) $lastProcessedDateTime = $updatedLastProcessedDateTime Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() + $Conn = Get-SqlConnection $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandText = $sqlStatement $Cmd.CommandTimeout = $SqlTimeout - $Cmd.ExecuteReader() - $Conn.Close() - $Conn.Dispose() + $Cmd.ExecuteNonQuery() | Out-Null + $Cmd.Dispose() $chunkLines = @() $chunkLines += $header @@ -339,17 +382,13 @@ foreach ($blob in $unprocessedBlobs) $updatedLastProcessedDateTime = $newProcessedTime Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() + $Conn = Get-SqlConnection $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandText = $sqlStatement $Cmd.CommandTimeout = $SqlTimeout - $Cmd.ExecuteReader() - $Conn.Close() - $Conn.Dispose() + $Cmd.ExecuteNonQuery() | Out-Null + $Cmd.Dispose() } else { @@ -359,4 +398,6 @@ foreach ($blob in $unprocessedBlobs) Remove-Item -Path $blobFilePath -Force } +Close-SqlConnection + Write-Output "DONE" \ No newline at end of file diff --git a/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 b/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 index a28957e0c..b6237515e 100644 --- a/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 +++ b/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1 @@ -21,12 +21,11 @@ if ([string]::IsNullOrEmpty($sqldatabase)) { $sqldatabase = "azureoptimization" } -$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" -$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$dceEndpoint = Get-AutomationVariable -Name "AzureOptimization_DCEIngestionEndpoint" $LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) if (-not($LogAnalyticsChunkSize -gt 0)) { - $LogAnalyticsChunkSize = 6000 + $LogAnalyticsChunkSize = 150 } $lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue if ([string]::IsNullOrEmpty($lognamePrefix)) @@ -53,80 +52,64 @@ if (-not($StorageBlobsPageSize -gt 0)) #region Functions -# Function to create the authorization signature -function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) +# Sends data to a Log Analytics custom table via the DCR-based Logs Ingestion API. +function Send-LogIngestionData($accessToken, $dceEndpoint, $dcrImmutableId, $streamName, $body) { - $xHeaders = "x-ms-date:" + $date - $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource - $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) - $keyBytes = [Convert]::FromBase64String($sharedKey) - $sha256 = New-Object System.Security.Cryptography.HMACSHA256 - $sha256.Key = $keyBytes - $calculatedHash = $sha256.ComputeHash($bytesToHash) - $encodedHash = [Convert]::ToBase64String($calculatedHash) - $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash - return $authorization -} - -# Function to create and post the request -function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) -{ - $method = "POST" - $contentType = "application/json" - $resource = "/api/logs" - $rfc1123date = [DateTime]::UtcNow.ToString("r") - $contentLength = $body.Length - $signature = Build-OMSSignature ` - -workspaceId $workspaceId ` - -sharedKey $sharedKey ` - -date $rfc1123date ` - -contentLength $contentLength ` - -method $method ` - -contentType $contentType ` - -resource $resource - - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" - if ($AzureEnvironment -eq "AzureChinaCloud") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" - } - if ($AzureEnvironment -eq "AzureUSGovernment") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" + $uri = "$dceEndpoint/dataCollectionRules/$dcrImmutableId/streams/$streamName`?api-version=2023-01-01" + $headers = @{ + "Authorization" = "Bearer $accessToken" + "Content-Type" = "application/json" } - if ($AzureEnvironment -eq "AzureGermanCloud") - { - throw "Azure Germany isn't supported for the Log Analytics Data Collector API" - } - - $OMSheaders = @{ - "Authorization" = $signature - "Log-Type" = $logType - "x-ms-date" = $rfc1123date - "time-generated-field" = $TimeStampField - } - try { - - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + $response = Invoke-WebRequest -Uri $uri -Method POST -Headers $headers -Body $body -UseBasicParsing -TimeoutSec 1000 -ErrorAction Stop + return $response.StatusCode } catch { - if ($_.Exception.Response.StatusCode.Value__ -eq 401) + if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { - "REAUTHENTICATING" - - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + return [int]$_.Exception.Response.StatusCode } - else + throw + } +} + +function Close-SqlConnection() +{ + if ($null -ne $script:SqlConnection) + { + if ($script:SqlConnection.State -ne [System.Data.ConnectionState]::Closed) { - return $_.Exception.Response.StatusCode.Value__ + $script:SqlConnection.Close() } + $script:SqlConnection.Dispose() + $script:SqlConnection = $null + $script:SqlTokenExpiresOn = $null } +} - return $response.StatusCode +function Get-SqlConnection() +{ + $refreshWindow = (Get-Date).ToUniversalTime().AddMinutes(5) + if ($null -ne $script:SqlConnection -and + $script:SqlConnection.State -eq [System.Data.ConnectionState]::Open -and + $script:SqlTokenExpiresOn -gt $refreshWindow) + { + return $script:SqlConnection + } + + Close-SqlConnection + + $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" + $script:SqlTokenExpiresOn = $dbToken.ExpiresOn.UtcDateTime + $script:SqlConnection = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;Pooling=False") + $script:SqlConnection.AccessToken = $dbToken.Token + $script:SqlConnection.Open() + + return $script:SqlConnection } + #endregion Functions @@ -175,10 +158,7 @@ do $tries++ try { - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() + $Conn = Get-SqlConnection $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandTimeout = $SqlTimeout @@ -192,20 +172,31 @@ do } catch { + Close-SqlConnection Write-Output "Failed to contact SQL at try $tries." Write-Output $Error[0] Start-Sleep -Seconds ($tries * 20) } + finally + { + if ($null -ne $sqlAdapter) + { + $sqlAdapter.Dispose() + } + if ($null -ne $Cmd) + { + $Cmd.Dispose() + } + } } while (-not($connectionSuccess) -and $tries -lt 3) +Close-SqlConnection + if (-not($connectionSuccess)) { throw "Could not establish connection to SQL." } -$Conn.Close() -$Conn.Dispose() - if ($controlRows.Count -eq 0 -or -not($controlRows[0].LastProcessedDateTime)) { throw "Could not find a valid ingestion control row for $storageAccountSinkContainer" @@ -215,9 +206,25 @@ $controlRow = $controlRows[0] $lastProcessedLine = $controlRow.LastProcessedLine $lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") $LogAnalyticsSuffix = $controlRow.LogAnalyticsSuffix +$dcrImmutableId = $controlRow.DCRImmutableId $logname = $lognamePrefix + $LogAnalyticsSuffix +$streamName = "Custom-$lognamePrefix$LogAnalyticsSuffix" + +if ([string]::IsNullOrEmpty($dcrImmutableId)) +{ + throw "DCRImmutableId is not set for container $storageAccountSinkContainer. Run Setup-LogAnalyticsTablesAndDCRs.ps1 first." +} -Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table..." +# Obtain a bearer token for the Logs Ingestion API +switch ($cloudEnvironment) +{ + "AzureChinaCloud" { $monitorAudience = "https://monitor.azure.cn/" } + "AzureUSGovernment" { $monitorAudience = "https://monitor.azure.us/" } + default { $monitorAudience = "https://monitor.azure.com/" } +} +$monitorToken = (Get-AzAccessToken -ResourceUrl $monitorAudience).Token + +Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the $($logname)_CL table (stream $streamName)..." $newProcessedTime = $null @@ -289,10 +296,16 @@ foreach ($blob in $unprocessedBlobs) $jsonObjectSplitted[$j][$i].RecommendationAction = $jsonObjectSplitted[$j][$i].RecommendationAction.Replace("'", "") $jsonObjectSplitted[$j][$i].AdditionalInfo = $jsonObjectSplitted[$j][$i].AdditionalInfo | ConvertTo-Json -Compress $jsonObjectSplitted[$j][$i].Tags = $jsonObjectSplitted[$j][$i].Tags | ConvertTo-Json -Compress + # Rename Timestamp to TimeGenerated as required by Log Analytics custom tables + $jsonObjectSplitted[$j][$i] | Add-Member -MemberType NoteProperty -Name 'TimeGenerated' -Value $jsonObjectSplitted[$j][$i].Timestamp -Force } $jsonObject = ConvertTo-Json -InputObject $jsonObjectSplitted[$j] - $res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment + # Refresh token before each chunk upload + $monitorToken = (Get-AzAccessToken -ResourceUrl $monitorAudience).Token + $res = Send-LogIngestionData -accessToken $monitorToken -dceEndpoint $dceEndpoint ` + -dcrImmutableId $dcrImmutableId -streamName $streamName ` + -body ([System.Text.Encoding]::UTF8.GetBytes($jsonObject)) if ($res -ge 200 -and $res -lt 300) { Write-Output "Successfully uploaded $currentObjectLines $LogAnalyticsSuffix rows to Log Analytics" @@ -315,17 +328,13 @@ foreach ($blob in $unprocessedBlobs) $lastProcessedDateTime = $updatedLastProcessedDateTime Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" $sqlStatement = "UPDATE [$LogAnalyticsIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() + $Conn = Get-SqlConnection $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandText = $sqlStatement $Cmd.CommandTimeout = $SqlTimeout - $Cmd.ExecuteReader() - $Conn.Close() - $Conn.Dispose() + $Cmd.ExecuteNonQuery() | Out-Null + $Cmd.Dispose() } else { @@ -344,4 +353,6 @@ foreach ($blob in $unprocessedBlobs) Remove-Item -Path $blob.Name -Force } +Close-SqlConnection + Write-Output "DONE" \ No newline at end of file diff --git a/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 b/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 index eb9d9e963..3736f3d14 100644 --- a/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 +++ b/src/optimization-engine/runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1 @@ -32,7 +32,8 @@ $storageAccountSink = Get-AutomationVariable -Name "AzureOptimization_StorageSi $storageAccountSinkContainer = Get-AutomationVariable -Name "AzureOptimization_RecommendationsContainer" -ErrorAction SilentlyContinue -if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) { +if ([string]::IsNullOrEmpty($storageAccountSinkContainer)) +{ $storageAccountSinkContainer = "recommendationsexports" } $StorageBlobsPageSize = [int] (Get-AutomationVariable -Name "AzureOptimization_StorageBlobsPageSize" -ErrorAction SilentlyContinue) @@ -41,14 +42,54 @@ if (-not($StorageBlobsPageSize -gt 0)) $StorageBlobsPageSize = 1000 } +function Close-SqlConnection() +{ + if ($null -ne $script:SqlConnection) + { + if ($script:SqlConnection.State -ne [System.Data.ConnectionState]::Closed) + { + $script:SqlConnection.Close() + } + $script:SqlConnection.Dispose() + $script:SqlConnection = $null + $script:SqlTokenExpiresOn = $null + } +} + +function Get-SqlConnection() +{ + $refreshWindow = (Get-Date).ToUniversalTime().AddMinutes(5) + if ($null -ne $script:SqlConnection -and + $script:SqlConnection.State -eq [System.Data.ConnectionState]::Open -and + $script:SqlTokenExpiresOn -gt $refreshWindow) + { + return $script:SqlConnection + } + + Close-SqlConnection + + $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" + $script:SqlTokenExpiresOn = $dbToken.ExpiresOn.UtcDateTime + $script:SqlConnection = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;Pooling=False") + $script:SqlConnection.AccessToken = $dbToken.Token + $script:SqlConnection.Open() + + return $script:SqlConnection +} + + "Logging in to Azure with $authenticationOption..." -switch ($authenticationOption) { - "UserAssignedManagedIdentity" { +switch ($authenticationOption) +{ + "UserAssignedManagedIdentity" + { Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment -AccountId $uamiClientID break } - Default { #ManagedIdentity + Default + { + #ManagedIdentity Connect-AzAccount -Identity -EnvironmentName $cloudEnvironment break } @@ -71,7 +112,7 @@ do $blobs = Get-AzStorageBlob -Container $storageAccountSinkContainer -MaxCount $StorageBlobsPageSize -ContinuationToken $continuationToken -Context $saCtx | Sort-Object -Property LastModified if ($blobs.Count -le 0) { break } $allblobs += $blobs - $continuationToken = $blobs[$blobs.Count -1].ContinuationToken; + $continuationToken = $blobs[$blobs.Count - 1].ContinuationToken } While ($null -ne $continuationToken) @@ -81,14 +122,13 @@ $recommendationsTable = "Recommendations" $tries = 0 $connectionSuccess = $false -do { +do +{ $tries++ - try { - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() - $Cmd=new-object system.Data.SqlClient.SqlCommand + try + { + $Conn = Get-SqlConnection + $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandTimeout = $SqlTimeout $Cmd.CommandText = "SELECT * FROM [dbo].[$SqlServerIngestControlTable] WHERE StorageContainerName = '$storageAccountSinkContainer' and SqlTableName = '$recommendationsTable'" @@ -99,13 +139,17 @@ do { $sqlAdapter.Fill($controlRows) | Out-Null $connectionSuccess = $true } - catch { + catch + { + Close-SqlConnection Write-Output "Failed to contact SQL at try $tries." Write-Output $Error[0] Start-Sleep -Seconds ($tries * 20) } } while (-not($connectionSuccess) -and $tries -lt 3) +Close-SqlConnection + if (-not($connectionSuccess)) { throw "Could not establish connection to SQL." @@ -120,8 +164,6 @@ $controlRow = $controlRows[0] $lastProcessedLine = $controlRow.LastProcessedLine $lastProcessedDateTime = $controlRow.LastProcessedDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") -$Conn.Close() -$Conn.Dispose() Write-Output "Processing blobs modified after $lastProcessedDateTime (line $lastProcessedLine) and ingesting them into the Recommendations SQL table..." @@ -129,10 +171,12 @@ $newProcessedTime = $null $unprocessedBlobs = @() -foreach ($blob in $allblobs) { +foreach ($blob in $allblobs) +{ $blobLastModified = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") if ($lastProcessedDateTime -lt $blobLastModified -or ` - ($lastProcessedDateTime -eq $blobLastModified -and $lastProcessedLine -gt 0)) { + ($lastProcessedDateTime -eq $blobLastModified -and $lastProcessedLine -gt 0)) + { Write-Output "$($blob.Name) found (modified on $blobLastModified)" $unprocessedBlobs += $blob } @@ -142,7 +186,8 @@ $unprocessedBlobs = $unprocessedBlobs | Sort-Object -Property LastModified Write-Output "Found $($unprocessedBlobs.Count) new blobs to process..." -foreach ($blob in $unprocessedBlobs) { +foreach ($blob in $unprocessedBlobs) +{ $newProcessedTime = $blob.LastModified.UtcDateTime.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'") Write-Output "About to process $($blob.Name)..." Get-AzStorageBlobContent -CloudBlob $blob.ICloudBlob -Context $saCtx -Force @@ -167,8 +212,9 @@ foreach ($blob in $unprocessedBlobs) { if ($recCount -gt 1) { - for ($i = 0; $i -lt $recCount; $i += $ChunkSize) { - $jsonObjectSplitted += , @($jsonObject[$i..($i + ($ChunkSize - 1))]); + for ($i = 0; $i -lt $recCount; $i += $ChunkSize) + { + $jsonObjectSplitted += , @($jsonObject[$i..($i + ($ChunkSize - 1))]) } } else @@ -223,60 +269,58 @@ foreach ($blob in $unprocessedBlobs) { $sqlStatement += ", '$($jsonObjectSplitted[$j][$i].InstanceId)', '$($jsonObjectSplitted[$j][$i].InstanceName)', '$additionalInfoString'" $sqlStatement += ", $resourceGroup, $subscriptionGuid, $subscriptionName, '$($jsonObjectSplitted[$j][$i].TenantGuid)'" $sqlStatement += ", $($jsonObjectSplitted[$j][$i].FitScore), '$tagsString', '$($jsonObjectSplitted[$j][$i].DetailsURL)')" - if ($i -ne ($jsonObjectSplitted[$j].Count-1)) + if ($i -ne ($jsonObjectSplitted[$j].Count - 1)) { $sqlStatement += "," } } - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn2 = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn2.AccessToken = $dbToken.Token - $Conn2.Open() - - $Cmd=new-object system.Data.SqlClient.SqlCommand - $Cmd.Connection = $Conn2 + $Conn = Get-SqlConnection + $Cmd = New-Object system.Data.SqlClient.SqlCommand + $Cmd.Connection = $Conn $Cmd.CommandText = $sqlStatement $Cmd.CommandTimeout = $SqlTimeout try { - $Cmd.ExecuteReader() + $Cmd.ExecuteNonQuery() | Out-Null } catch { Write-Output "Failed statement: $sqlStatement" throw } - - $Conn2.Close() + finally + { + $Cmd.Dispose() + } $linesProcessed += $currentObjectLines Write-Output "Processed $linesProcessed lines..." - if ($j -eq ($jsonObjectSplitted.Count - 1)) { + if ($j -eq ($jsonObjectSplitted.Count - 1)) + { $lastProcessedLine = -1 } - else { + else + { $lastProcessedLine = $linesProcessed - 1 } $updatedLastProcessedLine = $lastProcessedLine $updatedLastProcessedDateTime = $lastProcessedDateTime - if ($j -eq ($jsonObjectSplitted.Count - 1)) { + if ($j -eq ($jsonObjectSplitted.Count - 1)) + { $updatedLastProcessedDateTime = $newProcessedTime } $lastProcessedDateTime = $updatedLastProcessedDateTime Write-Output "Updating last processed time / line to $($updatedLastProcessedDateTime) / $updatedLastProcessedLine" $sqlStatement = "UPDATE [$SqlServerIngestControlTable] SET LastProcessedLine = $updatedLastProcessedLine, LastProcessedDateTime = '$updatedLastProcessedDateTime' WHERE StorageContainerName = '$storageAccountSinkContainer'" - $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" - $Conn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") - $Conn.AccessToken = $dbToken.Token - $Conn.Open() - $Cmd=new-object system.Data.SqlClient.SqlCommand + $Conn = Get-SqlConnection + $Cmd = New-Object system.Data.SqlClient.SqlCommand $Cmd.Connection = $Conn $Cmd.CommandText = $sqlStatement $Cmd.CommandTimeout = $SqlTimeout - $Cmd.ExecuteReader() - $Conn.Close() + $Cmd.ExecuteNonQuery() | Out-Null + $Cmd.Dispose() } else { @@ -288,4 +332,6 @@ foreach ($blob in $unprocessedBlobs) { Remove-Item -Path $blob.Name -Force } +Close-SqlConnection + Write-Output "DONE" \ No newline at end of file diff --git a/src/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 b/src/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 index db867ee93..0d779d8d1 100644 --- a/src/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 +++ b/src/optimization-engine/runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1 @@ -6,12 +6,11 @@ if ([string]::IsNullOrEmpty($cloudEnvironment)) $cloudEnvironment = "AzureCloud" } -$workspaceId = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceId" -$sharedKey = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsWorkspaceKey" +$dceEndpoint = Get-AutomationVariable -Name "AzureOptimization_DCEIngestionEndpoint" $LogAnalyticsChunkSize = [int] (Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsChunkSize" -ErrorAction SilentlyContinue) if (-not($LogAnalyticsChunkSize -gt 0)) { - $LogAnalyticsChunkSize = 6000 + $LogAnalyticsChunkSize = 150 } $lognamePrefix = Get-AutomationVariable -Name "AzureOptimization_LogAnalyticsLogPrefix" -ErrorAction SilentlyContinue if ([string]::IsNullOrEmpty($lognamePrefix)) @@ -41,80 +40,29 @@ $FiltersTable = "Filters" #region Functions -# Function to create the authorization signature -function Build-OMSSignature ($workspaceId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) +# Sends data to a Log Analytics custom table via the DCR-based Logs Ingestion API. +function Send-LogIngestionData($accessToken, $dceEndpoint, $dcrImmutableId, $streamName, $body) { - $xHeaders = "x-ms-date:" + $date - $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource - $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) - $keyBytes = [Convert]::FromBase64String($sharedKey) - $sha256 = New-Object System.Security.Cryptography.HMACSHA256 - $sha256.Key = $keyBytes - $calculatedHash = $sha256.ComputeHash($bytesToHash) - $encodedHash = [Convert]::ToBase64String($calculatedHash) - $authorization = 'SharedKey {0}:{1}' -f $workspaceId, $encodedHash - return $authorization -} - -# Function to create and post the request -function Post-OMSData($workspaceId, $sharedKey, $body, $logType, $TimeStampField, $AzureEnvironment) -{ - $method = "POST" - $contentType = "application/json" - $resource = "/api/logs" - $rfc1123date = [DateTime]::UtcNow.ToString("r") - $contentLength = $body.Length - $signature = Build-OMSSignature ` - -workspaceId $workspaceId ` - -sharedKey $sharedKey ` - -date $rfc1123date ` - -contentLength $contentLength ` - -method $method ` - -contentType $contentType ` - -resource $resource - - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" - if ($AzureEnvironment -eq "AzureChinaCloud") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.cn" + $resource + "?api-version=2016-04-01" + $uri = "$dceEndpoint/dataCollectionRules/$dcrImmutableId/streams/$streamName`?api-version=2023-01-01" + $headers = @{ + "Authorization" = "Bearer $accessToken" + "Content-Type" = "application/json" } - if ($AzureEnvironment -eq "AzureUSGovernment") - { - $uri = "https://" + $workspaceId + ".ods.opinsights.azure.us" + $resource + "?api-version=2016-04-01" - } - if ($AzureEnvironment -eq "AzureGermanCloud") - { - throw "Azure Germany isn't supported for the Log Analytics Data Collector API" - } - - $OMSheaders = @{ - "Authorization" = $signature - "Log-Type" = $logType - "x-ms-date" = $rfc1123date - "time-generated-field" = $TimeStampField - } - try { - - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 + $response = Invoke-WebRequest -Uri $uri -Method POST -Headers $headers -Body $body -UseBasicParsing -TimeoutSec 1000 -ErrorAction Stop + return $response.StatusCode } catch { - if ($_.Exception.Response.StatusCode.Value__ -eq 401) + if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { - "REAUTHENTICATING" - - $response = Invoke-WebRequest -Uri $uri -Method POST -ContentType $contentType -Headers $OMSheaders -Body $body -UseBasicParsing -TimeoutSec 1000 - } - else - { - return $_.Exception.Response.StatusCode.Value__ + return [int]$_.Exception.Response.StatusCode } + throw } - - return $response.StatusCode } + #endregion Functions "Logging in to Azure with $authenticationOption..." @@ -137,6 +85,14 @@ switch ($authenticationOption) $cloudDetails = Get-AzEnvironment -Name $CloudEnvironment $azureSqlDomain = $cloudDetails.SqlDatabaseDnsSuffix.Substring(1) +# Determine the Logs Ingestion API audience URL for this cloud +switch ($cloudEnvironment) +{ + "AzureChinaCloud" { $monitorAudience = "https://monitor.azure.cn/" } + "AzureUSGovernment" { $monitorAudience = "https://monitor.azure.us/" } + default { $monitorAudience = "https://monitor.azure.com/" } +} + Write-Output "Getting excluded recommendation sub-type IDs..." $tries = 0 @@ -246,12 +202,53 @@ foreach ($filter in $filters) $filterObjects += $filterObject } -$filtersJson = $filterObjects | ConvertTo-Json +$filtersJson = $filterObjects | ForEach-Object { + $_ | Add-Member -MemberType NoteProperty -Name 'TimeGenerated' -Value $_.Timestamp -Force -PassThru +} | ConvertTo-Json $LogAnalyticsSuffix = "SuppressionsV1" $logname = $lognamePrefix + $LogAnalyticsSuffix +$streamName = "Custom-$logname" + +# Retrieve DCR immutable ID from SQL control table +$dcrImmutableId = $null +$tries = 0 +$dcrQuerySuccess = $false +do +{ + $tries++ + try + { + $dbToken = Get-AzAccessToken -ResourceUrl "https://$azureSqlDomain/" + $dcrConn = New-Object System.Data.SqlClient.SqlConnection("Server=tcp:$sqlserver,1433;Database=$sqldatabase;Encrypt=True;Connection Timeout=$SqlTimeout;") + $dcrConn.AccessToken = $dbToken.Token + $dcrConn.Open() + $dcrCmd = New-Object system.Data.SqlClient.SqlCommand + $dcrCmd.Connection = $dcrConn + $dcrCmd.CommandTimeout = $SqlTimeout + $dcrCmd.CommandText = "SELECT DCRImmutableId FROM [dbo].[LogAnalyticsIngestControl] WHERE StorageContainerName = 'suppressions'" + $dcrImmutableId = $dcrCmd.ExecuteScalar() + $dcrConn.Close() + $dcrConn.Dispose() + $dcrQuerySuccess = $true + } + catch + { + Write-Output "Failed to retrieve DCR immutable ID from SQL at try $tries." + Write-Output $Error[0] + Start-Sleep -Seconds ($tries * 20) + } +} while (-not($dcrQuerySuccess) -and $tries -lt 3) + +if ([string]::IsNullOrEmpty($dcrImmutableId)) +{ + throw "DCRImmutableId is not set for suppressions. Run Setup-LogAnalyticsTablesAndDCRs.ps1 first." +} -$res = Post-OMSData -workspaceId $workspaceId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($filtersJson)) -logType $logname -TimeStampField "Timestamp" -AzureEnvironment $cloudEnvironment +$monitorToken = (Get-AzAccessToken -ResourceUrl $monitorAudience).Token +$res = Send-LogIngestionData -accessToken $monitorToken -dceEndpoint $dceEndpoint ` + -dcrImmutableId $dcrImmutableId -streamName $streamName ` + -body ([System.Text.Encoding]::UTF8.GetBytes($filtersJson)) if ($res -ge 200 -and $res -lt 300) { Write-Output "Successfully uploaded $($filterObjects.Count) $LogAnalyticsSuffix rows to Log Analytics" diff --git a/src/optimization-engine/upgrade-manifest.json b/src/optimization-engine/upgrade-manifest.json index 362f489c6..8e2a1596d 100644 --- a/src/optimization-engine/upgrade-manifest.json +++ b/src/optimization-engine/upgrade-manifest.json @@ -266,17 +266,17 @@ }, { "name": "AzureOptimization_ExportPricesWeekly", - "offset": "PT1H35M", + "offset": "PT1H05M", "frequency": "Week" }, { "name": "AzureOptimization_IngestPricesheetWeekly", - "offset": "PT2H10M", + "offset": "PT1H35M", "frequency": "Week" }, { "name": "AzureOptimization_IngestReservationsPriceWeekly", - "offset": "PT2H10M", + "offset": "PT1H35M", "frequency": "Week" }, { @@ -304,14 +304,14 @@ { "runbook": { "name": "runbooks/data-collection/Ingest-OptimizationCSVExportsToLogAnalytics.ps1", - "version": "1.6.2.0" + "version": "2.0.0.0" }, "source": "dataCollection" }, { "runbook": { "name": "runbooks/recommendations/Ingest-RecommendationsToSQLServer.ps1", - "version": "1.7.1.0" + "version": "1.7.2.0" }, "source": "recommendations", "schedule": "AzureOptimization_IngestRecommendationsWeekly" @@ -319,7 +319,7 @@ { "runbook": { "name": "runbooks/recommendations/Ingest-RecommendationsToLogAnalytics.ps1", - "version": "1.1.1.0" + "version": "2.0.0.0" }, "source": "recommendations", "schedule": "AzureOptimization_IngestRecommendationsWeekly" @@ -327,7 +327,7 @@ { "runbook": { "name": "runbooks/recommendations/Ingest-SuppressionsToLogAnalytics.ps1", - "version": "1.1.0.0" + "version": "2.0.0.0" }, "source": "recommendations", "schedule": "AzureOptimization_IngestSuppressionsWeekly" @@ -521,7 +521,7 @@ { "runbook": { "name": "runbooks/data-collection/Export-ConsumptionToBlobStorage.ps1", - "version": "2.1.1.0" + "version": "2.1.2.0" }, "container": "consumptionexports", "requiredVariables": [ @@ -1052,7 +1052,7 @@ "overwriteVariables": [ { "name": "AzureOptimization_LogAnalyticsChunkSize", - "value": 6000 + "value": 150 } ] } \ No newline at end of file