From 38bb7cd4bdbf86520d57ecdc12b90d57d93eedec Mon Sep 17 00:00:00 2001 From: JohnLBevan Date: Mon, 5 Dec 2022 13:41:46 +0000 Subject: [PATCH 1/4] #43 - cleanup whitespace --- Tasks/Tokenization/task/tokenization.ps1 | 188 +++++++++++------------ 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/Tasks/Tokenization/task/tokenization.ps1 b/Tasks/Tokenization/task/tokenization.ps1 index 7050d43..56e78b4 100644 --- a/Tasks/Tokenization/task/tokenization.ps1 +++ b/Tasks/Tokenization/task/tokenization.ps1 @@ -2,15 +2,15 @@ param ( [String] [Parameter(Mandatory = $true)] $SourcePath, - [String] [Parameter(Mandatory = $true)] $TargetFileNames, - [String] [Parameter(Mandatory = $false)] $RecursiveSearch, + [String] [Parameter(Mandatory = $true)] $TargetFileNames, + [String] [Parameter(Mandatory = $false)] $RecursiveSearch, [String] [Parameter(Mandatory = $true)] $TokenStart, [String] [Parameter(Mandatory = $true)] $TokenEnd, [String] [Parameter(Mandatory = $false)] $RequireVariable ) -import-module "Microsoft.TeamFoundation.DistributedTask.Task.Internal" -import-module "Microsoft.TeamFoundation.DistributedTask.Task.Common" +import-module "Microsoft.TeamFoundation.DistributedTask.Task.Internal" +import-module "Microsoft.TeamFoundation.DistributedTask.Task.Common" $patterns = @() $regex = $TokenStart + '([A-Za-z0-9.]*|_)*' + $TokenEnd @@ -22,100 +22,100 @@ Write-Host (Get-LocalizedString -Key 'Target File Name: {0}...' -ArgumentList $T function ProcessMatches($fileMatches) { - ForEach($targetFileMatch in $fileMatches) - { - $fileEncoding = Get-FileEncoding($targetFileMatch.FullName) - - Write-Host (Get-LocalizedString -Key 'Targeted FileName Encoding: {0}...' -ArgumentList $fileEncoding) - - $targetFilePath = $targetFileMatch.Directory.FullName - $tempFile = $targetFileMatch.FullName + '.tmp' - - #Write-Host (Get-LocalizedString -Key 'Target File Path: {0}...' -ArgumentList $targetFilePath) - Write-Host (Get-LocalizedString -Key 'Target File Match: {0}...' -ArgumentList $targetFileMatch.FullName) - Write-Host (Get-LocalizedString -Key 'Temp File: {0}...' -ArgumentList $tempFile) - - Copy-Item -Force $targetFileMatch.FullName $tempFile - + ForEach($targetFileMatch in $fileMatches) + { + $fileEncoding = Get-FileEncoding($targetFileMatch.FullName) + + Write-Host (Get-LocalizedString -Key 'Targeted FileName Encoding: {0}...' -ArgumentList $fileEncoding) + + $targetFilePath = $targetFileMatch.Directory.FullName + $tempFile = $targetFileMatch.FullName + '.tmp' + + #Write-Host (Get-LocalizedString -Key 'Target File Path: {0}...' -ArgumentList $targetFilePath) + Write-Host (Get-LocalizedString -Key 'Target File Match: {0}...' -ArgumentList $targetFileMatch.FullName) + Write-Host (Get-LocalizedString -Key 'Temp File: {0}...' -ArgumentList $tempFile) + + Copy-Item -Force $targetFileMatch.FullName $tempFile + $matches = select-string -Path $tempFile -Pattern $regex -AllMatches | % { $_.Matches } | % { $_.Value } ForEach ($match in $matches) { $matchedItem = $match $matchedItem = $matchedItem.TrimStart($TokenStart) $matchedItem = $matchedItem.TrimEnd($TokenEnd) $matchedItem = $matchedItem -replace '\.','_' - + Write-Host (Get-LocalizedString -Key 'Token {0}...' -ArgumentList $matchedItem) -ForegroundColor Green - + $matchValue = Get-TaskVariable $distributedTaskContext $matchedItem if ([System.Convert]::ToBoolean($RequireVariable) -and !$matchValue) { throw "$matchedItem variable not set" } - + Write-Host (Get-LocalizedString -Key 'Token Value: {0}...' -ArgumentList $matchValue) -ForegroundColor Green - - (Get-Content $tempFile) | - Foreach-Object { - $_ -replace $match,$matchValue - } | - Out-File $tempFile -Force -Encoding $fileEncoding - } - - Copy-Item -Force $tempFile $targetFileMatch.FullName - Remove-Item -Force $tempFile - } + + (Get-Content $tempFile) | + Foreach-Object { + $_ -replace $match,$matchValue + } | + Out-File $tempFile -Force -Encoding $fileEncoding + } + + Copy-Item -Force $tempFile $targetFileMatch.FullName + Remove-Item -Force $tempFile + } } function Get-FileEncoding($targetFilePath) { - [byte[]]$byte = get-content -Encoding byte -ReadCount 4 -TotalCount 4 -Path $targetFilePath - - # EF BB BF (UTF8) - if ( $byte[0] -eq 0xef -and $byte[1] -eq 0xbb -and $byte[2] -eq 0xbf ) - { return 'UTF8' } - - # FE FF (UTF-16 Big-Endian) - elseif ($byte[0] -eq 0xfe -and $byte[1] -eq 0xff) - { return 'Unicode UTF-16 Big-Endian' } - - # FF FE (UTF-16 Little-Endian) - elseif ($byte[0] -eq 0xff -and $byte[1] -eq 0xfe) - { return 'Unicode UTF-16 Little-Endian' } - - # 00 00 FE FF (UTF32 Big-Endian) - elseif ($byte[0] -eq 0 -and $byte[1] -eq 0 -and $byte[2] -eq 0xfe -and $byte[3] -eq 0xff) - { return 'UTF32 Big-Endian' } - - # FE FF 00 00 (UTF32 Little-Endian) - elseif ($byte[0] -eq 0xfe -and $byte[1] -eq 0xff -and $byte[2] -eq 0 -and $byte[3] -eq 0) - { return 'UTF32 Little-Endian' } - - # 2B 2F 76 (38 | 38 | 2B | 2F) - elseif ($byte[0] -eq 0x2b -and $byte[1] -eq 0x2f -and $byte[2] -eq 0x76 -and ($byte[3] -eq 0x38 -or $byte[3] -eq 0x39 -or $byte[3] -eq 0x2b -or $byte[3] -eq 0x2f) ) - { return 'UTF7'} - - # F7 64 4C (UTF-1) - elseif ( $byte[0] -eq 0xf7 -and $byte[1] -eq 0x64 -and $byte[2] -eq 0x4c ) - { return 'UTF-1' } - - # DD 73 66 73 (UTF-EBCDIC) - elseif ($byte[0] -eq 0xdd -and $byte[1] -eq 0x73 -and $byte[2] -eq 0x66 -and $byte[3] -eq 0x73) - { return 'UTF-EBCDIC' } - - # 0E FE FF (SCSU) - elseif ( $byte[0] -eq 0x0e -and $byte[1] -eq 0xfe -and $byte[2] -eq 0xff ) - { return 'SCSU' } - - # FB EE 28 (BOCU-1) - elseif ( $byte[0] -eq 0xfb -and $byte[1] -eq 0xee -and $byte[2] -eq 0x28 ) - { return 'BOCU-1' } - - # 84 31 95 33 (GB-18030) - elseif ($byte[0] -eq 0x84 -and $byte[1] -eq 0x31 -and $byte[2] -eq 0x95 -and $byte[3] -eq 0x33) - { return 'GB-18030' } - - else - { return 'ASCII' } + [byte[]]$byte = get-content -Encoding byte -ReadCount 4 -TotalCount 4 -Path $targetFilePath + + # EF BB BF (UTF8) + if ( $byte[0] -eq 0xef -and $byte[1] -eq 0xbb -and $byte[2] -eq 0xbf ) + { return 'UTF8' } + + # FE FF (UTF-16 Big-Endian) + elseif ($byte[0] -eq 0xfe -and $byte[1] -eq 0xff) + { return 'Unicode UTF-16 Big-Endian' } + + # FF FE (UTF-16 Little-Endian) + elseif ($byte[0] -eq 0xff -and $byte[1] -eq 0xfe) + { return 'Unicode UTF-16 Little-Endian' } + + # 00 00 FE FF (UTF32 Big-Endian) + elseif ($byte[0] -eq 0 -and $byte[1] -eq 0 -and $byte[2] -eq 0xfe -and $byte[3] -eq 0xff) + { return 'UTF32 Big-Endian' } + + # FE FF 00 00 (UTF32 Little-Endian) + elseif ($byte[0] -eq 0xfe -and $byte[1] -eq 0xff -and $byte[2] -eq 0 -and $byte[3] -eq 0) + { return 'UTF32 Little-Endian' } + + # 2B 2F 76 (38 | 38 | 2B | 2F) + elseif ($byte[0] -eq 0x2b -and $byte[1] -eq 0x2f -and $byte[2] -eq 0x76 -and ($byte[3] -eq 0x38 -or $byte[3] -eq 0x39 -or $byte[3] -eq 0x2b -or $byte[3] -eq 0x2f) ) + { return 'UTF7'} + + # F7 64 4C (UTF-1) + elseif ( $byte[0] -eq 0xf7 -and $byte[1] -eq 0x64 -and $byte[2] -eq 0x4c ) + { return 'UTF-1' } + + # DD 73 66 73 (UTF-EBCDIC) + elseif ($byte[0] -eq 0xdd -and $byte[1] -eq 0x73 -and $byte[2] -eq 0x66 -and $byte[3] -eq 0x73) + { return 'UTF-EBCDIC' } + + # 0E FE FF (SCSU) + elseif ( $byte[0] -eq 0x0e -and $byte[1] -eq 0xfe -and $byte[2] -eq 0xff ) + { return 'SCSU' } + + # FB EE 28 (BOCU-1) + elseif ( $byte[0] -eq 0xfb -and $byte[1] -eq 0xee -and $byte[2] -eq 0x28 ) + { return 'BOCU-1' } + + # 84 31 95 33 (GB-18030) + elseif ($byte[0] -eq 0x84 -and $byte[1] -eq 0x31 -and $byte[2] -eq 0x95 -and $byte[3] -eq 0x33) + { return 'GB-18030' } + + else + { return 'ASCII' } } Write-Host (Get-LocalizedString -Key 'RecursiveSearch: {0}...' -ArgumentList $RecursiveSearch) @@ -125,18 +125,16 @@ $targetedFiles = $TargetFileNames.Split(',') ForEach($targetedFileName in $targetedFiles) { - Write-Host (Get-LocalizedString -Key 'Targeted FileName: {0}...' -ArgumentList $targetedFileName) - - if ([System.Convert]::ToBoolean($RecursiveSearch)) - { - $fileMatches = Get-ChildItem -Path $SourcePath -Filter $targetedFileName -Recurse - - ProcessMatches($fileMatches) - } - else - { - $fileMatches = Get-ChildItem -Path $SourcePath -Filter $targetedFileName - - ProcessMatches($fileMatches) - } + Write-Host (Get-LocalizedString -Key 'Targeted FileName: {0}...' -ArgumentList $targetedFileName) + + if ([System.Convert]::ToBoolean($RecursiveSearch)) + { + $fileMatches = Get-ChildItem -Path $SourcePath -Filter $targetedFileName -Recurse + ProcessMatches($fileMatches) + } + else + { + $fileMatches = Get-ChildItem -Path $SourcePath -Filter $targetedFileName + ProcessMatches($fileMatches) + } } \ No newline at end of file From 24559cbc0a3ba5b8339b76df64bb538fd1ed4183 Mon Sep 17 00:00:00 2001 From: JohnLBevan Date: Mon, 5 Dec 2022 13:44:18 +0000 Subject: [PATCH 2/4] #43 - correct typo --- Tasks/Tokenization/task/tokenization.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tasks/Tokenization/task/tokenization.ps1 b/Tasks/Tokenization/task/tokenization.ps1 index 56e78b4..45ee371 100644 --- a/Tasks/Tokenization/task/tokenization.ps1 +++ b/Tasks/Tokenization/task/tokenization.ps1 @@ -12,13 +12,11 @@ param import-module "Microsoft.TeamFoundation.DistributedTask.Task.Internal" import-module "Microsoft.TeamFoundation.DistributedTask.Task.Common" -$patterns = @() $regex = $TokenStart + '([A-Za-z0-9.]*|_)*' + $TokenEnd -$matches = @() Write-Host (Get-LocalizedString -Key 'Regex: {0}...' -ArgumentList $regex) Write-Host (Get-LocalizedString -Key 'Source Path: {0}...' -ArgumentList $SourcePath) -Write-Host (Get-LocalizedString -Key 'Target File Name: {0}...' -ArgumentList $TargetFileName) +Write-Host (Get-LocalizedString -Key 'Target File Name: {0}...' -ArgumentList $TargetFileNames) function ProcessMatches($fileMatches) { @@ -28,7 +26,6 @@ function ProcessMatches($fileMatches) Write-Host (Get-LocalizedString -Key 'Targeted FileName Encoding: {0}...' -ArgumentList $fileEncoding) - $targetFilePath = $targetFileMatch.Directory.FullName $tempFile = $targetFileMatch.FullName + '.tmp' #Write-Host (Get-LocalizedString -Key 'Target File Path: {0}...' -ArgumentList $targetFilePath) From 9c6f0504dacb5b36d5b7b9df8ccd9ab8582a0193 Mon Sep 17 00:00:00 2001 From: JohnLBevan Date: Mon, 5 Dec 2022 13:49:09 +0000 Subject: [PATCH 3/4] #43 - remove need for temp file --- Tasks/Tokenization/task/tokenization.ps1 | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Tasks/Tokenization/task/tokenization.ps1 b/Tasks/Tokenization/task/tokenization.ps1 index 45ee371..dcbc287 100644 --- a/Tasks/Tokenization/task/tokenization.ps1 +++ b/Tasks/Tokenization/task/tokenization.ps1 @@ -25,16 +25,11 @@ function ProcessMatches($fileMatches) $fileEncoding = Get-FileEncoding($targetFileMatch.FullName) Write-Host (Get-LocalizedString -Key 'Targeted FileName Encoding: {0}...' -ArgumentList $fileEncoding) - - $tempFile = $targetFileMatch.FullName + '.tmp' - - #Write-Host (Get-LocalizedString -Key 'Target File Path: {0}...' -ArgumentList $targetFilePath) Write-Host (Get-LocalizedString -Key 'Target File Match: {0}...' -ArgumentList $targetFileMatch.FullName) - Write-Host (Get-LocalizedString -Key 'Temp File: {0}...' -ArgumentList $tempFile) - Copy-Item -Force $targetFileMatch.FullName $tempFile + $tempString = Get-Content -Path $targetFileMatch.FullName -Encoding $fileEncoding - $matches = select-string -Path $tempFile -Pattern $regex -AllMatches | % { $_.Matches } | % { $_.Value } + $matches = $tempString | select-string -Pattern $regex -AllMatches | % { $_.Matches } | % { $_.Value } ForEach ($match in $matches) { $matchedItem = $match $matchedItem = $matchedItem.TrimStart($TokenStart) @@ -51,15 +46,13 @@ function ProcessMatches($fileMatches) Write-Host (Get-LocalizedString -Key 'Token Value: {0}...' -ArgumentList $matchValue) -ForegroundColor Green - (Get-Content $tempFile) | + $tempString = $tempString | Foreach-Object { $_ -replace $match,$matchValue - } | - Out-File $tempFile -Force -Encoding $fileEncoding + } } - Copy-Item -Force $tempFile $targetFileMatch.FullName - Remove-Item -Force $tempFile + $tempString | Out-File $targetFileMatch.FullName -Force -Encoding $fileEncoding } } From 8ea2b86789c3f7407bd20bdfb435b6b174c4cafa Mon Sep 17 00:00:00 2001 From: JohnLBevan Date: Mon, 5 Dec 2022 13:50:35 +0000 Subject: [PATCH 4/4] #43 - matches is an automatic variable in PS; renamed --- Tasks/Tokenization/task/tokenization.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tasks/Tokenization/task/tokenization.ps1 b/Tasks/Tokenization/task/tokenization.ps1 index dcbc287..e0a54f5 100644 --- a/Tasks/Tokenization/task/tokenization.ps1 +++ b/Tasks/Tokenization/task/tokenization.ps1 @@ -29,8 +29,8 @@ function ProcessMatches($fileMatches) $tempString = Get-Content -Path $targetFileMatch.FullName -Encoding $fileEncoding - $matches = $tempString | select-string -Pattern $regex -AllMatches | % { $_.Matches } | % { $_.Value } - ForEach ($match in $matches) { + $allMatches = $tempString | select-string -Pattern $regex -AllMatches | % { $_.Matches } | % { $_.Value } + ForEach ($match in $allMatches) { $matchedItem = $match $matchedItem = $matchedItem.TrimStart($TokenStart) $matchedItem = $matchedItem.TrimEnd($TokenEnd)