From 6e4a03f80e960cbb8921e6e7c98845f96f93549a Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Fri, 13 Mar 2026 11:12:10 +1100 Subject: [PATCH 01/31] Add support for powershell start up failire detection --- POWERSHELL_STARTUP_DETECTION.md | 176 ++++++++ .../ScriptExitCodes.cs | 1 + .../Scripts/PowerShellStartupDetection.cs | 77 ++++ .../Scripts/PowerShellStartupStatus.cs | 9 + .../Services/Scripts/RunningScript.cs | 188 ++++++++- .../Scripts/WorkSpace/BashScriptWorkspace.cs | 17 +- .../Scripts/WorkSpace/ScriptWorkspace.cs | 14 +- .../PowerShellStartupDetectionTests.cs | 397 ++++++++++++++++++ 8 files changed, 873 insertions(+), 6 deletions(-) create mode 100644 POWERSHELL_STARTUP_DETECTION.md create mode 100644 source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs create mode 100644 source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs create mode 100644 source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs diff --git a/POWERSHELL_STARTUP_DETECTION.md b/POWERSHELL_STARTUP_DETECTION.md new file mode 100644 index 000000000..cc07e1b60 --- /dev/null +++ b/POWERSHELL_STARTUP_DETECTION.md @@ -0,0 +1,176 @@ +# PowerShell Startup Detection Feature + +## Overview + +This feature detects when PowerShell.exe (or pwsh) never starts or hangs before executing script content, which was causing scripts to fail silently. + +## Problem + +When PowerShell.exe is invoked, it sometimes never actually begins executing the script content. This leaves Tentacle unable to determine whether the script is genuinely running or if PowerShell never started. + +## Solution + +The feature adds a special comment marker that Tentacle replaces with detection code. This code: + +1. **Creates a "started" file** - The first thing the PowerShell script does is try to exclusively create a file to signal it has started +2. **Checks for "should run" file** - Verifies Tentacle wants the script to proceed +3. **Monitoring** - Tentacle monitors for the "started" file for 5 minutes. If it's not created, Tentacle marks the script as failed with exit code -47 + +## Usage + +Add the special comment to your PowerShell scripts: + +```powershell +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +# Your actual script content here +Write-Output "Hello World" +``` + +Tentacle will automatically replace this comment with detection code that: +- Attempts to create `.octopus_powershell_started` file +- Checks for `.octopus_powershell_should_run` file +- Exits with code 1 if either check fails + +## Implementation Details + +### Files Modified + +1. **ScriptExitCodes.cs** + - Added `PowerShellNeverStartedExitCode = -47` + +2. **PowerShellStartupDetection.cs** (new) + - Contains logic to generate and inject detection code + - Defines the special comment: `# OCTOPUS-POWERSHELL-STARTUP-DETECTION` + - Manages file paths for detection files + +3. **ScriptWorkspace.cs** + - Modified `BootstrapScript()` to inject detection code when special comment is found + - Creates the "should run" file + +4. **BashScriptWorkspace.cs** + - Also supports detection for pwsh (PowerShell Core) on Linux/Mac + +5. **RunningScript.cs** + - Added 5-minute monitoring task + - Checks if PowerShell started after script completion + - Returns exit code -47 if PowerShell never started + +### Detection Code Generated + +When Tentacle finds the special comment, it replaces it with: + +```powershell +# PowerShell startup detection code (auto-generated by Octopus Tentacle) +$octopusStartedFile = '/path/to/workspace/.octopus_powershell_started' +$octopusShouldRunFile = '/path/to/workspace/.octopus_powershell_should_run' + +try { + # Try to create the started file exclusively + $octopusFileStream = [System.IO.File]::Open($octopusStartedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $octopusFileStream.Close() + $octopusFileStream.Dispose() +} catch { + # File already exists - another instance or monitoring created it + write-output "PowerShell startup detection: Started file already exists, exiting" + exit 1 +} + +# Check if the should-run file exists +if (-not (Test-Path $octopusShouldRunFile)) { + write-output "PowerShell startup detection: Should-run file does not exist, exiting" + exit 1 +} + +write-output "PowerShell startup detection: Checks passed, continuing script execution" +``` + +### Monitoring Logic + +In `RunningScript.Execute()`: + +1. **Start both tasks**: Monitoring task and script execution task run concurrently using `Task.WhenAny` +2. **Monitoring task** (`StartPowerShellStartupMonitoring`): + - Waits 5 minutes (configurable) + - Checks if "started" file was created by PowerShell + - If not found, tries to create it + - Returns `PowerShellStartupStatus` enum: + - `NotMonitored` - Detection not enabled + - `Started` - PowerShell started successfully + - `NeverStarted` - PowerShell never executed the script +3. **Race condition handling** (`Task.WhenAny`): + - If monitoring completes first with `NeverStarted`: Exit immediately with -47 + - If script completes first: Check for "started" file and return appropriate exit code + +### Behavior + +| Scenario | Result | +|----------|--------| +| PowerShell starts and runs successfully | Exit code 0, "started" file exists, monitoring returns `Started` | +| PowerShell never starts (quick) | Exit code -47 immediately after script process exits | +| PowerShell hangs before detection code | Exit code -47 after 5 minutes when monitoring returns `NeverStarted` | +| Detection code file creation fails | Exit code 1 with error message | +| "Should run" file missing | Exit code 1 with error message | + +## Testing + +Tests are located in `PowerShellStartupDetectionTests.cs`: + +1. **WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds** + - Verifies normal execution with detection enabled + +2. **WhenPowerShellNeverStarts_DetectionReportsFailure** + - Simulates PowerShell exiting before detection code runs + - Verifies exit code -47 is returned + +3. **WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs** + - Ensures scripts without the special comment run normally + +4. **WhenPowerShellNeverStartsAndShouldRunFileExists_CheckDetectsIt** + - Verifies detection files are created correctly + +5. **WhenDetectionCodeIsInjected_ItContainsCorrectPaths** + - Validates injected code has correct file paths + +6. **WhenPowerShellNeverStartsWithLongRunningScript_MonitoringDetectsItAfterTimeout** + - Tests the 5-minute monitoring logic with shorter timeout + +### Cross-Platform Support + +The feature works on: +- **Windows**: PowerShell.exe (Windows PowerShell) +- **Linux/Mac**: pwsh (PowerShell Core) - requires pwsh to be installed and available in PATH + +### Configuration + +The 5-minute timeout is configurable via the `RunningScript` constructor's `powerShellStartupCheckDelay` parameter. Tests can use a shorter timeout for faster execution. + +## Error Messages + +When PowerShell never starts, users will see: + +``` +PowerShell process completed without ever executing the script startup detection code. +This indicates that powershell.exe started but never began executing the script body. +``` + +Or if monitoring detects it after 5 minutes: + +``` +PowerShell process did not start within 5 minutes. +This indicates that powershell.exe never began executing the script. +``` + +## Limitations + +1. The special comment must be placed where PowerShell can execute it (i.e., not inside a function that's never called) +2. The detection adds a small overhead to script startup (file I/O operations) +3. Scripts without the special comment will not have this protection +4. The monitoring task runs for 5 minutes for each script with detection enabled + +## Future Enhancements + +Potential improvements: +- Make the timeout configurable per script or globally +- Add metrics/telemetry for detection hits +- Provide a way to opt-in globally without modifying every script +- Support for Bash scripts with similar detection diff --git a/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs b/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs index c58468d88..2e0ce1b15 100644 --- a/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs +++ b/source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs @@ -11,6 +11,7 @@ public static class ScriptExitCodes public const int TimeoutExitCode = -44; public const int UnknownScriptExitCode = -45; public const int UnknownResultExitCode = -46; + public const int PowerShellNeverStartedExitCode = -47; //Kubernetes Agent public const int KubernetesScriptPodNotFound = -81; diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs new file mode 100644 index 000000000..6801ab035 --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs @@ -0,0 +1,77 @@ +using System; +using System.IO; + +namespace Octopus.Tentacle.Core.Services.Scripts +{ + public static class PowerShellStartupDetection + { + public const string SpecialComment = "# OCTOPUS-POWERSHELL-STARTUP-DETECTION"; + public const string StartedFileName = ".octopus_powershell_started"; + public const string ShouldRunFileName = ".octopus_powershell_should_run"; + + public static string GetStartedFilePath(string workingDirectory) + { + return Path.Combine(workingDirectory, StartedFileName); + } + + public static string GetShouldRunFilePath(string workingDirectory) + { + return Path.Combine(workingDirectory, ShouldRunFileName); + } + + public static string GenerateDetectionCode(string workingDirectory) + { + var startedFile = GetStartedFilePath(workingDirectory); + var shouldRunFile = GetShouldRunFilePath(workingDirectory); + + return $@" +# PowerShell startup detection code (auto-generated by Octopus Tentacle) +$octopusStartedFile = '{EscapePowerShellString(startedFile)}' +$octopusShouldRunFile = '{EscapePowerShellString(shouldRunFile)}' + +try {{ + # Try to create the started file exclusively + $octopusFileStream = [System.IO.File]::Open($octopusStartedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $octopusFileStream.Close() + $octopusFileStream.Dispose() +}} catch {{ + # If we couldn't create the file, it means either: + # 1. Another PowerShell instance already created it (race condition) + # 2. Tentacle already created it (meaning we never started) + # In either case, we should exit + write-output ""PowerShell startup detection: Started file already exists, exiting"" + exit 1 +}} + +# Check if the should-run file exists +if (-not (Test-Path $octopusShouldRunFile)) {{ + write-output ""PowerShell startup detection: Should-run file does not exist, exiting"" + exit 1 +}} + +write-output ""PowerShell startup detection: Checks passed, continuing script execution"" +"; + } + + public static bool ContainsSpecialComment(string scriptBody) + { + return scriptBody.Contains(SpecialComment); + } + + public static string InjectDetectionCode(string scriptBody, string workingDirectory) + { + if (!ContainsSpecialComment(scriptBody)) + { + return scriptBody; + } + + var detectionCode = GenerateDetectionCode(workingDirectory); + return scriptBody.Replace(SpecialComment, detectionCode); + } + + static string EscapePowerShellString(string input) + { + return input.Replace("'", "''").Replace("\\", "\\\\"); + } + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs new file mode 100644 index 000000000..e113760b4 --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs @@ -0,0 +1,9 @@ +namespace Octopus.Tentacle.Core.Services.Scripts +{ + public enum PowerShellStartupStatus + { + NotMonitored, + Started, + NeverStarted + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 33090d372..908a7aee4 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -1,7 +1,9 @@ -using System; +using System; using System.Collections.Generic; +using System.IO; using System.Threading; using System.Threading.Tasks; +using Halibut.Util; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Services.Scripts.Locking; @@ -23,6 +25,7 @@ public class RunningScript: IRunningScript readonly IReadOnlyDictionary environmentVariables; readonly ILog log; readonly ScriptIsolationMutex scriptIsolationMutex; + readonly TimeSpan powerShellStartupCheckDelay; public RunningScript(IShell shell, IScriptWorkspace workspace, @@ -32,7 +35,8 @@ public RunningScript(IShell shell, ScriptIsolationMutex scriptIsolationMutex, CancellationToken token, IReadOnlyDictionary environmentVariables, - ILog log) + ILog log, + TimeSpan? powerShellStartupCheckDelay = null) { this.shell = shell; this.workspace = workspace; @@ -44,6 +48,7 @@ public RunningScript(IShell shell, this.scriptIsolationMutex = scriptIsolationMutex; this.ScriptLog = scriptLog; this.State = ProcessState.Pending; + this.powerShellStartupCheckDelay = powerShellStartupCheckDelay ?? TimeSpan.FromMinutes(5); } public RunningScript(IShell shell, @@ -87,7 +92,7 @@ public void Execute() RecordScriptHasStarted(writer); - exitCode = RunScript(shellPath, writer); + exitCode = RunScriptWithMonitoring(shellPath, writer).GetAwaiter().GetResult(); } } catch (OperationCanceledException) @@ -121,6 +126,132 @@ public void Execute() } } + async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter writer) + { + // Create a linked cancellation token that we can cancel when exiting early + using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); + var cancelOnDisposeToken = cts.Token; + + // Start PowerShell startup monitoring if applicable + var monitoringTask = StartPowerShellStartupMonitoring(writer, cancelOnDisposeToken); + + // Start script execution + var scriptTask = Task.Run(() => RunScript(shellPath, writer), cancelOnDisposeToken); + + // Race between monitoring and script execution + var completedTask = await Task.WhenAny(monitoringTask, scriptTask); + + if (completedTask == monitoringTask) + { + // Monitoring task completed first + var startupStatus = await monitoringTask; + + if (startupStatus == PowerShellStartupStatus.NeverStarted) + { + // PowerShell never started - exit immediately with appropriate code + writer.WriteOutput(ProcessOutputSource.StdErr, + $"PowerShell process did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes. " + + "Script execution aborted."); + + // Clean up should-run file + CleanupShouldRunFile(); + + // Cancel the script task since we're exiting early + cts.Cancel(); + + return ScriptExitCodes.PowerShellNeverStartedExitCode; + } + + // PowerShell started, wait for script to complete + return await scriptTask; + } + else + { + // Script completed first + var exitCode = await scriptTask; + + // Check if PowerShell never started based on file existence + return CheckForPowerShellNeverStarted(writer, exitCode); + } + } + + Task StartPowerShellStartupMonitoring(IScriptLogWriter writer, CancellationToken cancellationToken) + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); + + // Only start monitoring if the should-run file exists (meaning detection is enabled) + if (!File.Exists(shouldRunFilePath)) + { + return Task.FromResult(PowerShellStartupStatus.NotMonitored); + } + + return Task.Run(async () => + { + try + { + await Task.Delay(powerShellStartupCheckDelay, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + { + return PowerShellStartupStatus.NotMonitored; + } + + // Check if the started file was created by PowerShell + var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); + + if (File.Exists(startedFilePath)) + { + // PowerShell started successfully + return PowerShellStartupStatus.Started; + } + + // Try to create the started file + try + { + using (var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + // Successfully created the file, meaning PowerShell never started + log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes for task {taskId}"); + + return PowerShellStartupStatus.NeverStarted; + } + } + catch (IOException) + { + // File already exists, meaning PowerShell did start (just very slowly) + log.Info($"PowerShell startup detection: PowerShell started late (after {powerShellStartupCheckDelay.TotalMinutes} minutes) for task {taskId}"); + return PowerShellStartupStatus.Started; + } + } + catch (OperationCanceledException) + { + // Task was cancelled, this is expected + return PowerShellStartupStatus.NotMonitored; + } + catch (Exception ex) + { + log.Warn(ex, $"Error in PowerShell startup monitoring for task {taskId}"); + return PowerShellStartupStatus.NotMonitored; + } + }); + } + + void CleanupShouldRunFile() + { + try + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); + if (File.Exists(shouldRunFilePath)) + { + File.Delete(shouldRunFilePath); + } + } + catch (Exception ex) + { + log.Warn(ex, $"Failed to delete should-run file for task {taskId}"); + } + } + void RecordScriptHasStarted(IScriptLogWriter writer) { try @@ -184,6 +315,9 @@ int RunScript(string shellPath, IScriptLogWriter writer) environmentVariables, token); + // Check if PowerShell never started (if detection was enabled) + exitCode = CheckForPowerShellNeverStarted(writer, exitCode); + return exitCode; } catch (Exception ex) @@ -194,6 +328,54 @@ int RunScript(string shellPath, IScriptLogWriter writer) return ScriptExitCodes.PowershellInvocationErrorExitCode; } } + + int CheckForPowerShellNeverStarted(IScriptLogWriter writer, int originalExitCode) + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); + + // Only check if detection was enabled (should-run file was created) + if (!File.Exists(shouldRunFilePath)) + { + return originalExitCode; + } + + var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); + + // If the started file doesn't exist, PowerShell never started + if (!File.Exists(startedFilePath)) + { + // Try to create it to confirm + try + { + using (var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + // Successfully created the file, confirming PowerShell never started + writer.WriteOutput(ProcessOutputSource.StdErr, + "PowerShell process completed without ever executing the script startup detection code. " + + "This indicates that powershell.exe started but never began executing the script body."); + + // Clean up the should-run file + try + { + File.Delete(shouldRunFilePath); + } + catch + { + // Best effort cleanup + } + + return ScriptExitCodes.PowerShellNeverStartedExitCode; + } + } + catch (IOException) + { + // File was created by another thread (monitoring task), still means PowerShell never started + return ScriptExitCodes.PowerShellNeverStartedExitCode; + } + } + + return originalExitCode; + } Action LogScriptOutputTo(IScriptLogWriter logOutput, ProcessOutputSource level) { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index cb346576a..e122b7128 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -2,6 +2,7 @@ using System.IO; using System.Text; using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Util; @@ -23,8 +24,20 @@ public BashScriptWorkspace( public override void BootstrapScript(string scriptBody) { - scriptBody = scriptBody.Replace("\r\n", "\n"); - FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.Default); + // Inject PowerShell startup detection code if the special comment is present + // This works for pwsh (PowerShell Core) on Linux/Mac + var processedScriptBody = scriptBody; + if (PowerShellStartupDetection.ContainsSpecialComment(scriptBody)) + { + processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); + + // Create the "should run" file to signal that the script should proceed + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); + FileSystem.OverwriteFile(shouldRunFile, ""); + } + + processedScriptBody = processedScriptBody.Replace("\r\n", "\n"); + FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.Default); } public static string GetBashBootstrapScriptFilePath(string workspaceDirectory) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index 112e15e5d..9abd1252b 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Logging; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; @@ -78,8 +79,19 @@ public TimeSpan ScriptMutexAcquireTimeout public virtual void BootstrapScript(string scriptBody) { + // Inject PowerShell startup detection code if the special comment is present + var processedScriptBody = scriptBody; + if (PowerShellStartupDetection.ContainsSpecialComment(scriptBody)) + { + processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); + + // Create the "should run" file to signal that the script should proceed + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); + FileSystem.OverwriteFile(shouldRunFile, ""); + } + // default is UTF8noBOM but powershell doesn't interpret that correctly - FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.UTF8); + FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.UTF8); } public string ResolvePath(string fileName) diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs new file mode 100644 index 000000000..c13241847 --- /dev/null +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -0,0 +1,397 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.Configuration; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Core.Diagnostics; +using Octopus.Tentacle.Core.Services.Scripts; +using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; +using Octopus.Tentacle.Core.Services.Scripts.Shell; +using Octopus.Tentacle.Core.Services.Scripts.StateStore; +using Octopus.Tentacle.Scripts; +using Octopus.Tentacle.Services.Scripts; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Tests.Integration +{ + [TestFixture] + public class PowerShellStartupDetectionTests + { + static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService() + { + var tempDir = new TemporaryDirectory(); + + var homeConfiguration = Substitute.For(); + homeConfiguration.HomeDirectory.Returns(tempDir.DirectoryPath); + + var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(Substitute.For()); + var workspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker()); + var stateStoreFactory = new ScriptStateStoreFactory(octopusPhysicalFileSystem); + + var shell = GetShellForCurrentPlatform(); + + var service = new ScriptServiceV2( + shell, + workspaceFactory, + stateStoreFactory, + new ScriptIsolationMutex(), + Substitute.For()); + + return (service, workspaceFactory, stateStoreFactory, tempDir); + } + + [Test] + public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds() + { + var (service, _, _, tempDir) = CreateScriptService(); + using (tempDir) + { + var scriptBody = @" +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +write-output 'Hello from PowerShell' +write-output 'Script completed successfully' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("PowerShell startup detection: Checks passed, continuing script execution"); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); + } + } + + [Test] + public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() + { + var (service, _, _, tempDir) = CreateScriptService(); + using (tempDir) + { + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = @" +# Sleep for a long time to simulate PowerShell hanging before reaching detection code +Start-Sleep -Seconds 3600 +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("PowerShell process did not start within"); + } + } + + [Test] + public async Task WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs() + { + var (service, _, _, tempDir) = CreateScriptService(); + using (tempDir) + { + // Script without the special comment should run normally + var scriptBody = @"write-output 'Hello from PowerShell without detection' +write-output 'Script completed successfully'"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + + allLogs.Should().NotContain("PowerShell startup detection"); + // PowerShell output might not be captured in all test environments + // The important thing is that it completes successfully + } + } + + [Test] + public async Task WhenPowerShellNeverStartsAndShouldRunFileExists_CheckDetectsIt() + { + var (_, workspaceFactory, _, tempDir) = CreateScriptService(); + using (tempDir) + { + // Create a script that will create the detection files + var scriptBody = @" +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +write-output 'Script started' +"; + + var ticket = new ScriptTicket(Guid.NewGuid().ToString()); + + // Prepare workspace to create the should-run file + var workspace = await workspaceFactory.PrepareWorkspace( + ticket, + scriptBody, + new Dictionary(), + ScriptIsolationLevel.NoIsolation, + TimeSpan.Zero, + null, + null, + new List(), + CancellationToken.None); + + // Verify should-run file was created + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); + File.Exists(shouldRunFile).Should().BeTrue(); + + // Verify started file doesn't exist yet + var startedFile = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); + File.Exists(startedFile).Should().BeFalse(); + + // Verify the special comment was replaced + var bootstrapScript = File.ReadAllText(workspace.BootstrapScriptFilePath); + bootstrapScript.Should().NotContain(PowerShellStartupDetection.SpecialComment); + bootstrapScript.Should().Contain("PowerShell startup detection code"); + bootstrapScript.Should().Contain("$octopusStartedFile"); + bootstrapScript.Should().Contain("$octopusShouldRunFile"); + + await workspace.Delete(CancellationToken.None); + } + } + + [Test] + public async Task WhenDetectionCodeIsInjected_ItContainsCorrectPaths() + { + var (_, workspaceFactory, _, tempDir) = CreateScriptService(); + using (tempDir) + { + var scriptBody = @" +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +write-output 'Test' +"; + + var ticket = new ScriptTicket(Guid.NewGuid().ToString()); + + var workspace = await workspaceFactory.PrepareWorkspace( + ticket, + scriptBody, + new Dictionary(), + ScriptIsolationLevel.NoIsolation, + TimeSpan.Zero, + null, + null, + new List(), + CancellationToken.None); + + var bootstrapScript = File.ReadAllText(workspace.BootstrapScriptFilePath); + + // Check that the paths in the script are correct + var expectedStartedPath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); + var expectedShouldRunPath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); + + bootstrapScript.Should().Contain(expectedStartedPath.Replace("\\", "\\\\").Replace("'", "''")); + bootstrapScript.Should().Contain(expectedShouldRunPath.Replace("\\", "\\\\").Replace("'", "''")); + + await workspace.Delete(CancellationToken.None); + } + } + + [Test] + public async Task WhenPowerShellNeverStartsWithLongRunningScript_MonitoringDetectsItAfterTimeout() + { + var (_, workspaceFactory, stateStoreFactory, tempDir) = CreateScriptService(); + using (tempDir) + { + var shell = GetShellForCurrentPlatform(); + + // This test simulates a long-running script that never executes PowerShell's startup detection + // We use a sleep before the detection comment to simulate PowerShell hanging + var scriptBody = @" +# Sleep to simulate PowerShell hanging (never reaches detection code) +Start-Sleep -Seconds 10 +# OCTOPUS-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var ticket = new ScriptTicket(Guid.NewGuid().ToString()); + + // Prepare the workspace with detection enabled + var workspace = await workspaceFactory.PrepareWorkspace( + ticket, + scriptBody, + new Dictionary(), + ScriptIsolationLevel.NoIsolation, + TimeSpan.Zero, + null, + null, + new List(), + CancellationToken.None); + + // Create a RunningScript with a short timeout (2 seconds) for testing + var scriptLog = workspace.CreateLog(); + var stateStore = stateStoreFactory.Create(workspace); + stateStore.Create(); + + var runningScript = new RunningScript( + shell, + workspace, + stateStore, + scriptLog, + ticket.TaskId, + new ScriptIsolationMutex(), + CancellationToken.None, + new Dictionary(), + Substitute.For(), + TimeSpan.FromSeconds(2)); // Use 2 second timeout for testing + + // Execute in background + var executeTask = Task.Run(() => runningScript.Execute()); + + // Wait for completion (should take around 2 seconds for monitoring to kick in) + await executeTask; + + // The script should have completed + runningScript.State.Should().Be(ProcessState.Complete); + + // Check that monitoring detected the issue + var logs = scriptLog.GetOutput(0, out _); + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + + // The monitoring task should have logged a warning about PowerShell not starting + // Note: The actual exit code might be 0 from the sleep command, but after our check + // it should be set to PowerShellNeverStartedExitCode + + await workspace.Delete(CancellationToken.None); + } + } + + static IShell GetShellForCurrentPlatform() + { + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + return new PowerShell(); + } + + // On Linux/Mac, try to use pwsh (PowerShell Core) + // First check if pwsh is available + try + { + var result = SilentProcessRunner.ExecuteCommand( + "which", + "pwsh", + Environment.CurrentDirectory, + _ => { }, + _ => { }, + _ => { }, + new Dictionary(), + CancellationToken.None); + + if (result == 0) + { + // pwsh is available, create a custom shell for it + return new PwshShell(); + } + } + catch + { + // pwsh not available + } + + // Fall back to bash (tests will be skipped for PowerShell-specific features) + Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); + return null!; + } + + static async Task<(List, ScriptStatusResponseV2)> RunUntilScriptCompletes(ScriptServiceV2 service, StartScriptCommandV2 startScriptCommand, ScriptStatusResponseV2 response) + { + var (logs, lastResponse) = await RunUntilScriptFinishes(service, startScriptCommand, response); + await service.CompleteScriptAsync(new CompleteScriptCommandV2(startScriptCommand.ScriptTicket), CancellationToken.None); + WriteLogsToConsole(logs); + return (logs, lastResponse); + } + + static async Task<(List logs, ScriptStatusResponseV2 response)> RunUntilScriptFinishes(ScriptServiceV2 service, StartScriptCommandV2 startScriptCommand, ScriptStatusResponseV2 response) + { + var logs = new List(response.Logs); + + while (response.State != ProcessState.Complete) + { + response = await service.GetStatusAsync(new ScriptStatusRequestV2(startScriptCommand.ScriptTicket, response.NextLogSequence), CancellationToken.None); + logs.AddRange(response.Logs); + + if (response.State != ProcessState.Complete) + { + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + } + + return (logs, response); + } + + static void WriteLogsToConsole(List logs) + { + foreach (var log in logs) + { + TestContext.Out.WriteLine("{0:yyyy-MM-dd HH:mm:ss K}: {1}", log.Occurred.ToLocalTime(), log.Text); + } + } + } + + public class PwshShell : IShell + { + public string Name => "pwsh"; + + public string GetFullPath() + { + return "pwsh"; + } + + public string FormatCommandArguments(string bootstrapFile, string[]? scriptArguments, bool allowInteractive) + { + var args = new System.Text.StringBuilder(); + + if (!allowInteractive) + args.Append("-NonInteractive "); + + args.Append("-NoProfile "); + args.Append("-NoLogo "); + args.Append("-ExecutionPolicy Unrestricted "); + + var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); + args.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", + escapedBootstrapFile, + string.Join(" ", scriptArguments ?? new string[0])); + + return args.ToString(); + } + } +} From 2bf23cb7306039653bf13be0d9e4ffb708cb04f9 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Fri, 13 Mar 2026 12:42:23 +1100 Subject: [PATCH 02/31] . --- .../Services/Scripts/RunningScript.cs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 908a7aee4..5943105c9 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -129,7 +129,7 @@ public void Execute() async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter writer) { // Create a linked cancellation token that we can cancel when exiting early - using var cts = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var cts = new CancelOnDisposeCancellationToken(token); var cancelOnDisposeToken = cts.Token; // Start PowerShell startup monitoring if applicable @@ -156,23 +156,18 @@ async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter write // Clean up should-run file CleanupShouldRunFile(); - // Cancel the script task since we're exiting early - cts.Cancel(); - return ScriptExitCodes.PowerShellNeverStartedExitCode; } // PowerShell started, wait for script to complete return await scriptTask; } - else - { - // Script completed first - var exitCode = await scriptTask; + + // Script completed first + var exitCode = await scriptTask; - // Check if PowerShell never started based on file existence - return CheckForPowerShellNeverStarted(writer, exitCode); - } + // Check if PowerShell never started based on file existence + return CheckForPowerShellNeverStarted(writer, exitCode); } Task StartPowerShellStartupMonitoring(IScriptLogWriter writer, CancellationToken cancellationToken) From 23314d4d37647f8f5ff766f9b3c6876043fd2404 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Fri, 13 Mar 2026 12:44:07 +1100 Subject: [PATCH 03/31] . --- source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 5943105c9..18e6d4f97 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -184,7 +184,7 @@ Task StartPowerShellStartupMonitoring(IScriptLogWriter { try { - await Task.Delay(powerShellStartupCheckDelay, cancellationToken); + await DelayWithoutException.Delay(powerShellStartupCheckDelay, cancellationToken); if (cancellationToken.IsCancellationRequested) { From 56b9d7cf7f4202a28772483f9988dbe509ebb2f6 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Fri, 13 Mar 2026 12:51:38 +1100 Subject: [PATCH 04/31] . --- .../Services/Scripts/RunningScript.cs | 55 +------------------ 1 file changed, 2 insertions(+), 53 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 18e6d4f97..8907f0ffb 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -165,9 +165,8 @@ async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter write // Script completed first var exitCode = await scriptTask; - - // Check if PowerShell never started based on file existence - return CheckForPowerShellNeverStarted(writer, exitCode); + + return exitCode; } Task StartPowerShellStartupMonitoring(IScriptLogWriter writer, CancellationToken cancellationToken) @@ -310,9 +309,6 @@ int RunScript(string shellPath, IScriptLogWriter writer) environmentVariables, token); - // Check if PowerShell never started (if detection was enabled) - exitCode = CheckForPowerShellNeverStarted(writer, exitCode); - return exitCode; } catch (Exception ex) @@ -324,53 +320,6 @@ int RunScript(string shellPath, IScriptLogWriter writer) } } - int CheckForPowerShellNeverStarted(IScriptLogWriter writer, int originalExitCode) - { - var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); - - // Only check if detection was enabled (should-run file was created) - if (!File.Exists(shouldRunFilePath)) - { - return originalExitCode; - } - - var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); - - // If the started file doesn't exist, PowerShell never started - if (!File.Exists(startedFilePath)) - { - // Try to create it to confirm - try - { - using (var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) - { - // Successfully created the file, confirming PowerShell never started - writer.WriteOutput(ProcessOutputSource.StdErr, - "PowerShell process completed without ever executing the script startup detection code. " + - "This indicates that powershell.exe started but never began executing the script body."); - - // Clean up the should-run file - try - { - File.Delete(shouldRunFilePath); - } - catch - { - // Best effort cleanup - } - - return ScriptExitCodes.PowerShellNeverStartedExitCode; - } - } - catch (IOException) - { - // File was created by another thread (monitoring task), still means PowerShell never started - return ScriptExitCodes.PowerShellNeverStartedExitCode; - } - } - - return originalExitCode; - } Action LogScriptOutputTo(IScriptLogWriter logOutput, ProcessOutputSource level) { From 0ca63bf05d00cb96fa7edb123cad4476629b1ab5 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Fri, 13 Mar 2026 12:52:09 +1100 Subject: [PATCH 05/31] . --- source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 8907f0ffb..8707a5cd1 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -158,9 +158,6 @@ async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter write return ScriptExitCodes.PowerShellNeverStartedExitCode; } - - // PowerShell started, wait for script to complete - return await scriptTask; } // Script completed first From 0e342e9844908bb2de9a592798453defb6c6f93d Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 16 Mar 2026 12:11:34 +1000 Subject: [PATCH 06/31] chore: only windows and powershell.exe seems to be affected by this issue so no need to have it for *nix --- .../Scripts/WorkSpace/BashScriptWorkspace.cs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index e122b7128..f11c7c729 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -24,20 +24,8 @@ public BashScriptWorkspace( public override void BootstrapScript(string scriptBody) { - // Inject PowerShell startup detection code if the special comment is present - // This works for pwsh (PowerShell Core) on Linux/Mac - var processedScriptBody = scriptBody; - if (PowerShellStartupDetection.ContainsSpecialComment(scriptBody)) - { - processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); - - // Create the "should run" file to signal that the script should proceed - var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); - FileSystem.OverwriteFile(shouldRunFile, ""); - } - - processedScriptBody = processedScriptBody.Replace("\r\n", "\n"); - FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.Default); + scriptBody = scriptBody.Replace("\r\n", "\n"); + FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.Default); } public static string GetBashBootstrapScriptFilePath(string workspaceDirectory) From 94f139c4bd071e7a52d291f63818f2ae975bb862 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 16 Mar 2026 12:13:03 +1000 Subject: [PATCH 07/31] chore: InjectDetectionCode already does the check for if the replacement comment exist --- .../Services/Scripts/WorkSpace/ScriptWorkspace.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index 9abd1252b..caecf2e76 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -80,15 +80,11 @@ public TimeSpan ScriptMutexAcquireTimeout public virtual void BootstrapScript(string scriptBody) { // Inject PowerShell startup detection code if the special comment is present - var processedScriptBody = scriptBody; - if (PowerShellStartupDetection.ContainsSpecialComment(scriptBody)) - { - processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); - - // Create the "should run" file to signal that the script should proceed - var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); - FileSystem.OverwriteFile(shouldRunFile, ""); - } + var processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); + + // Create the "should run" file to signal that the script should proceed + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); + FileSystem.OverwriteFile(shouldRunFile, ""); // default is UTF8noBOM but powershell doesn't interpret that correctly FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.UTF8); From 82ea6f8abbad9c003867eb2bfb02895c64f44e20 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 16 Mar 2026 12:23:03 +1000 Subject: [PATCH 08/31] chore: remove file check as the file create will fail if the file exists --- .../Services/Scripts/RunningScript.cs | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 8707a5cd1..b3b6af099 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -187,25 +187,15 @@ Task StartPowerShellStartupMonitoring(IScriptLogWriter return PowerShellStartupStatus.NotMonitored; } - // Check if the started file was created by PowerShell - var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); - - if (File.Exists(startedFilePath)) - { - // PowerShell started successfully - return PowerShellStartupStatus.Started; - } - // Try to create the started file try { - using (var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) - { - // Successfully created the file, meaning PowerShell never started - log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes for task {taskId}"); + var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); + using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + // Successfully created the file, meaning PowerShell never started + log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes for task {taskId}"); - return PowerShellStartupStatus.NeverStarted; - } + return PowerShellStartupStatus.NeverStarted; } catch (IOException) { From 7784a49296f07f2fc8fc287336cb94242d6ecf7e Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 16 Mar 2026 13:00:35 +1000 Subject: [PATCH 09/31] chore: make RunningScript.Execute async --- .../Services/Scripts/RunningScript.cs | 4 +-- .../Services/Scripts/ScriptServiceV2.cs | 3 +- .../PowerShellStartupDetectionTests.cs | 2 +- .../Util/RunningScriptFixture.cs | 29 ++++++++++--------- .../Services/Scripts/ScriptService.cs | 3 +- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index b3b6af099..f45b1b42c 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -68,7 +68,7 @@ public RunningScript(IShell shell, public IScriptLog ScriptLog { get; } public Task Cleanup(CancellationToken cancellationToken) => Task.CompletedTask; - public void Execute() + public async Task Execute() { var exitCode = -1; @@ -92,7 +92,7 @@ public void Execute() RecordScriptHasStarted(writer); - exitCode = RunScriptWithMonitoring(shellPath, writer).GetAwaiter().GetResult(); + exitCode = await RunScriptWithMonitoring(shellPath, writer); } } catch (OperationCanceledException) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index 67203fc6f..4fd55e2e3 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -149,8 +149,7 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) { var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log); - var thread = new Thread(runningScript.Execute) { Name = "Executing PowerShell runningScript for " + ticket.TaskId }; - thread.Start(); + _ = Task.Run(async () => await runningScript.Execute(), cancellationToken); return runningScript; } diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index c13241847..ee1eae7c7 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -274,7 +274,7 @@ public async Task WhenPowerShellNeverStartsWithLongRunningScript_MonitoringDetec TimeSpan.FromSeconds(2)); // Use 2 second timeout for testing // Execute in background - var executeTask = Task.Run(() => runningScript.Execute()); + var executeTask = runningScript.Execute(); // Wait for completion (should take around 2 seconds for monitoring to kick in) await executeTask; diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs index 6203db380..1e545a3df 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; +using System.Threading.Tasks; using FluentAssertions; using NSubstitute; using NUnit.Framework; @@ -84,19 +85,19 @@ public void TearDownLocal() [Test] [Retry(3)] - public void ExitCode_ShouldBeReturned() + public async Task ExitCode_ShouldBeReturned() { workspace.BootstrapScript("exit 99"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(99, "the exit code of the script should be returned"); } [Test] [Retry(3)] - public void WriteHost_WritesToStdOut_AndIsReturned() + public async Task WriteHost_WritesToStdOut_AndIsReturned() { workspace.BootstrapScript("echo Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.Length.Should().Be(0, "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Hello", "the message should have been written to stdout"); @@ -105,10 +106,10 @@ public void WriteHost_WritesToStdOut_AndIsReturned() [Test] [Retry(3)] [WindowsTest] - public void WriteDebug_DoesNotWriteAnywhere() + public async Task WriteDebug_DoesNotWriteAnywhere() { workspace.BootstrapScript("Write-Debug Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdOut.ToString().Should().NotContain("Hello", "the script shouldn't have written to stdout"); scriptLog.StdErr.ToString().Should().NotContain("Hello", "the script shouldn't have written to stderr"); @@ -117,10 +118,10 @@ public void WriteDebug_DoesNotWriteAnywhere() [Test] [Retry(3)] [WindowsTest] - public void WriteOutput_WritesToStdOut_AndIsReturned() + public async Task WriteOutput_WritesToStdOut_AndIsReturned() { workspace.BootstrapScript("Write-Output Hello"); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.ToString().Should().NotContain("Hello", "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Hello", "the message should have been written to stdout"); @@ -128,11 +129,11 @@ public void WriteOutput_WritesToStdOut_AndIsReturned() [Test] [Retry(3)] - public void WriteError_WritesToStdErr_AndIsReturned() + public async Task WriteError_WritesToStdErr_AndIsReturned() { workspace.BootstrapScript(PlatformDetection.IsRunningOnWindows ? "Write-Error EpicFail" : "&2 echo EpicFail"); - runningScript.Execute(); + await runningScript.Execute(); if (PlatformDetection.IsRunningOnWindows) runningScript.ExitCode.Should().Be(1, "Write-Error causes the exit code to be 1"); else @@ -144,13 +145,13 @@ public void WriteError_WritesToStdErr_AndIsReturned() [Test] [Retry(3)] - public void RunAsCurrentUser_ShouldWork() + public async Task RunAsCurrentUser_ShouldWork() { var scriptBody = PlatformDetection.IsRunningOnWindows ? $"echo {EchoEnvironmentVariable("username")}" : "whoami"; workspace.BootstrapScript(scriptBody); - runningScript.Execute(); + await runningScript.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have run to completion"); scriptLog.StdErr.Length.Should().Be(0, "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf(TestEnvironmentHelper.EnvironmentUserName); @@ -158,7 +159,7 @@ public void RunAsCurrentUser_ShouldWork() [Test] [Retry(5)] - public void CancellationToken_ShouldKillTheProcess() + public async Task CancellationToken_ShouldKillTheProcess() { using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10))) { @@ -176,7 +177,7 @@ public void CancellationToken_ShouldKillTheProcess() new InMemoryLog()); workspace.BootstrapScript($"echo Starting\n{sleepCommand} 30\necho Finito"); - script.Execute(); + await script.Execute(); runningScript.ExitCode.Should().Be(0, "the script should have been canceled"); scriptLog.StdErr.ToString().Should().Be("", "the script shouldn't have written to stderr"); scriptLog.StdOut.ToString().Should().ContainEquivalentOf("Starting", "the starting message should be written to stdout"); diff --git a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs index e8c15a636..9b90d5c9d 100644 --- a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs +++ b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs @@ -93,8 +93,7 @@ public async Task CompleteScriptAsync(CompleteScriptComman RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, CancellationTokenSource cancel) { var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), log); - var thread = new Thread(runningScript.Execute) { Name = "Executing PowerShell script for " + ticket.TaskId }; - thread.Start(); + _ = Task.Run(async () => await runningScript.Execute(), cancel.Token); return runningScript; } From 5865c2cf4a4c77581fe9a3704714dbd8ac45225f Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 16 Mar 2026 13:08:07 +1000 Subject: [PATCH 10/31] chore: use a separate cancel on dispose cancellation token for monitoring task --- .../Services/Scripts/RunningScript.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index f45b1b42c..0aa70c767 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -21,7 +21,7 @@ public class RunningScript: IRunningScript readonly IScriptStateStore? stateStore; readonly IShell shell; readonly string taskId; - readonly CancellationToken token; + readonly CancellationToken runningScriptToken; readonly IReadOnlyDictionary environmentVariables; readonly ILog log; readonly ScriptIsolationMutex scriptIsolationMutex; @@ -33,7 +33,7 @@ public RunningScript(IShell shell, IScriptLog scriptLog, string taskId, ScriptIsolationMutex scriptIsolationMutex, - CancellationToken token, + CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, ILog log, TimeSpan? powerShellStartupCheckDelay = null) @@ -42,7 +42,7 @@ public RunningScript(IShell shell, this.workspace = workspace; this.stateStore = stateStore; this.taskId = taskId; - this.token = token; + this.runningScriptToken = runningScriptToken; this.environmentVariables = environmentVariables; this.log = log; this.scriptIsolationMutex = scriptIsolationMutex; @@ -56,9 +56,9 @@ public RunningScript(IShell shell, IScriptLog scriptLog, string taskId, ScriptIsolationMutex scriptIsolationMutex, - CancellationToken token, + CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, - ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, token, environmentVariables, log) + ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, runningScriptToken, environmentVariables, log) { } @@ -85,7 +85,7 @@ public async Task Execute() workspace.ScriptMutexName ?? nameof(RunningScript), message => writer.WriteOutput(ProcessOutputSource.StdOut, message), taskId, - token, + runningScriptToken, log)) { State = ProcessState.Running; @@ -129,14 +129,17 @@ public async Task Execute() async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter writer) { // Create a linked cancellation token that we can cancel when exiting early - await using var cts = new CancelOnDisposeCancellationToken(token); - var cancelOnDisposeToken = cts.Token; + await using var scriptTaskCts = new CancelOnDisposeCancellationToken(runningScriptToken); + var scriptTaskCancelOnDisposeToken = scriptTaskCts.Token; + + await using var monitoringTaskCts = new CancelOnDisposeCancellationToken(); + var monitoringTaskCancelOnDisposeToken = monitoringTaskCts.Token; // Start PowerShell startup monitoring if applicable - var monitoringTask = StartPowerShellStartupMonitoring(writer, cancelOnDisposeToken); + var monitoringTask = StartPowerShellStartupMonitoring(writer, monitoringTaskCancelOnDisposeToken); // Start script execution - var scriptTask = Task.Run(() => RunScript(shellPath, writer), cancelOnDisposeToken); + var scriptTask = Task.Run(() => RunScript(shellPath, writer), scriptTaskCancelOnDisposeToken); // Race between monitoring and script execution var completedTask = await Task.WhenAny(monitoringTask, scriptTask); @@ -294,7 +297,7 @@ int RunScript(string shellPath, IScriptLogWriter writer) LogScriptOutputTo(writer, ProcessOutputSource.StdOut), LogScriptOutputTo(writer, ProcessOutputSource.StdErr), environmentVariables, - token); + runningScriptToken); return exitCode; } From 4b91a6e3d8a42862085a9f02c475d422a288bdfc Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Tue, 17 Mar 2026 09:06:24 +1000 Subject: [PATCH 11/31] chore: tests should only run on windows --- .../PowerShellStartupDetectionTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index ee1eae7c7..ec809be4d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -20,11 +20,13 @@ using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; using Octopus.Tentacle.Services.Scripts; +using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Tests.Integration { [TestFixture] + [WindowsTest] public class PowerShellStartupDetectionTests { static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService() From 80fe29bdba144bb5558a2495c22909ee61bcf0e3 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Fri, 20 Mar 2026 16:52:52 +1000 Subject: [PATCH 12/31] chore: include startup detection code --- .../Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs b/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs index 746cec825..67a90dbc5 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs @@ -50,7 +50,7 @@ public string FormatCommandArguments(string bootstrapFile, string[]? scriptArgum commandArguments.Append("-NoLogo "); commandArguments.Append("-ExecutionPolicy Unrestricted "); var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); - commandArguments.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", + commandArguments.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; # OCTOPUS-POWERSHELL-STARTUP-DETECTION ; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", escapedBootstrapFile, string.Join(" ", scriptArguments ?? new string[0])); return commandArguments.ToString(); From 7b92380781789c94311f6495f63922e8f56ab114 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Mon, 23 Mar 2026 09:20:27 +1000 Subject: [PATCH 13/31] Revert "chore: include startup detection code" This reverts commit 4300b376304b728c0e6ee79d4c85770a2832ebd8. --- .../Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs b/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs index 67a90dbc5..746cec825 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/Shell/PowerShell.cs @@ -50,7 +50,7 @@ public string FormatCommandArguments(string bootstrapFile, string[]? scriptArgum commandArguments.Append("-NoLogo "); commandArguments.Append("-ExecutionPolicy Unrestricted "); var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); - commandArguments.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; # OCTOPUS-POWERSHELL-STARTUP-DETECTION ; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", + commandArguments.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", escapedBootstrapFile, string.Join(" ", scriptArguments ?? new string[0])); return commandArguments.ToString(); From 3c112f419f2d525bcb97bfc7cb8dfc05fbc0a2b2 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Tue, 24 Mar 2026 15:15:44 +1000 Subject: [PATCH 14/31] fix: only start script with monitoring if we detect the special comment in the script body --- .../Services/Scripts/PowerShellStartupDetection.cs | 8 ++++---- .../Services/Scripts/RunningScript.cs | 7 ++++--- .../Services/Scripts/WorkSpace/IScriptWorkspace.cs | 1 + .../Services/Scripts/WorkSpace/ScriptWorkspace.cs | 5 ++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs index 6801ab035..11d6a2f5a 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs @@ -58,17 +58,17 @@ public static bool ContainsSpecialComment(string scriptBody) return scriptBody.Contains(SpecialComment); } - public static string InjectDetectionCode(string scriptBody, string workingDirectory) + public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody, string workingDirectory) { if (!ContainsSpecialComment(scriptBody)) { - return scriptBody; + return (scriptBody, false); } var detectionCode = GenerateDetectionCode(workingDirectory); - return scriptBody.Replace(SpecialComment, detectionCode); + return (scriptBody.Replace(SpecialComment, detectionCode), true); } - + static string EscapePowerShellString(string input) { return input.Replace("'", "''").Replace("\\", "\\\\"); diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 0aa70c767..9fc5a3cba 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -92,7 +92,9 @@ public async Task Execute() RecordScriptHasStarted(writer); - exitCode = await RunScriptWithMonitoring(shellPath, writer); + exitCode = workspace.ShouldMonitorPowerShellStartup + ? await RunScriptWithMonitoring(shellPath, writer) + : RunScript(shellPath, writer); } } catch (OperationCanceledException) @@ -153,8 +155,7 @@ async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter write { // PowerShell never started - exit immediately with appropriate code writer.WriteOutput(ProcessOutputSource.StdErr, - $"PowerShell process did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes. " + - "Script execution aborted."); + $"{shellPath} process did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes. Script execution aborted."); // Clean up should-run file CleanupShouldRunFile(); diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs index ee6fdb849..710c589fa 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs @@ -16,6 +16,7 @@ public interface IScriptWorkspace ScriptIsolationLevel IsolationLevel { get; set; } TimeSpan ScriptMutexAcquireTimeout { get; set; } string? ScriptMutexName { get; set; } + bool ShouldMonitorPowerShellStartup { get; set; } void BootstrapScript(string scriptBody); string ResolvePath(string fileName); Task Delete(CancellationToken cancellationToken); diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index caecf2e76..8998ff322 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -71,6 +71,8 @@ public TimeSpan ScriptMutexAcquireTimeout public string? ScriptMutexName { get; set; } + public bool ShouldMonitorPowerShellStartup { get; set; } + public string[]? ScriptArguments { get; set; } public string WorkingDirectory { get; } @@ -80,7 +82,8 @@ public TimeSpan ScriptMutexAcquireTimeout public virtual void BootstrapScript(string scriptBody) { // Inject PowerShell startup detection code if the special comment is present - var processedScriptBody = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); + var (processedScriptBody, shouldMonitorPowerShellStartup) = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); + ShouldMonitorPowerShellStartup = shouldMonitorPowerShellStartup; // Create the "should run" file to signal that the script should proceed var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); From 578c344d63419462c36c865af460d472742df9e0 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Tue, 24 Mar 2026 15:16:24 +1000 Subject: [PATCH 15/31] feat: make duration to wait for powershell to start configurable --- .../Scripts/Models/ExecuteShellScriptCommand.cs | 5 ++++- .../Scripts/ScriptServiceV2Executor.cs | 1 + .../Builders/StartScriptCommandV2Builder.cs | 8 ++++++++ .../ScriptServiceV2/StartScriptCommandV2.cs | 11 +++++++++-- .../Services/Scripts/ScriptServiceV2.cs | 6 +++--- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs b/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs index c4cb4b653..0908522d9 100644 --- a/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs +++ b/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs @@ -14,12 +14,15 @@ public ExecuteShellScriptCommand( ScriptIsolationConfiguration isolationConfiguration, Dictionary? additionalScripts = null, ScriptFile[]? additionalFiles = null, - TimeSpan? durationToWaitForScriptToFinish = null) + TimeSpan? durationToWaitForScriptToFinish = null, + TimeSpan? durationToWaitForPowerShellToStart = null) : base(scriptTicket, taskId, scriptBody, arguments, isolationConfiguration, additionalScripts, additionalFiles) { DurationToWaitForScriptToFinish = durationToWaitForScriptToFinish; + DurationToWaitForPowerShellToStart = durationToWaitForPowerShellToStart; } public TimeSpan? DurationToWaitForScriptToFinish { get; } + public TimeSpan? DurationToWaitForPowerShellToStart { get; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs index c3fc68898..282a86912 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs @@ -54,6 +54,7 @@ StartScriptCommandV2 Map(ExecuteScriptCommand command) shellScriptCommand.TaskId, shellScriptCommand.ScriptTicket, shellScriptCommand.DurationToWaitForScriptToFinish, + shellScriptCommand.DurationToWaitForPowerShellToStart, shellScriptCommand.Scripts, shellScriptCommand.Files.ToArray()); } diff --git a/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs b/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs index 5f82c99aa..c358f5593 100644 --- a/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs +++ b/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs @@ -19,6 +19,7 @@ public class StartScriptCommandV2Builder string taskId = Guid.NewGuid().ToString(); ScriptTicket scriptTicket = new UniqueScriptTicketBuilder().Build(); TimeSpan? durationStartScriptCanWaitForScriptToFinish = TimeSpan.FromSeconds(5); + TimeSpan? durationToWaitForPowerShellToStart = TimeSpan.FromSeconds(5); public StartScriptCommandV2Builder WithScriptBody(string scriptBody) { @@ -84,6 +85,12 @@ public StartScriptCommandV2Builder WithDurationStartScriptCanWaitForScriptToFini return this; } + public StartScriptCommandV2Builder WithDurationToWaitForPowerShellToStart(TimeSpan? duration) + { + this.durationToWaitForPowerShellToStart = duration; + return this; + } + public StartScriptCommandV2 Build() => new(scriptBody.ToString(), isolation, @@ -93,6 +100,7 @@ public StartScriptCommandV2 Build() taskId, scriptTicket, durationStartScriptCanWaitForScriptToFinish, + durationToWaitForPowerShellToStart, additionalScripts, files.ToArray()); } diff --git a/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs b/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs index e8cd3f0cd..d8dedf65a 100644 --- a/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs +++ b/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs @@ -15,12 +15,14 @@ public StartScriptCommandV2(string scriptBody, string[] arguments, string taskId, ScriptTicket scriptTicket, - TimeSpan? durationToWaitForScriptToFinish) + TimeSpan? durationToWaitForScriptToFinish, + TimeSpan? durationToWaitForPowerShellToStartup) { Arguments = arguments; TaskId = taskId; ScriptTicket = scriptTicket; DurationToWaitForScriptToFinish = durationToWaitForScriptToFinish; + DurationToWaitForPowerShellToStartup = durationToWaitForPowerShellToStartup; ScriptBody = scriptBody; Isolation = isolation; ScriptIsolationMutexTimeout = scriptIsolationMutexTimeout; @@ -35,6 +37,7 @@ public StartScriptCommandV2(string scriptBody, string taskId, ScriptTicket scriptTicket, TimeSpan? durationToWaitForScriptToFinish, + TimeSpan? durationToWaitForPowerShellToStartup, params ScriptFile[]? additionalFiles) : this(scriptBody, isolation, @@ -43,7 +46,8 @@ public StartScriptCommandV2(string scriptBody, arguments, taskId, scriptTicket, - durationToWaitForScriptToFinish) + durationToWaitForScriptToFinish, + durationToWaitForPowerShellToStartup) { if (additionalFiles != null) Files.AddRange(additionalFiles); @@ -57,6 +61,7 @@ public StartScriptCommandV2(string scriptBody, string taskId, ScriptTicket scriptTicket, TimeSpan? durationToWaitForScriptToFinish, + TimeSpan? durationToWaitForPowerShellToStartup, Dictionary? additionalScripts, params ScriptFile[]? additionalFiles) : this(scriptBody, @@ -67,6 +72,7 @@ public StartScriptCommandV2(string scriptBody, taskId, scriptTicket, durationToWaitForScriptToFinish, + durationToWaitForPowerShellToStartup, additionalFiles) { if (additionalScripts == null || !additionalScripts.Any()) @@ -82,6 +88,7 @@ public StartScriptCommandV2(string scriptBody, public string ScriptBody { get; } public string TaskId { get; } public TimeSpan? DurationToWaitForScriptToFinish { get; } + public TimeSpan? DurationToWaitForPowerShellToStartup { get; } public ScriptIsolationLevel Isolation { get; } public TimeSpan ScriptIsolationMutexTimeout { get; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index 4fd55e2e3..f409ea59c 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -96,7 +96,7 @@ public async Task StartScriptAsync(StartScriptCommandV2 runningScript.ScriptStateStore.Create(); } - var process = LaunchShell(command.ScriptTicket, command.TaskId, workspace, runningScript.ScriptStateStore, runningScript.CancellationToken); + var process = LaunchShell(command.ScriptTicket, command.TaskId, workspace, runningScript.ScriptStateStore, command.DurationToWaitForPowerShellToStartup, runningScript.CancellationToken); runningScript.Process = process; @@ -146,9 +146,9 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat await workspace.Delete(cancellationToken); } - RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) + RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, TimeSpan? durationToWaitForPowerShellToStart, CancellationToken cancellationToken) { - var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log); + var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log, durationToWaitForPowerShellToStart); _ = Task.Run(async () => await runningScript.Execute(), cancellationToken); return runningScript; } From 7f2fe33e4282f7e06e87e0fe0119fdb4af461dbf Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Tue, 24 Mar 2026 16:08:54 +1000 Subject: [PATCH 16/31] chore: fix failing test due to message mismatch --- .../PowerShellStartupDetectionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index ec809be4d..10324d99f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -111,7 +111,7 @@ public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("PowerShell process did not start within"); + allLogs.Should().Contain("PowerShell.exe process did not start within"); } } From f09bea54e1e93e6d031fa30147ab0e4257aa3623 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Thu, 26 Mar 2026 16:24:17 +1000 Subject: [PATCH 17/31] chore: add method for specifying timeout for monitor powershell startup --- .../Builders/ExecuteShellScriptCommandBuilder.cs | 10 +++++++++- .../Builders/TestExecuteShellScriptCommandBuilder.cs | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs b/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs index 1e5a97d92..63c3a3afb 100644 --- a/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs +++ b/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs @@ -6,6 +6,7 @@ namespace Octopus.Tentacle.Client.Scripts.Models.Builders public class ExecuteShellScriptCommandBuilder : ExecuteScriptCommandBuilder { TimeSpan? durationStartScriptCanWaitForScriptToFinish = TimeSpan.FromSeconds(5); // The UI refreshes every 5 seconds, so 5 seconds here might be reasonable. + TimeSpan? durationToWaitForPowerShellToStart = TimeSpan.FromMinutes(5); public ExecuteShellScriptCommandBuilder(string taskId, ScriptIsolationLevel defaultIsolationLevel) : base(taskId, defaultIsolationLevel) { @@ -17,6 +18,12 @@ public ExecuteScriptCommandBuilder WithDurationStartScriptCanWaitForScriptToFini return this; } + public ExecuteScriptCommandBuilder WithDurationToWaitForPowerShellToStart(TimeSpan? duration) + { + durationToWaitForPowerShellToStart = duration; + return this; + } + public override ExecuteScriptCommand Build() => new ExecuteShellScriptCommand( ScriptTicket, @@ -26,6 +33,7 @@ public override ExecuteScriptCommand Build() IsolationConfiguration, AdditionalScripts, Files.ToArray(), - durationStartScriptCanWaitForScriptToFinish); + durationStartScriptCanWaitForScriptToFinish, + durationToWaitForPowerShellToStart); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs index b1301ac8c..f050925bd 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs @@ -10,6 +10,7 @@ public TestExecuteShellScriptCommandBuilder() : base(Guid.NewGuid().ToString(), ScriptIsolationLevel.NoIsolation) { WithDurationStartScriptCanWaitForScriptToFinish(null); + WithDurationToWaitForPowerShellToStart(null); } } } \ No newline at end of file From 42f02c373013facdac9b6197b92465f98af03a98 Mon Sep 17 00:00:00 2001 From: hnrkndrssn Date: Fri, 27 Mar 2026 15:52:20 +1000 Subject: [PATCH 18/31] chore: some cleanup --- .../Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs | 1 - .../Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs | 4 ++-- .../Services/Scripts/WorkSpace/BashScriptWorkspace.cs | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 9fc5a3cba..5e4fdb32f 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -310,7 +310,6 @@ int RunScript(string shellPath, IScriptLogWriter writer) return ScriptExitCodes.PowershellInvocationErrorExitCode; } } - Action LogScriptOutputTo(IScriptLogWriter logOutput, ProcessOutputSource level) { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index f409ea59c..700279908 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -146,9 +146,9 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat await workspace.Delete(cancellationToken); } - RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, TimeSpan? durationToWaitForPowerShellToStart, CancellationToken cancellationToken) + RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, TimeSpan? durationToWaitForPowerShellToStartup, CancellationToken cancellationToken) { - var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log, durationToWaitForPowerShellToStart); + var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log, durationToWaitForPowerShellToStartup); _ = Task.Run(async () => await runningScript.Execute(), cancellationToken); return runningScript; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index f11c7c729..cb346576a 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -2,7 +2,6 @@ using System.IO; using System.Text; using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Util; From 58bcbc990f95f2dde3354ee97b3b0f6eb1ed919c Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 12:14:51 +1100 Subject: [PATCH 19/31] Refactor test to a higher level, and don't change contracts --- POWERSHELL_STARTUP_DETECTION.md | 4 +- .../ExecuteShellScriptCommandBuilder.cs | 10 +- .../Models/ExecuteShellScriptCommand.cs | 5 +- .../Scripts/ScriptServiceV2Executor.cs | 1 - .../Builders/StartScriptCommandV2Builder.cs | 8 - .../Diagnostics/SerilogSystemLog.cs | 39 ++ .../ScriptServiceV2/StartScriptCommandV2.cs | 11 +- .../PowerShellStartupDetection.cs | 52 +-- .../PowerShellStartupMonitor.cs | 95 +++++ .../PowerShellStartupStatus.cs | 2 +- .../Services/Scripts/RunningScript.cs | 115 +---- .../Services/Scripts/ScriptServiceV2.cs | 19 +- .../Scripts/WorkSpace/BashScriptWorkspace.cs | 2 + .../Scripts/WorkSpace/IScriptWorkspace.cs | 3 +- .../Scripts/WorkSpace/ScriptWorkspace.cs | 15 +- .../WorkSpace/ScriptWorkspaceFactory.cs | 14 +- .../Util/EnvironmentVariables.cs | 1 + .../PowerShellStartupDetectionTests.cs | 392 ++++++++++-------- .../TestExecuteShellScriptCommandBuilder.cs | 1 - .../Util/RunningScriptFixture.cs | 3 + .../Util/TestPwshShell.cs | 34 ++ .../Services/Scripts/ScriptService.cs | 3 +- 22 files changed, 482 insertions(+), 347 deletions(-) create mode 100644 source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs rename source/Octopus.Tentacle.Core/Services/Scripts/{ => PowerShellStartup}/PowerShellStartupDetection.cs (62%) create mode 100644 source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs rename source/Octopus.Tentacle.Core/Services/Scripts/{ => PowerShellStartup}/PowerShellStartupStatus.cs (63%) create mode 100644 source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs diff --git a/POWERSHELL_STARTUP_DETECTION.md b/POWERSHELL_STARTUP_DETECTION.md index cc07e1b60..681d4a2a9 100644 --- a/POWERSHELL_STARTUP_DETECTION.md +++ b/POWERSHELL_STARTUP_DETECTION.md @@ -21,7 +21,7 @@ The feature adds a special comment marker that Tentacle replaces with detection Add the special comment to your PowerShell scripts: ```powershell -# OCTOPUS-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION # Your actual script content here Write-Output "Hello World" ``` @@ -40,7 +40,7 @@ Tentacle will automatically replace this comment with detection code that: 2. **PowerShellStartupDetection.cs** (new) - Contains logic to generate and inject detection code - - Defines the special comment: `# OCTOPUS-POWERSHELL-STARTUP-DETECTION` + - Defines the special comment: `# TENTACLE-POWERSHELL-STARTUP-DETECTION` - Manages file paths for detection files 3. **ScriptWorkspace.cs** diff --git a/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs b/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs index 63c3a3afb..1e5a97d92 100644 --- a/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs +++ b/source/Octopus.Tentacle.Client/Scripts/Models/Builders/ExecuteShellScriptCommandBuilder.cs @@ -6,7 +6,6 @@ namespace Octopus.Tentacle.Client.Scripts.Models.Builders public class ExecuteShellScriptCommandBuilder : ExecuteScriptCommandBuilder { TimeSpan? durationStartScriptCanWaitForScriptToFinish = TimeSpan.FromSeconds(5); // The UI refreshes every 5 seconds, so 5 seconds here might be reasonable. - TimeSpan? durationToWaitForPowerShellToStart = TimeSpan.FromMinutes(5); public ExecuteShellScriptCommandBuilder(string taskId, ScriptIsolationLevel defaultIsolationLevel) : base(taskId, defaultIsolationLevel) { @@ -18,12 +17,6 @@ public ExecuteScriptCommandBuilder WithDurationStartScriptCanWaitForScriptToFini return this; } - public ExecuteScriptCommandBuilder WithDurationToWaitForPowerShellToStart(TimeSpan? duration) - { - durationToWaitForPowerShellToStart = duration; - return this; - } - public override ExecuteScriptCommand Build() => new ExecuteShellScriptCommand( ScriptTicket, @@ -33,7 +26,6 @@ public override ExecuteScriptCommand Build() IsolationConfiguration, AdditionalScripts, Files.ToArray(), - durationStartScriptCanWaitForScriptToFinish, - durationToWaitForPowerShellToStart); + durationStartScriptCanWaitForScriptToFinish); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs b/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs index 0908522d9..c4cb4b653 100644 --- a/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs +++ b/source/Octopus.Tentacle.Client/Scripts/Models/ExecuteShellScriptCommand.cs @@ -14,15 +14,12 @@ public ExecuteShellScriptCommand( ScriptIsolationConfiguration isolationConfiguration, Dictionary? additionalScripts = null, ScriptFile[]? additionalFiles = null, - TimeSpan? durationToWaitForScriptToFinish = null, - TimeSpan? durationToWaitForPowerShellToStart = null) + TimeSpan? durationToWaitForScriptToFinish = null) : base(scriptTicket, taskId, scriptBody, arguments, isolationConfiguration, additionalScripts, additionalFiles) { DurationToWaitForScriptToFinish = durationToWaitForScriptToFinish; - DurationToWaitForPowerShellToStart = durationToWaitForPowerShellToStart; } public TimeSpan? DurationToWaitForScriptToFinish { get; } - public TimeSpan? DurationToWaitForPowerShellToStart { get; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs index 282a86912..c3fc68898 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Executor.cs @@ -54,7 +54,6 @@ StartScriptCommandV2 Map(ExecuteScriptCommand command) shellScriptCommand.TaskId, shellScriptCommand.ScriptTicket, shellScriptCommand.DurationToWaitForScriptToFinish, - shellScriptCommand.DurationToWaitForPowerShellToStart, shellScriptCommand.Scripts, shellScriptCommand.Files.ToArray()); } diff --git a/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs b/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs index c358f5593..5f82c99aa 100644 --- a/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs +++ b/source/Octopus.Tentacle.CommonTestUtils/Builders/StartScriptCommandV2Builder.cs @@ -19,7 +19,6 @@ public class StartScriptCommandV2Builder string taskId = Guid.NewGuid().ToString(); ScriptTicket scriptTicket = new UniqueScriptTicketBuilder().Build(); TimeSpan? durationStartScriptCanWaitForScriptToFinish = TimeSpan.FromSeconds(5); - TimeSpan? durationToWaitForPowerShellToStart = TimeSpan.FromSeconds(5); public StartScriptCommandV2Builder WithScriptBody(string scriptBody) { @@ -85,12 +84,6 @@ public StartScriptCommandV2Builder WithDurationStartScriptCanWaitForScriptToFini return this; } - public StartScriptCommandV2Builder WithDurationToWaitForPowerShellToStart(TimeSpan? duration) - { - this.durationToWaitForPowerShellToStart = duration; - return this; - } - public StartScriptCommandV2 Build() => new(scriptBody.ToString(), isolation, @@ -100,7 +93,6 @@ public StartScriptCommandV2 Build() taskId, scriptTicket, durationStartScriptCanWaitForScriptToFinish, - durationToWaitForPowerShellToStart, additionalScripts, files.ToArray()); } diff --git a/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs b/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs new file mode 100644 index 000000000..3a2b9471d --- /dev/null +++ b/source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs @@ -0,0 +1,39 @@ +using System; +using Octopus.Tentacle.Core.Diagnostics; +using Octopus.Tentacle.Diagnostics; +using Serilog; +using Serilog.Events; +using LogEvent = Octopus.Tentacle.Diagnostics.LogEvent; + +namespace Octopus.Tentacle.CommonTestUtils.Diagnostics +{ + public class SerilogSystemLog : SystemLog + { + readonly ILogger logger; + + public SerilogSystemLog(ILogger logger) + { + this.logger = logger; + } + + protected override void WriteEvent(LogEvent logEvent) + { + var level = ToSerilogLevel(logEvent.Category); + if (logEvent.Error != null) + logger.Write(level, logEvent.Error, logEvent.MessageText); + else + logger.Write(level, logEvent.MessageText); + } + + static LogEventLevel ToSerilogLevel(LogCategory category) => category switch + { + LogCategory.Trace => LogEventLevel.Verbose, + LogCategory.Verbose => LogEventLevel.Debug, + LogCategory.Info => LogEventLevel.Information, + LogCategory.Warning => LogEventLevel.Warning, + LogCategory.Error => LogEventLevel.Error, + LogCategory.Fatal => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; + } +} diff --git a/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs b/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs index d8dedf65a..e8cd3f0cd 100644 --- a/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs +++ b/source/Octopus.Tentacle.Contracts/ScriptServiceV2/StartScriptCommandV2.cs @@ -15,14 +15,12 @@ public StartScriptCommandV2(string scriptBody, string[] arguments, string taskId, ScriptTicket scriptTicket, - TimeSpan? durationToWaitForScriptToFinish, - TimeSpan? durationToWaitForPowerShellToStartup) + TimeSpan? durationToWaitForScriptToFinish) { Arguments = arguments; TaskId = taskId; ScriptTicket = scriptTicket; DurationToWaitForScriptToFinish = durationToWaitForScriptToFinish; - DurationToWaitForPowerShellToStartup = durationToWaitForPowerShellToStartup; ScriptBody = scriptBody; Isolation = isolation; ScriptIsolationMutexTimeout = scriptIsolationMutexTimeout; @@ -37,7 +35,6 @@ public StartScriptCommandV2(string scriptBody, string taskId, ScriptTicket scriptTicket, TimeSpan? durationToWaitForScriptToFinish, - TimeSpan? durationToWaitForPowerShellToStartup, params ScriptFile[]? additionalFiles) : this(scriptBody, isolation, @@ -46,8 +43,7 @@ public StartScriptCommandV2(string scriptBody, arguments, taskId, scriptTicket, - durationToWaitForScriptToFinish, - durationToWaitForPowerShellToStartup) + durationToWaitForScriptToFinish) { if (additionalFiles != null) Files.AddRange(additionalFiles); @@ -61,7 +57,6 @@ public StartScriptCommandV2(string scriptBody, string taskId, ScriptTicket scriptTicket, TimeSpan? durationToWaitForScriptToFinish, - TimeSpan? durationToWaitForPowerShellToStartup, Dictionary? additionalScripts, params ScriptFile[]? additionalFiles) : this(scriptBody, @@ -72,7 +67,6 @@ public StartScriptCommandV2(string scriptBody, taskId, scriptTicket, durationToWaitForScriptToFinish, - durationToWaitForPowerShellToStartup, additionalFiles) { if (additionalScripts == null || !additionalScripts.Any()) @@ -88,7 +82,6 @@ public StartScriptCommandV2(string scriptBody, public string ScriptBody { get; } public string TaskId { get; } public TimeSpan? DurationToWaitForScriptToFinish { get; } - public TimeSpan? DurationToWaitForPowerShellToStartup { get; } public ScriptIsolationLevel Isolation { get; } public TimeSpan ScriptIsolationMutexTimeout { get; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs similarity index 62% rename from source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs rename to source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index 11d6a2f5a..c94a79578 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -1,33 +1,40 @@ using System; using System.IO; +using Octopus.Tentacle.Core.Util; -namespace Octopus.Tentacle.Core.Services.Scripts +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup { public static class PowerShellStartupDetection { - public const string SpecialComment = "# OCTOPUS-POWERSHELL-STARTUP-DETECTION"; + public const string PowershellTentacleStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; public const string StartedFileName = ".octopus_powershell_started"; public const string ShouldRunFileName = ".octopus_powershell_should_run"; - + + public static TimeSpan PowerShellStartupTimeout + { + get + { + var raw = Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePowerShellStartupTimeout); + return TimeSpan.TryParse(raw, out var value) ? value : TimeSpan.FromMinutes(5); + } + } + public static string GetStartedFilePath(string workingDirectory) { return Path.Combine(workingDirectory, StartedFileName); } - + public static string GetShouldRunFilePath(string workingDirectory) { return Path.Combine(workingDirectory, ShouldRunFileName); } - - public static string GenerateDetectionCode(string workingDirectory) + + public static string GenerateDetectionCode() { - var startedFile = GetStartedFilePath(workingDirectory); - var shouldRunFile = GetShouldRunFilePath(workingDirectory); - return $@" # PowerShell startup detection code (auto-generated by Octopus Tentacle) -$octopusStartedFile = '{EscapePowerShellString(startedFile)}' -$octopusShouldRunFile = '{EscapePowerShellString(shouldRunFile)}' +$octopusStartedFile = '{StartedFileName}' +$octopusShouldRunFile = '{ShouldRunFileName}' try {{ # Try to create the started file exclusively @@ -40,38 +47,31 @@ public static string GenerateDetectionCode(string workingDirectory) # 2. Tentacle already created it (meaning we never started) # In either case, we should exit write-output ""PowerShell startup detection: Started file already exists, exiting"" - exit 1 + exit -47 }} # Check if the should-run file exists if (-not (Test-Path $octopusShouldRunFile)) {{ write-output ""PowerShell startup detection: Should-run file does not exist, exiting"" - exit 1 + exit -47 }} - -write-output ""PowerShell startup detection: Checks passed, continuing script execution"" "; } - + public static bool ContainsSpecialComment(string scriptBody) { - return scriptBody.Contains(SpecialComment); + return scriptBody.Contains(PowershellTentacleStartupDetectionComment); } - - public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody, string workingDirectory) + + public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody) { if (!ContainsSpecialComment(scriptBody)) { return (scriptBody, false); } - - var detectionCode = GenerateDetectionCode(workingDirectory); - return (scriptBody.Replace(SpecialComment, detectionCode), true); - } - static string EscapePowerShellString(string input) - { - return input.Replace("'", "''").Replace("\\", "\\\\"); + var detectionCode = GenerateDetectionCode(); + return (scriptBody.Replace(PowershellTentacleStartupDetectionComment, detectionCode), true); } } } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs new file mode 100644 index 000000000..ca8209af0 --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Halibut.Util; +using Octopus.Tentacle.Core.Diagnostics; + +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup +{ + class PowerShellStartupMonitor + { + readonly string workSpaceWorkingDirectory; + readonly TimeSpan powerShellStartupTimeout; + readonly ILog log; + readonly string taskId; + + public PowerShellStartupMonitor(string workSpaceWorkingDirectory, TimeSpan powerShellStartupTimeout, ILog log, string taskId) + { + this.workSpaceWorkingDirectory = workSpaceWorkingDirectory; + this.powerShellStartupTimeout = powerShellStartupTimeout; + this.log = log; + this.taskId = taskId; + } + + public Task WaitForStartup(CancellationToken cancellationToken) + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory); + + if (!File.Exists(shouldRunFilePath)) + { + return Task.FromResult(PowerShellStartupStatus.NotMonitored); + } + + return Task.Run(async () => + { + try + { + await DelayWithoutException.Delay(powerShellStartupTimeout, cancellationToken); + + if (cancellationToken.IsCancellationRequested) + { + return PowerShellStartupStatus.NotMonitored; + } + + try + { + var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workSpaceWorkingDirectory); + using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); + log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupTimeout.TotalMinutes} minutes for task {taskId}"); + DeleteShouldRunFileToEnsureThePowerShellCanNeverStart(); + return PowerShellStartupStatus.NeverStarted; + } + catch (IOException) + { + log.Info($"PowerShell startup detection: PowerShell started late (after {powerShellStartupTimeout.TotalMinutes} minutes) for task {taskId}"); + return PowerShellStartupStatus.Started; + } + } + catch (OperationCanceledException) + { + return PowerShellStartupStatus.NotMonitored; + } + catch (Exception ex) + { + log.Warn(ex, $"Error in PowerShell startup monitoring for task {taskId}"); + return PowerShellStartupStatus.NotMonitored; + } + }); + } + + /// + /// Since the powershell guard works by only running if it can create a file, we run into an + /// interesting situation when the powershell is blocked from running and the workspace is + /// cleaned up. If the workspace is cleaned up, then the file that the guard must create in + /// order to processed wont exist. This means it could proceed with execution. To ensure this + /// never happens, we delete the should-run file. Since the guard will only run if this file + /// exists, the workspace deleting problem goes away. + /// + public void DeleteShouldRunFileToEnsureThePowerShellCanNeverStart() + { + try + { + var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory); + if (File.Exists(shouldRunFilePath)) + { + File.Delete(shouldRunFilePath); + } + } + catch (Exception ex) + { + log.Warn(ex, $"Failed to delete should-run file for task {taskId}"); + } + } + } +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs similarity index 63% rename from source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs rename to source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs index e113760b4..0f48c31ff 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartupStatus.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs @@ -1,4 +1,4 @@ -namespace Octopus.Tentacle.Core.Services.Scripts +namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup { public enum PowerShellStartupStatus { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 5e4fdb32f..3ed08eb59 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Threading; using System.Threading.Tasks; using Halibut.Util; @@ -8,6 +7,7 @@ using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Logging; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; @@ -25,7 +25,7 @@ public class RunningScript: IRunningScript readonly IReadOnlyDictionary environmentVariables; readonly ILog log; readonly ScriptIsolationMutex scriptIsolationMutex; - readonly TimeSpan powerShellStartupCheckDelay; + readonly TimeSpan powerShellStartupTimeout; public RunningScript(IShell shell, IScriptWorkspace workspace, @@ -35,8 +35,9 @@ public RunningScript(IShell shell, ScriptIsolationMutex scriptIsolationMutex, CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, - ILog log, - TimeSpan? powerShellStartupCheckDelay = null) + TimeSpan powerShellStartupTimeout, + ILog log + ) { this.shell = shell; this.workspace = workspace; @@ -48,7 +49,7 @@ public RunningScript(IShell shell, this.scriptIsolationMutex = scriptIsolationMutex; this.ScriptLog = scriptLog; this.State = ProcessState.Pending; - this.powerShellStartupCheckDelay = powerShellStartupCheckDelay ?? TimeSpan.FromMinutes(5); + this.powerShellStartupTimeout = powerShellStartupTimeout; } public RunningScript(IShell shell, @@ -58,7 +59,8 @@ public RunningScript(IShell shell, ScriptIsolationMutex scriptIsolationMutex, CancellationToken runningScriptToken, IReadOnlyDictionary environmentVariables, - ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, runningScriptToken, environmentVariables, log) + TimeSpan powerShellStartupTimeout, + ILog log) : this(shell, workspace, null, scriptLog, taskId, scriptIsolationMutex, runningScriptToken, environmentVariables, powerShellStartupTimeout, log) { } @@ -92,9 +94,9 @@ public async Task Execute() RecordScriptHasStarted(writer); - exitCode = workspace.ShouldMonitorPowerShellStartup - ? await RunScriptWithMonitoring(shellPath, writer) - : RunScript(shellPath, writer); + exitCode = workspace.ShouldMonitorPowerShellStartup() + ? await RunPowershellScriptWithMonitoring(shellPath, writer) + : RunScript(shellPath, writer, runningScriptToken); } } catch (OperationCanceledException) @@ -128,115 +130,42 @@ public async Task Execute() } } - async Task RunScriptWithMonitoring(string shellPath, IScriptLogWriter writer) + async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWriter writer) { - // Create a linked cancellation token that we can cancel when exiting early + // We want to be able to make some effort to cancel the running script, if the monitor task detects it as hung. + // Hence, we make a linked cancellation token with the runningScriptToken await using var scriptTaskCts = new CancelOnDisposeCancellationToken(runningScriptToken); - var scriptTaskCancelOnDisposeToken = scriptTaskCts.Token; + // The monitoring task is NOT linked to the runningScriptToken, since it should keep monitoring even if an attempt to + // cancel the script is made. Remember, these hung powershell scripts WILL NOT CANCEL, so we must continue to monitor. await using var monitoringTaskCts = new CancelOnDisposeCancellationToken(); - var monitoringTaskCancelOnDisposeToken = monitoringTaskCts.Token; - // Start PowerShell startup monitoring if applicable - var monitoringTask = StartPowerShellStartupMonitoring(writer, monitoringTaskCancelOnDisposeToken); + var monitor = new PowerShellStartupMonitor(workspace.WorkingDirectory, powerShellStartupTimeout, log, taskId); - // Start script execution - var scriptTask = Task.Run(() => RunScript(shellPath, writer), scriptTaskCancelOnDisposeToken); + var monitoringTask = monitor.WaitForStartup(monitoringTaskCts.Token); + var scriptTask = Task.Run(() => RunScript(shellPath, writer, scriptTaskCts.Token), scriptTaskCts.Token); - // Race between monitoring and script execution var completedTask = await Task.WhenAny(monitoringTask, scriptTask); if (completedTask == monitoringTask) { - // Monitoring task completed first var startupStatus = await monitoringTask; if (startupStatus == PowerShellStartupStatus.NeverStarted) { // PowerShell never started - exit immediately with appropriate code writer.WriteOutput(ProcessOutputSource.StdErr, - $"{shellPath} process did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes. Script execution aborted."); - - // Clean up should-run file - CleanupShouldRunFile(); + $"{shellPath} process did not start within {powerShellStartupTimeout.TotalMinutes} minutes. Script execution aborted."); return ScriptExitCodes.PowerShellNeverStartedExitCode; } } - // Script completed first var exitCode = await scriptTask; return exitCode; } - Task StartPowerShellStartupMonitoring(IScriptLogWriter writer, CancellationToken cancellationToken) - { - var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); - - // Only start monitoring if the should-run file exists (meaning detection is enabled) - if (!File.Exists(shouldRunFilePath)) - { - return Task.FromResult(PowerShellStartupStatus.NotMonitored); - } - - return Task.Run(async () => - { - try - { - await DelayWithoutException.Delay(powerShellStartupCheckDelay, cancellationToken); - - if (cancellationToken.IsCancellationRequested) - { - return PowerShellStartupStatus.NotMonitored; - } - - // Try to create the started file - try - { - var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); - using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); - // Successfully created the file, meaning PowerShell never started - log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupCheckDelay.TotalMinutes} minutes for task {taskId}"); - - return PowerShellStartupStatus.NeverStarted; - } - catch (IOException) - { - // File already exists, meaning PowerShell did start (just very slowly) - log.Info($"PowerShell startup detection: PowerShell started late (after {powerShellStartupCheckDelay.TotalMinutes} minutes) for task {taskId}"); - return PowerShellStartupStatus.Started; - } - } - catch (OperationCanceledException) - { - // Task was cancelled, this is expected - return PowerShellStartupStatus.NotMonitored; - } - catch (Exception ex) - { - log.Warn(ex, $"Error in PowerShell startup monitoring for task {taskId}"); - return PowerShellStartupStatus.NotMonitored; - } - }); - } - - void CleanupShouldRunFile() - { - try - { - var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); - if (File.Exists(shouldRunFilePath)) - { - File.Delete(shouldRunFilePath); - } - } - catch (Exception ex) - { - log.Warn(ex, $"Failed to delete should-run file for task {taskId}"); - } - } - void RecordScriptHasStarted(IScriptLogWriter writer) { try @@ -286,7 +215,7 @@ void RecordScriptHasCompleted(int exitCode) } } - int RunScript(string shellPath, IScriptLogWriter writer) + int RunScript(string shellPath, IScriptLogWriter writer, CancellationToken cancellationToken) { try { @@ -298,7 +227,7 @@ int RunScript(string shellPath, IScriptLogWriter writer) LogScriptOutputTo(writer, ProcessOutputSource.StdOut), LogScriptOutputTo(writer, ProcessOutputSource.StdErr), environmentVariables, - runningScriptToken); + cancellationToken); return exitCode; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index 700279908..1ca3775b1 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -9,10 +9,10 @@ using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Maintenance; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; -using Octopus.Tentacle.Services; using Octopus.Tentacle.Services.Scripts; using Octopus.Tentacle.Util; @@ -28,6 +28,7 @@ public class ScriptServiceV2 : IAsyncScriptServiceV2, IRunningScriptReporter readonly ScriptIsolationMutex scriptIsolationMutex; readonly ConcurrentDictionary runningScripts = new(); readonly IReadOnlyDictionary environmentVariables; + readonly TimeSpan powerShellStartupTimeout; public ScriptServiceV2( IShell shell, @@ -35,13 +36,15 @@ public ScriptServiceV2( IScriptStateStoreFactory scriptStateStoreFactory, ScriptIsolationMutex scriptIsolationMutex, ISystemLog log, - IReadOnlyDictionary environmentVariables) + IReadOnlyDictionary environmentVariables, + TimeSpan powerShellStartupTimeout) { this.shell = shell; this.workspaceFactory = workspaceFactory; this.scriptStateStoreFactory = scriptStateStoreFactory; this.log = log; this.environmentVariables = environmentVariables; + this.powerShellStartupTimeout = powerShellStartupTimeout; this.scriptIsolationMutex = scriptIsolationMutex; } @@ -50,7 +53,7 @@ public ScriptServiceV2( IScriptWorkspaceFactory workspaceFactory, IScriptStateStoreFactory scriptStateStoreFactory, ScriptIsolationMutex scriptIsolationMutex, - ISystemLog log) : this(shell, workspaceFactory, scriptStateStoreFactory, scriptIsolationMutex, log, new Dictionary()) + ISystemLog log) : this(shell, workspaceFactory, scriptStateStoreFactory, scriptIsolationMutex, log, new Dictionary(), PowerShellStartupDetection.PowerShellStartupTimeout) { } @@ -96,7 +99,11 @@ public async Task StartScriptAsync(StartScriptCommandV2 runningScript.ScriptStateStore.Create(); } - var process = LaunchShell(command.ScriptTicket, command.TaskId, workspace, runningScript.ScriptStateStore, command.DurationToWaitForPowerShellToStartup, runningScript.CancellationToken); + var process = LaunchShell(command.ScriptTicket, + command.TaskId, + workspace, + runningScript.ScriptStateStore, + runningScript.CancellationToken); runningScript.Process = process; @@ -146,9 +153,9 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat await workspace.Delete(cancellationToken); } - RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, TimeSpan? durationToWaitForPowerShellToStartup, CancellationToken cancellationToken) + RunningScript LaunchShell(ScriptTicket ticket,string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) { - var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, log, durationToWaitForPowerShellToStartup); + var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, powerShellStartupTimeout, log); _ = Task.Run(async () => await runningScript.Execute(), cancellationToken); return runningScript; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index cb346576a..2b32e4114 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -27,6 +27,8 @@ public override void BootstrapScript(string scriptBody) FileSystem.OverwriteFile(BootstrapScriptFilePath, scriptBody, Encoding.Default); } + public override bool ShouldMonitorPowerShellStartup() => false; + public static string GetBashBootstrapScriptFilePath(string workspaceDirectory) => Path.Combine(workspaceDirectory, BootstrapScriptFileName); } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs index 710c589fa..c28d9e58c 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/IScriptWorkspace.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.Threading; using System.Threading.Tasks; using Octopus.Tentacle.Contracts; @@ -16,7 +15,7 @@ public interface IScriptWorkspace ScriptIsolationLevel IsolationLevel { get; set; } TimeSpan ScriptMutexAcquireTimeout { get; set; } string? ScriptMutexName { get; set; } - bool ShouldMonitorPowerShellStartup { get; set; } + bool ShouldMonitorPowerShellStartup(); void BootstrapScript(string scriptBody); string ResolvePath(string fileName); Task Delete(CancellationToken cancellationToken); diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index 8998ff322..32254226f 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -5,10 +5,10 @@ using System.Threading.Tasks; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Services.Scripts; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Logging; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; -using Octopus.Tentacle.Services.Scripts; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Scripts @@ -70,8 +70,6 @@ public TimeSpan ScriptMutexAcquireTimeout } public string? ScriptMutexName { get; set; } - - public bool ShouldMonitorPowerShellStartup { get; set; } public string[]? ScriptArguments { get; set; } @@ -81,10 +79,9 @@ public TimeSpan ScriptMutexAcquireTimeout public virtual void BootstrapScript(string scriptBody) { - // Inject PowerShell startup detection code if the special comment is present - var (processedScriptBody, shouldMonitorPowerShellStartup) = PowerShellStartupDetection.InjectDetectionCode(scriptBody, WorkingDirectory); - ShouldMonitorPowerShellStartup = shouldMonitorPowerShellStartup; - + var (processedScriptBody, shouldMonitorPowerShellStartup) = PowerShellStartupDetection.InjectDetectionCode(scriptBody); + this.shouldMonitorPowerShellStartup = shouldMonitorPowerShellStartup; + // Create the "should run" file to signal that the script should proceed var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); FileSystem.OverwriteFile(shouldRunFile, ""); @@ -92,6 +89,10 @@ public virtual void BootstrapScript(string scriptBody) // default is UTF8noBOM but powershell doesn't interpret that correctly FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.UTF8); } + + public virtual bool ShouldMonitorPowerShellStartup() => shouldMonitorPowerShellStartup; + + bool shouldMonitorPowerShellStartup; public string ResolvePath(string fileName) { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs index 5fc53a003..7a3821a14 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs @@ -19,15 +19,25 @@ public class ScriptWorkspaceFactory : IScriptWorkspaceFactory readonly IOctopusFileSystem fileSystem; readonly IHomeDirectoryProvider home; readonly SensitiveValueMasker sensitiveValueMasker; + readonly bool useBashWorkspace; public ScriptWorkspaceFactory( IOctopusFileSystem fileSystem, IHomeDirectoryProvider home, - SensitiveValueMasker sensitiveValueMasker) + SensitiveValueMasker sensitiveValueMasker, + bool useBashWorkspace) { this.fileSystem = fileSystem; this.home = home; this.sensitiveValueMasker = sensitiveValueMasker; + this.useBashWorkspace = useBashWorkspace; + } + + public ScriptWorkspaceFactory( + IOctopusFileSystem fileSystem, + IHomeDirectoryProvider home, + SensitiveValueMasker sensitiveValueMasker) : this(fileSystem, home, sensitiveValueMasker, useBashWorkspace: !PlatformDetection.IsRunningOnWindows) + { } public IScriptWorkspace GetWorkspace(ScriptTicket ticket, WorkspaceReadinessCheck readinessCheck) @@ -107,7 +117,7 @@ string FindWorkingDirectory(ScriptTicket ticket) IScriptWorkspace CreateWorkspace(ScriptTicket scriptTicket, string workingDirectory) { - if (!PlatformDetection.IsRunningOnWindows) + if (useBashWorkspace) { return new BashScriptWorkspace(scriptTicket, workingDirectory, fileSystem, sensitiveValueMasker); } diff --git a/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs b/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs index 6a455bb2d..2b5f83a59 100644 --- a/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs +++ b/source/Octopus.Tentacle.Core/Util/EnvironmentVariables.cs @@ -28,6 +28,7 @@ public static class EnvironmentVariables public const string TentacleEnableDataStreamLengthChecks = "TentacleEnableDataStreamLengthChecks"; public const string TentacleMachineConfigurationHomeDirectory = "TentacleMachineConfigurationHomeDirectory"; public const string TentaclePollingConnectionCount = "TentaclePollingConnectionCount"; + public const string TentaclePowerShellStartupTimeout = "TentaclePowerShellStartupTimeout"; public const string NfsWatchdogDirectory = "watchdog_directory"; public static string TentacleUseTcpNoDelay = "TentacleUseTcpNoDelay"; public static string TentacleUseAsyncListener = "TentacleUseAsyncListener"; diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index 10324d99f..d43e75d7c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -9,35 +9,40 @@ using NUnit.Framework; using Octopus.Tentacle.CommonTestUtils; using Octopus.Tentacle.CommonTestUtils.Builders; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; using Octopus.Tentacle.Configuration; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Contracts.ScriptServiceV2; -using Octopus.Tentacle.Core.Diagnostics; using Octopus.Tentacle.Core.Services.Scripts; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Locking; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; using Octopus.Tentacle.Scripts; -using Octopus.Tentacle.Services.Scripts; -using Octopus.Tentacle.Tests.Integration.Support.TestAttributes; +using Octopus.Tentacle.Tests.Integration.Support; +using Octopus.Tentacle.Tests.Integration.Util; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Tests.Integration { - [TestFixture] - [WindowsTest] - public class PowerShellStartupDetectionTests + [IntegrationTestTimeout] + public class PowerShellStartupDetectionTests : IntegrationTest { - static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService() + static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService( + TimeSpan? powerShellStartupTimeout = null) { var tempDir = new TemporaryDirectory(); + var systemLog = new SerilogSystemLog(new SerilogLoggerBuilder().Build()); + var homeConfiguration = Substitute.For(); homeConfiguration.HomeDirectory.Returns(tempDir.DirectoryPath); - var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(Substitute.For()); - var workspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker()); + var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(systemLog); + var workspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker(), + useBashWorkspace: false // Force the powershell workspace to be used + ); var stateStoreFactory = new ScriptStateStoreFactory(octopusPhysicalFileSystem); var shell = GetShellForCurrentPlatform(); @@ -47,7 +52,10 @@ public class PowerShellStartupDetectionTests workspaceFactory, stateStoreFactory, new ScriptIsolationMutex(), - Substitute.For()); + systemLog, + new Dictionary(), + powerShellStartupTimeout ?? PowerShellStartupDetection.PowerShellStartupTimeout); + return (service, workspaceFactory, stateStoreFactory, tempDir); } @@ -55,11 +63,42 @@ public class PowerShellStartupDetectionTests [Test] public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds() { - var (service, _, _, tempDir) = CreateScriptService(); + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(60)); using (tempDir) { var scriptBody = @" -# OCTOPUS-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'Hello from PowerShell' +write-output 'Script completed successfully' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); + + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); + } + } + + + [Test] + public async Task WhenPowerShellScriptHasDetectionComment_AndPowershellScriptRunsLongerThanThePowerShellStartupTimeout_ScriptSucceeds() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(10)); + using (tempDir) + { + var scriptBody = @" +# TENTACLE-POWERSHELL-STARTUP-DETECTION +Start-Sleep -Seconds 20 write-output 'Hello from PowerShell' write-output 'Script completed successfully' "; @@ -67,7 +106,6 @@ public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_Sc var startScriptCommand = new StartScriptCommandV2Builder() .WithScriptBody(scriptBody) .WithIsolation(ScriptIsolationLevel.NoIsolation) - .WithDurationStartScriptCanWaitForScriptToFinish(null) .Build(); var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); @@ -77,7 +115,6 @@ public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_Sc finalResponse.ExitCode.Should().Be(0); var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("PowerShell startup detection: Checks passed, continuing script execution"); allLogs.Should().Contain("Hello from PowerShell"); allLogs.Should().Contain("Script completed successfully"); } @@ -86,22 +123,20 @@ public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_Sc [Test] public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() { - var (service, _, _, tempDir) = CreateScriptService(); + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); using (tempDir) { // Simulate PowerShell hanging before the detection code by sleeping for a long time // This tests the scenario where PowerShell.exe starts but hangs before executing our script - var scriptBody = @" + var scriptBody = $@" # Sleep for a long time to simulate PowerShell hanging before reaching detection code Start-Sleep -Seconds 3600 -# OCTOPUS-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION write-output 'This should never be printed' "; var startScriptCommand = new StartScriptCommandV2Builder() .WithScriptBody(scriptBody) - .WithIsolation(ScriptIsolationLevel.NoIsolation) - .WithDurationStartScriptCanWaitForScriptToFinish(null) .Build(); var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); @@ -111,18 +146,72 @@ public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("PowerShell.exe process did not start within"); + allLogs.Should().Contain("process did not start within"); + } + } + + [Test] + public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndAttemptToCancelTheScript() + { + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + using (tempDir) + { + var stillRunning = Path.Combine(tempDir.DirectoryPath, "stillRunning"); + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = $@" +while ($true) {{ + Add-Content -Path '{stillRunning}' -Value 'This is the appended text.' + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' +"; + + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); + + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + + await DeletePotentiallyInUseFile(stillRunning); + await Task.Delay(TimeSpan.FromSeconds(5)); + File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made not effort to cancel it."); + } + } + + async Task DeletePotentiallyInUseFile(string file) + { + while (true) + { + this.CancellationToken.ThrowIfCancellationRequested(); + try + { + File.Delete(file); + break; + } + catch (Exception) { } + Logger.Information("Will re-attempt to delete {file} in 100ms", file); + await Task.Delay(TimeSpan.FromMilliseconds(100), this.CancellationToken); } } [Test] public async Task WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs() { - var (service, _, _, tempDir) = CreateScriptService(); + var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); using (tempDir) { - // Script without the special comment should run normally - var scriptBody = @"write-output 'Hello from PowerShell without detection' + // If we have a long-running script that does not have the detection comment, + // then tentacle should not bother with any detection logic. This includes not terminating the script + // because it never reported as running. + var scriptBody = @" +Start-Sleep -Seconds 10 +write-output 'Hello from PowerShell without detection' write-output 'Script completed successfully'"; var startScriptCommand = new StartScriptCommandV2Builder() @@ -144,155 +233,137 @@ public async Task WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOcc // The important thing is that it completes successfully } } - + + [Test] - public async Task WhenPowerShellNeverStartsAndShouldRunFileExists_CheckDetectsIt() + public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndTheScriptShouldNotBeAbleToStartAgain() { - var (_, workspaceFactory, _, tempDir) = CreateScriptService(); + var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); using (tempDir) { - // Create a script that will create the detection files - var scriptBody = @" -# OCTOPUS-POWERSHELL-STARTUP-DETECTION -write-output 'Script started' + var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); + + var scriptBody = $@" +while (Test-Path -Path '{shouldSleep}') {{ + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION +write-output 'This should never be printed' "; - var ticket = new ScriptTicket(Guid.NewGuid().ToString()); - - // Prepare workspace to create the should-run file - var workspace = await workspaceFactory.PrepareWorkspace( - ticket, - scriptBody, - new Dictionary(), - ScriptIsolationLevel.NoIsolation, - TimeSpan.Zero, - null, - null, - new List(), - CancellationToken.None); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - // Verify should-run file was created - var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); - File.Exists(shouldRunFile).Should().BeTrue(); + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); - // Verify started file doesn't exist yet - var startedFile = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); - File.Exists(startedFile).Should().BeFalse(); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - // Verify the special comment was replaced - var bootstrapScript = File.ReadAllText(workspace.BootstrapScriptFilePath); - bootstrapScript.Should().NotContain(PowerShellStartupDetection.SpecialComment); - bootstrapScript.Should().Contain("PowerShell startup detection code"); - bootstrapScript.Should().Contain("$octopusStartedFile"); - bootstrapScript.Should().Contain("$octopusShouldRunFile"); + // At this point the monitor has: + // - Created the "started" file (prevents PowerShell from creating it) + // - Deleted the "should-run" file (prevents PowerShell from running even if workspace is cleaned up) - await workspace.Delete(CancellationToken.None); - } - } + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); - [Test] - public async Task WhenDetectionCodeIsInjected_ItContainsCorrectPaths() - { - var (_, workspaceFactory, _, tempDir) = CreateScriptService(); - using (tempDir) - { - var scriptBody = @" -# OCTOPUS-POWERSHELL-STARTUP-DETECTION -write-output 'Test' -"; + // Re-invoke the bootstrap script directly - the detection code should block it from running + var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(workspace.BootstrapScriptFilePath)); + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(workspace.BootstrapScriptFilePath, null, allowInteractive: false); - var ticket = new ScriptTicket(Guid.NewGuid().ToString()); - - var workspace = await workspaceFactory.PrepareWorkspace( - ticket, - scriptBody, - new Dictionary(), - ScriptIsolationLevel.NoIsolation, - TimeSpan.Zero, - null, - null, - new List(), + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), CancellationToken.None); - var bootstrapScript = File.ReadAllText(workspace.BootstrapScriptFilePath); - - // Check that the paths in the script are correct - var expectedStartedPath = PowerShellStartupDetection.GetStartedFilePath(workspace.WorkingDirectory); - var expectedShouldRunPath = PowerShellStartupDetection.GetShouldRunFilePath(workspace.WorkingDirectory); - - bootstrapScript.Should().Contain(expectedStartedPath.Replace("\\", "\\\\").Replace("'", "''")); - bootstrapScript.Should().Contain(expectedShouldRunPath.Replace("\\", "\\\\").Replace("'", "''")); + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); - await workspace.Delete(CancellationToken.None); + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running"); + } } } [Test] - public async Task WhenPowerShellNeverStartsWithLongRunningScript_MonitoringDetectsItAfterTimeout() + public async Task WhenPowerShellNeverStarts_AndWorkspaceIsDeletedBeforeScriptRuns_TheScriptShouldStillNotBeAbleToStart() { - var (_, workspaceFactory, stateStoreFactory, tempDir) = CreateScriptService(); + var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); using (tempDir) { - var shell = GetShellForCurrentPlatform(); - - // This test simulates a long-running script that never executes PowerShell's startup detection - // We use a sleep before the detection comment to simulate PowerShell hanging - var scriptBody = @" -# Sleep to simulate PowerShell hanging (never reaches detection code) -Start-Sleep -Seconds 10 -# OCTOPUS-POWERSHELL-STARTUP-DETECTION + var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); + + var scriptBody = $@" +while (Test-Path -Path '{shouldSleep}') {{ + Start-Sleep -Seconds 1 +}} +# TENTACLE-POWERSHELL-STARTUP-DETECTION write-output 'This should never be printed' "; - var ticket = new ScriptTicket(Guid.NewGuid().ToString()); - - // Prepare the workspace with detection enabled - var workspace = await workspaceFactory.PrepareWorkspace( - ticket, - scriptBody, - new Dictionary(), - ScriptIsolationLevel.NoIsolation, - TimeSpan.Zero, - null, - null, - new List(), - CancellationToken.None); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - // Create a RunningScript with a short timeout (2 seconds) for testing - var scriptLog = workspace.CreateLog(); - var stateStore = stateStoreFactory.Create(workspace); - stateStore.Create(); - - var runningScript = new RunningScript( - shell, - workspace, - stateStore, - scriptLog, - ticket.TaskId, - new ScriptIsolationMutex(), - CancellationToken.None, - new Dictionary(), - Substitute.For(), - TimeSpan.FromSeconds(2)); // Use 2 second timeout for testing + var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); - // Execute in background - var executeTask = runningScript.Execute(); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - // Wait for completion (should take around 2 seconds for monitoring to kick in) - await executeTask; + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); - // The script should have completed - runningScript.State.Should().Be(ProcessState.Complete); - - // Check that monitoring detected the issue - var logs = scriptLog.GetOutput(0, out _); - var allLogs = string.Join("\n", logs.Select(l => l.Text)); - - // The monitoring task should have logged a warning about PowerShell not starting - // Note: The actual exit code might be 0 from the sleep command, but after our check - // it should be set to PowerShellNeverStartedExitCode - - await workspace.Delete(CancellationToken.None); + // Simulate the workspace being cleaned up while the script is still in memory: + // delete every file in the workspace except the bootstrap script itself + var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + var bootstrapScriptFilePath = workspace.BootstrapScriptFilePath; + foreach (var file in Directory.GetFiles(workspace.WorkingDirectory)) + { + if (!string.Equals(file, bootstrapScriptFilePath, StringComparison.OrdinalIgnoreCase)) + File.Delete(file); + } + + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(bootstrapScriptFilePath)); + + // Re-invoke the bootstrap script directly - even without the workspace files it should be blocked + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(bootstrapScriptFilePath, null, allowInteractive: false); + + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), + CancellationToken.None); + + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); + + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running even when the workspace files are gone"); + } } } @@ -320,7 +391,7 @@ static IShell GetShellForCurrentPlatform() if (result == 0) { // pwsh is available, create a custom shell for it - return new PwshShell(); + return new TestPwshShell(); } } catch @@ -329,14 +400,14 @@ static IShell GetShellForCurrentPlatform() } // Fall back to bash (tests will be skipped for PowerShell-specific features) - Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); + //Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); + Assert.Fail("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); return null!; } static async Task<(List, ScriptStatusResponseV2)> RunUntilScriptCompletes(ScriptServiceV2 service, StartScriptCommandV2 startScriptCommand, ScriptStatusResponseV2 response) { var (logs, lastResponse) = await RunUntilScriptFinishes(service, startScriptCommand, response); - await service.CompleteScriptAsync(new CompleteScriptCommandV2(startScriptCommand.ScriptTicket), CancellationToken.None); WriteLogsToConsole(logs); return (logs, lastResponse); } @@ -367,33 +438,4 @@ static void WriteLogsToConsole(List logs) } } } - - public class PwshShell : IShell - { - public string Name => "pwsh"; - - public string GetFullPath() - { - return "pwsh"; - } - - public string FormatCommandArguments(string bootstrapFile, string[]? scriptArguments, bool allowInteractive) - { - var args = new System.Text.StringBuilder(); - - if (!allowInteractive) - args.Append("-NonInteractive "); - - args.Append("-NoProfile "); - args.Append("-NoLogo "); - args.Append("-ExecutionPolicy Unrestricted "); - - var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); - args.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", - escapedBootstrapFile, - string.Join(" ", scriptArguments ?? new string[0])); - - return args.ToString(); - } - } } diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs index f050925bd..b1301ac8c 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/Builders/TestExecuteShellScriptCommandBuilder.cs @@ -10,7 +10,6 @@ public TestExecuteShellScriptCommandBuilder() : base(Guid.NewGuid().ToString(), ScriptIsolationLevel.NoIsolation) { WithDurationStartScriptCanWaitForScriptToFinish(null); - WithDurationToWaitForPowerShellToStart(null); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs index 1e545a3df..c0e3b05ad 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/RunningScriptFixture.cs @@ -12,6 +12,7 @@ using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Diagnostics; @@ -73,6 +74,7 @@ public void SetUpLocal() scriptIsolationMutex, cancellationTokenSource.Token, new Dictionary(), + PowerShellStartupDetection.PowerShellStartupTimeout, log); } @@ -174,6 +176,7 @@ public async Task CancellationToken_ShouldKillTheProcess() scriptIsolationMutex, cts.Token, new Dictionary(), + PowerShellStartupDetection.PowerShellStartupTimeout, new InMemoryLog()); workspace.BootstrapScript($"echo Starting\n{sleepCommand} 30\necho Finito"); diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs new file mode 100644 index 000000000..40088bf8d --- /dev/null +++ b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs @@ -0,0 +1,34 @@ +using System; +using Octopus.Tentacle.Core.Services.Scripts.Shell; + +namespace Octopus.Tentacle.Tests.Integration.Util +{ + public class TestPwshShell : IShell + { + public string Name => "pwsh"; + + public string GetFullPath() + { + return "pwsh"; + } + + public string FormatCommandArguments(string bootstrapFile, string[]? scriptArguments, bool allowInteractive) + { + var args = new System.Text.StringBuilder(); + + if (!allowInteractive) + args.Append("-NonInteractive "); + + args.Append("-NoProfile "); + args.Append("-NoLogo "); + args.Append("-ExecutionPolicy Unrestricted "); + + var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); + args.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", + escapedBootstrapFile, + string.Join(" ", scriptArguments ?? new string[0])); + + return args.ToString(); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs index 9b90d5c9d..2f684f433 100644 --- a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs +++ b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs @@ -9,6 +9,7 @@ using Octopus.Tentacle.Core.Services; using Octopus.Tentacle.Core.Services.Scripts; using Octopus.Tentacle.Core.Services.Scripts.Locking; +using Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Maintenance; using Octopus.Tentacle.Scripts; @@ -92,7 +93,7 @@ public async Task CompleteScriptAsync(CompleteScriptComman RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, CancellationTokenSource cancel) { - var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), log); + var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), PowerShellStartupDetection.PowerShellStartupTimeout, log); _ = Task.Run(async () => await runningScript.Execute(), cancel.Token); return runningScript; } From 1d6aaedb2960496dd0c728fd89b57d4939d96fe8 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 12:54:44 +1100 Subject: [PATCH 20/31] . --- .../integration/common/setup-vm-agent.sh | 3 ++ docs/powershell-startup-detection.md | 48 +++++++++++++++++++ .../PowerShellStartupDetection.cs | 42 ++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 docs/powershell-startup-detection.md diff --git a/build/test-scripts/integration/common/setup-vm-agent.sh b/build/test-scripts/integration/common/setup-vm-agent.sh index 730eaa782..e39ec6d49 100755 --- a/build/test-scripts/integration/common/setup-vm-agent.sh +++ b/build/test-scripts/integration/common/setup-vm-agent.sh @@ -6,3 +6,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/dotnet-install.sh install_dotnet 8.0 + +# Install pwsh (PowerShell Core) as a dotnet global tool so integration tests can use it +$HOME/.dotnet/dotnet tool install --global PowerShell || true diff --git a/docs/powershell-startup-detection.md b/docs/powershell-startup-detection.md new file mode 100644 index 000000000..d9cc3c3ab --- /dev/null +++ b/docs/powershell-startup-detection.md @@ -0,0 +1,48 @@ +# PowerShell Startup Detection + +## Background + +When `powershell.exe` is invoked to run a script it can occasionally start the OS process but silently stall before executing +any script content — typically due to Group Policy, antivirus, or other startup hooks that block or hang the PowerShell host. +Tentacle had no way to distinguish this from a legitimately long-running script, so affected deployments would hang indefinitely +with no useful error. + +The startup detection mechanism lets Tentacle detect and report when PowerShell starts but never executes the script body. + +## Opting in + +Detection is opt-in. To enable it, include the following marker comment somewhere near the top of your script body (before any code you want to run): + +```powershell +# TENTACLE-POWERSHELL-STARTUP-DETECTION +``` + +When Tentacle bootstraps the script it replaces this marker with generated detection code. Scripts that do not include the marker are completely unaffected. + +## What happens at runtime + +When the marker is present, Tentacle: + +1. Writes a `.octopus_powershell_should_run` file to the script workspace before launching PowerShell. +2. Replaces the marker with generated PowerShell that, at the point it executes: + - Exclusively creates a `.octopus_powershell_started` sentinel file. + - Verifies the `.octopus_powershell_should_run` file still exists. + - Exits with code `-47` if either check fails (meaning the monitor already concluded PowerShell never started). +3. Runs a monitoring task concurrently with script execution. If the sentinel file is not created within the startup timeout the monitor: + - Creates the sentinel itself (so any late-starting PowerShell process will exit immediately). + - Deletes the should-run file (so any late-starting process that somehow missed the sentinel will also exit). + - Returns exit code `-47` (`PowerShellNeverStartedExitCode`) with a "process did not start within…" message written to the task log. + +## Startup timeout + +The default timeout is **5 minutes**. It can be overridden by setting the environment variable: + +``` +OCTOPUS_TENTACLE_POWERSHELL_STARTUP_TIMEOUT= +``` + +For example, `00:02:00` for a 2-minute timeout. + +## Platform support + +Currently scoped to `powershell.exe`. \ No newline at end of file diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index c94a79578..39fc8bfc9 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -4,6 +4,48 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup { + /// + /// Provides opt-in detection of PowerShell processes that start but never execute the script body. + /// + /// + /// + /// When powershell.exe is invoked to run a script it can occasionally start the OS process + /// but silently stall before executing any script content. This was seen happening because + /// crowdstrike, prevented the script body from running. + /// + /// When this happens, we get no output from the script AND the script is un-killable. + /// + /// To work-around this, powershell scripts can place: # TENTACLE-POWERSHELL-STARTUP-DETECTION + /// at the start. This will result in Tentacle being able to detect if the script has started + /// or not. If it has not started, Tentacle will prevent the script from ever progressing and + /// report the script as failed to start. + /// + /// + /// How it works
+ /// Scripts opt in by including the marker comment # TENTACLE-POWERSHELL-STARTUP-DETECTION + /// at the start of the script body, before any work is done in the scropt. + /// When Tentacle bootstraps the script via + /// , the marker is replaced with generated PowerShell that: + /// + /// Attempts to exclusively create a .octopus_powershell_started sentinel file. + /// If the file already exists (because beat it to + /// the punch after the timeout), the script exits with code -47. + /// Checks that the .octopus_powershell_should_run file written by Tentacle before + /// launch is still present. If it has been deleted by cleanup workspace or by the monitor, + /// the script exits with code -47. + /// + /// When we run the script we also run which waits for the timeout window + /// (default 5 minutes, overridable via OCTOPUS_TENTACLE_POWERSHELL_STARTUP_TIMEOUT). It will then attempt + /// to exclusively create the .octopus_powershell_started. + /// - If it can make the file, the running script is cancelled, although it likely will not cancel. Tentacle + /// returns exit code -47, without waiting for the script to finish. + /// - If it can't make then file, then script started and tentacle simply waits for the script to complete. + /// + /// + /// Design notes
+ /// Detection is entirely opt-in; scripts without the marker are unaffected. + /// + /// public static class PowerShellStartupDetection { public const string PowershellTentacleStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; From cc66c45dda847708b34c70c20a1d4f564807693d Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 12:56:39 +1100 Subject: [PATCH 21/31] . --- POWERSHELL_STARTUP_DETECTION.md | 176 -------------------------------- 1 file changed, 176 deletions(-) delete mode 100644 POWERSHELL_STARTUP_DETECTION.md diff --git a/POWERSHELL_STARTUP_DETECTION.md b/POWERSHELL_STARTUP_DETECTION.md deleted file mode 100644 index 681d4a2a9..000000000 --- a/POWERSHELL_STARTUP_DETECTION.md +++ /dev/null @@ -1,176 +0,0 @@ -# PowerShell Startup Detection Feature - -## Overview - -This feature detects when PowerShell.exe (or pwsh) never starts or hangs before executing script content, which was causing scripts to fail silently. - -## Problem - -When PowerShell.exe is invoked, it sometimes never actually begins executing the script content. This leaves Tentacle unable to determine whether the script is genuinely running or if PowerShell never started. - -## Solution - -The feature adds a special comment marker that Tentacle replaces with detection code. This code: - -1. **Creates a "started" file** - The first thing the PowerShell script does is try to exclusively create a file to signal it has started -2. **Checks for "should run" file** - Verifies Tentacle wants the script to proceed -3. **Monitoring** - Tentacle monitors for the "started" file for 5 minutes. If it's not created, Tentacle marks the script as failed with exit code -47 - -## Usage - -Add the special comment to your PowerShell scripts: - -```powershell -# TENTACLE-POWERSHELL-STARTUP-DETECTION -# Your actual script content here -Write-Output "Hello World" -``` - -Tentacle will automatically replace this comment with detection code that: -- Attempts to create `.octopus_powershell_started` file -- Checks for `.octopus_powershell_should_run` file -- Exits with code 1 if either check fails - -## Implementation Details - -### Files Modified - -1. **ScriptExitCodes.cs** - - Added `PowerShellNeverStartedExitCode = -47` - -2. **PowerShellStartupDetection.cs** (new) - - Contains logic to generate and inject detection code - - Defines the special comment: `# TENTACLE-POWERSHELL-STARTUP-DETECTION` - - Manages file paths for detection files - -3. **ScriptWorkspace.cs** - - Modified `BootstrapScript()` to inject detection code when special comment is found - - Creates the "should run" file - -4. **BashScriptWorkspace.cs** - - Also supports detection for pwsh (PowerShell Core) on Linux/Mac - -5. **RunningScript.cs** - - Added 5-minute monitoring task - - Checks if PowerShell started after script completion - - Returns exit code -47 if PowerShell never started - -### Detection Code Generated - -When Tentacle finds the special comment, it replaces it with: - -```powershell -# PowerShell startup detection code (auto-generated by Octopus Tentacle) -$octopusStartedFile = '/path/to/workspace/.octopus_powershell_started' -$octopusShouldRunFile = '/path/to/workspace/.octopus_powershell_should_run' - -try { - # Try to create the started file exclusively - $octopusFileStream = [System.IO.File]::Open($octopusStartedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) - $octopusFileStream.Close() - $octopusFileStream.Dispose() -} catch { - # File already exists - another instance or monitoring created it - write-output "PowerShell startup detection: Started file already exists, exiting" - exit 1 -} - -# Check if the should-run file exists -if (-not (Test-Path $octopusShouldRunFile)) { - write-output "PowerShell startup detection: Should-run file does not exist, exiting" - exit 1 -} - -write-output "PowerShell startup detection: Checks passed, continuing script execution" -``` - -### Monitoring Logic - -In `RunningScript.Execute()`: - -1. **Start both tasks**: Monitoring task and script execution task run concurrently using `Task.WhenAny` -2. **Monitoring task** (`StartPowerShellStartupMonitoring`): - - Waits 5 minutes (configurable) - - Checks if "started" file was created by PowerShell - - If not found, tries to create it - - Returns `PowerShellStartupStatus` enum: - - `NotMonitored` - Detection not enabled - - `Started` - PowerShell started successfully - - `NeverStarted` - PowerShell never executed the script -3. **Race condition handling** (`Task.WhenAny`): - - If monitoring completes first with `NeverStarted`: Exit immediately with -47 - - If script completes first: Check for "started" file and return appropriate exit code - -### Behavior - -| Scenario | Result | -|----------|--------| -| PowerShell starts and runs successfully | Exit code 0, "started" file exists, monitoring returns `Started` | -| PowerShell never starts (quick) | Exit code -47 immediately after script process exits | -| PowerShell hangs before detection code | Exit code -47 after 5 minutes when monitoring returns `NeverStarted` | -| Detection code file creation fails | Exit code 1 with error message | -| "Should run" file missing | Exit code 1 with error message | - -## Testing - -Tests are located in `PowerShellStartupDetectionTests.cs`: - -1. **WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds** - - Verifies normal execution with detection enabled - -2. **WhenPowerShellNeverStarts_DetectionReportsFailure** - - Simulates PowerShell exiting before detection code runs - - Verifies exit code -47 is returned - -3. **WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs** - - Ensures scripts without the special comment run normally - -4. **WhenPowerShellNeverStartsAndShouldRunFileExists_CheckDetectsIt** - - Verifies detection files are created correctly - -5. **WhenDetectionCodeIsInjected_ItContainsCorrectPaths** - - Validates injected code has correct file paths - -6. **WhenPowerShellNeverStartsWithLongRunningScript_MonitoringDetectsItAfterTimeout** - - Tests the 5-minute monitoring logic with shorter timeout - -### Cross-Platform Support - -The feature works on: -- **Windows**: PowerShell.exe (Windows PowerShell) -- **Linux/Mac**: pwsh (PowerShell Core) - requires pwsh to be installed and available in PATH - -### Configuration - -The 5-minute timeout is configurable via the `RunningScript` constructor's `powerShellStartupCheckDelay` parameter. Tests can use a shorter timeout for faster execution. - -## Error Messages - -When PowerShell never starts, users will see: - -``` -PowerShell process completed without ever executing the script startup detection code. -This indicates that powershell.exe started but never began executing the script body. -``` - -Or if monitoring detects it after 5 minutes: - -``` -PowerShell process did not start within 5 minutes. -This indicates that powershell.exe never began executing the script. -``` - -## Limitations - -1. The special comment must be placed where PowerShell can execute it (i.e., not inside a function that's never called) -2. The detection adds a small overhead to script startup (file I/O operations) -3. Scripts without the special comment will not have this protection -4. The monitoring task runs for 5 minutes for each script with detection enabled - -## Future Enhancements - -Potential improvements: -- Make the timeout configurable per script or globally -- Add metrics/telemetry for detection hits -- Provide a way to opt-in globally without modifying every script -- Support for Bash scripts with similar detection From 645d8ceb1ad8cddda0934e8737ab02b5cb0eb2aa Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:04:09 +1100 Subject: [PATCH 22/31] . --- .../PowerShellStartupDetection.cs | 50 ++++++++++--------- .../PowerShellStartupMonitor.cs | 1 + 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index 39fc8bfc9..d57cb7560 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -48,7 +48,7 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup /// public static class PowerShellStartupDetection { - public const string PowershellTentacleStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; + public const string PowershellStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; public const string StartedFileName = ".octopus_powershell_started"; public const string ShouldRunFileName = ".octopus_powershell_should_run"; @@ -75,45 +75,47 @@ public static string GenerateDetectionCode() { return $@" # PowerShell startup detection code (auto-generated by Octopus Tentacle) -$octopusStartedFile = '{StartedFileName}' -$octopusShouldRunFile = '{ShouldRunFileName}' +& {{ + $startedFile = '{StartedFileName}' + $shouldRunFile = '{ShouldRunFileName}' -try {{ - # Try to create the started file exclusively - $octopusFileStream = [System.IO.File]::Open($octopusStartedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) - $octopusFileStream.Close() - $octopusFileStream.Dispose() -}} catch {{ - # If we couldn't create the file, it means either: - # 1. Another PowerShell instance already created it (race condition) - # 2. Tentacle already created it (meaning we never started) - # In either case, we should exit - write-output ""PowerShell startup detection: Started file already exists, exiting"" - exit -47 -}} + try {{ + # Try to create the started file exclusively + $fileStream = [System.IO.File]::Open($startedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $fileStream.Close() + $fileStream.Dispose() + }} catch {{ + # If we couldn't create the file, it means either: + # 1. Another PowerShell instance already created it (race condition) + # 2. Tentacle already created it (meaning we never started) + # In either case, we should exit + write-output ""PowerShell startup detection: Started file already exists, exiting"" + exit -47 + }} -# Check if the should-run file exists -if (-not (Test-Path $octopusShouldRunFile)) {{ - write-output ""PowerShell startup detection: Should-run file does not exist, exiting"" - exit -47 + # Check if the should-run file exists + if (-not (Test-Path $shouldRunFile)) {{ + write-output ""PowerShell startup detection: Should-run file does not exist, exiting"" + exit -47 + }} }} "; } - public static bool ContainsSpecialComment(string scriptBody) + public static bool ScriptContainsPowershellStartupDetectionComment(string scriptBody) { - return scriptBody.Contains(PowershellTentacleStartupDetectionComment); + return scriptBody.Contains(PowershellStartupDetectionComment); } public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody) { - if (!ContainsSpecialComment(scriptBody)) + if (!ScriptContainsPowershellStartupDetectionComment(scriptBody)) { return (scriptBody, false); } var detectionCode = GenerateDetectionCode(); - return (scriptBody.Replace(PowershellTentacleStartupDetectionComment, detectionCode), true); + return (scriptBody.Replace(PowershellStartupDetectionComment, detectionCode), true); } } } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs index ca8209af0..ee66d7c2e 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs @@ -45,6 +45,7 @@ public Task WaitForStartup(CancellationToken cancellati try { var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workSpaceWorkingDirectory); + // If we can make the file, then the script has not started. using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None); log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupTimeout.TotalMinutes} minutes for task {taskId}"); DeleteShouldRunFileToEnsureThePowerShellCanNeverStart(); From e932a6db3dc5daf67ef9ac95fb2eb418e369c144 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:33:27 +1100 Subject: [PATCH 23/31] Install pwsh on agents --- .../integration/common/dotnet-install.sh | 29 ------------------- .../integration/common/setup-vm-agent.sh | 3 +- 2 files changed, 1 insertion(+), 31 deletions(-) delete mode 100755 build/test-scripts/integration/common/dotnet-install.sh diff --git a/build/test-scripts/integration/common/dotnet-install.sh b/build/test-scripts/integration/common/dotnet-install.sh deleted file mode 100755 index 81a7e5f43..000000000 --- a/build/test-scripts/integration/common/dotnet-install.sh +++ /dev/null @@ -1,29 +0,0 @@ -# From https://raw.githubusercontent.com/OctopusDeploy/BuildAgentAutomation/main/Common/shared/dotnet-install.sh -function install_dotnet { - echo "downloading dotnet-install.sh so we can install dotnet $*" - - curl --silent --fail -L -O https://dot.net/v1/dotnet-install.sh || exit 1 - - echo "marking script as executable" - chmod +x dotnet-install.sh || exit 1 - - # Parse the arguments with flags --runtime and --sdk - while [ -n "$1" ]; do - if [ "$1" == "--sdk" ]; then - echo "Installing dotnet SDK $2" - ./dotnet-install.sh --channel "$2" --verbose || exit 1 - shift - elif [ "$1" == "--runtime" ]; then - echo "Installing dotnet runtime $2" - ./dotnet-install.sh --channel "$2" --runtime dotnet --verbose || exit 1 - shift - else - echo "Installing dotnet $1" - ./dotnet-install.sh --channel "$1" --verbose || exit 1 - fi - shift - done - - echo "Removing dotnet-install.sh script" - rm -f dotnet-install.sh || exit 1 -} \ No newline at end of file diff --git a/build/test-scripts/integration/common/setup-vm-agent.sh b/build/test-scripts/integration/common/setup-vm-agent.sh index e39ec6d49..76099efc1 100755 --- a/build/test-scripts/integration/common/setup-vm-agent.sh +++ b/build/test-scripts/integration/common/setup-vm-agent.sh @@ -7,5 +7,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/dotnet-install.sh install_dotnet 8.0 -# Install pwsh (PowerShell Core) as a dotnet global tool so integration tests can use it -$HOME/.dotnet/dotnet tool install --global PowerShell || true +$SCRIPT_DIR/install-pwsh.sh From cf46bac038bddb4b2fb7c4c43adf857b56133b20 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:36:06 +1100 Subject: [PATCH 24/31] . --- .../integration/common/dotnet-install.sh | 29 +++++++++++++++++++ .../integration/common/setup-vm-agent.sh | 2 -- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 build/test-scripts/integration/common/dotnet-install.sh diff --git a/build/test-scripts/integration/common/dotnet-install.sh b/build/test-scripts/integration/common/dotnet-install.sh new file mode 100755 index 000000000..81a7e5f43 --- /dev/null +++ b/build/test-scripts/integration/common/dotnet-install.sh @@ -0,0 +1,29 @@ +# From https://raw.githubusercontent.com/OctopusDeploy/BuildAgentAutomation/main/Common/shared/dotnet-install.sh +function install_dotnet { + echo "downloading dotnet-install.sh so we can install dotnet $*" + + curl --silent --fail -L -O https://dot.net/v1/dotnet-install.sh || exit 1 + + echo "marking script as executable" + chmod +x dotnet-install.sh || exit 1 + + # Parse the arguments with flags --runtime and --sdk + while [ -n "$1" ]; do + if [ "$1" == "--sdk" ]; then + echo "Installing dotnet SDK $2" + ./dotnet-install.sh --channel "$2" --verbose || exit 1 + shift + elif [ "$1" == "--runtime" ]; then + echo "Installing dotnet runtime $2" + ./dotnet-install.sh --channel "$2" --runtime dotnet --verbose || exit 1 + shift + else + echo "Installing dotnet $1" + ./dotnet-install.sh --channel "$1" --verbose || exit 1 + fi + shift + done + + echo "Removing dotnet-install.sh script" + rm -f dotnet-install.sh || exit 1 +} \ No newline at end of file diff --git a/build/test-scripts/integration/common/setup-vm-agent.sh b/build/test-scripts/integration/common/setup-vm-agent.sh index 76099efc1..730eaa782 100755 --- a/build/test-scripts/integration/common/setup-vm-agent.sh +++ b/build/test-scripts/integration/common/setup-vm-agent.sh @@ -6,5 +6,3 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/dotnet-install.sh install_dotnet 8.0 - -$SCRIPT_DIR/install-pwsh.sh From cc05ef40bd7be6aa3b5156cb886bd8db67796e9c Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:36:54 +1100 Subject: [PATCH 25/31] . --- .../PowerShellStartupDetectionTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index d43e75d7c..d06b6ac4f 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -400,8 +400,7 @@ static IShell GetShellForCurrentPlatform() } // Fall back to bash (tests will be skipped for PowerShell-specific features) - //Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); - Assert.Fail("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); + Assert.Inconclusive("PowerShell (pwsh) not available on this platform. Install PowerShell Core to run these tests."); return null!; } From 09c9e8693dfa73536b0719010f463d426229c93b Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:44:58 +1100 Subject: [PATCH 26/31] . --- .../PowerShellStartup/PowerShellStartupDetection.cs | 10 +++++----- .../PowerShellStartup/PowerShellStartupMonitor.cs | 13 +++++++------ .../Services/Scripts/ScriptServiceV2.cs | 4 ++-- .../Services/Scripts/WorkSpace/ScriptWorkspace.cs | 11 +++++++---- .../Services/Scripts/ScriptService.cs | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index d57cb7560..0c53e3a07 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -11,7 +11,7 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup /// /// When powershell.exe is invoked to run a script it can occasionally start the OS process /// but silently stall before executing any script content. This was seen happening because - /// crowdstrike, prevented the script body from running. + /// CrowdStrike prevented the script body from running. /// /// When this happens, we get no output from the script AND the script is un-killable. /// @@ -23,7 +23,7 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup /// /// How it works
/// Scripts opt in by including the marker comment # TENTACLE-POWERSHELL-STARTUP-DETECTION - /// at the start of the script body, before any work is done in the scropt. + /// at the start of the script body, before any work is done in the script. /// When Tentacle bootstraps the script via /// , the marker is replaced with generated PowerShell that: /// @@ -34,12 +34,12 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup /// launch is still present. If it has been deleted by cleanup workspace or by the monitor, /// the script exits with code -47. /// - /// When we run the script we also run which waits for the timeout window - /// (default 5 minutes, overridable via OCTOPUS_TENTACLE_POWERSHELL_STARTUP_TIMEOUT). It will then attempt + /// When we run the script we also run which waits for the timeout window + /// (default 5 minutes, overridable via TentaclePowerShellStartupTimeout). It will then attempt /// to exclusively create the .octopus_powershell_started. /// - If it can make the file, the running script is cancelled, although it likely will not cancel. Tentacle /// returns exit code -47, without waiting for the script to finish. - /// - If it can't make then file, then script started and tentacle simply waits for the script to complete. + /// - If it can't make the file, then the script started and Tentacle simply waits for the script to complete. /// /// /// Design notes
diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs index ee66d7c2e..1c40c86dd 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs @@ -70,12 +70,13 @@ public Task WaitForStartup(CancellationToken cancellati } /// - /// Since the powershell guard works by only running if it can create a file, we run into an - /// interesting situation when the powershell is blocked from running and the workspace is - /// cleaned up. If the workspace is cleaned up, then the file that the guard must create in - /// order to processed wont exist. This means it could proceed with execution. To ensure this - /// never happens, we delete the should-run file. Since the guard will only run if this file - /// exists, the workspace deleting problem goes away. + /// Deletes the should-run file so that if PowerShell does eventually start, the startup guard + /// will detect its absence and exit immediately. + /// + /// Without this, a race exists: if the workspace is cleaned up while PowerShell is blocked, + /// the started sentinel file disappears. The guard would then be able to create it successfully + /// and incorrectly conclude that it is safe to proceed. Deleting the should-run file closes + /// that gap — the guard always checks for it and exits with code -47 if it is missing. /// public void DeleteShouldRunFileToEnsureThePowerShellCanNeverStart() { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs index 1ca3775b1..0a200fc19 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/ScriptServiceV2.cs @@ -153,10 +153,10 @@ public async Task CompleteScriptAsync(CompleteScriptCommandV2 command, Cancellat await workspace.Delete(cancellationToken); } - RunningScript LaunchShell(ScriptTicket ticket,string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) + RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, IScriptStateStore stateStore, CancellationToken cancellationToken) { var runningScript = new RunningScript(shell, workspace, stateStore, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancellationToken, environmentVariables, powerShellStartupTimeout, log); - _ = Task.Run(async () => await runningScript.Execute(), cancellationToken); + _ = Task.Run(async () => await runningScript.Execute()); return runningScript; } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs index 32254226f..f8ed13c4d 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspace.cs @@ -82,10 +82,13 @@ public virtual void BootstrapScript(string scriptBody) var (processedScriptBody, shouldMonitorPowerShellStartup) = PowerShellStartupDetection.InjectDetectionCode(scriptBody); this.shouldMonitorPowerShellStartup = shouldMonitorPowerShellStartup; - // Create the "should run" file to signal that the script should proceed - var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); - FileSystem.OverwriteFile(shouldRunFile, ""); - + if (shouldMonitorPowerShellStartup) + { + // Create the "should run" file to signal that the script should proceed + var shouldRunFile = PowerShellStartupDetection.GetShouldRunFilePath(WorkingDirectory); + FileSystem.OverwriteFile(shouldRunFile, ""); + } + // default is UTF8noBOM but powershell doesn't interpret that correctly FileSystem.OverwriteFile(BootstrapScriptFilePath, processedScriptBody, Encoding.UTF8); } diff --git a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs index 2f684f433..e1b952202 100644 --- a/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs +++ b/source/Octopus.Tentacle/Services/Scripts/ScriptService.cs @@ -94,7 +94,7 @@ public async Task CompleteScriptAsync(CompleteScriptComman RunningScript LaunchShell(ScriptTicket ticket, string serverTaskId, IScriptWorkspace workspace, CancellationTokenSource cancel) { var runningScript = new RunningScript(shell, workspace, workspace.CreateLog(), serverTaskId, scriptIsolationMutex, cancel.Token, new Dictionary(), PowerShellStartupDetection.PowerShellStartupTimeout, log); - _ = Task.Run(async () => await runningScript.Execute(), cancel.Token); + _ = Task.Run(async () => await runningScript.Execute()); return runningScript; } From 5bb0dd644a6cd5705e0e26c2d3c512c38137ef38 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 13:45:15 +1100 Subject: [PATCH 27/31] . --- docs/powershell-startup-detection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/powershell-startup-detection.md b/docs/powershell-startup-detection.md index d9cc3c3ab..3937d0933 100644 --- a/docs/powershell-startup-detection.md +++ b/docs/powershell-startup-detection.md @@ -38,7 +38,7 @@ When the marker is present, Tentacle: The default timeout is **5 minutes**. It can be overridden by setting the environment variable: ``` -OCTOPUS_TENTACLE_POWERSHELL_STARTUP_TIMEOUT= +TentaclePowerShellStartupTimeout= ``` For example, `00:02:00` for a 2-minute timeout. From 7b82b5a5716d6fc08a84fc0a46ad621adf621a3a Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Wed, 1 Apr 2026 15:27:06 +1100 Subject: [PATCH 28/31] . --- .../PowerShellStartupDetectionTemplateValues.cs | 7 +++++++ .../PowerShellStartup/PowerShellStartupDetection.cs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs diff --git a/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs new file mode 100644 index 000000000..a9b754306 --- /dev/null +++ b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs @@ -0,0 +1,7 @@ +namespace Octopus.Tentacle.Contracts +{ + public static class PowerShellStartupDetectionTemplateValues + { + public const string PowershellStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index 0c53e3a07..51c8e7bce 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Util; namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup @@ -48,7 +49,6 @@ namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup /// public static class PowerShellStartupDetection { - public const string PowershellStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; public const string StartedFileName = ".octopus_powershell_started"; public const string ShouldRunFileName = ".octopus_powershell_should_run"; @@ -104,7 +104,7 @@ public static string GenerateDetectionCode() public static bool ScriptContainsPowershellStartupDetectionComment(string scriptBody) { - return scriptBody.Contains(PowershellStartupDetectionComment); + return scriptBody.Contains(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment); } public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody) @@ -115,7 +115,7 @@ public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) } var detectionCode = GenerateDetectionCode(); - return (scriptBody.Replace(PowershellStartupDetectionComment, detectionCode), true); + return (scriptBody.Replace(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment, detectionCode), true); } } } From 80dd551ef151d0fe61a5f1fb3951f1c95ab14436 Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Thu, 9 Apr 2026 12:28:23 +1000 Subject: [PATCH 29/31] From review --- ...owerShellStartupDetectionTemplateValues.cs | 9 +- .../PowerShellStartupDetection.cs | 4 +- .../PowerShellStartupMonitor.cs | 5 +- .../Services/Scripts/RunningScript.cs | 11 +- .../Scripts/WorkSpace/BashScriptWorkspace.cs | 2 +- .../WorkSpace/ScriptWorkspaceFactory.cs | 13 +- .../Scripts/WorkSpace/ScriptWorkspaceType.cs | 19 + .../PowerShellStartupDetectionTests.cs | 406 +++++++++--------- .../Util/TestPwshShell.cs | 11 +- 9 files changed, 255 insertions(+), 225 deletions(-) create mode 100644 source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs diff --git a/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs index a9b754306..00670c199 100644 --- a/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs +++ b/source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs @@ -2,6 +2,11 @@ namespace Octopus.Tentacle.Contracts { public static class PowerShellStartupDetectionTemplateValues { - public const string PowershellStartupDetectionComment = "# TENTACLE-POWERSHELL-STARTUP-DETECTION"; + /// + /// Tentacle will use this to prevent the script from running in some cases. + /// If this is used, it MUST be above anything else run in the PowerShell script. + /// Otherwise, Tentacle may falsely assume the script has not started. + /// + public const string PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript = "# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT"; } -} \ No newline at end of file +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index 51c8e7bce..7624d6838 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -104,7 +104,7 @@ public static string GenerateDetectionCode() public static bool ScriptContainsPowershellStartupDetectionComment(string scriptBody) { - return scriptBody.Contains(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment); + return scriptBody.Contains(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript); } public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody) @@ -115,7 +115,7 @@ public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) } var detectionCode = GenerateDetectionCode(); - return (scriptBody.Replace(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionComment, detectionCode), true); + return (scriptBody.Replace(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript, detectionCode), true); } } } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs index 1c40c86dd..32ac7fa87 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs @@ -83,10 +83,7 @@ public void DeleteShouldRunFileToEnsureThePowerShellCanNeverStart() try { var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory); - if (File.Exists(shouldRunFilePath)) - { - File.Delete(shouldRunFilePath); - } + File.Delete(shouldRunFilePath); } catch (Exception ex) { diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 3ed08eb59..95660f3e5 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -95,7 +95,7 @@ public async Task Execute() RecordScriptHasStarted(writer); exitCode = workspace.ShouldMonitorPowerShellStartup() - ? await RunPowershellScriptWithMonitoring(shellPath, writer) + ? await RunPowershellScriptWithMonitoring(shellPath, writer, runningScriptToken) : RunScript(shellPath, writer, runningScriptToken); } } @@ -130,7 +130,7 @@ public async Task Execute() } } - async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWriter writer) + async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWriter writer, CancellationToken runningScriptToken) { // We want to be able to make some effort to cancel the running script, if the monitor task detects it as hung. // Hence, we make a linked cancellation token with the runningScriptToken @@ -138,6 +138,10 @@ async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWr // The monitoring task is NOT linked to the runningScriptToken, since it should keep monitoring even if an attempt to // cancel the script is made. Remember, these hung powershell scripts WILL NOT CANCEL, so we must continue to monitor. + // Note: We don't bother reacting to the runningScriptToken, since under normal circumstances cancellation will be + // strictly after the script has started. Additionally, scripts can be killed. The only case that reacting to + // the runningScriptToken would help is when we are in those situations where the script never starts AND won't + // respond to being killed. The Additional effort doesn't seem worth it. await using var monitoringTaskCts = new CancelOnDisposeCancellationToken(); var monitor = new PowerShellStartupMonitor(workspace.WorkingDirectory, powerShellStartupTimeout, log, taskId); @@ -157,6 +161,9 @@ async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWr writer.WriteOutput(ProcessOutputSource.StdErr, $"{shellPath} process did not start within {powerShellStartupTimeout.TotalMinutes} minutes. Script execution aborted."); + // The script has not started, and the files on disk have been arranged, so it will never meaningfully progress. + // We will now abandon the script, as we do we will cancell its cancellation token. Which will result in + // the script possibly dieing, although from what we have seen, the script will never die. return ScriptExitCodes.PowerShellNeverStartedExitCode; } } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs index 2b32e4114..958d30bbe 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/BashScriptWorkspace.cs @@ -32,4 +32,4 @@ public override void BootstrapScript(string scriptBody) public static string GetBashBootstrapScriptFilePath(string workspaceDirectory) => Path.Combine(workspaceDirectory, BootstrapScriptFileName); } -} \ No newline at end of file +} diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs index 7a3821a14..065c76c5d 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceFactory.cs @@ -7,6 +7,7 @@ using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Core.Configuration; using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; +using Octopus.Tentacle.Core.Services.Scripts.WorkSpace; using Octopus.Tentacle.Security; using Octopus.Tentacle.Util; @@ -19,24 +20,24 @@ public class ScriptWorkspaceFactory : IScriptWorkspaceFactory readonly IOctopusFileSystem fileSystem; readonly IHomeDirectoryProvider home; readonly SensitiveValueMasker sensitiveValueMasker; - readonly bool useBashWorkspace; + readonly ScriptWorkspaceType workspaceType; public ScriptWorkspaceFactory( IOctopusFileSystem fileSystem, IHomeDirectoryProvider home, SensitiveValueMasker sensitiveValueMasker, - bool useBashWorkspace) + ScriptWorkspaceType workspaceType) { this.fileSystem = fileSystem; this.home = home; this.sensitiveValueMasker = sensitiveValueMasker; - this.useBashWorkspace = useBashWorkspace; + this.workspaceType = workspaceType; } - + public ScriptWorkspaceFactory( IOctopusFileSystem fileSystem, IHomeDirectoryProvider home, - SensitiveValueMasker sensitiveValueMasker) : this(fileSystem, home, sensitiveValueMasker, useBashWorkspace: !PlatformDetection.IsRunningOnWindows) + SensitiveValueMasker sensitiveValueMasker) : this(fileSystem, home, sensitiveValueMasker, ScriptWorkspaceTypeFromOs.ForCurrentOs()) { } @@ -117,7 +118,7 @@ string FindWorkingDirectory(ScriptTicket ticket) IScriptWorkspace CreateWorkspace(ScriptTicket scriptTicket, string workingDirectory) { - if (useBashWorkspace) + if (workspaceType == ScriptWorkspaceType.Bash) { return new BashScriptWorkspace(scriptTicket, workingDirectory, fileSystem, sensitiveValueMasker); } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs new file mode 100644 index 000000000..02012ac02 --- /dev/null +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs @@ -0,0 +1,19 @@ +using System; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Core.Services.Scripts.WorkSpace +{ + public static class ScriptWorkspaceTypeFromOs + { + public static ScriptWorkspaceType ForCurrentOs() + { + return PlatformDetection.IsRunningOnWindows ? ScriptWorkspaceType.PowerShell : ScriptWorkspaceType.Bash; + } + } + + public enum ScriptWorkspaceType + { + Bash, + PowerShell + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index d06b6ac4f..9e4e6400d 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -19,6 +19,7 @@ using Octopus.Tentacle.Core.Services.Scripts.Security.Masking; using Octopus.Tentacle.Core.Services.Scripts.Shell; using Octopus.Tentacle.Core.Services.Scripts.StateStore; +using Octopus.Tentacle.Core.Services.Scripts.WorkSpace; using Octopus.Tentacle.Scripts; using Octopus.Tentacle.Tests.Integration.Support; using Octopus.Tentacle.Tests.Integration.Util; @@ -29,159 +30,156 @@ namespace Octopus.Tentacle.Tests.Integration [IntegrationTestTimeout] public class PowerShellStartupDetectionTests : IntegrationTest { - static (ScriptServiceV2 service, ScriptWorkspaceFactory workspaceFactory, ScriptStateStoreFactory stateStoreFactory, TemporaryDirectory tempDir) CreateScriptService( - TimeSpan? powerShellStartupTimeout = null) + class ScriptServiceContext : IDisposable { - var tempDir = new TemporaryDirectory(); - - var systemLog = new SerilogSystemLog(new SerilogLoggerBuilder().Build()); - - var homeConfiguration = Substitute.For(); - homeConfiguration.HomeDirectory.Returns(tempDir.DirectoryPath); - - var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(systemLog); - var workspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker(), - useBashWorkspace: false // Force the powershell workspace to be used - ); - var stateStoreFactory = new ScriptStateStoreFactory(octopusPhysicalFileSystem); - - var shell = GetShellForCurrentPlatform(); - - var service = new ScriptServiceV2( - shell, - workspaceFactory, - stateStoreFactory, - new ScriptIsolationMutex(), - systemLog, - new Dictionary(), - powerShellStartupTimeout ?? PowerShellStartupDetection.PowerShellStartupTimeout); - - - return (service, workspaceFactory, stateStoreFactory, tempDir); + readonly TemporaryDirectory _tempDir; + + public ScriptServiceV2 Service { get; } + public ScriptWorkspaceFactory WorkspaceFactory { get; } + public string DirectoryPath => _tempDir.DirectoryPath; + + public ScriptServiceContext(TimeSpan? powerShellStartupTimeout = null) + { + _tempDir = new TemporaryDirectory(); + + var systemLog = new SerilogSystemLog(new SerilogLoggerBuilder().Build()); + + var homeConfiguration = Substitute.For(); + homeConfiguration.HomeDirectory.Returns(_tempDir.DirectoryPath); + + var octopusPhysicalFileSystem = new OctopusPhysicalFileSystem(systemLog); + WorkspaceFactory = new ScriptWorkspaceFactory(octopusPhysicalFileSystem, homeConfiguration, new SensitiveValueMasker(), ScriptWorkspaceType.PowerShell); + var stateStoreFactory = new ScriptStateStoreFactory(octopusPhysicalFileSystem); + + var shell = GetShellForCurrentPlatform(); + + Service = new ScriptServiceV2( + shell, + WorkspaceFactory, + stateStoreFactory, + new ScriptIsolationMutex(), + systemLog, + new Dictionary(), + powerShellStartupTimeout ?? PowerShellStartupDetection.PowerShellStartupTimeout); + } + + public void Dispose() => _tempDir.Dispose(); } [Test] public async Task WhenPowerShellScriptHasDetectionComment_AndRunsSuccessfully_ScriptSucceeds() { - var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(60)); - using (tempDir) - { - var scriptBody = @" -# TENTACLE-POWERSHELL-STARTUP-DETECTION + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(60)); + + var scriptBody = @" +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT write-output 'Hello from PowerShell' write-output 'Script completed successfully' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .WithIsolation(ScriptIsolationLevel.NoIsolation) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); - var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("Hello from PowerShell"); - allLogs.Should().Contain("Script completed successfully"); - } + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); } [Test] public async Task WhenPowerShellScriptHasDetectionComment_AndPowershellScriptRunsLongerThanThePowerShellStartupTimeout_ScriptSucceeds() { - var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(10)); - using (tempDir) - { - var scriptBody = @" -# TENTACLE-POWERSHELL-STARTUP-DETECTION + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(10)); + + var scriptBody = @" +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT Start-Sleep -Seconds 20 write-output 'Hello from PowerShell' write-output 'Script completed successfully' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .WithIsolation(ScriptIsolationLevel.NoIsolation) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); - var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("Hello from PowerShell"); - allLogs.Should().Contain("Script completed successfully"); - } + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("Hello from PowerShell"); + allLogs.Should().Contain("Script completed successfully"); } [Test] public async Task WhenPowerShellNeverStarts_DetectionReportsFailure() { - var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - using (tempDir) - { - // Simulate PowerShell hanging before the detection code by sleeping for a long time - // This tests the scenario where PowerShell.exe starts but hangs before executing our script - var scriptBody = $@" + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = $@" # Sleep for a long time to simulate PowerShell hanging before reaching detection code Start-Sleep -Seconds 3600 -# TENTACLE-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT write-output 'This should never be printed' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - var allLogs = string.Join("\n", logs.Select(l => l.Text)); - allLogs.Should().Contain("process did not start within"); - } + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + allLogs.Should().Contain("process did not start within"); } [Test] public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndAttemptToCancelTheScript() { - var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - using (tempDir) - { - var stillRunning = Path.Combine(tempDir.DirectoryPath, "stillRunning"); - // Simulate PowerShell hanging before the detection code by sleeping for a long time - // This tests the scenario where PowerShell.exe starts but hangs before executing our script - var scriptBody = $@" + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + + var stillRunning = Path.Combine(ctx.DirectoryPath, "stillRunning"); + // Simulate PowerShell hanging before the detection code by sleeping for a long time + // This tests the scenario where PowerShell.exe starts but hangs before executing our script + var scriptBody = $@" while ($true) {{ Add-Content -Path '{stillRunning}' -Value 'This is the appended text.' Start-Sleep -Seconds 1 }} -# TENTACLE-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT write-output 'This should never be printed' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - await DeletePotentiallyInUseFile(stillRunning); - await Task.Delay(TimeSpan.FromSeconds(5)); - File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made not effort to cancel it."); - } + await DeletePotentiallyInUseFile(stillRunning); + await Task.Delay(TimeSpan.FromSeconds(5)); + File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made not effort to cancel it."); } async Task DeletePotentiallyInUseFile(string file) @@ -203,167 +201,161 @@ async Task DeletePotentiallyInUseFile(string file) [Test] public async Task WhenPowerShellScriptWithoutDetectionComment_NormalExecutionOccurs() { - var (service, _, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - using (tempDir) - { - // If we have a long-running script that does not have the detection comment, - // then tentacle should not bother with any detection logic. This includes not terminating the script - // because it never reported as running. - var scriptBody = @" + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + + // If we have a long-running script that does not have the detection comment, + // then tentacle should not bother with any detection logic. This includes not terminating the script + // because it never reported as running. + var scriptBody = @" Start-Sleep -Seconds 10 write-output 'Hello from PowerShell without detection' write-output 'Script completed successfully'"; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .WithIsolation(ScriptIsolationLevel.NoIsolation) - .WithDurationStartScriptCanWaitForScriptToFinish(null) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .WithIsolation(ScriptIsolationLevel.NoIsolation) + .WithDurationStartScriptCanWaitForScriptToFinish(null) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (logs, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (logs, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(0); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(0); - var allLogs = string.Join("\n", logs.Select(l => l.Text)); - - allLogs.Should().NotContain("PowerShell startup detection"); - // PowerShell output might not be captured in all test environments - // The important thing is that it completes successfully - } + var allLogs = string.Join("\n", logs.Select(l => l.Text)); + + allLogs.Should().NotContain("PowerShell startup detection"); + // PowerShell output might not be captured in all test environments + // The important thing is that it completes successfully } [Test] public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_AndTheScriptShouldNotBeAbleToStartAgain() { - var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - using (tempDir) - { - var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); - File.WriteAllText(shouldSleep, ""); + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - var scriptBody = $@" + var shouldSleep = Path.Combine(ctx.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); + + var scriptBody = $@" while (Test-Path -Path '{shouldSleep}') {{ Start-Sleep -Seconds 1 }} -# TENTACLE-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT write-output 'This should never be printed' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - // At this point the monitor has: - // - Created the "started" file (prevents PowerShell from creating it) - // - Deleted the "should-run" file (prevents PowerShell from running even if workspace is cleaned up) + // At this point the monitor has: + // - Created the "started" file (prevents PowerShell from creating it) + // - Deleted the "should-run" file (prevents PowerShell from running even if workspace is cleaned up) - // Delete shouldSleep so the script can proceed past the loop when re-invoked directly - File.Delete(shouldSleep); + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); - // Re-invoke the bootstrap script directly - the detection code should block it from running - var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); - Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(workspace.BootstrapScriptFilePath)); - var shell = GetShellForCurrentPlatform(); - var args = shell.FormatCommandArguments(workspace.BootstrapScriptFilePath, null, allowInteractive: false); + // Re-invoke the bootstrap script directly - the detection code should block it from running + var workspace = ctx.WorkspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(workspace.BootstrapScriptFilePath)); + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(workspace.BootstrapScriptFilePath, null, allowInteractive: false); - var directOutput = new List(); - var directExitCode = SilentProcessRunner.ExecuteCommand( - shell.GetFullPath(), - args, - workspace.WorkingDirectory, - _ => { }, - line => directOutput.Add(line), - line => directOutput.Add(line), - CancellationToken.None); + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), + CancellationToken.None); - var directOutputText = string.Join("\n", directOutput); - Logger.Information("Direct invocation output:\n{Output}", directOutputText); + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); - directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); - // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 - if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) - { - directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running"); - } + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running"); } } [Test] public async Task WhenPowerShellNeverStarts_AndWorkspaceIsDeletedBeforeScriptRuns_TheScriptShouldStillNotBeAbleToStart() { - var (service, workspaceFactory, _, tempDir) = CreateScriptService(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); - using (tempDir) - { - var shouldSleep = Path.Combine(tempDir.DirectoryPath, "shouldSleep"); - File.WriteAllText(shouldSleep, ""); + using var ctx = new ScriptServiceContext(powerShellStartupTimeout: TimeSpan.FromSeconds(2)); + + var shouldSleep = Path.Combine(ctx.DirectoryPath, "shouldSleep"); + File.WriteAllText(shouldSleep, ""); - var scriptBody = $@" + var scriptBody = $@" while (Test-Path -Path '{shouldSleep}') {{ Start-Sleep -Seconds 1 }} -# TENTACLE-POWERSHELL-STARTUP-DETECTION +# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT write-output 'This should never be printed' "; - var startScriptCommand = new StartScriptCommandV2Builder() - .WithScriptBody(scriptBody) - .Build(); + var startScriptCommand = new StartScriptCommandV2Builder() + .WithScriptBody(scriptBody) + .Build(); - var startScriptResponse = await service.StartScriptAsync(startScriptCommand, CancellationToken.None); - var (_, finalResponse) = await RunUntilScriptCompletes(service, startScriptCommand, startScriptResponse); + var startScriptResponse = await ctx.Service.StartScriptAsync(startScriptCommand, CancellationToken.None); + var (_, finalResponse) = await RunUntilScriptCompletes(ctx.Service, startScriptCommand, startScriptResponse); - finalResponse.State.Should().Be(ProcessState.Complete); - finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); + finalResponse.State.Should().Be(ProcessState.Complete); + finalResponse.ExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode); - // Delete shouldSleep so the script can proceed past the loop when re-invoked directly - File.Delete(shouldSleep); + // Delete shouldSleep so the script can proceed past the loop when re-invoked directly + File.Delete(shouldSleep); - // Simulate the workspace being cleaned up while the script is still in memory: - // delete every file in the workspace except the bootstrap script itself - var workspace = workspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); - var bootstrapScriptFilePath = workspace.BootstrapScriptFilePath; - foreach (var file in Directory.GetFiles(workspace.WorkingDirectory)) - { - if (!string.Equals(file, bootstrapScriptFilePath, StringComparison.OrdinalIgnoreCase)) - File.Delete(file); - } + // Simulate the workspace being cleaned up while the script is still in memory: + // delete every file in the workspace except the bootstrap script itself + var workspace = ctx.WorkspaceFactory.GetWorkspace(startScriptCommand.ScriptTicket, WorkspaceReadinessCheck.Skip); + var bootstrapScriptFilePath = workspace.BootstrapScriptFilePath; + foreach (var file in Directory.GetFiles(workspace.WorkingDirectory)) + { + if (!string.Equals(file, bootstrapScriptFilePath, StringComparison.OrdinalIgnoreCase)) + File.Delete(file); + } - Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(bootstrapScriptFilePath)); + Logger.Information("Bootstrap script contents:\n{BootstrapScript}", File.ReadAllText(bootstrapScriptFilePath)); - // Re-invoke the bootstrap script directly - even without the workspace files it should be blocked - var shell = GetShellForCurrentPlatform(); - var args = shell.FormatCommandArguments(bootstrapScriptFilePath, null, allowInteractive: false); + // Re-invoke the bootstrap script directly - even without the workspace files it should be blocked + var shell = GetShellForCurrentPlatform(); + var args = shell.FormatCommandArguments(bootstrapScriptFilePath, null, allowInteractive: false); - var directOutput = new List(); - var directExitCode = SilentProcessRunner.ExecuteCommand( - shell.GetFullPath(), - args, - workspace.WorkingDirectory, - _ => { }, - line => directOutput.Add(line), - line => directOutput.Add(line), - CancellationToken.None); + var directOutput = new List(); + var directExitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + args, + workspace.WorkingDirectory, + _ => { }, + line => directOutput.Add(line), + line => directOutput.Add(line), + CancellationToken.None); - var directOutputText = string.Join("\n", directOutput); - Logger.Information("Direct invocation output:\n{Output}", directOutputText); + var directOutputText = string.Join("\n", directOutput); + Logger.Information("Direct invocation output:\n{Output}", directOutputText); - directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); + directOutputText.Should().Contain("PowerShell startup detection", "The detection code should have run and reported why it exited"); - // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 - if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) - { - directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running even when the workspace files are gone"); - } + // On Mac/Linux exit codes are unsigned 8-bit, so -47 wraps to 209 + if (Octopus.Tentacle.Util.PlatformDetection.IsRunningOnWindows) + { + directExitCode.Should().Be(ScriptExitCodes.PowerShellNeverStartedExitCode, "The detection code should prevent the script from running even when the workspace files are gone"); } } diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs index 40088bf8d..c0c0a46d2 100644 --- a/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs +++ b/source/Octopus.Tentacle.Tests.Integration/Util/TestPwshShell.cs @@ -24,6 +24,15 @@ public string FormatCommandArguments(string bootstrapFile, string[]? scriptArgum args.Append("-ExecutionPolicy Unrestricted "); var escapedBootstrapFile = bootstrapFile.Replace("'", "''"); + + // This is all copied from PowerShell.cs, the intention here is that this matches that as best we can for + // testing powershell stuff on linux/mac for local dev. + + // $ErrorActionPreference = 'Stop': make all PS errors terminating. + // `. { . 'file' args }`: double dot-source — outer `. { }` runs the block in current scope + // so $LastExitCode set inside the bootstrap script remains visible after it exits. + // `if (test-path variable:global:lastexitcode)`: $LastExitCode only exists if a native + // process ran; guard prevents `exit $null` on pure-PS scripts. args.AppendFormat("-Command \"$ErrorActionPreference = 'Stop'; . {{. '{0}' {1}; if ((test-path variable:global:lastexitcode)) {{ exit $LastExitCode }}}}\"", escapedBootstrapFile, string.Join(" ", scriptArguments ?? new string[0])); @@ -31,4 +40,4 @@ public string FormatCommandArguments(string bootstrapFile, string[]? scriptArgum return args.ToString(); } } -} \ No newline at end of file +} From a33a74a77e8e30d22d37aa7b92697f89d4c0d8ae Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Thu, 9 Apr 2026 15:13:53 +1000 Subject: [PATCH 30/31] . --- .../Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs | 4 ++-- .../Services/Scripts/WorkSpace/ScriptWorkspaceType.cs | 2 +- .../PowerShellStartupDetectionTests.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs index 95660f3e5..fca5dfa15 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/RunningScript.cs @@ -162,8 +162,8 @@ async Task RunPowershellScriptWithMonitoring(string shellPath, IScriptLogWr $"{shellPath} process did not start within {powerShellStartupTimeout.TotalMinutes} minutes. Script execution aborted."); // The script has not started, and the files on disk have been arranged, so it will never meaningfully progress. - // We will now abandon the script, as we do we will cancell its cancellation token. Which will result in - // the script possibly dieing, although from what we have seen, the script will never die. + // We will now abandon the script, as we do we will cancel its cancellation token. Which will result in + // the script possibly dying, although from what we have seen, the script will never die. return ScriptExitCodes.PowerShellNeverStartedExitCode; } } diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs index 02012ac02..717fac792 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/WorkSpace/ScriptWorkspaceType.cs @@ -16,4 +16,4 @@ public enum ScriptWorkspaceType Bash, PowerShell } -} \ No newline at end of file +} diff --git a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs index 9e4e6400d..814523b42 100644 --- a/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs +++ b/source/Octopus.Tentacle.Tests.Integration/PowerShellStartupDetectionTests.cs @@ -179,7 +179,7 @@ public async Task WhenPowerShellNeverStarts_WeShouldDetectTheScriptDidNotStart_A await DeletePotentiallyInUseFile(stillRunning); await Task.Delay(TimeSpan.FromSeconds(5)); - File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made not effort to cancel it."); + File.Exists(stillRunning).Should().BeFalse("Otherwise the script is still running and we made no effort to cancel it."); } async Task DeletePotentiallyInUseFile(string file) From 5bf8b4c4f9030c3c2f2e4c07c631c5aea253ca1a Mon Sep 17 00:00:00 2001 From: Luke Butters Date: Thu, 9 Apr 2026 15:20:59 +1000 Subject: [PATCH 31/31] Change timeout to 13minutes a unique timeout, that is well above the time we expect a powershell to start in --- .../Scripts/PowerShellStartup/PowerShellStartupDetection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs index 7624d6838..2faadd323 100644 --- a/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs +++ b/source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs @@ -57,7 +57,7 @@ public static TimeSpan PowerShellStartupTimeout get { var raw = Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePowerShellStartupTimeout); - return TimeSpan.TryParse(raw, out var value) ? value : TimeSpan.FromMinutes(5); + return TimeSpan.TryParse(raw, out var value) ? value : TimeSpan.FromMinutes(13); } }