Tentacle can detect Powershell scripts that don't start, and can abandon them.

docs/powershell-startup-detection.md

@@ -0,0 +1,48 @@
+# PowerShell Startup Detection
+
+## Background
+
+When `powershell.exe` is invoked to run a script it can occasionally start the OS process but silently stall before executing 
+any script content — typically due to Group Policy, antivirus, or other startup hooks that block or hang the PowerShell host. 
+Tentacle had no way to distinguish this from a legitimately long-running script, so affected deployments would hang indefinitely 
+with no useful error.
+
+The startup detection mechanism lets Tentacle detect and report when PowerShell starts but never executes the script body.
+
+## Opting in
+
+Detection is opt-in. To enable it, include the following marker comment somewhere near the top of your script body (before any code you want to run):
+
+```powershell
+# TENTACLE-POWERSHELL-STARTUP-DETECTION
+```
+
+When Tentacle bootstraps the script it replaces this marker with generated detection code. Scripts that do not include the marker are completely unaffected.
+
+## What happens at runtime
+
+When the marker is present, Tentacle:
+
+1. Writes a `.octopus_powershell_should_run` file to the script workspace before launching PowerShell.
+2. Replaces the marker with generated PowerShell that, at the point it executes:
+   - Exclusively creates a `.octopus_powershell_started` sentinel file.
+   - Verifies the `.octopus_powershell_should_run` file still exists.
+   - Exits with code `-47` if either check fails (meaning the monitor already concluded PowerShell never started).
+3. Runs a monitoring task concurrently with script execution. If the sentinel file is not created within the startup timeout the monitor:
+   - Creates the sentinel itself (so any late-starting PowerShell process will exit immediately).
+   - Deletes the should-run file (so any late-starting process that somehow missed the sentinel will also exit).
+   - Returns exit code `-47` (`PowerShellNeverStartedExitCode`) with a "process did not start within…" message written to the task log.
+
+## Startup timeout
+
+The default timeout is **5 minutes**. It can be overridden by setting the environment variable:
+
+```
+TentaclePowerShellStartupTimeout=<TimeSpan>
+```
+
+For example, `00:02:00` for a 2-minute timeout.
+
+## Platform support
+
+Currently scoped to `powershell.exe`.

source/Octopus.Tentacle.CommonTestUtils/Diagnostics/SerilogSystemLog.cs

@@ -0,0 +1,39 @@
+using System;
+using Octopus.Tentacle.Core.Diagnostics;
+using Octopus.Tentacle.Diagnostics;
+using Serilog;
+using Serilog.Events;
+using LogEvent = Octopus.Tentacle.Diagnostics.LogEvent;
+
+namespace Octopus.Tentacle.CommonTestUtils.Diagnostics
+{
+    public class SerilogSystemLog : SystemLog
+    {
+        readonly ILogger logger;
+
+        public SerilogSystemLog(ILogger logger)
+        {
+            this.logger = logger;
+        }
+
+        protected override void WriteEvent(LogEvent logEvent)
+        {
+            var level = ToSerilogLevel(logEvent.Category);
+            if (logEvent.Error != null)
+                logger.Write(level, logEvent.Error, logEvent.MessageText);
+            else
+                logger.Write(level, logEvent.MessageText);
+        }
+
+        static LogEventLevel ToSerilogLevel(LogCategory category) => category switch
+        {
+            LogCategory.Trace => LogEventLevel.Verbose,
+            LogCategory.Verbose => LogEventLevel.Debug,
+            LogCategory.Info => LogEventLevel.Information,
+            LogCategory.Warning => LogEventLevel.Warning,
+            LogCategory.Error => LogEventLevel.Error,
+            LogCategory.Fatal => LogEventLevel.Fatal,
+            _ => LogEventLevel.Information
+        };
+    }
+}

source/Octopus.Tentacle.Contracts/PowerShellStartupDetectionTemplateValues.cs

@@ -0,0 +1,12 @@
+namespace Octopus.Tentacle.Contracts
+{
+    public static class PowerShellStartupDetectionTemplateValues
+    {
+        /// <summary>
+        /// Tentacle will use this to prevent the script from running in some cases.
+        /// If this is used, it MUST be above anything else run in the PowerShell script.
+        /// Otherwise, Tentacle may falsely assume the script has not started.
+        /// </summary>
+        public const string PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript = "# TENTACLE-POWERSHELL-STARTUP-DETECTION-AND-GUARD-MUST-BE-AT-THE-START-OF-THE-SCRIPT";
+    }
+}

source/Octopus.Tentacle.Contracts/ScriptExitCodes.cs

@@ -11,6 +11,7 @@ public static class ScriptExitCodes
         public const int TimeoutExitCode = -44;
         public const int UnknownScriptExitCode = -45;
         public const int UnknownResultExitCode = -46;
+        public const int PowerShellNeverStartedExitCode = -47;
 
         //Kubernetes Agent
         public const int KubernetesScriptPodNotFound = -81;

source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupDetection.cs

@@ -0,0 +1,121 @@
+using System;
+using System.IO;
+using Octopus.Tentacle.Contracts;
+using Octopus.Tentacle.Core.Util;
+
+namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup
+{
+    /// <summary>
+    /// Provides opt-in detection of PowerShell processes that start but never execute the script body.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// When <c>powershell.exe</c> is invoked to run a script it can occasionally start the OS process
+    /// but silently stall before executing any script content. This was seen happening because
+    /// CrowdStrike prevented the script body from running.
+    /// 
+    /// When this happens, we get no output from the script AND the script is un-killable.
+    ///
+    /// To work-around this, powershell scripts can place: # TENTACLE-POWERSHELL-STARTUP-DETECTION
+    /// at the start. This will result in Tentacle being able to detect if the script has started
+    /// or not. If it has not started, Tentacle will prevent the script from ever progressing and
+    /// report the script as failed to start.
+    /// </para>
+    /// <para>
+    /// <b>How it works</b><br/>
+    /// Scripts opt in by including the marker comment <c># TENTACLE-POWERSHELL-STARTUP-DETECTION</c>
+    /// at the start of the script body, before any work is done in the script.
+    /// When Tentacle bootstraps the script via
+    /// <see cref="InjectDetectionCode"/>, the marker is replaced with generated PowerShell that:
+    /// <list type="number">
+    ///   <item>Attempts to exclusively create a <c>.octopus_powershell_started</c> sentinel file.
+    ///         If the file already exists (because <see cref="PowerShellStartupMonitor"/> beat it to
+    ///         the punch after the timeout), the script exits with code <c>-47</c>.</item>
+    ///   <item>Checks that the <c>.octopus_powershell_should_run</c> file written by Tentacle before
+    ///         launch is still present.  If it has been deleted by cleanup workspace or by the monitor,
+    ///         the script exits with code <c>-47</c>.</item>
+    /// </list>
+    /// When we run the script we also run <see cref="PowerShellStartupMonitor"/> which waits for the timeout window
+    /// (default 5 minutes, overridable via <c>TentaclePowerShellStartupTimeout</c>). It will then attempt
+    /// to exclusively create the <c>.octopus_powershell_started</c>.
+    ///   - If it can make the file, the running script is cancelled, although it likely will not cancel. Tentacle
+    ///    returns exit code <c>-47</c>, without waiting for the script to finish.
+    ///   - If it can't make the file, then the script started and Tentacle simply waits for the script to complete.
+    /// </para>
+    /// <para>
+    /// <b>Design notes</b><br/>
+    /// Detection is entirely opt-in; scripts without the marker are unaffected.
+    /// </para>
+    /// </remarks>
+    public static class PowerShellStartupDetection
+    {
+        public const string StartedFileName = ".octopus_powershell_started";
+        public const string ShouldRunFileName = ".octopus_powershell_should_run";
+
+        public static TimeSpan PowerShellStartupTimeout
+        {
+            get
+            {
+                var raw = Environment.GetEnvironmentVariable(EnvironmentVariables.TentaclePowerShellStartupTimeout);
+                return TimeSpan.TryParse(raw, out var value) ? value : TimeSpan.FromMinutes(13);
+            }
+        }
+
+        public static string GetStartedFilePath(string workingDirectory)
+        {
+            return Path.Combine(workingDirectory, StartedFileName);
+        }
+
+        public static string GetShouldRunFilePath(string workingDirectory)
+        {
+            return Path.Combine(workingDirectory, ShouldRunFileName);
+        }
+
+        public static string GenerateDetectionCode()
+        {
+            return $@"
+# PowerShell startup detection code (auto-generated by Octopus Tentacle)
+& {{
+    $startedFile = '{StartedFileName}'
+    $shouldRunFile = '{ShouldRunFileName}'
+
+    try {{
+        # Try to create the started file exclusively
+        $fileStream = [System.IO.File]::Open($startedFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)
+        $fileStream.Close()
+        $fileStream.Dispose()
+    }} catch {{
+        # If we couldn't create the file, it means either:
+        # 1. Another PowerShell instance already created it (race condition)
+        # 2. Tentacle already created it (meaning we never started)
+        # In either case, we should exit
+        write-output ""PowerShell startup detection: Started file already exists, exiting""
+        exit -47
+    }}
+
+    # Check if the should-run file exists
+    if (-not (Test-Path $shouldRunFile)) {{
+        write-output ""PowerShell startup detection: Should-run file does not exist, exiting""
+        exit -47
+    }}
+}}
+";
+        }
+
+        public static bool ScriptContainsPowershellStartupDetectionComment(string scriptBody)
+        {
+            return scriptBody.Contains(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript);
+        }
+
+        public static (string processedScriptBody, bool shouldMonitorPowerShellStartup) InjectDetectionCode(string scriptBody)
+        {
+            if (!ScriptContainsPowershellStartupDetectionComment(scriptBody))
+            {
+                return (scriptBody, false);
+            }
+
+            var detectionCode = GenerateDetectionCode();
+            return (scriptBody.Replace(PowerShellStartupDetectionTemplateValues.PowershellStartupDetectionCommentMustBeAtTheStartOfTheScript, detectionCode), true);
+        }
+    }
+}

source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupMonitor.cs

@@ -0,0 +1,94 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Halibut.Util;
+using Octopus.Tentacle.Core.Diagnostics;
+
+namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup
+{
+    class PowerShellStartupMonitor
+    {
+        readonly string workSpaceWorkingDirectory;
+        readonly TimeSpan powerShellStartupTimeout;
+        readonly ILog log;
+        readonly string taskId;
+
+        public PowerShellStartupMonitor(string workSpaceWorkingDirectory, TimeSpan powerShellStartupTimeout, ILog log, string taskId)
+        {
+            this.workSpaceWorkingDirectory = workSpaceWorkingDirectory;
+            this.powerShellStartupTimeout = powerShellStartupTimeout;
+            this.log = log;
+            this.taskId = taskId;
+        }
+
+        public Task<PowerShellStartupStatus> WaitForStartup(CancellationToken cancellationToken)
+        {
+            var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory);
+
+            if (!File.Exists(shouldRunFilePath))
+            {
+                return Task.FromResult(PowerShellStartupStatus.NotMonitored);
+            }
+
+            return Task.Run(async () =>
+            {
+                try
+                {
+                    await DelayWithoutException.Delay(powerShellStartupTimeout, cancellationToken);
+
+                    if (cancellationToken.IsCancellationRequested)
+                    {
+                        return PowerShellStartupStatus.NotMonitored;
+                    }
+
+                    try
+                    {
+                        var startedFilePath = PowerShellStartupDetection.GetStartedFilePath(workSpaceWorkingDirectory);
+                        // If we can make the file, then the script has not started.
+                        using var fileStream = File.Open(startedFilePath, FileMode.CreateNew, FileAccess.Write, FileShare.None);
+                        log.Warn($"PowerShell startup detection: PowerShell did not start within {powerShellStartupTimeout.TotalMinutes} minutes for task {taskId}");
+                        DeleteShouldRunFileToEnsureThePowerShellCanNeverStart();
+                        return PowerShellStartupStatus.NeverStarted;
+                    }
+                    catch (IOException)
+                    {
+                        log.Info($"PowerShell startup detection: PowerShell started late (after {powerShellStartupTimeout.TotalMinutes} minutes) for task {taskId}");
+                        return PowerShellStartupStatus.Started;
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    return PowerShellStartupStatus.NotMonitored;
+                }
+                catch (Exception ex)
+                {
+                    log.Warn(ex, $"Error in PowerShell startup monitoring for task {taskId}");
+                    return PowerShellStartupStatus.NotMonitored;
+                }
+            });
+        }
+
+        /// <summary>
+        /// Deletes the should-run file so that if PowerShell does eventually start, the startup guard
+        /// will detect its absence and exit immediately.
+        ///
+        /// Without this, a race exists: if the workspace is cleaned up while PowerShell is blocked,
+        /// the started sentinel file disappears. The guard would then be able to create it successfully
+        /// and incorrectly conclude that it is safe to proceed. Deleting the should-run file closes
+        /// that gap — the guard always checks for it and exits with code -47 if it is missing.
+        /// </summary>
+        public void DeleteShouldRunFileToEnsureThePowerShellCanNeverStart()
+        {
+            try
+            {
+                var shouldRunFilePath = PowerShellStartupDetection.GetShouldRunFilePath(workSpaceWorkingDirectory);
+                File.Delete(shouldRunFilePath);
+            }
+            catch (Exception ex)
+            {
+                log.Warn(ex, $"Failed to delete should-run file for task {taskId}");
+            }
+        }
+    }
+}

source/Octopus.Tentacle.Core/Services/Scripts/PowerShellStartup/PowerShellStartupStatus.cs

@@ -0,0 +1,9 @@
+namespace Octopus.Tentacle.Core.Services.Scripts.PowerShellStartup
+{
+    public enum PowerShellStartupStatus
+    {
+        NotMonitored,
+        Started,
+        NeverStarted
+    }
+}