Implement Script Isolation

source/Calamari.Common/CalamariFlavourProgram.cs

@@ -12,6 +12,7 @@
 using Calamari.Common.Features.FunctionScriptContributions;
 using Calamari.Common.Features.Packages;
 using Calamari.Common.Features.Processes;
+using Calamari.Common.Features.Processes.ScriptIsolation;
 using Calamari.Common.Features.Scripting;
 using Calamari.Common.Features.Scripting.DotnetScript;
 using Calamari.Common.Features.StructuredVariables;
@@ -78,6 +79,7 @@ protected virtual int Run(string[] args)
                 }
 #endif
 
+                using var _ = Isolation.Enforce(options.ScriptIsolation);
                 return ResolveAndExecuteCommand(container, options);
             }
             catch (Exception ex)

source/Calamari.Common/CalamariFlavourProgramAsync.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
 using System.Threading.Tasks;
 using Autofac;
 using Autofac.Core;
@@ -14,6 +15,7 @@
 using Calamari.Common.Features.FunctionScriptContributions;
 using Calamari.Common.Features.Packages;
 using Calamari.Common.Features.Processes;
+using Calamari.Common.Features.Processes.ScriptIsolation;
 using Calamari.Common.Features.Scripting;
 using Calamari.Common.Features.Scripting.DotnetScript;
 using Calamari.Common.Features.StructuredVariables;
@@ -143,6 +145,7 @@ protected async Task<int> Run(string[] args)
                 }
 #endif
 
+                await using var _ = await Isolation.EnforceAsync(options.ScriptIsolation, CancellationToken.None);
                 await ResolveAndExecuteCommand(container, options);
                 return 0;
             }

source/Calamari.Common/Features/Processes/ScriptIsolation/FileLock.cs

@@ -0,0 +1,69 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public static class FileLock
+{
+    public static ILockHandle Acquire(LockOptions lockOptions)
+    {
+        var fileShareMode = GetFileShareMode(lockOptions.Type);
+        try
+        {
+            var fileStream = lockOptions.LockFile.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite, fileShareMode);
+            return new LockHandle(fileStream);
+        }
+        catch (IOException e) when (IsFileLocked(e))
+        {
+            throw new LockRejectedException(e);
+        }
+    }
+
+    const int WindowsErrorSharingViolation = unchecked((int)0x80070020); // ERROR_SHARING_VIOLATION
+    const int LinuxErrorAgainWouldBlock = 11; // EAGAIN / EWOULDBLOCK
+    const int MacOsErrorAgainWouldBlock = 35; // EAGAIN / EWOULDBLOCK
+
+    static bool IsFileLocked(IOException ioException)
+    {
+        if (OperatingSystem.IsWindows())
+        {
+            return ioException.HResult == WindowsErrorSharingViolation;
+        }
+
+        if (OperatingSystem.IsLinux())
+        {
+            return ioException.HResult == LinuxErrorAgainWouldBlock;
+        }
+
+        if (OperatingSystem.IsMacOS())
+        {
+            return ioException.HResult == MacOsErrorAgainWouldBlock;
+        }
+
+        return false;
+    }
+
+    static FileShare GetFileShareMode(LockType isolationLevel)
+    {
+        return isolationLevel switch
+        {
+            LockType.Exclusive => FileShare.None,
+            LockType.Shared => FileShare.ReadWrite,
+            _ => throw new ArgumentOutOfRangeException(nameof(isolationLevel), isolationLevel, null)
+        };
+    }
+
+    sealed class LockHandle(FileStream fileStream) : ILockHandle
+    {
+        public void Dispose()
+        {
+            fileStream.Dispose();
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            await fileStream.DisposeAsync();
+        }
+    }
+}

source/Calamari.Common/Features/Processes/ScriptIsolation/ILockHandle.cs

@@ -0,0 +1,5 @@
+using System;
+
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public interface ILockHandle : IAsyncDisposable, IDisposable;

source/Calamari.Common/Features/Processes/ScriptIsolation/Isolation.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Calamari.Common.Plumbing.Commands;
+using Calamari.Common.Plumbing.Logging;
+
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public static class Isolation
+{
+    public static ILockHandle Enforce(CommonOptions.ScriptIsolationOptions scriptIsolationOptions)
+    {
+        var lockOptions = LockOptions.FromScriptIsolationOptionsOrNull(scriptIsolationOptions);
+        if (lockOptions is null)
+        {
+            return new NoLock();
+        }
+
+        var pipeline = lockOptions.BuildLockAcquisitionPipeline();
+        LogIsolation(lockOptions);
+        try
+        {
+            return pipeline.Execute(FileLock.Acquire, lockOptions);
+        }
+        catch (Exception exception)
+        {
+            LockRejectedException.Throw(exception);
+            throw; // Satisfy the compiler
+        }
+    }
+
+    public static async Task<ILockHandle> EnforceAsync(
+        CommonOptions.ScriptIsolationOptions scriptIsolationOptions,
+        CancellationToken cancellationToken
+    )
+    {
+        var lockOptions = LockOptions.FromScriptIsolationOptionsOrNull(scriptIsolationOptions);
+        if (lockOptions is null)
+        {
+            return new NoLock();
+        }
+
+        var pipeline = lockOptions.BuildLockAcquisitionPipeline();
+        LogIsolation(lockOptions);
+        try
+        {
+            return await pipeline.ExecuteAsync(static (o, _) => ValueTask.FromResult(FileLock.Acquire(o)), lockOptions, cancellationToken);
+        }
+        catch (Exception exception)
+        {
+            LockRejectedException.Throw(exception);
+            throw; // Satisfy the compiler
+        }
+    }
+
+    static void LogIsolation(LockOptions lockOptions)
+    {
+        Log.Verbose($"Acquiring script isolation mutex {lockOptions.Name} with {lockOptions.Type} lock");
+    }
+
+    class NoLock : ILockHandle
+    {
+        public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+
+        public void Dispose()
+        {
+        }
+    }
+}

source/Calamari.Common/Features/Processes/ScriptIsolation/LockOptions.cs

@@ -0,0 +1,145 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+using Calamari.Common.Plumbing.Commands;
+using Calamari.Common.Plumbing.Logging;
+using Polly;
+using Polly.Timeout;
+
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public sealed record LockOptions(
+    LockType Type,
+    string Name,
+    FileInfo LockFile,
+    TimeSpan Timeout
+)
+{
+    static readonly TimeSpan RetryInitialDelay = TimeSpan.FromMilliseconds(10);
+    static readonly TimeSpan RetryMaxDelay = TimeSpan.FromMilliseconds(500);
+
+    public ResiliencePipeline<ILockHandle> BuildLockAcquisitionPipeline()
+    {
+        var builder = new ResiliencePipelineBuilder<ILockHandle>();
+        return AddLockOptions(builder).Build();
+    }
+
+    public ResiliencePipelineBuilder<ILockHandle> AddLockOptions(ResiliencePipelineBuilder<ILockHandle> builder)
+    {
+        // If it's 10ms or less, we'll skip timeout and limit retries
+        var retryAttempts = Timeout <= TimeSpan.FromMilliseconds(10) && Timeout != System.Threading.Timeout.InfiniteTimeSpan
+            ? 1
+            : int.MaxValue;
+        if (Timeout > TimeSpan.FromMilliseconds(10))
+        {
+            builder.AddTimeout(
+                               new TimeoutStrategyOptions
+                               {
+                                   // Using a timeout generator does not constrain the timeout to
+                                   // a maximum of 1 day
+                                   TimeoutGenerator = _ => ValueTask.FromResult(Timeout)
+                               }
+                              );
+        }
+
+        builder.AddRetry(
+                         new()
+                         {
+                             BackoffType = DelayBackoffType.Exponential,
+                             Delay = RetryInitialDelay,
+                             MaxDelay = RetryMaxDelay,
+                             MaxRetryAttempts = retryAttempts,
+                             ShouldHandle = new PredicateBuilder<ILockHandle>().Handle<LockRejectedException>(),
+                             UseJitter = true
+                         }
+                        );
+        return builder;
+    }
+
+    public static LockOptions? FromScriptIsolationOptionsOrNull(CommonOptions.ScriptIsolationOptions options)
+    {
+        if (!options.FullyConfigured)
+        {
+            LogIfPartiallyConfigured(options);
+            return null;
+        }
+
+        var lockType = MapScriptIsolationLevelToLockTypeOrNull(options.Level);
+        if (lockType == null)
+        {
+            Log.Verbose($"Failed to map script isolation level '{options.Level}' to a valid LockType. Expected 'FullIsolation' or 'NoIsolation' (case-insensitive).");
+            LogIsolationWillNotBeEnforced();
+            return null;
+        }
+
+        TimeSpan timeout;
+
+        if (string.IsNullOrWhiteSpace(options.Timeout))
+        {
+            timeout = System.Threading.Timeout.InfiniteTimeSpan;
+        }
+        else if (!TimeSpan.TryParse(options.Timeout, out timeout))
+        {
+            Log.Verbose($"Failed to parse mutex timeout value '{options.Timeout}' as TimeSpan. Defaulting to Infinite.");
+            timeout = System.Threading.Timeout.InfiniteTimeSpan;
+        }
+
+        var lockFileInfo = GetLockFileInfo(options.TentacleHome, options.MutexName);
+        return new LockOptions(lockType.Value, options.MutexName, lockFileInfo, timeout);
+    }
+
+    static void LogIfPartiallyConfigured(CommonOptions.ScriptIsolationOptions options)
+    {
+        if (!options.PartiallyConfigured)
+        {
+            return;
+        }
+
+        var missingOptions = new List<string>();
+        if (string.IsNullOrWhiteSpace(options.Level))
+        {
+            missingOptions.Add("scriptIsolationLevel");
+        }
+
+        if (string.IsNullOrWhiteSpace(options.MutexName))
+        {
+            missingOptions.Add("scriptIsolationMutexName");
+        }
+
+        if (string.IsNullOrWhiteSpace(options.TentacleHome))
+        {
+            missingOptions.Add("TentacleHome (Environment Variable)");
+        }
+
+        var optionIsOrAre = missingOptions.Count > 1 ? "options are" : "option is";
+        Log.Verbose($"Some script isolation options were provided, but the following required {optionIsOrAre} missing: {string.Join(", ", missingOptions)}");
+        LogIsolationWillNotBeEnforced();
+    }
+
+    static void LogIsolationWillNotBeEnforced()
+    {
+        Log.Verbose("Script isolation will not be enforced.");
+    }
+
+    static FileInfo GetLockFileInfo(string tentacleHome, string mutexName)
+    {
+        foreach (var invalidChar in Path.GetInvalidFileNameChars())
+        {
+            if (mutexName.Contains(invalidChar))
+            {
+                throw new ArgumentException($"Invalid mutex name '{mutexName}'.");
+            }
+        }
+
+        return new FileInfo(Path.Combine(tentacleHome, $"ScriptIsolation.{mutexName}.lock"));
+    }
+
+    static LockType? MapScriptIsolationLevelToLockTypeOrNull(string isolationLevel) =>
+        isolationLevel.ToLowerInvariant() switch
+        {
+            "fullisolation" => LockType.Exclusive,
+            "noisolation" => LockType.Shared,
+            _ => null
+        };
+}

source/Calamari.Common/Features/Processes/ScriptIsolation/LockRejectedException.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Polly.Timeout;
+
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public sealed class LockRejectedException(string message, Exception? innerException = null)
+    : Exception(message, innerException)
+{
+    public LockRejectedException(Exception innerException) : this("Lock acquisition failed", innerException)
+    {
+    }
+
+    [DoesNotReturn]
+    public static void Throw(Exception innerException)
+    {
+        if (innerException is LockRejectedException lockRejectedException)
+        {
+            throw lockRejectedException;
+        }
+
+        if (innerException is TimeoutRejectedException timeoutRejectedException)
+        {
+            var message = $"Lock acquisition failed after {timeoutRejectedException.Timeout}";
+            throw new LockRejectedException(message, timeoutRejectedException);
+        }
+
+        throw new LockRejectedException("Lock acquisition failed", innerException);
+    }
+}

source/Calamari.Common/Features/Processes/ScriptIsolation/LockType.cs

@@ -0,0 +1,7 @@
+namespace Calamari.Common.Features.Processes.ScriptIsolation;
+
+public enum LockType
+{
+    Shared,
+    Exclusive
+}