check bash exec & associative array commands are available and use printf instead of xxd

IsaacCalligeros95 · IsaacCalligeros95 · commit 700695b0a5d0 · 2025-06-02T09:13:10.000+09:30
diff --git a/source/Calamari.Common/Features/Scripting/Bash/Bootstrap.sh b/source/Calamari.Common/Features/Scripting/Bash/Bootstrap.sh
@@ -265,6 +265,24 @@ function log_environment_information
 
 log_environment_information
 
+function hex_to_bin_fd() {
+    local hex="$1"
+    local fd="$2"
+    
+    # Process hex in reasonable chunks
+    for ((i=0; i<${#hex}; i+=100)); do
+        local chunk="${hex:$i:100}"
+        local cmd="printf '"
+        
+        for ((j=0; j<${#chunk}; j+=2)); do
+            cmd+="\\x${chunk:$j:2}"
+        done
+        
+        cmd+="'"
+        eval "$cmd" >&$fd
+    done
+}
+
 function decrypt_and_parse_variables {
     local encrypted="$1"
     local iv="$2"
@@ -287,7 +305,9 @@ function decrypt_and_parse_variables {
     local concatenated_hex
     concatenated_hex=$(printf "%s" "${hex_parts[@]}")
 
-    exec 3< <(echo -n "$concatenated_hex" | xxd -r -p)
+    exec 3<> <(:)
+    hex_to_bin_fd "$concatenated_hex" 3
+    exec 3<&3
 
     local idx
     for idx in "${!key_byte_lengths[@]}"; do
@@ -374,14 +394,15 @@ function report_kubernetes_manifest_file
 bashParametersArrayFeatureToggle=#### BashParametersArrayFeatureToggle ####
 
 if [ "$bashParametersArrayFeatureToggle" = true ]; then
-    if (( ${BASH_VERSINFO[0]:-0} > 4 || (${BASH_VERSINFO[0]:-0} == 4 && ${BASH_VERSINFO[1]:-0} > 2) )); then
-        if command -v xxd > /dev/null; then
+    if declare -gA test_array 2>/dev/null; then
+        if exec 3<> <(:); then
+            exec 3<&-
             decrypt_and_parse_variables "#### VARIABLESTRING.ENCRYPTED ####" "#### VARIABLESTRING.IV ####"
         else
-            echo "xxd is not installed, this is required to use octopus_parameters"
+            echo "Process substitution with exec is not supported. This is required to use octopus_parameters"
         fi
     else
-        echo "Bash version 4.2 or later is required to use octopus_parameters"
+        echo "Associative arrays are not supported in this Bash version. This is required to use octopus_parameters"
     fi
 fi
 
diff --git a/source/Calamari.Tests/Fixtures/Bash/BashFixture.cs b/source/Calamari.Tests/Fixtures/Bash/BashFixture.cs
@@ -2,12 +2,15 @@
 using System.Collections.Generic;
 using System.IO;
 using Calamari.Common.Features.Processes;
+using System.Diagnostics;
+using System.Linq;
 using Calamari.Common.FeatureToggles;
 using Calamari.Common.Plumbing;
 using Calamari.Common.Plumbing.Variables;
 using Calamari.Deployment;
 using Calamari.Testing.Requirements;
 using Calamari.Tests.Helpers;
+using FluentAssertions;
 using NUnit.Framework;
 
 namespace Calamari.Tests.Fixtures.Bash
@@ -34,7 +37,7 @@ public void ShouldPrintEncodedVariable(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldPrintSensitiveVariable(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("print-sensitive-variable.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("print-sensitive-variable.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -48,7 +51,7 @@ public void ShouldPrintSensitiveVariable(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldCreateArtifact(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("create-artifact.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("create-artifact.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -62,21 +65,21 @@ public void ShouldCreateArtifact(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldUpdateProgress(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("update-progress.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("update-progress.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
                                 output.AssertSuccess();
                                 output.AssertOutput("##octopus[progress percentage='NTA=' message='SGFsZiBXYXk=']");
                             });
         }
-        
+
         [TestCase(FeatureToggle.BashParametersArrayFeatureToggle)]
         [TestCase(null)]
         [RequiresBashDotExeIfOnWindows]
         public void ShouldReportKubernetesManifest(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("report-kubernetes-manifest.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("report-kubernetes-manifest.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -87,7 +90,7 @@ public void ShouldReportKubernetesManifest(FeatureToggle? featureToggle)
                                 output.AssertOutput("##octopus[k8s-manifest-applied manifest='ImFwaVZlcnNpb24iOiAidjEiXG4ia2luZCI6ICJOYW1lc3BhY2UiXG4ibWV0YWRhdGEiOlxuICAibmFtZSI6ICJkaWZmcyJcbiJsYWJlbHMiOlxuICAgICJuYW1lIjogImRpZmZzIlxu' ns='bXk=']");
                             });
         }
-        
+
         [TestCase(FeatureToggle.BashParametersArrayFeatureToggle)]
         [TestCase(null)]
         [RequiresBashDotExeIfOnWindows]
@@ -107,20 +110,23 @@ public void ShouldReportKubernetesManifestFile(FeatureToggle? featureToggle)
   ""name"": ""diffs""
 ""labels"":
     ""name"": ""diffs""".ReplaceLineEndings("\n");
-            
+
             var filePath = Path.Combine(tempPath, "ShouldWriteServiceMessageForKubernetesManifestFile.manifest.yaml");
             File.WriteAllText(filePath, manifest);
 
             //if we are running on windows, we must be running via bash.exe, so we need to translate this to a wsl path
             var updatedFilePath = filePath;
             if (CalamariEnvironment.IsRunningOnWindows)
             {
-                var qualifiedPath = filePath.Replace(@"\",@"\\");
+                var qualifiedPath = filePath.Replace(@"\", @"\\");
 
                 var path = string.Empty;
-                var result = SilentProcessRunner.ExecuteCommand("wsl", $"wslpath -a -u {qualifiedPath}", tempPath, output => path = output,
+                var result = SilentProcessRunner.ExecuteCommand("wsl",
+                                                                $"wslpath -a -u {qualifiedPath}",
+                                                                tempPath,
+                                                                output => path = output,
                                                                 _ => { });
-                
+
                 if (result.ExitCode != 0)
                 {
                     Assert.Fail("Failed to convert windows path to WSL path");
@@ -135,7 +141,6 @@ public void ShouldReportKubernetesManifestFile(FeatureToggle? featureToggle)
 
             try
             {
-
                 var (output, _) = RunScript("report-kubernetes-manifest-file.sh", additionalVariables);
 
                 Assert.Multiple(() =>
@@ -160,7 +165,7 @@ public void ShouldConsumeParametersWithQuotes(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("parameters.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.Script.ScriptParameters] = "\"Para meter0\" 'Para meter1'" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.Script.ScriptParameters] = "\"Para meter0\" 'Para meter1'" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -174,8 +179,10 @@ public void ShouldConsumeParametersWithQuotes(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldNotReceiveParametersIfNoneProvided(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("parameters.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+            var (output, _) = RunScript("parameters.sh",
+                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -197,7 +204,7 @@ public void ShouldCallHello(FeatureToggle? featureToggle)
                                             ["Variable3"] = "GHI",
                                             ["Foo_bar"] = "Hello",
                                             ["Host"] = "Never",
-                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -213,8 +220,9 @@ public void ShouldCallHelloWithSensitiveVariable(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = "NameToEncrypt" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+                                            { ["Name"] = "NameToEncrypt" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -230,7 +238,7 @@ public void ShouldCallHelloWithNullVariable(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -246,8 +254,9 @@ public void ShouldCallHelloWithNullSensitiveVariable(FeatureToggle? featureToggl
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -262,7 +271,7 @@ public void ShouldCallHelloWithNullSensitiveVariable(FeatureToggle? featureToggl
         public void ShouldNotFailOnStdErr(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("stderr.sh",
-                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -278,7 +287,7 @@ public void ShouldFailOnStdErrWithTreatScriptWarningsAsErrors(FeatureToggle? fea
         {
             var (output, _) = RunScript("stderr.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -294,7 +303,7 @@ public void ShouldNotFailOnStdErrFromServiceMessagesWithTreatScriptWarningsAsErr
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             output.AssertSuccess();
         }
@@ -304,7 +313,7 @@ public void ShouldNotFailOnStdErrFromServiceMessagesWithTreatScriptWarningsAsErr
         [TestCase(null)]
         public void ShouldSupportStrictVariableUnset(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("strict-mode.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("strict-mode.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -336,8 +345,9 @@ public void ShouldBeAbleToEnumerateVariableValues(FeatureToggle? featureToggle)
                                             ["VariableName \n 11"] = "Value \n 11",
                                             ["VariableName.prop.anotherprop 12"] = "Value.prop.12",
                                             ["VariableName`prop`anotherprop` 13"] = "Value`prop`13",
+                                            ["VariableName 14 😭🙈👀"] = "Value 14 😭🙈👀",
                                             [specialCharacters] = specialCharacters
-                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             output.AssertSuccess();
             if (featureToggle == FeatureToggle.BashParametersArrayFeatureToggle)
@@ -366,9 +376,75 @@ public void ShouldBeAbleToEnumerateVariableValues(FeatureToggle? featureToggle)
 
                 output.AssertOutput("Key: VariableName.prop.anotherprop 12, Value: Value.prop.12");
                 output.AssertOutput("Key: VariableName`prop`anotherprop` 13, Value: Value`prop`13");
+                output.AssertOutput("Key: VariableName 14 😭🙈👀, Value: Value 14 😭🙈👀");
                 output.AssertOutput($"Key: {specialCharacters}, Value: {specialCharacters}");
             }
         }
+
+        [TestCase(FeatureToggle.BashParametersArrayFeatureToggle)]
+        [TestCase(null)]
+        [RequiresBashDotExeIfOnWindows]
+        public void ShouldBeAbleToEnumerateLargeVariableSetsEfficiently(FeatureToggle? featureToggle)
+        {
+            // Create a dictionary with 10,000 variables with diverse characters
+            var variables = new Dictionary<string, string>();
+            var random = new Random(42); // Seed for reproducibility
+
+            // Generate 10,000 unique variables with diverse content
+            for (int i = 0; i < 10000; i++)
+            {
+                string key = $"Key{i}_{Guid.NewGuid().ToString("N")}";
+                string value = $"Value{i}_{Convert.ToBase64String(Guid.NewGuid().ToByteArray())}";
+
+                // Mix in some random Unicode characters
+                if (random.Next(5) == 0)
+                {
+                    key += (char)random.Next(0x1F600, 0x1F64F); // Emoji range
+                    value += Environment.NewLine + (char)random.Next(0x2600, 0x26FF); // Unicode symbols
+                }
+
+                variables[key] = value;
+            }
+
+            var sw = Stopwatch.StartNew();
+            // Run the script with all these variables
+            var (output, _) = RunScript("enumerate-variables.sh",
+                                        variables.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
+            sw.Stop();
+            // This depends on the running machine, locally ~1000ms, in CI sometimes this is ~2000ms. We're being very conservative
+            // but if there's a scenario where this test fails this should be increased. This test exists because there are 
+            // potential performance problems in encoding/decoding of variables in bash, this is a sanity check.
+            sw.Elapsed.TotalMilliseconds.Should().BeLessThan(4000);
+
+            output.AssertSuccess();
+            if (featureToggle == FeatureToggle.BashParametersArrayFeatureToggle)
+            {
+                var fullOutput = string.Join(Environment.NewLine, output.CapturedOutput.Infos);
+                if (fullOutput.Contains("Bash version 4.2 or later is required to use octopus_parameters"))
+                {
+                    output.AssertOutput("Still ran this script");
+                    return;
+                }
+
+                // Get all output lines that start with "Key: "
+                var outputLines = output.CapturedOutput.Infos
+                                        .Where(line => line.StartsWith("Key: "))
+                                        .ToList();
+
+                // Verify count matches
+                Assert.That(outputLines.Count,
+                            Is.EqualTo(variables.Count),
+                            "Not all variables were processed");
+
+                // For each variable, construct the expected output format and verify it exists
+                foreach (var kvp in variables)
+                {
+                    string expectedOutput = $"Key: {kvp.Key}, Value: {kvp.Value}";
+                    Assert.That(outputLines.Contains(expectedOutput),
+                                $"Expected output line not found: '{expectedOutput}'");
+                }
+            }
+        }
     }
 
     public static class AdditionalVariablesExtensions
@@ -379,4 +455,4 @@ public static Dictionary<string, string> AddFeatureToggleToDictionary(this Dicti
             return variables;
         }
     }
-}
+}
