diff --git a/aks-node-controller/helpers/const.go b/aks-node-controller/helpers/const.go index e9d49da5ab8..e51261f25d3 100644 --- a/aks-node-controller/helpers/const.go +++ b/aks-node-controller/helpers/const.go @@ -8,7 +8,7 @@ const ( NetworkPolicyAzure = "azure" NetworkPolicyCalico = "calico" LoadBalancerBasic = "basic" - LoadBalancerStandard = "Standard" + LoadBalancerStandard = "standard" VMSizeStandardDc2s = "Standard_DC2s" VMSizeStandardDc4s = "Standard_DC4s" DefaultLinuxUser = "azureuser" diff --git a/aks-node-controller/helpers/utils_test.go b/aks-node-controller/helpers/utils_test.go index 2cd80e02d80..83ac4fd74b8 100644 --- a/aks-node-controller/helpers/utils_test.go +++ b/aks-node-controller/helpers/utils_test.go @@ -23,14 +23,14 @@ func Test_getLoadBalancerSKU(t *testing.T) { { name: "LoadBalancerSKU Standard", args: args{ - sku: "Standard", + sku: "standard", }, want: aksnodeconfigv1.LoadBalancerSku_LOAD_BALANCER_SKU_STANDARD, }, { name: "LoadBalancerSKU Basic", args: args{ - sku: "Basic", + sku: "basic", }, want: aksnodeconfigv1.LoadBalancerSku_LOAD_BALANCER_SKU_BASIC, }, @@ -299,3 +299,110 @@ func TestIsKubeletServingCertificateRotationEnabled(t *testing.T) { }) } } + +func TestValidateAndSetLinuxKubeletFlags_RemovesDeprecatedFlags(t *testing.T) { + kubeletFlags := map[string]string{ + "--dynamic-config-dir": "/var/lib/kubelet", + "--non-masquerade-cidr": "10.240.0.0/12", + "--cni-bin-dir": "/opt/cni/bin", + "--cni-cache-dir": "/var/lib/cni", + "--cni-conf-dir": "/etc/cni/net.d", + "--docker-endpoint": "npipe:////./pipe/docker_engine", + "--image-pull-progress-deadline": "30m", + "--network-plugin": "cni", + "--network-plugin-mtu": "1500", + "--feature-gates": "", + } + + ValidateAndSetLinuxKubeletFlags(kubeletFlags, newTestContainerService("1.27.3"), &datamodel.AgentPoolProfile{}) + + removedFlags := []string{ + "--dynamic-config-dir", + "--non-masquerade-cidr", + "--cni-bin-dir", + "--cni-cache-dir", + "--cni-conf-dir", + "--docker-endpoint", + "--image-pull-progress-deadline", + "--network-plugin", + "--network-plugin-mtu", + } + + for _, flag := range removedFlags { + if _, exists := kubeletFlags[flag]; exists { + t.Fatalf("expected flag %s to be removed", flag) + } + } +} + +func TestValidateAndSetLinuxKubeletFlags_FeatureGatesByVersion(t *testing.T) { + testCases := []struct { + name string + version string + initialFeatureGates string + rotateServerCerts bool + expectedFeatureGates map[string]bool + }{ + { + name: "removes dynamic gate when version >= 1.24", + version: "1.26.0", + initialFeatureGates: "DynamicKubeletConfig=false,OtherFeature=true", + expectedFeatureGates: map[string]bool{ + "OtherFeature": true, + }, + }, + { + name: "adds dynamic and disable accelerator gates for 1.22", + version: "1.22.6", + initialFeatureGates: "FooBar=true", + expectedFeatureGates: map[string]bool{ + "FooBar": true, + "DynamicKubeletConfig": false, + "DisableAcceleratorUsageMetrics": false, + }, + }, + { + name: "does not add dynamic gate before 1.11", + version: "1.10.13", + initialFeatureGates: "", + expectedFeatureGates: map[string]bool{}, + }, + { + name: "adds rotate kubelet server certificate gate when enabled", + version: "1.28.2", + initialFeatureGates: "", + rotateServerCerts: true, + expectedFeatureGates: map[string]bool{ + "RotateKubeletServerCertificate": true, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kubeletFlags := map[string]string{ + "--feature-gates": tc.initialFeatureGates, + } + if tc.rotateServerCerts { + kubeletFlags["--rotate-server-certificates"] = "true" + } + + ValidateAndSetLinuxKubeletFlags(kubeletFlags, newTestContainerService(tc.version), &datamodel.AgentPoolProfile{}) + + featureGateMap := strKeyValToMapBool(kubeletFlags["--feature-gates"], ",", "=") + if !reflect.DeepEqual(featureGateMap, tc.expectedFeatureGates) { + t.Fatalf("unexpected feature gates: got %v, want %v", featureGateMap, tc.expectedFeatureGates) + } + }) + } +} + +func newTestContainerService(version string) *datamodel.ContainerService { + return &datamodel.ContainerService{ + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: version, + }, + }, + } +} diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index a2b53403d34..afc7704f5c5 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -596,13 +596,6 @@ func getDisableSSH(v *aksnodeconfigv1.Configuration) bool { return !v.GetEnableSsh() } -func getServicePrincipalFileContent(authConfig *aksnodeconfigv1.AuthConfig) string { - if authConfig.GetServicePrincipalSecret() == "" { - return "" - } - return base64.StdEncoding.EncodeToString([]byte(authConfig.GetServicePrincipalSecret())) -} - func getKubeletFlags(kubeletConfig *aksnodeconfigv1.KubeletConfig) string { return createSortedKeyValuePairs(kubeletConfig.GetKubeletFlags(), " ") } diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 0cc1e66c329..e2e874d8519 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -136,7 +136,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string { "DHCPV6_CONFIG_FILEPATH": getDHCPV6ConfigFilepath(), "THP_ENABLED": config.GetCustomLinuxOsConfig().GetTransparentHugepageSupport(), "THP_DEFRAG": config.GetCustomLinuxOsConfig().GetTransparentDefrag(), - "SERVICE_PRINCIPAL_FILE_CONTENT": getServicePrincipalFileContent(config.AuthConfig), + "SERVICE_PRINCIPAL_FILE_CONTENT": config.GetAuthConfig().GetServicePrincipalSecret(), "KUBELET_CLIENT_CONTENT": config.GetKubeletConfig().GetKubeletClientKey(), "KUBELET_CLIENT_CERT_CONTENT": config.GetKubeletConfig().GetKubeletClientCertContent(), "KUBELET_CONFIG_FILE_ENABLED": fmt.Sprintf("%v", config.GetKubeletConfig().GetEnableKubeletConfigFile()), diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index 0eeab692e7b..36d641cac97 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -334,6 +334,20 @@ oom_score = -999 } } +func TestBuildCSECmd_SetsServicePrincipalFileContent(t *testing.T) { + secret := "super-secret-value" + cmd, err := BuildCSECmd(context.TODO(), &aksnodeconfigv1.Configuration{ + AuthConfig: &aksnodeconfigv1.AuthConfig{ServicePrincipalSecret: secret}, + }) + require.NoError(t, err) + + vars := environToMap(cmd.Env) + + // The value should be exactly the secret, without additional base64 encoding. + // Actually the client which passes the secret to aks-node-controller should have base64 encoded it first. + assert.Equal(t, secret, vars["SERVICE_PRINCIPAL_FILE_CONTENT"]) +} + func TestAKSNodeConfigCompatibilityFromJsonToCSECommand(t *testing.T) { tests := []struct { name string diff --git a/e2e/node_config.go b/e2e/node_config.go index c9b50ade3d0..e70e95e19e5 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -532,7 +532,7 @@ func baseTemplateLinux(t testing.TB, location string, k8sVersion string, arch st }, ServicePrincipalProfile: &datamodel.ServicePrincipalProfile{ ClientID: "msi", - Secret: "**msi**", + Secret: base64.StdEncoding.EncodeToString([]byte("msi")), }, CertificateProfile: &datamodel.CertificateProfile{}, HostedMasterProfile: &datamodel.HostedMasterProfile{}, @@ -816,7 +816,7 @@ func baseTemplateWindows(t testing.TB, location string) *datamodel.NodeBootstrap }, ServicePrincipalProfile: &datamodel.ServicePrincipalProfile{ ClientID: "msi", - Secret: "**msi**", + Secret: base64.StdEncoding.EncodeToString([]byte("msi")), }, FeatureFlags: &datamodel.FeatureFlags{ EnableWinDSR: true, diff --git a/e2e/validators.go b/e2e/validators.go index dab642d2d05..cf99e0c2178 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -199,7 +199,7 @@ func ValidateLeakedSecrets(ctx context.Context, s *Scenario) { for _, logFile := range []string{"/var/log/azure/cluster-provision.log", "/var/log/azure/aks-node-controller.log"} { for _, secretValue := range secrets { if secretValue != "" { - ValidateFileExcludesContent(ctx, s, logFile, secretValue) + ValidateFileExcludesExactContent(ctx, s, logFile, secretValue) } } } @@ -384,6 +384,38 @@ func fileHasContent(ctx context.Context, s *Scenario, fileName string, contents } } +func fileHasExactContent(ctx context.Context, s *Scenario, fileName string, contents string) bool { + s.T.Helper() + require.NotEmpty(s.T, contents, "Test setup failure: Can't validate that a file has contents with an empty string. Filename: %s", fileName) + encodedPattern := base64.StdEncoding.EncodeToString([]byte(contents)) + if s.IsWindows() { + steps := []string{ + "$ErrorActionPreference = \"Stop\"", + fmt.Sprintf("if ( -not ( Test-Path -Path %s ) ) { exit 2 }", fileName), + fmt.Sprintf("$pattern = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('%s'))", encodedPattern), + fmt.Sprintf("$content = Get-Content -Path %s -Raw", fileName), + "$escaped = [regex]::Escape($pattern)", + "if ([regex]::Match($content, \"(?