Skip to content

fix: deprecate kubelet_flags in favour of kubelet_cmd_flags #7779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
awesomenix wants to merge 1 commit into from
Closed

fix: deprecate kubelet_flags in favour of kubelet_cmd_flags #7779

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 7 additions & 14 deletions aks-node-controller/helpers/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,29 +131,26 @@ func Test_getNetworkPolicyType(t *testing.T) {
}
}

func Test_getKubeletNodeLabels(t *testing.T) {
func Test_getKubeletCmdNodeLabels(t *testing.T) {
type args struct {
ap *datamodel.AgentPoolProfile
}
tests := []struct {
name string
args args
want map[string]string
want string
}{
{
name: "KubeletNodeLabels default labels",
name: "KubeletCmdNodeLabels default labels",
args: args{
ap: &datamodel.AgentPoolProfile{
Name: "agentPool0",
},
},
want: map[string]string{
"agentpool": "agentPool0",
"kubernetes.azure.com/agentpool": "agentPool0",
},
want: "agentpool=agentPool0,kubernetes.azure.com/agentpool=agentPool0",
},
{
name: "KubeletNodeLabels with CustomNodeLabels",
name: "KubeletCmdNodeLabels with CustomNodeLabels",
args: args{
ap: &datamodel.AgentPoolProfile{
Name: "agentPool0",
Expand All @@ -162,16 +159,12 @@ func Test_getKubeletNodeLabels(t *testing.T) {
},
},
},
want: map[string]string{
"agentpool": "agentPool0",
"kubernetes.azure.com/agentpool": "agentPool0",
"a": "b",
},
want: "agentpool=agentPool0,kubernetes.azure.com/agentpool=agentPool0,a=b",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := GetKubeletNodeLabels(tt.args.ap); !reflect.DeepEqual(got, tt.want) {
if got := tt.args.ap.GetKubernetesLabels(); got != tt.want {
t.Errorf("GetKubeletNodeLabels() = %v, want %v", got, tt.want)
}
})
Expand Down
12 changes: 12 additions & 0 deletions aks-node-controller/parser/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,9 +603,21 @@ func getKubeletFlags(kubeletConfig *aksnodeconfigv1.KubeletConfig) string {
if kubeletConfig.GetKubeletCmdFlags() != "" {
return kubeletConfig.GetKubeletCmdFlags()
}
//nolint:staticcheck // deprecated field required for compatibility
return createSortedKeyValuePairs(kubeletConfig.GetKubeletFlags(), " ")
}

// getKubeletNodeLabels returns the kubelet command-line node labels as a string.
// When kubelet_cmd_node_labels is present, it takes precedence and is returned as-is (raw string).
// Otherwise, it falls back to constructing flags from the kubelet_node_labels map (sorted key-value pairs).
func getKubeletNodeLabels(kubeletConfig *aksnodeconfigv1.KubeletConfig) string {
if kubeletConfig.GetKubeletCmdNodeLabels() != "" {
Copy link
Contributor

@cameronmeissner cameronmeissner Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so the bootstrapper now specifies node labels within a dedicated proto field? - is this PR mainly to separate out node labels from kubelet flags?

return kubeletConfig.GetKubeletCmdNodeLabels()
}
//nolint:staticcheck // deprecated field required for compatibility
return createSortedKeyValuePairs(kubeletConfig.GetKubeletNodeLabels(), ",")
}

func marshalToJSON(v any) ([]byte, error) {
// Originally we can set the Multiline here and it will marshal to a JSON we can use.
// However the protojson team intentionally randomly add extra whitespace after the key in the key-value.
Expand Down
2 changes: 1 addition & 1 deletion aks-node-controller/parser/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func getCSEEnv(config *aksnodeconfigv1.Configuration) map[string]string {
"NEEDS_CGROUPV2": fmt.Sprintf("%v", config.GetNeedsCgroupv2()),
"KUBELET_FLAGS": getKubeletFlags(config.GetKubeletConfig()),
"NETWORK_POLICY": getStringFromNetworkPolicyType(config.GetNetworkConfig().GetNetworkPolicy()),
"KUBELET_NODE_LABELS": createSortedKeyValuePairs(config.GetKubeletConfig().GetKubeletNodeLabels(), ","),
"KUBELET_NODE_LABELS": getKubeletNodeLabels(config.GetKubeletConfig()),
"AZURE_ENVIRONMENT_FILEPATH": getAzureEnvironmentFilepath(config),
"KUBE_CA_CRT": config.GetKubernetesCaCert(),
"KUBENET_TEMPLATE": getKubenetTemplate(),
Expand Down
2 changes: 1 addition & 1 deletion aks-node-controller/parser/parser_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ oom_score = -999
KubeletConfig: &aksnodeconfigv1.KubeletConfig{
EnableKubeletConfigFile: false,
KubeletCmdFlags: agent.GetOrderedKubeletConfigFlagString(config),
KubeletNodeLabels: helpers.GetKubeletNodeLabels(agentPool),
KubeletCmdNodeLabels: agentPool.GetKubernetesLabels(),
},
CustomCloudConfig: &aksnodeconfigv1.CustomCloudConfig{},
}
Expand Down
554 changes: 287 additions & 267 deletions aks-node-controller/pkg/gen/aksnodeconfig/v1/kubelet_config.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions aks-node-controller/proto/aksnodeconfig/v1/kubelet_config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ option go_package = "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aks

message KubeletConfig {
// A map of kubelet flags to their values. The map is sorted by key.
map<string, string> kubelet_flags = 1;
map<string, string> kubelet_flags = 1 [deprecated = true];

// A map of node labels to their values.
map<string, string> kubelet_node_labels = 2;
map<string, string> kubelet_node_labels = 2 [deprecated = true];

// The type of disk to use for the kubelet.
KubeletDisk kubelet_disk_type = 3;
Expand All @@ -31,6 +31,10 @@ message KubeletConfig {
// Kubelet cmd flags, provided as is, so we can deprecate kubelet flags
// +optional.
string kubelet_cmd_flags = 9;

// Kubelet cmd node labels, provided as is, so we can deprecate kubelet_node_labels
// +optional.
string kubelet_cmd_node_labels = 10;
}

enum KubeletDisk {
Expand Down
87 changes: 4 additions & 83 deletions e2e/node_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,88 +16,6 @@ import (
"github.com/stretchr/testify/require"
)

// this is a base kubelet config for Scriptless e2e test
func baseKubeletConfig() *aksnodeconfigv1.KubeletConfig {
return &aksnodeconfigv1.KubeletConfig{
EnableKubeletConfigFile: true,
KubeletFlags: map[string]string{
"--cloud-provider": "external",
"--kubeconfig": "/var/lib/kubelet/kubeconfig",
"--pod-infra-container-image": "mcr.microsoft.com/oss/v2/kubernetes/pause:3.6",
},
KubeletNodeLabels: map[string]string{
"agentpool": "nodepool2",
"kubernetes.azure.com/agentpool": "nodepool2",
"kubernetes.azure.com/cluster": "test-cluster",
"kubernetes.azure.com/mode": "system",
"kubernetes.azure.com/node-image-version": "AKSUbuntu-2404gen2containerd-2025.06.02",
},
KubeletConfigFileConfig: &aksnodeconfigv1.KubeletConfigFileConfig{
Kind: "KubeletConfiguration",
ApiVersion: "kubelet.config.k8s.io/v1beta1",
StaticPodPath: "/etc/kubernetes/manifests",
Address: "0.0.0.0",
TlsCertFile: "/etc/kubernetes/certs/kubeletserver.crt",
TlsPrivateKeyFile: "/etc/kubernetes/certs/kubeletserver.key",
TlsCipherSuites: []string{
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_128_GCM_SHA256",
},
RotateCertificates: true,
ServerTlsBootstrap: true,
Authentication: &aksnodeconfigv1.KubeletAuthentication{
X509: &aksnodeconfigv1.KubeletX509Authentication{
ClientCaFile: "/etc/kubernetes/certs/ca.crt",
},
Webhook: &aksnodeconfigv1.KubeletWebhookAuthentication{
Enabled: true,
},
},
Authorization: &aksnodeconfigv1.KubeletAuthorization{
Mode: "Webhook",
},
EventRecordQps: to.Ptr(int32(0)),
ClusterDomain: "cluster.local",
ClusterDns: []string{
"10.0.0.10",
},
StreamingConnectionIdleTimeout: "4h",
NodeStatusUpdateFrequency: "10s",
ImageGcHighThresholdPercent: to.Ptr(int32(85)),
ImageGcLowThresholdPercent: to.Ptr(int32(80)),
CgroupsPerQos: to.Ptr(true),
MaxPods: to.Ptr(int32(110)),
PodPidsLimit: to.Ptr(int32(-1)),
ResolvConf: "/run/systemd/resolve/resolv.conf",
EvictionHard: map[string]string{
"memory.available": "750Mi",
"nodefs.available": "10%",
"nodefs.inodesFree": "5%",
},
ProtectKernelDefaults: true,
FeatureGates: map[string]bool{},
FailSwapOn: to.Ptr(false),
KubeReserved: map[string]string{
"cpu": "100m",
"memory": "1638Mi",
},
EnforceNodeAllocatable: []string{
"pods",
},
AllowedUnsafeSysctls: []string{
"kernel.msg*",
"net.ipv4.route.min_pmtu",
},
},
}
}

func getBaseNBC(t testing.TB, cluster *Cluster, vhd *config.Image) (*datamodel.NodeBootstrappingConfiguration, error) {
var nbc *datamodel.NodeBootstrappingConfiguration

Expand Down Expand Up @@ -285,7 +203,10 @@ func nbcToAKSNodeConfigV1(nbc *datamodel.NodeBootstrappingConfiguration) *aksnod
// Before scriptless, absvc combined kubelet configs from multiple sources such as nbc.AgentPoolProfile.CustomKubeletConfig, nbc.KubeletConfig and more.
// Now in scriptless, we don't have absvc to process nbc and nbc is no longer a dependency.
// Therefore, we require client (e.g. AKS-RP) to provide the final kubelet config that is ready to be written to the final kubelet config file on a node.
KubeletConfig: baseKubeletConfig(),
KubeletConfig: &aksnodeconfigv1.KubeletConfig{
KubeletCmdFlags: agent.GetOrderedKubeletConfigFlagString(nbc),
KubeletCmdNodeLabels: nbc.AgentPoolProfile.GetKubernetesLabels(),
},
}
}

Expand Down
18 changes: 5 additions & 13 deletions e2e/scenario_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package e2e
import (
"context"
"fmt"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -757,8 +758,8 @@ func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations_Scriptless(t *testing
Config: Config{
Cluster: ClusterKubenet,
VHD: config.VHDUbuntu2204Gen2Containerd,
AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) {
config.KubeletConfig.KubeletFlags["--register-with-taints"] = "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule"
AKSNodeConfigMutator: func(aksConfig *aksnodeconfigv1.Configuration) {
aksConfig.KubeletConfig.KubeletCmdFlags = strings.Join([]string{aksConfig.KubeletConfig.KubeletCmdFlags, "--register-with-taints=testkey1=value1:NoSchedule,testkey2=value2:NoSchedule"}, " ")
},
Validator: func(ctx context.Context, s *Scenario) {
ValidateFileHasContent(ctx, s, "/etc/systemd/system/chronyd.service.d/10-chrony-restarts.conf", "Restart=always")
Expand Down Expand Up @@ -2178,15 +2179,10 @@ func Test_Ubuntu2204Gen2_ImagePullIdentityBinding_Enabled_Scriptless(t *testing.
DefaultTenantId: "test-tenant-id-67890",
LocalAuthoritySni: "test.sni.local",
}
// Set kubelet flags to enable credential provider
if aksConfig.KubeletConfig == nil {
aksConfig.KubeletConfig = &aksnodeconfigv1.KubeletConfig{}
}
if aksConfig.KubeletConfig.KubeletFlags == nil {
aksConfig.KubeletConfig.KubeletFlags = make(map[string]string)
}
aksConfig.KubeletConfig.KubeletFlags["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
aksConfig.KubeletConfig.KubeletFlags["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
aksConfig.KubeletConfig.KubeletCmdFlags = strings.Join([]string{aksConfig.KubeletConfig.KubeletCmdFlags, "--image-credential-provider-config=/var/lib/kubelet/credential-provider-config.yaml", "--image-credential-provider-bin-dir=/var/lib/kubelet/credential-provider"}, " ")
},
Validator: func(ctx context.Context, s *Scenario) {
// Verify aks-node-controller completed successfully
Expand Down Expand Up @@ -2227,11 +2223,7 @@ func Test_Ubuntu2204Gen2_ImagePullIdentityBinding_Disabled_Scriptless(t *testing
if aksConfig.KubeletConfig == nil {
aksConfig.KubeletConfig = &aksnodeconfigv1.KubeletConfig{}
}
if aksConfig.KubeletConfig.KubeletFlags == nil {
aksConfig.KubeletConfig.KubeletFlags = make(map[string]string)
}
aksConfig.KubeletConfig.KubeletFlags["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
aksConfig.KubeletConfig.KubeletFlags["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
aksConfig.KubeletConfig.KubeletCmdFlags = strings.Join([]string{aksConfig.KubeletConfig.KubeletCmdFlags, "--image-credential-provider-config=/var/lib/kubelet/credential-provider-config.yaml", "--image-credential-provider-bin-dir=/var/lib/kubelet/credential-provider"}, " ")
},
Validator: func(ctx context.Context, s *Scenario) {
// Verify aks-node-controller completed successfully
Expand Down
Loading