feat: node-exporter into vhd build

e2e/validators.go

@@ -1168,6 +1168,50 @@ func ValidateNodeProblemDetector(ctx context.Context, s *Scenario) {
 	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Node Problem Detector (NPD) service validation failed")
 }
 
+func ValidateNodeExporter(ctx context.Context, s *Scenario) {
+	s.T.Helper()
+
+	skipFile := "/etc/node-exporter.d/skip_vhd_node_exporter"
+	serviceName := "node-exporter.service"
+
+	// Check if node-exporter is installed on this VHD by looking for the skip sentinel file.
+	// The skip file is only present on VHDs that have node-exporter installed (Ubuntu, Mariner, Azure Linux).
+	// Flatcar, OSGuard, and older VHDs do not have node-exporter installed and will not have the skip file.
+	if !fileExist(ctx, s, skipFile) {
+		s.T.Logf("Skipping node-exporter validation: sentinel file %s not found (VHD does not have node-exporter installed)", skipFile)
+		return
+	}
+
+	s.T.Logf("skip_vhd_node_exporter sentinel file found, validating node-exporter installation")
+
+	// Validate service is running
+	ValidateSystemdUnitIsRunning(ctx, s, serviceName)
+	ValidateSystemdUnitIsNotFailed(ctx, s, serviceName)
+
+	// Validate service is enabled
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-enabled %s", serviceName), 0, fmt.Sprintf("%s should be enabled", serviceName))
+
+	// Validate binary exists and is executable
+	// The binary is installed at /usr/bin and symlinked to /opt/bin for consistency with other binaries (kubelet, etc.)
+	ValidateFileExists(ctx, s, "/usr/bin/node-exporter")
+	ValidateFileExists(ctx, s, "/opt/bin/node-exporter")
+	ValidateFileExists(ctx, s, "/opt/bin/node-exporter-startup.sh")
+
+	// Validate configuration files exist
+	ValidateFileExists(ctx, s, skipFile)
+	ValidateFileExists(ctx, s, "/etc/node-exporter.d/web-config.yml")
+
+	// Validate that node-exporter is listening on port 9100 (metrics endpoint)
+	s.T.Logf("Validating node-exporter metrics endpoint is accessible")
+	command := []string{
+		"set -ex",
+		"curl -sf http://localhost:9100/metrics > /dev/null",
+	}
+	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "node-exporter metrics endpoint should be accessible")
+
+	s.T.Logf("node-exporter validation passed")
+}
+
 func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) {
 	command := []string{
 		"set -ex",

parts/common/components.json

@@ -1764,6 +1764,42 @@
           }
         }
       }
+    },
+    {
+      "name": "node-exporter",
+      "downloadLocation": "/opt/node-exporter",
+      "downloadURIs": {
+        "ubuntu": {
+          "current": {
+            "versionsV2": [
+              {
+                "renovateTag": "name=node-exporter-kubernetes, repository=production, os=ubuntu, release=18.04",
+                "latestVersion": "1.9.1-ubuntu18.04u5"
+              }
+            ]
+          }
+        },
+        "mariner": {
+          "current": {
+            "versionsV2": [
+              {
+                "renovateTag": "RPM_registry=https://packages.microsoft.com/cbl-mariner/2.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=mariner, release=2.0",
+                "latestVersion": "1.9.1-8.cm2"
+              }
+            ]
+          }
+        },
+        "azurelinux": {
+          "v3.0": {
+            "versionsV2": [
+              {
+                "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-exporter-kubernetes, os=azurelinux, release=3.0",
+                "latestVersion": "1.9.1-8.azl3"
+              }
+            ]
+          }
+        }
+      }
     }
   ],
   "OCIArtifacts": [

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -821,6 +821,25 @@ EOF
     systemctlEnableAndStart mig-partition 300
 }
 
+configureNodeExporter() {
+    echo "Configuring Node Exporter"
+    # Check for skip file to determine if node-exporter was installed on this VHD
+    if [ ! -f /etc/node-exporter.d/skip_vhd_node_exporter ]; then
+        echo "Node Exporter assets not found on this VHD (missing /etc/node-exporter.d/skip_vhd_node_exporter); skipping configuration."
+        return 0
+    fi
+
+    if ! systemctlEnableAndStart node-exporter 30; then
+        echo "Failed to start node-exporter service"
+        return $ERR_NODE_EXPORTER_START_FAIL
+    fi
+    if ! systemctlEnableAndStart node-exporter-restart.path 30; then
+        echo "Failed to start node-exporter-restart.path"
+        return $ERR_NODE_EXPORTER_START_FAIL
+    fi
+    echo "Node Exporter started successfully"
+}
+
 ensureSysctl() {
     SYSCTL_CONFIG_FILE=/etc/sysctl.d/999-sysctl-aks.conf
     mkdir -p "$(dirname "${SYSCTL_CONFIG_FILE}")"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -76,6 +76,8 @@ ERR_ENABLE_MANAGED_GPU_EXPERIENCE=123 # Error confguring managed GPU experience
 # Error code 124 is returned when a `timeout` command times out, and --preserve-status is not specified: https://man7.org/linux/man-pages/man1/timeout.1.html
 ERR_VHD_BUILD_ERROR=125 # Reserved for VHD CI exit conditions
 
+ERR_NODE_EXPORTER_START_FAIL=128 # Error starting or enabling node-exporter service
+
 ERR_SWAP_CREATE_FAIL=130 # Error allocating swap file
 ERR_SWAP_CREATE_INSUFFICIENT_DISK_SPACE=131 # Error insufficient disk space for swap file creation
 
@@ -938,10 +940,10 @@ fallbackToKubeBinaryInstall() {
         if [ "${SHOULD_ENFORCE_KUBE_PMC_INSTALL}" = "true" ]; then
             echo "Kube PMC install is enforced, skipping fallback to kube binary install for ${packageName}"
             return 1
-        elif [ -f "/opt/bin/${packageName}-${packageVersion}" ]; then
-            mv "/opt/bin/${packageName}-${packageVersion}" "/opt/bin/${packageName}"
-            chmod a+x /opt/bin/${packageName}
-            rm -rf /opt/bin/${packageName}-* &
+        elif [ -f "/usr/local/bin/${packageName}-${packageVersion}" ]; then
+            mv "/usr/local/bin/${packageName}-${packageVersion}" "/usr/local/bin/${packageName}"
+            chmod a+x /usr/local/bin/${packageName}
+            rm -rf /usr/local/bin/${packageName}-* &
             return 0
         else
             echo "No binary fallback found for ${packageName} version ${packageVersion}"

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -455,6 +455,8 @@ function nodePrep {
 
     logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet
 
+    logs_to_events "AKS.CSE.configureNodeExporter" configureNodeExporter
+
     if $REBOOTREQUIRED; then
         echo 'reboot required, rebooting node in 1 minute'
         /bin/bash -c "shutdown -r 1 &"

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/node-exporter.d/web-config.yml

@@ -0,0 +1,5 @@
+tls_server_config:
+  cert_file: "/etc/kubernetes/certs/kubeletserver.crt"
+  key_file: "/etc/kubernetes/certs/kubeletserver.key"
+  client_auth_type: "RequireAndVerifyClientCert"
+  client_ca_file: "/etc/kubernetes/certs/ca.crt"

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/systemd/system/node-exporter-restart.path

@@ -0,0 +1,9 @@
+[Path]
+# Watch server cert paths - one will exist depending on whether kubelet serving cert rotation is enabled
+# Rotation enabled: kubelet-server-current.pem (symlink updated on rotation)
+# Rotation disabled: kubeletserver.crt (static cert)
+PathModified=/var/lib/kubelet/pki/kubelet-server-current.pem
+PathModified=/etc/kubernetes/certs/kubeletserver.crt
+
+[Install]
+WantedBy=multi-user.target

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/systemd/system/node-exporter-restart.service

@@ -0,0 +1,3 @@
+[Service]
+Type=OneShot
+ExecStart=/bin/systemctl restart node-exporter.service

parts/linux/cloud-init/artifacts/node-exporter/baseline/etc/systemd/system/node-exporter.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Prometheus Node Exporter
+Documentation=https://github.com/prometheus/node_exporter
+
+[Service]
+ExecStart=/opt/bin/node-exporter-startup.sh
+
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

parts/linux/cloud-init/artifacts/node-exporter/node-exporter-startup.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+if [ "$(cat /etc/os-release | grep ^ID= | cut -c 4-)" = "flatcar" ]; then
+    NODE_IP=$(ip -o -4 addr show dev eth0 | awk '{print $4}' | cut -d '/' -f 1)
+else
+    NODE_IP=$(hostname -I | awk '{print $1}')
+fi
+
+TLS_CONFIG_PATH="/etc/node-exporter.d/web-config.yml"
+TLS_CONFIG_ARG=""
+KUBELET_DEFAULTS="/etc/default/kubelet"
+
+# Detect TLS cert paths from kubelet configuration
+# Priority: rotation cert > static cert paths from kubelet flags > skip TLS
+CERT_FILE=""
+KEY_FILE=""
+
+# Check for rotation cert first (used when --rotate-server-certificates=true)
+if [ -f "/var/lib/kubelet/pki/kubelet-server-current.pem" ]; then
+    CERT_FILE="/var/lib/kubelet/pki/kubelet-server-current.pem"
+    KEY_FILE="/var/lib/kubelet/pki/kubelet-server-current.pem"
+elif [ -f "$KUBELET_DEFAULTS" ]; then
+    # Parse kubelet flags for static cert paths
+    KUBELET_FLAGS=$(grep "^KUBELET_FLAGS=" "$KUBELET_DEFAULTS" | cut -d'=' -f2-)
+    TLS_CERT=$(echo "$KUBELET_FLAGS" | grep -o '\--tls-cert-file=[^ ]*' | cut -d'=' -f2)
+    TLS_KEY=$(echo "$KUBELET_FLAGS" | grep -o '\--tls-private-key-file=[^ ]*' | cut -d'=' -f2)
+
+    if [ -n "$TLS_CERT" ] && [ -n "$TLS_KEY" ] && [ -f "$TLS_CERT" ] && [ -f "$TLS_KEY" ]; then
+        CERT_FILE="$TLS_CERT"
+        KEY_FILE="$TLS_KEY"
+    fi
+fi
+
+# Configure TLS if we found valid cert paths
+if [ -n "$CERT_FILE" ] && [ -n "$KEY_FILE" ]; then
+    cat > "$TLS_CONFIG_PATH" <<EOF
+tls_server_config:
+  cert_file: "$CERT_FILE"
+  key_file: "$KEY_FILE"
+  client_auth_type: "RequireAndVerifyClientCert"
+  client_ca_file: "/etc/kubernetes/certs/ca.crt"
+EOF
+    TLS_CONFIG_ARG="--web.config.file=${TLS_CONFIG_PATH}"
+fi
+
+exec /opt/bin/node-exporter \
+    --web.listen-address=${NODE_IP}:19100 \
+    ${TLS_CONFIG_ARG} \
+    --no-collector.wifi \
+    --no-collector.hwmon \
+    --collector.cpu.info \
+    --collector.filesystem.mount-points-exclude="^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)" \
+    --collector.netclass.ignored-devices="^(azv.*|veth.*|[a-f0-9]{15})$" \
+    --collector.netclass.netlink \
+    --collector.netdev.device-exclude="^(azv.*|veth.*|[a-f0-9]{15})$" \
+    --no-collector.arp.netlink