F 145/update oneke from 1.29 to 1.31

appliances/OneKE/cilium.rb

@@ -7,6 +7,14 @@
 require_relative 'config.rb'
 require_relative 'helpers.rb'
 
+# NOTE: We added the ONEAPP_K8S_CILIUM_BGP_ENABLED flag for being able to disable
+# BGP control plane and CiliumLoadBalancerIPPool CRD creation in order to avoid
+# conflicts with other LB controllers managing services without LBClass set.
+# From cilium v1.17, we will be able to add `defaultLBServiceIPAM: none` in the
+# HelmChartConfig for letting cilium ignoring services without LBClass set.
+# More info: https://github.com/cilium/cilium/pull/33351
+# and https://docs.cilium.io/en/latest/network/lb-ipam/#loadbalancerclass
+
 def configure_cilium(manifest_dir = K8S_MANIFEST_DIR, endpoint = ONEAPP_K8S_CONTROL_PLANE_EP)
     msg :info, 'Configure Cilium'
 
@@ -24,32 +32,38 @@ def configure_cilium(manifest_dir = K8S_MANIFEST_DIR, endpoint = ONEAPP_K8S_CONT
           namespace: kube-system
         spec:
           valuesContent: |-
-            kubeProxyReplacement: strict
+            kubeProxyReplacement: true
             k8sServiceHost: "#{ep.host}"
             k8sServicePort: #{ep.port}
             cni:
               chainingMode: "none"
               exclusive: false
             bgpControlPlane:
-              enabled: true
-        ---
-        apiVersion: cilium.io/v2alpha1
-        kind: CiliumLoadBalancerIPPool
-        metadata:
-          name: default
-          namespace: kube-system
-        spec:
-          cidrs: {}
+              enabled: #{ONEAPP_K8S_CILIUM_BGP_ENABLED}
         MANIFEST
 
-        unless ONEAPP_K8S_CILIUM_RANGES.empty?
-            ip_address_pool = documents.find do |doc|
-                doc['kind'] == 'CiliumLoadBalancerIPPool' && doc.dig('metadata', 'name') == 'default'
-            end
-            ip_address_pool['spec']['cidrs'] = extract_cilium_ranges.map do |item|
-                { 'cidr' => item.join('/') }
+        if ONEAPP_K8S_CILIUM_BGP_ENABLED
+            documents += YAML.load_stream <<~MANIFEST
+            ---
+            apiVersion: cilium.io/v2alpha1
+            kind: CiliumLoadBalancerIPPool
+            metadata:
+                name: default
+                namespace: kube-system
+            spec:
+                blocks: {}
+                allowFirstLastIPs: "No"
+            MANIFEST
+            unless ONEAPP_K8S_CILIUM_RANGES.empty?
+                ip_address_pool = documents.find do |doc|
+                    doc['kind'] == 'CiliumLoadBalancerIPPool' && doc.dig('metadata', 'name') == 'default'
+                end
+                ip_address_pool['spec']['blocks'] = extract_cilium_ranges.map do |item|
+                    { 'cidr' => item.join('/') }
+                end
             end
         end
+
     else
         msg :info, 'Use Cilium user-provided config'
         documents = YAML.load_stream Base64.decode64 ONEAPP_K8S_CNI_CONFIG

appliances/OneKE/cilium_spec.rb

@@ -35,11 +35,10 @@
 end
 
 RSpec.describe 'configure_cilium' do
-    it 'should apply user-defined ranges (empty)' do
+    it 'should apply user-defined ranges' do
         stub_const 'ONEAPP_K8S_CONTROL_PLANE_EP', '192.168.150.86:6443'
         stub_const 'ONEAPP_K8S_CNI_PLUGIN', 'cilium'
-        stub_const 'ONEAPP_K8S_CNI_CONFIG', nil
-        stub_const 'ONEAPP_K8S_CILIUM_RANGES', []
+        stub_const 'ONEAPP_K8S_CILIUM_RANGES', ['192.168.150.128/25', '10.11.12.0/24']
         output = YAML.load_stream <<~MANIFEST
         ---
         apiVersion: helm.cattle.io/v1
@@ -49,7 +48,37 @@
           namespace: kube-system
         spec:
           valuesContent: |-
-            kubeProxyReplacement: strict
+            kubeProxyReplacement: true
+            k8sServiceHost: "192.168.150.86"
+            k8sServicePort: 6443
+            cni:
+              chainingMode: "none"
+              exclusive: false
+            bgpControlPlane:
+              enabled: false
+        MANIFEST
+        Dir.mktmpdir do |temp_dir|
+            configure_cilium temp_dir
+            result = YAML.load_stream File.read "#{temp_dir}/rke2-cilium-config.yaml"
+            expect(result).to eq output
+        end
+    end
+
+    it 'should define ip ranges when ONEAPP_K8S_CILIUM_BGP_ENABLED is true and ONEAPP_K8S_CILIUM_RANGES is not empty' do
+        stub_const 'ONEAPP_K8S_CONTROL_PLANE_EP', '192.168.150.86:6443'
+        stub_const 'ONEAPP_K8S_CNI_PLUGIN', 'cilium'
+        stub_const 'ONEAPP_K8S_CILIUM_BGP_ENABLED', true
+        stub_const 'ONEAPP_K8S_CILIUM_RANGES', ['192.168.150.128/25', '10.11.12.0/24']
+        output = YAML.load_stream <<~MANIFEST
+        ---
+        apiVersion: helm.cattle.io/v1
+        kind: HelmChartConfig
+        metadata:
+          name: rke2-cilium
+          namespace: kube-system
+        spec:
+          valuesContent: |-
+            kubeProxyReplacement: true
             k8sServiceHost: "192.168.150.86"
             k8sServicePort: 6443
             cni:
@@ -64,7 +93,10 @@
           name: default
           namespace: kube-system
         spec:
-          cidrs: {}
+          blocks:
+          - cidr: 192.168.150.128/25
+          - cidr: 10.11.12.0/24
+          allowFirstLastIPs: "No"
         MANIFEST
         Dir.mktmpdir do |temp_dir|
             configure_cilium temp_dir
@@ -73,10 +105,11 @@
         end
     end
 
-    it 'should apply user-defined ranges' do
+    it 'should define ip ranges when ONEAPP_K8S_CILIUM_BGP_ENABLED is true and ONEAPP_K8S_CILIUM_RANGES is empty' do
         stub_const 'ONEAPP_K8S_CONTROL_PLANE_EP', '192.168.150.86:6443'
         stub_const 'ONEAPP_K8S_CNI_PLUGIN', 'cilium'
-        stub_const 'ONEAPP_K8S_CILIUM_RANGES', ['192.168.150.128/25', '10.11.12.0/24']
+        stub_const 'ONEAPP_K8S_CILIUM_BGP_ENABLED', true
+        stub_const 'ONEAPP_K8S_CILIUM_RANGES', []
         output = YAML.load_stream <<~MANIFEST
         ---
         apiVersion: helm.cattle.io/v1
@@ -86,7 +119,7 @@
           namespace: kube-system
         spec:
           valuesContent: |-
-            kubeProxyReplacement: strict
+            kubeProxyReplacement: true
             k8sServiceHost: "192.168.150.86"
             k8sServicePort: 6443
             cni:
@@ -101,9 +134,8 @@
           name: default
           namespace: kube-system
         spec:
-          cidrs:
-          - cidr: 192.168.150.128/25
-          - cidr: 10.11.12.0/24
+          blocks: {}
+          allowFirstLastIPs: "No"
         MANIFEST
         Dir.mktmpdir do |temp_dir|
             configure_cilium temp_dir
@@ -122,7 +154,7 @@
           namespace: kube-system
         spec:
           valuesContent: |-
-            kubeProxyReplacement: strict
+            kubeProxyReplacement: true
             k8sServiceHost: "192.168.150.86"
             k8sServicePort: 6443
             cni:
@@ -137,9 +169,10 @@
           name: default
           namespace: kube-system
         spec:
-          cidrs:
+          blocks:
           - cidr: 192.168.150.128/25
           - cidr: 10.11.12.0/24
+          allowFirstLastIPs: "No"
         MANIFEST
         stub_const 'ONEAPP_K8S_CNI_PLUGIN', 'cilium'
         stub_const 'ONEAPP_K8S_CNI_CONFIG', Base64.encode64(manifest)
@@ -151,5 +184,4 @@
             expect(result).to eq output
         end
     end
-
 end

appliances/OneKE/config.rb

@@ -6,29 +6,31 @@
     require_relative '../lib/helpers.rb'
 end
 
-ONE_SERVICE_VERSION   = env :ONE_SERVICE_VERSION, '1.29'
+ONE_SERVICE_VERSION   = env :ONE_SERVICE_VERSION, '1.31'
 ONE_SERVICE_AIRGAPPED = env :ONE_SERVICE_AIRGAPPED, 'NO'
 ONE_SERVICE_SETUP_DIR = env :ONE_SERVICE_SETUP_DIR, '/opt/one-appliance'
 
-ONE_SERVICE_RKE2_RELEASE = env :ONE_SERVICE_RKE2_RELEASE, "#{ONE_SERVICE_VERSION}.4"
+ONE_SERVICE_RKE2_RELEASE = env :ONE_SERVICE_RKE2_RELEASE, "#{ONE_SERVICE_VERSION}.3"
 ONE_SERVICE_RKE2_VERSION = env :ONE_SERVICE_RKE2_VERSION, "v#{ONE_SERVICE_RKE2_RELEASE}+rke2r1"
-ONE_SERVICE_HELM_VERSION = env :ONE_SERVICE_HELM_VERSION, '3.14.4'
+ONE_SERVICE_HELM_VERSION = env :ONE_SERVICE_HELM_VERSION, '3.16.3'
 
 ONEAPP_K8S_MULTUS_ENABLED = env :ONEAPP_K8S_MULTUS_ENABLED, 'NO'
 ONEAPP_K8S_MULTUS_CONFIG  = env :ONEAPP_K8S_MULTUS_CONFIG, nil
 
-ONEAPP_K8S_CNI_PLUGIN    = env :ONEAPP_K8S_CNI_PLUGIN, 'cilium'
-ONEAPP_K8S_CNI_CONFIG    = env :ONEAPP_K8S_CNI_CONFIG, nil
-ONEAPP_K8S_CILIUM_RANGES = ENV.select { |key, _| key.start_with? 'ONEAPP_K8S_CILIUM_RANGE' } .values
+ONEAPP_K8S_CNI_PLUGIN         = env :ONEAPP_K8S_CNI_PLUGIN, 'cilium'
+ONEAPP_K8S_CNI_CONFIG         = env :ONEAPP_K8S_CNI_CONFIG, nil
+ONEAPP_K8S_CILIUM_BGP_ENABLED = env :ONEAPP_K8S_CILIUM_BGP_ENABLED, 'NO'
+ONEAPP_K8S_CILIUM_RANGES      = ENV.select { |key, _| key.start_with? 'ONEAPP_K8S_CILIUM_RANGE' } .values
 
-ONEAPP_K8S_LONGHORN_CHART_VERSION = env :ONEAPP_K8S_LONGHORN_CHART_VERSION, '1.6.1'
+ONEAPP_K8S_LONGHORN_CHART_VERSION = env :ONEAPP_K8S_LONGHORN_CHART_VERSION, '1.7.2'
 ONEAPP_K8S_LONGHORN_ENABLED       = env :ONEAPP_K8S_LONGHORN_ENABLED, 'NO'
 
 ONEAPP_STORAGE_DEVICE     = env :ONEAPP_STORAGE_DEVICE, nil # for example '/dev/vdb'
 ONEAPP_STORAGE_FILESYSTEM = env :ONEAPP_STORAGE_FILESYSTEM, 'xfs'
 ONEAPP_STORAGE_MOUNTPOINT = env :ONEAPP_STORAGE_MOUNTPOINT, '/var/lib/longhorn'
 
-ONEAPP_K8S_METALLB_CHART_VERSION = env :ONEAPP_K8S_METALLB_CHART_VERSION, '0.14.5'
+ONEAPP_K8S_METALLB_CHART_VERSION = env :ONEAPP_K8S_METALLB_CHART_VERSION, '0.14.8'
+ONEAPP_K8S_METALLB_CLASS         = env :ONEAPP_K8S_METALLB_CLASS, 'metallb'
 ONEAPP_K8S_METALLB_ENABLED       = env :ONEAPP_K8S_METALLB_ENABLED, 'NO'
 ONEAPP_K8S_METALLB_CONFIG        = env :ONEAPP_K8S_METALLB_CONFIG, nil
 ONEAPP_K8S_METALLB_RANGES        = ENV.select { |key, _| key.start_with? 'ONEAPP_K8S_METALLB_RANGE' } .values
@@ -38,6 +40,8 @@
 
 ONEAPP_K8S_RUBY_VERSION = env :ONEAPP_K8S_RUBY_VERSION, '3.3-alpine3.18'
 
+ONEAPP_RKE2_CLOUD_CONTROLLER_ENABLED = env :ONEAPP_RKE2_CLOUD_CONTROLLER_ENABLED, 'YES'
+
 ONEAPP_VROUTER_ETH0_VIP0 = env :ONEAPP_VROUTER_ETH0_VIP0, nil
 ONEAPP_VROUTER_ETH1_VIP0 = env :ONEAPP_VROUTER_ETH1_VIP0, nil
 
@@ -56,6 +60,11 @@
 ONEAPP_K8S_CONTROL_PLANE_EP = env :ONEAPP_K8S_CONTROL_PLANE_EP, "#{ONEAPP_VROUTER_ETH0_VIP0}:#{ONEAPP_VNF_HAPROXY_LB1_PORT}"
 ONEAPP_K8S_EXTRA_SANS       = env :ONEAPP_K8S_EXTRA_SANS, 'localhost,127.0.0.1'
 
+# Proxy config for RKE2: https://docs.rke2.io/advanced#configuring-an-http-proxy
+ONEAPP_K8S_HTTP_PROXY  = env :ONEAPP_K8S_HTTP_PROXY, nil
+ONEAPP_K8S_HTTPS_PROXY = env :ONEAPP_K8S_HTTPS_PROXY, nil
+ONEAPP_K8S_NO_PROXY    = env :ONEAPP_K8S_NO_PROXY, nil
+
 FALLBACK_GW  = env :FALLBACK_GW, nil
 FALLBACK_DNS = env :FALLBACK_DNS, nil
 

appliances/OneKE/kubernetes.rb

@@ -186,18 +186,21 @@ def init_master
     sans << cp.host
 
     server_config = {
-        'node-name'          => name,
-        'token'              => SecureRandom.uuid,
-        'tls-san'            => sans.uniq,
-        'node-taint'         => ['CriticalAddonsOnly=true:NoExecute'],
-        'disable'            => ['rke2-ingress-nginx'],
-        'cni'                => cni,
-        'disable-kube-proxy' => ONEAPP_K8S_CNI_PLUGIN == 'cilium'
+        'node-name'                => name,
+        'token'                    => SecureRandom.uuid,
+        'tls-san'                  => sans.uniq,
+        'node-taint'               => ['CriticalAddonsOnly=true:NoExecute'],
+        'disable'                  => ['rke2-ingress-nginx'],
+        'cni'                      => cni,
+        'disable-kube-proxy'       => ONEAPP_K8S_CNI_PLUGIN == 'cilium',
+        'disable-cloud-controller' => ONEAPP_RKE2_CLOUD_CONTROLLER_ENABLED == false
     }
 
     msg :info, 'Prepare initial rke2-server config'
     file '/etc/rancher/rke2/config.yaml', YAML.dump(server_config), overwrite: false
 
+    configure_rke2_proxy 'master'
+
     msg :info, "Initialize first master: #{name}"
     bash 'systemctl enable rke2-server.service --now'
 
@@ -239,14 +242,15 @@ def join_master(token, retries = RETRIES, seconds = SECONDS)
     sans << cp.host
 
     server_config = {
-        'node-name'          => name,
-        'server'             => "https://#{ONEAPP_RKE2_SUPERVISOR_EP}",
-        'token'              => token,
-        'tls-san'            => sans.uniq,
-        'node-taint'         => ['CriticalAddonsOnly=true:NoExecute'],
-        'disable'            => ['rke2-ingress-nginx'],
-        'cni'                => cni,
-        'disable-kube-proxy' => ONEAPP_K8S_CNI_PLUGIN == 'cilium'
+        'node-name'                => name,
+        'server'                   => "https://#{ONEAPP_RKE2_SUPERVISOR_EP}",
+        'token'                    => token,
+        'tls-san'                  => sans.uniq,
+        'node-taint'               => ['CriticalAddonsOnly=true:NoExecute'],
+        'disable'                  => ['rke2-ingress-nginx'],
+        'cni'                      => cni,
+        'disable-kube-proxy'       => ONEAPP_K8S_CNI_PLUGIN == 'cilium',
+        'disable-cloud-controller' => ONEAPP_RKE2_CLOUD_CONTROLLER_ENABLED == false
     }
 
     msg :info, 'Prepare rke2-server config'
@@ -303,6 +307,8 @@ def join_worker(token)
     msg :info, 'Prepare rke2-agent config'
     file '/etc/rancher/rke2/config.yaml', YAML.dump(agent_config), overwrite: true
 
+    configure_rke2_proxy 'worker'
+
     msg :info, "Join worker: #{name}"
     bash 'systemctl enable rke2-agent.service --now'
 end
@@ -327,6 +333,8 @@ def join_storage(token)
     msg :info, 'Prepare rke2-agent config'
     file '/etc/rancher/rke2/config.yaml', YAML.dump(agent_config), overwrite: true
 
+    configure_rke2_proxy 'worker'
+
     msg :info, "Join storage: #{name}"
     bash 'systemctl enable rke2-agent.service --now'
 end
@@ -359,3 +367,33 @@ def detect_node
     msg :debug, "detect_node / #{results}"
     results
 end
+
+def configure_rke2_proxy(current_role)
+    return if ONEAPP_K8S_HTTP_PROXY.to_s.empty? && ONEAPP_K8S_HTTPS_PROXY.to_s.empty?
+
+    rke2_role = current_role == 'master' ? 'server' : 'agent'
+    filepath  = "/etc/default/rke2-#{rke2_role}"
+
+    msg :info, "Prepare rke2-#{rke2_role} proxy config in #{filepath}"
+
+    proxy_config = []
+    proxy_config << "HTTP_PROXY=#{ONEAPP_K8S_HTTP_PROXY}" unless ONEAPP_K8S_HTTP_PROXY.nil?
+    proxy_config << "HTTPS_PROXY=#{ONEAPP_K8S_HTTPS_PROXY}" unless ONEAPP_K8S_HTTPS_PROXY.nil?
+    if ONEAPP_K8S_NO_PROXY.to_s.empty?
+        no_proxy = ['127.0.0.1/32', 'localhost']
+        no_proxy << retrieve_endpoint_host(ONEAPP_K8S_CONTROL_PLANE_EP) if ONEAPP_K8S_CONTROL_PLANE_EP
+        no_proxy << retrieve_endpoint_host(ONEAPP_RKE2_SUPERVISOR_EP) if ONEAPP_RKE2_SUPERVISOR_EP
+        proxy_config << "NO_PROXY=#{no_proxy.uniq.join(',')}"
+    else
+        proxy_config << "NO_PROXY=#{ONEAPP_K8S_NO_PROXY}"
+    end
+
+    file filepath, proxy_config.join("\n"), overwrite: true
+end
+
+def retrieve_endpoint_host(endpoint)
+    uri = URI.parse(endpoint.include?('://') ? endpoint : "http://#{endpoint}")
+    host = uri.host
+    host = "#{host}/32" if host =~ Resolv::IPv4::Regex
+    host
+end

appliances/OneKE/metallb.rb

@@ -82,6 +82,7 @@ def fetch_metallb(addon_dir = ONE_ADDON_DIR)
         speaker:
           image:
             pullPolicy: IfNotPresent
+        loadBalancerClass: #{ONEAPP_K8S_METALLB_CLASS}
     MANIFEST
 
     msg :info, "Generate MetalLB addon manifest: #{ONEAPP_K8S_METALLB_CHART_VERSION}"