Skip to content

Commit 2fb20b4

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #3799 from Azure/mvacula/aro-23474
E2E test for back-level cluster version (4.19) installation
2 parents 2ff7bb3 + fbc7ae8 commit 2fb20b4

9 files changed

+200
-4
lines changed

‎test/e2e/cluster_version_backlevel.go‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
// Copyright 2025 Microsoft Corporation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package e2e
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"strings"
21+
"time"
22+
23+
. "github.com/onsi/ginkgo/v2"
24+
. "github.com/onsi/gomega"
25+
26+
"github.com/Azure/ARO-HCP/test/util/framework"
27+
"github.com/Azure/ARO-HCP/test/util/labels"
28+
"github.com/Azure/ARO-HCP/test/util/verifiers"
29+
)
30+
31+
var _ = Describe("Customer", func() {
32+
33+
backlevelVersions := []string{"4.19"}
34+
35+
for _, version := range backlevelVersions {
36+
version := version // capture loop variable
37+
It("should be able to create an HCP cluster with back-level version "+version,
38+
labels.RequireNothing,
39+
labels.Critical,
40+
labels.Positive,
41+
labels.AroRpApiCompatible,
42+
func(ctx context.Context) {
43+
const (
44+
customerNetworkSecurityGroupName = "customer-nsg-name-"
45+
customerVnetName = "customer-vnet-name-"
46+
customerVnetSubnetName = "customer-vnet-subnet-"
47+
customerClusterName = "cluster-ver-"
48+
customerNodePoolName = "np-ver-"
49+
)
50+
tc := framework.NewTestContext()
51+
52+
if tc.UsePooledIdentities() {
53+
err := tc.AssignIdentityContainers(ctx, 1, 60*time.Second)
54+
Expect(err).NotTo(HaveOccurred())
55+
}
56+
57+
By("creating a resource group")
58+
resourceGroup, err := tc.NewResourceGroup(ctx, "rg-cluster-back-version", tc.Location())
59+
Expect(err).NotTo(HaveOccurred())
60+
61+
clusterSuffix := strings.ReplaceAll(version, ".", "-")
62+
clusterName := customerClusterName + clusterSuffix
63+
64+
clusterParams := framework.NewDefaultClusterParams()
65+
clusterParams.ClusterName = clusterName
66+
managedResourceGroupName := framework.SuffixName(*resourceGroup.Name+"-"+clusterSuffix, "-managed", 64)
67+
clusterParams.ManagedResourceGroupName = managedResourceGroupName
68+
clusterParams.OpenshiftVersionId = version
69+
70+
// copied 4.19 defaults from 01/22/2026 snapshot of NewDefaultClusterParams
71+
clusterParams.Network = framework.NetworkConfig{
72+
NetworkType: "OVNKubernetes",
73+
PodCIDR: "10.128.0.0/14",
74+
ServiceCIDR: "172.30.0.0/16",
75+
MachineCIDR: "10.0.0.0/16",
76+
HostPrefix: 23,
77+
}
78+
clusterParams.EncryptionKeyManagementMode = "CustomerManaged"
79+
clusterParams.EncryptionType = "KMS"
80+
clusterParams.APIVisibility = "Public"
81+
clusterParams.ImageRegistryState = "Enabled"
82+
clusterParams.ChannelGroup = "stable"
83+
84+
clusterParams, err = tc.CreateClusterCustomerResources(ctx,
85+
resourceGroup,
86+
clusterParams,
87+
map[string]any{
88+
"customerNsgName": customerNetworkSecurityGroupName + clusterSuffix,
89+
"customerVnetName": customerVnetName + clusterSuffix,
90+
"customerVnetSubnetName": customerVnetSubnetName + clusterSuffix,
91+
},
92+
TestArtifactsFS,
93+
)
94+
Expect(err).NotTo(HaveOccurred())
95+
96+
By("creating HCP cluster version " + version)
97+
err = tc.CreateHCPClusterFromParam(
98+
ctx,
99+
GinkgoLogr,
100+
*resourceGroup.Name,
101+
clusterParams,
102+
45*time.Minute,
103+
)
104+
Expect(err).NotTo(HaveOccurred())
105+
106+
adminRESTConfig, err := tc.GetAdminRESTConfigForHCPCluster(
107+
ctx,
108+
tc.Get20240610ClientFactoryOrDie(ctx).NewHcpOpenShiftClustersClient(),
109+
*resourceGroup.Name,
110+
clusterName,
111+
10*time.Minute,
112+
)
113+
Expect(err).NotTo(HaveOccurred())
114+
115+
err = verifiers.VerifyHCPCluster(ctx, adminRESTConfig)
116+
Expect(err).NotTo(HaveOccurred())
117+
118+
By("creating node pool with back-level version")
119+
backlevelNodePoolVersions := []string{"4.19.7"}
120+
121+
var matchingNodePoolVersion string
122+
for _, nodePoolVersion := range backlevelNodePoolVersions {
123+
if strings.HasPrefix(nodePoolVersion, version+".") {
124+
matchingNodePoolVersion = nodePoolVersion
125+
break
126+
}
127+
}
128+
129+
if matchingNodePoolVersion != "" {
130+
nodePoolSuffix := strings.ReplaceAll(matchingNodePoolVersion, ".", "-")
131+
nodePoolName := customerNodePoolName + nodePoolSuffix
132+
nodePoolParams := framework.NewDefaultNodePoolParams()
133+
nodePoolParams.ClusterName = clusterName
134+
nodePoolParams.NodePoolName = nodePoolName
135+
nodePoolParams.OpenshiftVersionId = matchingNodePoolVersion
136+
137+
// copied 4.19 defaults from 01/22/2026 snapshot of NewDefaultNodePoolParams
138+
nodePoolParams.Replicas = int32(2)
139+
nodePoolParams.VMSize = "Standard_D8s_v3"
140+
nodePoolParams.OSDiskSizeGiB = int32(64)
141+
nodePoolParams.DiskStorageAccountType = "StandardSSD_LRS"
142+
nodePoolParams.ChannelGroup = "stable"
143+
144+
By("creating node pool version " + matchingNodePoolVersion + " and verifying a simple web app can run")
145+
err = tc.CreateNodePoolFromParam(ctx,
146+
*resourceGroup.Name,
147+
clusterName,
148+
nodePoolParams,
149+
45*time.Minute,
150+
)
151+
Expect(err).NotTo(HaveOccurred())
152+
153+
nodePoolLabel := fmt.Sprintf("%s-%s", clusterName, nodePoolName)
154+
nodeSelector := map[string]string{"hypershift.openshift.io/nodePool": nodePoolLabel}
155+
err = verifiers.VerifySimpleWebApp(nodeSelector).Verify(ctx, adminRESTConfig)
156+
Expect(err).NotTo(HaveOccurred())
157+
}
158+
159+
})
160+
}
161+
})

‎test/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ require (
2424
github.com/openshift-eng/openshift-tests-extension v0.0.0-20251217181008-4f0b29a50e82
2525
github.com/sirupsen/logrus v1.9.3
2626
github.com/spf13/cobra v1.10.2
27+
go.yaml.in/yaml/v2 v2.4.2
2728
golang.org/x/crypto v0.46.0
2829
golang.org/x/sync v0.19.0
2930
k8s.io/api v0.34.1
@@ -245,7 +246,6 @@ require (
245246
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
246247
go.uber.org/multierr v1.11.0 // indirect
247248
go.uber.org/zap v1.27.0 // indirect
248-
go.yaml.in/yaml/v2 v2.4.2 // indirect
249249
go.yaml.in/yaml/v3 v3.0.4 // indirect
250250
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
251251
gocloud.dev v0.40.0 // indirect

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_dev_cd_check_paralleldev_cd_check_parallel.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Customer should be able to create an HCP cluster with custom autoscaling
55
Customer should be able to create a HCP cluster without CNI
66
Customer should be able to list HCP clusters without node pools at both subscription and resource group levels
77
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
8+
Customer should be able to create an HCP cluster with back-level version 4.19
89
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
910
Customer should be able to create an HCP cluster
1011
Customer should be able to lifecycle and confirm external auth on a cluster

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_integration_parallelintegration_parallel.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ Customer should not be able to reuse subnets and NSGs between clusters
1212
Customer should create an HCP cluster and validate TLS certificates
1313
Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request
1414
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
15+
Customer should be able to create an HCP cluster with back-level version 4.19
1516
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
1617
Customer should be able to create an HCP cluster
1718
Customer should be able to create a cluster with an external auth config and get the external auth config

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_prod_parallelprod_parallel.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Customer should not be able to reuse subnets and NSGs between clusters
1111
Customer should create an HCP cluster and validate TLS certificates
1212
Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request
1313
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
14+
Customer should be able to create an HCP cluster with back-level version 4.19
1415
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
1516
Customer should be able to create an HCP cluster
1617
Customer should be able to create a cluster with an external auth config and get the external auth config

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallel_01rp_api_compat_all_parallel_development.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ Customer should be able to list HCP clusters without node pools at both subscrip
88
Customer should not be able to deploy 2 identically named clusters within the same resource group
99
Customer should not be able to reuse subnets and NSGs between clusters
1010
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
11+
Customer should be able to create an HCP cluster with back-level version 4.19
1112
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
1213
Customer should be able to create an HCP cluster
1314
Customer should be able to lifecycle and confirm external auth on a cluster

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_rp_api_compat_all_parallelrp_api_compat_all_parallel.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ Customer should be able to list HCP clusters without node pools at both subscrip
88
Customer should not be able to deploy 2 identically named clusters within the same resource group
99
Customer should not be able to reuse subnets and NSGs between clusters
1010
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
11+
Customer should be able to create an HCP cluster with back-level version 4.19
1112
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
1213
Customer should be able to create an HCP cluster
1314
Customer should be able to lifecycle and confirm external auth on a cluster

‎test/testdata/zz_fixture_TestMainListSuitesForEachSuite_stage_parallelstage_parallel.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Customer should not be able to reuse subnets and NSGs between clusters
1111
Customer should create an HCP cluster and validate TLS certificates
1212
Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request
1313
Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request
14+
Customer should be able to create an HCP cluster with back-level version 4.19
1415
Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group
1516
Customer should be able to create an HCP cluster
1617
Customer should be able to create a cluster with an external auth config and get the external auth config

‎test/util/verifiers/serving_app.go‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"time"
2525

2626
"github.com/onsi/ginkgo/v2"
27+
"go.yaml.in/yaml/v2"
2728

2829
corev1 "k8s.io/api/core/v1"
2930
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -44,6 +45,7 @@ var staticFiles embed.FS
4445

4546
type verifySimpleWebApp struct {
4647
namespaceName string
48+
nodeSelector map[string]string
4749
}
4850

4951
func (v verifySimpleWebApp) Name() string {
@@ -80,7 +82,30 @@ func (v verifySimpleWebApp) Verify(ctx context.Context, adminRESTConfig *rest.Co
8082
return fmt.Errorf("failed to create dynamic client: %w", err)
8183
}
8284

83-
deployment, err := createArbitraryResource(ctx, dynamicClient, namespace.Name, must(staticFiles.ReadFile("artifacts/serving_app/deployment.yaml")))
85+
deploymentYAML := must(staticFiles.ReadFile("artifacts/serving_app/deployment.yaml"))
86+
87+
if v.nodeSelector != nil {
88+
89+
var deploymentMap map[string]any
90+
if err := yaml.Unmarshal(deploymentYAML, &deploymentMap); err != nil {
91+
return fmt.Errorf("failed to unmarshal deployment YAML: %w", err)
92+
}
93+
94+
if spec, ok := deploymentMap["spec"].(map[string]any); ok {
95+
if template, ok := spec["template"].(map[string]any); ok {
96+
if templateSpec, ok := template["spec"].(map[string]any); ok {
97+
templateSpec["nodeSelector"] = v.nodeSelector
98+
}
99+
}
100+
}
101+
102+
deploymentYAML, err = yaml.Marshal(deploymentMap)
103+
if err != nil {
104+
return fmt.Errorf("failed to marshal modified deployment: %w", err)
105+
}
106+
}
107+
108+
deployment, err := createArbitraryResource(ctx, dynamicClient, namespace.Name, deploymentYAML)
84109
if err != nil {
85110
return fmt.Errorf("failed to create deployment: %w", err)
86111
}
@@ -266,8 +291,12 @@ func (v verifySimpleWebApp) cleanup(ctx context.Context, adminRESTConfig *rest.C
266291
return nil
267292
}
268293

269-
func VerifySimpleWebApp() HostedClusterVerifier {
270-
return verifySimpleWebApp{}
294+
func VerifySimpleWebApp(nodeSelector ...map[string]string) HostedClusterVerifier {
295+
var ns map[string]string
296+
if len(nodeSelector) > 0 {
297+
ns = nodeSelector[0]
298+
}
299+
return verifySimpleWebApp{nodeSelector: ns}
271300
}
272301

273302
func must[T any](v T, err error) T {

0 commit comments

Comments
 (0)