Add retry logic for transient errors during cluster deletion

Author: bizz001

pkg/util/azureerrors/error.go

@@ -291,3 +291,28 @@ func detectVMProfile(errStr string) VMProfileType {
 	}
 	return VMProfileUnknown
 }
+
+// IsRetryableError returns true if the error is a transient/retryable error
+// such as 429 Too Many Requests or contains RetryableError code
+func IsRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	var responseError *azcore.ResponseError
+	if errors.As(err, &responseError) {
+		if responseError.StatusCode == http.StatusTooManyRequests {
+			return true
+		}
+	}
+
+	var detailedErr autorest.DetailedError
+	if errors.As(err, &detailedErr) {
+		if detailedErr.StatusCode == http.StatusTooManyRequests {
+			return true
+		}
+	}
+
+	// Check for RetryableError in error message (nested Azure errors)
+	return strings.Contains(err.Error(), "RetryableError")
+}

pkg/util/cluster/cluster.go

@@ -1354,7 +1354,31 @@ func (c *Cluster) deleteWimiRoleAssignments(ctx context.Context, vnetResourceGro
 
 func (c *Cluster) deleteCluster(ctx context.Context, resourceGroup, clusterName string) error {
 	c.log.Printf("deleting cluster %s", clusterName)
-	if err := c.openshiftclusters.DeleteAndWait(ctx, resourceGroup, clusterName); err != nil {
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	defer cancel()
+
+	var lastErr error
+	err := wait.PollImmediateUntil(30*time.Second, func() (bool, error) {
+		err := c.openshiftclusters.DeleteAndWait(timeoutCtx, resourceGroup, clusterName)
+		if err == nil {
+			return true, nil
+		}
+		if azureerrors.IsRetryableError(err) {
+			c.log.Warnf("retryable error deleting cluster %s, will retry: %v", clusterName, err)
+			lastErr = err
+			return false, nil
+		}
+		return false, err
+	}, timeoutCtx.Done())
+
+	if err != nil {
+		// If err is wait.ErrWaitTimeout, the timeout expired while retrying
+		// In that case, return lastErr if we have one (more informative)
+		if err == wait.ErrWaitTimeout && lastErr != nil {
+			return fmt.Errorf("error deleting cluster %s: %w", clusterName, lastErr)
+		}
+		// Otherwise return the actual error (e.g., non-retryable error from poll function)
 		return fmt.Errorf("error deleting cluster %s: %w", clusterName, err)
 	}
 	return nil
@@ -1365,14 +1389,39 @@ func (c *Cluster) ensureResourceGroupDeleted(ctx context.Context, resourceGroupN
 	timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer cancel()
 
-	return wait.PollImmediateUntil(5*time.Second, func() (bool, error) {
-		_, err := c.groups.Get(ctx, resourceGroupName)
+	var lastErr error
+	err := wait.PollImmediateUntil(5*time.Second, func() (bool, error) {
+		_, err := c.groups.Get(timeoutCtx, resourceGroupName)
 		if azureerrors.ResourceGroupNotFound(err) {
 			c.log.Infof("finished deleting resource group %s", resourceGroupName)
 			return true, nil
 		}
-		return false, fmt.Errorf("failed to delete resource group %s with %s", resourceGroupName, err)
+		if err != nil {
+			// Non-retryable errors should fail immediately
+			if !azureerrors.IsRetryableError(err) {
+				return false, fmt.Errorf("non-retryable error checking resource group %s: %w", resourceGroupName, err)
+			}
+			lastErr = err
+			c.log.Warnf("retryable error checking resource group %s, will retry: %v", resourceGroupName, err)
+		} else {
+			c.log.Infof("resource group %s still exists, waiting for deletion", resourceGroupName)
+		}
+		return false, nil
 	}, timeoutCtx.Done())
+
+	if err != nil {
+		// If err is wait.ErrWaitTimeout, the timeout expired while retrying
+		// In that case, return lastErr if we have one (more informative)
+		if err == wait.ErrWaitTimeout && lastErr != nil {
+			return fmt.Errorf("timed out waiting for resource group %s to be deleted, last error: %w", resourceGroupName, lastErr)
+		}
+		// Otherwise return the actual error (e.g., non-retryable error or timeout with no lastErr)
+		if err == wait.ErrWaitTimeout {
+			return fmt.Errorf("timed out waiting for resource group %s to be deleted", resourceGroupName)
+		}
+		return err
+	}
+	return nil
 }
 
 func (c *Cluster) deleteResourceGroup(ctx context.Context, resourceGroup string) error {