Skip to content

s3: Read S3 secrets from live hub cluster instead of gathered data #382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nirs merged 8 commits into from
Feb 17, 2026
Merged

s3: Read S3 secrets from live hub cluster instead of gathered data #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
bcce295
s3: rename Profile credentials fields and use []byte type
parikshithb Feb 17, 2026
a4e9d41
backend: add GetSecret to validation and testing interfaces
parikshithb Feb 12, 2026
6b5999c
helpers: rename fingerprint constants to avoid gosec warnings
parikshithb Feb 17, 2026
bce723a
test: improve gatherS3Data logging and console output
parikshithb Feb 12, 2026
d3f09c6
s3: Read S3 secrets from live hub cluster
parikshithb Jan 30, 2026
59cb81b
add comments for S3 gather and check functions
parikshithb Feb 10, 2026
c9ba633
test: add S3 profile inspection cancellation tests
parikshithb Feb 6, 2026
e160cfb
test: add GetSecret failure tests for gather and validate
parikshithb Feb 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 28 additions & 5 deletions pkg/gather/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,8 +218,13 @@ func (c *Command) inspectS3Profiles(
profiles, prefix, err := c.applicationS3Info(drpcName, drpcNamespace)
if err != nil {
step.Duration = time.Since(start).Seconds()
step.Status = report.Failed
console.Error("Failed to %s", step.Name)
if errors.Is(err, context.Canceled) {
step.Status = report.Canceled
console.Error("Canceled %s", step.Name)
} else {
step.Status = report.Failed
console.Error("Failed to %s", step.Name)
}
c.Logger().Errorf("Step %q %s: %s", c.current.Name, step.Status, err)
c.current.AddStep(step)
return nil, "", false
Expand All @@ -235,6 +240,8 @@ func (c *Command) inspectS3Profiles(
return profiles, prefix, true
}

// gatherApplicationS3Data gathers S3 data from the given profiles using the specified prefix.
// Returns true only if all profiles were gathered successfully, false on failure or cancellation.
func (c *Command) gatherApplicationS3Data(profiles []*s3.Profile, prefix string) bool {
start := time.Now()
outputDir := c.dataDir()
Expand Down Expand Up @@ -314,16 +321,32 @@ func (c *Command) applicationS3Info(
) ([]*s3.Profile, string, error) {
// Read S3 profiles from the ramen hub configmap, the source of truth
// synced to managed clusters.
reader := c.outputReader(c.Env().Hub.Name)

hub := c.Env().Hub
reader := c.outputReader(hub.Name)
configMapName := ramen.HubOperatorConfigMapName
configMapNamespace := c.config.Namespaces.RamenHubNamespace

profiles, err := ramen.ClusterProfiles(reader, configMapName, configMapNamespace)
storeProfiles, err := ramen.ClusterProfiles(reader, configMapName, configMapNamespace)
if err != nil {
return nil, "", err
}

// Get S3 secrets from live hub cluster since gathered data may contain
// sanitized secrets. On cancellation, return immediately. On other failures,
// empty credentials will cause S3 operations to fail during gatherS3.
var profiles []*s3.Profile
for _, sp := range storeProfiles {
secret, err := c.backend.GetSecret(c, hub, sp.S3SecretRef.Name, sp.S3SecretRef.Namespace)
if err != nil {
if errors.Is(err, context.Canceled) {
return nil, "", err
}
c.Logger().Warnf("Failed to get S3 secret \"%s/%s\" from cluster %q: %s",
sp.S3SecretRef.Namespace, sp.S3SecretRef.Name, hub.Name, err)
}
profiles = append(profiles, ramen.S3ProfileFromStore(sp, secret))
}

prefix, err := ramen.ApplicationS3Prefix(reader, drpcName, drpcNamespace)
if err != nil {
return nil, "", err
Expand Down
106 changes: 106 additions & 0 deletions pkg/gather/command_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

e2econfig "github.com/ramendr/ramen/e2e/config"
"github.com/ramendr/ramen/e2e/types"
corev1 "k8s.io/api/core/v1"

"github.com/ramendr/ramenctl/pkg/command"
"github.com/ramendr/ramenctl/pkg/config"
Expand Down Expand Up @@ -93,6 +94,29 @@ var (
},
}

inspectS3ProfilesCanceled = &validation.Mock{
GetSecretFunc: func(ctx validation.Context, cluster *types.Cluster, name, namespace string) (*corev1.Secret, error) {
return nil, context.Canceled
},
}

getSecretFailed = &validation.Mock{
GetSecretFunc: func(ctx validation.Context, cluster *types.Cluster, name, namespace string) (*corev1.Secret, error) {
return nil, errors.New("secret not found")
},
}

getSecretInvalid = &validation.Mock{
GetSecretFunc: func(ctx validation.Context, cluster *types.Cluster, name, namespace string) (*corev1.Secret, error) {
return &corev1.Secret{
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("invalid id"),
"AWS_SECRET_ACCESS_KEY": []byte("invalid key"),
},
}, nil
},
}

gatherS3Failed = &validation.Mock{
GatherS3Func: func(
ctx validation.Context,
Expand Down Expand Up @@ -279,6 +303,88 @@ func TestGatherApplicationInspectS3ProfilesFailed(t *testing.T) {
checkItems(t, cmd.report.Steps[1], items)
}

func TestGatherApplicationInspectS3ProfilesCanceled(t *testing.T) {
cmd := testCommand(t, inspectS3ProfilesCanceled)
helpers.AddGatheredData(t, cmd.dataDir(), "appset-deploy-rbd", "validate-application")
if err := cmd.Application(drpcName, drpcNamespace); err == nil {
t.Fatal("command did not fail")
}
checkReport(t, cmd.report, report.Canceled)
checkApplication(t, cmd.report, testApplication)

if len(cmd.report.Steps) != 2 {
t.Fatalf("unexpected steps %+v", cmd.report.Steps)
}
checkStep(t, cmd.report.Steps[0], "validate config", report.Passed)
checkStep(t, cmd.report.Steps[1], "gather data", report.Canceled)

items := []*report.Step{
{Name: "inspect application", Status: report.Passed},
{Name: "gather \"hub\"", Status: report.Passed},
{Name: "gather \"dr1\"", Status: report.Passed},
{Name: "gather \"dr2\"", Status: report.Passed},
{Name: "inspect S3 profiles", Status: report.Canceled},
}
checkItems(t, cmd.report.Steps[1], items)
}

func TestGatherApplicationGetSecretFailed(t *testing.T) {
cmd := testCommand(t, getSecretFailed)
helpers.AddGatheredData(t, cmd.dataDir(), "appset-deploy-rbd", "validate-application")
if err := cmd.Application(drpcName, drpcNamespace); err == nil {
t.Fatal("command did not fail")
}
checkReport(t, cmd.report, report.Failed)
checkApplication(t, cmd.report, testApplication)

if len(cmd.report.Steps) != 2 {
t.Fatalf("unexpected steps %+v", cmd.report.Steps)
}
checkStep(t, cmd.report.Steps[0], "validate config", report.Passed)
checkStep(t, cmd.report.Steps[1], "gather data", report.Failed)

// When GetSecret returns an error. The profile will have empty credentials
// causing S3 gather to fail.
items := []*report.Step{
{Name: "inspect application", Status: report.Passed},
{Name: "gather \"hub\"", Status: report.Passed},
{Name: "gather \"dr1\"", Status: report.Passed},
{Name: "gather \"dr2\"", Status: report.Passed},
{Name: "inspect S3 profiles", Status: report.Passed},
{Name: "gather S3 profile \"minio-on-dr1\"", Status: report.Failed},
{Name: "gather S3 profile \"minio-on-dr2\"", Status: report.Failed},
}
checkItems(t, cmd.report.Steps[1], items)
}

func TestGatherApplicationGetSecretInvalid(t *testing.T) {
cmd := testCommand(t, getSecretInvalid)
helpers.AddGatheredData(t, cmd.dataDir(), "appset-deploy-rbd", "validate-application")
if err := cmd.Application(drpcName, drpcNamespace); err == nil {
t.Fatal("command did not fail")
}
checkReport(t, cmd.report, report.Failed)
checkApplication(t, cmd.report, testApplication)

if len(cmd.report.Steps) != 2 {
t.Fatalf("unexpected steps %+v", cmd.report.Steps)
}
checkStep(t, cmd.report.Steps[0], "validate config", report.Passed)
checkStep(t, cmd.report.Steps[1], "gather data", report.Failed)

// When GetSecret returns a secret with invalid value, causing S3 gather to fail.
items := []*report.Step{
{Name: "inspect application", Status: report.Passed},
{Name: "gather \"hub\"", Status: report.Passed},
{Name: "gather \"dr1\"", Status: report.Passed},
{Name: "gather \"dr2\"", Status: report.Passed},
{Name: "inspect S3 profiles", Status: report.Passed},
{Name: "gather S3 profile \"minio-on-dr1\"", Status: report.Failed},
{Name: "gather S3 profile \"minio-on-dr2\"", Status: report.Failed},
}
checkItems(t, cmd.report.Steps[1], items)
}

func TestGatherApplicationS3DataFailed(t *testing.T) {
cmd := testCommand(t, gatherS3Failed)
helpers.AddGatheredData(t, cmd.dataDir(), "appset-deploy-rbd", "validate-application")
Expand Down
12 changes: 7 additions & 5 deletions pkg/helpers/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ import (
const (
Modified = "modified"

// Secret key fingerprints (SHA-256 hashes) for testdata.
// Both K8s and OCP testdata secrets have the same data values.
AccessKeyFingerprint = "F3:1C:B8:5A:2C:33:BA:C3:57:84:22:D5:11:F5:35:40:FF:A8:6A:34:B8:CD:42:AC:86:65:E2:2B:E1:05:EA:23"
//nolint:gosec
SecretKeyFingerprint = "BC:42:FE:14:DB:F0:91:1C:91:1F:8F:CF:72:AF:CE:C5:83:5C:AF:93:AC:08:40:CE:31:D8:67:CA:AC:BC:E4:16"
// Secrets from testdata. Both k8s and ocp use the same values.

FakeAWSKeyID = "this is not a real secret"
FakeAWSKey = "this is not a real key"

FakeAWSKeyIDFingerprint = "F3:1C:B8:5A:2C:33:BA:C3:57:84:22:D5:11:F5:35:40:FF:A8:6A:34:B8:CD:42:AC:86:65:E2:2B:E1:05:EA:23"
FakeAWSKeyFingerprint = "BC:42:FE:14:DB:F0:91:1C:91:1F:8F:CF:72:AF:CE:C5:83:5C:AF:93:AC:08:40:CE:31:D8:67:CA:AC:BC:E4:16"
)

func MarshalYAML(t *testing.T, a any) string {
Expand Down
61 changes: 29 additions & 32 deletions pkg/ramen/ramen.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

ramenapi "github.com/ramendr/ramen/api/v1alpha1"
e2etypes "github.com/ramendr/ramen/e2e/types"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/yaml"

Expand Down Expand Up @@ -248,39 +249,49 @@ func ListDRClusters(reader gathering.OutputReader) ([]string, error) {
return reader.ListResources("", resource)
}

// ClusterProfiles extracts S3 profiles with credentials from the ramen configmap.
// ClusterProfiles extracts S3 store profiles from the ramen configmap.
func ClusterProfiles(
reader gathering.OutputReader,
configMapName, configMapNamespace string,
) ([]*s3.Profile, error) {
) ([]*ramenapi.S3StoreProfile, error) {
configData, err := getRamenConfigMapData(reader, configMapName, configMapNamespace)
if err != nil {
return nil, err
}
if len(configData.S3StoreProfiles) == 0 {
return nil, fmt.Errorf("no S3 profiles found in ramen config")
}
profiles := make([]*s3.Profile, 0, len(configData.S3StoreProfiles))
for _, storeProfile := range configData.S3StoreProfiles {
secretName := storeProfile.S3SecretRef.Name
secretNamespace := storeProfile.S3SecretRef.Namespace
if secretNamespace == "" {
secretNamespace = configMapNamespace
profiles := make([]*ramenapi.S3StoreProfile, 0, len(configData.S3StoreProfiles))
for i := range configData.S3StoreProfiles {
profile := &configData.S3StoreProfiles[i]
// Default empty namespace to configmap namespace.
if profile.S3SecretRef.Namespace == "" {
profile.S3SecretRef.Namespace = configMapNamespace
}
accessKeyID, secretAccessKey := getS3SecretKeys(reader, secretName, secretNamespace)
profiles = append(profiles, &s3.Profile{
Name: storeProfile.S3ProfileName,
Bucket: storeProfile.S3Bucket,
Region: storeProfile.S3Region,
Endpoint: storeProfile.S3CompatibleEndpoint,
CACertificate: storeProfile.CACertificates,
AccessKey: accessKeyID,
SecretKey: secretAccessKey,
})
profiles = append(profiles, profile)
}
return profiles, nil
}

// S3ProfileFromStore creates an s3.Profile from a ramen S3StoreProfile and secret.
// If secret is nil, the profile will have empty credentials.
func S3ProfileFromStore(storeProfile *ramenapi.S3StoreProfile, secret *corev1.Secret) *s3.Profile {
var accessKey, secretKey []byte
if secret != nil {
accessKey = secret.Data["AWS_ACCESS_KEY_ID"]
secretKey = secret.Data["AWS_SECRET_ACCESS_KEY"]
}
return &s3.Profile{
Name: storeProfile.S3ProfileName,
Bucket: storeProfile.S3Bucket,
Region: storeProfile.S3Region,
Endpoint: storeProfile.S3CompatibleEndpoint,
CACertificate: storeProfile.CACertificates,
AWSAccessKeyID: accessKey,
AWSSecretAccessKey: secretKey,
}
}

// ApplicationS3Prefix returns the s3 object prefix for an application's s3 data.
func ApplicationS3Prefix(
reader gathering.OutputReader,
Expand Down Expand Up @@ -317,20 +328,6 @@ func getRamenConfigMapData(
return configData, nil
}

// getS3SecretKeys reads S3 credentials from a ramen s3 profile secret.
func getS3SecretKeys(
reader gathering.OutputReader,
name, namespace string,
) (string, string) {
secret, err := core.ReadSecret(reader, name, namespace)
if err != nil {
return "", ""
}
accessKeyID := string(secret.Data["AWS_ACCESS_KEY_ID"])
secretAccessKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
return accessKeyID, secretAccessKey
}

func primaryClusterName(drpc *ramenapi.DRPlacementControl) string {
if drpc.Spec.Action == ramenapi.ActionFailover {
return drpc.Spec.FailoverCluster
Expand Down
24 changes: 12 additions & 12 deletions pkg/report/clusters_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -688,11 +688,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
// CACertificate is optional, empty is OK if hub also has no cert.
Expand Down Expand Up @@ -728,11 +728,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
CACertificate: report.ValidatedFingerprint{
Expand Down Expand Up @@ -824,11 +824,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
CACertificate: report.ValidatedFingerprint{
Expand Down Expand Up @@ -863,11 +863,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
CACertificate: report.ValidatedFingerprint{
Expand Down Expand Up @@ -958,11 +958,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
CACertificate: report.ValidatedFingerprint{
Expand Down Expand Up @@ -997,11 +997,11 @@ func testClusterStatus() *report.ClustersStatus {
},
AWSAccessKeyID: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.AccessKeyFingerprint,
Value: helpers.FakeAWSKeyIDFingerprint,
},
AWSSecretAccessKey: report.ValidatedFingerprint{
Validated: report.Validated{State: report.OK},
Value: helpers.SecretKeyFingerprint,
Value: helpers.FakeAWSKeyFingerprint,
},
},
CACertificate: report.ValidatedFingerprint{
Expand Down
Loading