feat: add ignoreFiles option to suppress file warnings in test directories (#651)

Adds a new TestSuite configuration option `ignoreFiles` that accepts
file patterns (e.g., "*.md", "README*") to ignore when collecting test
step files. Files matching these patterns are silently skipped without
generating warnings, allowing documentation and other non-test files to
coexist in test case directories. Defaults to "README*".

Signed-off-by: Martin André <[email protected]>

Author: mandre

docs/testing/reference.md

@@ -13,6 +13,10 @@ kindContainers:
 testDirs:
 - tests/e2e/
 timeout: 120
+ignoreFiles:
+- "*.md"
+- "README*"
+- ".gitignore"
 ```
 
 Supported settings:
@@ -39,6 +43,7 @@ reportGranularity | string           | What granularity to report failures at. O
 reportName        | string           | The name of report to create. This field is not used unless reportFormat is set.         | "kuttl-test"
 namespace         | string           | The namespace to use for tests. This namespace will be created if it does not exist and removed if it was created (unless `skipDelete` is set). If no namespace is set, one will be auto-generated. |
 suppress          | list of strings  | Suppresses log collection of the specified types. Currently only `events` is supported.  |
+ignoreFiles       | list of strings  | File patterns (e.g., `*.md`, `README*`) to ignore when collecting test steps. Files matching these patterns will not generate warnings about not matching the expected test file pattern. Setting this field (even to an empty list) overrides the defaults. | `["README*"]`
 
 ## TestStep
 

internal/harness/harness.go

@@ -88,6 +88,7 @@ func (h *Harness) LoadTests(dir string) ([]*testcase.Case, error) {
 			testcase.WithNamespace(h.TestSuite.Namespace),
 			testcase.WithTimeout(timeout),
 			testcase.WithLogSuppressions(h.TestSuite.Suppress),
+			testcase.WithIgnoreFiles(h.TestSuite.IgnoreFiles),
 			testcase.WithRunLabels(h.RunLabels),
 			testcase.WithClients(h.Client, h.DiscoveryClient),
 			testcase.WithTemplateVars(h.TemplateVars)))

internal/testcase/case.go

@@ -71,6 +71,13 @@ func WithLogSuppressions(suppressions []string) CaseOption {
 	}
 }
 
+// WithIgnoreFiles sets the list of file patterns to ignore.
+func WithIgnoreFiles(patterns []string) CaseOption {
+	return func(c *Case) {
+		c.ignoreFiles = patterns
+	}
+}
+
 // WithRunLabels sets the run labels.
 func WithRunLabels(runLabels labels.Set) CaseOption {
 	return func(c *Case) {
@@ -118,6 +125,8 @@ type Case struct {
 	logger testutils.Logger
 	// List of log types which should be suppressed.
 	suppressions []string
+	// List of file patterns to ignore when collecting test steps.
+	ignoreFiles []string
 	// Caution: the Vars element of this struct may be shared with other Case objects.
 	templateEnv template.Env
 }
@@ -378,7 +387,7 @@ func (c *Case) getEagerClients() ([]clientWithKubeConfig, error) {
 
 // LoadTestSteps loads all the test steps for a test case.
 func (c *Case) LoadTestSteps() error {
-	testStepFiles, err := files.CollectTestStepFiles(c.dir, c.logger)
+	testStepFiles, err := files.CollectTestStepFiles(c.dir, c.logger, c.ignoreFiles)
 	if err != nil {
 		return err
 	}

internal/utils/files/files.go

@@ -8,17 +8,44 @@ import (
 	testutils "github.com/kudobuilder/kuttl/internal/utils"
 )
 
+var defaultIgnorePatterns = []string{"README*"}
+
+// matchesAnyPattern checks if a filename matches any of the given patterns.
+// Returns true if a match is found, false otherwise.
+func matchesAnyPattern(filename string, patterns []string, logger testutils.Logger) bool {
+	for _, pattern := range patterns {
+		matched, err := filepath.Match(pattern, filename)
+		if err != nil {
+			logger.Logf("Invalid ignore pattern %q: %v", pattern, err)
+			continue
+		}
+		if matched {
+			return true
+		}
+	}
+	return false
+}
+
 // CollectTestStepFiles collects a map of test steps and their associated files
 // from a directory.
-func CollectTestStepFiles(dir string, logger testutils.Logger) (map[int64][]string, error) {
+func CollectTestStepFiles(dir string, logger testutils.Logger, ignorePatterns []string) (map[int64][]string, error) {
 	testStepFiles := map[int64][]string{}
 
+	patterns := ignorePatterns
+	if patterns == nil {
+		patterns = defaultIgnorePatterns
+	}
+
 	files, err := os.ReadDir(dir)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, file := range files {
+		if matchesAnyPattern(file.Name(), patterns, logger) {
+			continue
+		}
+
 		f := kfile.Parse(file.Name())
 		if f.Type == kfile.TypeUnknown {
 			logger.Logf("Ignoring %q: %v.", file.Name(), f.Error)

internal/utils/files/files_test.go

@@ -1,6 +1,8 @@
 package files
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -9,6 +11,38 @@ import (
 	testutils "github.com/kudobuilder/kuttl/internal/utils"
 )
 
+// mockLogger is a simple logger that captures log messages for testing
+type mockLogger struct {
+	messages []string
+}
+
+func (m *mockLogger) Log(args ...interface{}) {
+	m.messages = append(m.messages, fmt.Sprint(args...))
+}
+
+func (m *mockLogger) Logf(format string, args ...interface{}) {
+	m.messages = append(m.messages, fmt.Sprintf(format, args...))
+}
+
+func (m *mockLogger) WithPrefix(_ string) testutils.Logger {
+	return m
+}
+
+func (m *mockLogger) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (m *mockLogger) Flush() {}
+
+func (m *mockLogger) hasMessageContaining(substring string) bool {
+	for _, msg := range m.messages {
+		if strings.Contains(msg, substring) {
+			return true
+		}
+	}
+	return false
+}
+
 func TestCollectTestStepFiles(t *testing.T) {
 	for _, tt := range []struct {
 		path     string
@@ -48,9 +82,38 @@ func TestCollectTestStepFiles(t *testing.T) {
 		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
-			testStepFiles, err := CollectTestStepFiles(tt.path, testutils.NewTestLogger(t, tt.path))
+			testStepFiles, err := CollectTestStepFiles(tt.path, testutils.NewTestLogger(t, tt.path), nil)
 			require.NoError(t, err)
 			assert.Equal(t, tt.expected, testStepFiles)
 		})
 	}
 }
+
+func TestCollectTestStepFilesWithIgnorePatterns(t *testing.T) {
+	t.Run("default patterns ignore README files", func(t *testing.T) {
+		logger := &mockLogger{}
+		_, err := CollectTestStepFiles("test_data/with-overrides", logger, nil)
+		require.NoError(t, err)
+
+		assert.False(t, logger.hasMessageContaining("Ignoring \"README.md\""),
+			"README.md should be silently ignored with default patterns")
+	})
+
+	t.Run("explicit patterns override defaults", func(t *testing.T) {
+		logger := &mockLogger{}
+		_, err := CollectTestStepFiles("test_data/with-overrides", logger, []string{})
+		require.NoError(t, err)
+
+		assert.True(t, logger.hasMessageContaining("Ignoring \"README.md\""),
+			"README.md should generate warning when default patterns are overridden with empty list")
+	})
+
+	t.Run("custom patterns silently ignore matching files", func(t *testing.T) {
+		logger := &mockLogger{}
+		_, err := CollectTestStepFiles("test_data/with-overrides", logger, []string{"*.txt", "*.md"})
+		require.NoError(t, err)
+
+		assert.False(t, logger.hasMessageContaining("Ignoring \"README.md\""),
+			"README.md should be silently ignored with matching custom pattern")
+	})
+}

pkg/apis/testharness/v1beta1/test_types.go

@@ -97,6 +97,10 @@ type TestSuite struct {
 	// Suppress is used to suppress logs
 	Suppress []string `json:"suppress"`
 
+	// IgnoreFiles is a list of file patterns (e.g., "*.md", "README*") to ignore when collecting test steps.
+	// Files matching these patterns will not generate warnings about not matching the expected test file pattern.
+	IgnoreFiles []string `json:"ignoreFiles"`
+
 	Config *RestConfig `json:"config,omitempty"`
 }